Skip to content

Commit 8f5f06a

Browse files
authored
some Tokenizer construction and related other cleanups (#4799)
1 parent dac578e commit 8f5f06a

15 files changed

+138
-147
lines changed

Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -694,7 +694,7 @@ test/testbool.o: test/testbool.cpp lib/check.h lib/checkbool.h lib/color.h lib/c
694694
test/testboost.o: test/testboost.cpp lib/check.h lib/checkboost.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
695695
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testboost.cpp
696696

697-
test/testbufferoverrun.o: test/testbufferoverrun.cpp externals/simplecpp/simplecpp.h externals/tinyxml2/tinyxml2.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
697+
test/testbufferoverrun.o: test/testbufferoverrun.cpp externals/simplecpp/simplecpp.h externals/tinyxml2/tinyxml2.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
698698
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testbufferoverrun.cpp
699699

700700
test/testcharvar.o: test/testcharvar.cpp lib/check.h lib/checkother.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
@@ -703,7 +703,7 @@ test/testcharvar.o: test/testcharvar.cpp lib/check.h lib/checkother.h lib/color.
703703
test/testclangimport.o: test/testclangimport.cpp lib/check.h lib/clangimport.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
704704
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testclangimport.cpp
705705

706-
test/testclass.o: test/testclass.cpp externals/tinyxml2/tinyxml2.h lib/check.h lib/checkclass.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
706+
test/testclass.o: test/testclass.cpp externals/simplecpp/simplecpp.h externals/tinyxml2/tinyxml2.h lib/check.h lib/checkclass.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
707707
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testclass.cpp
708708

709709
test/testcmdlineparser.o: test/testcmdlineparser.cpp cli/cmdlineparser.h cli/cppcheckexecutor.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/timer.h lib/utils.h test/fixture.h test/redirect.h
@@ -730,7 +730,7 @@ test/testfilelister.o: test/testfilelister.cpp cli/filelister.h lib/check.h lib/
730730
test/testfunctions.o: test/testfunctions.cpp externals/tinyxml2/tinyxml2.h lib/check.h lib/checkfunctions.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
731731
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testfunctions.cpp
732732

733-
test/testgarbage.o: test/testgarbage.cpp lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
733+
test/testgarbage.o: test/testgarbage.cpp externals/simplecpp/simplecpp.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/timer.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
734734
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testgarbage.cpp
735735

736736
test/testimportproject.o: test/testimportproject.cpp lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/timer.h lib/utils.h test/fixture.h

cli/cppcheckexecutor.cpp

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -147,13 +147,13 @@ bool CppCheckExecutor::parseFromArgs(CppCheck *cppcheck, int argc, const char* c
147147
#else
148148
const bool caseSensitive = true;
149149
#endif
150-
if (!mSettings->project.fileSettings.empty() && !mSettings->fileFilters.empty()) {
150+
if (!settings.project.fileSettings.empty() && !settings.fileFilters.empty()) {
151151
// filter only for the selected filenames from all project files
152152
std::list<ImportProject::FileSettings> newList;
153153

154154
const std::list<ImportProject::FileSettings>& fileSettings = settings.project.fileSettings;
155155
std::copy_if(fileSettings.cbegin(), fileSettings.cend(), std::back_inserter(newList), [&](const ImportProject::FileSettings& fs) {
156-
return matchglobs(mSettings->fileFilters, fs.filename);
156+
return matchglobs(settings.fileFilters, fs.filename);
157157
});
158158
if (!newList.empty())
159159
settings.project.fileSettings = newList;
@@ -165,7 +165,7 @@ bool CppCheckExecutor::parseFromArgs(CppCheck *cppcheck, int argc, const char* c
165165
// Execute recursiveAddFiles() to each given file parameter
166166
const PathMatch matcher(ignored, caseSensitive);
167167
for (const std::string &pathname : pathnames) {
168-
std::string err = FileLister::recursiveAddFiles(mFiles, Path::toNativeSeparators(pathname), mSettings->library.markupExtensions(), matcher);
168+
std::string err = FileLister::recursiveAddFiles(mFiles, Path::toNativeSeparators(pathname), settings.library.markupExtensions(), matcher);
169169
if (!err.empty()) {
170170
std::cout << "cppcheck: " << err << std::endl;
171171
}
@@ -177,10 +177,10 @@ bool CppCheckExecutor::parseFromArgs(CppCheck *cppcheck, int argc, const char* c
177177
if (!ignored.empty())
178178
std::cout << "cppcheck: Maybe all paths were ignored?" << std::endl;
179179
return false;
180-
} else if (!mSettings->fileFilters.empty() && settings.project.fileSettings.empty()) {
180+
} else if (!settings.fileFilters.empty() && settings.project.fileSettings.empty()) {
181181
std::map<std::string, std::size_t> newMap;
182182
for (std::map<std::string, std::size_t>::const_iterator i = mFiles.cbegin(); i != mFiles.cend(); ++i)
183-
if (matchglobs(mSettings->fileFilters, i->first)) {
183+
if (matchglobs(settings.fileFilters, i->first)) {
184184
newMap[i->first] = i->second;
185185
}
186186
mFiles = newMap;
@@ -226,11 +226,6 @@ int CppCheckExecutor::check(int argc, const char* const argv[])
226226
return ret;
227227
}
228228

229-
void CppCheckExecutor::setSettings(const Settings &settings)
230-
{
231-
mSettings = &settings;
232-
}
233-
234229
int CppCheckExecutor::check_wrapper(CppCheck& cppcheck)
235230
{
236231
#ifdef USE_WINDOWS_SEH
@@ -266,7 +261,6 @@ bool CppCheckExecutor::reportSuppressions(const Settings &settings, bool unusedF
266261
int CppCheckExecutor::check_internal(CppCheck& cppcheck)
267262
{
268263
Settings& settings = cppcheck.settings();
269-
mSettings = &settings;
270264
const bool std = tryLoadLibrary(settings.library, settings.exename, "std.cfg");
271265

272266
for (const std::string &lib : settings.libraries) {
@@ -330,8 +324,8 @@ int CppCheckExecutor::check_internal(CppCheck& cppcheck)
330324
unsigned int c = 0;
331325
if (settings.project.fileSettings.empty()) {
332326
for (std::map<std::string, std::size_t>::const_iterator i = mFiles.cbegin(); i != mFiles.cend(); ++i) {
333-
if (!mSettings->library.markupFile(i->first)
334-
|| !mSettings->library.processMarkupAfterCode(i->first)) {
327+
if (!settings.library.markupFile(i->first)
328+
|| !settings.library.processMarkupAfterCode(i->first)) {
335329
returnValue += cppcheck.check(i->first);
336330
processedsize += i->second;
337331
if (!settings.quiet)
@@ -355,7 +349,7 @@ int CppCheckExecutor::check_internal(CppCheck& cppcheck)
355349
// second loop to parse all markup files which may not work until all
356350
// c/cpp files have been parsed and checked
357351
for (std::map<std::string, std::size_t>::const_iterator i = mFiles.cbegin(); i != mFiles.cend(); ++i) {
358-
if (mSettings->library.markupFile(i->first) && mSettings->library.processMarkupAfterCode(i->first)) {
352+
if (settings.library.markupFile(i->first) && settings.library.processMarkupAfterCode(i->first)) {
359353
returnValue += cppcheck.check(i->first);
360354
processedsize += i->second;
361355
if (!settings.quiet)
@@ -374,7 +368,7 @@ int CppCheckExecutor::check_internal(CppCheck& cppcheck)
374368
returnValue = executor.check();
375369
}
376370

377-
cppcheck.analyseWholeProgram(mSettings->buildDir, mFiles);
371+
cppcheck.analyseWholeProgram(settings.buildDir, mFiles);
378372

379373
if (settings.severity.isEnabled(Severity::information) || settings.checkConfiguration) {
380374
const bool err = reportSuppressions(settings, cppcheck.isUnusedFunctionCheckEnabled(), mFiles, *this);
@@ -413,7 +407,6 @@ int CppCheckExecutor::check_internal(CppCheck& cppcheck)
413407
reportErr(ErrorMessage::getXMLFooter());
414408
}
415409

416-
mSettings = nullptr;
417410
if (returnValue)
418411
return settings.exitCode;
419412
return 0;

cli/cppcheckexecutor.h

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,6 @@ class CppCheckExecutor : public ErrorLogger {
137137
*/
138138
bool parseFromArgs(CppCheck *cppcheck, int argc, const char* const argv[]);
139139

140-
/**
141-
* Helper function to supply settings. This can be used for testing.
142-
* @param settings Reference to an Settings instance
143-
*/
144-
void setSettings(const Settings &settings);
145-
146140
private:
147141

148142
/**
@@ -166,7 +160,7 @@ class CppCheckExecutor : public ErrorLogger {
166160
int check_internal(CppCheck& cppcheck);
167161

168162
/**
169-
* Pointer to current settings; set while check() is running.
163+
* Pointer to current settings; set while check() is running for reportError().
170164
*/
171165
const Settings* mSettings;
172166

lib/cppcheck.cpp

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -800,10 +800,10 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
800800
}
801801

802802
// Run define rules on raw code
803-
for (const Settings::Rule &rule : mSettings.rules) {
804-
if (rule.tokenlist != "define")
805-
continue;
806-
803+
const auto it = std::find_if(mSettings.rules.cbegin(), mSettings.rules.cend(), [](const Settings::Rule& rule) {
804+
return rule.tokenlist == "define";
805+
});
806+
if (it != mSettings.rules.cend()) {
807807
std::string code;
808808
const std::list<Directive> &directives = preprocessor.getDirectives();
809809
for (const Directive &dir : directives) {
@@ -814,7 +814,6 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
814814
std::istringstream istr2(code);
815815
tokenizer2.list.createTokens(istr2);
816816
executeRules("define", tokenizer2);
817-
break;
818817
}
819818

820819
if (!mSettings.force && configurations.size() > mSettings.maxConfigs) {
@@ -871,8 +870,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
871870
continue;
872871
}
873872

874-
Tokenizer tokenizer(&mSettings, this);
875-
tokenizer.setPreprocessor(&preprocessor);
873+
Tokenizer tokenizer(&mSettings, this, &preprocessor);
876874
if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
877875
tokenizer.setTimerResults(&s_timerResults);
878876

lib/importproject.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -588,7 +588,7 @@ namespace {
588588
// TODO : Better evaluation
589589
Settings s;
590590
std::istringstream istr(c);
591-
Tokenizer tokenizer(&s, nullptr);
591+
Tokenizer tokenizer(&s);
592592
tokenizer.tokenize(istr,"vcxproj");
593593
for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) {
594594
if (tok->str() == "(" && tok->astOperand1() && tok->astOperand2()) {

lib/tokenize.cpp

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -155,35 +155,20 @@ static bool isClassStructUnionEnumStart(const Token * tok)
155155

156156
//---------------------------------------------------------------------------
157157

158-
Tokenizer::Tokenizer() :
159-
list(nullptr),
160-
mSettings(nullptr),
161-
mErrorLogger(nullptr),
162-
mSymbolDatabase(nullptr),
163-
mTemplateSimplifier(nullptr),
164-
mVarId(0),
165-
mUnnamedCount(0),
166-
mCodeWithTemplates(false), //is there any templates?
167-
mTimerResults(nullptr),
168-
mPreprocessor(nullptr)
169-
{}
170-
171-
Tokenizer::Tokenizer(const Settings *settings, ErrorLogger *errorLogger) :
158+
Tokenizer::Tokenizer(const Settings *settings, ErrorLogger *errorLogger, const Preprocessor *preprocessor) :
172159
list(settings),
173160
mSettings(settings),
174161
mErrorLogger(errorLogger),
175162
mSymbolDatabase(nullptr),
176-
mTemplateSimplifier(nullptr),
163+
mTemplateSimplifier(new TemplateSimplifier(this)),
177164
mVarId(0),
178165
mUnnamedCount(0),
179166
mCodeWithTemplates(false), //is there any templates?
180167
mTimerResults(nullptr),
181-
mPreprocessor(nullptr)
168+
mPreprocessor(preprocessor)
182169
{
183170
// make sure settings are specified
184171
assert(mSettings);
185-
186-
mTemplateSimplifier = new TemplateSimplifier(this);
187172
}
188173

189174
Tokenizer::~Tokenizer()
@@ -2773,16 +2758,11 @@ bool Tokenizer::simplifyUsing()
27732758
bool Tokenizer::createTokens(std::istream &code,
27742759
const std::string& FileName)
27752760
{
2776-
// make sure settings specified
2777-
assert(mSettings);
2778-
27792761
return list.createTokens(code, FileName);
27802762
}
27812763

27822764
void Tokenizer::createTokens(simplecpp::TokenList&& tokenList)
27832765
{
2784-
// make sure settings specified
2785-
assert(mSettings);
27862766
list.createTokens(std::move(tokenList));
27872767
}
27882768

@@ -9811,8 +9791,8 @@ void Tokenizer::simplifyNamespaceAliases()
98119791

98129792
bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
98139793
{
9814-
if (!mPreprocessor)
9815-
return false;
9794+
assert(mPreprocessor);
9795+
98169796
return std::any_of(mPreprocessor->getDirectives().cbegin(), mPreprocessor->getDirectives().cend(), [&](const Directive& d) {
98179797
return d.str.compare(0, 3, "#if") == 0 &&
98189798
d.linenr >= start->linenr() &&

lib/tokenize.h

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "errortypes.h"
2626
#include "tokenlist.h"
2727

28+
#include <cassert>
2829
#include <iosfwd>
2930
#include <list>
3031
#include <map>
@@ -59,8 +60,7 @@ class CPPCHECKLIB Tokenizer {
5960
friend class TemplateSimplifier;
6061

6162
public:
62-
Tokenizer();
63-
Tokenizer(const Settings * settings, ErrorLogger *errorLogger);
63+
explicit Tokenizer(const Settings * settings, ErrorLogger *errorLogger = nullptr, const Preprocessor *preprocessor = nullptr);
6464
~Tokenizer();
6565

6666
void setTimerResults(TimerResults *tr) {
@@ -377,10 +377,8 @@ class CPPCHECKLIB Tokenizer {
377377
*/
378378
static const Token * isFunctionHead(const Token *tok, const std::string &endsWith, bool cpp);
379379

380-
void setPreprocessor(const Preprocessor *preprocessor) {
381-
mPreprocessor = preprocessor;
382-
}
383380
const Preprocessor *getPreprocessor() const {
381+
assert(mPreprocessor);
384382
return mPreprocessor;
385383
}
386384

@@ -621,12 +619,6 @@ class CPPCHECKLIB Tokenizer {
621619
return mCodeWithTemplates;
622620
}
623621

624-
625-
void setSettings(const Settings *settings) {
626-
mSettings = settings;
627-
list.setSettings(settings);
628-
}
629-
630622
const SymbolDatabase *getSymbolDatabase() const {
631623
return mSymbolDatabase;
632624
}
@@ -703,15 +695,15 @@ class CPPCHECKLIB Tokenizer {
703695
void setPodTypes();
704696

705697
/** settings */
706-
const Settings * mSettings;
698+
const Settings * const mSettings;
707699

708700
/** errorlogger */
709701
ErrorLogger* const mErrorLogger;
710702

711703
/** Symbol database that all checks etc can use */
712704
SymbolDatabase *mSymbolDatabase;
713705

714-
TemplateSimplifier *mTemplateSimplifier;
706+
TemplateSimplifier * const mTemplateSimplifier;
715707

716708
/** E.g. "A" for code where "#ifdef A" is true. This is used to
717709
print additional information in error situations. */
@@ -746,7 +738,7 @@ class CPPCHECKLIB Tokenizer {
746738
*/
747739
TimerResults *mTimerResults;
748740

749-
const Preprocessor *mPreprocessor;
741+
const Preprocessor * const mPreprocessor;
750742
};
751743

752744
/// @}

test/testbufferoverrun.cpp

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
#include "errortypes.h"
2424
#include "standards.h"
2525
#include "library.h"
26-
#include "preprocessor.h"
2726
#include "settings.h"
2827
#include "fixture.h"
2928
#include "tokenize.h"
@@ -99,14 +98,10 @@ class TestBufferOverrun : public TestFixture {
9998
std::map<std::string, simplecpp::TokenList*> filedata;
10099
simplecpp::preprocess(tokens2, tokens1, files, filedata, simplecpp::DUI());
101100

102-
Preprocessor preprocessor(*settings, nullptr);
103-
preprocessor.setDirectives(tokens1);
104-
105101
// Tokenizer..
106102
Tokenizer tokenizer(settings, this);
107103
tokenizer.createTokens(std::move(tokens2));
108104
tokenizer.simplifyTokens1("");
109-
tokenizer.setPreprocessor(&preprocessor);
110105

111106
// Check for buffer overruns..
112107
runChecks<CheckBufferOverrun>(&tokenizer, settings, this);

0 commit comments

Comments
 (0)