Skip to content

testrunner: refactored givenACodeSampleToTokenize into SimpleTokenizer and SimpleTokenList / several related cleanups #6084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 10, 2024
Merged

testrunner: refactored givenACodeSampleToTokenize into SimpleTokenizer and SimpleTokenList / several related cleanups #6084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e25d126
testrunner: refactored `givenACodeSampleToTokenize` into `SimpleToken…
firewave Mar 4, 2024
25a6859
testrunner: pass `ErrorLogger` into `SimpleTokenizer`
firewave Mar 5, 2024
cc9b7a7
testrunner: renamed `SimpleTokenList` members
firewave Mar 5, 2024
b9e616b
testrunner: more usage of `Simple*` helpers
firewave Mar 5, 2024
6c37b31
testrunner: made `Simple*` objects `const` if possible
firewave Mar 5, 2024
202ae05
testrunner: pass `Standards::Language` to `SimpleTokenList`
firewave Mar 5, 2024
24cb9cd
testrunner: added missing `TokenList::createTokens()` checks
firewave Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -856,10 +856,10 @@ test/testtoken.o: test/testtoken.cpp lib/addoninfo.h lib/check.h lib/color.h lib
test/testtokenize.o: test/testtokenize.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenize.cpp

test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenlist.cpp

test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h
test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenrange.cpp

test/testtype.o: test/testtype.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/checktype.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
Expand Down
48 changes: 37 additions & 11 deletions test/helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,34 +20,32 @@
#define helpersH

#include "settings.h"
#include "standards.h"
#include "tokenize.h"
#include "tokenlist.h"

#include <cstddef>
#include <stdexcept>
#include <sstream>
#include <string>
#include <vector>

class Token;
class Preprocessor;
class SuppressionList;
class ErrorLogger;
namespace simplecpp {
struct DUI;
}

class givenACodeSampleToTokenize {
private:
const Settings settings;
Tokenizer tokenizer;

class SimpleTokenizer {
public:
explicit givenACodeSampleToTokenize(const char sample[], bool createOnly = false, bool cpp = true)
: tokenizer(settings, nullptr) {
SimpleTokenizer(ErrorLogger& errorlogger, const char sample[], bool cpp = true)
: tokenizer{settings, &errorlogger}
{
std::istringstream iss(sample);
if (createOnly)
tokenizer.list.createTokens(iss, cpp ? "test.cpp" : "test.c");
else
tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c");
if (!tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c"))
throw std::runtime_error("creating tokens failed");
}

Token* tokens() {
Expand All @@ -57,6 +55,34 @@ class givenACodeSampleToTokenize {
const Token* tokens() const {
return tokenizer.tokens();
}

private:
const Settings settings;
Tokenizer tokenizer;
};

class SimpleTokenList
{
public:

explicit SimpleTokenList(const char code[], Standards::Language lang = Standards::Language::CPP)
{
std::istringstream iss(code);
if (!list.createTokens(iss, lang))
throw std::runtime_error("creating tokens failed");
}

Token* front() {
return list.front();
}

const Token* front() const {
return list.front();
}

private:
const Settings settings;
TokenList list{&settings};
};


Expand Down
83 changes: 36 additions & 47 deletions test/testlibrary.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,8 @@ class TestLibrary : public TestFixture {
" </function>\n"
"</def>";

TokenList tokenList(&settings);
std::istringstream istr("foo();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo();";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

Library library;
Expand All @@ -140,16 +139,14 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
{
TokenList tokenList(&settings);
std::istringstream istr("fred.foo(123);"); // <- wrong scope, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "fred.foo(123);"; // <- wrong scope, not library function
const SimpleTokenList tokenList(code);

ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2)));
}
{
TokenList tokenList(&settings);
std::istringstream istr("Fred::foo(123);"); // <- wrong scope, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "Fred::foo(123);"; // <- wrong scope, not library function
const SimpleTokenList tokenList(code);

ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2)));
}
Expand All @@ -165,7 +162,7 @@ class TestLibrary : public TestFixture {

TokenList tokenList(&settings);
std::istringstream istr("foo();"); // <- too few arguments, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -189,7 +186,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo();"); // <- too few arguments, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -198,7 +195,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo(a);"); // <- library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -207,7 +204,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo(a, b);"); // <- library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -216,7 +213,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo(a, b, c);"); // <- too much arguments, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -232,9 +229,8 @@ class TestLibrary : public TestFixture {
" </function>\n"
"</def>";

TokenList tokenList(&settings);
std::istringstream istr("Fred foo(123);"); // <- Variable declaration, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "Fred foo(123);"; // <- Variable declaration, not library function
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());
tokenList.front()->next()->varId(1);

Expand Down Expand Up @@ -292,9 +288,8 @@ class TestLibrary : public TestFixture {
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
ASSERT_EQUALS(0, library.functions["foo"].argumentChecks[-1].notuninit);

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d,e);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d,e);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

ASSERT_EQUALS(false, library.isuninitargbad(tokenList.front(), 1));
Expand All @@ -317,9 +312,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

ASSERT(Library::ArgumentChecks::Direction::DIR_IN == library.getArgDirection(tokenList.front(), 1));
Expand Down Expand Up @@ -349,9 +343,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d,e,f,g,h,i,j,k);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d,e,f,g,h,i,j,k);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

// 1-
Expand Down Expand Up @@ -491,9 +484,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d,e);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d,e);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

// arg1: type=strlen arg2
Expand Down Expand Up @@ -554,16 +546,14 @@ class TestLibrary : public TestFixture {
ASSERT(library.functions.at("bar").argumentChecks.empty());

{
TokenList tokenList(&settings);
std::istringstream istr("Foo::foo();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "Foo::foo();";
const SimpleTokenList tokenList(code);
ASSERT(library.isnotnoreturn(tokenList.front()->tokAt(2)));
}

{
TokenList tokenList(&settings);
std::istringstream istr("bar();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "bar();";
const SimpleTokenList tokenList(code);
ASSERT(library.isnotnoreturn(tokenList.front()));
}
}
Expand Down Expand Up @@ -635,9 +625,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("a(); b();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "a(); b();";
const SimpleTokenList tokenList(code);

const Library::WarnInfo* a = library.getWarnInfo(tokenList.front());
const Library::WarnInfo* b = library.getWarnInfo(tokenList.front()->tokAt(4));
Expand Down Expand Up @@ -791,7 +780,7 @@ class TestLibrary : public TestFixture {
}
}

void container() const {
void container() {
constexpr char xmldata[] = "<?xml version=\"1.0\"?>\n"
"<def>\n"
" <container id=\"A\" startPattern=\"std :: A &lt;\" endPattern=\"&gt; !!::\" itEndPattern=\"&gt; :: iterator\">\n"
Expand Down Expand Up @@ -871,7 +860,7 @@ class TestLibrary : public TestFixture {
ASSERT_EQUALS(C.arrayLike_indexOp, true);

{
givenACodeSampleToTokenize var("std::A<int> a;");
const SimpleTokenizer var(*this, "std::A<int> a;");
ASSERT_EQUALS(&A, library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -880,14 +869,14 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("std::A<int>::size_type a_s;");
const SimpleTokenizer var(*this, "std::A<int>::size_type a_s;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
givenACodeSampleToTokenize var("std::A<int>::iterator a_it;");
const SimpleTokenizer var(*this, "std::A<int>::iterator a_it;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT_EQUALS(&A, library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -896,7 +885,7 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("std::B<int> b;");
const SimpleTokenizer var(*this, "std::B<int> b;");
ASSERT_EQUALS(&B, library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -905,14 +894,14 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("std::B<int>::size_type b_s;");
const SimpleTokenizer var(*this, "std::B<int>::size_type b_s;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
givenACodeSampleToTokenize var("std::B<int>::iterator b_it;");
const SimpleTokenizer var(*this, "std::B<int>::iterator b_it;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT_EQUALS(&B, library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -921,14 +910,14 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("C c;");
const SimpleTokenizer var(*this, "C c;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
givenACodeSampleToTokenize var("D d;");
const SimpleTokenizer var(*this, "D d;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
Expand Down
12 changes: 8 additions & 4 deletions test/testsimplifytemplate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5290,7 +5290,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down Expand Up @@ -5357,7 +5358,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down Expand Up @@ -5427,7 +5429,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down Expand Up @@ -5456,7 +5459,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down
Loading