testrunner: added options -n (no summary) and -d (dry run) / added script to run test cases separately / adjusted some tests (#6049)

This script runs each test case separately so we can uncover cases where
it depends on left-over data on the previous one. `CTest` is only
running the fixtures separately so that does not.

Author: firewave

test/fixture.cpp

@@ -106,7 +106,7 @@ bool TestFixture::prepareTest(const char testname[])
         } else {
             std::cout << classname << "::" << mTestname << std::endl;
         }
-        return true;
+        return !dry_run;
     }
     return false;
 }
@@ -301,8 +301,25 @@ void TestFixture::assertThrowFail(const char * const filename, const unsigned in
 void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
 {
     ++fails_counter;
+
+    std::string ex_msg;
+
+    try {
+        // cppcheck-suppress rethrowNoCurrentException
+        throw;
+    }
+    catch (const InternalError& e) {
+        ex_msg = e.errorMessage;
+    }
+    catch (const std::exception& e) {
+        ex_msg = e.what();
+    }
+    catch (...) {
+        ex_msg = "unknown exception";
+    }
+
     errmsg << getLocationStr(filename, linenr) << ": Assertion failed. "
-           << "Unexpected exception was thrown"  << std::endl << "_____" << std::endl;
+           << "Unexpected exception was thrown: " << ex_msg << std::endl << "_____" << std::endl;
 
 }
 
@@ -323,7 +340,9 @@ void TestFixture::printHelp()
         "\n"
         "Options:\n"
         "    -q                   Do not print the test cases that have run.\n"
-        "    -h, --help           Print this help.\n";
+        "    -h, --help           Print this help.\n"
+        "    -n                   Print no summaries.\n"
+        "    -d                   Do not execute the tests.\n";
 }
 
 void TestFixture::run(const std::string &str)
@@ -355,6 +374,7 @@ void TestFixture::run(const std::string &str)
 void TestFixture::processOptions(const options& args)
 {
     quiet_tests = args.quiet();
+    dry_run = args.dry_run();
     exename = args.exe();
 }
 
@@ -378,15 +398,19 @@ std::size_t TestFixture::runTests(const options& args)
         }
     }
 
-    std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl;
-    std::cout << "Number of todos: " << todos_counter;
-    if (succeeded_todos_counter > 0)
-        std::cout << " (" << succeeded_todos_counter << " succeeded)";
-    std::cout << std::endl;
+    if (args.summary() && !args.dry_run()) {
+        std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl;
+        std::cout << "Number of todos: " << todos_counter;
+        if (succeeded_todos_counter > 0)
+            std::cout << " (" << succeeded_todos_counter << " succeeded)";
+        std::cout << std::endl;
+    }
     // calling flush here, to do all output before the error messages (in case the output is buffered)
     std::cout.flush();
 
-    std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl;
+    if (args.summary() && !args.dry_run()) {
+        std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl;
+    }
     std::cerr << errmsg.str();
 
     std::cerr.flush();

test/fixture.h

@@ -59,6 +59,7 @@ class TestFixture : public ErrorLogger {
     std::string exename;
     std::string testToRun;
     bool quiet_tests{};
+    bool dry_run{};
 
     virtual void run() = 0;
 
@@ -255,6 +256,8 @@ class TestFixture : public ErrorLogger {
 
     std::ostringstream errout;
 
+    const Settings settingsDefault;
+
 private:
     std::ostringstream mOutput;
 
@@ -276,7 +279,9 @@ class TestFixture : public ErrorLogger {
 #define ASSERT( CONDITION )  if (!assert_(__FILE__, __LINE__, (CONDITION))) return
 #define ASSERT_LOC( CONDITION, FILE_, LINE_ )  assert_(FILE_, LINE_, (CONDITION))
 #define CHECK_EQUALS( EXPECTED, ACTUAL )  assertEquals(__FILE__, __LINE__, (EXPECTED), (ACTUAL))
-#define ASSERT_EQUALS( EXPECTED, ACTUAL )  if (!assertEquals(__FILE__, __LINE__, (EXPECTED), (ACTUAL))) return
+// *INDENT-OFF*
+#define ASSERT_EQUALS( EXPECTED, ACTUAL ) do { try { if (!assertEquals(__FILE__, __LINE__, (EXPECTED), (ACTUAL))) return; } catch (...) { assertNoThrowFail(__FILE__, __LINE__); } } while (false)
+// *INDENT-ON*
 #define ASSERT_EQUALS_WITHOUT_LINENUMBERS( EXPECTED, ACTUAL )  assertEqualsWithoutLineNumbers(__FILE__, __LINE__, EXPECTED, ACTUAL)
 #define ASSERT_EQUALS_DOUBLE( EXPECTED, ACTUAL, TOLERANCE )  assertEqualsDouble(__FILE__, __LINE__, EXPECTED, ACTUAL, TOLERANCE)
 #define ASSERT_EQUALS_MSG( EXPECTED, ACTUAL, MSG )  assertEquals(__FILE__, __LINE__, EXPECTED, ACTUAL, MSG)

test/options.cpp

@@ -20,6 +20,8 @@ options::options(int argc, const char* const argv[])
     : mWhichTests(argv + 1, argv + argc)
     ,mQuiet(mWhichTests.count("-q") != 0)
     ,mHelp(mWhichTests.count("-h") != 0 || mWhichTests.count("--help"))
+    ,mSummary(mWhichTests.count("-n") == 0)
+    ,mDryRun(mWhichTests.count("-d") != 0)
     ,mExe(argv[0])
 {
     for (std::set<std::string>::const_iterator it = mWhichTests.cbegin(); it != mWhichTests.cend();) {
@@ -44,6 +46,16 @@ bool options::help() const
     return mHelp;
 }
 
+bool options::summary() const
+{
+    return mSummary;
+}
+
+bool options::dry_run() const
+{
+    return mDryRun;
+}
+
 const std::set<std::string>& options::which_test() const
 {
     return mWhichTests;

test/options.h

@@ -33,6 +33,10 @@ class options {
     bool quiet() const;
     /** Print help. */
     bool help() const;
+    /** Print summary. */
+    bool summary() const;
+    /** Perform dry run. */
+    bool dry_run() const;
     /** Which test should be run. Empty string means 'all tests' */
     const std::set<std::string>& which_test() const;
 
@@ -46,6 +50,8 @@ class options {
     std::set<std::string> mWhichTests;
     const bool mQuiet;
     const bool mHelp;
+    const bool mSummary;
+    const bool mDryRun;
     std::string mExe;
 };
 

test/scripts/testrunner-single.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ -n "$1" ]; then
+  testrunner_bin=$1
+else
+  make -s -C "$SCRIPT_DIR/../.." -j"$(nproc)" testrunner # CXXFLAGS="-g -O2 -w -DHAVE_BOOST"
+  testrunner_bin=$SCRIPT_DIR/../../testrunner
+fi
+
+ec=0
+
+tests=$($testrunner_bin -d | cut -d'(' -f2 | cut -d')' -f1)
+for test in $tests; do
+  $testrunner_bin -n "$test" || ec=1
+done
+
+exit $ec

test/testastutils.cpp

@@ -92,8 +92,7 @@ class TestAstUtils : public TestFixture {
 
 #define findLambdaStartToken(code) findLambdaStartToken_(code, __FILE__, __LINE__)
     bool findLambdaStartToken_(const char code[], const char* file, int line) {
-        const Settings settings;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
         const Token * const tokStart = (::findLambdaStartToken)(tokenizer.list.back());
@@ -125,8 +124,7 @@ class TestAstUtils : public TestFixture {
 
 #define isNullOperand(code) isNullOperand_(code, __FILE__, __LINE__)
     bool isNullOperand_(const char code[], const char* file, int line) {
-        const Settings settings;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
         return (::isNullOperand)(tokenizer.tokens());
@@ -147,8 +145,7 @@ class TestAstUtils : public TestFixture {
 
 #define isReturnScope(code, offset) isReturnScope_(code, offset, __FILE__, __LINE__)
     bool isReturnScope_(const char code[], int offset, const char* file, int line) {
-        const Settings settings;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
         const Token * const tok = (offset < 0)
@@ -178,9 +175,8 @@ class TestAstUtils : public TestFixture {
 
 #define isSameExpression(...) isSameExpression_(__FILE__, __LINE__, __VA_ARGS__)
     bool isSameExpression_(const char* file, int line, const char code[], const char tokStr1[], const char tokStr2[], bool cpp) {
-        const Settings settings;
         Library library;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         ASSERT_LOC(tokenizer.tokenize(istr, cpp ? "test.cpp" : "test.c"), file, line);
         const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), tokStr1, strlen(tokStr1));
@@ -228,13 +224,12 @@ class TestAstUtils : public TestFixture {
 
 #define isVariableChanged(code, startPattern, endPattern) isVariableChanged_(code, startPattern, endPattern, __FILE__, __LINE__)
     bool isVariableChanged_(const char code[], const char startPattern[], const char endPattern[], const char* file, int line) {
-        const Settings settings;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
         const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), startPattern, strlen(startPattern));
         const Token * const tok2 = Token::findsimplematch(tokenizer.tokens(), endPattern, strlen(endPattern));
-        return (isVariableChanged)(tok1, tok2, 1, false, &settings, /*cpp*/ true);
+        return (isVariableChanged)(tok1, tok2, 1, false, &settingsDefault, /*cpp*/ true);
     }
 
     void isVariableChangedTest() {
@@ -263,14 +258,13 @@ class TestAstUtils : public TestFixture {
 
 #define isVariableChangedByFunctionCall(code, pattern, inconclusive) isVariableChangedByFunctionCall_(code, pattern, inconclusive, __FILE__, __LINE__)
     bool isVariableChangedByFunctionCall_(const char code[], const char pattern[], bool *inconclusive, const char* file, int line) {
-        const Settings settings;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
         const Token * const argtok = Token::findmatch(tokenizer.tokens(), pattern);
         ASSERT_LOC(argtok, file, line);
         int indirect = (argtok->variable() && argtok->variable()->isArray());
-        return (isVariableChangedByFunctionCall)(argtok, indirect, &settings, inconclusive);
+        return (isVariableChangedByFunctionCall)(argtok, indirect, &settingsDefault, inconclusive);
     }
 
     void isVariableChangedByFunctionCallTest() {
@@ -438,8 +432,7 @@ class TestAstUtils : public TestFixture {
 
 #define nextAfterAstRightmostLeaf(code, parentPattern, rightPattern) nextAfterAstRightmostLeaf_(code, parentPattern, rightPattern, __FILE__, __LINE__)
     bool nextAfterAstRightmostLeaf_(const char code[], const char parentPattern[], const char rightPattern[], const char* file, int line) {
-        const Settings settings;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
         const Token * tok = Token::findsimplematch(tokenizer.tokens(), parentPattern, strlen(parentPattern));
@@ -463,8 +456,7 @@ class TestAstUtils : public TestFixture {
     enum class Result {False, True, Fail};
 
     Result isUsedAsBool(const char code[], const char pattern[]) {
-        const Settings settings;
-        Tokenizer tokenizer(settings, this);
+        Tokenizer tokenizer(settingsDefault, this);
         std::istringstream istr(code);
         if (!tokenizer.tokenize(istr, "test.cpp"))
             return Result::Fail;

test/testbufferoverrun.cpp

@@ -3451,9 +3451,7 @@ class TestBufferOverrun : public TestFixture {
                       "test.cpp:2:note:Assign p, buffer with size 10\n"
                       "test.cpp:3:note:Buffer overrun\n", errout.str());
 
-        // TODO: need to reset this but it breaks other tests
-        (void)settingsOld;
-        //settings0 = settingsOld;
+        settings0 = settingsOld;
     }
 
     void buffer_overrun_bailoutIfSwitch() {
@@ -3805,7 +3803,7 @@ class TestBufferOverrun : public TestFixture {
               "    std::string hi = \"hi\" + val;\n"
               "    std::cout << hi << std::endl;\n"
               "}\n");
-        ASSERT_EQUALS("[test.cpp:2] -> [test.cpp:3]: (portability) Undefined behaviour, pointer arithmetic '\"hi\"+val' is out of bounds.\n", errout.str());
+        ASSERT_EQUALS("[test.cpp:3]: (portability) Undefined behaviour, pointer arithmetic '\"hi\"+val' is out of bounds.\n", errout.str());
 
         check("void f(const char* s, int len) {\n" // #11026
               "    const char* end = s + len;\n"
@@ -4053,14 +4051,14 @@ class TestBufferOverrun : public TestFixture {
               "  for (int i = 0; i < 3; i++)\n"
               "    a[i] = NULL;\n"
               "}");
-        ASSERT_EQUALS("[test.cpp:3] -> [test.cpp:4]: (error) Array 'a[2]' accessed at index 2, which is out of bounds.\n", errout.str());
+        ASSERT_EQUALS("[test.cpp:4]: (error) Array 'a[2]' accessed at index 2, which is out of bounds.\n", errout.str());
 
         check("void f() {\n"
               "  int **a = new int*[2];\n"
               "  for (int i = 0; i < 3; i++)\n"
               "    a[i] = NULL;\n"
               "}");
-        ASSERT_EQUALS("[test.cpp:3] -> [test.cpp:4]: (error) Array 'a[2]' accessed at index 2, which is out of bounds.\n", errout.str());
+        ASSERT_EQUALS("[test.cpp:4]: (error) Array 'a[2]' accessed at index 2, which is out of bounds.\n", errout.str());
     }
 
     // statically allocated buffer
@@ -5136,7 +5134,7 @@ class TestBufferOverrun : public TestFixture {
               "    int* p = new int[d];\n"
               "    return p;\n"
               "}\n");
-        ASSERT_EQUALS("[test.cpp:2] -> [test.cpp:3] -> [test.cpp:5]: (warning, inconclusive) Memory allocation size is negative.\n", errout.str());
+        ASSERT_EQUALS("[test.cpp:5]: (warning, inconclusive) Memory allocation size is negative.\n", errout.str());
     }
 
     void negativeArraySize() {

test/testclass.cpp

@@ -8945,21 +8945,20 @@ class TestClass : public TestFixture {
 
 
     void ctu(const std::vector<std::string> &code) {
-        const Settings settings;
         Check &check = getCheck<CheckClass>();
 
         // getFileInfo
         std::list<Check::FileInfo*> fileInfo;
         for (const std::string& c: code) {
-            Tokenizer tokenizer(settings, this);
+            Tokenizer tokenizer(settingsDefault, this);
             std::istringstream istr(c);
             ASSERT(tokenizer.tokenize(istr, (std::to_string(fileInfo.size()) + ".cpp").c_str()));
-            fileInfo.push_back(check.getFileInfo(&tokenizer, &settings));
+            fileInfo.push_back(check.getFileInfo(&tokenizer, &settingsDefault));
         }
 
         // Check code..
         errout.str("");
-        check.analyseWholeProgram(nullptr, fileInfo, settings, *this);
+        check.analyseWholeProgram(nullptr, fileInfo, settingsDefault, *this);
 
         while (!fileInfo.empty()) {
             delete fileInfo.back();

test/testmemleak.cpp

@@ -27,11 +27,6 @@
 #include <list>
 #include <sstream> // IWYU pragma: keep
 
-class TestMemleakInClass;
-class TestMemleakNoVar;
-class TestMemleakStructMember;
-
-
 class TestMemleak : private TestFixture {
 public:
     TestMemleak() : TestFixture("TestMemleak") {}

test/testnullpointer.cpp

@@ -4147,8 +4147,7 @@ class TestNullPointer : public TestFixture {
     }
 
     void functioncalllibrary() {
-        const Settings settings1;
-        Tokenizer tokenizer(settings1,this);
+        Tokenizer tokenizer(settingsDefault,this);
         std::istringstream code("void f() { int a,b,c; x(a,b,c); }");
         ASSERT_EQUALS(true, tokenizer.tokenize(code, "test.c"));
         const Token *xtok = Token::findsimplematch(tokenizer.tokens(), "x");