From 87526e0eddd752a256feb8f2c083e5cd5a525e7a Mon Sep 17 00:00:00 2001 From: Chris Ryder Date: Mon, 5 Mar 2018 14:44:25 +0000 Subject: [PATCH 1/2] Enable a coarse-grained prallelism when running regression test jobs via Makefiles Previously, the makefiles ran the regression test target sequentially, regardless of any -jN option passed to make. In this commit, an extra 'test-batch' target is added which enables running the same regression tests, but with each top-level test directory run as a seperate target. To avoid horribly interleaved output from parallel test jobs, GNU parallel is used to execute the test jobs. This provides line bufferering, such that output from each job is prefixed with a job number. This matches the behaviour of ctest used in the CMake based builds. --- .travis.yml | 8 +++++--- regression/Makefile | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 33152480cb9..48d29fea89b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,6 +62,7 @@ jobs: - libwww-perl - g++-5 - libubsan0 + - parallel before_install: - mkdir bin ; ln -s /usr/bin/gcc-5 bin/gcc # env: COMPILER=g++-5 SAN_FLAGS="-fsanitize=undefined -fno-sanitize-recover -fno-omit-frame-pointer" @@ -76,7 +77,7 @@ jobs: compiler: gcc cache: ccache before_install: - - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache + - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache parallel - export PATH=$PATH:/usr/local/opt/ccache/libexec env: COMPILER="ccache g++" @@ -87,7 +88,7 @@ jobs: compiler: clang cache: ccache before_install: - - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache + - HOMEBREW_NO_AUTO_UPDATE=1 brew install ccache parallel - export PATH=$PATH:/usr/local/opt/ccache/libexec env: - COMPILER="ccache clang++" @@ -132,6 +133,7 @@ jobs: - clang-3.7 - libstdc++-5-dev - libubsan0 + - parallel before_install: - mkdir bin ; ln -s /usr/bin/clang-3.7 bin/gcc - export CCACHE_CPP2=yes @@ -258,7 +260,7 @@ install: script: - if [ -e bin/gcc ] ; then export PATH=$PWD/bin:$PATH ; fi ; - - env UBSAN_OPTIONS=print_stacktrace=1 make -C regression test "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2 + - env UBSAN_OPTIONS=print_stacktrace=1 make -C regression test-parallel "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2 JOBS=2 - make -C unit "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2 - make -C unit test diff --git a/regression/Makefile b/regression/Makefile index 947bb6ef843..21f68712b70 100644 --- a/regression/Makefile +++ b/regression/Makefile @@ -20,15 +20,40 @@ DIRS = ansi-c \ test-script \ # Empty last line -# Check for the existence of $dir. Tests under goto-gcc cannot be run on -# Windows, so appveyor.yml unlinks the entire directory under Windows. +# Tests under goto-gcc cannot be run on Windows, so appveyor.yml unlinks +# the entire directory under Windows. This variable will contain the list +# of directories that actually exist on the current platform. +PLATFORM_DIRS = $(wildcard $(DIRS)) + +# Run all test directories in sequence +.PHONY: test test: - @for dir in $(DIRS); do \ - if [ -d "$$dir" ]; then \ - $(MAKE) -C "$$dir" test || exit 1; \ - fi; \ + @for dir in $(PLATFORM_DIRS); do \ + $(MAKE) "$$dir" || exit 1; \ done; +# Pattern to execute a single test suite directory +.PHONY: $(PLATFORM_DIRS) +$(PLATFORM_DIRS): + @echo "Running $@..." ; + $(MAKE) -C "$@" test || exit 1; + +# Run all test directories using GNU Parallel +.PHONY: test-parallel +.NOTPARALLEL: test-parallel +test-parallel: + @echo "Building with $(JOBS) jobs" + parallel \ + --halt soon,fail=1 \ + --tag \ + --tagstring '{#}:' \ + --linebuffer \ + --jobs $(JOBS) \ + $(MAKE) "{}" \ + ::: $(PLATFORM_DIRS) + + +.PHONY: clean clean: @for dir in *; do \ if [ -d "$$dir" ]; then \ From 691816ba0d089cc574c548ca9f068c7528f74488 Mon Sep 17 00:00:00 2001 From: Chris Ryder Date: Tue, 6 Mar 2018 14:34:20 +0000 Subject: [PATCH 2/2] Improve job scheduling when running regression tests with -jN Currently the regression test suite directories are listed and executed in alphabetical order. However, because the runtimes of the various test suites are not uniform, this leads to poor utilisation of the parallel jobs. For instance, consider four tests, A, B, C and D. Lets assume the runtimes for these hypothetical tests are: A 100 B 50 C 50 D 100 further, lets assume we have two cores to run on. The tests are run in alphabetical order, leading to a job schedule that looks something like: Time Core Test 0 1 A 0 2 B 50 2 C 100 1 D 100 2 200 - All tests complete However, if jobs are scheduled in roughly decending runtime order, we get a job schedule that looks roughly like: Time Core Test 0 1 A 0 2 D 100 1 B 100 2 C 150 - All tests complete This commit re-orders the list of regression suite directories into roughly decending runtime order. --- regression/CMakeLists.txt | 32 ++++++++++++++++++-------------- regression/Makefile | 31 +++++++++++++++++-------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/regression/CMakeLists.txt b/regression/CMakeLists.txt index 9b22e036e09..27009895802 100644 --- a/regression/CMakeLists.txt +++ b/regression/CMakeLists.txt @@ -20,25 +20,29 @@ macro(add_test_pl_tests cmdline) add_test_pl_profile("${TEST_DIR_NAME}" "${cmdline}" -K KNOWNBUG ${ARGN}) endmacro(add_test_pl_tests) -add_subdirectory(ansi-c) +# For the best possible utilisation of multiple cores when +# running tests in parallel, it is important that these directories are +# listed with decreasing runtimes (i.e. longest running at the top) add_subdirectory(cbmc) -add_subdirectory(cbmc-cover) -add_subdirectory(cbmc-cpp) add_subdirectory(cbmc-java) -add_subdirectory(cbmc-java-inheritance) -add_subdirectory(cpp) add_subdirectory(goto-analyzer) -add_subdirectory(goto-analyzer-taint) -add_subdirectory(goto-cc-cbmc) -add_subdirectory(goto-cc-goto-analyzer) -add_subdirectory(goto-diff) +add_subdirectory(ansi-c) +add_subdirectory(jbmc-strings) add_subdirectory(goto-instrument) +add_subdirectory(cpp) +add_subdirectory(strings-smoke-tests) +add_subdirectory(cbmc-cover) add_subdirectory(goto-instrument-typedef) +add_subdirectory(strings) +add_subdirectory(invariants) +add_subdirectory(goto-diff) +add_subdirectory(test-script) +add_subdirectory(goto-analyzer-taint) +add_subdirectory(cbmc-java-inheritance) if(NOT WIN32) add_subdirectory(goto-gcc) endif() -add_subdirectory(invariants) -add_subdirectory(jbmc-strings) -add_subdirectory(strings) -add_subdirectory(strings-smoke-tests) -add_subdirectory(test-script) +add_subdirectory(goto-cc-cbmc) +add_subdirectory(cbmc-cpp) +add_subdirectory(goto-cc-goto-analyzer) + diff --git a/regression/Makefile b/regression/Makefile index 21f68712b70..c5f7c41764f 100644 --- a/regression/Makefile +++ b/regression/Makefile @@ -1,23 +1,26 @@ -DIRS = ansi-c \ - cbmc \ - cbmc-cover \ - cbmc-cpp \ +# For the best possible utilisation of multiple cores when +# running tests in parallel, it is important that these directories are +# listed with decreasing runtimes (i.e. longest running at the top) +DIRS = cbmc \ cbmc-java \ - cbmc-java-inheritance \ - cpp \ goto-analyzer \ - goto-analyzer-taint \ - goto-cc-cbmc \ - goto-cc-goto-analyzer \ - goto-diff \ - goto-gcc \ + ansi-c \ + jbmc-strings \ goto-instrument \ + cpp \ + strings-smoke-tests \ + cbmc-cover \ goto-instrument-typedef \ - invariants \ strings \ - jbmc-strings \ - strings-smoke-tests \ + invariants \ + goto-diff \ test-script \ + goto-analyzer-taint \ + cbmc-java-inheritance \ + goto-gcc \ + goto-cc-cbmc \ + cbmc-cpp \ + goto-cc-goto-analyzer \ # Empty last line # Tests under goto-gcc cannot be run on Windows, so appveyor.yml unlinks