From ef786af12c899442b35a1aadc9f9503b083dc1d7 Mon Sep 17 00:00:00 2001 From: "Romanov, Vlad" Date: Thu, 5 Jan 2023 07:56:42 -0800 Subject: [PATCH 1/7] [SYCL] Enable stack printing on crashes in post commit --- .../workflows/sycl_linux_build_and_test.yml | 7 +++++- .github/workflows/sycl_post_commit.yml | 2 ++ sycl/CMakeLists.txt | 4 ++++ sycl/source/CMakeLists.txt | 11 ++++++++++ sycl/source/detail/global_handler.cpp | 22 +++++++++++++++++++ sycl/test/Unit/lit.cfg.py | 3 +++ sycl/test/lit.cfg.py | 3 +++ 7 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sycl_linux_build_and_test.yml b/.github/workflows/sycl_linux_build_and_test.yml index a49bb0153059b..4aeae141368f1 100644 --- a/.github/workflows/sycl_linux_build_and_test.yml +++ b/.github/workflows/sycl_linux_build_and_test.yml @@ -18,6 +18,10 @@ on: build_ref: type: string required: false + build_type: + type: string + required: false + default: "Release" build_cache_root: type: string required: true @@ -97,12 +101,13 @@ jobs: CACHE_SIZE: ${{ inputs.build_cache_size }} ARGS: ${{ inputs.build_configure_extra_args }} CUDA_LIB_PATH: "/usr/local/cuda/lib64/stubs" + BUILD_TYPE: ${{ inputs.build_type }} run: | mkdir -p $CACHE_ROOT/build_cache_$CACHE_SUFFIX mkdir -p $GITHUB_WORKSPACE/build cd $GITHUB_WORKSPACE/build python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \ - -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \ + -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t $BUILD_TYPE \ --ci-defaults $ARGS \ --cmake-opt="-DLLVM_CCACHE_BUILD=ON" \ --cmake-opt="-DLLVM_CCACHE_DIR=$CACHE_ROOT/build_cache_$CACHE_SUFFIX" \ diff --git a/.github/workflows/sycl_post_commit.yml b/.github/workflows/sycl_post_commit.yml index c149effcb2e52..8175643f1a7f0 100644 --- a/.github/workflows/sycl_post_commit.yml +++ b/.github/workflows/sycl_post_commit.yml @@ -34,6 +34,8 @@ jobs: lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }} cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }} lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }} + build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" + build_type: RelWithDebInfo linux_no_assert: name: Linux (no assert) uses: ./.github/workflows/sycl_linux_build_and_test.yml diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt index fe9050e8f8571..72457a468b091 100644 --- a/sycl/CMakeLists.txt +++ b/sycl/CMakeLists.txt @@ -324,6 +324,10 @@ if (SYCL_ENABLE_XPTI_TRACING) endif() endif() +if (SYCL_ENABLE_STACK_PRINTING) + add_dependencies(sycl-toolchain llvm-symbolizer) +endif() + option(SYCL_INCLUDE_TESTS "Generate build targets for the SYCL unit tests." ${LLVM_INCLUDE_TESTS}) diff --git a/sycl/source/CMakeLists.txt b/sycl/source/CMakeLists.txt index ecd54a4ced599..27dcc690f0941 100644 --- a/sycl/source/CMakeLists.txt +++ b/sycl/source/CMakeLists.txt @@ -26,6 +26,17 @@ function(add_sycl_rt_library LIB_NAME LIB_OBJ_NAME) $ ${CMAKE_CURRENT_BINARY_DIR}/version.rc) + # Unlike for sycl library, for LLVMSupport we have only one version for a given build, + # so, we link LLVMSupport lib to matching sycl version only. + if (SYCL_ENABLE_STACK_PRINTING) + if(NOT MSVC OR (CMAKE_BUILD_TYPE STREQUAL "Debug" AND ARG_COMPILE_OPTIONS MATCHES ".*MDd.*") OR + (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT ARG_COMPILE_OPTIONS MATCHES ".*MDd.*")) + add_dependencies(${LIB_NAME} LLVMSupport) + target_compile_definitions(${LIB_OBJ_NAME} PUBLIC ENABLE_STACK_TRACE) + target_link_libraries(${LIB_NAME} PRIVATE LLVMSupport) + endif() + endif() + if (SYCL_ENABLE_COVERAGE) target_compile_options(${LIB_OBJ_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index 78b44330aedb7..feb6e1ab13293 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -6,6 +6,11 @@ // //===----------------------------------------------------------------------===// +#ifdef ENABLE_STACK_TRACE +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Signals.h" +#endif + #include #include #include @@ -87,9 +92,25 @@ void GlobalHandler::attachScheduler(Scheduler *Scheduler) { MScheduler.Inst.reset(Scheduler); } +static void enableOnCrashStackPrinting() { +#ifdef ENABLE_STACK_TRACE + static std::once_flag PrintStackFlag; + std::call_once(PrintStackFlag, []() { + llvm::sys::PrintStackTraceOnErrorSignal(llvm::StringRef()); + }); +#endif +} + Scheduler &GlobalHandler::getScheduler() { getOrCreate(MScheduler); registerSchedulerUsage(); + // On Windows the regestration of the signal handler before main function + // (e.g. from DLLMain or from constructors of program scope objects) doesn't + // work. So, registering signal handler here because: + // 1) getScheduler is likely to be called for any non-trivial application; + // 2) first call to getScheduler is likely to be done after main starts + // The same is done in getPlugins + enableOnCrashStackPrinting(); return *MScheduler.Inst; } @@ -125,6 +146,7 @@ std::mutex &GlobalHandler::getFilterMutex() { return getOrCreate(MFilterMutex); } std::vector &GlobalHandler::getPlugins() { + enableOnCrashStackPrinting(); return getOrCreate(MPlugins); } device_filter_list & diff --git a/sycl/test/Unit/lit.cfg.py b/sycl/test/Unit/lit.cfg.py index 9ad08d5cde55c..162ff0ebbee3b 100644 --- a/sycl/test/Unit/lit.cfg.py +++ b/sycl/test/Unit/lit.cfg.py @@ -50,6 +50,9 @@ if symbolizer in os.environ: config.environment[symbolizer] = os.environ[symbolizer] +llvm_symbolizer = os.path.join(config.llvm_tools_dir, 'llvm-symbolizer') +config.environment['LLVM_SYMBOLIZER_PATH'] = llvm_symbolizer + def find_shlibpath_var(): if platform.system() in ['Linux', 'FreeBSD', 'NetBSD', 'SunOS']: yield 'LD_LIBRARY_PATH' diff --git a/sycl/test/lit.cfg.py b/sycl/test/lit.cfg.py index 76f9f8b29d48a..2b5c91b111e82 100644 --- a/sycl/test/lit.cfg.py +++ b/sycl/test/lit.cfg.py @@ -91,6 +91,9 @@ config.substitutions.append( ('%llvm_build_lib_dir', config.llvm_build_lib_dir ) ) config.substitutions.append( ('%llvm_build_bin_dir', config.llvm_build_bin_dir ) ) +llvm_symbolizer = os.path.join(config.llvm_build_bin_dir, 'llvm-symbolizer') +llvm_config.with_environment('LLVM_SYMBOLIZER_PATH', llvm_symbolizer) + config.substitutions.append( ('%fsycl-host-only', '-std=c++17 -Xclang -fsycl-is-host -isystem %s -isystem %s -isystem %s -isystem %s' % (config.sycl_include, config.level_zero_include_dir, config.opencl_include_dir, config.sycl_include + '/sycl/') ) ) config.substitutions.append( ('%sycl_lib', ' -lsycl6' if platform.system() == "Windows" else '-lsycl') ) From 17cc118849567f7a463f6d6a0597af8c08e2147b Mon Sep 17 00:00:00 2001 From: "Romanov, Vlad" Date: Mon, 9 Jan 2023 04:08:18 -0800 Subject: [PATCH 2/7] Build without debug info --- .github/workflows/sycl_post_commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl_post_commit.yml b/.github/workflows/sycl_post_commit.yml index 8175643f1a7f0..28b928bf0fc3b 100644 --- a/.github/workflows/sycl_post_commit.yml +++ b/.github/workflows/sycl_post_commit.yml @@ -35,7 +35,7 @@ jobs: cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }} lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }} build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" - build_type: RelWithDebInfo + build_type: Release linux_no_assert: name: Linux (no assert) uses: ./.github/workflows/sycl_linux_build_and_test.yml From bf7aab17955ae7818031d87fb389bff933c05bec Mon Sep 17 00:00:00 2001 From: "Romanov, Vlad" Date: Mon, 9 Jan 2023 04:09:39 -0800 Subject: [PATCH 3/7] grammar --- sycl/source/detail/global_handler.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sycl/source/detail/global_handler.cpp b/sycl/source/detail/global_handler.cpp index feb6e1ab13293..a3280a4a3b3ee 100644 --- a/sycl/source/detail/global_handler.cpp +++ b/sycl/source/detail/global_handler.cpp @@ -104,12 +104,12 @@ static void enableOnCrashStackPrinting() { Scheduler &GlobalHandler::getScheduler() { getOrCreate(MScheduler); registerSchedulerUsage(); - // On Windows the regestration of the signal handler before main function + // On Windows the registration of the signal handler before main function // (e.g. from DLLMain or from constructors of program scope objects) doesn't // work. So, registering signal handler here because: // 1) getScheduler is likely to be called for any non-trivial application; - // 2) first call to getScheduler is likely to be done after main starts - // The same is done in getPlugins + // 2) first call to getScheduler is likely to be done after main starts. + // The same is done in getPlugins. enableOnCrashStackPrinting(); return *MScheduler.Inst; } From 922e4e1d2edc44efb894c3b39e0293bb29723c9a Mon Sep 17 00:00:00 2001 From: "Romanov, Vlad" Date: Tue, 10 Jan 2023 02:47:33 -0800 Subject: [PATCH 4/7] Limit link threads --- .github/workflows/sycl_post_commit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl_post_commit.yml b/.github/workflows/sycl_post_commit.yml index 28b928bf0fc3b..850b665c43e90 100644 --- a/.github/workflows/sycl_post_commit.yml +++ b/.github/workflows/sycl_post_commit.yml @@ -34,8 +34,8 @@ jobs: lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }} cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }} lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }} - build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" - build_type: Release + build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON -DLLVM_PARALLEL_LINK_JOBS=4" + build_type: RelWithDebInfo linux_no_assert: name: Linux (no assert) uses: ./.github/workflows/sycl_linux_build_and_test.yml From 4eed011fc84eaafa95a78f6ebfb1fd960f1a0186 Mon Sep 17 00:00:00 2001 From: "Romanov, Vlad" Date: Tue, 10 Jan 2023 04:01:55 -0800 Subject: [PATCH 5/7] Limit link threads --- .github/workflows/sycl_post_commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl_post_commit.yml b/.github/workflows/sycl_post_commit.yml index 850b665c43e90..38e7cbfea9782 100644 --- a/.github/workflows/sycl_post_commit.yml +++ b/.github/workflows/sycl_post_commit.yml @@ -34,7 +34,7 @@ jobs: lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }} cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }} lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }} - build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON -DLLVM_PARALLEL_LINK_JOBS=4" + build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" --cmake-opt="-DLLVM_PARALLEL_LINK_JOBS=4" build_type: RelWithDebInfo linux_no_assert: name: Linux (no assert) From a324c1bd822a36c4b74a0971bcc221a2ce740006 Mon Sep 17 00:00:00 2001 From: "Romanov, Vlad" Date: Tue, 10 Jan 2023 06:39:52 -0800 Subject: [PATCH 6/7] debug info for sycl.so only --- .github/workflows/sycl_linux_build_and_test.yml | 7 +------ .github/workflows/sycl_post_commit.yml | 4 ++-- sycl/CMakeLists.txt | 2 ++ sycl/cmake/modules/AddSYCL.cmake | 6 ++++++ sycl/source/CMakeLists.txt | 6 ++++++ 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sycl_linux_build_and_test.yml b/.github/workflows/sycl_linux_build_and_test.yml index 4aeae141368f1..a49bb0153059b 100644 --- a/.github/workflows/sycl_linux_build_and_test.yml +++ b/.github/workflows/sycl_linux_build_and_test.yml @@ -18,10 +18,6 @@ on: build_ref: type: string required: false - build_type: - type: string - required: false - default: "Release" build_cache_root: type: string required: true @@ -101,13 +97,12 @@ jobs: CACHE_SIZE: ${{ inputs.build_cache_size }} ARGS: ${{ inputs.build_configure_extra_args }} CUDA_LIB_PATH: "/usr/local/cuda/lib64/stubs" - BUILD_TYPE: ${{ inputs.build_type }} run: | mkdir -p $CACHE_ROOT/build_cache_$CACHE_SUFFIX mkdir -p $GITHUB_WORKSPACE/build cd $GITHUB_WORKSPACE/build python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \ - -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t $BUILD_TYPE \ + -s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \ --ci-defaults $ARGS \ --cmake-opt="-DLLVM_CCACHE_BUILD=ON" \ --cmake-opt="-DLLVM_CCACHE_DIR=$CACHE_ROOT/build_cache_$CACHE_SUFFIX" \ diff --git a/.github/workflows/sycl_post_commit.yml b/.github/workflows/sycl_post_commit.yml index 38e7cbfea9782..fd223c3b3a4ea 100644 --- a/.github/workflows/sycl_post_commit.yml +++ b/.github/workflows/sycl_post_commit.yml @@ -34,8 +34,8 @@ jobs: lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }} cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }} lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }} - build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" --cmake-opt="-DLLVM_PARALLEL_LINK_JOBS=4" - build_type: RelWithDebInfo + build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" \ + --cmake-opt="-DSYCL_LIB_WITH_DEBUG_SYMBOL=ON" linux_no_assert: name: Linux (no assert) uses: ./.github/workflows/sycl_linux_build_and_test.yml diff --git a/sycl/CMakeLists.txt b/sycl/CMakeLists.txt index 72457a468b091..dddb5ecd5c921 100644 --- a/sycl/CMakeLists.txt +++ b/sycl/CMakeLists.txt @@ -9,6 +9,8 @@ option(SYCL_ENABLE_WERROR "Treat all warnings as errors in SYCL project" OFF) option(SYCL_DISABLE_STL_ASSERTIONS "Disable assertions in STL containers" OFF) option(SYCL_ADD_DEV_VERSION_POSTFIX "Adds -V postfix to version string" ON) option(SYCL_ENABLE_COVERAGE "Enables code coverage for runtime and unit tests" OFF) +option(SYCL_ENABLE_STACK_PRINTING "Enables stack printing on crashes of SYCL applications" OFF) +option(SYCL_LIB_WITH_DEBUG_SYMBOLS "Builds SYCL runtime libraries with debug symbols" OFF) if (NOT SYCL_COVERAGE_PATH) set(SYCL_COVERAGE_PATH "${CMAKE_CURRENT_BINARY_DIR}/profiles") diff --git a/sycl/cmake/modules/AddSYCL.cmake b/sycl/cmake/modules/AddSYCL.cmake index b9869c94696b1..07d3d7f91e005 100644 --- a/sycl/cmake/modules/AddSYCL.cmake +++ b/sycl/cmake/modules/AddSYCL.cmake @@ -24,6 +24,12 @@ function(add_sycl_library LIB_NAME TYPE) add_stripped_pdb(${LIB_NAME}) endif() + # TODO: Enabled for MSVC + if (NOT MSVC AND SYCL_LIB_WITH_DEBUG_SYMBOLS) + separate_arguments(CMAKE_CXX_FLAGS_DEBUG_SEPARATED UNIX_COMMAND "${CMAKE_CXX_FLAGS_DEBUG}") + target_compile_options(${LIB_NAME} PRIVATE ${CMAKE_CXX_FLAGS_DEBUG_SEPARATED}) + endif() + # TODO remove add_common_options add_common_options(${LIB_NAME}) endfunction() diff --git a/sycl/source/CMakeLists.txt b/sycl/source/CMakeLists.txt index 27dcc690f0941..aea374592ded4 100644 --- a/sycl/source/CMakeLists.txt +++ b/sycl/source/CMakeLists.txt @@ -37,6 +37,12 @@ function(add_sycl_rt_library LIB_NAME LIB_OBJ_NAME) endif() endif() + # TODO: Enabled for MSVC + if (NOT MSVC AND SYCL_LIB_WITH_DEBUG_SYMBOLS) + separate_arguments(CMAKE_CXX_FLAGS_DEBUG_SEPARATED UNIX_COMMAND "${CMAKE_CXX_FLAGS_DEBUG}") + target_compile_options(${LIB_NAME} PRIVATE ${CMAKE_CXX_FLAGS_DEBUG_SEPARATED}) + endif() + if (SYCL_ENABLE_COVERAGE) target_compile_options(${LIB_OBJ_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping From bf12e22fa95665a5bf4728bd956679e3495c2368 Mon Sep 17 00:00:00 2001 From: "Romanov, Vlad" Date: Wed, 11 Jan 2023 04:01:49 -0800 Subject: [PATCH 7/7] fix multiple lines --- .github/workflows/sycl_post_commit.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sycl_post_commit.yml b/.github/workflows/sycl_post_commit.yml index fd223c3b3a4ea..838dc65760954 100644 --- a/.github/workflows/sycl_post_commit.yml +++ b/.github/workflows/sycl_post_commit.yml @@ -34,8 +34,7 @@ jobs: lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }} cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }} lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }} - build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" \ - --cmake-opt="-DSYCL_LIB_WITH_DEBUG_SYMBOL=ON" + build_configure_extra_args: --cmake-opt="-DSYCL_ENABLE_STACK_PRINTING=ON" --cmake-opt="-DSYCL_LIB_WITH_DEBUG_SYMBOL=ON" linux_no_assert: name: Linux (no assert) uses: ./.github/workflows/sycl_linux_build_and_test.yml