diff --git a/.evergreen/abi-compliance-check-setup.sh b/.evergreen/abi-compliance-check-setup.sh new file mode 100755 index 0000000000..fcbc97a56e --- /dev/null +++ b/.evergreen/abi-compliance-check-setup.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +declare working_dir +working_dir="$(pwd)" + +export PATH +PATH="${working_dir:?}/install/bin:${PATH:-}" + +# Install prefix to use for ABI compatibility scripts. +[[ -d "${working_dir}/install" ]] + +declare parallel_level +parallel_level="$(("$(nproc)" + 1))" + +# Obtain abi-compliance-checker. +echo "Fetching abi-compliance-checker..." +[[ -d checker ]] || { + git clone -b "2.3" --depth 1 https://github.com/lvc/abi-compliance-checker.git checker + pushd checker + make -j "${parallel_level:?}" --no-print-directory install prefix="${working_dir:?}/install" + popd # checker +} >/dev/null +echo "Fetching abi-compliance-checker... done." + +# Obtain ctags. +echo "Fetching ctags..." +[[ -d ctags ]] || { + git clone -b "v6.0.0" --depth 1 https://github.com/universal-ctags/ctags.git ctags + pushd ctags + ./autogen.sh + ./configure --prefix="${working_dir}/install" + make -j "${parallel_level:?}" + make install + popd # ctags +} >/dev/null +echo "Fetching ctags... done." + +command -V abi-compliance-checker diff --git a/.evergreen/abi-compliance-check-test.sh b/.evergreen/abi-compliance-check-test.sh new file mode 100755 index 0000000000..d359840b89 --- /dev/null +++ b/.evergreen/abi-compliance-check-test.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +declare working_dir +working_dir="$(pwd)" + +declare base current +base="$(cat base-commit.txt)" +current="$(cat current-commit.txt)" + +export PATH +PATH="${working_dir:?}/install/bin:${PATH:-}" + +# Remove 'r' prefix in version string. +declare old_ver new_ver +old_ver="${base:1}-base" +new_ver="${current:1}-current" + +command -V abi-compliance-checker >/dev/null + +mkdir cxx-abi cxx-noabi + +cat >cxx-abi/old.xml < + ${old_ver:?} + + + + ../install/old/include/bsoncxx/ + ../install/old/include/mongocxx/ + + + + /v_noabi/ + + + + ../install/old/lib + + + + ../install/old/include/ + + + + bsoncxx/enums/ + /config/ + +DOC + +cat >cxx-abi/new.xml < + ${new_ver:?} + + + + ../install/new/include/mongocxx/ + ../install/new/include/bsoncxx/ + + + + /v_noabi/ + + + + ../install/new/lib + + + + ../install/new/include/ + + + + bsoncxx/enums/ + /config/ + +DOC + +cat >cxx-noabi/old.xml < + ${old_ver:?} + + + + ../install/old/include/bsoncxx/v_noabi + ../install/old/include/mongocxx/v_noabi + + + + ../install/old/lib + + + + ../install/old/include/ + + + + bsoncxx/enums/ + /config/ + +DOC + +cat >cxx-noabi/new.xml < + ${new_ver:?} + + + + ../install/new/include/bsoncxx/v_noabi + ../install/new/include/mongocxx/v_noabi + + + + ../install/new/lib + + + + ../install/new/include/ + + + + bsoncxx/enums/ + /config/ + +DOC + +# Allow task to upload the HTML report despite failed status. +echo "Generating stable ABI report..." +pushd cxx-abi +if ! abi-compliance-checker -lib mongo-cxx-driver -old old.xml -new new.xml; then + : # CXX-2812: enable code below once stable ABI symbols exist in the base commit libraries. + # declare status + # status='{"status":"failed", "type":"test", "should_continue":true, "desc":"abi-compliance-checker emitted one or more errors"}' + # curl -sS -d "${status:?}" -H "Content-Type: application/json" -X POST localhost:2285/task_status || true +fi +popd # cxx-abi +echo "Generating stable ABI report... done." + +# Also create a report for the unstable ABI. Errors are expected and OK. +echo "Generating unstable ABI report..." +pushd cxx-noabi +abi-compliance-checker -lib mongo-cxx-driver -old old.xml -new new.xml || true +popd # cxx-noabi +echo "Generating unstable ABI report... done." diff --git a/.evergreen/abi-prohibited-symbols-test.sh b/.evergreen/abi-prohibited-symbols-test.sh new file mode 100755 index 0000000000..3fd77b3a96 --- /dev/null +++ b/.evergreen/abi-prohibited-symbols-test.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +command -V nm >/dev/null + +declare -a libs +libs=( + install/new/lib/libbsoncxx.so + install/new/lib/libmongocxx.so +) + +for lib in "${libs[@]}"; do + [[ -f "${lib:?}" ]] || { + echo "error: missing ${lib:?}" + exit 1 + } 1>&2 +done + +# Patterns for library symbols to check. +match_pattern=( + -e '(bsoncxx|mongocxx)::' +) + +# Patterns for bad symbols. +bad_pattern=( + -e '(bsoncxx|mongocxx)::(.+::)?detail::' + -e '(bsoncxx|mongocxx)::(.+::)?test::' +) + +# Ensure implementation details do not leak into the ABI. +mapfile -t bad_symbols < <( + nm --demangle --dynamic --defined-only --extern-only --just-symbols "${libs[@]}" | + grep -E "${match_pattern[@]}" | + grep -E "${bad_pattern[@]}" +) + +# Print list of bad symbols. +(("${#bad_symbols[*]}" == 0)) || { + echo "error: found ${#bad_symbols[@]} prohibited symbols in exported ABI:" + printf " - %s\n" "${bad_symbols[@]}" + exit 1 +} 1>&2 diff --git a/.evergreen/abi-stability-setup.sh b/.evergreen/abi-stability-setup.sh new file mode 100755 index 0000000000..39ccdcf02a --- /dev/null +++ b/.evergreen/abi-stability-setup.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +: "${cxx_standard:?}" # Set by abi-stability-checks-* build variant definition. + +command -V git >/dev/null + +# Files prepared by EVG config. +[[ -d "mongoc" ]] || { + echo "missing mongoc" 1>&2 + exit 1 +} +[[ -d "mongo-cxx-driver" ]] || { + echo "missing mongo-cxx-driver" 1>&2 + exit 1 +} + +declare working_dir +working_dir="$(pwd)" + +declare cmake_binary +# shellcheck source=/dev/null +. ./mongoc/.evergreen/scripts/find-cmake-latest.sh +cmake_binary="$(find_cmake_latest)" +command -V "${cmake_binary:?}" + +# To use a different base commit, replace `--abbrev 0` with the intended commit. +# Note: EVG treat all changes relative to the EVG base commit as staged changes! +declare base current +base="$(git -C mongo-cxx-driver describe --tags --abbrev=0)" +current="$(git -C mongo-cxx-driver describe --tags)" + +echo "Old Version (Base): ${base:?}" +echo "New Version (Current): ${current:?}" + +printf "%s" "${base:?}" >base-commit.txt +printf "%s" "${current:?}" >current-commit.txt + +# Remove 'r' prefix in version string. +declare old_ver new_ver +old_ver="${base:1}" +new_ver="${current:1}" + +declare parallel_level +parallel_level="$(("$(nproc)" + 1))" + +# Use Ninja if available. +if command -V ninja; then + export CMAKE_GENERATOR + CMAKE_GENERATOR="Ninja" +else + export CMAKE_BUILD_PARALLEL_LEVEL + CMAKE_BUILD_PARALLEL_LEVEL="${parallel_level:?}" +fi + +# Use ccache if available. +if command -V ccache; then + export CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER + CMAKE_C_COMPILER_LAUNCHER="ccache" + CMAKE_CXX_COMPILER_LAUNCHER="ccache" +fi + +# Install prefix to use for ABI compatibility scripts. +mkdir -p "${working_dir}/install" + +# As encouraged by ABI compatibility checkers. +export CFLAGS CXXFLAGS +CFLAGS="-g -Og" +CXXFLAGS="-g -Og" + +# Build and install the base commit first. +git -C mongo-cxx-driver stash push -u +git -C mongo-cxx-driver reset --hard "${base:?}" + +# Install old (base) to install/old. +echo "Building old libraries..." +{ + "${cmake_binary:?}" \ + -S mongo-cxx-driver \ + -B build/old \ + -DCMAKE_INSTALL_PREFIX="install/old" \ + -DCMAKE_PREFIX_PATH="${working_dir:?}/mongoc" \ + -DBUILD_VERSION="${old_ver:?}-base" \ + -DCMAKE_CXX_STANDARD="${cxx_standard:?}" + "${cmake_binary:?}" --build build/old + "${cmake_binary:?}" --install build/old +} &>old.log || { + cat old.log 1>&2 + exit 1 +} +echo "Building old libraries... done." + +# Restore all pending changes. +git -C mongo-cxx-driver reset --hard "HEAD@{1}" +git -C mongo-cxx-driver stash pop -q || true # Only patch builds have stashed changes. + +# Install new (current) to install/new. +echo "Building new libraries..." +{ + "${cmake_binary:?}" \ + -S mongo-cxx-driver \ + -B build/new \ + -DCMAKE_INSTALL_PREFIX="install/new" \ + -DCMAKE_PREFIX_PATH="${working_dir:?}/mongoc" \ + -DBUILD_VERSION="${new_ver:?}-current" \ + -DCMAKE_CXX_STANDARD="${cxx_standard:?}" + "${cmake_binary:?}" --build build/new + "${cmake_binary:?}" --install build/new +} &>new.log || { + cat new.log 1>&2 + exit 1 +} +echo "Building new libraries... done." diff --git a/.evergreen/abidiff-setup.sh b/.evergreen/abidiff-setup.sh new file mode 100755 index 0000000000..c39f15a85b --- /dev/null +++ b/.evergreen/abidiff-setup.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +declare working_dir +working_dir="$(pwd)" + +export PATH +PATH="${working_dir:?}/install/bin:${PATH:-}" + +# Install prefix to use for ABI compatibility scripts. +[[ -d "${working_dir}/install" ]] + +if command -V abidiff 2>/dev/null; then + exit # Already available. +fi + +# Expected to be run on Ubuntu. +echo "Installing abigail-tools..." +sudo apt-get install -q -y abigail-tools >/dev/null +echo "Installing abigail-tools... done." + +command -V abidiff diff --git a/.evergreen/abidiff-test.sh b/.evergreen/abidiff-test.sh new file mode 100755 index 0000000000..bac301f3c2 --- /dev/null +++ b/.evergreen/abidiff-test.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +declare working_dir +working_dir="$(pwd)" + +export PATH +PATH="${working_dir:?}/install/bin:${PATH:-}" + +declare -a common_flags +flags=( + --headers-dir1 install/old/include + --headers-dir2 install/new/include + --non-reachable-types + --fail-no-debug-info +) + +declare -a abi_flags=("${common_flags[@]}" --suppressions cxx-abi/abignore) +declare -a noabi_flags=("${common_flags[@]}" --suppressions cxx-noabi/abignore) + +command -V abidiff >/dev/null + +mkdir cxx-abi cxx-noabi + +cat >cxx-abi/abignore <cxx-noabi/abignore <cxx-abi/bsoncxx.txt; then + declare status + status='{"status":"failed", "type":"test", "should_continue":true, "desc":"abidiff returned an error for bsoncxx (stable)"}' + curl -sS -d "${status:?}" -H "Content-Type: application/json" -X POST localhost:2285/task_status || true +fi +echo "Comparing stable ABI for bsoncxx... done." + +# Allow task to upload the diff reports despite failed status. +echo "Comparing stable ABI for mongocxx..." +if ! abidiff "${abi_flags[@]}" install/old/lib/libmongocxx.so install/new/lib/libmongocxx.so &>cxx-abi/mongocxx.txt; then + declare status + status='{"status":"failed", "type":"test", "should_continue":true, "desc":"abidiff returned an error for mongocxx (stable)"}' + curl -sS -d "${status:?}" -H "Content-Type: application/json" -X POST localhost:2285/task_status || true +fi +echo "Comparing stable ABI for mongocxx... done." + +echo "Comparing unstable ABI for bsoncxx..." +abidiff "${noabi_flags[@]}" install/old/lib/libbsoncxx.so install/new/lib/libbsoncxx.so &>cxx-noabi/bsoncxx.txt || true +echo "Comparing unstable ABI for bsoncxx... done." + +echo "Comparing unstable ABI for mongocxx..." +abidiff "${noabi_flags[@]}" install/old/lib/libmongocxx.so install/new/lib/libmongocxx.so &>cxx-noabi/mongocxx.txt || true +echo "Comparing unstable ABI for mongocxx... done." + +# Ensure files have content even when abidiff produces no output. +printf "\n" >>cxx-abi/bsoncxx.txt +printf "\n" >>cxx-abi/mongocxx.txt +printf "\n" >>cxx-noabi/bsoncxx.txt +printf "\n" >>cxx-noabi/mongocxx.txt diff --git a/.mci.yml b/.mci.yml index b0df280423..0f9592bae1 100644 --- a/.mci.yml +++ b/.mci.yml @@ -56,6 +56,103 @@ variables: ####################################### functions: + "abi-compliance-check": + - command: subprocess.exec + type: setup + params: + binary: bash + args: [mongo-cxx-driver/.evergreen/abi-compliance-check-setup.sh] + - command: subprocess.exec + type: test + params: + binary: bash + args: [mongo-cxx-driver/.evergreen/abi-compliance-check-test.sh] + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: text/html + display_name: "ABI Compliance Check (Stable): " + local_files_include_filter: cxx-abi/compat_reports/**/compat_report.html + permissions: public-read + remote_file: mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/abi/ + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: text/plain + display_name: "ABI Compliance Check (Stable): " + local_files_include_filter: cxx-abi/logs/**/log.txt + permissions: public-read + remote_file: mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/abi/ + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: text/html + display_name: "ABI Compliance Check (Unstable): " + local_files_include_filter: cxx-noabi/compat_reports/**/compat_report.html + permissions: public-read + remote_file: mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/noabi/ + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: text/plain + display_name: "ABI Compliance Check (Unstable): " + local_files_include_filter: cxx-noabi/logs/**/log.txt + permissions: public-read + remote_file: mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/noabi/ + + "abidiff": + - command: subprocess.exec + type: setup + params: + binary: bash + args: [mongo-cxx-driver/.evergreen/abidiff-setup.sh] + - command: subprocess.exec + type: test + params: + binary: bash + args: [mongo-cxx-driver/.evergreen/abidiff-test.sh] + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: text/plain + display_name: "abidiff (Stable): " + local_files_include_filter: cxx-abi/*.txt + permissions: public-read + remote_file: mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abidiff/abi/ + - command: s3.put + type: system + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: text/plain + display_name: "abidiff (Unstable): " + local_files_include_filter: cxx-noabi/*.txt + permissions: public-read + remote_file: mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abidiff/noabi/ + + "abi-prohibited-symbols": + - command: subprocess.exec + type: test + params: + binary: bash + args: [mongo-cxx-driver/.evergreen/abi-prohibited-symbols-test.sh] + "setup": - command: shell.exec params: @@ -563,6 +660,22 @@ post: ####################################### tasks: + - name: abi-compliance-check + tags: ["abi-stability", "abi-compliance-check"] + run_on: "ubuntu2204-large" + commands: + - func: "abi-compliance-check" + - name: abidiff + tags: ["abi-stability", "abidiff"] + run_on: "ubuntu2204-large" + commands: + - func: "abidiff" + - name: abi-prohibited-symbols + tags: ["abi-stability", "abi-prohibited-symbols"] + run_on: "ubuntu2204-large" + commands: + - func: "abi-prohibited-symbols" + - name: lint run_on: ubuntu1804-large commands: @@ -987,6 +1100,27 @@ tasks: ./build/src/mongocxx/test/test_driver "atlas search indexes prose tests" task_groups: + - name: tg-abi-stability + max_hosts: -1 + setup_task_can_fail_task: true + setup_task: + - command: git.get_project + params: + directory: "mongo-cxx-driver" + - func: install_c_driver + - command: subprocess.exec + params: + binary: bash + args: [mongo-cxx-driver/.evergreen/abi-stability-setup.sh] + include_expansions_in_env: [cxx_standard] + tasks: [".abi-stability"] + teardown_task_can_fail_task: true + teardown_task: + - command: subprocess.exec + params: + binary: bash + args: [-c, rm -rf *] + - name: test_atlas_task_group_search_indexes setup_group: - func: "setup" @@ -1023,6 +1157,29 @@ task_groups: ####################################### buildvariants: + ########################## + # ABI Stability Checks # + ########################## + - name: abi-stability-polyfill + display_name: "ABI Stability Checks (polyfill)" + expansions: + cxx_standard: 11 # Use a polyfill library. + tasks: + - "tg-abi-stability" + display_tasks: + - name: "ABI Stability Checks (polyfill)" + execution_tasks: [".abi-stability"] + + - name: abi-stability-stdlib + display_name: "ABI Stability Checks (stdlib)" + expansions: + cxx_standard: 17 # Use the stdlib. + tasks: + - "tg-abi-stability" + display_tasks: + - name: "ABI Stability Checks (stdlib)" + execution_tasks: [".abi-stability"] + ####################################### # Standard MongoDB Integration Tests # #######################################