diff --git a/.evergreen/config_generator/components/sbom.py b/.evergreen/config_generator/components/sbom.py new file mode 100644 index 0000000000..5922c108dc --- /dev/null +++ b/.evergreen/config_generator/components/sbom.py @@ -0,0 +1,159 @@ +from config_generator.components.funcs.setup import Setup + +from config_generator.etc.distros import find_small_distro +from config_generator.etc.function import Function, merge_defns +from config_generator.etc.utils import bash_exec + +from shrub.v3.evg_build_variant import BuildVariant +from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, expansions_update, s3_put +from shrub.v3.evg_task import EvgTask, EvgTaskRef + +from pydantic import ConfigDict +from typing import Optional + + +TAG = 'sbom' + + +class CustomCommand(BuiltInCommand): + command: str + model_config = ConfigDict(arbitrary_types_allowed=True) + + +def ec2_assume_role( + role_arn: Optional[str] = None, + policy: Optional[str] = None, + duration_seconds: Optional[int] = None, + command_type: Optional[EvgCommandType] = None, +) -> CustomCommand: + return CustomCommand( + command="ec2.assume_role", + params={ + "role_arn": role_arn, + "policy": policy, + "duration_seconds": duration_seconds, + }, + type=command_type, + ) + + +class CheckAugmentedSBOM(Function): + name = 'check augmented sbom' + commands = [ + ec2_assume_role( + command_type=EvgCommandType.SETUP, + role_arn='${KONDUKTO_ROLE_ARN}', + ), + bash_exec( + command_type=EvgCommandType.SETUP, + include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'], + script='''\ + set -o errexit + set -o pipefail + kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" + printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml + ''', + ), + expansions_update( + command_type=EvgCommandType.SETUP, + file='expansions.kondukto.yml', + ), + bash_exec( + command_type=EvgCommandType.TEST, + working_dir='mongo-cxx-driver', + include_expansions_in_env=[ + 'ARTIFACTORY_PASSWORD', + 'ARTIFACTORY_USER', + 'branch_name', + 'KONDUKTO_TOKEN', + ], + script='.evergreen/scripts/sbom.sh', + ), + ] + + +class UploadAugmentedSBOM(Function): + name = 'upload augmented sbom' + commands = [ + # The current Augmented SBOM, ignoring version and timestamp fields. + s3_put( + command_type=EvgCommandType.SYSTEM, + aws_key='${aws_key}', + aws_secret='${aws_secret}', + bucket='mciuploads', + content_type='application/json', + display_name='Augmented SBOM (Old)', + local_file='mongo-cxx-driver/old.json', + permissions='public-read', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/old.json', + ), + # The updated Augmented SBOM, ignoring version and timestamp fields. + s3_put( + command_type=EvgCommandType.SYSTEM, + aws_key='${aws_key}', + aws_secret='${aws_secret}', + bucket='mciuploads', + content_type='application/json', + display_name='Augmented SBOM (New)', + local_file='mongo-cxx-driver/new.json', + permissions='public-read', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/new.json', + ), + # The difference between the current and updated Augmented SBOM. + s3_put( + command_type=EvgCommandType.SYSTEM, + aws_key='${aws_key}', + aws_secret='${aws_secret}', + bucket='mciuploads', + content_type='application/json', + display_name='Augmented SBOM (Diff)', + local_file='mongo-cxx-driver/diff.txt', + permissions='public-read', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/diff.txt', + ), + # The updated Augmented SBOM without any filtering or modifications. + s3_put( + command_type=EvgCommandType.SYSTEM, + aws_key='${aws_key}', + aws_secret='${aws_secret}', + bucket='mciuploads', + content_type='application/json', + display_name='Augmented SBOM (Updated)', + local_file='mongo-cxx-driver/etc/augmented.sbom.json.new', + permissions='public-read', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented.sbom.json', + ), + ] + + +def functions(): + return merge_defns( + CheckAugmentedSBOM.defn(), + UploadAugmentedSBOM.defn(), + ) + + +def tasks(): + distro_name = 'rhel80' + distro = find_small_distro(distro_name) + + yield EvgTask( + name='sbom', + tags=[TAG, distro_name], + run_on=distro.name, + commands=[ + Setup.call(), + CheckAugmentedSBOM.call(), + UploadAugmentedSBOM.call(), + ], + ) + + +def variants(): + return [ + BuildVariant( + name=TAG, + display_name='SBOM', + tasks=[EvgTaskRef(name=f'.{TAG}')], + ), + ] diff --git a/.evergreen/config_generator/components/silk.py b/.evergreen/config_generator/components/silk.py deleted file mode 100644 index f133024eff..0000000000 --- a/.evergreen/config_generator/components/silk.py +++ /dev/null @@ -1,90 +0,0 @@ -from config_generator.components.funcs.setup import Setup - -from config_generator.etc.distros import find_small_distro -from config_generator.etc.function import Function, merge_defns -from config_generator.etc.utils import bash_exec - -from shrub.v3.evg_build_variant import BuildVariant -from shrub.v3.evg_command import EvgCommandType, s3_put -from shrub.v3.evg_task import EvgTask, EvgTaskRef - - -TAG = 'silk' - - -class CheckAugmentedSBOM(Function): - name = 'check augmented sbom' - commands = bash_exec( - command_type=EvgCommandType.TEST, - working_dir='mongo-cxx-driver', - include_expansions_in_env=[ - 'ARTIFACTORY_USER', - 'ARTIFACTORY_PASSWORD', - 'SILK_CLIENT_ID', - 'SILK_CLIENT_SECRET', - ], - script='.evergreen/scripts/check-augmented-sbom.sh', - ) - - -class UploadAugmentedSBOM(Function): - name = 'upload augmented sbom' - commands = [ - s3_put( - command_type=EvgCommandType.SYSTEM, - aws_key='${aws_key}', - aws_secret='${aws_secret}', - bucket='mciuploads', - content_type='application/json', - display_name='Augmented SBOM', - local_file='mongo-cxx-driver/etc/augmented.sbom.json.new', - permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json', - ), - s3_put( - command_type=EvgCommandType.SYSTEM, - aws_key='${aws_key}', - aws_secret='${aws_secret}', - bucket='mciuploads', - content_type='application/json', - display_name='Augmented SBOM (Diff)', - local_file='mongo-cxx-driver/diff.txt', - permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json.diff', - ), - ] - - -def functions(): - return merge_defns( - CheckAugmentedSBOM.defn(), - UploadAugmentedSBOM.defn(), - ) - - -def tasks(): - distro_name = 'rhel8-latest' - distro = find_small_distro(distro_name) - - return [ - EvgTask( - name='silk-check-augmented-sbom', - tags=[TAG, distro_name], - run_on=distro.name, - commands=[ - Setup.call(), - CheckAugmentedSBOM.call(), - UploadAugmentedSBOM.call(), - ], - ), - ] - - -def variants(): - return [ - BuildVariant( - name=TAG, - display_name='Silk', - tasks=[EvgTaskRef(name=f'.{TAG}')], - ), - ] diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index b98a419381..c3cc68d105 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -203,19 +203,42 @@ functions: .evergreen/atlas_data_lake/pull-mongohouse-image.sh check augmented sbom: - command: subprocess.exec - type: test - params: - binary: bash - working_dir: mongo-cxx-driver - include_expansions_in_env: - - ARTIFACTORY_USER - - ARTIFACTORY_PASSWORD - - SILK_CLIENT_ID - - SILK_CLIENT_SECRET - args: - - -c - - .evergreen/scripts/check-augmented-sbom.sh + - command: ec2.assume_role + type: setup + params: + role_arn: ${KONDUKTO_ROLE_ARN} + - command: subprocess.exec + type: setup + params: + binary: bash + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + args: + - -c + - | + set -o errexit + set -o pipefail + kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" + printf "KONDUKTO_TOKEN: %s\n" "$kondukto_token" >|expansions.kondukto.yml + - command: expansions.update + type: setup + params: + file: expansions.kondukto.yml + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: mongo-cxx-driver + include_expansions_in_env: + - ARTIFACTORY_PASSWORD + - ARTIFACTORY_USER + - branch_name + - KONDUKTO_TOKEN + args: + - -c + - .evergreen/scripts/sbom.sh clang-tidy: command: subprocess.exec type: test @@ -572,14 +595,25 @@ functions: - command: s3.put type: system params: - display_name: Augmented SBOM + display_name: Augmented SBOM (Old) aws_key: ${aws_key} aws_secret: ${aws_secret} bucket: mciuploads content_type: application/json - local_file: mongo-cxx-driver/etc/augmented.sbom.json.new + local_file: mongo-cxx-driver/old.json + permissions: public-read + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/old.json + - command: s3.put + type: system + params: + display_name: Augmented SBOM (New) + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: application/json + local_file: mongo-cxx-driver/new.json permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/new.json - command: s3.put type: system params: @@ -590,7 +624,18 @@ functions: content_type: application/json local_file: mongo-cxx-driver/diff.txt permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json.diff + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/diff.txt + - command: s3.put + type: system + params: + display_name: Augmented SBOM (Updated) + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: application/json + local_file: mongo-cxx-driver/etc/augmented.sbom.json.new + permissions: public-read + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented.sbom.json upload code coverage: command: subprocess.exec type: system diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 1398b75554..102ae42bfa 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -4146,6 +4146,13 @@ tasks: example_projects_cxx: clang++ example_projects_cxxflags: -D_GLIBCXX_USE_CXX11_ABI=0 -fsanitize=undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer example_projects_ldflags: -fsanitize=undefined -fno-sanitize-recover=undefined + - name: sbom + run_on: rhel80-small + tags: [sbom, rhel80] + commands: + - func: setup + - func: check augmented sbom + - func: upload augmented sbom - name: scan-build-ubuntu2204-std11-default run_on: ubuntu2204-large tags: [scan-build, ubuntu2204, std11] @@ -4209,13 +4216,6 @@ tasks: BSONCXX_POLYFILL: impls CXX_STANDARD: 17 - func: upload scan artifacts - - name: silk-check-augmented-sbom - run_on: rhel8-latest-small - tags: [silk, rhel8-latest] - commands: - - func: setup - - func: check augmented sbom - - func: upload augmented sbom - name: test_mongohouse run_on: ubuntu2204-large tags: [mongohouse, ubuntu2204] diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 9ed4d6756c..6ba8d2791f 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -88,6 +88,10 @@ buildvariants: - .sanitizers tasks: - name: .sanitizers + - name: sbom + display_name: SBOM + tasks: + - name: .sbom - name: scan-build-matrix display_name: scan-build-matrix display_tasks: @@ -96,10 +100,6 @@ buildvariants: - .scan-build tasks: - name: .scan-build - - name: silk - display_name: Silk - tasks: - - name: .silk - name: uninstall-check display_name: Uninstall Check display_tasks: diff --git a/.evergreen/scripts/check-augmented-sbom.sh b/.evergreen/scripts/check-augmented-sbom.sh deleted file mode 100755 index 19cd4cd5ee..0000000000 --- a/.evergreen/scripts/check-augmented-sbom.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail - -command -v podman >/dev/null || { - echo "missing required program podman" 1>&2 - exit 1 -} - -command -v jq >/dev/null || { - echo "missing required program jq" 1>&2 - exit 1 -} - -podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}" - -# Ensure latest version of SilkBomb is being used. -podman pull artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 - -silkbomb_download_flags=( - # Avoid bumping version or timestamp in diff. - --no-update-sbom-version - --no-update-timestamp - - --silk-asset-group mongo-cxx-driver-4.0 - -o /pwd/etc/augmented.sbom.json.new -) - -podman run \ - --env-file <( - echo "SILK_CLIENT_ID=${SILK_CLIENT_ID:?}" - echo "SILK_CLIENT_SECRET=${SILK_CLIENT_SECRET:?}" - ) \ - -it --rm -v "$(pwd):/pwd" \ - artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 \ - download "${silkbomb_download_flags[@]:?}" - -[[ -f ./etc/augmented.sbom.json.new ]] || { - echo "failed to download Augmented SBOM from Silk" 1>&2 - exit 1 -} - -echo "Comparing Augmented SBOM..." - -jq -S '.' ./etc/augmented.sbom.json >|old.json -jq -S '.' ./etc/augmented.sbom.json.new >|new.json - -# Allow task to upload the augmented SBOM despite failed diff. -if ! diff -sty --left-column -W 200 old.json new.json >|diff.txt; then - declare status - status='{"status":"failed", "type":"test", "should_continue":true, "desc":"detected significant changes in Augmented SBOM"}' - curl -sS -d "${status:?}" -H "Content-Type: application/json" -X POST localhost:2285/task_status || true -fi - -cat diff.txt - -echo "Comparing Augmented SBOM... done." diff --git a/.evergreen/scripts/sbom.sh b/.evergreen/scripts/sbom.sh new file mode 100755 index 0000000000..2791c5fcde --- /dev/null +++ b/.evergreen/scripts/sbom.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +: "${ARTIFACTORY_USER:?}" +: "${ARTIFACTORY_PASSWORD:?}" +: "${branch_name:?}" +: "${KONDUKTO_TOKEN:?}" + +command -v podman >/dev/null || { + echo "missing required program podman" 1>&2 + exit 1 +} + +command -v jq >/dev/null || { + echo "missing required program jq" 1>&2 + exit 1 +} + +podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}" + +silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0" + +# Ensure latest version of SilkBomb is being used. +podman pull "${silkbomb:?}" + +silkbomb_augment_flags=( + --repo mongodb/mongo-cxx-driver + --branch "${branch_name:?}" + --sbom-in /pwd/etc/cyclonedx.sbom.json + --sbom-out /pwd/etc/augmented.sbom.json.new + + # Any notable updates to the Augmented SBOM version should be done manually after careful inspection. + # Otherwise, it should be equal to the SBOM Lite version, which should normally be `1`. + --no-update-sbom-version +) + +# First validate the SBOM Lite. +podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \ + validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira + +# Allow the timestamp to be updated in the Augmented SBOM for update purposes. +podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" augment "${silkbomb_augment_flags[@]:?}" + +[[ -f ./etc/augmented.sbom.json.new ]] || { + echo "failed to download Augmented SBOM" 1>&2 + exit 1 +} + +echo "Comparing Augmented SBOM..." + +# Format for easier diff while ignoring the timestamp field. +jq -S 'del(.metadata.timestamp)' ./etc/augmented.sbom.json >|old.json +jq -S 'del(.metadata.timestamp)' ./etc/augmented.sbom.json.new >|new.json + +# Allow the task to upload the Augmented SBOM even if the diff failed. +if ! diff -sty --left-column -W 200 old.json new.json >|diff.txt; then + declare status + status='{"status":"failed", "type":"test", "should_continue":true, "desc":"detected significant changes in Augmented SBOM"}' + curl -sS -d "${status:?}" -H "Content-Type: application/json" -X POST localhost:2285/task_status || true +fi + +cat diff.txt + +echo "Comparing Augmented SBOM... done."