From 7d01a54ece275bd9e3c11e35f85facaad78c96af Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 20 Feb 2025 10:33:03 -0600 Subject: [PATCH 1/3] CXX-3228 update scripts for SilkBomb 2.0 --- .../config_generator/components/silk.py | 163 ++++++++++++++---- .evergreen/generated_configs/functions.yml | 116 +++++++++++-- .evergreen/generated_configs/tasks.yml | 11 +- .evergreen/scripts/check-augmented-sbom.sh | 58 ------- .../scripts/silk-check-augmented-sbom.sh | 62 +++++++ .evergreen/scripts/silk-upload-sbom-lite.sh | 29 ++++ 6 files changed, 332 insertions(+), 107 deletions(-) delete mode 100755 .evergreen/scripts/check-augmented-sbom.sh create mode 100755 .evergreen/scripts/silk-check-augmented-sbom.sh create mode 100755 .evergreen/scripts/silk-upload-sbom-lite.sh diff --git a/.evergreen/config_generator/components/silk.py b/.evergreen/config_generator/components/silk.py index f133024eff..139061fc2b 100644 --- a/.evergreen/config_generator/components/silk.py +++ b/.evergreen/config_generator/components/silk.py @@ -5,42 +5,122 @@ from config_generator.etc.utils import bash_exec from shrub.v3.evg_build_variant import BuildVariant -from shrub.v3.evg_command import EvgCommandType, s3_put -from shrub.v3.evg_task import EvgTask, EvgTaskRef +from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, expansions_update, s3_put +from shrub.v3.evg_task import EvgTask, EvgTaskRef, EvgTaskDependency + +from pydantic import ConfigDict +from typing import Optional TAG = 'silk' +class CustomCommand(BuiltInCommand): + command: str + model_config = ConfigDict(arbitrary_types_allowed=True) + + +def ec2_assume_role( + role_arn: Optional[str] = None, + policy: Optional[str] = None, + duration_seconds: Optional[int] = None, + command_type: Optional[EvgCommandType] = None, +) -> CustomCommand: + return CustomCommand( + command="ec2.assume_role", + params={ + "role_arn": role_arn, + "policy": policy, + "duration_seconds": duration_seconds, + }, + type=command_type, + ) + + +EC2_ASSUME_ROLE_COMMANDS = [ + ec2_assume_role( + command_type=EvgCommandType.SETUP, + role_arn='${KONDUKTO_ROLE_ARN}', + ), + bash_exec( + command_type=EvgCommandType.SETUP, + include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'], + script='''\ + set -o errexit + set -o pipefail + kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" + printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml + ''' + ), + expansions_update( + command_type=EvgCommandType.SETUP, + file='expansions.kondukto.yml' + ), +] + + +class UploadSBOMLite(Function): + name = 'upload sbom lite' + commands = EC2_ASSUME_ROLE_COMMANDS + [ + bash_exec( + command_type=EvgCommandType.TEST, + working_dir='mongo-cxx-driver', + include_expansions_in_env=[ + 'ARTIFACTORY_USER', + 'ARTIFACTORY_PASSWORD', + 'branch_name', + 'KONDUKTO_TOKEN', + ], + script='.evergreen/scripts/silk-upload-sbom-lite.sh', + ), + ] + + class CheckAugmentedSBOM(Function): name = 'check augmented sbom' - commands = bash_exec( - command_type=EvgCommandType.TEST, - working_dir='mongo-cxx-driver', - include_expansions_in_env=[ - 'ARTIFACTORY_USER', - 'ARTIFACTORY_PASSWORD', - 'SILK_CLIENT_ID', - 'SILK_CLIENT_SECRET', - ], - script='.evergreen/scripts/check-augmented-sbom.sh', - ) + commands = EC2_ASSUME_ROLE_COMMANDS + [ + bash_exec( + command_type=EvgCommandType.TEST, + working_dir='mongo-cxx-driver', + include_expansions_in_env=[ + 'ARTIFACTORY_PASSWORD', + 'ARTIFACTORY_USER', + 'branch_name', + 'KONDUKTO_TOKEN', + ], + script='.evergreen/scripts/silk-check-augmented-sbom.sh', + ), + ] class UploadAugmentedSBOM(Function): name = 'upload augmented sbom' commands = [ + # The current Augmented SBOM, ignoring version and timestamp fields. s3_put( command_type=EvgCommandType.SYSTEM, aws_key='${aws_key}', aws_secret='${aws_secret}', bucket='mciuploads', content_type='application/json', - display_name='Augmented SBOM', - local_file='mongo-cxx-driver/etc/augmented.sbom.json.new', + display_name='Augmented SBOM (Old)', + local_file='mongo-cxx-driver/old.json', permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/old.json', ), + # The updated Augmented SBOM, ignoring version and timestamp fields. + s3_put( + command_type=EvgCommandType.SYSTEM, + aws_key='${aws_key}', + aws_secret='${aws_secret}', + bucket='mciuploads', + content_type='application/json', + display_name='Augmented SBOM (New)', + local_file='mongo-cxx-driver/new.json', + permissions='public-read', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/new.json', + ), + # The difference between the current and updated Augmented SBOM. s3_put( command_type=EvgCommandType.SYSTEM, aws_key='${aws_key}', @@ -50,34 +130,57 @@ class UploadAugmentedSBOM(Function): display_name='Augmented SBOM (Diff)', local_file='mongo-cxx-driver/diff.txt', permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json.diff', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/diff.txt', + ), + # The updated Augmented SBOM without any filtering or modifications. + s3_put( + command_type=EvgCommandType.SYSTEM, + aws_key='${aws_key}', + aws_secret='${aws_secret}', + bucket='mciuploads', + content_type='application/json', + display_name='Augmented SBOM (Updated)', + local_file='mongo-cxx-driver/etc/augmented.sbom.json.new', + permissions='public-read', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json', ), ] def functions(): return merge_defns( + UploadSBOMLite.defn(), CheckAugmentedSBOM.defn(), UploadAugmentedSBOM.defn(), ) def tasks(): - distro_name = 'rhel8-latest' + distro_name = 'rhel80' distro = find_small_distro(distro_name) - return [ - EvgTask( - name='silk-check-augmented-sbom', - tags=[TAG, distro_name], - run_on=distro.name, - commands=[ - Setup.call(), - CheckAugmentedSBOM.call(), - UploadAugmentedSBOM.call(), - ], - ), - ] + upload_task = EvgTask( + name='silk-upload-sbom-lite', + tags=[TAG, distro_name], + run_on=distro.name, + commands=[ + Setup.call(), + UploadSBOMLite.call(), + ], + ) + + yield upload_task + yield EvgTask( + name='silk-check-augmented-sbom', + tags=[TAG, distro_name], + depends_on=[EvgTaskDependency(name=upload_task.name)], + run_on=distro.name, + commands=[ + Setup.call(), + CheckAugmentedSBOM.call(), + UploadAugmentedSBOM.call(), + ], + ) def variants(): diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index b98a419381..b4ed513e6e 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -203,19 +203,42 @@ functions: .evergreen/atlas_data_lake/pull-mongohouse-image.sh check augmented sbom: - command: subprocess.exec - type: test - params: - binary: bash - working_dir: mongo-cxx-driver - include_expansions_in_env: - - ARTIFACTORY_USER - - ARTIFACTORY_PASSWORD - - SILK_CLIENT_ID - - SILK_CLIENT_SECRET - args: - - -c - - .evergreen/scripts/check-augmented-sbom.sh + - command: ec2.assume_role + type: setup + params: + role_arn: ${KONDUKTO_ROLE_ARN} + - command: subprocess.exec + type: setup + params: + binary: bash + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + args: + - -c + - | + set -o errexit + set -o pipefail + kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" + printf "KONDUKTO_TOKEN: %s\n" "$kondukto_token" >|expansions.kondukto.yml + - command: expansions.update + type: setup + params: + file: expansions.kondukto.yml + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: mongo-cxx-driver + include_expansions_in_env: + - ARTIFACTORY_PASSWORD + - ARTIFACTORY_USER + - branch_name + - KONDUKTO_TOKEN + args: + - -c + - .evergreen/scripts/silk-check-augmented-sbom.sh clang-tidy: command: subprocess.exec type: test @@ -572,14 +595,25 @@ functions: - command: s3.put type: system params: - display_name: Augmented SBOM + display_name: Augmented SBOM (Old) aws_key: ${aws_key} aws_secret: ${aws_secret} bucket: mciuploads content_type: application/json - local_file: mongo-cxx-driver/etc/augmented.sbom.json.new + local_file: mongo-cxx-driver/old.json permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/old.json + - command: s3.put + type: system + params: + display_name: Augmented SBOM (New) + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: application/json + local_file: mongo-cxx-driver/new.json + permissions: public-read + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/new.json - command: s3.put type: system params: @@ -590,7 +624,18 @@ functions: content_type: application/json local_file: mongo-cxx-driver/diff.txt permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json.diff + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/diff.txt + - command: s3.put + type: system + params: + display_name: Augmented SBOM (Updated) + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + content_type: application/json + local_file: mongo-cxx-driver/etc/augmented.sbom.json.new + permissions: public-read + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json upload code coverage: command: subprocess.exec type: system @@ -629,6 +674,43 @@ functions: optional: true permissions: public-read remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz + upload sbom lite: + - command: ec2.assume_role + type: setup + params: + role_arn: ${KONDUKTO_ROLE_ARN} + - command: subprocess.exec + type: setup + params: + binary: bash + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + args: + - -c + - | + set -o errexit + set -o pipefail + kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" + printf "KONDUKTO_TOKEN: %s\n" "$kondukto_token" >|expansions.kondukto.yml + - command: expansions.update + type: setup + params: + file: expansions.kondukto.yml + - command: subprocess.exec + type: test + params: + binary: bash + working_dir: mongo-cxx-driver + include_expansions_in_env: + - ARTIFACTORY_USER + - ARTIFACTORY_PASSWORD + - branch_name + - KONDUKTO_TOKEN + args: + - -c + - .evergreen/scripts/silk-upload-sbom-lite.sh upload scan artifacts: - command: subprocess.exec type: test diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 1398b75554..2a91f3f5b4 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -4210,12 +4210,19 @@ tasks: CXX_STANDARD: 17 - func: upload scan artifacts - name: silk-check-augmented-sbom - run_on: rhel8-latest-small - tags: [silk, rhel8-latest] + run_on: rhel80-small + tags: [silk, rhel80] + depends_on: [{ name: silk-upload-sbom-lite }] commands: - func: setup - func: check augmented sbom - func: upload augmented sbom + - name: silk-upload-sbom-lite + run_on: rhel80-small + tags: [silk, rhel80] + commands: + - func: setup + - func: upload sbom lite - name: test_mongohouse run_on: ubuntu2204-large tags: [mongohouse, ubuntu2204] diff --git a/.evergreen/scripts/check-augmented-sbom.sh b/.evergreen/scripts/check-augmented-sbom.sh deleted file mode 100755 index 19cd4cd5ee..0000000000 --- a/.evergreen/scripts/check-augmented-sbom.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail - -command -v podman >/dev/null || { - echo "missing required program podman" 1>&2 - exit 1 -} - -command -v jq >/dev/null || { - echo "missing required program jq" 1>&2 - exit 1 -} - -podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}" - -# Ensure latest version of SilkBomb is being used. -podman pull artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 - -silkbomb_download_flags=( - # Avoid bumping version or timestamp in diff. - --no-update-sbom-version - --no-update-timestamp - - --silk-asset-group mongo-cxx-driver-4.0 - -o /pwd/etc/augmented.sbom.json.new -) - -podman run \ - --env-file <( - echo "SILK_CLIENT_ID=${SILK_CLIENT_ID:?}" - echo "SILK_CLIENT_SECRET=${SILK_CLIENT_SECRET:?}" - ) \ - -it --rm -v "$(pwd):/pwd" \ - artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 \ - download "${silkbomb_download_flags[@]:?}" - -[[ -f ./etc/augmented.sbom.json.new ]] || { - echo "failed to download Augmented SBOM from Silk" 1>&2 - exit 1 -} - -echo "Comparing Augmented SBOM..." - -jq -S '.' ./etc/augmented.sbom.json >|old.json -jq -S '.' ./etc/augmented.sbom.json.new >|new.json - -# Allow task to upload the augmented SBOM despite failed diff. -if ! diff -sty --left-column -W 200 old.json new.json >|diff.txt; then - declare status - status='{"status":"failed", "type":"test", "should_continue":true, "desc":"detected significant changes in Augmented SBOM"}' - curl -sS -d "${status:?}" -H "Content-Type: application/json" -X POST localhost:2285/task_status || true -fi - -cat diff.txt - -echo "Comparing Augmented SBOM... done." diff --git a/.evergreen/scripts/silk-check-augmented-sbom.sh b/.evergreen/scripts/silk-check-augmented-sbom.sh new file mode 100755 index 0000000000..e8d83b2107 --- /dev/null +++ b/.evergreen/scripts/silk-check-augmented-sbom.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +: "${ARTIFACTORY_USER:?}" +: "${ARTIFACTORY_PASSWORD:?}" +: "${branch_name:?}" +: "${KONDUKTO_TOKEN:?}" + +command -v podman >/dev/null || { + echo "missing required program podman" 1>&2 + exit 1 +} + +command -v jq >/dev/null || { + echo "missing required program jq" 1>&2 + exit 1 +} + +podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}" + +silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0" + +# Ensure latest version of SilkBomb is being used. +podman pull "${silkbomb:?}" + +silkbomb_augment_flags=( + --repo mongodb/mongo-cxx-driver + --branch "${branch_name:?}" + --sbom-in /pwd/etc/cyclonedx.sbom.json + --sbom-out /pwd/etc/augmented.sbom.json.new + + # Any notable updates to the Augmented SBOM version should be done manually after careful inspection. + # Otherwise, it should be equal to the SBOM Lite version, which should normally be `1`. + --no-update-sbom-version +) + +# Allow the timestamp to be updated in the Augmented SBOM for update purposes. +podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" augment "${silkbomb_augment_flags[@]:?}" + +[[ -f ./etc/augmented.sbom.json.new ]] || { + echo "failed to download Augmented SBOM" 1>&2 + exit 1 +} + +echo "Comparing Augmented SBOM..." + +# Format for easier diff while ignoring the timestamp field. +jq -S 'del(.metadata.timestamp)' ./etc/augmented.sbom.json >|old.json +jq -S 'del(.metadata.timestamp)' ./etc/augmented.sbom.json.new >|new.json + +# Allow the task to upload the Augmented SBOM even if the diff failed. +if ! diff -sty --left-column -W 200 old.json new.json >|diff.txt; then + declare status + status='{"status":"failed", "type":"test", "should_continue":true, "desc":"detected significant changes in Augmented SBOM"}' + curl -sS -d "${status:?}" -H "Content-Type: application/json" -X POST localhost:2285/task_status || true +fi + +cat diff.txt + +echo "Comparing Augmented SBOM... done." diff --git a/.evergreen/scripts/silk-upload-sbom-lite.sh b/.evergreen/scripts/silk-upload-sbom-lite.sh new file mode 100755 index 0000000000..705f4d9a59 --- /dev/null +++ b/.evergreen/scripts/silk-upload-sbom-lite.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +: "${ARTIFACTORY_USER:?}" +: "${ARTIFACTORY_PASSWORD:?}" +: "${branch_name:?}" +: "${KONDUKTO_TOKEN:?}" + +command -v podman >/dev/null || { + echo "missing required program podman" 1>&2 + exit 1 +} + +podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}" + +silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0" + +# Ensure latest version of SilkBomb is being used. +podman pull "${silkbomb:?}" + +# First validate the SBOM Lite. +podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \ + validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira + +# Then upload the SBOM Lite. +podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" \ + upload --repo mongodb/mongo-cxx-driver --branch "${branch_name:?}" --sbom-in /pwd/etc/cyclonedx.sbom.json From 236635245af3439dc9de18b12bfa9865297f27cb Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 20 Feb 2025 13:34:21 -0600 Subject: [PATCH 2/3] Upload SBOM Lite using the augment command --- .../config_generator/components/silk.py | 82 ++++++------------- .evergreen/generated_configs/functions.yml | 47 ++--------- .evergreen/generated_configs/tasks.yml | 21 ++--- .evergreen/generated_configs/variants.yml | 8 +- .../{silk-check-augmented-sbom.sh => sbom.sh} | 4 + .evergreen/scripts/silk-upload-sbom-lite.sh | 29 ------- 6 files changed, 44 insertions(+), 147 deletions(-) rename .evergreen/scripts/{silk-check-augmented-sbom.sh => sbom.sh} (91%) delete mode 100755 .evergreen/scripts/silk-upload-sbom-lite.sh diff --git a/.evergreen/config_generator/components/silk.py b/.evergreen/config_generator/components/silk.py index 139061fc2b..be51041b96 100644 --- a/.evergreen/config_generator/components/silk.py +++ b/.evergreen/config_generator/components/silk.py @@ -12,7 +12,7 @@ from typing import Optional -TAG = 'silk' +TAG = 'sbom' class CustomCommand(BuiltInCommand): @@ -37,48 +37,27 @@ def ec2_assume_role( ) -EC2_ASSUME_ROLE_COMMANDS = [ - ec2_assume_role( - command_type=EvgCommandType.SETUP, - role_arn='${KONDUKTO_ROLE_ARN}', - ), - bash_exec( - command_type=EvgCommandType.SETUP, - include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'], - script='''\ +class CheckAugmentedSBOM(Function): + name = 'check augmented sbom' + commands = [ + ec2_assume_role( + command_type=EvgCommandType.SETUP, + role_arn='${KONDUKTO_ROLE_ARN}', + ), + bash_exec( + command_type=EvgCommandType.SETUP, + include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'], + script='''\ set -o errexit set -o pipefail kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml - ''' - ), - expansions_update( - command_type=EvgCommandType.SETUP, - file='expansions.kondukto.yml' - ), -] - - -class UploadSBOMLite(Function): - name = 'upload sbom lite' - commands = EC2_ASSUME_ROLE_COMMANDS + [ - bash_exec( - command_type=EvgCommandType.TEST, - working_dir='mongo-cxx-driver', - include_expansions_in_env=[ - 'ARTIFACTORY_USER', - 'ARTIFACTORY_PASSWORD', - 'branch_name', - 'KONDUKTO_TOKEN', - ], - script='.evergreen/scripts/silk-upload-sbom-lite.sh', + ''', + ), + expansions_update( + command_type=EvgCommandType.SETUP, + file='expansions.kondukto.yml', ), - ] - - -class CheckAugmentedSBOM(Function): - name = 'check augmented sbom' - commands = EC2_ASSUME_ROLE_COMMANDS + [ bash_exec( command_type=EvgCommandType.TEST, working_dir='mongo-cxx-driver', @@ -88,7 +67,7 @@ class CheckAugmentedSBOM(Function): 'branch_name', 'KONDUKTO_TOKEN', ], - script='.evergreen/scripts/silk-check-augmented-sbom.sh', + script='.evergreen/scripts/sbom.sh', ), ] @@ -106,7 +85,7 @@ class UploadAugmentedSBOM(Function): display_name='Augmented SBOM (Old)', local_file='mongo-cxx-driver/old.json', permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/old.json', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/old.json', ), # The updated Augmented SBOM, ignoring version and timestamp fields. s3_put( @@ -118,7 +97,7 @@ class UploadAugmentedSBOM(Function): display_name='Augmented SBOM (New)', local_file='mongo-cxx-driver/new.json', permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/new.json', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/new.json', ), # The difference between the current and updated Augmented SBOM. s3_put( @@ -130,7 +109,7 @@ class UploadAugmentedSBOM(Function): display_name='Augmented SBOM (Diff)', local_file='mongo-cxx-driver/diff.txt', permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/diff.txt', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/diff.txt', ), # The updated Augmented SBOM without any filtering or modifications. s3_put( @@ -142,14 +121,13 @@ class UploadAugmentedSBOM(Function): display_name='Augmented SBOM (Updated)', local_file='mongo-cxx-driver/etc/augmented.sbom.json.new', permissions='public-read', - remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json', + remote_file='mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented.sbom.json', ), ] def functions(): return merge_defns( - UploadSBOMLite.defn(), CheckAugmentedSBOM.defn(), UploadAugmentedSBOM.defn(), ) @@ -159,21 +137,9 @@ def tasks(): distro_name = 'rhel80' distro = find_small_distro(distro_name) - upload_task = EvgTask( - name='silk-upload-sbom-lite', - tags=[TAG, distro_name], - run_on=distro.name, - commands=[ - Setup.call(), - UploadSBOMLite.call(), - ], - ) - - yield upload_task yield EvgTask( - name='silk-check-augmented-sbom', + name='sbom', tags=[TAG, distro_name], - depends_on=[EvgTaskDependency(name=upload_task.name)], run_on=distro.name, commands=[ Setup.call(), @@ -187,7 +153,7 @@ def variants(): return [ BuildVariant( name=TAG, - display_name='Silk', + display_name='SBOM', tasks=[EvgTaskRef(name=f'.{TAG}')], ), ] diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index b4ed513e6e..c3cc68d105 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -238,7 +238,7 @@ functions: - KONDUKTO_TOKEN args: - -c - - .evergreen/scripts/silk-check-augmented-sbom.sh + - .evergreen/scripts/sbom.sh clang-tidy: command: subprocess.exec type: test @@ -602,7 +602,7 @@ functions: content_type: application/json local_file: mongo-cxx-driver/old.json permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/old.json + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/old.json - command: s3.put type: system params: @@ -613,7 +613,7 @@ functions: content_type: application/json local_file: mongo-cxx-driver/new.json permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/new.json + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/new.json - command: s3.put type: system params: @@ -624,7 +624,7 @@ functions: content_type: application/json local_file: mongo-cxx-driver/diff.txt permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/diff.txt + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/diff.txt - command: s3.put type: system params: @@ -635,7 +635,7 @@ functions: content_type: application/json local_file: mongo-cxx-driver/etc/augmented.sbom.json.new permissions: public-read - remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/silk/augmented.sbom.json + remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented.sbom.json upload code coverage: command: subprocess.exec type: system @@ -674,43 +674,6 @@ functions: optional: true permissions: public-read remote_file: mongo-cxx-driver/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz - upload sbom lite: - - command: ec2.assume_role - type: setup - params: - role_arn: ${KONDUKTO_ROLE_ARN} - - command: subprocess.exec - type: setup - params: - binary: bash - include_expansions_in_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - args: - - -c - - | - set -o errexit - set -o pipefail - kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" - printf "KONDUKTO_TOKEN: %s\n" "$kondukto_token" >|expansions.kondukto.yml - - command: expansions.update - type: setup - params: - file: expansions.kondukto.yml - - command: subprocess.exec - type: test - params: - binary: bash - working_dir: mongo-cxx-driver - include_expansions_in_env: - - ARTIFACTORY_USER - - ARTIFACTORY_PASSWORD - - branch_name - - KONDUKTO_TOKEN - args: - - -c - - .evergreen/scripts/silk-upload-sbom-lite.sh upload scan artifacts: - command: subprocess.exec type: test diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 2a91f3f5b4..102ae42bfa 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -4146,6 +4146,13 @@ tasks: example_projects_cxx: clang++ example_projects_cxxflags: -D_GLIBCXX_USE_CXX11_ABI=0 -fsanitize=undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer example_projects_ldflags: -fsanitize=undefined -fno-sanitize-recover=undefined + - name: sbom + run_on: rhel80-small + tags: [sbom, rhel80] + commands: + - func: setup + - func: check augmented sbom + - func: upload augmented sbom - name: scan-build-ubuntu2204-std11-default run_on: ubuntu2204-large tags: [scan-build, ubuntu2204, std11] @@ -4209,20 +4216,6 @@ tasks: BSONCXX_POLYFILL: impls CXX_STANDARD: 17 - func: upload scan artifacts - - name: silk-check-augmented-sbom - run_on: rhel80-small - tags: [silk, rhel80] - depends_on: [{ name: silk-upload-sbom-lite }] - commands: - - func: setup - - func: check augmented sbom - - func: upload augmented sbom - - name: silk-upload-sbom-lite - run_on: rhel80-small - tags: [silk, rhel80] - commands: - - func: setup - - func: upload sbom lite - name: test_mongohouse run_on: ubuntu2204-large tags: [mongohouse, ubuntu2204] diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index 9ed4d6756c..6ba8d2791f 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -88,6 +88,10 @@ buildvariants: - .sanitizers tasks: - name: .sanitizers + - name: sbom + display_name: SBOM + tasks: + - name: .sbom - name: scan-build-matrix display_name: scan-build-matrix display_tasks: @@ -96,10 +100,6 @@ buildvariants: - .scan-build tasks: - name: .scan-build - - name: silk - display_name: Silk - tasks: - - name: .silk - name: uninstall-check display_name: Uninstall Check display_tasks: diff --git a/.evergreen/scripts/silk-check-augmented-sbom.sh b/.evergreen/scripts/sbom.sh similarity index 91% rename from .evergreen/scripts/silk-check-augmented-sbom.sh rename to .evergreen/scripts/sbom.sh index e8d83b2107..2791c5fcde 100755 --- a/.evergreen/scripts/silk-check-augmented-sbom.sh +++ b/.evergreen/scripts/sbom.sh @@ -36,6 +36,10 @@ silkbomb_augment_flags=( --no-update-sbom-version ) +# First validate the SBOM Lite. +podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \ + validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira + # Allow the timestamp to be updated in the Augmented SBOM for update purposes. podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" augment "${silkbomb_augment_flags[@]:?}" diff --git a/.evergreen/scripts/silk-upload-sbom-lite.sh b/.evergreen/scripts/silk-upload-sbom-lite.sh deleted file mode 100755 index 705f4d9a59..0000000000 --- a/.evergreen/scripts/silk-upload-sbom-lite.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail - -: "${ARTIFACTORY_USER:?}" -: "${ARTIFACTORY_PASSWORD:?}" -: "${branch_name:?}" -: "${KONDUKTO_TOKEN:?}" - -command -v podman >/dev/null || { - echo "missing required program podman" 1>&2 - exit 1 -} - -podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}" - -silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0" - -# Ensure latest version of SilkBomb is being used. -podman pull "${silkbomb:?}" - -# First validate the SBOM Lite. -podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \ - validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira - -# Then upload the SBOM Lite. -podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" \ - upload --repo mongodb/mongo-cxx-driver --branch "${branch_name:?}" --sbom-in /pwd/etc/cyclonedx.sbom.json From 0ad43616e80634e74d0f97f16aead6c304a2735a Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 20 Feb 2025 15:56:17 -0600 Subject: [PATCH 3/3] Sync changes following PR review --- .evergreen/config_generator/components/{silk.py => sbom.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .evergreen/config_generator/components/{silk.py => sbom.py} (98%) diff --git a/.evergreen/config_generator/components/silk.py b/.evergreen/config_generator/components/sbom.py similarity index 98% rename from .evergreen/config_generator/components/silk.py rename to .evergreen/config_generator/components/sbom.py index be51041b96..5922c108dc 100644 --- a/.evergreen/config_generator/components/silk.py +++ b/.evergreen/config_generator/components/sbom.py @@ -6,7 +6,7 @@ from shrub.v3.evg_build_variant import BuildVariant from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, expansions_update, s3_put -from shrub.v3.evg_task import EvgTask, EvgTaskRef, EvgTaskDependency +from shrub.v3.evg_task import EvgTask, EvgTaskRef from pydantic import ConfigDict from typing import Optional