From f0a3b52a0561d8aca70db23f114530678c24221b Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Fri, 11 Jul 2025 10:44:11 -0500 Subject: [PATCH 01/14] Address Earthly 0.8 compatibility warnings --- Earthfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Earthfile b/Earthfile index 6d668088f7..89cb4e1365 100644 --- a/Earthfile +++ b/Earthfile @@ -1,4 +1,4 @@ -VERSION --arg-scope-and-set --pass-args 0.7 +VERSION --arg-scope-and-set --pass-args --use-function-keyword 0.7 LOCALLY IMPORT ./tools/ AS tools @@ -114,7 +114,7 @@ test-cxx-driver: # PREP_CMAKE "warms up" the CMake installation cache for the current environment PREP_CMAKE: - COMMAND + FUNCTION LET scratch=/opt/mongoc-cmake # Copy the minimal amount that we need, as to avoid cache invalidation COPY tools/use.sh tools/platform.sh tools/paths.sh tools/base.sh tools/download.sh \ @@ -462,7 +462,7 @@ env.centos7: DO --pass-args +CENTOS_ENV --version=7 ALPINE_ENV: - COMMAND + FUNCTION ARG --required version FROM --pass-args tools+init-env --from alpine:$version # XXX: On Alpine, we just use the system's CMake. At time of writing, it is @@ -484,7 +484,7 @@ ALPINE_ENV: DO --pass-args tools+ADD_C_COMPILER --clang_pkg="gcc clang compiler-rt" UBUNTU_ENV: - COMMAND + FUNCTION ARG --required version FROM --pass-args tools+init-env --from ubuntu:$version RUN __install curl build-essential @@ -502,7 +502,7 @@ UBUNTU_ENV: DO +PREP_CMAKE CENTOS_ENV: - COMMAND + FUNCTION ARG --required version FROM --pass-args tools+init-env --from centos:$version # Update repositories to use vault.centos.org From 49e0004f262339fc6aecafcd12c4da648b3cb3d8 Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Fri, 11 Jul 2025 10:44:11 -0500 Subject: [PATCH 02/14] Workaround repository metalink retrieval errors on CentOS --- Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Earthfile b/Earthfile index 89cb4e1365..fb66a43ed9 100644 --- a/Earthfile +++ b/Earthfile @@ -508,7 +508,7 @@ CENTOS_ENV: # Update repositories to use vault.centos.org RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - RUN yum -y install epel-release && yum -y update + RUN yum -y --enablerepo=extras install epel-release && yum -y update RUN yum -y install curl gcc gcc-c++ make ARG --required purpose From 6bc0ae805753083817beae1fbdb2346cab26a232 Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Fri, 11 Jul 2025 10:44:11 -0500 Subject: [PATCH 03/14] CDRIVER-5971 Use Amazon ECR to obtain all OCI images --- .../config_generator/components/earthly.py | 48 ++++++++++++++++++- .evergreen/generated_configs/functions.yml | 22 ++++++++- .evergreen/generated_configs/tasks.yml | 28 +++++++++++ Earthfile | 20 ++++---- tools/earthly.sh | 2 +- 5 files changed, 106 insertions(+), 14 deletions(-) diff --git a/.evergreen/config_generator/components/earthly.py b/.evergreen/config_generator/components/earthly.py index 4dac11f31d..ea1baf1bf7 100644 --- a/.evergreen/config_generator/components/earthly.py +++ b/.evergreen/config_generator/components/earthly.py @@ -5,9 +5,18 @@ from typing import Iterable, Literal, Mapping, NamedTuple, TypeVar from shrub.v3.evg_build_variant import BuildVariant -from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, subprocess_exec +from shrub.v3.evg_command import ( + BuiltInCommand, + EvgCommandType, + KeyValueParam, + ec2_assume_role, + expansions_update, + subprocess_exec, +) from shrub.v3.evg_task import EvgTask, EvgTaskRef +from config_generator.etc.function import Function + from ..etc.utils import all_possible T = TypeVar("T") @@ -38,7 +47,7 @@ "Valid options for the SASL configuration parameter" TLSOption = Literal["OpenSSL", "off"] "Options for the TLS backend configuration parameter (AKA 'ENABLE_SSL')" -CxxVersion = Literal["none"] # TODO: Once CXX-3103 is released, add latest C++ release tag. +CxxVersion = Literal["none"] # TODO: Once CXX-3103 is released, add latest C++ release tag. "C++ driver refs that are under CI test" # A separator character, since we cannot use whitespace @@ -136,6 +145,34 @@ def suffix(self) -> str: return _SEPARATOR.join(f"{k}={v}" for k, v in self._asdict().items()) +# Use DevProd-provided Amazon ECR instance to obtain earthly-buildkitd in advance. +class DockerLoginAmazonECR(Function): + name = 'docker-login-amazon-ecr' + commands = [ + # Avoid inadvertently using a pre-existing and potentially conflicting Docker config. + expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]), + ec2_assume_role(role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"), + subprocess_exec( + binary="bash", + command_type=EvgCommandType.SETUP, + include_expansions_in_env=[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "DOCKER_CONFIG", + ], + args=[ + "-c", + 'aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com', + ], + ), + ] + + @classmethod + def call(cls, **kwargs): + return cls.default_call(**kwargs) + + def task_filter(env: EarthlyVariant, conf: Configuration) -> bool: """ Control which tasks are actually defined by matching on the platform and @@ -175,6 +212,7 @@ def earthly_exec( *(f"--{arg}={val}" for arg, val in (args or {}).items()), ], command_type=EvgCommandType(kind), + include_expansions_in_env=["DOCKER_CONFIG"], env=env if env else None, working_dir="mongoc", ) @@ -209,10 +247,12 @@ def earthly_task( return EvgTask( name=name, commands=[ + DockerLoginAmazonECR.call(), # Ensure subsequent Docker commands are authenticated. subprocess_exec( binary="bash", command_type=EvgCommandType.SETUP, + include_expansions_in_env=["DOCKER_CONFIG"], args=[ "-c", r'docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"', @@ -249,6 +289,10 @@ def earthly_task( ] +def functions(): + return DockerLoginAmazonECR.defn() + + def tasks() -> Iterable[EvgTask]: for conf in all_possible(Configuration): # test-example is a target in all configurations diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 88e91847e8..fb33ae2889 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -123,7 +123,7 @@ functions: DRYRUN: "1" args: - -c - - uv run --frozen --only-group format tools/format.py --mode=check + - uv run --frozen --only-group=format tools/format.py --mode=check cse-sasl-cyrus-darwinssl-compile: - command: expansions.update params: @@ -175,6 +175,26 @@ functions: args: - -c - EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON ${EXTRA_CONFIGURE_FLAGS}" .evergreen/scripts/compile.sh + docker-login-amazon-ecr: + - command: expansions.update + params: + updates: + - { key: DOCKER_CONFIG, value: "${workdir}/.docker" } + - command: ec2.assume_role + params: + role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro + - command: subprocess.exec + type: setup + params: + binary: bash + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + - DOCKER_CONFIG + args: + - -c + - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com fetch-build: - command: subprocess.exec type: setup diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index b5abdbc7d7..a3b0e035e8 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1133,10 +1133,13 @@ tasks: - ubuntu2404-large tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: + - func: docker-login-amazon-ecr - command: subprocess.exec type: setup params: binary: bash + include_expansions_in_env: + - DOCKER_CONFIG args: - -c - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" @@ -1145,6 +1148,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +env-warmup - --sasl=Cyrus @@ -1157,6 +1162,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +run - --targets=test-example @@ -1174,10 +1181,13 @@ tasks: - ubuntu2404-large tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: + - func: docker-login-amazon-ecr - command: subprocess.exec type: setup params: binary: bash + include_expansions_in_env: + - DOCKER_CONFIG args: - -c - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" @@ -1186,6 +1196,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +env-warmup - --sasl=Cyrus @@ -1198,6 +1210,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +run - --targets=test-example @@ -1215,10 +1229,13 @@ tasks: - ubuntu2404-large tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: + - func: docker-login-amazon-ecr - command: subprocess.exec type: setup params: binary: bash + include_expansions_in_env: + - DOCKER_CONFIG args: - -c - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" @@ -1227,6 +1244,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +env-warmup - --sasl=off @@ -1239,6 +1258,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +run - --targets=test-example @@ -1256,10 +1277,13 @@ tasks: - ubuntu2404-large tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: + - func: docker-login-amazon-ecr - command: subprocess.exec type: setup params: binary: bash + include_expansions_in_env: + - DOCKER_CONFIG args: - -c - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" @@ -1268,6 +1292,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +env-warmup - --sasl=off @@ -1280,6 +1306,8 @@ tasks: params: binary: ./tools/earthly.sh working_dir: mongoc + include_expansions_in_env: + - DOCKER_CONFIG args: - +run - --targets=test-example diff --git a/Earthfile b/Earthfile index fb66a43ed9..f8adbcecc5 100644 --- a/Earthfile +++ b/Earthfile @@ -148,7 +148,7 @@ multibuild: # release-archive : # Create a release archive of the source tree. (Refer to dev docs) release-archive: - FROM alpine:3.20 + FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20 RUN apk add git bash ARG --required prefix ARG --required ref @@ -193,7 +193,7 @@ release-archive: # Obtain the signing public key. Exported as an artifact /c-driver.pub signing-pubkey: - FROM alpine:3.20 + FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20 RUN apk add curl RUN curl --location --silent --fail "https://pgp.mongodb.com/c-driver.pub" -o /c-driver.pub SAVE ARTIFACT /c-driver.pub @@ -203,7 +203,7 @@ signing-pubkey: # to be used to access them. (Refer to dev docs) sign-file: # Pull from Garasign: - FROM artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-gpg + FROM artifactory.corp.mongodb.com/release-infrastructure/garasign-gpg # Copy the file to be signed ARG --required file COPY $file /s/file @@ -223,7 +223,7 @@ sign-file: # Generate a signed release artifact. Refer to the "Earthly" page of our dev docs for more information. # (Refer to dev docs) signed-release: - FROM alpine:3.20 + FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20 RUN apk add git # The version of the release. This affects the filepaths of the output and is the default for --ref ARG --required version @@ -312,7 +312,7 @@ sbom-validate: --exclude jira snyk: - FROM --platform=linux/amd64 ubuntu:24.04 + FROM --platform=linux/amd64 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/ubuntu:24.04 RUN apt-get update && apt-get -y install curl RUN curl --location https://github.com/snyk/cli/releases/download/v1.1291.1/snyk-linux -o /usr/local/bin/snyk RUN chmod a+x /usr/local/bin/snyk @@ -384,7 +384,7 @@ test-vcpkg-manifest-mode: make test-manifest-mode vcpkg-base: - FROM alpine:3.18 + FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.18 RUN apk add cmake curl gcc g++ musl-dev ninja-is-really-ninja zip unzip tar \ build-base git pkgconf perl bash linux-headers ENV VCPKG_ROOT=/opt/vcpkg-git @@ -443,7 +443,7 @@ env.alpine3.19: DO --pass-args +ALPINE_ENV --version=3.19 env.archlinux: - FROM --pass-args tools+init-env --from archlinux + FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/archlinux RUN pacman-key --init ARG --required purpose @@ -464,7 +464,7 @@ env.centos7: ALPINE_ENV: FUNCTION ARG --required version - FROM --pass-args tools+init-env --from alpine:$version + FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:$version # XXX: On Alpine, we just use the system's CMake. At time of writing, it is # very up-to-date and much faster than building our own from source (since # Kitware does not (yet) provide libmuslc builds of CMake) @@ -486,7 +486,7 @@ ALPINE_ENV: UBUNTU_ENV: FUNCTION ARG --required version - FROM --pass-args tools+init-env --from ubuntu:$version + FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/ubuntu:$version RUN __install curl build-essential ARG --required purpose @@ -504,7 +504,7 @@ UBUNTU_ENV: CENTOS_ENV: FUNCTION ARG --required version - FROM --pass-args tools+init-env --from centos:$version + FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/centos:$version # Update repositories to use vault.centos.org RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* diff --git a/tools/earthly.sh b/tools/earthly.sh index 93eda0bc5b..efcff89888 100755 --- a/tools/earthly.sh +++ b/tools/earthly.sh @@ -36,7 +36,7 @@ if ! is-file "$EARTHLY_EXE"; then fi run-earthly() { - "$EARTHLY_EXE" "$@" + "$EARTHLY_EXE" --buildkit-image 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 "$@" } if is-main; then From 4e8a720710d1952ef553ce6abda5b9e7430e14eb Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Fri, 11 Jul 2025 10:44:12 -0500 Subject: [PATCH 04/14] Update release instructions to use AWS ECR instead of Artifactory --- docs/dev/earthly.rst | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/docs/dev/earthly.rst b/docs/dev/earthly.rst index d216fc81c4..c985cde187 100644 --- a/docs/dev/earthly.rst +++ b/docs/dev/earthly.rst @@ -149,8 +149,7 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository .. earthly-target:: +sign-file Signs a file using Garasign. Use of this target requires authenticating - against the MongoDB Artifactory installation! (Refer to: - `earthly.artifactory-auth`) + against MongoDB's AWS ECR instance! (Refer to: `earthly.aws-ecr`) .. earthly-artifact:: +sign-file/signature.asc @@ -175,23 +174,34 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository .. seealso:: `earthly.secrets` - .. _earthly.artifactory-auth: + .. _earthly.aws-ecr: - Authenticating with Artifactory - =============================== + Authenticating with AWS ECR + =========================== In order to run `+sign-file` or any target that depends upon it, the container engine client\ [#oci]_ will need to be authenticated with the - MongoDB Artifactory instance. + MongoDB's AWS ECR pull-through cache using AWS CLI v2:: - Authenticating can be done using the container engine's command-line - interface. For example, with Podman:: + # Forward the short-term AWS credentials to the container engine client. + $ aws ecr get-login-password --profile | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com - $ podman login "artifactory.corp.mongodb.com" + Configure the AWS profile using ``aws configure sso`` or modifying the + ``$HOME/.aws/config`` file such that: - Which will prompt you for a username and password if you are not already - authenticated with the host.\ [#creds]_ If you are already authenticated, this - command will have no effect. + - The SSO start URL is ``https://d-9067613a84.awsapps.com/start#/``. + - The SSO and client region are ``us-east-1``. + - The SSO registration scope is ``sso:account:access`` (default). + - The SSO account ID is ``901841024863`` (aka ``devprod-platforms-ecr``). + - The SSO role name is ``ECRScopedAccess`` (default). + + To refresh short-term credentials when they have expired, run + ``aws sso login --profile `` followed by the same + ``aws ecr get-login-password ... | podman login ...`` command described + above. + + .. seealso:: `"DevProd Platforms Container Registry" + `_ and `"Configuring IAM Identity Center authentication with the AWS CLI" _`. .. earthly-target:: +sbom-generate From 4facc36c606b589f3918576459ea998541834932 Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Tue, 15 Jul 2025 10:28:41 -0500 Subject: [PATCH 05/14] Replace more instances of Artifactory with Amazon ECR --- Earthfile | 4 ++-- docs/dev/earthly.rst | 11 ++++++----- docs/dev/releasing.rst | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Earthfile b/Earthfile index f8adbcecc5..aa70d7c749 100644 --- a/Earthfile +++ b/Earthfile @@ -203,7 +203,7 @@ signing-pubkey: # to be used to access them. (Refer to dev docs) sign-file: # Pull from Garasign: - FROM artifactory.corp.mongodb.com/release-infrastructure/garasign-gpg + FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-gpg # Copy the file to be signed ARG --required file COPY $file /s/file @@ -253,7 +253,7 @@ signed-release: # This target is simply an environment in which the SilkBomb executable is available. silkbomb: - FROM artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0 + FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0 # Alias the silkbomb executable to a simpler name: RUN ln -s /python/src/sbom/silkbomb/bin /usr/local/bin/silkbomb diff --git a/docs/dev/earthly.rst b/docs/dev/earthly.rst index c985cde187..d062c10ed3 100644 --- a/docs/dev/earthly.rst +++ b/docs/dev/earthly.rst @@ -149,7 +149,8 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository .. earthly-target:: +sign-file Signs a file using Garasign. Use of this target requires authenticating - against MongoDB's AWS ECR instance! (Refer to: `earthly.aws-ecr`) + against the DevProd-provided Amazon ECR instance! (Refer to: + `earthly.amazon-ecr`) .. earthly-artifact:: +sign-file/signature.asc @@ -174,14 +175,14 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository .. seealso:: `earthly.secrets` - .. _earthly.aws-ecr: + .. _earthly.amazon-ecr: - Authenticating with AWS ECR - =========================== + Authenticating with Amazon ECR + ============================== In order to run `+sign-file` or any target that depends upon it, the container engine client\ [#oci]_ will need to be authenticated with the - MongoDB's AWS ECR pull-through cache using AWS CLI v2:: + DevProd-provided Amazon ECR pull-through cache using AWS CLI v2:: # Forward the short-term AWS credentials to the container engine client. $ aws ecr get-login-password --profile | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com diff --git a/docs/dev/releasing.rst b/docs/dev/releasing.rst index be177eef93..d41460c9b4 100644 --- a/docs/dev/releasing.rst +++ b/docs/dev/releasing.rst @@ -396,7 +396,7 @@ Specifically, it is generated using the :any:`+signed-release` target. Before running :any:`+signed-release`, one will need to set up some environment that is required for it to succeed: -1. :ref:`Authenticate with Artifactory ` +1. :ref:`Authenticate with the DevProd-provided Amazon ECR instance ` 2. Set the Earthly secrets required for the :any:`+sign-file` target. 3. Download an augmented SBOM from a recent execution of the ``sbom`` task in an Evergreen patch or commit build and save it to ``etc/augmented-sbom.json``. From de0240093b2f57c25ee35d9eef36119f9225acac Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Tue, 15 Jul 2025 10:30:29 -0500 Subject: [PATCH 06/14] Wording consistency tweak --- docs/dev/earthly.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/earthly.rst b/docs/dev/earthly.rst index d062c10ed3..f5e8866ceb 100644 --- a/docs/dev/earthly.rst +++ b/docs/dev/earthly.rst @@ -182,7 +182,7 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository In order to run `+sign-file` or any target that depends upon it, the container engine client\ [#oci]_ will need to be authenticated with the - DevProd-provided Amazon ECR pull-through cache using AWS CLI v2:: + DevProd-provided Amazon ECR instance using AWS CLI v2:: # Forward the short-term AWS credentials to the container engine client. $ aws ecr get-login-password --profile | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com From e82983cbaa95dca394b665dc5dd8cfe317e28b6d Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Tue, 15 Jul 2025 12:01:43 -0500 Subject: [PATCH 07/14] Artifactory authentication is no longer necessary --- .../config_generator/components/earthly.py | 10 ------ .evergreen/generated_configs/tasks.yml | 36 ------------------- 2 files changed, 46 deletions(-) diff --git a/.evergreen/config_generator/components/earthly.py b/.evergreen/config_generator/components/earthly.py index ea1baf1bf7..84b2f6de96 100644 --- a/.evergreen/config_generator/components/earthly.py +++ b/.evergreen/config_generator/components/earthly.py @@ -248,16 +248,6 @@ def earthly_task( name=name, commands=[ DockerLoginAmazonECR.call(), - # Ensure subsequent Docker commands are authenticated. - subprocess_exec( - binary="bash", - command_type=EvgCommandType.SETUP, - include_expansions_in_env=["DOCKER_CONFIG"], - args=[ - "-c", - r'docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"', - ], - ), # First, just build the "env-warmup" which will prepare the build environment. # This won't generate any output, but allows EVG to track it as a separate build step # for timing and logging purposes. The subequent build step will cache-hit the diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index a3b0e035e8..1a230db3c3 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1134,15 +1134,6 @@ tasks: tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: - func: docker-login-amazon-ecr - - command: subprocess.exec - type: setup - params: - binary: bash - include_expansions_in_env: - - DOCKER_CONFIG - args: - - -c - - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" - command: subprocess.exec type: setup params: @@ -1182,15 +1173,6 @@ tasks: tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: - func: docker-login-amazon-ecr - - command: subprocess.exec - type: setup - params: - binary: bash - include_expansions_in_env: - - DOCKER_CONFIG - args: - - -c - - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" - command: subprocess.exec type: setup params: @@ -1230,15 +1212,6 @@ tasks: tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: - func: docker-login-amazon-ecr - - command: subprocess.exec - type: setup - params: - binary: bash - include_expansions_in_env: - - DOCKER_CONFIG - args: - - -c - - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" - command: subprocess.exec type: setup params: @@ -1278,15 +1251,6 @@ tasks: tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc] commands: - func: docker-login-amazon-ecr - - command: subprocess.exec - type: setup - params: - binary: bash - include_expansions_in_env: - - DOCKER_CONFIG - args: - - -c - - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}" - command: subprocess.exec type: setup params: From ccd8a013388440eda03d567f5a22eb7687b966ac Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Tue, 15 Jul 2025 11:23:38 -0500 Subject: [PATCH 08/14] Use `$default_search_registry` to avoid forcing use of Amazon ECR --- .../config_generator/components/earthly.py | 4 ++++ .evergreen/generated_configs/tasks.yml | 24 +++++++++++++++++++ Earthfile | 21 +++++++++------- tools/earthly.sh | 2 +- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.evergreen/config_generator/components/earthly.py b/.evergreen/config_generator/components/earthly.py index 84b2f6de96..849497fb91 100644 --- a/.evergreen/config_generator/components/earthly.py +++ b/.evergreen/config_generator/components/earthly.py @@ -207,8 +207,12 @@ def earthly_exec( return subprocess_exec( "./tools/earthly.sh", args=[ + "--buildkit-image", + "901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3", *(f"--secret={k}" for k in (secrets or ())), f"+{target}", + # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits. + "--default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub", *(f"--{arg}={val}" for arg, val in (args or {}).items()), ], command_type=EvgCommandType(kind), diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 1a230db3c3..48c7a64745 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1142,7 +1142,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=Cyrus - --tls=OpenSSL - --test_mongocxx_ref=none @@ -1156,7 +1159,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example - --sasl=Cyrus - --tls=OpenSSL @@ -1181,7 +1187,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=Cyrus - --tls=off - --test_mongocxx_ref=none @@ -1195,7 +1204,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example - --sasl=Cyrus - --tls=off @@ -1220,7 +1232,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=off - --tls=OpenSSL - --test_mongocxx_ref=none @@ -1234,7 +1249,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example - --sasl=off - --tls=OpenSSL @@ -1259,7 +1277,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=off - --tls=off - --test_mongocxx_ref=none @@ -1273,7 +1294,10 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: + - --buildkit-image + - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run + - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example - --sasl=off - --tls=off diff --git a/Earthfile b/Earthfile index aa70d7c749..032b07942f 100644 --- a/Earthfile +++ b/Earthfile @@ -1,6 +1,9 @@ VERSION --arg-scope-and-set --pass-args --use-function-keyword 0.7 LOCALLY +# Allow setting the "default" container image registry to use for image short names (e.g. to Amazon ECR). +ARG --global default_search_registry=docker.io + IMPORT ./tools/ AS tools # For target names, descriptions, and build parameters, run the "doc" Earthly subcommand. @@ -148,7 +151,7 @@ multibuild: # release-archive : # Create a release archive of the source tree. (Refer to dev docs) release-archive: - FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20 + FROM $default_search_registry/library/alpine:3.20 RUN apk add git bash ARG --required prefix ARG --required ref @@ -193,7 +196,7 @@ release-archive: # Obtain the signing public key. Exported as an artifact /c-driver.pub signing-pubkey: - FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20 + FROM $default_search_registry/library/alpine:3.20 RUN apk add curl RUN curl --location --silent --fail "https://pgp.mongodb.com/c-driver.pub" -o /c-driver.pub SAVE ARTIFACT /c-driver.pub @@ -223,7 +226,7 @@ sign-file: # Generate a signed release artifact. Refer to the "Earthly" page of our dev docs for more information. # (Refer to dev docs) signed-release: - FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20 + FROM $default_search_registry/library/alpine:3.20 RUN apk add git # The version of the release. This affects the filepaths of the output and is the default for --ref ARG --required version @@ -312,7 +315,7 @@ sbom-validate: --exclude jira snyk: - FROM --platform=linux/amd64 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/ubuntu:24.04 + FROM --platform=linux/amd64 $default_search_registry/library/ubuntu:24.04 RUN apt-get update && apt-get -y install curl RUN curl --location https://github.com/snyk/cli/releases/download/v1.1291.1/snyk-linux -o /usr/local/bin/snyk RUN chmod a+x /usr/local/bin/snyk @@ -384,7 +387,7 @@ test-vcpkg-manifest-mode: make test-manifest-mode vcpkg-base: - FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.18 + FROM $default_search_registry/library/alpine:3.18 RUN apk add cmake curl gcc g++ musl-dev ninja-is-really-ninja zip unzip tar \ build-base git pkgconf perl bash linux-headers ENV VCPKG_ROOT=/opt/vcpkg-git @@ -443,7 +446,7 @@ env.alpine3.19: DO --pass-args +ALPINE_ENV --version=3.19 env.archlinux: - FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/archlinux + FROM --pass-args tools+init-env --from $default_search_registry/library/archlinux RUN pacman-key --init ARG --required purpose @@ -464,7 +467,7 @@ env.centos7: ALPINE_ENV: FUNCTION ARG --required version - FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:$version + FROM --pass-args tools+init-env --from $default_search_registry/library/alpine:$version # XXX: On Alpine, we just use the system's CMake. At time of writing, it is # very up-to-date and much faster than building our own from source (since # Kitware does not (yet) provide libmuslc builds of CMake) @@ -486,7 +489,7 @@ ALPINE_ENV: UBUNTU_ENV: FUNCTION ARG --required version - FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/ubuntu:$version + FROM --pass-args tools+init-env --from $default_search_registry/library/ubuntu:$version RUN __install curl build-essential ARG --required purpose @@ -504,7 +507,7 @@ UBUNTU_ENV: CENTOS_ENV: FUNCTION ARG --required version - FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/centos:$version + FROM --pass-args tools+init-env --from $default_search_registry/library/centos:$version # Update repositories to use vault.centos.org RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \ sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* diff --git a/tools/earthly.sh b/tools/earthly.sh index efcff89888..93eda0bc5b 100755 --- a/tools/earthly.sh +++ b/tools/earthly.sh @@ -36,7 +36,7 @@ if ! is-file "$EARTHLY_EXE"; then fi run-earthly() { - "$EARTHLY_EXE" --buildkit-image 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 "$@" + "$EARTHLY_EXE" "$@" } if is-main; then From aafc97f63862c07469ad9a1c8969a3b39d6597ba Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Tue, 15 Jul 2025 12:29:52 -0500 Subject: [PATCH 09/14] Use `--buildkit-image=...` syntax and add doc comment --- .../config_generator/components/earthly.py | 4 ++-- .evergreen/generated_configs/tasks.yml | 24 +++++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.evergreen/config_generator/components/earthly.py b/.evergreen/config_generator/components/earthly.py index 849497fb91..12dd3b262c 100644 --- a/.evergreen/config_generator/components/earthly.py +++ b/.evergreen/config_generator/components/earthly.py @@ -207,8 +207,8 @@ def earthly_exec( return subprocess_exec( "./tools/earthly.sh", args=[ - "--buildkit-image", - "901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3", + # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits. + "--buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3", *(f"--secret={k}" for k in (secrets or ())), f"+{target}", # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits. diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 48c7a64745..8188744a0b 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -1142,8 +1142,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=Cyrus @@ -1159,8 +1158,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example @@ -1187,8 +1185,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=Cyrus @@ -1204,8 +1201,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example @@ -1232,8 +1228,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=off @@ -1249,8 +1244,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example @@ -1277,8 +1271,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +env-warmup - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --sasl=off @@ -1294,8 +1287,7 @@ tasks: include_expansions_in_env: - DOCKER_CONFIG args: - - --buildkit-image - - 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 + - --buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3 - +run - --default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub - --targets=test-example From 9af26b421aa29b171c9c5422ee18411d09f46c8f Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Wed, 16 Jul 2025 09:34:08 -0500 Subject: [PATCH 10/14] Fix comment documenting DockerLoginAmazonECR --- .evergreen/config_generator/components/earthly.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config_generator/components/earthly.py b/.evergreen/config_generator/components/earthly.py index 12dd3b262c..debf6dd39a 100644 --- a/.evergreen/config_generator/components/earthly.py +++ b/.evergreen/config_generator/components/earthly.py @@ -145,7 +145,7 @@ def suffix(self) -> str: return _SEPARATOR.join(f"{k}={v}" for k, v in self._asdict().items()) -# Use DevProd-provided Amazon ECR instance to obtain earthly-buildkitd in advance. +# Authenticate with DevProd-provided Amazon ECR instance to use as pull-through cache for DockerHub. class DockerLoginAmazonECR(Function): name = 'docker-login-amazon-ecr' commands = [ From e7786b118d86c89213d735e2b51796393b86fc80 Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Wed, 16 Jul 2025 16:35:02 -0500 Subject: [PATCH 11/14] Also migrate sbom task from Artifactory to Amazon ECR --- .../config_generator/components/sbom.py | 86 ++++++++++--------- .evergreen/generated_configs/functions.yml | 22 ++++- .evergreen/scripts/sbom.sh | 7 +- 3 files changed, 69 insertions(+), 46 deletions(-) diff --git a/.evergreen/config_generator/components/sbom.py b/.evergreen/config_generator/components/sbom.py index dca242148d..67fcff9a9a 100644 --- a/.evergreen/config_generator/components/sbom.py +++ b/.evergreen/config_generator/components/sbom.py @@ -3,11 +3,17 @@ from config_generator.etc.utils import bash_exec from shrub.v3.evg_build_variant import BuildVariant -from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, expansions_update, s3_put +from shrub.v3.evg_command import ( + BuiltInCommand, + EvgCommandType, + KeyValueParam, + ec2_assume_role, + expansions_update, + s3_put, +) from shrub.v3.evg_task import EvgTask, EvgTaskRef from pydantic import ConfigDict -from typing import Optional TAG = 'sbom' @@ -18,56 +24,58 @@ class CustomCommand(BuiltInCommand): model_config = ConfigDict(arbitrary_types_allowed=True) -def ec2_assume_role( - role_arn: Optional[str] = None, - policy: Optional[str] = None, - duration_seconds: Optional[int] = None, - command_type: Optional[EvgCommandType] = None, -) -> CustomCommand: - return CustomCommand( - command="ec2.assume_role", - params={ - "role_arn": role_arn, - "policy": policy, - "duration_seconds": duration_seconds, - }, - type=command_type, - ) - - class SBOM(Function): name = 'sbom' commands = [ - ec2_assume_role( - command_type=EvgCommandType.SETUP, - role_arn='${kondukto_role_arn}', - ), - bash_exec( - command_type=EvgCommandType.SETUP, - include_expansions_in_env=[ - 'AWS_ACCESS_KEY_ID', - 'AWS_SECRET_ACCESS_KEY', - 'AWS_SESSION_TOKEN', - ], - script='''\ + # Authenticate with Kondukto. + *[ + ec2_assume_role( + command_type=EvgCommandType.SETUP, + role_arn='${kondukto_role_arn}', + ), + bash_exec( + command_type=EvgCommandType.SETUP, + include_expansions_in_env=[ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SESSION_TOKEN', + ], + script='''\ set -o errexit set -o pipefail kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)" printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml ''', - ), - expansions_update( - command_type=EvgCommandType.SETUP, - file='expansions.kondukto.yml', - ), + ), + expansions_update( + command_type=EvgCommandType.SETUP, + file='expansions.kondukto.yml', + ), + ], + # Authenticate with Amazon ECR. + *[ + # Avoid inadvertently using a pre-existing and potentially conflicting Podman config. + # Note: podman understands and uses DOCKER_CONFIG despite the name. + expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]), + ec2_assume_role(role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"), + bash_exec( + command_type=EvgCommandType.SETUP, + include_expansions_in_env=[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "DOCKER_CONFIG", + ], + script='aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com', + ), + ], bash_exec( command_type=EvgCommandType.TEST, working_dir='mongoc', include_expansions_in_env=[ - 'artifactory_password', - 'artifactory_username', 'branch_name', - 'KONDUKTO_TOKEN', + 'DOCKER_CONFIG', + "KONDUKTO_TOKEN", ], script='.evergreen/scripts/sbom.sh', ), diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index fb33ae2889..ec2f3b7683 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -526,15 +526,33 @@ functions: type: setup params: file: expansions.kondukto.yml + - command: expansions.update + params: + updates: + - { key: DOCKER_CONFIG, value: "${workdir}/.docker" } + - command: ec2.assume_role + params: + role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro + - command: subprocess.exec + type: setup + params: + binary: bash + include_expansions_in_env: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + - DOCKER_CONFIG + args: + - -c + - aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com - command: subprocess.exec type: test params: binary: bash working_dir: mongoc include_expansions_in_env: - - artifactory_password - - artifactory_username - branch_name + - DOCKER_CONFIG - KONDUKTO_TOKEN args: - -c diff --git a/.evergreen/scripts/sbom.sh b/.evergreen/scripts/sbom.sh index b683b73787..4a2a01dff3 100755 --- a/.evergreen/scripts/sbom.sh +++ b/.evergreen/scripts/sbom.sh @@ -3,9 +3,8 @@ set -o errexit set -o pipefail -: "${artifactory_username:?}" -: "${artifactory_password:?}" : "${branch_name:?}" +: "${DOCKER_CONFIG:?}" : "${KONDUKTO_TOKEN:?}" command -v podman >/dev/null || { @@ -13,9 +12,7 @@ command -v podman >/dev/null || { exit 1 } -podman login --password-stdin --username "${artifactory_username:?}" artifactory.corp.mongodb.com <<<"${artifactory_password:?}" - -silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0" +silkbomb="901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0" # Ensure latest version of SilkBomb is being used. podman pull "${silkbomb:?}" From 08645e6e354061461b78c56716c1ad216a088e7f Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 17 Jul 2025 10:24:54 -0500 Subject: [PATCH 12/14] Fix RST link syntax --- docs/dev/earthly.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/earthly.rst b/docs/dev/earthly.rst index f5e8866ceb..bce683b7c7 100644 --- a/docs/dev/earthly.rst +++ b/docs/dev/earthly.rst @@ -202,7 +202,7 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository above. .. seealso:: `"DevProd Platforms Container Registry" - `_ and `"Configuring IAM Identity Center authentication with the AWS CLI" _`. + `_ and `"Configuring IAM Identity Center authentication with the AWS CLI" `_. .. earthly-target:: +sbom-generate From 2c73cdc10b6580ee3ff8aa890142090a425e6c3a Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 17 Jul 2025 10:27:16 -0500 Subject: [PATCH 13/14] RST link syntax must be on single line --- docs/dev/earthly.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/earthly.rst b/docs/dev/earthly.rst index bce683b7c7..04d83d05ff 100644 --- a/docs/dev/earthly.rst +++ b/docs/dev/earthly.rst @@ -201,8 +201,8 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository ``aws ecr get-login-password ... | podman login ...`` command described above. - .. seealso:: `"DevProd Platforms Container Registry" - `_ and `"Configuring IAM Identity Center authentication with the AWS CLI" `_. + .. seealso:: `"DevProd Platforms Container Registry" `_ + and `"Configuring IAM Identity Center authentication with the AWS CLI" `_. .. earthly-target:: +sbom-generate From fe679211f41f21e0fc19057b9c7526f6c9f05bf0 Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 17 Jul 2025 10:29:49 -0500 Subject: [PATCH 14/14] Actually it was indentation error --- docs/dev/earthly.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/dev/earthly.rst b/docs/dev/earthly.rst index 04d83d05ff..9a9a45b863 100644 --- a/docs/dev/earthly.rst +++ b/docs/dev/earthly.rst @@ -201,8 +201,10 @@ enumerated using ``earthly ls`` or ``earthly doc`` in the root of the repository ``aws ecr get-login-password ... | podman login ...`` command described above. - .. seealso:: `"DevProd Platforms Container Registry" `_ - and `"Configuring IAM Identity Center authentication with the AWS CLI" `_. + .. seealso:: `"DevProd Platforms Container Registry" + `_ and + `"Configuring IAM Identity Center authentication with the AWS CLI" + `_. .. earthly-target:: +sbom-generate