CXX-3265 Update release instructions to use Amazon ECR instead of Artifactory (#1425)

* Use inline script metadata for extras/docker/generate.py
* Migrate EVG tasks from Artifactory to Amazon ECR

Author: eramongodb

.evergreen/config_generator/components/docker_build.py

@@ -7,6 +7,7 @@
 from shrub.v3.evg_build_variant import BuildVariant
 from shrub.v3.evg_command import EvgCommandType
 from shrub.v3.evg_task import EvgTask, EvgTaskRef
+from shrub.v3.evg_command import KeyValueParam, ec2_assume_role, expansions_update
 
 
 TAG = 'docker-build'
@@ -24,24 +25,43 @@
 
 class DockerImageBuild(Function):
     name = 'docker-image-build'
-    commands = bash_exec(
-        command_type=EvgCommandType.TEST,
-        working_dir='mongo-cxx-driver',
-        script='''\
-            set -o errexit
-            set -o pipefail
-            docker login -u "${ARTIFACTORY_USER}" --password-stdin artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD}"
-            set -x
-            echo "Building Alpine Docker image"
-            make -C extras/docker/alpine3.19 nocachebuild test
-            echo "Building Debian Docker image"
-            make -C extras/docker/bookworm nocachebuild test
-            echo "Building Red Hat UBI Docker image"
-            make -C extras/docker/redhat-ubi-9.4 nocachebuild test
-            echo "Building Ubuntu Docker image"
-            make -C extras/docker/noble nocachebuild test
-        '''
-    )
+    commands = [
+        # Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
+        expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
+        ec2_assume_role(role_arn='arn:aws:iam::901841024863:role/ecr-role-evergreen-ro'),
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            include_expansions_in_env=[
+                "AWS_ACCESS_KEY_ID",
+                "AWS_SECRET_ACCESS_KEY",
+                "AWS_SESSION_TOKEN",
+                "DOCKER_CONFIG",
+            ],
+            script='aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
+        ),
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            working_dir='mongo-cxx-driver',
+            env={
+                # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits.
+                "DEFAULT_SEARCH_REGISTRY": "901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub",
+            },
+            include_expansions_in_env=['DOCKER_CONFIG'],
+            script='''\
+                set -o errexit
+                set -o pipefail
+                set -x
+                echo "Building Alpine Docker image"
+                make -C extras/docker/alpine3.19 nocachebuild test
+                echo "Building Debian Docker image"
+                make -C extras/docker/bookworm nocachebuild test
+                echo "Building Red Hat UBI Docker image"
+                make -C extras/docker/redhat-ubi-9.4 nocachebuild test
+                echo "Building Ubuntu Docker image"
+                make -C extras/docker/noble nocachebuild test
+            ''',
+        ),
+    ]
 
 
 def functions():
@@ -62,7 +82,7 @@ def tasks():
                 commands=[
                     Setup.call(),
                     DockerImageBuild.call(),
-                ]
+                ],
             )
         )
 

.evergreen/config_generator/components/sbom.py

@@ -5,11 +5,17 @@
 from config_generator.etc.utils import bash_exec
 
 from shrub.v3.evg_build_variant import BuildVariant
-from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, ec2_assume_role, expansions_update, s3_put
+from shrub.v3.evg_command import (
+    BuiltInCommand,
+    EvgCommandType,
+    KeyValueParam,
+    ec2_assume_role,
+    expansions_update,
+    s3_put,
+)
 from shrub.v3.evg_task import EvgTask, EvgTaskRef
 
 from pydantic import ConfigDict
-from typing import Optional
 
 
 TAG = 'sbom'
@@ -23,31 +29,50 @@ class CustomCommand(BuiltInCommand):
 class CheckAugmentedSBOM(Function):
     name = 'check augmented sbom'
     commands = [
-        ec2_assume_role(
-            command_type=EvgCommandType.SETUP,
-            role_arn='${KONDUKTO_ROLE_ARN}',
-        ),
-        bash_exec(
-            command_type=EvgCommandType.SETUP,
-            include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'],
-            script='''\
-            set -o errexit
-            set -o pipefail
-            kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
-            printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml
-        ''',
-        ),
-        expansions_update(
-            command_type=EvgCommandType.SETUP,
-            file='expansions.kondukto.yml',
-        ),
+        # Authenticate with Kondukto.
+        *[
+            ec2_assume_role(
+                command_type=EvgCommandType.SETUP,
+                role_arn='${KONDUKTO_ROLE_ARN}',
+            ),
+            bash_exec(
+                command_type=EvgCommandType.SETUP,
+                include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'],
+                script='''\
+                    set -o errexit
+                    set -o pipefail
+                    kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
+                    printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml
+                ''',
+            ),
+            expansions_update(
+                command_type=EvgCommandType.SETUP,
+                file='expansions.kondukto.yml',
+            ),
+        ],
+        # Authenticate with Amazon ECR.
+        *[
+            # Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
+            # Note: podman understands and uses DOCKER_CONFIG despite the name.
+            expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
+            ec2_assume_role(role_arn='arn:aws:iam::901841024863:role/ecr-role-evergreen-ro'),
+            bash_exec(
+                command_type=EvgCommandType.SETUP,
+                include_expansions_in_env=[
+                    'AWS_ACCESS_KEY_ID',
+                    'AWS_SECRET_ACCESS_KEY',
+                    'AWS_SESSION_TOKEN',
+                    'DOCKER_CONFIG',
+                ],
+                script='aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
+            ),
+        ],
         bash_exec(
             command_type=EvgCommandType.TEST,
             working_dir='mongo-cxx-driver',
             include_expansions_in_env=[
-                'ARTIFACTORY_PASSWORD',
-                'ARTIFACTORY_USER',
                 'branch_name',
+                'DOCKER_CONFIG',
                 'KONDUKTO_TOKEN',
             ],
             script='.evergreen/scripts/sbom.sh',

.evergreen/generated_configs/functions.yml

@@ -246,15 +246,33 @@ functions:
       type: setup
       params:
         file: expansions.kondukto.yml
+    - command: expansions.update
+      params:
+        updates:
+          - { key: DOCKER_CONFIG, value: "${workdir}/.docker" }
+    - command: ec2.assume_role
+      params:
+        role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+          - DOCKER_CONFIG
+        args:
+          - -c
+          - aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com
     - command: subprocess.exec
       type: test
       params:
         binary: bash
         working_dir: mongo-cxx-driver
         include_expansions_in_env:
-          - ARTIFACTORY_PASSWORD
-          - ARTIFACTORY_USER
           - branch_name
+          - DOCKER_CONFIG
           - KONDUKTO_TOKEN
         args:
           - -c
@@ -328,26 +346,48 @@ functions:
         - -c
         - .evergreen/scripts/compile.sh
   docker-image-build:
-    command: subprocess.exec
-    type: test
-    params:
-      binary: bash
-      working_dir: mongo-cxx-driver
-      args:
-        - -c
-        - |
-          set -o errexit
-          set -o pipefail
-          docker login -u "${ARTIFACTORY_USER}" --password-stdin artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD}"
-          set -x
-          echo "Building Alpine Docker image"
-          make -C extras/docker/alpine3.19 nocachebuild test
-          echo "Building Debian Docker image"
-          make -C extras/docker/bookworm nocachebuild test
-          echo "Building Red Hat UBI Docker image"
-          make -C extras/docker/redhat-ubi-9.4 nocachebuild test
-          echo "Building Ubuntu Docker image"
-          make -C extras/docker/noble nocachebuild test
+    - command: expansions.update
+      params:
+        updates:
+          - { key: DOCKER_CONFIG, value: "${workdir}/.docker" }
+    - command: ec2.assume_role
+      params:
+        role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+          - DOCKER_CONFIG
+        args:
+          - -c
+          - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: mongo-cxx-driver
+        env:
+          DEFAULT_SEARCH_REGISTRY: 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub
+        include_expansions_in_env:
+          - DOCKER_CONFIG
+        args:
+          - -c
+          - |
+            set -o errexit
+            set -o pipefail
+            set -x
+            echo "Building Alpine Docker image"
+            make -C extras/docker/alpine3.19 nocachebuild test
+            echo "Building Debian Docker image"
+            make -C extras/docker/bookworm nocachebuild test
+            echo "Building Red Hat UBI Docker image"
+            make -C extras/docker/redhat-ubi-9.4 nocachebuild test
+            echo "Building Ubuntu Docker image"
+            make -C extras/docker/noble nocachebuild test
   fetch-det:
     - command: subprocess.exec
       type: setup

.evergreen/scripts/sbom.sh

@@ -3,9 +3,8 @@
 set -o errexit
 set -o pipefail
 
-: "${ARTIFACTORY_USER:?}"
-: "${ARTIFACTORY_PASSWORD:?}"
 : "${branch_name:?}"
+: "${DOCKER_CONFIG:?}"
 : "${KONDUKTO_TOKEN:?}"
 
 command -v podman >/dev/null || {
@@ -18,9 +17,7 @@ command -v jq >/dev/null || {
   exit 1
 }
 
-podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}"
-
-silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0"
+silkbomb="901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0"
 
 # Ensure latest version of SilkBomb is being used.
 podman pull "${silkbomb:?}"

etc/garasign_dist_file.sh

@@ -19,15 +19,8 @@ if ! command -v gpg >/dev/null; then
   echo "gpg is required to verify distribution tarball signature" 1>&2
 fi
 
-artifactory_creds=~/.secrets/artifactory-creds.txt
 garasign_creds=~/.secrets/garasign-creds.txt
 
-unset ARTIFACTORY_USER ARTIFACTORY_PASSWORD
-# shellcheck source=/dev/null
-. "${artifactory_creds:?}"
-: "${ARTIFACTORY_USER:?"missing ARTIFACTORY_USER in ${artifactory_creds:?}"}"
-: "${ARTIFACTORY_PASSWORD:?"missing ARTIFACTORY_PASSWORD in ${artifactory_creds:?}"}"
-
 unset GRS_CONFIG_USER1_USERNAME GRS_CONFIG_USER1_PASSWORD
 # shellcheck source=/dev/null
 . "${garasign_creds:?}"
@@ -37,10 +30,8 @@ unset GRS_CONFIG_USER1_USERNAME GRS_CONFIG_USER1_PASSWORD
 dist_file="${1:?}"
 dist_file_signed="${dist_file:?}.asc"
 
-"${launcher:?}" login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}"
-
 # Ensure latest version of Garasign is being used.
-"${launcher:?}" pull artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-gpg
+"${launcher:?}" pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-gpg
 
 plugin_commands=(
   gpg --yes -v --armor -o "${dist_file_signed:?}" --detach-sign "${dist_file:?}"
@@ -51,7 +42,7 @@ plugin_commands=(
   --rm \
   -v "$(pwd):$(pwd)" \
   -w "$(pwd)" \
-  artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-gpg
+  901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-gpg
 
 # Validate the signature file works as intended.
 (

etc/garasign_release_tag.sh

@@ -17,25 +17,16 @@ if ! command -v "${launcher:?}" >/dev/null; then
   echo "${launcher:?} is required to create a GPG-signed release tag" 1>&2
 fi
 
-artifactory_creds=~/.secrets/artifactory-creds.txt
 garasign_creds=~/.secrets/garasign-creds.txt
 
-unset ARTIFACTORY_USER ARTIFACTORY_PASSWORD
-# shellcheck source=/dev/null
-. "${artifactory_creds:?}"
-: "${ARTIFACTORY_USER:?"missing ARTIFACTORY_USER in ${artifactory_creds:?}"}"
-: "${ARTIFACTORY_PASSWORD:?"missing ARTIFACTORY_PASSWORD in ${artifactory_creds:?}"}"
-
 unset GRS_CONFIG_USER1_USERNAME GRS_CONFIG_USER1_PASSWORD
 # shellcheck source=/dev/null
 . "${garasign_creds:?}"
 : "${GRS_CONFIG_USER1_USERNAME:?"missing GRS_CONFIG_USER1_USERNAME in ${garasign_creds:?}"}"
 : "${GRS_CONFIG_USER1_PASSWORD:?"missing GRS_CONFIG_USER1_PASSWORD in ${garasign_creds:?}"}"
 
-"${launcher:?}" login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}"
-
 # Ensure latest version of Garasign is being used.
-"${launcher:?}" pull artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-git
+"${launcher:?}" pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-git
 
 # Sign using "MongoDB C++ Release Signing Key <[email protected]>" from https://pgp.mongodb.com/ (cpp-driver).
 git_tag_command=(
@@ -57,7 +48,7 @@ plugin_commands+=" && ${git_tag_command[*]:?}"
   --rm \
   -v "$(pwd):$(pwd)" \
   -w "$(pwd)" \
-  artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-git
+  901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-git
 
 # Validate the release tag is signed as intended.
 (