CDRIVER-5971 Use Amazon ECR to obtain OCI images in EVG (#2058)

* Address Earthly 0.8 compatibility warnings
* Workaround repository metalink retrieval errors on CentOS
* Update release instructions to use AWS ECR instead of Artifactory
* Use `$default_search_registry` to avoid forcing use of Amazon ECR
* Also migrate sbom task from Artifactory to Amazon ECR

Author: eramongodb

.evergreen/config_generator/components/earthly.py

@@ -5,9 +5,18 @@
 from typing import Iterable, Literal, Mapping, NamedTuple, TypeVar
 
 from shrub.v3.evg_build_variant import BuildVariant
-from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, subprocess_exec
+from shrub.v3.evg_command import (
+    BuiltInCommand,
+    EvgCommandType,
+    KeyValueParam,
+    ec2_assume_role,
+    expansions_update,
+    subprocess_exec,
+)
 from shrub.v3.evg_task import EvgTask, EvgTaskRef
 
+from config_generator.etc.function import Function
+
 from ..etc.utils import all_possible
 
 T = TypeVar("T")
@@ -38,7 +47,7 @@
 "Valid options for the SASL configuration parameter"
 TLSOption = Literal["OpenSSL", "off"]
 "Options for the TLS backend configuration parameter (AKA 'ENABLE_SSL')"
-CxxVersion = Literal["none"] # TODO: Once CXX-3103 is released, add latest C++ release tag.
+CxxVersion = Literal["none"]  # TODO: Once CXX-3103 is released, add latest C++ release tag.
 "C++ driver refs that are under CI test"
 
 # A separator character, since we cannot use whitespace
@@ -136,6 +145,34 @@ def suffix(self) -> str:
         return _SEPARATOR.join(f"{k}={v}" for k, v in self._asdict().items())
 
 
+# Authenticate with DevProd-provided Amazon ECR instance to use as pull-through cache for DockerHub.
+class DockerLoginAmazonECR(Function):
+    name = 'docker-login-amazon-ecr'
+    commands = [
+        # Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
+        expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
+        ec2_assume_role(role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"),
+        subprocess_exec(
+            binary="bash",
+            command_type=EvgCommandType.SETUP,
+            include_expansions_in_env=[
+                "AWS_ACCESS_KEY_ID",
+                "AWS_SECRET_ACCESS_KEY",
+                "AWS_SESSION_TOKEN",
+                "DOCKER_CONFIG",
+            ],
+            args=[
+                "-c",
+                'aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
+            ],
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
 def task_filter(env: EarthlyVariant, conf: Configuration) -> bool:
     """
     Control which tasks are actually defined by matching on the platform and
@@ -170,11 +207,16 @@ def earthly_exec(
     return subprocess_exec(
         "./tools/earthly.sh",
         args=[
+            # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits.
+            "--buildkit-image=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/earthly/buildkitd:v0.8.3",
             *(f"--secret={k}" for k in (secrets or ())),
             f"+{target}",
+            # Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits.
+            "--default_search_registry=901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub",
             *(f"--{arg}={val}" for arg, val in (args or {}).items()),
         ],
         command_type=EvgCommandType(kind),
+        include_expansions_in_env=["DOCKER_CONFIG"],
         env=env if env else None,
         working_dir="mongoc",
     )
@@ -209,15 +251,7 @@ def earthly_task(
     return EvgTask(
         name=name,
         commands=[
-            # Ensure subsequent Docker commands are authenticated.
-            subprocess_exec(
-                binary="bash",
-                command_type=EvgCommandType.SETUP,
-                args=[
-                    "-c",
-                    r'docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"',
-                ],
-            ),
+            DockerLoginAmazonECR.call(),
             # First, just build the "env-warmup" which will prepare the build environment.
             # This won't generate any output, but allows EVG to track it as a separate build step
             # for timing and logging purposes. The subequent build step will cache-hit the
@@ -249,6 +283,10 @@ def earthly_task(
 ]
 
 
+def functions():
+    return DockerLoginAmazonECR.defn()
+
+
 def tasks() -> Iterable[EvgTask]:
     for conf in all_possible(Configuration):
         # test-example is a target in all configurations

.evergreen/config_generator/components/sbom.py

@@ -3,11 +3,17 @@
 from config_generator.etc.utils import bash_exec
 
 from shrub.v3.evg_build_variant import BuildVariant
-from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, expansions_update, s3_put
+from shrub.v3.evg_command import (
+    BuiltInCommand,
+    EvgCommandType,
+    KeyValueParam,
+    ec2_assume_role,
+    expansions_update,
+    s3_put,
+)
 from shrub.v3.evg_task import EvgTask, EvgTaskRef
 
 from pydantic import ConfigDict
-from typing import Optional
 
 
 TAG = 'sbom'
@@ -18,56 +24,58 @@ class CustomCommand(BuiltInCommand):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
-def ec2_assume_role(
-    role_arn: Optional[str] = None,
-    policy: Optional[str] = None,
-    duration_seconds: Optional[int] = None,
-    command_type: Optional[EvgCommandType] = None,
-) -> CustomCommand:
-    return CustomCommand(
-        command="ec2.assume_role",
-        params={
-            "role_arn": role_arn,
-            "policy": policy,
-            "duration_seconds": duration_seconds,
-        },
-        type=command_type,
-    )
-
-
 class SBOM(Function):
     name = 'sbom'
     commands = [
-        ec2_assume_role(
-            command_type=EvgCommandType.SETUP,
-            role_arn='${kondukto_role_arn}',
-        ),
-        bash_exec(
-            command_type=EvgCommandType.SETUP,
-            include_expansions_in_env=[
-                'AWS_ACCESS_KEY_ID',
-                'AWS_SECRET_ACCESS_KEY',
-                'AWS_SESSION_TOKEN',
-            ],
-            script='''\
+        # Authenticate with Kondukto.
+        *[
+            ec2_assume_role(
+                command_type=EvgCommandType.SETUP,
+                role_arn='${kondukto_role_arn}',
+            ),
+            bash_exec(
+                command_type=EvgCommandType.SETUP,
+                include_expansions_in_env=[
+                    'AWS_ACCESS_KEY_ID',
+                    'AWS_SECRET_ACCESS_KEY',
+                    'AWS_SESSION_TOKEN',
+                ],
+                script='''\
                 set -o errexit
                 set -o pipefail
                 kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
                 printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml
             ''',
-        ),
-        expansions_update(
-            command_type=EvgCommandType.SETUP,
-            file='expansions.kondukto.yml',
-        ),
+            ),
+            expansions_update(
+                command_type=EvgCommandType.SETUP,
+                file='expansions.kondukto.yml',
+            ),
+        ],
+        # Authenticate with Amazon ECR.
+        *[
+            # Avoid inadvertently using a pre-existing and potentially conflicting Podman config.
+            # Note: podman understands and uses DOCKER_CONFIG despite the name.
+            expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
+            ec2_assume_role(role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"),
+            bash_exec(
+                command_type=EvgCommandType.SETUP,
+                include_expansions_in_env=[
+                    "AWS_ACCESS_KEY_ID",
+                    "AWS_SECRET_ACCESS_KEY",
+                    "AWS_SESSION_TOKEN",
+                    "DOCKER_CONFIG",
+                ],
+                script='aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
+            ),
+        ],
         bash_exec(
             command_type=EvgCommandType.TEST,
             working_dir='mongoc',
             include_expansions_in_env=[
-                'artifactory_password',
-                'artifactory_username',
                 'branch_name',
-                'KONDUKTO_TOKEN',
+                'DOCKER_CONFIG',
+                "KONDUKTO_TOKEN",
             ],
             script='.evergreen/scripts/sbom.sh',
         ),

.evergreen/generated_configs/functions.yml

@@ -175,6 +175,26 @@ functions:
         args:
           - -c
           - EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON ${EXTRA_CONFIGURE_FLAGS}" .evergreen/scripts/compile.sh
+  docker-login-amazon-ecr:
+    - command: expansions.update
+      params:
+        updates:
+          - { key: DOCKER_CONFIG, value: "${workdir}/.docker" }
+    - command: ec2.assume_role
+      params:
+        role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+          - DOCKER_CONFIG
+        args:
+          - -c
+          - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com
   fetch-build:
     - command: subprocess.exec
       type: setup
@@ -506,15 +526,33 @@ functions:
       type: setup
       params:
         file: expansions.kondukto.yml
+    - command: expansions.update
+      params:
+        updates:
+          - { key: DOCKER_CONFIG, value: "${workdir}/.docker" }
+    - command: ec2.assume_role
+      params:
+        role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+          - DOCKER_CONFIG
+        args:
+          - -c
+          - aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com
     - command: subprocess.exec
       type: test
       params:
         binary: bash
         working_dir: mongoc
         include_expansions_in_env:
-          - artifactory_password
-          - artifactory_username
           - branch_name
+          - DOCKER_CONFIG
           - KONDUKTO_TOKEN
         args:
           - -c