[Draft] CLOUDP-333692: Re-design images building

.evergreen-functions.yml

@@ -538,7 +538,43 @@ functions:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/evergreen/run_python.sh pipeline.py --include ${image_name} --parallel --sign
+        binary: scripts/evergreen/run_python.sh scripts/release/main.py --parallel ${image_name}
+
+  legacy_pipeline:
+    - *switch_context
+    - command: shell.exec
+      type: setup
+      params:
+        shell: bash
+        script: |
+          # Docker Hub workaround
+          # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror
+          docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use
+          docker buildx inspect --bootstrap
+    - command: ec2.assume_role
+      display_name: Assume IAM role with permissions to pull Kondukto API token
+      params:
+        role_arn: ${kondukto_role_arn}
+    - command: shell.exec
+      display_name: Pull Kondukto API token from AWS Secrets Manager and write it to file
+      params:
+        silent: true
+        shell: bash
+        include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
+        script: |
+          set -e
+          # use AWS CLI to get the Kondukto API token from AWS Secrets Manager
+          kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)
+          # write the KONDUKTO_TOKEN environment variable to Silkbomb environment file
+          echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/silkbomb.env
+    - command: subprocess.exec
+      retry_on_failure: true
+      type: setup
+      params:
+        shell: bash
+        <<: *e2e_include_expansions_in_env
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        binary: scripts/evergreen/run_python.sh pipeline.py --parallel ${image_name} --sign
 
   teardown_cloud_qa_all:
     - *switch_context

.evergreen-periodic-builds.yaml

@@ -21,7 +21,7 @@ variables:
 tasks:
   - name: periodic_build_operator
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: operator-daily
 
@@ -35,49 +35,49 @@ tasks:
 
   - name: periodic_build_init_appdb
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-appdb-daily
 
   - name: periodic_build_init_database
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-database-daily
 
   - name: periodic_build_init_opsmanager
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-ops-manager-daily
 
   - name: periodic_build_database
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: database-daily
 
   - name: periodic_build_sbom_cli
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: cli
 
   - name: periodic_build_ops_manager_6
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager-6-daily
 
   - name: periodic_build_ops_manager_7
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager-7-daily
 
   - name: periodic_build_ops_manager_8
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager-8-daily
 
@@ -91,35 +91,35 @@ tasks:
     exec_timeout_secs: 43200
     commands:
       - func: enable_QEMU
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: mongodb-agent-daily
 
   - name: periodic_build_agent_1
     exec_timeout_secs: 43200
     commands:
       - func: enable_QEMU
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: mongodb-agent-1-daily
 
   #TODO should we still build the community operator?
   - name: periodic_build_community_operator
     commands:
       - func: enable_QEMU
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: mongodb-kubernetes-operator-daily
 
   - name: periodic_build_readiness_probe
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: readinessprobe-daily
 
   - name: periodic_build_version_upgrade_post_start_hook
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: operator-version-upgrade-post-start-hook-daily
 

.evergreen.yml

@@ -283,7 +283,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: operator
 
@@ -296,7 +296,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-appdb
 
@@ -308,7 +308,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-database
 
@@ -320,7 +320,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-ops-manager
 
@@ -332,7 +332,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent
 
@@ -345,7 +345,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent-pct
           include_tags: release
@@ -390,7 +390,7 @@ tasks:
     commands:
       - func: clone
       - func: setup_building_host
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent-pct
           skip_tags: release
@@ -405,7 +405,7 @@ tasks:
     commands:
       - func: clone
       - func: setup_building_host
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent-pct
           skip_tags: release
@@ -549,7 +549,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: database
 
@@ -568,7 +568,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager
           include_tags: release

.gitignore

@@ -44,11 +44,6 @@ public/architectures/**/secrets/*
 
 docker/mongodb-kubernetes-appdb/content/readinessprobe
 mongodb-kubernetes
-docker/mongodb-kubernetes-operator/Dockerfile
-docker/mongodb-kubernetes-database/Dockerfile
-docker/mongodb-enterprise-ops-manager/Dockerfile
-docker/mongodb-kubernetes-init-database/Dockerfile
-docker/mongodb-kubernetes-init-ops-manager/Dockerfile
 docker/mongodb-kubernetes-operator/content/mongodb-kubernetes-operator.tar
 docker/mongodb-kubernetes-tests/helm_chart/
 docker/mongodb-kubernetes-tests/public/

docker/mongodb-agent-non-matrix/Dockerfile

@@ -1,5 +1,14 @@
-ARG imagebase
-FROM ${imagebase} as base
+FROM scratch AS base
+
+ARG agent_version
+ARG agent_distro
+ARG tools_version
+ARG tools_distro
+
+ADD https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-${agent_version}.${agent_distro}.tar.gz /data/mongodb-agent.tar.gz
+ADD https://downloads.mongodb.org/tools/db/mongodb-database-tools-${tools_distro}-${tools_version}.tgz /data/mongodb-tools.tgz
+
+COPY ./docker/mongodb-kubernetes-init-database/content/LICENSE /data/LICENSE
 
 FROM registry.access.redhat.com/ubi9/ubi-minimal
 

docker/mongodb-agent-non-matrix/README.md

@@ -0,0 +1,17 @@
+### Building locally
+
+For building the MongoDB Agent (non-static) image locally use the example command:
+
+TODO: What to do with label quay.expires-after=48h?
+```bash
+AGENT_VERSION="108.0.7.8810-1"
+TOOLS_VERSION="100.12.0"
+AGENT_DISTRO="rhel9_x86_64"
+TOOLS_DISTRO="rhel93-x86_64"
+docker buildx build --load --progress plain . -f docker/mongodb-agent/Dockerfile -t "mongodb-agent:${AGENT_VERSION}" \
+ --build-arg version="${VERSION}" \
+ --build-arg agent_version="${AGENT_VERSION}" \
+ --build-arg tools_version="${TOOLS_VERSION}" \
+ --build-arg agent_distro="${AGENT_DISTRO}" \
+ --build-arg tools_distro="${TOOLS_DISTRO}"
+```

docker/mongodb-agent/Dockerfile

@@ -1,5 +1,40 @@
-ARG imagebase
-FROM ${imagebase} as base
+# the init database image gets supplied by pipeline.py and corresponds to the operator version we want to release
+# the agent with. This enables us to release the agent for older operator.
+ARG init_database_image
+FROM ${init_database_image} AS init_database
+
+FROM public.ecr.aws/docker/library/golang:1.24 AS dependency_downloader
+
+WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes/
+
+COPY go.mod go.sum ./
+
+RUN go mod download
+
+FROM public.ecr.aws/docker/library/golang:1.24 AS readiness_builder
+
+WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes/
+
+COPY --from=dependency_downloader /go/pkg /go/pkg
+COPY . /go/src/github.com/mongodb/mongodb-kubernetes
+
+RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /readinessprobe ./mongodb-community-operator/cmd/readiness/main.go
+RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go
+
+FROM scratch AS base
+ARG mongodb_tools_url_ubi
+ARG mongodb_agent_url_ubi
+
+COPY --from=readiness_builder /readinessprobe /data/
+COPY --from=readiness_builder /version-upgrade-hook /data/
+
+ADD ${mongodb_tools_url_ubi} /data/mongodb_tools_ubi.tgz
+ADD ${mongodb_agent_url_ubi} /data/mongodb_agent_ubi.tgz
+
+COPY --from=init_database /probes/probe.sh /data/probe.sh
+COPY --from=init_database /scripts/agent-launcher-lib.sh /data/
+COPY --from=init_database /scripts/agent-launcher.sh /data/
+COPY --from=init_database /licenses/LICENSE /data/
 
 FROM registry.access.redhat.com/ubi9/ubi-minimal
 

docker/mongodb-agent/README.md

@@ -1,4 +1,20 @@
 # Mongodb-Agent
 The agent gets released in a matrix style with the init-database image, which gets tagged with the operator version.
-This works by using the multi-stage pattern and build-args. First - retrieve the `init-database:<version>` and retrieve the 
-binaries from there. Then we continue with the other steps to fully build the image.
+This works by using the multi-stage pattern and build-args. First - retrieve the `init-database:<version>` and retrieve the
+binaries from there. Then we continue with the other steps to fully build the image.
+
+### Building locally
+
+For building the MongoDB Agent image locally use the example command:
+
+```bash
+VERSION="108.0.7.8810-1"
+INIT_DATABASE_IMAGE="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-database:1.1.0"
+MONGODB_TOOLS_URL_UBI="https://downloads.mongodb.org/tools/db/mongodb-database-tools-rhel93-x86_64-100.12.0.tgz"
+MONGODB_AGENT_URL_UBI="https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-108.0.7.8810-1.rhel9_x86_64.tar.gz"
+docker buildx build --load --progress plain . -f docker/mongodb-agent/Dockerfile -t "mongodb-agent:${VERSION}_1.1.0" \
+ --build-arg version="${VERSION}" \
+ --build-arg init_database_image="${INIT_DATABASE_IMAGE}" \
+ --build-arg mongodb_tools_url_ubi="${MONGODB_TOOLS_URL_UBI}" \
+ --build-arg mongodb_agent_url_ubi="${MONGODB_AGENT_URL_UBI}"
+```