CLOUDP-333692: Re-design images building

.evergreen-functions.yml

@@ -505,6 +505,43 @@ functions:
           - ${workdir}
 
   pipeline:
+    - *switch_context
+    - command: shell.exec
+      type: setup
+      params:
+        shell: bash
+        script: |
+          # Docker Hub workaround
+          # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror
+          docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use
+          docker buildx inspect --bootstrap
+    - command: ec2.assume_role
+      display_name: Assume IAM role with permissions to pull Kondukto API token
+      params:
+        role_arn: ${kondukto_role_arn}
+    - command: shell.exec
+      display_name: Pull Kondukto API token from AWS Secrets Manager and write it to file
+      params:
+        silent: true
+        shell: bash
+        include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
+        script: |
+          set -e
+          # use AWS CLI to get the Kondukto API token from AWS Secrets Manager
+          kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)
+          # write the KONDUKTO_TOKEN environment variable to Silkbomb environment file
+          echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/silkbomb.env
+    - command: subprocess.exec
+      retry_on_failure: true
+      type: setup
+      params:
+        shell: bash
+        <<: *e2e_include_expansions_in_env
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name}
+
+  # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function
+  legacy_pipeline:
     - *switch_context
     - command: shell.exec
       type: setup

.evergreen-periodic-builds.yaml

@@ -21,7 +21,7 @@ variables:
 tasks:
   - name: periodic_build_operator
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: operator-daily
 
@@ -35,49 +35,49 @@ tasks:
 
   - name: periodic_build_init_appdb
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-appdb-daily
 
   - name: periodic_build_init_database
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-database-daily
 
   - name: periodic_build_init_opsmanager
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-ops-manager-daily
 
   - name: periodic_build_database
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: database-daily
 
   - name: periodic_build_sbom_cli
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: cli
 
   - name: periodic_build_ops_manager_6
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager-6-daily
 
   - name: periodic_build_ops_manager_7
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager-7-daily
 
   - name: periodic_build_ops_manager_8
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager-8-daily
 
@@ -91,15 +91,15 @@ tasks:
     exec_timeout_secs: 43200
     commands:
       - func: enable_QEMU
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: mongodb-agent-daily
 
   - name: periodic_build_agent_1
     exec_timeout_secs: 43200
     commands:
       - func: enable_QEMU
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: mongodb-agent-1-daily
 
@@ -123,19 +123,19 @@ tasks:
   - name: periodic_build_community_operator
     commands:
       - func: enable_QEMU
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: mongodb-kubernetes-operator-daily
 
   - name: periodic_build_readiness_probe
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: readinessprobe-daily
 
   - name: periodic_build_version_upgrade_post_start_hook
     commands:
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: operator-version-upgrade-post-start-hook-daily
 

.evergreen.yml

@@ -283,7 +283,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: operator
           include_tags: release
@@ -297,7 +297,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-appdb
           include_tags: release
@@ -310,7 +310,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-database
           include_tags: release
@@ -323,7 +323,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: init-ops-manager
           include_tags: release
@@ -336,7 +336,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent
           include_tags: release
@@ -350,7 +350,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent-pct
           include_tags: release
@@ -395,7 +395,7 @@ tasks:
     commands:
       - func: clone
       - func: setup_building_host
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent-pct
           skip_tags: release
@@ -410,7 +410,7 @@ tasks:
     commands:
       - func: clone
       - func: setup_building_host
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: agent-pct
           skip_tags: release
@@ -554,7 +554,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: database
 
@@ -573,7 +573,7 @@ tasks:
       - func: setup_building_host
       - func: quay_login
       - func: setup_docker_sbom
-      - func: pipeline
+      - func: legacy_pipeline
         vars:
           image_name: ops-manager
           include_tags: release

Makefile

@@ -75,13 +75,13 @@ operator: configure-operator build-and-push-operator-image
 
 # build-push, (todo) restart database
 database: aws_login
-	@ scripts/dev/run_python.sh pipeline.py --include database
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py database
 
 readiness_probe: aws_login
-	@ scripts/dev/run_python.sh pipeline.py --include readiness-probe
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py readiness-probe
 
 upgrade_hook: aws_login
-	@ scripts/dev/run_python.sh pipeline.py --include upgrade-hook
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py upgrade-hook
 
 # ensures cluster is up, cleans Kubernetes + OM, build-push-deploy operator,
 # push-deploy database, create secrets, config map, resources etc
@@ -90,7 +90,7 @@ full: build-and-push-images
 
 # build-push appdb image
 appdb: aws_login
-	@ scripts/dev/run_python.sh pipeline.py --include appdb
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py --include appdb
 
 # runs the e2e test: make e2e test=e2e_sharded_cluster_pv. The Operator is redeployed before the test, the namespace is cleaned.
 # The e2e test image is built and pushed together with all main ones (operator, database, init containers)
@@ -154,19 +154,19 @@ aws_cleanup:
 	@ scripts/evergreen/prepare_aws.sh
 
 build-and-push-operator-image: aws_login
-	@ scripts/dev/run_python.sh pipeline.py --include operator-quick
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py operator
 
 build-and-push-database-image: aws_login
 	@ scripts/dev/build_push_database_image
 
 build-and-push-test-image: aws_login build-multi-cluster-binary
 	@ if [[ -z "$(local)" ]]; then \
-		scripts/dev/run_python.sh pipeline.py --include test; \
+		scripts/dev/run_python.sh scripts/release/pipeline_main.py test; \
 	fi
 
 build-and-push-mco-test-image: aws_login
 	@ if [[ -z "$(local)" ]]; then \
-		scripts/dev/run_python.sh pipeline.py --include mco-test; \
+		scripts/dev/run_python.sh scripts/release/pipeline_main.py mco-test; \
 	fi
 
 build-multi-cluster-binary:
@@ -181,27 +181,27 @@ build-and-push-images: build-and-push-operator-image appdb-init-image om-init-im
 build-and-push-init-images: appdb-init-image om-init-image database-init-image
 
 database-init-image:
-	@ scripts/dev/run_python.sh pipeline.py --include init-database
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py init-database
 
 appdb-init-image:
-	@ scripts/dev/run_python.sh pipeline.py --include init-appdb
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py init-appdb
 
 # Not setting a parallel-factor will default to 0 which will lead to using all CPUs, that can cause docker to die.
 # Here we are defaulting to 6, a higher value might work for you.
 agent-image:
-	@ scripts/dev/run_python.sh pipeline.py --include agent --all-agents --parallel --parallel-factor 6
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel --parallel-factor 6 agent
 
 agent-image-slow:
-	@ scripts/dev/run_python.sh pipeline.py --include agent --parallel-factor 1
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel-factor 1 agent
 
 operator-image:
-	@ scripts/dev/run_python.sh pipeline.py --include operator
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py operator
 
 om-init-image:
-	@ scripts/dev/run_python.sh pipeline.py --include init-ops-manager
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py init-ops-manager
 
 om-image:
-	@ scripts/dev/run_python.sh pipeline.py --include ops-manager
+	@ scripts/dev/run_python.sh scripts/release/pipeline_main.py ops-manager
 
 configure-operator:
 	@ scripts/dev/configure_operator.sh

docker/mongodb-kubernetes-readinessprobe/Dockerfile

@@ -4,7 +4,8 @@ WORKDIR /go/src
 ADD . .
 
 ARG TARGETARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -a -o /data/scripts/readinessprobe ./mongodb-community-operator/cmd/readiness/main.go
+ARG TARGETOS
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -o /data/scripts/readinessprobe ./mongodb-community-operator/cmd/readiness/main.go
 
 FROM registry.access.redhat.com/ubi9/ubi-minimal
 

docker/mongodb-kubernetes-upgrade-hook/Dockerfile

@@ -4,7 +4,8 @@ WORKDIR /go/src
 ADD . .
 
 ARG TARGETARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -a -o /data/scripts/version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go
+ARG TARGETOS
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -o /data/scripts/version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go
 
 FROM registry.access.redhat.com/ubi9/ubi-minimal
 

requirements.txt

@@ -34,6 +34,7 @@ wrapt==1.17.2
 botocore==1.39.4
 boto3==1.39.4
 python-frontmatter==1.1.0
+python-on-whales==0.78.0
 
 # from kubeobject
 freezegun==1.5.3