chore: Automate Releases

.gitignore

@@ -24,3 +24,5 @@ local_tests/serverless-init/datadog-agent
 local_tests/serverless-init/logs.txt
 bottlecap/target
 bottlecap/proptest-regressions
+
+.gitlab/pipeline-**

.gitlab-ci.yml

@@ -1,134 +1,82 @@
 variables:
-  GIT_DEPTH: 1
-  REGION_TO_DEPLOY:
-    description: "use sa-east-1 for dev, us-east-1 for RC, all for all regions"
-    value: sa-east-1
+  DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension
+  DOCKER_TARGET_VERSION: latest
+  # Manual trigger variables
   AGENT_BRANCH:
-    description: "datadog-agent branch you want to release"
+    description: "Branch of the datadog-agent repository to use."
     value: main
   LAYER_SUFFIX:
-    description: "Suffix to be appended to the layer name (default empty)"
+    description: "Suffix to be appended to the layer name (default empty)."
     value: ""
 
-image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3
-
 stages:
-  - build_tools_if_needed
-  - build_layer
-  - prepare_multi_region
-  - trigger
+  - generate
+  - build
 
-build_tools:
-  stage: build_tools_if_needed
-  variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-    TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension
+ci image:
+  stage: build
+  image: registry.ddbuild.io/images/docker:20.10
+  tags: ["arch:arm64"]
   rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
-      when: never
-    - changes:
-      - build-tools/**/*
-  tags: ["runner:docker"]
-  script:
-    - cd build-tools && docker buildx build --tag ${TARGET} --push .
-
-build_and_deploy_layer:
-  stage: build_layer
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"'
+      changes:
+        - .gitlab/Dockerfile
+      when: on_success
   variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-    ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer
-    TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension
-  tags: ["runner:docker"]
-  artifacts:
-    paths:
-      - tmp/serverless/datadog_extension_signed.zip
+    DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION}
   script:
-    - mkdir tmp
-    - git clone --branch ${AGENT_BRANCH} --depth=1 https://github.com/DataDog/datadog-agent.git
-    - dockerId=$(docker create --platform linux/amd64 ${TARGET})
-    - docker cp $dockerId:/build_tools .
-    - EXTERNAL_ID=$(aws ssm get-parameter 
-      --region us-east-1 
-      --name ci.datadog-lambda-extension.externalid 
-      --with-decryption 
-      --query "Parameter.Value" 
-      --out text)
-
-    # build
-    - ./build_tools
-      build
-      --version 1
-      --agent-version 1
-      --architecture amd64
-      --context-path .
-      --destination-path tmp/serverless
-      --docker-path "scripts_v2/Dockerfile.build"
-      --artifact-name "datadog_extension.zip"
+    - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile .
 
-    # sign
-    - ./build_tools
-      sign 
-      --layer-path tmp/serverless/datadog_extension.zip 
-      --destination-path tmp/serverless/datadog_extension_signed.zip 
-      --assume-role "$ROLE_TO_ASSUME"
-      --external-id "$EXTERNAL_ID"
+.go-cache: &go-cache
+  key: datadog-lambda-extension-go-cache
+  policy: pull
 
-    # ls artifacts
-    - ls tmp/serverless
-
-    # deploy to single region if needed
-    - if [ "${REGION_TO_DEPLOY}" = "all" ]; then exit 0; fi
-    - ./build_tools
-      deploy
-      --layer-path tmp/serverless/datadog_extension_signed.zip
-      --architecture amd64
-      --layer-name "Datadog-Extension"
-      --layer-suffix "$LAYER_SUFFIX"
-      --region "$REGION_TO_DEPLOY"
-      --assume-role "$ROLE_TO_ASSUME"
-      --external-id "$EXTERNAL_ID"
-
-prepare_multi_region:
-  stage: prepare_multi_region
-  tags: ["runner:docker"]
+generator:
+  stage: generate
+  image: registry.ddbuild.io/images/mirror/golang:alpine
+  tags: ["arch:amd64"]
+  cache: *go-cache
+  script:
+    - apk add --no-cache gomplate
+    - gomplate --config .gitlab/config.yaml
   artifacts:
     paths:
-      - trigger_region.yaml
-      - tmp/serverless/datadog_extension_signed.zip
+      - .gitlab/pipeline-bottlecap.yaml
+      - .gitlab/pipeline-go-agent.yaml
+      - .gitlab/pipeline-lambda-extension.yaml
+
+bottlecap-only:
+  stage: build
+  trigger:
+    include:
+      - artifact: .gitlab/pipeline-bottlecap.yaml
+        job: generator
+    strategy: depend
+  rules:
+    - when: on_success
+
+go-agent-only:
+  stage: build
+  trigger:
+    include:
+      - artifact: .gitlab/pipeline-go-agent.yaml
+        job: generator
+    strategy: depend
   rules:
-    - if: $REGION_TO_DEPLOY != "all"
-      when: never
     - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_PIPELINE_SOURCE == "external"
-    - if: $CI_PIPELINE_SOURCE == "trigger"
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
   variables:
-    TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension
-    ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer
-  script:
-    - echo $CI_PIPELINE_SOURCE
-    - if [ "${REGION_TO_DEPLOY}" != "all" ]; then exit 0; fi
-    - EXTERNAL_ID=$(aws ssm get-parameter
-        --region us-east-1
-        --name ci.datadog-lambda-extension.externalid
-        --with-decryption
-        --query "Parameter.Value"
-        --out text)
-    - dockerId=$(docker create --platform linux/amd64 ${TARGET})
-    - docker cp $dockerId:/build_tools .
-    - regions=$(./build_tools list_region --assume-role "$ROLE_TO_ASSUME" --external-id "$EXTERNAL_ID")
-    - sed "s/xxx_layer_sufix_xxx/${LAYER_SUFFIX}/" trigger_region.orig.yaml > trigger_region.tmp.yaml
-    - sed "s/xxx_aws_regions_xxx/${regions}/" trigger_region.tmp.yaml > trigger_region.yaml
-    - cat trigger_region.yaml
+    AGENT_BRANCH: $AGENT_BRANCH
+    LAYER_SUFFIX: $LAYER_SUFFIX
 
-multi_region:
-  rules:
-    - if: $REGION_TO_DEPLOY == "all"
-  stage: trigger
+lambda-extension:
+  stage: build
   trigger:
     include:
-      - artifact: trigger_region.yaml
-        job: prepare_multi_region
+      - artifact: .gitlab/pipeline-lambda-extension.yaml
+        job: generator
+    strategy: depend
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web"
+  variables:
+    AGENT_BRANCH: $AGENT_BRANCH
+    LAYER_SUFFIX: $LAYER_SUFFIX

.gitlab/Dockerfile

@@ -0,0 +1,23 @@
+FROM registry.ddbuild.io/images/docker:24.0.5
+
+RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
+  curl gcc gnupg g++ make cmake unzip openssl g++ uuid-runtime
+
+# Install AWS CLI
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip && ./aws/install
+
+#  Install Protocol Buffers compiler by hand
+COPY ./scripts/install-protoc.sh /
+RUN chmod +x /install-protoc.sh && /install-protoc.sh
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+  sh -s -- --profile minimal --default-toolchain nightly -y
+
+RUN source $HOME/.cargo/env
+ENV PATH /root/.cargo/bin/:$PATH
+
+RUN rustup component add rust-src --toolchain nightly
+
+

.gitlab/config.yaml

@@ -0,0 +1,21 @@
+# gomplate template generation pipeline
+
+inputFiles:
+  - .gitlab/templates/bottlecap.yaml.tpl
+  - .gitlab/templates/go-agent.yaml.tpl
+  - .gitlab/templates/lambda-extension.yaml.tpl
+
+outputFiles:
+  - .gitlab/pipeline-bottlecap.yaml
+  - .gitlab/pipeline-go-agent.yaml
+  - .gitlab/pipeline-lambda-extension.yaml
+
+datasources:
+  architectures:
+    url: .gitlab/datasources/architectures.yaml
+
+  environments:
+    url: .gitlab/datasources/environments.yaml
+
+  regions:
+    url: .gitlab/datasources/regions.yaml

.gitlab/datasources/architectures.yaml

@@ -0,0 +1,3 @@
+architectures:
+  - name: amd64
+  - name: arm64

.gitlab/datasources/environments.yaml

@@ -0,0 +1,9 @@
+environments:
+  - name: sandbox
+    external_id: sandbox-publish-externalid
+    role_to_assume: sandbox-layer-deployer
+    account: 425362996713
+  - name: prod
+    external_id: prod-publish-externalid
+    role_to_assume: dd-serverless-layer-deployer-role
+    account: 464622532012

.gitlab/datasources/regions.yaml

@@ -0,0 +1,30 @@
+regions:
+  - code: "us-east-1"
+  - code: "us-east-2"
+  - code: "us-west-1"
+  - code: "us-west-2"
+  - code: "af-south-1"
+  - code: "ap-east-1"
+  - code: "ap-south-1"
+  - code: "ap-south-2"
+  - code: "ap-southeast-1"
+  - code: "ap-southeast-2"
+  - code: "ap-southeast-3"
+  - code: "ap-southeast-4"
+  - code: "ap-northeast-1"
+  - code: "ap-northeast-2"
+  - code: "ap-northeast-3"
+  - code: "ca-central-1"
+  - code: "ca-west-1"
+  - code: "eu-central-1"
+  - code: "eu-central-2"
+  - code: "eu-north-1"
+  - code: "eu-west-1"
+  - code: "eu-west-2"
+  - code: "eu-west-3"
+  - code: "eu-south-1"
+  - code: "eu-south-2"
+  - code: "il-central-1"
+  - code: "me-south-1"
+  - code: "me-central-1"
+  - code: "sa-east-1"

.gitlab/scripts/build_bottlecap.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2024 Datadog, Inc.
+
+set -e
+
+if [ -z "$ARCHITECTURE" ]; then
+    printf "[ERROR]: ARCHITECTURE not specified\n"
+    exit 1
+fi
+
+if [ -z "$ALPINE" ]; then
+    printf "Building bottlecap"
+else
+    echo "Building bottlecap for alpine"
+    BUILD_SUFFIX="-alpine"
+fi
+
+prepare_folders() {
+    # Move into the root directory, so this script can be called from any directory
+    SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+    ROOT_DIR=$SCRIPTS_DIR/../..
+    cd $ROOT_DIR
+
+    echo $ROOT_DIR
+
+    EXTENSION_DIR=".layers"
+    TARGET_DIR=$(pwd)/$EXTENSION_DIR
+
+    rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX} 2>/dev/null
+    rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX}.zip 2>/dev/null
+
+    cd $ROOT_DIR
+}
+
+
+docker_build() {
+    local arch=$1
+    if [ "$arch" == "amd64" ]; then
+        PLATFORM="x86_64"
+    else
+        PLATFORM="aarch64"
+    fi
+
+    docker buildx build --platform linux/${arch} \
+        -t datadog/build-bottlecap-${arch} \
+        -f ./scripts/Dockerfile.bottlecap.build \
+        --build-arg PLATFORM=$PLATFORM \
+        --build-arg GO_AGENT_PATH="datadog_extension-${arch}${BUILD_SUFFIX}" \
+        . -o $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}
+
+    cp $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}.zip
+
+    unzip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}
+    rm -rf $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip
+    rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}
+    rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}.zip
+}
+
+prepare_folders
+docker_build $ARCHITECTURE