chore: Automate Releases (#295)

* update image registry

* remove `prepare_multi_region` jobs

i will use pipeline generation later

* refactor pipeline

to comply with newer standards

* update `build_bottlecap_layer.sh`

to stop using `docker` commands, since it will not work on Gitlab

* first iteration

* add path to build repo image and build in container

* update image to build

* update `.gitlab-ci.yml`

* update `.gitlab/Dockerfile

* change layer builder tag

* forgot `buildx`

* update `.gitlab/Dockerfile`

* make `cargo` available

* make `cargo` available again

* `cd bottlecap`

* in `Dockerfile` install `gcc, make, openssl, ...`

* update how `c++` is installed

* remove installing `openssl-devel`

* make `cargo` available

* install protobuf in `.gitlab/Dockerfile`

* update `.gitlab/Dockerfile`

* typo

* update arch on `check` job

trying to see if its an architecture problem

* make `protoc` available

* make most jobs to run on `amd64`

* use script to install `protoc`

also updated a job back to `arm64` to test if it works with any arch

* dont specify platform for nightly

* make pipeline dynamic

* update layer size numbers

zipped should be around 15, while unzipped is around 43 w bottlecap

* update layer size numbers

also added `get_secrets.sh` script

* add `regions.yaml`

also added `architectures.yaml`

* update `.gitlab/Dockerfile` for a dependency

* allow build for `go agent`

also modified the whole structure so we can run multiple pipelines if needed

* pass any path starting with `pipeline-`

* specify paths for pipelines

* set to include artifact for `go-agent`

* typo on pipeline to use

* try different cloning strategy

* typo on scrip to use to build

* fix script to copy stuff from `datadog-agent`

* typo

* update where the get secrets script comes from

* add `sign_layers.sh`

* allow signing layers for both environments

* abstract scripts to use `LAYER_FILE`

for size check and signing

* add `awscli` to `Dockerfile`

* update templates to use publishing script properly

* revert image to be built always

* update go template for signing only in prod

* allow suffix to be used when publishing

* move when we add suffix

we were checking the wrong version number for sandbox

* eol

* allow a new pipeline for combined go+rust extension

* add pipeline in `.gitlab.yml`

* update `go-agent.yaml.tpl`

added arch as variable

* make bottlecap dev use another dockerfile

* update `build_go_agent.sh`

mainly to unzip before removingthe zip

* build bottlecap for gitlab

* add extension with bottlecap pipeline

* typo in `config.yaml`

* clone `datadog-agent` before building the go agent

facepalm

* see if agent branch is defaulted to main

* its not printing, try again

* disallow specific runtime pipelines to publish to prod

* `lambda-extension` to only be triggered through web

* fix how version is set for `build_go_agent.sh`

* maybe copy isnt finding the right file?

* debug ls

* forgot to add path to directory including the binaries

* update layer size

* remove all regions but `sa-east-1` temporarily

* increase layer size again

* add all regions back

* only add permissions on prod

Author: duncanista

.gitignore

@@ -24,3 +24,5 @@ local_tests/serverless-init/datadog-agent
 local_tests/serverless-init/logs.txt
 bottlecap/target
 bottlecap/proptest-regressions
+
+.gitlab/pipeline-**

.gitlab-ci.yml

@@ -1,134 +1,82 @@
 variables:
-  GIT_DEPTH: 1
-  REGION_TO_DEPLOY:
-    description: "use sa-east-1 for dev, us-east-1 for RC, all for all regions"
-    value: sa-east-1
+  DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension
+  DOCKER_TARGET_VERSION: latest
+  # Manual trigger variables
   AGENT_BRANCH:
-    description: "datadog-agent branch you want to release"
+    description: "Branch of the datadog-agent repository to use."
     value: main
   LAYER_SUFFIX:
-    description: "Suffix to be appended to the layer name (default empty)"
+    description: "Suffix to be appended to the layer name (default empty)."
     value: ""
 
-image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3
-
 stages:
-  - build_tools_if_needed
-  - build_layer
-  - prepare_multi_region
-  - trigger
+  - generate
+  - build
 
-build_tools:
-  stage: build_tools_if_needed
-  variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-    TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension
+ci image:
+  stage: build
+  image: registry.ddbuild.io/images/docker:20.10
+  tags: ["arch:arm64"]
   rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
-      when: never
-    - changes:
-      - build-tools/**/*
-  tags: ["runner:docker"]
-  script:
-    - cd build-tools && docker buildx build --tag ${TARGET} --push .
-
-build_and_deploy_layer:
-  stage: build_layer
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"'
+      changes:
+        - .gitlab/Dockerfile
+      when: on_success
   variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-    ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer
-    TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension
-  tags: ["runner:docker"]
-  artifacts:
-    paths:
-      - tmp/serverless/datadog_extension_signed.zip
+    DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION}
   script:
-    - mkdir tmp
-    - git clone --branch ${AGENT_BRANCH} --depth=1 https://github.com/DataDog/datadog-agent.git
-    - dockerId=$(docker create --platform linux/amd64 ${TARGET})
-    - docker cp $dockerId:/build_tools .
-    - EXTERNAL_ID=$(aws ssm get-parameter 
-      --region us-east-1 
-      --name ci.datadog-lambda-extension.externalid 
-      --with-decryption 
-      --query "Parameter.Value" 
-      --out text)
-
-    # build
-    - ./build_tools
-      build
-      --version 1
-      --agent-version 1
-      --architecture amd64
-      --context-path .
-      --destination-path tmp/serverless
-      --docker-path "scripts_v2/Dockerfile.build"
-      --artifact-name "datadog_extension.zip"
+    - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile .
 
-    # sign
-    - ./build_tools
-      sign 
-      --layer-path tmp/serverless/datadog_extension.zip 
-      --destination-path tmp/serverless/datadog_extension_signed.zip 
-      --assume-role "$ROLE_TO_ASSUME"
-      --external-id "$EXTERNAL_ID"
+.go-cache: &go-cache
+  key: datadog-lambda-extension-go-cache
+  policy: pull
 
-    # ls artifacts
-    - ls tmp/serverless
-
-    # deploy to single region if needed
-    - if [ "${REGION_TO_DEPLOY}" = "all" ]; then exit 0; fi
-    - ./build_tools
-      deploy
-      --layer-path tmp/serverless/datadog_extension_signed.zip
-      --architecture amd64
-      --layer-name "Datadog-Extension"
-      --layer-suffix "$LAYER_SUFFIX"
-      --region "$REGION_TO_DEPLOY"
-      --assume-role "$ROLE_TO_ASSUME"
-      --external-id "$EXTERNAL_ID"
-
-prepare_multi_region:
-  stage: prepare_multi_region
-  tags: ["runner:docker"]
+generator:
+  stage: generate
+  image: registry.ddbuild.io/images/mirror/golang:alpine
+  tags: ["arch:amd64"]
+  cache: *go-cache
+  script:
+    - apk add --no-cache gomplate
+    - gomplate --config .gitlab/config.yaml
   artifacts:
     paths:
-      - trigger_region.yaml
-      - tmp/serverless/datadog_extension_signed.zip
+      - .gitlab/pipeline-bottlecap.yaml
+      - .gitlab/pipeline-go-agent.yaml
+      - .gitlab/pipeline-lambda-extension.yaml
+
+bottlecap-only:
+  stage: build
+  trigger:
+    include:
+      - artifact: .gitlab/pipeline-bottlecap.yaml
+        job: generator
+    strategy: depend
+  rules:
+    - when: on_success
+
+go-agent-only:
+  stage: build
+  trigger:
+    include:
+      - artifact: .gitlab/pipeline-go-agent.yaml
+        job: generator
+    strategy: depend
   rules:
-    - if: $REGION_TO_DEPLOY != "all"
-      when: never
     - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_PIPELINE_SOURCE == "external"
-    - if: $CI_PIPELINE_SOURCE == "trigger"
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
   variables:
-    TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension
-    ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer
-  script:
-    - echo $CI_PIPELINE_SOURCE
-    - if [ "${REGION_TO_DEPLOY}" != "all" ]; then exit 0; fi
-    - EXTERNAL_ID=$(aws ssm get-parameter
-        --region us-east-1
-        --name ci.datadog-lambda-extension.externalid
-        --with-decryption
-        --query "Parameter.Value"
-        --out text)
-    - dockerId=$(docker create --platform linux/amd64 ${TARGET})
-    - docker cp $dockerId:/build_tools .
-    - regions=$(./build_tools list_region --assume-role "$ROLE_TO_ASSUME" --external-id "$EXTERNAL_ID")
-    - sed "s/xxx_layer_sufix_xxx/${LAYER_SUFFIX}/" trigger_region.orig.yaml > trigger_region.tmp.yaml
-    - sed "s/xxx_aws_regions_xxx/${regions}/" trigger_region.tmp.yaml > trigger_region.yaml
-    - cat trigger_region.yaml
+    AGENT_BRANCH: $AGENT_BRANCH
+    LAYER_SUFFIX: $LAYER_SUFFIX
 
-multi_region:
-  rules:
-    - if: $REGION_TO_DEPLOY == "all"
-  stage: trigger
+lambda-extension:
+  stage: build
   trigger:
     include:
-      - artifact: trigger_region.yaml
-        job: prepare_multi_region
+      - artifact: .gitlab/pipeline-lambda-extension.yaml
+        job: generator
+    strategy: depend
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web"
+  variables:
+    AGENT_BRANCH: $AGENT_BRANCH
+    LAYER_SUFFIX: $LAYER_SUFFIX

.gitlab/Dockerfile

@@ -0,0 +1,23 @@
+FROM registry.ddbuild.io/images/docker:24.0.5
+
+RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
+  curl gcc gnupg g++ make cmake unzip openssl g++ uuid-runtime
+
+# Install AWS CLI
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip && ./aws/install
+
+#  Install Protocol Buffers compiler by hand
+COPY ./scripts/install-protoc.sh /
+RUN chmod +x /install-protoc.sh && /install-protoc.sh
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+  sh -s -- --profile minimal --default-toolchain nightly -y
+
+RUN source $HOME/.cargo/env
+ENV PATH /root/.cargo/bin/:$PATH
+
+RUN rustup component add rust-src --toolchain nightly
+
+

.gitlab/config.yaml

@@ -0,0 +1,21 @@
+# gomplate template generation pipeline
+
+inputFiles:
+  - .gitlab/templates/bottlecap.yaml.tpl
+  - .gitlab/templates/go-agent.yaml.tpl
+  - .gitlab/templates/lambda-extension.yaml.tpl
+
+outputFiles:
+  - .gitlab/pipeline-bottlecap.yaml
+  - .gitlab/pipeline-go-agent.yaml
+  - .gitlab/pipeline-lambda-extension.yaml
+
+datasources:
+  architectures:
+    url: .gitlab/datasources/architectures.yaml
+  
+  environments:
+    url: .gitlab/datasources/environments.yaml
+
+  regions:
+    url: .gitlab/datasources/regions.yaml

.gitlab/datasources/architectures.yaml

@@ -0,0 +1,3 @@
+architectures:
+  - name: amd64
+  - name: arm64

.gitlab/datasources/environments.yaml

@@ -0,0 +1,9 @@
+environments:
+  - name: sandbox
+    external_id: sandbox-publish-externalid
+    role_to_assume: sandbox-layer-deployer
+    account: 425362996713
+  - name: prod
+    external_id: prod-publish-externalid
+    role_to_assume: dd-serverless-layer-deployer-role
+    account: 464622532012

.gitlab/datasources/regions.yaml

@@ -0,0 +1,30 @@
+regions:
+  - code: "us-east-1"
+  - code: "us-east-2"
+  - code: "us-west-1"
+  - code: "us-west-2"
+  - code: "af-south-1"
+  - code: "ap-east-1"
+  - code: "ap-south-1"
+  - code: "ap-south-2"
+  - code: "ap-southeast-1"
+  - code: "ap-southeast-2"
+  - code: "ap-southeast-3"
+  - code: "ap-southeast-4"
+  - code: "ap-northeast-1"
+  - code: "ap-northeast-2"
+  - code: "ap-northeast-3"
+  - code: "ca-central-1"
+  - code: "ca-west-1"
+  - code: "eu-central-1"
+  - code: "eu-central-2"
+  - code: "eu-north-1"
+  - code: "eu-west-1"
+  - code: "eu-west-2"
+  - code: "eu-west-3"
+  - code: "eu-south-1"
+  - code: "eu-south-2"
+  - code: "il-central-1"
+  - code: "me-south-1"
+  - code: "me-central-1"
+  - code: "sa-east-1"

.gitlab/scripts/build_bottlecap.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2024 Datadog, Inc.
+
+set -e
+
+if [ -z "$ARCHITECTURE" ]; then
+    printf "[ERROR]: ARCHITECTURE not specified\n"
+    exit 1
+fi
+
+if [ -z "$ALPINE" ]; then
+    printf "Building bottlecap"
+else
+    echo "Building bottlecap for alpine"
+    BUILD_SUFFIX="-alpine"
+fi
+
+prepare_folders() {
+    # Move into the root directory, so this script can be called from any directory
+    SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+    ROOT_DIR=$SCRIPTS_DIR/../..
+    cd $ROOT_DIR
+
+    echo $ROOT_DIR
+
+    EXTENSION_DIR=".layers"
+    TARGET_DIR=$(pwd)/$EXTENSION_DIR
+
+    rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX} 2>/dev/null
+    rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX}.zip 2>/dev/null
+
+    cd $ROOT_DIR
+}
+
+
+docker_build() {
+    local arch=$1
+    if [ "$arch" == "amd64" ]; then
+        PLATFORM="x86_64"
+    else
+        PLATFORM="aarch64"
+    fi
+
+    docker buildx build --platform linux/${arch} \
+        -t datadog/build-bottlecap-${arch} \
+        -f ./scripts/Dockerfile.bottlecap.build \
+        --build-arg PLATFORM=$PLATFORM \
+        --build-arg GO_AGENT_PATH="datadog_extension-${arch}${BUILD_SUFFIX}" \
+        . -o $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}
+
+    cp $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}.zip
+
+    unzip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}
+    rm -rf $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip
+    rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}
+    rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}.zip
+}
+
+prepare_folders
+docker_build $ARCHITECTURE