From 5a3b1c9f77d17b7695060ff61722e0e994815156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 8 May 2024 13:28:27 -0400 Subject: [PATCH 01/74] update image registry --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b59bcd75f..e98ff2c12 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: description: "Suffix to be appended to the layer name (default empty)" value: "" -image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3 +image: registry.ddbuild.io/images/docker:20.10-py3 stages: - build_tools_if_needed @@ -22,7 +22,7 @@ build_tools: stage: build_tools_if_needed variables: CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension + TARGET: registry.ddbuild.io/ci/datadog-lambda-extension rules: - if: $CI_PIPELINE_SOURCE == "web" when: never @@ -39,7 +39,7 @@ build_and_deploy_layer: variables: CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer - TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension + TARGET: registry.ddbuild.io/ci/datadog-lambda-extension tags: ["runner:docker"] artifacts: paths: @@ -106,7 +106,7 @@ prepare_multi_region: - if: $CI_PIPELINE_SOURCE == "pipeline" - if: $CI_PIPELINE_SOURCE == "parent_pipeline" variables: - TARGET: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-lambda-extension + TARGET: registry.ddbuild.io/ci/datadog-lambda-extension ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer script: - echo $CI_PIPELINE_SOURCE From 07266c79ff615184fd187ab64338aa2331f08c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 8 May 2024 13:29:18 -0400 Subject: [PATCH 02/74] remove `prepare_multi_region` jobs i will use pipeline generation later --- .gitlab-ci.yml | 45 +-------------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e98ff2c12..ee3f1f12c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -88,47 +88,4 @@ build_and_deploy_layer: --layer-suffix "$LAYER_SUFFIX" --region "$REGION_TO_DEPLOY" --assume-role "$ROLE_TO_ASSUME" - --external-id "$EXTERNAL_ID" - -prepare_multi_region: - stage: prepare_multi_region - tags: ["runner:docker"] - artifacts: - paths: - - trigger_region.yaml - - tmp/serverless/datadog_extension_signed.zip - rules: - - if: $REGION_TO_DEPLOY != "all" - when: never - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_PIPELINE_SOURCE == "external" - - if: $CI_PIPELINE_SOURCE == "trigger" - - if: $CI_PIPELINE_SOURCE == "pipeline" - - if: $CI_PIPELINE_SOURCE == "parent_pipeline" - variables: - TARGET: registry.ddbuild.io/ci/datadog-lambda-extension - ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer - script: - - echo $CI_PIPELINE_SOURCE - - if [ "${REGION_TO_DEPLOY}" != "all" ]; then exit 0; fi - - EXTERNAL_ID=$(aws ssm get-parameter - --region us-east-1 - --name ci.datadog-lambda-extension.externalid - --with-decryption - --query "Parameter.Value" - --out text) - - dockerId=$(docker create --platform linux/amd64 ${TARGET}) - - docker cp $dockerId:/build_tools . - - regions=$(./build_tools list_region --assume-role "$ROLE_TO_ASSUME" --external-id "$EXTERNAL_ID") - - sed "s/xxx_layer_sufix_xxx/${LAYER_SUFFIX}/" trigger_region.orig.yaml > trigger_region.tmp.yaml - - sed "s/xxx_aws_regions_xxx/${regions}/" trigger_region.tmp.yaml > trigger_region.yaml - - cat trigger_region.yaml - -multi_region: - rules: - - if: $REGION_TO_DEPLOY == "all" - stage: trigger - trigger: - include: - - artifact: trigger_region.yaml - job: prepare_multi_region \ No newline at end of file + --external-id "$EXTERNAL_ID" \ No newline at end of file From e97dd726923e14a82fc9356b777a3368d67cc6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 8 May 2024 13:40:06 -0400 Subject: [PATCH 03/74] refactor pipeline to comply with newer standards --- .gitlab-ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee3f1f12c..955dc3320 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,34 +13,34 @@ variables: image: registry.ddbuild.io/images/docker:20.10-py3 stages: - - build_tools_if_needed - - build_layer - - prepare_multi_region + - build + - build-layer - trigger -build_tools: - stage: build_tools_if_needed - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - TARGET: registry.ddbuild.io/ci/datadog-lambda-extension +build-tools: + stage: build + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:amd64"] rules: - if: $CI_PIPELINE_SOURCE == "web" when: never - changes: - build-tools/**/* - tags: ["runner:docker"] + variables: + DOCKER_TARGET: registry.ddbuild.io/ci/datadog-lambda-extension script: - - cd build-tools && docker buildx build --tag ${TARGET} --push . + - cd build-tools + - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} . -build_and_deploy_layer: - stage: build_layer +build-and-deploy-layer: + stage: build-layer rules: - if: $CI_PIPELINE_SOURCE == "web" variables: CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer TARGET: registry.ddbuild.io/ci/datadog-lambda-extension - tags: ["runner:docker"] + tags: ["arch:amd64"] artifacts: paths: - tmp/serverless/datadog_extension_signed.zip From 5a4860e4b41cbbbf4ee63e577b884e6cd09f8b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:14:47 -0400 Subject: [PATCH 04/74] update `build_bottlecap_layer.sh` to stop using `docker` commands, since it will not work on Gitlab --- scripts/build_bottlecap_layer.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/build_bottlecap_layer.sh b/scripts/build_bottlecap_layer.sh index 3e1f40fd4..1e7ebd1ca 100755 --- a/scripts/build_bottlecap_layer.sh +++ b/scripts/build_bottlecap_layer.sh @@ -41,11 +41,12 @@ _docker_build_bottlecap_zip() { -t datadog/build-bottlecap-${arch} \ -f ./scripts/Dockerfile.bottlecap.build \ --build-arg PLATFORM=$PLATFORM \ - . --load - local dockerId=$(docker create datadog/build-bottlecap-${arch}) - docker cp $dockerId:/datadog_extension.zip $TARGET_DIR/datadog_bottlecap-${arch}.zip - docker rm $dockerId - unzip $TARGET_DIR/datadog_bottlecap-${arch}.zip -d $TARGET_DIR/datadog_bottlecap-${arch} + . -o $TARGET_DIR/datadog_bottlecap-${arch} + + cp $TARGET_DIR/datadog_bottlecap-${arch}/datadog_extension.zip $TARGET_DIR/datadog_bottlecap-${arch}.zip + + unzip $TARGET_DIR/datadog_bottlecap-${arch}/datadog_extension.zip -d $TARGET_DIR/datadog_bottlecap-${arch} + rm -rf $TARGET_DIR/datadog_bottlecap-${arch}/datadog_extension.zip } build_for_arch() { From c2eafdafbcba5053a33fc5a565074d76a07c04b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:34:34 -0400 Subject: [PATCH 05/74] first iteration --- .gitlab-ci.yml | 114 ++++++++++++++++----------------------------- .gitlab/Dockerfile | 3 ++ 2 files changed, 42 insertions(+), 75 deletions(-) create mode 100644 .gitlab/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 955dc3320..3a97ded6b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,91 +1,55 @@ variables: - GIT_DEPTH: 1 - REGION_TO_DEPLOY: - description: "use sa-east-1 for dev, us-east-1 for RC, all for all regions" - value: sa-east-1 - AGENT_BRANCH: - description: "datadog-agent branch you want to release" - value: main - LAYER_SUFFIX: - description: "Suffix to be appended to the layer name (default empty)" - value: "" - -image: registry.ddbuild.io/images/docker:20.10-py3 + DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension + DOCKER_TARGET_VERSION: latest stages: - build - - build-layer - - trigger + - test + - deploy -build-tools: +ci image: stage: build image: registry.ddbuild.io/images/docker:20.10-py3 - tags: ["arch:amd64"] - rules: - - if: $CI_PIPELINE_SOURCE == "web" - when: never - - changes: - - build-tools/**/* + tags: ["arch:arm64"] + # rules: + # - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' + # when: on_success variables: - DOCKER_TARGET: registry.ddbuild.io/ci/datadog-lambda-extension + DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} script: - - cd build-tools - - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} . + - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} .gitlab/Dockerfile -build-and-deploy-layer: - stage: build-layer - rules: - - if: $CI_PIPELINE_SOURCE == "web" - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - ROLE_TO_ASSUME: arn:aws:iam::425362996713:role/sandbox-layer-deployer - TARGET: registry.ddbuild.io/ci/datadog-lambda-extension - tags: ["arch:amd64"] +build layer: + stage: build + tags: ["arch:arm64"] + image: registry.ddbuild.io/images/docker:24.0.5 artifacts: + expire_in: 1 hr paths: - - tmp/serverless/datadog_extension_signed.zip + - .layers/datadog_bottlecap-arm64.zip script: - - mkdir tmp - - git clone --branch ${AGENT_BRANCH} --depth=1 https://github.com/DataDog/datadog-agent.git - - dockerId=$(docker create --platform linux/amd64 ${TARGET}) - - docker cp $dockerId:/build_tools . - - EXTERNAL_ID=$(aws ssm get-parameter - --region us-east-1 - --name ci.datadog-lambda-extension.externalid - --with-decryption - --query "Parameter.Value" - --out text) - - # build - - ./build_tools - build - --version 1 - --agent-version 1 - --architecture amd64 - --context-path . - --destination-path tmp/serverless - --docker-path "scripts_v2/Dockerfile.build" - --artifact-name "datadog_extension.zip" + - ARCHITECTURE=arm64 ./scripts/build_bottlecap_layer.sh - # sign - - ./build_tools - sign - --layer-path tmp/serverless/datadog_extension.zip - --destination-path tmp/serverless/datadog_extension_signed.zip - --assume-role "$ROLE_TO_ASSUME" - --external-id "$EXTERNAL_ID" +fmt: + stage: test + tags: ["arch:arm64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + # needs: [] + script: + - cargo fmt - # ls artifacts - - ls tmp/serverless +check: + stage: test + tags: ["arch:arm64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + # needs: [] + script: + - cargo check - # deploy to single region if needed - - if [ "${REGION_TO_DEPLOY}" = "all" ]; then exit 0; fi - - ./build_tools - deploy - --layer-path tmp/serverless/datadog_extension_signed.zip - --architecture amd64 - --layer-name "Datadog-Extension" - --layer-suffix "$LAYER_SUFFIX" - --region "$REGION_TO_DEPLOY" - --assume-role "$ROLE_TO_ASSUME" - --external-id "$EXTERNAL_ID" \ No newline at end of file +clippy: + stage: test + tags: ["arch:arm64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + # needs: [] + script: + - cargo clippy --all-features diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile new file mode 100644 index 000000000..1b1842457 --- /dev/null +++ b/.gitlab/Dockerfile @@ -0,0 +1,3 @@ +FROM registry.ddbuild.io/images/docker:24.0.5 + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh From 6dce2fd6ea6680df13abec6b80f15e6b75031933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:42:33 -0400 Subject: [PATCH 06/74] add path to build repo image and build in container --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a97ded6b..f085ab68a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ ci image: variables: DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} script: - - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} .gitlab/Dockerfile + - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f ./gitlab/Dockerfile . build layer: stage: build @@ -27,6 +27,8 @@ build layer: expire_in: 1 hr paths: - .layers/datadog_bottlecap-arm64.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" script: - ARCHITECTURE=arm64 ./scripts/build_bottlecap_layer.sh From 774d69e77038aeb78cbe8e1f889c5cc9b0d2a824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:44:02 -0400 Subject: [PATCH 07/74] update image to build --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f085ab68a..294868fb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,12 +17,12 @@ ci image: variables: DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} script: - - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f ./gitlab/Dockerfile . + - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile . build layer: stage: build tags: ["arch:arm64"] - image: registry.ddbuild.io/images/docker:24.0.5 + image: registry.ddbuild.io/images/docker:20.10-py3 artifacts: expire_in: 1 hr paths: From ed4d3c1ab66ad95240588d4ed51a4b8f1114f9fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:49:00 -0400 Subject: [PATCH 08/74] update `.gitlab-ci.yml` --- .gitlab-ci.yml | 2 -- .gitlab/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 294868fb7..f9b157243 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,8 +27,6 @@ build layer: expire_in: 1 hr paths: - .layers/datadog_bottlecap-arm64.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" script: - ARCHITECTURE=arm64 ./scripts/build_bottlecap_layer.sh diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 1b1842457..3b6344a8f 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,3 +1,3 @@ FROM registry.ddbuild.io/images/docker:24.0.5 -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +RUN curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh From c8a65c6182ea41d12851ff26865f106b17feb2ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:53:23 -0400 Subject: [PATCH 09/74] update `.gitlab/Dockerfile --- .gitlab-ci.yml | 4 +++- .gitlab/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9b157243..ce397083c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,11 +22,13 @@ ci image: build layer: stage: build tags: ["arch:arm64"] - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:20.10 artifacts: expire_in: 1 hr paths: - .layers/datadog_bottlecap-arm64.zip + variables: + CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" script: - ARCHITECTURE=arm64 ./scripts/build_bottlecap_layer.sh diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 3b6344a8f..6b663dda0 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,3 +1,3 @@ FROM registry.ddbuild.io/images/docker:24.0.5 -RUN curl -y --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -y From bf73b79d63bb30321790b1a3edd93c4f2737dfc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:59:08 -0400 Subject: [PATCH 10/74] change layer builder tag --- .gitlab-ci.yml | 2 +- .gitlab/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ce397083c..37bfdbf22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ ci image: build layer: stage: build - tags: ["arch:arm64"] + tags: ["arch:amd64"] image: registry.ddbuild.io/images/docker:20.10 artifacts: expire_in: 1 hr diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 6b663dda0..d2ff6da46 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,3 +1,3 @@ FROM registry.ddbuild.io/images/docker:24.0.5 -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -y +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -- -y From 67c2dfbeb41122e6b119bfd0f1ad623d77e92241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:11:11 -0400 Subject: [PATCH 11/74] forgot `buildx` --- .gitlab-ci.yml | 2 +- scripts/build_bottlecap_layer.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 37bfdbf22..86ef64237 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ stages: ci image: stage: build - image: registry.ddbuild.io/images/docker:20.10-py3 + image: registry.ddbuild.io/images/docker:20.10 tags: ["arch:arm64"] # rules: # - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' diff --git a/scripts/build_bottlecap_layer.sh b/scripts/build_bottlecap_layer.sh index 1e7ebd1ca..e6620b155 100755 --- a/scripts/build_bottlecap_layer.sh +++ b/scripts/build_bottlecap_layer.sh @@ -37,7 +37,7 @@ _docker_build_bottlecap_zip() { PLATFORM="aarch64" fi - docker build --platform linux/${arch} \ + docker buildx build --platform linux/${arch} \ -t datadog/build-bottlecap-${arch} \ -f ./scripts/Dockerfile.bottlecap.build \ --build-arg PLATFORM=$PLATFORM \ From 22f0b9452545db4b6169cd660b61fe617bb37ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:13:11 -0400 Subject: [PATCH 12/74] update `.gitlab/Dockerfile` --- .gitlab/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index d2ff6da46..ff5533ddb 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,3 +1,3 @@ FROM registry.ddbuild.io/images/docker:24.0.5 -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -- -y +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From 1482b69d715f3766a2acb6be05b556091db09db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:16:44 -0400 Subject: [PATCH 13/74] make `cargo` available --- .gitlab/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index ff5533ddb..af50df9f4 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,3 +1,4 @@ FROM registry.ddbuild.io/images/docker:24.0.5 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" From d090eab95efdb84d849b39c464f16410864b1395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:24:08 -0400 Subject: [PATCH 14/74] make `cargo` available again --- .gitlab/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index af50df9f4..97d8a3eb6 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,4 +1,5 @@ FROM registry.ddbuild.io/images/docker:24.0.5 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -ENV PATH="/root/.cargo/bin:${PATH}" +RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc +ENV PATH="$HOME/.cargo/bin:${PATH}" From 442e3d0b1eb53c9bc836e2ec1847d8b346085610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:28:52 -0400 Subject: [PATCH 15/74] `cd bottlecap` --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86ef64237..36a8c8a80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ fmt: image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} # needs: [] script: - - cargo fmt + - cd bottlecap && cargo fmt check: stage: test @@ -46,7 +46,7 @@ check: image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} # needs: [] script: - - cargo check + - cd bottlecap && cargo check clippy: stage: test @@ -54,4 +54,4 @@ clippy: image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} # needs: [] script: - - cargo clippy --all-features + - cd bottlecap && cargo clippy --all-features From eb56cbf0b51cf69a838a2d17f72f9d2f621b0a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:43:23 -0400 Subject: [PATCH 16/74] in `Dockerfile` install `gcc, make, openssl, ...` --- .gitlab/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 97d8a3eb6..2527df002 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,5 +1,10 @@ FROM registry.ddbuild.io/images/docker:24.0.5 +RUN apt-get update && apt-get install -y \ + curl gcc gcc-c++ make unzip openssl openssl-devel + +# Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc ENV PATH="$HOME/.cargo/bin:${PATH}" From 89122938a9b985845674374f3a9f6b66e2c6e408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:47:58 -0400 Subject: [PATCH 17/74] update how `c++` is installed --- .gitlab/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 2527df002..a720911a9 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,7 +1,7 @@ FROM registry.ddbuild.io/images/docker:24.0.5 -RUN apt-get update && apt-get install -y \ - curl gcc gcc-c++ make unzip openssl openssl-devel +RUN apt-get update && apt-get install -y --fix-missing \ + curl gcc gnupg g++ make unzip openssl openssl-devel # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From 5db91403a62c21d1713d32ac53ca66438022153d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:50:13 -0400 Subject: [PATCH 18/74] remove installing `openssl-devel` --- .gitlab/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index a720911a9..241e4eec6 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,7 +1,7 @@ FROM registry.ddbuild.io/images/docker:24.0.5 RUN apt-get update && apt-get install -y --fix-missing \ - curl gcc gnupg g++ make unzip openssl openssl-devel + curl gcc gnupg g++ make unzip openssl g++ # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From cae1e5742c063e517c1c88a702dcf17ee6676493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:06:04 -0400 Subject: [PATCH 19/74] make `cargo` available --- .gitlab/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 241e4eec6..26d4cb77a 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -6,5 +6,5 @@ RUN apt-get update && apt-get install -y --fix-missing \ # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc -ENV PATH="$HOME/.cargo/bin:${PATH}" +RUN source $HOME/.cargo/env +ENV PATH /root/.cargo/bin/:$PATH From 6825ce07c04cb46747bffb0fd21f7fe70281fdb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:18:34 -0400 Subject: [PATCH 20/74] install protobuf in `.gitlab/Dockerfile` --- .gitlab/Dockerfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 26d4cb77a..1513c9534 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,7 +1,16 @@ FROM registry.ddbuild.io/images/docker:24.0.5 RUN apt-get update && apt-get install -y --fix-missing \ - curl gcc gnupg g++ make unzip openssl g++ + curl gcc gnupg g++ make cmake unzip openssl g++ + +ENV PB_VERSION 25.3 +ENV PB_URL https://github.com/protocolbuffers/protobuf/releases/download +ENV PB_TARGET linux-x86_64 + +# Install protobuy 25.3 compiler from binary for linux-86_64 +RUN curl -LO ${PB_URL}/v${PB_VERSION}/protoc-${PB_VERSION}-${PB_TARGET}.zip +RUN unzip protoc-${PB_VERSION}-${PB_TARGET}.zip -d "$HOME/.local" +ENV PATH $PATH:$HOME/.local/bin # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y From 1e9a02d4c3db054d0bd093866aa99302a77b8d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:30:35 -0400 Subject: [PATCH 21/74] update `.gitlab/Dockerfile` --- .gitlab/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 1513c9534..5fa45d858 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -13,7 +13,10 @@ RUN unzip protoc-${PB_VERSION}-${PB_TARGET}.zip -d "$HOME/.local" ENV PATH $PATH:$HOME/.local/bin # Install Rust -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- --profile minimal --default-toolchain nightly-x86_84-unknown-linux-gnu -y RUN source $HOME/.cargo/env ENV PATH /root/.cargo/bin/:$PATH + +RUN rustup component add rust-src --toolchain nightly-x86_84-unknown-linux-gnu \ No newline at end of file From 3784240cc4c4e751c0e5df4946bfb8386f9bf452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:32:49 -0400 Subject: [PATCH 22/74] typo --- .gitlab/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 5fa45d858..4677622ae 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -14,9 +14,9 @@ ENV PATH $PATH:$HOME/.local/bin # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- --profile minimal --default-toolchain nightly-x86_84-unknown-linux-gnu -y + sh -s -- --profile minimal --default-toolchain nightly-x86_64-unknown-linux-gnu -y RUN source $HOME/.cargo/env ENV PATH /root/.cargo/bin/:$PATH -RUN rustup component add rust-src --toolchain nightly-x86_84-unknown-linux-gnu \ No newline at end of file +RUN rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu \ No newline at end of file From 004e81e0b9a9349045d6e1a9bdffd3f0b7720741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:52:04 -0400 Subject: [PATCH 23/74] update arch on `check` job trying to see if its an architecture problem --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36a8c8a80..49090c516 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ fmt: check: stage: test - tags: ["arch:arm64"] + tags: ["arch:amd64"] image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} # needs: [] script: From 36c0cefc51e053449337f2bb58f19e6c63b17a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:53:36 -0400 Subject: [PATCH 24/74] make `protoc` available --- .gitlab/Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 4677622ae..2a2a56cc6 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -7,11 +7,6 @@ ENV PB_VERSION 25.3 ENV PB_URL https://github.com/protocolbuffers/protobuf/releases/download ENV PB_TARGET linux-x86_64 -# Install protobuy 25.3 compiler from binary for linux-86_64 -RUN curl -LO ${PB_URL}/v${PB_VERSION}/protoc-${PB_VERSION}-${PB_TARGET}.zip -RUN unzip protoc-${PB_VERSION}-${PB_TARGET}.zip -d "$HOME/.local" -ENV PATH $PATH:$HOME/.local/bin - # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ sh -s -- --profile minimal --default-toolchain nightly-x86_64-unknown-linux-gnu -y @@ -19,4 +14,11 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ RUN source $HOME/.cargo/env ENV PATH /root/.cargo/bin/:$PATH -RUN rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu \ No newline at end of file +RUN rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + +# Install protobuy 25.3 compiler from binary for linux-86_64 +RUN curl -LO ${PB_URL}/v${PB_VERSION}/protoc-${PB_VERSION}-${PB_TARGET}.zip +RUN unzip protoc-${PB_VERSION}-${PB_TARGET}.zip -d protoc +RUN mv protoc/bin/* /usr/local/bin/ +RUN mv protoc/include/* /usr/local/include/ +RUN rm -rf protoc* From acc9a1e4ee6f9155914674994c9bf4a0c2d97841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:59:08 -0400 Subject: [PATCH 25/74] make most jobs to run on `amd64` --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49090c516..5cdb8a2ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,7 +34,7 @@ build layer: fmt: stage: test - tags: ["arch:arm64"] + tags: ["arch:amd64"] image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} # needs: [] script: @@ -50,7 +50,7 @@ check: clippy: stage: test - tags: ["arch:arm64"] + tags: ["arch:amd64"] image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} # needs: [] script: From b9f4c68052723888ca32d53f7eedc036bd682fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:20:19 -0400 Subject: [PATCH 26/74] use script to install `protoc` also updated a job back to `arm64` to test if it works with any arch --- .gitlab-ci.yml | 2 +- .gitlab/Dockerfile | 18 +++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cdb8a2ff..ee4231525 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ check: clippy: stage: test - tags: ["arch:amd64"] + tags: ["arch:arm64"] image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} # needs: [] script: diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index 2a2a56cc6..fe22594ce 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,24 +1,20 @@ FROM registry.ddbuild.io/images/docker:24.0.5 +ARG PLATFORM RUN apt-get update && apt-get install -y --fix-missing \ curl gcc gnupg g++ make cmake unzip openssl g++ -ENV PB_VERSION 25.3 -ENV PB_URL https://github.com/protocolbuffers/protobuf/releases/download -ENV PB_TARGET linux-x86_64 +# Install Protocol Buffers compiler by hand +COPY ./scripts/install-protoc.sh / +RUN chmod +x /install-protoc.sh && /install-protoc.sh # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- --profile minimal --default-toolchain nightly-x86_64-unknown-linux-gnu -y + sh -s -- --profile minimal --default-toolchain nightly-$PLATFORM-unknown-linux-gnu -y RUN source $HOME/.cargo/env ENV PATH /root/.cargo/bin/:$PATH -RUN rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu +RUN rustup component add rust-src --toolchain nightly-$PLATFORM-unknown-linux-gnu + -# Install protobuy 25.3 compiler from binary for linux-86_64 -RUN curl -LO ${PB_URL}/v${PB_VERSION}/protoc-${PB_VERSION}-${PB_TARGET}.zip -RUN unzip protoc-${PB_VERSION}-${PB_TARGET}.zip -d protoc -RUN mv protoc/bin/* /usr/local/bin/ -RUN mv protoc/include/* /usr/local/include/ -RUN rm -rf protoc* From 4dd52b7fd13819a36b0aba571ad3828ea8ba7aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:35:39 -0400 Subject: [PATCH 27/74] dont specify platform for nightly --- .gitlab/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index fe22594ce..d8cabfaae 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,6 +1,5 @@ FROM registry.ddbuild.io/images/docker:24.0.5 -ARG PLATFORM RUN apt-get update && apt-get install -y --fix-missing \ curl gcc gnupg g++ make cmake unzip openssl g++ @@ -10,11 +9,11 @@ RUN chmod +x /install-protoc.sh && /install-protoc.sh # Install Rust RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- --profile minimal --default-toolchain nightly-$PLATFORM-unknown-linux-gnu -y + sh -s -- --profile minimal --default-toolchain nightly -y RUN source $HOME/.cargo/env ENV PATH /root/.cargo/bin/:$PATH -RUN rustup component add rust-src --toolchain nightly-$PLATFORM-unknown-linux-gnu +RUN rustup component add rust-src --toolchain nightly From 0b0aacf63285c475799f58299de23efdee20d074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:44:12 -0400 Subject: [PATCH 28/74] make pipeline dynamic --- .gitignore | 2 + .gitlab-ci.yml | 62 +++++++++++---------------- .gitlab/architectures.yaml | 3 ++ .gitlab/config.yaml | 11 +++++ .gitlab/scripts/check_layer_size.sh | 42 +++++++++++++++++++ .gitlab/template.yaml.tpl | 65 +++++++++++++++++++++++++++++ 6 files changed, 148 insertions(+), 37 deletions(-) create mode 100644 .gitlab/architectures.yaml create mode 100644 .gitlab/config.yaml create mode 100755 .gitlab/scripts/check_layer_size.sh create mode 100644 .gitlab/template.yaml.tpl diff --git a/.gitignore b/.gitignore index 00557a628..42fa99691 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ local_tests/serverless-init/datadog-agent local_tests/serverless-init/logs.txt bottlecap/target bottlecap/proptest-regressions + +.gitlab/pipeline.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee4231525..f5db4c885 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,55 +3,43 @@ variables: DOCKER_TARGET_VERSION: latest stages: + - generate - build - - test - - deploy ci image: stage: build image: registry.ddbuild.io/images/docker:20.10 tags: ["arch:arm64"] - # rules: - # - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' - # when: on_success + rules: + - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' + when: on_success variables: DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} script: - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile . -build layer: - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_bottlecap-arm64.zip - variables: - CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true" - script: - - ARCHITECTURE=arm64 ./scripts/build_bottlecap_layer.sh +.go-cache: &go-cache + key: datadog-lambda-extension-go-cache + policy: pull -fmt: - stage: test +generator: + stage: generate + image: registry.ddbuild.io/images/mirror/golang:alpine tags: ["arch:amd64"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - # needs: [] + cache: *go-cache script: - - cd bottlecap && cargo fmt - -check: - stage: test - tags: ["arch:amd64"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - # needs: [] - script: - - cd bottlecap && cargo check + - apk add --no-cache gomplate + - gomplate --config .gitlab/config.yaml + artifacts: + paths: + - .gitlab/pipeline.yaml -clippy: - stage: test - tags: ["arch:arm64"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - # needs: [] - script: - - cd bottlecap && cargo clippy --all-features +bottlecap: + stage: build + trigger: + include: + - artifact: .gitlab/pipeline.yaml + job: generator + strategy: depend + rules: + - when: on_success \ No newline at end of file diff --git a/.gitlab/architectures.yaml b/.gitlab/architectures.yaml new file mode 100644 index 000000000..b56abbf75 --- /dev/null +++ b/.gitlab/architectures.yaml @@ -0,0 +1,3 @@ +architectures: + - name: amd64 + - name: arm64 \ No newline at end of file diff --git a/.gitlab/config.yaml b/.gitlab/config.yaml new file mode 100644 index 000000000..d53e9ad05 --- /dev/null +++ b/.gitlab/config.yaml @@ -0,0 +1,11 @@ +# gomplate template generation pipeline + +inputFiles: + - .gitlab/template.yaml.tpl + +outputFiles: + - .gitlab/pipeline.yaml + +datasources: + architectures: + url: .gitlab/architectures.yaml \ No newline at end of file diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh new file mode 100755 index 000000000..68b525cc5 --- /dev/null +++ b/.gitlab/scripts/check_layer_size.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +# Compares layer size to threshold, and fails if below that threshold + +set -e + +if [ -z "$ARCHITECTURE" ]; then + echo "ARCHITECTURE not specified" + echo "" + exit 1 +fi + +MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 5 \* 1024) # 5MB +MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 12 \* 1024) # 12MB + + +LAYER_FILES_PREFIX="datadog_bottlecap" +LAYER_DIR=".layers" + +FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}-${ARCHITECTURE}.zip +FILE_SIZE=$(stat --printf="%s" $FILE) +FILE_SIZE_KB="$(( ${FILE_SIZE%% *} / 1024))" +echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb" +if [ "$FILE_SIZE_KB" -gt "$MAX_LAYER_COMPRESSED_SIZE_KB" ]; then + echo "Zipped size exceeded limit $MAX_LAYER_COMPRESSED_SIZE_KB kb" + exit 1 +fi +mkdir tmp +unzip -q $FILE -d tmp +UNZIPPED_FILE_SIZE=$(du -shb tmp/ | cut -f1) +UNZIPPED_FILE_SIZE_KB="$(( ${UNZIPPED_FILE_SIZE%% *} / 1024))" +rm -rf tmp +echo "Layer file ${FILE} has unzipped size ${UNZIPPED_FILE_SIZE_KB} kb" +if [ "$UNZIPPED_FILE_SIZE_KB" -gt "$MAX_LAYER_UNCOMPRESSED_SIZE_KB" ]; then + echo "Unzipped size exceeded limit $MAX_LAYER_UNCOMPRESSED_SIZE_KB kb" + exit 1 +fi \ No newline at end of file diff --git a/.gitlab/template.yaml.tpl b/.gitlab/template.yaml.tpl new file mode 100644 index 000000000..221bab080 --- /dev/null +++ b/.gitlab/template.yaml.tpl @@ -0,0 +1,65 @@ +stages: + - build + - test + - sign + - publish + +default: + retry: + max: 1 + when: + - runner_system_failure + +variables: + DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension + DOCKER_TARGET_VERSION: latest + +{{ range $architecture := (ds "architectures").architectures }} + +build layer (bottlecap) ({{ $architecture.name }}): + stage: build + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + artifacts: + expire_in: 1 hr + paths: + - .layers/datadog_bottlecap-{{ $architecture.name }}.zip + script: + - ARCHITECTURE={{ $architecture.name }} ./scripts/build_bottlecap_layer.sh + +check layer size (bottlecap) ({{ $architecture.name }}): + stage: test + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: + - build layer (bottlecap) ({{ $architecture.name }}) + dependencies: + - build layer (bottlecap) ({{ $architecture.name }}) + script: + - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/check_layer_size.sh + +fmt ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo fmt + +check ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo check + +clippy ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo clippy --all-features + +{{- end }} # architectures end \ No newline at end of file From 015b9545cbe1952178ff332d5989f24529e555ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:50:42 -0400 Subject: [PATCH 29/74] update layer size numbers zipped should be around 15, while unzipped is around 43 w bottlecap --- .gitlab/scripts/check_layer_size.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh index 68b525cc5..c69fe9a8f 100755 --- a/.gitlab/scripts/check_layer_size.sh +++ b/.gitlab/scripts/check_layer_size.sh @@ -15,8 +15,8 @@ if [ -z "$ARCHITECTURE" ]; then exit 1 fi -MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 5 \* 1024) # 5MB -MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 12 \* 1024) # 12MB +MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 15 \* 1024) # 15 MB +MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 43 \* 1024) # 43 MB LAYER_FILES_PREFIX="datadog_bottlecap" From 3a747169b6dd44ae53928bab281994fa338e31ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 4 Jul 2024 14:20:31 -0400 Subject: [PATCH 30/74] update layer size numbers also added `get_secrets.sh` script --- .gitlab/scripts/check_layer_size.sh | 4 +-- .gitlab/scripts/get_secrets.sh | 48 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100755 .gitlab/scripts/get_secrets.sh diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh index c69fe9a8f..dccc4a569 100755 --- a/.gitlab/scripts/check_layer_size.sh +++ b/.gitlab/scripts/check_layer_size.sh @@ -3,7 +3,7 @@ # Unless explicitly stated otherwise all files in this repository are licensed # under the Apache License Version 2.0. # This product includes software developed at Datadog (https://www.datadoghq.com/). -# Copyright 2019 Datadog, Inc. +# Copyright 2024 Datadog, Inc. # Compares layer size to threshold, and fails if below that threshold @@ -15,7 +15,7 @@ if [ -z "$ARCHITECTURE" ]; then exit 1 fi -MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 15 \* 1024) # 15 MB +MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 16 \* 1024) # 16 MB, amd64 is 16, while arm64 is 15 MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 43 \* 1024) # 43 MB diff --git a/.gitlab/scripts/get_secrets.sh b/.gitlab/scripts/get_secrets.sh new file mode 100755 index 000000000..6d5df92c2 --- /dev/null +++ b/.gitlab/scripts/get_secrets.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +set -e + +if [ -z "$EXTERNAL_ID_NAME" ]; then + printf "[Error] No EXTERNAL_ID_NAME found.\n" + printf "Exiting script...\n" + exit 1 +fi + +if [ -z "$ROLE_TO_ASSUME" ]; then + printf "[Error] No ROLE_TO_ASSUME found.\n" + printf "Exiting script...\n" + exit 1 +fi + +printf "Getting AWS External ID...\n" + +EXTERNAL_ID=$(aws ssm get-parameter \ + --region us-east-1 \ + --name "ci.datadog-lambda-extension.$EXTERNAL_ID_NAME" \ + --with-decryption \ + --query "Parameter.Value" \ + --out text) + +printf "Getting DD API KEY...\n" + +export DD_API_KEY=$(aws ssm get-parameter \ + --region us-east-1 \ + --name ci.datadog-lambda-extension.dd-api-key \ + --with-decryption \ + --query "Parameter.Value" \ + --out text) + +printf "Assuming role...\n" + +export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \ + $(aws sts assume-role \ + --role-arn "arn:aws:iam::$AWS_ACCOUNT:role/$ROLE_TO_ASSUME" \ + --role-session-name "ci.datadog-lambda-extension-$CI_JOB_ID-$CI_JOB_STAGE" \ + --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ + --external-id $EXTERNAL_ID \ + --output text)) \ No newline at end of file From d674797035d16f500f0bc4b72069f993a566c182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:17:35 -0400 Subject: [PATCH 31/74] add `regions.yaml` also added `architectures.yaml` --- .gitlab/datasources/architectures.yaml | 3 +++ .gitlab/datasources/regions.yaml | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 .gitlab/datasources/architectures.yaml create mode 100644 .gitlab/datasources/regions.yaml diff --git a/.gitlab/datasources/architectures.yaml b/.gitlab/datasources/architectures.yaml new file mode 100644 index 000000000..b56abbf75 --- /dev/null +++ b/.gitlab/datasources/architectures.yaml @@ -0,0 +1,3 @@ +architectures: + - name: amd64 + - name: arm64 \ No newline at end of file diff --git a/.gitlab/datasources/regions.yaml b/.gitlab/datasources/regions.yaml new file mode 100644 index 000000000..9f12a0118 --- /dev/null +++ b/.gitlab/datasources/regions.yaml @@ -0,0 +1,30 @@ +regions: + - code: "us-east-1" + - code: "us-east-2" + - code: "us-west-1" + - code: "us-west-2" + - code: "af-south-1" + - code: "ap-east-1" + - code: "ap-south-1" + - code: "ap-south-2" + - code: "ap-southeast-1" + - code: "ap-southeast-2" + - code: "ap-southeast-3" + - code: "ap-southeast-4" + - code: "ap-northeast-1" + - code: "ap-northeast-2" + - code: "ap-northeast-3" + - code: "ca-central-1" + - code: "ca-west-1" + - code: "eu-central-1" + - code: "eu-central-2" + - code: "eu-north-1" + - code: "eu-west-1" + - code: "eu-west-2" + - code: "eu-west-3" + - code: "eu-south-1" + - code: "eu-south-2" + - code: "il-central-1" + - code: "me-south-1" + - code: "me-central-1" + - code: "sa-east-1" From 00fdc2686653dec237ea19fac27c296ec7e6fe04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:18:26 -0400 Subject: [PATCH 32/74] update `.gitlab/Dockerfile` for a dependency --- .gitlab/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index d8cabfaae..bc1a8cf1f 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,7 +1,7 @@ FROM registry.ddbuild.io/images/docker:24.0.5 RUN apt-get update && apt-get install -y --fix-missing \ - curl gcc gnupg g++ make cmake unzip openssl g++ + curl gcc gnupg g++ make cmake unzip openssl g++ uuid-runtime # Install Protocol Buffers compiler by hand COPY ./scripts/install-protoc.sh / From c45b9298554a4f52659cb5baaf3ef588ca4718e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:03:06 -0400 Subject: [PATCH 33/74] allow build for `go agent` also modified the whole structure so we can run multiple pipelines if needed --- .gitignore | 2 +- .gitlab-ci.yml | 27 +++++- .gitlab/architectures.yaml | 3 - .gitlab/config.yaml | 14 ++- .gitlab/datasources/environments.yaml | 9 ++ .gitlab/scripts/build_go_agent.sh | 74 +++++++++++++++ .gitlab/template.yaml.tpl | 65 ------------- .gitlab/templates/bottlecap.yaml.tpl | 130 ++++++++++++++++++++++++++ .gitlab/templates/go-agent.yaml.tpl | 32 +++++++ 9 files changed, 281 insertions(+), 75 deletions(-) delete mode 100644 .gitlab/architectures.yaml create mode 100644 .gitlab/datasources/environments.yaml create mode 100755 .gitlab/scripts/build_go_agent.sh delete mode 100644 .gitlab/template.yaml.tpl create mode 100644 .gitlab/templates/bottlecap.yaml.tpl create mode 100644 .gitlab/templates/go-agent.yaml.tpl diff --git a/.gitignore b/.gitignore index 42fa99691..f91f8dbbf 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,4 @@ local_tests/serverless-init/logs.txt bottlecap/target bottlecap/proptest-regressions -.gitlab/pipeline.yaml +.gitlab/pipeline-** diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f5db4c885..d2e600b40 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,13 @@ variables: DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension DOCKER_TARGET_VERSION: latest + # Manual trigger variables + AGENT_BRANCH: + description: "Branch of the datadog-agent repository to use." + value: main + LAYER_SUFFIX: + description: "Suffix to be appended to the layer name (default empty)." + value: "" stages: - generate @@ -12,6 +19,8 @@ ci image: tags: ["arch:arm64"] rules: - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' + changes: + - .gitlab/Dockerfile when: on_success variables: DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} @@ -32,14 +41,26 @@ generator: - gomplate --config .gitlab/config.yaml artifacts: paths: - - .gitlab/pipeline.yaml + - .gitlab/pipeline-bottlecap.yaml bottlecap: stage: build trigger: include: - - artifact: .gitlab/pipeline.yaml + - artifact: .gitlab/pipeline-bottlecap.yaml job: generator strategy: depend rules: - - when: on_success \ No newline at end of file + - when: on_success + +go-agent: + stage: build + trigger: + include: + - local: .gitlab/pipeline-go-agent.yaml + strategy: depend + rules: + - if: $CI_PIPELINE_SOURCE == "web" + variables: + AGENT_BRANCH: $AGENT_BRANCH + LAYER_SUFFIX: $LAYER_SUFFIX \ No newline at end of file diff --git a/.gitlab/architectures.yaml b/.gitlab/architectures.yaml deleted file mode 100644 index b56abbf75..000000000 --- a/.gitlab/architectures.yaml +++ /dev/null @@ -1,3 +0,0 @@ -architectures: - - name: amd64 - - name: arm64 \ No newline at end of file diff --git a/.gitlab/config.yaml b/.gitlab/config.yaml index d53e9ad05..de680c281 100644 --- a/.gitlab/config.yaml +++ b/.gitlab/config.yaml @@ -1,11 +1,19 @@ # gomplate template generation pipeline inputFiles: - - .gitlab/template.yaml.tpl + - .gitlab/templates/bottlecap.yaml.tpl + - .gitlab/templates/go-agent.yaml.tpl outputFiles: - - .gitlab/pipeline.yaml + - .gitlab/pipeline-bottlecap.yaml + - .gitlab/pipeline-go-agent.yaml datasources: architectures: - url: .gitlab/architectures.yaml \ No newline at end of file + url: .gitlab/datasources/architectures.yaml + + environments: + url: .gitlab/datasources/environments.yaml + + regions: + url: .gitlab/datasources/regions.yaml diff --git a/.gitlab/datasources/environments.yaml b/.gitlab/datasources/environments.yaml new file mode 100644 index 000000000..90056ab00 --- /dev/null +++ b/.gitlab/datasources/environments.yaml @@ -0,0 +1,9 @@ +environments: + - name: sandbox + external_id: sandbox-publish-externalid + role_to_assume: sandbox-layer-deployer + account: 425362996713 + - name: prod + external_id: prod-publish-externalid + role_to_assume: dd-serverless-layer-deployer-role + account: 464622532012 diff --git a/.gitlab/scripts/build_go_agent.sh b/.gitlab/scripts/build_go_agent.sh new file mode 100755 index 000000000..fc9434a4e --- /dev/null +++ b/.gitlab/scripts/build_go_agent.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +# Usage +# ARCHITECTURE=arm64 ./scripts/build_go_agent.sh + +set -e + +if [ -z "$ARCHITECTURE" ]; then + echo "ARCHITECTURE not specified" + echo "" + exit 1 +fi + +# If version is not set, then this is not a release build +if [ -z "$VERSION" ]; then + VERSION="dev" +fi + +if [ -z "$SERVERLESS_INIT" ]; then + echo "Building Datadog Lambda Extension" + CMD_PATH="cmd/serverless" +else + echo "Building Serverless Init" + CMD_PATH="cmd/serverless-init" +fi + + +if [ -z "$ALPINE" ]; then + BUILD_FILE=Dockerfile.build +else + echo "Building for alpine" + BUILD_FILE=Dockerfile.alpine.build + BUILD_SUFFIX="-alpine" +fi + +# Allow to override the build tags +if [ -z "$BUILD_TAGS" ]; then + BUILD_TAGS="serverless otlp" +fi + +ROOT_DIR=$(pwd) + +EXTENSION_DIR=".layers" +TARGET_DIR=$ROOT_DIR/$EXTENSION_DIR + +# Make sure the folder does not exist +rm -rf $EXTENSION_DIR 2>/dev/null + +mkdir -p $EXTENSION_DIR + +function docker_build { + arch=$1 + file=$2 + + docker buildx build --platform linux/${arch} \ + -t datadog/build-go-agent-${arch}:${VERSION} \ + -f ${ROOT_DIR}/scripts/${file} \ + --build-arg EXTENSION_VERSION="${VERSION}" \ + --build-arg AGENT_VERSION="${AGENT_VERSION}" \ + --build-arg CMD_PATH="${CMD_PATH}" \ + --build-arg BUILD_TAGS="${BUILD_TAGS}" \ + . -o $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} + + cp $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}.zip + rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} +} + +docker_build $ARCHITECTURE $BUILD_FILE + diff --git a/.gitlab/template.yaml.tpl b/.gitlab/template.yaml.tpl deleted file mode 100644 index 221bab080..000000000 --- a/.gitlab/template.yaml.tpl +++ /dev/null @@ -1,65 +0,0 @@ -stages: - - build - - test - - sign - - publish - -default: - retry: - max: 1 - when: - - runner_system_failure - -variables: - DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension - DOCKER_TARGET_VERSION: latest - -{{ range $architecture := (ds "architectures").architectures }} - -build layer (bottlecap) ({{ $architecture.name }}): - stage: build - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_bottlecap-{{ $architecture.name }}.zip - script: - - ARCHITECTURE={{ $architecture.name }} ./scripts/build_bottlecap_layer.sh - -check layer size (bottlecap) ({{ $architecture.name }}): - stage: test - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - needs: - - build layer (bottlecap) ({{ $architecture.name }}) - dependencies: - - build layer (bottlecap) ({{ $architecture.name }}) - script: - - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/check_layer_size.sh - -fmt ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo fmt - -check ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo check - -clippy ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo clippy --all-features - -{{- end }} # architectures end \ No newline at end of file diff --git a/.gitlab/templates/bottlecap.yaml.tpl b/.gitlab/templates/bottlecap.yaml.tpl new file mode 100644 index 000000000..e6916e724 --- /dev/null +++ b/.gitlab/templates/bottlecap.yaml.tpl @@ -0,0 +1,130 @@ +stages: + - build + - test + - sign + - publish + +default: + retry: + max: 1 + when: + - runner_system_failure + +variables: + DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension + DOCKER_TARGET_VERSION: latest + +{{ range $architecture := (ds "architectures").architectures }} + +build layer ({{ $architecture.name }}): + stage: build + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + artifacts: + expire_in: 1 hr + paths: + - .layers/datadog_bottlecap-{{ $architecture.name }}.zip + script: + - ARCHITECTURE={{ $architecture.name }} ./scripts/build_bottlecap_layer.sh + +check layer size ({{ $architecture.name }}): + stage: test + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: + - build layer ({{ $architecture.name }}) + dependencies: + - build layer ({{ $architecture.name }}) + script: + - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/check_layer_size.sh + +fmt ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo fmt + +check ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo check + +clippy ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo clippy --all-features + +{{ range $environment := (ds "environments").environments }} + +{{ if or (eq $environment.name "prod") }} +sign layer ({{ $architecture.name }}): + stage: sign + tags: ["arch:amd64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build layer ({{ $architecture.name }}) + - check layer size ({{ $architecture.name }}) + - fmt ({{ $architecture.name }}) + - check ({{ $architecture.name }}) + - clippy ({{ $architecture.name }}) + dependencies: + - build layer ({{ $architecture.name }}) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_bottlecap-{{ $architecture.name }}.zip + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh + script: + - LAYER_FILE=datadog_bottlecap-{{ $architecture.name}}.zip .gitlab/scripts/sign_layers.sh {{ $environment.name }} +{{ end }} + +publish layer {{ $environment.name }} ({{ $architecture.name }}): + stage: publish + tags: ["arch:amd64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + rules: + - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: +{{ if or (eq $environment.name "prod") }} + - sign layer ({{ $architecture.name }}) +{{ else }} + - build layer ({{ $architecture.name }}) + - check layer size ({{ $architecture.name }}) + - fmt ({{ $architecture.name }}) + - check ({{ $architecture.name }}) + - clippy ({{ $architecture.name }}) +{{ end }} + dependencies: +{{ if or (eq $environment.name "prod") }} + - sign layer ({{ $architecture.name }}) +{{ else }} + - build layer ({{ $architecture.name }}) +{{ end }} + parallel: + matrix: + - REGION: {{ range (ds "regions").regions }} + - {{ .code }} + {{- end}} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh + script: + - STAGE={{ $environment.name }} ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/publish_layers.sh + +{{- end }} # environments end + +{{- end }} # architectures end diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl new file mode 100644 index 000000000..8e67e27ab --- /dev/null +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -0,0 +1,32 @@ +stages: + - build + - test + - sign + - publish + +default: + retry: + max: 1 + when: + - runner_system_failure + +variables: + DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension + DOCKER_TARGET_VERSION: latest + GIT_DEPTH: 1 + +{{ range $architecture := (ds "architectures").architectures }} + +build layer ({{ $architecture.name }}): + stage: build + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + artifacts: + expire_in: 1 hr + paths: + - .layers/datadog_extension-{{ $architecture.name }}.zip + script: + - git clone --branch ${AGENT_BRANCH} --depth=1 https://github.com/DataDog/datadog-agent.git + - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/build_layer.sh + +{{- end }} # architectures end \ No newline at end of file From 46da8b7f906a8f2bfabf4d2b6f9a502ca4e88b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:23:20 -0400 Subject: [PATCH 34/74] pass any path starting with `pipeline-` --- .gitlab-ci.yml | 2 +- .gitlab/scripts/check_layer_size.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2e600b40..7e35d0183 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ generator: - gomplate --config .gitlab/config.yaml artifacts: paths: - - .gitlab/pipeline-bottlecap.yaml + - .gitlab/pipeline-* bottlecap: stage: build diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh index dccc4a569..3e66e3a7a 100755 --- a/.gitlab/scripts/check_layer_size.sh +++ b/.gitlab/scripts/check_layer_size.sh @@ -16,7 +16,7 @@ if [ -z "$ARCHITECTURE" ]; then fi MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 16 \* 1024) # 16 MB, amd64 is 16, while arm64 is 15 -MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 43 \* 1024) # 43 MB +MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 44 \* 1024) # 44 MB, amd is 44, while arm64 is 43 LAYER_FILES_PREFIX="datadog_bottlecap" From 130974b32880c23cb05e577b11fdd8f1a7f13cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:26:29 -0400 Subject: [PATCH 35/74] specify paths for pipelines --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e35d0183..f68b18d18 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,8 @@ generator: - gomplate --config .gitlab/config.yaml artifacts: paths: - - .gitlab/pipeline-* + - .gitlab/pipeline-bottlecap.yaml + - .gitlab/pipeline-go-agent.yaml bottlecap: stage: build From f08d645d56509f2d7ef82d9385f6dcb3ed848728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:27:00 -0400 Subject: [PATCH 36/74] set to include artifact for `go-agent` --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f68b18d18..c74f3e2e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,8 @@ go-agent: stage: build trigger: include: - - local: .gitlab/pipeline-go-agent.yaml + - artifact: .gitlab/pipeline-bottlecap.yaml + job: generator strategy: depend rules: - if: $CI_PIPELINE_SOURCE == "web" From d3071b51cc50f1404f1892ee965b9e44ece3db7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:30:33 -0400 Subject: [PATCH 37/74] typo on pipeline to use --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c74f3e2e5..a8df9b534 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ go-agent: stage: build trigger: include: - - artifact: .gitlab/pipeline-bottlecap.yaml + - artifact: .gitlab/pipeline-go-agent.yaml job: generator strategy: depend rules: From ba7aef398f66a833483c862bd18cba9c617faf23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:45:36 -0400 Subject: [PATCH 38/74] try different cloning strategy --- .gitlab/templates/go-agent.yaml.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index 8e67e27ab..1f58aa89e 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -26,7 +26,8 @@ build layer ({{ $architecture.name }}): paths: - .layers/datadog_extension-{{ $architecture.name }}.zip script: - - git clone --branch ${AGENT_BRANCH} --depth=1 https://github.com/DataDog/datadog-agent.git + - echo $AGENT_BRANCH + - git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/build_layer.sh {{- end }} # architectures end \ No newline at end of file From 761e3210b1b5cebf0088c8f62e9bcfab4f1e3084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:49:07 -0400 Subject: [PATCH 39/74] typo on scrip to use to build --- .gitlab/templates/go-agent.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index 1f58aa89e..2b5beb868 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -28,6 +28,6 @@ build layer ({{ $architecture.name }}): script: - echo $AGENT_BRANCH - git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git - - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/build_layer.sh + - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/build_go_agent.sh {{- end }} # architectures end \ No newline at end of file From c936caa369a294ffaf3410f8b65db7087edbe418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:08:54 -0400 Subject: [PATCH 40/74] fix script to copy stuff from `datadog-agent` --- .gitlab/scripts/build_go_agent.sh | 23 ++++++++++++++++++++--- .gitlab/templates/go-agent.yaml.tpl | 3 +-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.gitlab/scripts/build_go_agent.sh b/.gitlab/scripts/build_go_agent.sh index fc9434a4e..36f985245 100755 --- a/.gitlab/scripts/build_go_agent.sh +++ b/.gitlab/scripts/build_go_agent.sh @@ -38,21 +38,38 @@ else BUILD_SUFFIX="-alpine" fi -# Allow to override the build tags +# Allow override build tags if [ -z "$BUILD_TAGS" ]; then BUILD_TAGS="serverless otlp" fi -ROOT_DIR=$(pwd) +# Allow override agent path +if [ -z "$AGENT_PATH" ]; then + AGENT_PATH="../datadog-agent" +fi + +MAIN_DIR=$(pwd) # datadog-lambda-extension EXTENSION_DIR=".layers" -TARGET_DIR=$ROOT_DIR/$EXTENSION_DIR +TARGET_DIR=$MAIN_DIR/$EXTENSION_DIR # Make sure the folder does not exist rm -rf $EXTENSION_DIR 2>/dev/null mkdir -p $EXTENSION_DIR +# Prepare folder with only *mod and *sum files to enable Docker caching capabilities +mkdir -p $MAIN_DIR/scripts/.src $MAIN_DIR/scripts/.cache +echo "Copy mod files to build a cache" +cp $AGENT_PATH/go.mod $MAIN_DIR/scripts/.cache +cp $AGENT_PATH/go.sum $MAIN_DIR/scripts/.cache + +# Compress all files to speed up docker copy +touch $MAIN_DIR/scripts/.src/datadog-agent.tgz +cd $AGENT_PATH/.. +tar --exclude=.git -czf $MAIN_DIR/scripts/.src/datadog-agent.tgz datadog-agent +cd $MAIN_DIR + function docker_build { arch=$1 file=$2 diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index 2b5beb868..c04cb5737 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -26,8 +26,7 @@ build layer ({{ $architecture.name }}): paths: - .layers/datadog_extension-{{ $architecture.name }}.zip script: - - echo $AGENT_BRANCH - - git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git + - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/build_go_agent.sh {{- end }} # architectures end \ No newline at end of file From 91950ad1b8829c548d0d19b9bc375af3d161b353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:38:17 -0400 Subject: [PATCH 41/74] typo --- .gitlab/scripts/build_go_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/build_go_agent.sh b/.gitlab/scripts/build_go_agent.sh index 36f985245..ee9ddcba8 100755 --- a/.gitlab/scripts/build_go_agent.sh +++ b/.gitlab/scripts/build_go_agent.sh @@ -76,7 +76,7 @@ function docker_build { docker buildx build --platform linux/${arch} \ -t datadog/build-go-agent-${arch}:${VERSION} \ - -f ${ROOT_DIR}/scripts/${file} \ + -f ${MAIN_DIR}/scripts/${file} \ --build-arg EXTENSION_VERSION="${VERSION}" \ --build-arg AGENT_VERSION="${AGENT_VERSION}" \ --build-arg CMD_PATH="${CMD_PATH}" \ From bbda3eebd0329900b28d960b4b12c77b5535ae03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:32:09 -0400 Subject: [PATCH 42/74] update where the get secrets script comes from --- .gitlab/templates/bottlecap.yaml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/templates/bottlecap.yaml.tpl b/.gitlab/templates/bottlecap.yaml.tpl index e6916e724..851794c67 100644 --- a/.gitlab/templates/bottlecap.yaml.tpl +++ b/.gitlab/templates/bottlecap.yaml.tpl @@ -85,7 +85,7 @@ sign layer ({{ $architecture.name }}): paths: - .layers/datadog_bottlecap-{{ $architecture.name }}.zip before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - LAYER_FILE=datadog_bottlecap-{{ $architecture.name}}.zip .gitlab/scripts/sign_layers.sh {{ $environment.name }} {{ end }} @@ -121,7 +121,7 @@ publish layer {{ $environment.name }} ({{ $architecture.name }}): - {{ .code }} {{- end}} before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - STAGE={{ $environment.name }} ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/publish_layers.sh From 341f0f2ea7fb50b1d6730c514d8a3074f090de94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:32:23 -0400 Subject: [PATCH 43/74] add `sign_layers.sh` --- .gitlab/scripts/sign_layers.sh | 121 +++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100755 .gitlab/scripts/sign_layers.sh diff --git a/.gitlab/scripts/sign_layers.sh b/.gitlab/scripts/sign_layers.sh new file mode 100755 index 000000000..717369366 --- /dev/null +++ b/.gitlab/scripts/sign_layers.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. +# +# Usage +# +# LAYER_FILE= ./scripts/sign_layers.sh +# +# LAYER_FILE=datadog_extension-amd64.zip ./scripts/sign_layers.sh sandbox + +set -e + +LAYER_DIR=".layers" +SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile" +VALID_ACCOUNTS=("sandbox" "prod") + +if [ -z "$LAYER_FILE" ]; then + echo "[ERROR]: $LAYER_FILE not specified." + exit 1 +fi + +# Check account parameter +if [ -z "$1" ]; then + echo "[ERROR]: Account parameter not specified." + exit 1 +fi + +if [[ ! "${VALID_ACCOUNTS[@]}" =~ $1 ]]; then + echo "[ERROR]: Account parameter is invalid. Not in `sandbox` or `prod`." + exit 1 +fi + +if [ "$1" = "sandbox" ]; then + REGION="sa-east-1" + S3_BUCKET_NAME="dd-lambda-signing-bucket-sandbox" +fi +if [ "$1" = "prod" ]; then + REGION="us-east-1" + S3_BUCKET_NAME="dd-lambda-signing-bucket" +fi + +echo "---" +echo "Signing layer for $LAYER_FILE" + +LAYER_LOCAL_PATH="${LAYER_DIR}/${LAYER_FILE}" + +# Upload the layer to S3 for signing +echo "---" +echo "Uploading layer to S3 for signing..." + +UUID=$(uuidgen) +S3_UNSIGNED_ZIP_KEY="${UUID}.zip" +S3_UNSIGNED_ZIP_URI="s3://${S3_BUCKET_NAME}/${S3_UNSIGNED_ZIP_KEY}" + +aws s3 cp $LAYER_LOCAL_PATH $S3_UNSIGNED_ZIP_URI + +# Start a signing job +echo "---" +echo "Starting the signing job..." +SIGNING_JOB_ID=$(aws signer start-signing-job \ + --source "s3={bucketName=${S3_BUCKET_NAME},key=${S3_UNSIGNED_ZIP_KEY},version=null}" \ + --destination "s3={bucketName=${S3_BUCKET_NAME}}" \ + --profile-name $SIGNING_PROFILE_NAME \ + --region $REGION \ + | jq -r '.jobId'\ +) + +# Wait for the signing job to complete +echo "---" +echo "Waiting for the signing job to complete..." +SECONDS_WAITED_SO_FAR=0 +while : +do + sleep 3 + SECONDS_WAITED_SO_FAR=$((SECONDS_WAITED_SO_FAR + 3)) + + SIGNING_JOB_DESCRIPTION=$(aws signer describe-signing-job \ + --job-id $SIGNING_JOB_ID \ + --region $REGION\ + ) + SIGNING_JOB_STATUS=$(echo $SIGNING_JOB_DESCRIPTION | jq -r '.status') + SIGNING_JOB_STATUS_REASON=$(echo $SIGNING_JOB_DESCRIPTION | jq -r '.statusReason') + + echo "---" + if [ $SIGNING_JOB_STATUS = "Succeeded" ]; then + echo "Signing job succeeded!" + break + fi + + if [ $SIGNING_JOB_STATUS = "Failed" ]; then + echo "[ERROR]: Signing job failed" + echo $SIGNING_JOB_STATUS_REASON + exit 1 + fi + + if [ $SECONDS_WAITED_SO_FAR -ge 60 ]; then + echo "[ERROR]: Timed out waiting for the signing job to complete" + exit 1 + fi + + echo "Signing job still in progress..." +done + +# Download the signed ZIP, overwriting the original ZIP +echo "---" +echo "Replacing the local layer with the signed layer from S3..." +S3_SIGNED_ZIP_KEY="${SIGNING_JOB_ID}.zip" +S3_SIGNED_ZIP_URI="s3://${S3_BUCKET_NAME}/${S3_SIGNED_ZIP_KEY}" +aws s3 cp $S3_SIGNED_ZIP_URI $LAYER_LOCAL_PATH + +# Delete the signed and unsigned ZIPs in S3 +echo "Cleaning up the S3 bucket..." +aws s3api delete-object --bucket $S3_BUCKET_NAME --key $S3_UNSIGNED_ZIP_KEY +aws s3api delete-object --bucket $S3_BUCKET_NAME --key $S3_SIGNED_ZIP_KEY + + +echo "---" +echo "Successfully signed layer ${LAYER_FILE}!" From 893657af950567929d1520e2b7c60b072c2cb4ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:53:29 -0400 Subject: [PATCH 44/74] allow signing layers for both environments --- .gitlab/scripts/sign_layers.sh | 2 +- .gitlab/templates/go-agent.yaml.tpl | 59 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/.gitlab/scripts/sign_layers.sh b/.gitlab/scripts/sign_layers.sh index 717369366..7792fbc72 100755 --- a/.gitlab/scripts/sign_layers.sh +++ b/.gitlab/scripts/sign_layers.sh @@ -35,7 +35,7 @@ fi if [ "$1" = "sandbox" ]; then REGION="sa-east-1" - S3_BUCKET_NAME="dd-lambda-signing-bucket-sandbox" + S3_BUCKET_NAME="dd-lambda-signing-bucket-serverless-sandbox" fi if [ "$1" = "prod" ]; then REGION="us-east-1" diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index c04cb5737..ad4be538a 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -29,4 +29,63 @@ build layer ({{ $architecture.name }}): - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/build_go_agent.sh +check layer size ({{ $architecture.name }}): + stage: test + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: + - build layer ({{ $architecture.name }}) + dependencies: + - build layer ({{ $architecture.name }}) + script: + - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/check_layer_size.sh + +{{ range $environment := (ds "environments").environments }} + +sign layer {{ $environment.name }} ({{ $architecture.name }}): + stage: sign + tags: ["arch:amd64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + # rules: + # - if: '$CI_COMMIT_TAG =~ /^v.*/' + # when: manual + needs: + - build layer ({{ $architecture.name }}) + - check layer size ({{ $architecture.name }}) + dependencies: + - build layer ({{ $architecture.name }}) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 hr # Signed layers should expire after 1 day TODO: modify to 1 day + paths: + - .layers/datadog_extension-{{ $architecture.name }}.zip + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - LAYER_FILE=datadog_extension-{{ $architecture.name}}.zip .gitlab/scripts/sign_layers.sh {{ $environment.name }} + +publish layer {{ $environment.name }} ({{ $architecture.name }}): + stage: publish + tags: ["arch:amd64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + rules: + - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + - sign layer {{ $environment.name }} ({{ $architecture.name }}) + dependencies: + - sign layer {{ $environment.name }} ({{ $architecture.name }}) + parallel: + matrix: + - REGION: {{ range (ds "regions").regions }} + - {{ .code }} + {{- end}} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - STAGE={{ $environment.name }} ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/publish_layers.sh + +{{- end }} # environments + {{- end }} # architectures end \ No newline at end of file From 8a191e60afe1bebe2c49fc0bf90bd9ade09cc284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 9 Jul 2024 13:09:57 -0400 Subject: [PATCH 45/74] abstract scripts to use `LAYER_FILE` for size check and signing --- .gitlab/scripts/check_layer_size.sh | 11 ++++------- .gitlab/templates/bottlecap.yaml.tpl | 12 +++++++++--- .gitlab/templates/go-agent.yaml.tpl | 8 ++++++-- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh index 3e66e3a7a..bca398d55 100755 --- a/.gitlab/scripts/check_layer_size.sh +++ b/.gitlab/scripts/check_layer_size.sh @@ -9,20 +9,17 @@ set -e -if [ -z "$ARCHITECTURE" ]; then - echo "ARCHITECTURE not specified" - echo "" +if [ -z "$LAYER_FILE" ]; then + echo "[ERROR]: LAYER_FILE not specified" exit 1 fi MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 16 \* 1024) # 16 MB, amd64 is 16, while arm64 is 15 MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 44 \* 1024) # 44 MB, amd is 44, while arm64 is 43 +LAYERS_DIR=".layers" -LAYER_FILES_PREFIX="datadog_bottlecap" -LAYER_DIR=".layers" - -FILE=$LAYER_DIR/${LAYER_FILES_PREFIX}-${ARCHITECTURE}.zip +FILE=$LAYERS_DIR/$LAYER_FILE FILE_SIZE=$(stat --printf="%s" $FILE) FILE_SIZE_KB="$(( ${FILE_SIZE%% *} / 1024))" echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb" diff --git a/.gitlab/templates/bottlecap.yaml.tpl b/.gitlab/templates/bottlecap.yaml.tpl index 851794c67..b78ad886c 100644 --- a/.gitlab/templates/bottlecap.yaml.tpl +++ b/.gitlab/templates/bottlecap.yaml.tpl @@ -24,8 +24,10 @@ build layer ({{ $architecture.name }}): expire_in: 1 hr paths: - .layers/datadog_bottlecap-{{ $architecture.name }}.zip + variables: + ARCHITECTURE: {{ $architecture.name }} script: - - ARCHITECTURE={{ $architecture.name }} ./scripts/build_bottlecap_layer.sh + - ./scripts/build_bottlecap_layer.sh check layer size ({{ $architecture.name }}): stage: test @@ -35,8 +37,10 @@ check layer size ({{ $architecture.name }}): - build layer ({{ $architecture.name }}) dependencies: - build layer ({{ $architecture.name }}) + variables: + LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip script: - - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/check_layer_size.sh + - .gitlab/scripts/check_layer_size.sh fmt ({{ $architecture.name }}): stage: test @@ -84,10 +88,12 @@ sign layer ({{ $architecture.name }}): expire_in: 1 day # Signed layers should expire after 1 day paths: - .layers/datadog_bottlecap-{{ $architecture.name }}.zip + variables: + LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - - LAYER_FILE=datadog_bottlecap-{{ $architecture.name}}.zip .gitlab/scripts/sign_layers.sh {{ $environment.name }} + - .gitlab/scripts/sign_layers.sh {{ $environment.name }} {{ end }} publish layer {{ $environment.name }} ({{ $architecture.name }}): diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index ad4be538a..f1e617d94 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -37,8 +37,10 @@ check layer size ({{ $architecture.name }}): - build layer ({{ $architecture.name }}) dependencies: - build layer ({{ $architecture.name }}) + variables: + LAYER_FILE: datadog_extension-{{ $architecture.name }}.zip script: - - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/check_layer_size.sh + - .gitlab/scripts/check_layer_size.sh {{ range $environment := (ds "environments").environments }} @@ -58,10 +60,12 @@ sign layer {{ $environment.name }} ({{ $architecture.name }}): expire_in: 1 hr # Signed layers should expire after 1 day TODO: modify to 1 day paths: - .layers/datadog_extension-{{ $architecture.name }}.zip + variables: + LAYER_FILE: datadog_extension-{{ $architecture.name }}.zip before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - - LAYER_FILE=datadog_extension-{{ $architecture.name}}.zip .gitlab/scripts/sign_layers.sh {{ $environment.name }} + - .gitlab/scripts/sign_layers.sh {{ $environment.name }} publish layer {{ $environment.name }} ({{ $architecture.name }}): stage: publish From 2a484265887a5d5f5699c87cce714077b7c5f266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:00:44 -0400 Subject: [PATCH 46/74] add `awscli` to `Dockerfile` --- .gitlab-ci.yml | 10 +++++----- .gitlab/Dockerfile | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8df9b534..6cf846bd3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,11 +17,11 @@ ci image: stage: build image: registry.ddbuild.io/images/docker:20.10 tags: ["arch:arm64"] - rules: - - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' - changes: - - .gitlab/Dockerfile - when: on_success + # rules: + # - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' + # changes: + # - .gitlab/Dockerfile + # when: on_success variables: DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} script: diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index bc1a8cf1f..d33cb7c69 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -1,8 +1,12 @@ FROM registry.ddbuild.io/images/docker:24.0.5 -RUN apt-get update && apt-get install -y --fix-missing \ +RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \ curl gcc gnupg g++ make cmake unzip openssl g++ uuid-runtime +# Install AWS CLI +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +RUN unzip awscliv2.zip && ./aws/install + # Install Protocol Buffers compiler by hand COPY ./scripts/install-protoc.sh / RUN chmod +x /install-protoc.sh && /install-protoc.sh From 915ec2b56737fa6471aa2eb20e5e01ab9a15d907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:24:10 -0400 Subject: [PATCH 47/74] update templates to use publishing script properly --- .gitlab/scripts/publish_layers.sh | 140 +++++++++++++++++++++++++++ .gitlab/templates/bottlecap.yaml.tpl | 6 +- .gitlab/templates/go-agent.yaml.tpl | 8 +- 3 files changed, 151 insertions(+), 3 deletions(-) create mode 100755 .gitlab/scripts/publish_layers.sh diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh new file mode 100755 index 000000000..326e9f073 --- /dev/null +++ b/.gitlab/scripts/publish_layers.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +set -e + +LAYER_DIR=".layers" +VALID_ACCOUNTS=("sandbox" "prod") + +publish_layer() { + region=$1 + layer=$2 + file=$3 + compatible_architectures=$4 + + version_nbr=$(aws lambda publish-layer-version --layer-name $layer \ + --description "Datadog Lambda Extension" \ + --compatible-architectures $compatible_architectures \ + --zip-file "fileb://${file}" \ + --region $region \ + | jq -r '.Version' + ) + + permission=$(aws lambda add-layer-version-permission --layer-name $layer \ + --version-number $version_nbr \ + --statement-id "release-$version_nbr" \ + --action lambda:GetLayerVersion \ + --principal "*" \ + --region $region + ) + + echo $version_nbr +} + + +if [ -z "$ARCHITECTURE "]; then + printf "[ERROR]: ARCHITECTURE not specified." + exit 1 +fi + + +if [ -z "$LAYER_FILE" ]; then + printf "[ERROR]: LAYER_FILE not specified." + exit 1 +fi + +LAYER_PATH="${LAYER_DIR}/${LAYER_FILE}" +# Check that the layer files exist +if [ ! -f $LAYER_PATH ]; then + printf "[ERROR]: Could not find ${LAYER_PATH}." + exit 1 +fi + +if [ "$ARCHITECTURE" == "amd64" ]; then + LAYER_NAME="Datadog-Extension" +else + LAYER_NAME="Datadog-Extension-ARM" +fi + +if [ -z "$LAYER_NAME" ]; then + printf "[ERROR]: LAYER_NAME not specified." + exit 1 +fi + +AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') + +if [ -z "$REGION" ]; then + printf "[ERROR]: REGION not specified." + exit 1 +else + echo "Region specified: $REGION" + if [[ ! "$AVAILABLE_REGIONS" == *"$REGION"* ]]; then + printf "Could not find $REGION in available regions: $AVAILABLE_REGIONS" + exit 1 + fi +fi + +if [ -z "$STAGE" ]; then + printf "[ERROR]: STAGE not specified.\n" + exit 1 +fi + +if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then + # Deploy latest version + latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`') + VERSION=$(($latest_version + 1)) +else + # Running on prod + if [ -z "$CI_COMMIT_TAG" ]; then + printf "[Error] No CI_COMMIT_TAG found.\n" + printf "Exiting script...\n" + exit 1 + else + printf "Tag found in environment: $CI_COMMIT_TAG\n" + fi + + VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) +fi + +if [ -z "$VERSION" ]; then + printf "[ERROR]: Layer VERSION not specified" + exit 1 +else + echo "Layer version parsed: $VERSION" +fi + +printf "[$REGION] Starting publishing layers...\n" + +# Compatible Architectures +if [ "$ARCHITECTURE" == "amd64" ]; then + architectures="x86_64" +else + architectures="arm64" +fi + +latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`') +if [ $latest_version -ge $VERSION ]; then + printf "[$REGION] Layer $layer version $VERSION already exists in region $REGION, skipping...\n" + exit 1 +elif [ $latest_version -lt $((VERSION-1)) ]; then + printf "[$REGION][WARNING] The latest version of layer $layer in region $REGION is $latest_version, this will publish all the missing versions including $VERSION\n" +fi + +while [ $latest_version -lt $VERSION ]; do + latest_version=$(publish_layer $REGION $LAYER_NAME $LAYER_PATH $architectures) + printf "[$REGION] Published version $latest_version for layer $LAYER_NAME in region $REGION\n" + + # This shouldn't happen unless someone manually deleted the latest version, say 28, and + # then tries to republish 28 again. The published version would actually be 29, because + # Lambda layers are immutable and AWS will skip deleted version and use the next number. + if [ $latest_version -gt $VERSION ]; then + printf "[$REGION] Published version $latest_version is greater than the desired version $VERSION!" + exit 1 + fi +done + +printf "[$REGION] Finished publishing layers...\n" diff --git a/.gitlab/templates/bottlecap.yaml.tpl b/.gitlab/templates/bottlecap.yaml.tpl index b78ad886c..69ebcccd6 100644 --- a/.gitlab/templates/bottlecap.yaml.tpl +++ b/.gitlab/templates/bottlecap.yaml.tpl @@ -126,10 +126,14 @@ publish layer {{ $environment.name }} ({{ $architecture.name }}): - REGION: {{ range (ds "regions").regions }} - {{ .code }} {{- end}} + variables: + ARCHITECTURE: {{ $architecture.name }} + LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip + STAGE: {{ $environment.name }} before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - - STAGE={{ $environment.name }} ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/publish_layers.sh + - .gitlab/scripts/publish_layers.sh {{- end }} # environments end diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index f1e617d94..a89049176 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -85,11 +85,15 @@ publish layer {{ $environment.name }} ({{ $architecture.name }}): - REGION: {{ range (ds "regions").regions }} - {{ .code }} {{- end}} + variables: + ARCHITECTURE: {{ $architecture.name }} + LAYER_FILE: datadog_extension-{{ $architecture.name }}.zip + STAGE: {{ $environment.name }} before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - - STAGE={{ $environment.name }} ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/publish_layers.sh + - .gitlab/scripts/publish_layers.sh -{{- end }} # environments +{{- end }} # environments end {{- end }} # architectures end \ No newline at end of file From 46cba36b552d226f31f95f1142e2e33ed5ca427a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:25:01 -0400 Subject: [PATCH 48/74] revert image to be built always --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6cf846bd3..a8df9b534 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,11 +17,11 @@ ci image: stage: build image: registry.ddbuild.io/images/docker:20.10 tags: ["arch:arm64"] - # rules: - # - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' - # changes: - # - .gitlab/Dockerfile - # when: on_success + rules: + - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"' + changes: + - .gitlab/Dockerfile + when: on_success variables: DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} script: From 9c29bb81dcf991001c16afa0c1765fa4e83ca082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 00:06:51 -0400 Subject: [PATCH 49/74] update go template for signing only in prod --- .gitlab/scripts/publish_layers.sh | 2 +- .gitlab/templates/go-agent.yaml.tpl | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index 326e9f073..5f12ef4bd 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -36,7 +36,7 @@ publish_layer() { } -if [ -z "$ARCHITECTURE "]; then +if [ -z "$ARCHITECTURE" ]; then printf "[ERROR]: ARCHITECTURE not specified." exit 1 fi diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index a89049176..39cca46c6 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -44,13 +44,14 @@ check layer size ({{ $architecture.name }}): {{ range $environment := (ds "environments").environments }} -sign layer {{ $environment.name }} ({{ $architecture.name }}): +{{ if or (eq $environment.name "prod") }} +sign layer ({{ $architecture.name }}): stage: sign tags: ["arch:amd64"] image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - # rules: - # - if: '$CI_COMMIT_TAG =~ /^v.*/' - # when: manual + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual needs: - build layer ({{ $architecture.name }}) - check layer size ({{ $architecture.name }}) @@ -66,6 +67,7 @@ sign layer {{ $environment.name }} ({{ $architecture.name }}): - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - .gitlab/scripts/sign_layers.sh {{ $environment.name }} +{{ end }} publish layer {{ $environment.name }} ({{ $architecture.name }}): stage: publish @@ -77,9 +79,17 @@ publish layer {{ $environment.name }} ({{ $architecture.name }}): allow_failure: true - if: '$CI_COMMIT_TAG =~ /^v.*/' needs: - - sign layer {{ $environment.name }} ({{ $architecture.name }}) +{{ if or (eq $environment.name "prod") }} + - sign layer ({{ $architecture.name }}) +{{ else }} + - build layer ({{ $architecture.name }}) +{{ end }} dependencies: - - sign layer {{ $environment.name }} ({{ $architecture.name }}) +{{ if or (eq $environment.name "prod") }} + - sign layer ({{ $architecture.name }}) +{{ else }} + - build layer ({{ $architecture.name }}) +{{ end }} parallel: matrix: - REGION: {{ range (ds "regions").regions }} @@ -96,4 +106,4 @@ publish layer {{ $environment.name }} ({{ $architecture.name }}): {{- end }} # environments end -{{- end }} # architectures end \ No newline at end of file +{{- end }} # architectures end From 1d724f868266ef2ec4ee2e9d337ee941af7e071c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 00:12:45 -0400 Subject: [PATCH 50/74] allow suffix to be used when publishing --- .gitlab/scripts/publish_layers.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index 5f12ef4bd..1f6cbca00 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -116,6 +116,13 @@ else architectures="arm64" fi +if [ -z "$LAYER_SUFFIX" ]; then + printf "[$REGION] Deploying layers without suffix\n" +else + printf "[$REGION] Deploying layers with specified suffix: ${LAYER_SUFFIX}\n" + LAYER_NAME="${LAYER_NAME}-${LAYER_SUFFIX}" +fi + latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`') if [ $latest_version -ge $VERSION ]; then printf "[$REGION] Layer $layer version $VERSION already exists in region $REGION, skipping...\n" From a2188a8948097cd35cf02b3f16ffa599070dcacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 00:22:35 -0400 Subject: [PATCH 51/74] move when we add suffix we were checking the wrong version number for sandbox --- .gitlab/scripts/publish_layers.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index 1f6cbca00..90d1802cd 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -83,6 +83,15 @@ if [ -z "$STAGE" ]; then exit 1 fi +printf "[$REGION] Starting publishing layers...\n" + +if [ -z "$LAYER_SUFFIX" ]; then + printf "[$REGION] Deploying layers without suffix\n" +else + printf "[$REGION] Deploying layers with specified suffix: ${LAYER_SUFFIX}\n" + LAYER_NAME="${LAYER_NAME}-${LAYER_SUFFIX}" +fi + if [[ "$STAGE" =~ ^(staging|sandbox)$ ]]; then # Deploy latest version latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`') @@ -107,8 +116,6 @@ else echo "Layer version parsed: $VERSION" fi -printf "[$REGION] Starting publishing layers...\n" - # Compatible Architectures if [ "$ARCHITECTURE" == "amd64" ]; then architectures="x86_64" @@ -116,13 +123,6 @@ else architectures="arm64" fi -if [ -z "$LAYER_SUFFIX" ]; then - printf "[$REGION] Deploying layers without suffix\n" -else - printf "[$REGION] Deploying layers with specified suffix: ${LAYER_SUFFIX}\n" - LAYER_NAME="${LAYER_NAME}-${LAYER_SUFFIX}" -fi - latest_version=$(aws lambda list-layer-versions --region $REGION --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`') if [ $latest_version -ge $VERSION ]; then printf "[$REGION] Layer $layer version $VERSION already exists in region $REGION, skipping...\n" From ac3f5c164c38e5cdec28e1a9ba7edddc3dc25701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 00:24:09 -0400 Subject: [PATCH 52/74] eol --- .gitlab-ci.yml | 2 +- .gitlab/datasources/architectures.yaml | 2 +- .gitlab/scripts/check_layer_size.sh | 2 +- .gitlab/scripts/get_secrets.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8df9b534..83bdb67e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,4 +65,4 @@ go-agent: - if: $CI_PIPELINE_SOURCE == "web" variables: AGENT_BRANCH: $AGENT_BRANCH - LAYER_SUFFIX: $LAYER_SUFFIX \ No newline at end of file + LAYER_SUFFIX: $LAYER_SUFFIX diff --git a/.gitlab/datasources/architectures.yaml b/.gitlab/datasources/architectures.yaml index b56abbf75..4b9b1a60b 100644 --- a/.gitlab/datasources/architectures.yaml +++ b/.gitlab/datasources/architectures.yaml @@ -1,3 +1,3 @@ architectures: - name: amd64 - - name: arm64 \ No newline at end of file + - name: arm64 diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh index bca398d55..f3af3d586 100755 --- a/.gitlab/scripts/check_layer_size.sh +++ b/.gitlab/scripts/check_layer_size.sh @@ -36,4 +36,4 @@ echo "Layer file ${FILE} has unzipped size ${UNZIPPED_FILE_SIZE_KB} kb" if [ "$UNZIPPED_FILE_SIZE_KB" -gt "$MAX_LAYER_UNCOMPRESSED_SIZE_KB" ]; then echo "Unzipped size exceeded limit $MAX_LAYER_UNCOMPRESSED_SIZE_KB kb" exit 1 -fi \ No newline at end of file +fi diff --git a/.gitlab/scripts/get_secrets.sh b/.gitlab/scripts/get_secrets.sh index 6d5df92c2..dd019061a 100755 --- a/.gitlab/scripts/get_secrets.sh +++ b/.gitlab/scripts/get_secrets.sh @@ -45,4 +45,4 @@ export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN --role-session-name "ci.datadog-lambda-extension-$CI_JOB_ID-$CI_JOB_STAGE" \ --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ --external-id $EXTERNAL_ID \ - --output text)) \ No newline at end of file + --output text)) From 9d65a1b39a41c36f2237925ab510390e696bcec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:42:39 -0400 Subject: [PATCH 53/74] allow a new pipeline for combined go+rust extension --- .gitlab/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/config.yaml b/.gitlab/config.yaml index de680c281..ef3010403 100644 --- a/.gitlab/config.yaml +++ b/.gitlab/config.yaml @@ -3,10 +3,12 @@ inputFiles: - .gitlab/templates/bottlecap.yaml.tpl - .gitlab/templates/go-agent.yaml.tpl + - .gitlab/templates/lambda-extension.yaml.tpl outputFiles: - .gitlab/pipeline-bottlecap.yaml - .gitlab/pipeline-go-agent.yaml + - .gitlab/pipeline-lambda-extension.yaml.tpl datasources: architectures: From 34863fbc4b53822d07541b7841bb09e2f7a34d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:43:14 -0400 Subject: [PATCH 54/74] add pipeline in `.gitlab.yml` --- .gitlab-ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83bdb67e9..7663da329 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,8 +43,9 @@ generator: paths: - .gitlab/pipeline-bottlecap.yaml - .gitlab/pipeline-go-agent.yaml + - .gitlab/pipeline-lambda-extension.yaml -bottlecap: +bottlecap-only: stage: build trigger: include: @@ -54,7 +55,7 @@ bottlecap: rules: - when: on_success -go-agent: +go-agent-only: stage: build trigger: include: @@ -66,3 +67,14 @@ go-agent: variables: AGENT_BRANCH: $AGENT_BRANCH LAYER_SUFFIX: $LAYER_SUFFIX + +lambda-extension: + stage: build + trigger: + include: + - artifact: .gitlab/pipeline-lambda-extension.yaml + job: generator + strategy: depend + variables: + AGENT_BRANCH: $AGENT_BRANCH + LAYER_SUFFIX: $LAYER_SUFFIX From c349181f1d6da9efa758f335a10f0a0c938ec97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:43:57 -0400 Subject: [PATCH 55/74] update `go-agent.yaml.tpl` added arch as variable --- .gitlab/templates/go-agent.yaml.tpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index 39cca46c6..6fa15173e 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -25,9 +25,11 @@ build layer ({{ $architecture.name }}): expire_in: 1 hr paths: - .layers/datadog_extension-{{ $architecture.name }}.zip + variables: + ARCHITECTURE: {{ $architecture.name }} script: - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - - ARCHITECTURE={{ $architecture.name }} .gitlab/scripts/build_go_agent.sh + - .gitlab/scripts/build_go_agent.sh check layer size ({{ $architecture.name }}): stage: test From d69aacc9279b2f4f3c8e7776e396632065bc4341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:44:27 -0400 Subject: [PATCH 56/74] make bottlecap dev use another dockerfile --- scripts/Dockerfile.bottlecap.build | 15 ++++++++--- scripts/Dockerfile.bottlecap.dev | 41 ++++++++++++++++++++++++++++++ scripts/build_bottlecap_layer.sh | 2 +- 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 scripts/Dockerfile.bottlecap.dev diff --git a/scripts/Dockerfile.bottlecap.build b/scripts/Dockerfile.bottlecap.build index 78d7d9400..221d48431 100644 --- a/scripts/Dockerfile.bottlecap.build +++ b/scripts/Dockerfile.bottlecap.build @@ -2,14 +2,19 @@ FROM public.ecr.aws/lambda/provided:al2 as bottlecap-builder ARG PLATFORM -RUN yum install -y curl gcc gcc-c++ make unzip openssl openssl-devel +RUN yum install -y curl gcc gcc-c++ make unzip + # Install Protocol Buffers compiler by hand, since AL2 does not have a recent enough version. COPY ./scripts/install-protoc.sh / RUN chmod +x /install-protoc.sh && /install-protoc.sh + +# Install Rust Toolchain RUN curl https://sh.rustup.rs -sSf | \ sh -s -- --profile minimal --default-toolchain nightly-$PLATFORM-unknown-linux-gnu -y ENV PATH=/root/.cargo/bin:$PATH RUN rustup component add rust-src --toolchain nightly-$PLATFORM-unknown-linux-gnu + +# Build Bottlecap RUN mkdir -p /tmp/dd COPY ./bottlecap/src /tmp/dd/bottlecap/src COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml @@ -19,13 +24,17 @@ WORKDIR /tmp/dd/bottlecap RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +nightly build -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort --release --target $PLATFORM-unknown-linux-gnu RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap -# zip the extension +# Zip Extension FROM ubuntu:latest as compresser ARG DATADOG_WRAPPER=datadog_wrapper +ARG GO_AGENT_PATH RUN apt-get update RUN apt-get install -y zip binutils -COPY --from=public.ecr.aws/datadog/lambda-extension:57 /opt/extensions/datadog-agent /datadog-agent-go + +COPY ./.layers/$GO_AGENT_PATH/extensions/datadog-agent /datadog-agent-go +RUN strip /datadog-agent-go # just in case + RUN mkdir /extensions WORKDIR /extensions diff --git a/scripts/Dockerfile.bottlecap.dev b/scripts/Dockerfile.bottlecap.dev new file mode 100644 index 000000000..78d7d9400 --- /dev/null +++ b/scripts/Dockerfile.bottlecap.dev @@ -0,0 +1,41 @@ +# syntax = docker/dockerfile:experimental + +FROM public.ecr.aws/lambda/provided:al2 as bottlecap-builder +ARG PLATFORM +RUN yum install -y curl gcc gcc-c++ make unzip openssl openssl-devel +# Install Protocol Buffers compiler by hand, since AL2 does not have a recent enough version. +COPY ./scripts/install-protoc.sh / +RUN chmod +x /install-protoc.sh && /install-protoc.sh +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- --profile minimal --default-toolchain nightly-$PLATFORM-unknown-linux-gnu -y +ENV PATH=/root/.cargo/bin:$PATH +RUN rustup component add rust-src --toolchain nightly-$PLATFORM-unknown-linux-gnu +RUN mkdir -p /tmp/dd +COPY ./bottlecap/src /tmp/dd/bottlecap/src +COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml +COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock +ENV RUSTFLAGS="-C panic=abort -Zlocation-detail=none" +WORKDIR /tmp/dd/bottlecap +RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +nightly build -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort --release --target $PLATFORM-unknown-linux-gnu +RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap + +# zip the extension +FROM ubuntu:latest as compresser +ARG DATADOG_WRAPPER=datadog_wrapper + +RUN apt-get update +RUN apt-get install -y zip binutils +COPY --from=public.ecr.aws/datadog/lambda-extension:57 /opt/extensions/datadog-agent /datadog-agent-go +RUN mkdir /extensions +WORKDIR /extensions + +COPY --from=bottlecap-builder /tmp/dd/bottlecap/bottlecap /extensions/datadog-agent + +COPY ./scripts/$DATADOG_WRAPPER /$DATADOG_WRAPPER +RUN chmod +x /$DATADOG_WRAPPER +RUN zip -r datadog_extension.zip /extensions /$DATADOG_WRAPPER /datadog-agent-go + +# keep the smallest possible docker image +FROM scratch +COPY --from=compresser /extensions/datadog_extension.zip / +ENTRYPOINT ["/datadog_extension.zip"] diff --git a/scripts/build_bottlecap_layer.sh b/scripts/build_bottlecap_layer.sh index e6620b155..cf8cd0d22 100755 --- a/scripts/build_bottlecap_layer.sh +++ b/scripts/build_bottlecap_layer.sh @@ -39,7 +39,7 @@ _docker_build_bottlecap_zip() { docker buildx build --platform linux/${arch} \ -t datadog/build-bottlecap-${arch} \ - -f ./scripts/Dockerfile.bottlecap.build \ + -f ./scripts/Dockerfile.bottlecap.dev \ --build-arg PLATFORM=$PLATFORM \ . -o $TARGET_DIR/datadog_bottlecap-${arch} From 6d61f4d15b63699b8227f7a46e330dff96828c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:45:25 -0400 Subject: [PATCH 57/74] update `build_go_agent.sh` mainly to unzip before removingthe zip --- .gitlab/scripts/build_go_agent.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts/build_go_agent.sh b/.gitlab/scripts/build_go_agent.sh index ee9ddcba8..7d91a5e0f 100755 --- a/.gitlab/scripts/build_go_agent.sh +++ b/.gitlab/scripts/build_go_agent.sh @@ -11,8 +11,7 @@ set -e if [ -z "$ARCHITECTURE" ]; then - echo "ARCHITECTURE not specified" - echo "" + printf "[ERROR]: ARCHITECTURE not specified\n" exit 1 fi @@ -84,7 +83,8 @@ function docker_build { . -o $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} cp $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}.zip - rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} + unzip $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} + rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}/datadog_extension.zip } docker_build $ARCHITECTURE $BUILD_FILE From 0a97c640017cefc580d8ce211a21d8195def90f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:45:43 -0400 Subject: [PATCH 58/74] build bottlecap for gitlab --- .gitlab/scripts/build_bottlecap.sh | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100755 .gitlab/scripts/build_bottlecap.sh diff --git a/.gitlab/scripts/build_bottlecap.sh b/.gitlab/scripts/build_bottlecap.sh new file mode 100755 index 000000000..bb774e665 --- /dev/null +++ b/.gitlab/scripts/build_bottlecap.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +set -e + +if [ -z "$ARCHITECTURE" ]; then + printf "[ERROR]: ARCHITECTURE not specified\n" + exit 1 +fi + +if [ -z "$ALPINE" ]; then + printf "Building bottlecap" +else + echo "Building bottlecap for alpine" + BUILD_SUFFIX="-alpine" +fi + +prepare_folders() { + # Move into the root directory, so this script can be called from any directory + SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + ROOT_DIR=$SCRIPTS_DIR/../.. + cd $ROOT_DIR + + echo $ROOT_DIR + + EXTENSION_DIR=".layers" + TARGET_DIR=$(pwd)/$EXTENSION_DIR + + rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX} 2>/dev/null + rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX}.zip 2>/dev/null + + cd $ROOT_DIR +} + + +docker_build() { + local arch=$1 + if [ "$arch" == "amd64" ]; then + PLATFORM="x86_64" + else + PLATFORM="aarch64" + fi + + docker buildx build --platform linux/${arch} \ + -t datadog/build-bottlecap-${arch} \ + -f ./scripts/Dockerfile.bottlecap.build \ + --build-arg PLATFORM=$PLATFORM \ + --build-arg GO_AGENT_PATH="datadog_extension-${arch}${BUILD_SUFFIX}" \ + . -o $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX} + + cp $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}.zip + + unzip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX} + rm -rf $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip + rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} + rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}.zip +} + +prepare_folders +docker_build $ARCHITECTURE From b09df3ccaea7dbda3b1f8841d37bb4bafa009703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:46:09 -0400 Subject: [PATCH 59/74] add extension with bottlecap pipeline --- .gitlab/templates/lambda-extension.yaml.tpl | 157 ++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 .gitlab/templates/lambda-extension.yaml.tpl diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl new file mode 100644 index 000000000..3b150386f --- /dev/null +++ b/.gitlab/templates/lambda-extension.yaml.tpl @@ -0,0 +1,157 @@ +stages: + - build + - test + - sign + - publish + +default: + retry: + max: 1 + when: + - runner_system_failure + +variables: + DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension + DOCKER_TARGET_VERSION: latest + +{{ range $architecture := (ds "architectures").architectures }} + +build go agent ({{ $architecture.name }}): + stage: build + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + artifacts: + expire_in: 1 hr + paths: + - .layers/datadog_extension-{{ $architecture.name }}.zip + variables: + ARCHITECTURE: {{ $architecture.name }} + script: + - .gitlab/scripts/build_go_agent.sh + +build bottlecap ({{ $architecture.name }}): + stage: build + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: + - build go agent ({{ $architecture.name }}) + dependencies: + - build go agent ({{ $architecture.name }}) + artifacts: + expire_in: 1 hr + paths: + - .layers/datadog_bottlecap-{{ $architecture.name }}.zip + variables: + ARCHITECTURE: {{ $architecture.name }} + script: + - .gitlab/scripts/build_bottlecap.sh + +check layer size ({{ $architecture.name }}): + stage: test + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: + - build bottlecap ({{ $architecture.name }}) + dependencies: + - build bottlecap ({{ $architecture.name }}) + variables: + LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip + script: + - .gitlab/scripts/check_layer_size.sh + +fmt ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo fmt + +check ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo check + +clippy ({{ $architecture.name }}): + stage: test + tags: ["arch:{{ $architecture.name }}"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo clippy --all-features + +{{ range $environment := (ds "environments").environments }} + +{{ if or (eq $environment.name "prod") }} +sign layer ({{ $architecture.name }}): + stage: sign + tags: ["arch:amd64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - build bottlecap ({{ $architecture.name }}) + - check layer size ({{ $architecture.name }}) + - fmt ({{ $architecture.name }}) + - check ({{ $architecture.name }}) + - clippy ({{ $architecture.name }}) + dependencies: + - build bottlecap ({{ $architecture.name }}) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_bottlecap-{{ $architecture.name }}.zip + variables: + LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/sign_layers.sh {{ $environment.name }} +{{ end }} + +publish layer {{ $environment.name }} ({{ $architecture.name }}): + stage: publish + tags: ["arch:amd64"] + image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} + rules: + - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: +{{ if or (eq $environment.name "prod") }} + - sign layer ({{ $architecture.name }}) +{{ else }} + - build bottlecap ({{ $architecture.name }}) + - check layer size ({{ $architecture.name }}) + - fmt ({{ $architecture.name }}) + - check ({{ $architecture.name }}) + - clippy ({{ $architecture.name }}) +{{ end }} + dependencies: +{{ if or (eq $environment.name "prod") }} + - sign layer ({{ $architecture.name }}) +{{ else }} + - build bottlecap ({{ $architecture.name }}) +{{ end }} + parallel: + matrix: + - REGION: {{ range (ds "regions").regions }} + - {{ .code }} + {{- end}} + variables: + ARCHITECTURE: {{ $architecture.name }} + LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip + STAGE: {{ $environment.name }} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/publish_layers.sh + +{{- end }} # environments end + +{{- end }} # architectures end From 17c778a3077f8815958a54c68553b811a8716ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:55:41 -0400 Subject: [PATCH 60/74] typo in `config.yaml` --- .gitlab/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/config.yaml b/.gitlab/config.yaml index ef3010403..3e3899556 100644 --- a/.gitlab/config.yaml +++ b/.gitlab/config.yaml @@ -8,7 +8,7 @@ inputFiles: outputFiles: - .gitlab/pipeline-bottlecap.yaml - .gitlab/pipeline-go-agent.yaml - - .gitlab/pipeline-lambda-extension.yaml.tpl + - .gitlab/pipeline-lambda-extension.yaml datasources: architectures: From 2f9b849d3d8ccd730476fef58fe6906096b57236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 17:00:45 -0400 Subject: [PATCH 61/74] clone `datadog-agent` before building the go agent facepalm --- .gitlab/templates/lambda-extension.yaml.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl index 3b150386f..3d938c10e 100644 --- a/.gitlab/templates/lambda-extension.yaml.tpl +++ b/.gitlab/templates/lambda-extension.yaml.tpl @@ -27,6 +27,7 @@ build go agent ({{ $architecture.name }}): variables: ARCHITECTURE: {{ $architecture.name }} script: + - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - .gitlab/scripts/build_go_agent.sh build bottlecap ({{ $architecture.name }}): From cdb29bfd973e59b4bc2927bc5a3ef3febdbecc2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 20:06:59 -0400 Subject: [PATCH 62/74] see if agent branch is defaulted to main --- .gitlab/templates/lambda-extension.yaml.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl index 3d938c10e..85cdddb0c 100644 --- a/.gitlab/templates/lambda-extension.yaml.tpl +++ b/.gitlab/templates/lambda-extension.yaml.tpl @@ -27,6 +27,7 @@ build go agent ({{ $architecture.name }}): variables: ARCHITECTURE: {{ $architecture.name }} script: + - echo "AGENT BRANCH should be main, right now it is $AGENT_BRANCH" - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - .gitlab/scripts/build_go_agent.sh From c0b5e05c81f95706ac82caef525e9bcdb313e8c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Jul 2024 20:08:38 -0400 Subject: [PATCH 63/74] its not printing, try again --- .gitlab/templates/lambda-extension.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl index 85cdddb0c..1df3db88d 100644 --- a/.gitlab/templates/lambda-extension.yaml.tpl +++ b/.gitlab/templates/lambda-extension.yaml.tpl @@ -27,7 +27,7 @@ build go agent ({{ $architecture.name }}): variables: ARCHITECTURE: {{ $architecture.name }} script: - - echo "AGENT BRANCH should be main, right now it is $AGENT_BRANCH" + - echo "AGENT BRANCH should be main, right now it is ${AGENT_BRANCH}" - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - .gitlab/scripts/build_go_agent.sh From c57706ac796f557f723b728161a2e380866d47a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 00:24:53 -0400 Subject: [PATCH 64/74] disallow specific runtime pipelines to publish to prod --- .gitlab-ci.yml | 2 ++ .gitlab/templates/bottlecap.yaml.tpl | 45 +++------------------------- .gitlab/templates/go-agent.yaml.tpl | 38 ++--------------------- 3 files changed, 8 insertions(+), 77 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7663da329..41a420938 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,6 +75,8 @@ lambda-extension: - artifact: .gitlab/pipeline-lambda-extension.yaml job: generator strategy: depend + rules: + - if: $CI_PIPELINE_SOURCE == "web" variables: AGENT_BRANCH: $AGENT_BRANCH LAYER_SUFFIX: $LAYER_SUFFIX diff --git a/.gitlab/templates/bottlecap.yaml.tpl b/.gitlab/templates/bottlecap.yaml.tpl index 69ebcccd6..3898a5b02 100644 --- a/.gitlab/templates/bottlecap.yaml.tpl +++ b/.gitlab/templates/bottlecap.yaml.tpl @@ -68,14 +68,14 @@ clippy ({{ $architecture.name }}): {{ range $environment := (ds "environments").environments }} -{{ if or (eq $environment.name "prod") }} -sign layer ({{ $architecture.name }}): - stage: sign +publish layer {{ $environment.name }} ({{ $architecture.name }}): + stage: publish tags: ["arch:amd64"] image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' + - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' when: manual + allow_failure: true needs: - build layer ({{ $architecture.name }}) - check layer size ({{ $architecture.name }}) @@ -84,43 +84,6 @@ sign layer ({{ $architecture.name }}): - clippy ({{ $architecture.name }}) dependencies: - build layer ({{ $architecture.name }}) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_bottlecap-{{ $architecture.name }}.zip - variables: - LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/sign_layers.sh {{ $environment.name }} -{{ end }} - -publish layer {{ $environment.name }} ({{ $architecture.name }}): - stage: publish - tags: ["arch:amd64"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - rules: - - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - needs: -{{ if or (eq $environment.name "prod") }} - - sign layer ({{ $architecture.name }}) -{{ else }} - - build layer ({{ $architecture.name }}) - - check layer size ({{ $architecture.name }}) - - fmt ({{ $architecture.name }}) - - check ({{ $architecture.name }}) - - clippy ({{ $architecture.name }}) -{{ end }} - dependencies: -{{ if or (eq $environment.name "prod") }} - - sign layer ({{ $architecture.name }}) -{{ else }} - - build layer ({{ $architecture.name }}) -{{ end }} parallel: matrix: - REGION: {{ range (ds "regions").regions }} diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl index 6fa15173e..bc2ed612e 100644 --- a/.gitlab/templates/go-agent.yaml.tpl +++ b/.gitlab/templates/go-agent.yaml.tpl @@ -46,31 +46,6 @@ check layer size ({{ $architecture.name }}): {{ range $environment := (ds "environments").environments }} -{{ if or (eq $environment.name "prod") }} -sign layer ({{ $architecture.name }}): - stage: sign - tags: ["arch:amd64"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build layer ({{ $architecture.name }}) - - check layer size ({{ $architecture.name }}) - dependencies: - - build layer ({{ $architecture.name }}) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 hr # Signed layers should expire after 1 day TODO: modify to 1 day - paths: - - .layers/datadog_extension-{{ $architecture.name }}.zip - variables: - LAYER_FILE: datadog_extension-{{ $architecture.name }}.zip - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/sign_layers.sh {{ $environment.name }} -{{ end }} - publish layer {{ $environment.name }} ({{ $architecture.name }}): stage: publish tags: ["arch:amd64"] @@ -79,19 +54,10 @@ publish layer {{ $environment.name }} ({{ $architecture.name }}): - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' when: manual allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' needs: -{{ if or (eq $environment.name "prod") }} - - sign layer ({{ $architecture.name }}) -{{ else }} - - build layer ({{ $architecture.name }}) -{{ end }} + - build layer ({{ $architecture.name }}) dependencies: -{{ if or (eq $environment.name "prod") }} - - sign layer ({{ $architecture.name }}) -{{ else }} - - build layer ({{ $architecture.name }}) -{{ end }} + - build layer ({{ $architecture.name }}) parallel: matrix: - REGION: {{ range (ds "regions").regions }} From c5cb5a7eaf55519abb7a51c99b0880b6a7626840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 00:25:25 -0400 Subject: [PATCH 65/74] `lambda-extension` to only be triggered through web --- .gitlab/templates/lambda-extension.yaml.tpl | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl index 1df3db88d..3d938c10e 100644 --- a/.gitlab/templates/lambda-extension.yaml.tpl +++ b/.gitlab/templates/lambda-extension.yaml.tpl @@ -27,7 +27,6 @@ build go agent ({{ $architecture.name }}): variables: ARCHITECTURE: {{ $architecture.name }} script: - - echo "AGENT BRANCH should be main, right now it is ${AGENT_BRANCH}" - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - .gitlab/scripts/build_go_agent.sh From f06c67c1fa4d82bde6b95831f5836f02cfbac22c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 00:41:53 -0400 Subject: [PATCH 66/74] fix how version is set for `build_go_agent.sh` --- .gitlab/scripts/build_go_agent.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts/build_go_agent.sh b/.gitlab/scripts/build_go_agent.sh index 7d91a5e0f..f3c9009c1 100755 --- a/.gitlab/scripts/build_go_agent.sh +++ b/.gitlab/scripts/build_go_agent.sh @@ -15,9 +15,14 @@ if [ -z "$ARCHITECTURE" ]; then exit 1 fi -# If version is not set, then this is not a release build -if [ -z "$VERSION" ]; then + +if [ -z "$CI_COMMIT_TAG" ]; then + # Running on dev + printf "Running on dev environment\n" VERSION="dev" +else + printf "Found version tag in environment\n" + VERSION=$(echo "${CI_COMMIT_TAG##*v}" | cut -d. -f2) fi if [ -z "$SERVERLESS_INIT" ]; then From bfb9c25a802a5c12e2a07003edd315b57e498a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 00:49:59 -0400 Subject: [PATCH 67/74] maybe copy isnt finding the right file? --- scripts/Dockerfile.bottlecap.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Dockerfile.bottlecap.build b/scripts/Dockerfile.bottlecap.build index 221d48431..cf4e765e0 100644 --- a/scripts/Dockerfile.bottlecap.build +++ b/scripts/Dockerfile.bottlecap.build @@ -32,7 +32,7 @@ ARG GO_AGENT_PATH RUN apt-get update RUN apt-get install -y zip binutils -COPY ./.layers/$GO_AGENT_PATH/extensions/datadog-agent /datadog-agent-go +COPY .layers/$GO_AGENT_PATH/extensions/datadog-agent /datadog-agent-go RUN strip /datadog-agent-go # just in case RUN mkdir /extensions From fff10a4c2a6f5d2f79630cb9e41199b65785de61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 00:53:54 -0400 Subject: [PATCH 68/74] debug ls --- .gitlab/templates/lambda-extension.yaml.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl index 3d938c10e..20aa721d2 100644 --- a/.gitlab/templates/lambda-extension.yaml.tpl +++ b/.gitlab/templates/lambda-extension.yaml.tpl @@ -45,6 +45,7 @@ build bottlecap ({{ $architecture.name }}): variables: ARCHITECTURE: {{ $architecture.name }} script: + - ls .layers - .gitlab/scripts/build_bottlecap.sh check layer size ({{ $architecture.name }}): From 190a6ec3df61a7eb688f70ebb779532e36915922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 01:03:03 -0400 Subject: [PATCH 69/74] forgot to add path to directory including the binaries --- .gitlab/templates/lambda-extension.yaml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl index 20aa721d2..5e44bd33e 100644 --- a/.gitlab/templates/lambda-extension.yaml.tpl +++ b/.gitlab/templates/lambda-extension.yaml.tpl @@ -24,6 +24,7 @@ build go agent ({{ $architecture.name }}): expire_in: 1 hr paths: - .layers/datadog_extension-{{ $architecture.name }}.zip + - .layers/datadog_extension-{{ $architecture.name }}/* variables: ARCHITECTURE: {{ $architecture.name }} script: @@ -45,7 +46,6 @@ build bottlecap ({{ $architecture.name }}): variables: ARCHITECTURE: {{ $architecture.name }} script: - - ls .layers - .gitlab/scripts/build_bottlecap.sh check layer size ({{ $architecture.name }}): From 18d294b2cf1450570966c39bdef1101ebbbf363d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:27:42 -0400 Subject: [PATCH 70/74] update layer size --- .gitlab/scripts/check_layer_size.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh index f3af3d586..93e15a403 100755 --- a/.gitlab/scripts/check_layer_size.sh +++ b/.gitlab/scripts/check_layer_size.sh @@ -14,7 +14,7 @@ if [ -z "$LAYER_FILE" ]; then exit 1 fi -MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 16 \* 1024) # 16 MB, amd64 is 16, while arm64 is 15 +MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 17 \* 1024) # 17 MB, amd64 is 17, while arm64 is 15 MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 44 \* 1024) # 44 MB, amd is 44, while arm64 is 43 LAYERS_DIR=".layers" From 7a19dd5846c26be0f78817e137af37450bd7811d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:40:12 -0400 Subject: [PATCH 71/74] remove all regions but `sa-east-1` temporarily --- .gitlab/datasources/regions.yaml | 56 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.gitlab/datasources/regions.yaml b/.gitlab/datasources/regions.yaml index 9f12a0118..e724bdf5e 100644 --- a/.gitlab/datasources/regions.yaml +++ b/.gitlab/datasources/regions.yaml @@ -1,30 +1,30 @@ regions: - - code: "us-east-1" - - code: "us-east-2" - - code: "us-west-1" - - code: "us-west-2" - - code: "af-south-1" - - code: "ap-east-1" - - code: "ap-south-1" - - code: "ap-south-2" - - code: "ap-southeast-1" - - code: "ap-southeast-2" - - code: "ap-southeast-3" - - code: "ap-southeast-4" - - code: "ap-northeast-1" - - code: "ap-northeast-2" - - code: "ap-northeast-3" - - code: "ca-central-1" - - code: "ca-west-1" - - code: "eu-central-1" - - code: "eu-central-2" - - code: "eu-north-1" - - code: "eu-west-1" - - code: "eu-west-2" - - code: "eu-west-3" - - code: "eu-south-1" - - code: "eu-south-2" - - code: "il-central-1" - - code: "me-south-1" - - code: "me-central-1" + # - code: "us-east-1" + # - code: "us-east-2" + # - code: "us-west-1" + # - code: "us-west-2" + # - code: "af-south-1" + # - code: "ap-east-1" + # - code: "ap-south-1" + # - code: "ap-south-2" + # - code: "ap-southeast-1" + # - code: "ap-southeast-2" + # - code: "ap-southeast-3" + # - code: "ap-southeast-4" + # - code: "ap-northeast-1" + # - code: "ap-northeast-2" + # - code: "ap-northeast-3" + # - code: "ca-central-1" + # - code: "ca-west-1" + # - code: "eu-central-1" + # - code: "eu-central-2" + # - code: "eu-north-1" + # - code: "eu-west-1" + # - code: "eu-west-2" + # - code: "eu-west-3" + # - code: "eu-south-1" + # - code: "eu-south-2" + # - code: "il-central-1" + # - code: "me-south-1" + # - code: "me-central-1" - code: "sa-east-1" From ac5d22b25eb1c7615c4cb2bb77f7b7dad982a237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:55:46 -0400 Subject: [PATCH 72/74] increase layer size again --- .gitlab/scripts/check_layer_size.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/check_layer_size.sh b/.gitlab/scripts/check_layer_size.sh index 93e15a403..d54b1daa4 100755 --- a/.gitlab/scripts/check_layer_size.sh +++ b/.gitlab/scripts/check_layer_size.sh @@ -15,7 +15,7 @@ if [ -z "$LAYER_FILE" ]; then fi MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 17 \* 1024) # 17 MB, amd64 is 17, while arm64 is 15 -MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 44 \* 1024) # 44 MB, amd is 44, while arm64 is 43 +MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 46 \* 1024) # 46 MB, amd is 46, while arm64 is 45 LAYERS_DIR=".layers" From 2440892b5239f0448a9c8476dfb6fba4de037712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:56:13 -0400 Subject: [PATCH 73/74] add all regions back --- .gitlab/datasources/regions.yaml | 56 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.gitlab/datasources/regions.yaml b/.gitlab/datasources/regions.yaml index e724bdf5e..9f12a0118 100644 --- a/.gitlab/datasources/regions.yaml +++ b/.gitlab/datasources/regions.yaml @@ -1,30 +1,30 @@ regions: - # - code: "us-east-1" - # - code: "us-east-2" - # - code: "us-west-1" - # - code: "us-west-2" - # - code: "af-south-1" - # - code: "ap-east-1" - # - code: "ap-south-1" - # - code: "ap-south-2" - # - code: "ap-southeast-1" - # - code: "ap-southeast-2" - # - code: "ap-southeast-3" - # - code: "ap-southeast-4" - # - code: "ap-northeast-1" - # - code: "ap-northeast-2" - # - code: "ap-northeast-3" - # - code: "ca-central-1" - # - code: "ca-west-1" - # - code: "eu-central-1" - # - code: "eu-central-2" - # - code: "eu-north-1" - # - code: "eu-west-1" - # - code: "eu-west-2" - # - code: "eu-west-3" - # - code: "eu-south-1" - # - code: "eu-south-2" - # - code: "il-central-1" - # - code: "me-south-1" - # - code: "me-central-1" + - code: "us-east-1" + - code: "us-east-2" + - code: "us-west-1" + - code: "us-west-2" + - code: "af-south-1" + - code: "ap-east-1" + - code: "ap-south-1" + - code: "ap-south-2" + - code: "ap-southeast-1" + - code: "ap-southeast-2" + - code: "ap-southeast-3" + - code: "ap-southeast-4" + - code: "ap-northeast-1" + - code: "ap-northeast-2" + - code: "ap-northeast-3" + - code: "ca-central-1" + - code: "ca-west-1" + - code: "eu-central-1" + - code: "eu-central-2" + - code: "eu-north-1" + - code: "eu-west-1" + - code: "eu-west-2" + - code: "eu-west-3" + - code: "eu-south-1" + - code: "eu-south-2" + - code: "il-central-1" + - code: "me-south-1" + - code: "me-central-1" - code: "sa-east-1" From 446a25bc67048820a3fcbace5d8efe768de889aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:22:26 -0400 Subject: [PATCH 74/74] only add permissions on prod --- .gitlab/scripts/publish_layers.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index 90d1802cd..336593d95 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -24,13 +24,16 @@ publish_layer() { | jq -r '.Version' ) - permission=$(aws lambda add-layer-version-permission --layer-name $layer \ - --version-number $version_nbr \ - --statement-id "release-$version_nbr" \ - --action lambda:GetLayerVersion \ - --principal "*" \ - --region $region - ) + # Add permissions only for prod + if [ "$STAGE" == "prod" ]; then + permission=$(aws lambda add-layer-version-permission --layer-name $layer \ + --version-number $version_nbr \ + --statement-id "release-$version_nbr" \ + --action lambda:GetLayerVersion \ + --principal "*" \ + --region $region + ) + fi echo $version_nbr }