From 8812080b20bddb81410f6a69d63d6ece879c658f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 10:44:42 +0000 Subject: [PATCH 01/41] ci: add release stage when a tag is created --- .ci/Jenkinsfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index d6de0b69..a713ec59 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -95,6 +95,15 @@ pipeline { } } } + stage('Release') { + options { skipDefaultCheckout() } + when { + tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP' + } + steps { + echo 'TBD' + } + } } post { cleanup { From fbbf42539347151d1976f363a190468f3a5dbcc5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 11:06:47 +0000 Subject: [PATCH 02/41] ci: add release dist --- .ci/Jenkinsfile | 48 +++++++++++++++++++++++++++++++++-- apm-lambda-extension/Makefile | 1 + 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a713ec59..8845173b 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -100,8 +100,52 @@ pipeline { when { tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP' } - steps { - echo 'TBD' + stages { + stage('BuildPublish') { + failFast false + matrix { + agent {label "${PLATFORM}"} + options { skipDefaultCheckout() } + axes { + axis { + name 'PLATFORM' + values 'arm', 'ubuntu-18 && immutable' + } + } + stages { + stage('Dist') { + steps { + withGithubNotify(context: "Dist-${PLATFORM}") { + deleteDir() + unstash 'source' + withGoEnv(){ + dir("${BASE_DIR}/apm-lambda-extension"){ + cmd(label: 'make dist', script: 'make dist') + stash(includes: 'bin/*.zip', name: "dist-${isArm() ? 'arm' : 'linux'}") + } + } + } + } + } + stage('Publish') { + steps { + withGithubNotify(context: "Publish-${PLATFORM}") { + //withAWSEnv(secret: 'TBD') { + dir("${BASE_DIR}/apm-lambda-extension"){ + cmd(label: 'make publish', script: 'echo publish') + } + //} + } + } + } + } + } + } + stage('Release Notes') { + steps { + echo 'TBD' + } + } } } } diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index ffe2e12a..9d358a35 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -29,3 +29,4 @@ ifndef GOARCH export GOARCH=amd64 endif env +dist: build test zip From e27ca6442e1ee803555d7cb4767b1289701cd2e4 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 11:07:24 +0000 Subject: [PATCH 03/41] chore: testing purposes --- .ci/Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 8845173b..1f964e15 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -97,9 +97,10 @@ pipeline { } stage('Release') { options { skipDefaultCheckout() } - when { - tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP' - } + // TODO: for testing purposes + //when { + // tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP' + //} stages { stage('BuildPublish') { failFast false From 0c89783c9d9f24b295add0fbc0e3a7a5723911cb Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 12:29:22 +0000 Subject: [PATCH 04/41] ci: stash is not needed when running a matrix --- .ci/Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 1f964e15..0c2d9234 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -122,7 +122,6 @@ pipeline { withGoEnv(){ dir("${BASE_DIR}/apm-lambda-extension"){ cmd(label: 'make dist', script: 'make dist') - stash(includes: 'bin/*.zip', name: "dist-${isArm() ? 'arm' : 'linux'}") } } } From f5ed12e25862bc4222dafe744abf2e1cadb059e0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 12:36:43 +0000 Subject: [PATCH 05/41] ci: add release-notes --- .ci/Jenkinsfile | 7 ++++++- apm-lambda-extension/Makefile | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 0c2d9234..d77e4c86 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { REPO = 'apm-aws-lambda' BASE_DIR = "src/github.com/elastic/${env.REPO}" PIPELINE_LOG_LEVEL = 'INFO' + CHANGELOG = 'changelog.md' } options { timeout(time: 1, unit: 'HOURS') @@ -143,7 +144,11 @@ pipeline { } stage('Release Notes') { steps { - echo 'TBD' + withGhEnv(forceInstallation: true, version: '2.4.0') { + dir("${BASE_DIR}/apm-lambda-extension"){ + cmd(label: 'make release-notes', script: 'make release-notes') + } + } } } } diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 9d358a35..a72d5b0e 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -30,3 +30,12 @@ ifndef GOARCH endif env dist: build test zip + +release-notes: +ifndef BRANCH_NAME + $(error BRANCH_NAME is undefined) +endif +ifndef CHANGELOG + $(error CHANGELOG is undefined) +endif + gh release create $(BRANCH_NAME) -F $(CHANGELOG) From 86021169bc8c50ff5a811826d7ea4a7e78e88621 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 12:39:34 +0000 Subject: [PATCH 06/41] chore: for testing purposes --- apm-lambda-extension/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index a72d5b0e..245306d3 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -38,4 +38,5 @@ endif ifndef CHANGELOG $(error CHANGELOG is undefined) endif - gh release create $(BRANCH_NAME) -F $(CHANGELOG) + @gh release list + @echo "gh release create $(BRANCH_NAME) -F $(CHANGELOG)" From ffa2caa9cf76f3d3538e42e2fdadb5bfa629221d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 13:23:38 +0000 Subject: [PATCH 07/41] ci: enable aws login --- .ci/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index d77e4c86..4893fb7c 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -131,11 +131,11 @@ pipeline { stage('Publish') { steps { withGithubNotify(context: "Publish-${PLATFORM}") { - //withAWSEnv(secret: 'TBD') { + withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { dir("${BASE_DIR}/apm-lambda-extension"){ cmd(label: 'make publish', script: 'echo publish') } - //} + } } } } From 7f3e9d03392621ec0b0f3f935463574db0f785be Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 13:33:06 +0000 Subject: [PATCH 08/41] ci: publish in all the AWS regions --- .ci/Jenkinsfile | 8 +++++++- .gitignore | 3 +++ apm-lambda-extension/Makefile | 32 +++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4893fb7c..a60e7b13 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -9,6 +9,7 @@ pipeline { BASE_DIR = "src/github.com/elastic/${env.REPO}" PIPELINE_LOG_LEVEL = 'INFO' CHANGELOG = 'changelog.md' + ELASTIC_LAYER_NAME = 'apm-lambda-extension' } options { timeout(time: 1, unit: 'HOURS') @@ -133,12 +134,17 @@ pipeline { withGithubNotify(context: "Publish-${PLATFORM}") { withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { dir("${BASE_DIR}/apm-lambda-extension"){ - cmd(label: 'make publish', script: 'echo publish') + cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') } } } } } + post { + always { + archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/apm-lambda-extension/.regions") + } + } } } } diff --git a/.gitignore b/.gitignore index 66fd13c9..4ae26517 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +# AWS regions file +.regions diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 245306d3..aefa219f 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -19,7 +19,7 @@ ifndef ELASTIC_LAYER_NAME endif GOARCH=${GOARCH} make build GOARCH=${GOARCH} make zip - aws lambda publish-layer-version --layer-name "${ELASTIC_LAYER_NAME}" --zip-file "fileb://./bin/extension.zip" + $(MAKE) publish zip: cd bin && rm -f extension.zip || true && zip -r extension.zip extensions && cp extension.zip ${GOARCH}.zip test: @@ -31,6 +31,36 @@ endif env dist: build test zip +# List all the AWS regions +get-all-aws-regions: + @aws \ + ec2 \ + describe-regions \ + --all-regions \ + --output json \ + --no-cli-pager \ + | jq -r '.Regions[].RegionName' > .regions + +# Publish the given LAYER in all the AWS regions +publish-in-all-aws-regions: get-all-aws-regions +ifndef ELASTIC_LAYER_NAME + $(error ELASTIC_LAYER_NAME is undefined) +endif + @while read AWS_DEFAULT_REGION; do \ + AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish; \ + done <.regions + +# Publish the given LAYER in the given AWS region +publish: +ifndef AWS_DEFAULT_REGION + $(error AWS_DEFAULT_REGION is undefined) +endif +ifndef ELASTIC_LAYER_NAME + $(error ELASTIC_LAYER_NAME is undefined) +endif + @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' + aws lambda publish-layer-version --layer-name "${ELASTIC_LAYER_NAME}" --zip-file "fileb://./bin/extension.zip" + release-notes: ifndef BRANCH_NAME $(error BRANCH_NAME is undefined) From 019d6f0b1f380b1bb27a37dea9572b45776c765c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 13:34:08 +0000 Subject: [PATCH 09/41] chore: for testing purposes --- apm-lambda-extension/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index aefa219f..b6adee02 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -59,7 +59,7 @@ ifndef ELASTIC_LAYER_NAME $(error ELASTIC_LAYER_NAME is undefined) endif @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' - aws lambda publish-layer-version --layer-name "${ELASTIC_LAYER_NAME}" --zip-file "fileb://./bin/extension.zip" + @echo 'aws lambda publish-layer-version --layer-name "${ELASTIC_LAYER_NAME}" --zip-file "fileb://./bin/extension.zip"' release-notes: ifndef BRANCH_NAME From 4d39efe1c6e83877c16d1781d6f013ba9fc57fb5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 13:37:14 +0000 Subject: [PATCH 10/41] ci: fix post location --- .ci/Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a60e7b13..a176cb81 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -139,10 +139,10 @@ pipeline { } } } - } - post { - always { - archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/apm-lambda-extension/.regions") + post { + always { + archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/apm-lambda-extension/.regions") + } } } } From 620d8b063cfbfab623b69296cc7cb795f5aabae0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 14:07:31 +0000 Subject: [PATCH 11/41] fix: You must specify a region. --- apm-lambda-extension/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index b6adee02..6013f0b7 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -36,6 +36,7 @@ get-all-aws-regions: @aws \ ec2 \ describe-regions \ + --region us-east-1 \ --all-regions \ --output json \ --no-cli-pager \ From 9ec1143abac42f0745d1b2713bcb829a66125e4b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 14:43:46 +0000 Subject: [PATCH 12/41] makefile: report pipe errors --- apm-lambda-extension/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 6013f0b7..e80d37d1 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -1,3 +1,5 @@ +SHELL = /bin/bash -eo pipefail + build: ifndef GOARCH GOARCH=amd64 From 3b9c99c598b1c508c8477c015210733b8c8a0fe7 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 16:17:27 +0000 Subject: [PATCH 13/41] Use generate-cli-skeleton for dry-run Prints a JSON skeleton to standard output without sending an API request --- apm-lambda-extension/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index e80d37d1..6bc75465 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -62,7 +62,12 @@ ifndef ELASTIC_LAYER_NAME $(error ELASTIC_LAYER_NAME is undefined) endif @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' - @echo 'aws lambda publish-layer-version --layer-name "${ELASTIC_LAYER_NAME}" --zip-file "fileb://./bin/extension.zip"' + aws lambda \ + --output json \ + publish-layer-version \ + --layer-name "${ELASTIC_LAYER_NAME}" \ + --zip-file "fileb://./bin/extension.zip" \ + --generate-cli-skeleton release-notes: ifndef BRANCH_NAME From 97e47db02ba85c2a7b6c6ab88bc50758cea518dc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 17:10:54 +0000 Subject: [PATCH 14/41] ci: create ARN table for each arch --- .ci/Jenkinsfile | 6 +++++- .ci/create-arn-table.sh | 22 ++++++++++++++++++++++ .gitignore | 1 + apm-lambda-extension/Makefile | 30 ++++++++++++++++++++++++------ 4 files changed, 52 insertions(+), 7 deletions(-) create mode 100755 .ci/create-arn-table.sh diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a176cb81..9dc93559 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -8,7 +8,7 @@ pipeline { REPO = 'apm-aws-lambda' BASE_DIR = "src/github.com/elastic/${env.REPO}" PIPELINE_LOG_LEVEL = 'INFO' - CHANGELOG = 'changelog.md' + PREFIX_ARN_FILE = 'arn-file.md' ELASTIC_LAYER_NAME = 'apm-lambda-extension' } options { @@ -135,6 +135,8 @@ pipeline { withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { dir("${BASE_DIR}/apm-lambda-extension"){ cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') + stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") + stash(allowEmpty: true, name: "arn-${isArm() ? 'arm' : 'amd'}", useDefaultExcludes: false) } } } @@ -152,6 +154,8 @@ pipeline { steps { withGhEnv(forceInstallation: true, version: '2.4.0') { dir("${BASE_DIR}/apm-lambda-extension"){ + unstash "arn-arm" + unstash "arn-amd" cmd(label: 'make release-notes', script: 'make release-notes') } } diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh new file mode 100755 index 00000000..0feb0aa5 --- /dev/null +++ b/.ci/create-arn-table.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -eo pipefail + +# +# Create the AWS ARN table given the below environment variables: +# +# - AWS_FOLDER - that's the location of the publish-layer-version output for each region +# - GOARCH - that's the supported architecture. +# - PREFIX_ARN_FILE - that's the output file. +# + +{ + echo "### ARCH: ${GOARCH}" + echo '' + echo '|Region|Arch|ARN|' + echo '|------|----|---|' + for f in $(ls "${AWS_FOLDER}"); do + # TODO: identify what field to be used. + echo "|${f}|${GOARCH}|$(cat $AWS_FOLDER/${f} | jq -r .LayerName)|" + done + echo '' +} > ${GOARCH}-${PREFIX_ARN_FILE} diff --git a/.gitignore b/.gitignore index 4ae26517..de9071ac 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ # AWS regions file .regions +.aws/ diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 6bc75465..f3f47c53 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -1,9 +1,14 @@ SHELL = /bin/bash -eo pipefail -build: +AWS_FOLDER = .aws + ifndef GOARCH GOARCH=amd64 endif + +export AWS_FOLDER GOARCH + +build: GOOS=linux go build -o bin/extensions/apm-lambda-extension main.go chmod +x bin/extensions/apm-lambda-extension build-and-publish: @@ -52,6 +57,7 @@ endif @while read AWS_DEFAULT_REGION; do \ AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish; \ done <.regions + $(MAKE) create-arn-file # Publish the given LAYER in the given AWS region publish: @@ -62,19 +68,31 @@ ifndef ELASTIC_LAYER_NAME $(error ELASTIC_LAYER_NAME is undefined) endif @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' - aws lambda \ + @mkdir -p $(AWS_FOLDER) + @aws lambda \ --output json \ publish-layer-version \ --layer-name "${ELASTIC_LAYER_NAME}" \ --zip-file "fileb://./bin/extension.zip" \ - --generate-cli-skeleton + --generate-cli-skeleton > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} + +# Generate the file with the ARN entries +create-arn-file: +ifndef BRANCH_NAME + $(error BRANCH_NAME is undefined) +endif +ifndef PREFIX_ARN_FILE + $(error PREFIX_ARN_FILE is undefined) +endif + @../.ci/create-arn-table.sh release-notes: ifndef BRANCH_NAME $(error BRANCH_NAME is undefined) endif -ifndef CHANGELOG - $(error CHANGELOG is undefined) +ifndef PREFIX_ARN_FILE + $(error PREFIX_ARN_FILE is undefined) endif @gh release list - @echo "gh release create $(BRANCH_NAME) -F $(CHANGELOG)" + @cat *-$(PREFIX_ARN_FILE) > $(PREFIX_ARN_FILE) + @echo "gh release create $(BRANCH_NAME) -F $(PREFIX_ARN_FILE)" From b5971158caef96989ad5956c79a0e46745218d0a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 20:59:47 +0000 Subject: [PATCH 15/41] ci: remove duplicated stash --- .ci/Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 9dc93559..a05dc017 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -136,7 +136,6 @@ pipeline { dir("${BASE_DIR}/apm-lambda-extension"){ cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") - stash(allowEmpty: true, name: "arn-${isArm() ? 'arm' : 'amd'}", useDefaultExcludes: false) } } } From 1a43178522cc0202601b6679a7cd3d9678955986 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 21:20:29 +0000 Subject: [PATCH 16/41] Support upload assets --- .ci/Jenkinsfile | 3 +++ apm-lambda-extension/Makefile | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index a05dc017..fcbc6bb5 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -136,6 +136,7 @@ pipeline { dir("${BASE_DIR}/apm-lambda-extension"){ cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") + stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") } } } @@ -155,6 +156,8 @@ pipeline { dir("${BASE_DIR}/apm-lambda-extension"){ unstash "arn-arm" unstash "arn-amd" + unstash "dist-arm" + unstash "dist-amd" cmd(label: 'make release-notes', script: 'make release-notes') } } diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index f3f47c53..2bcf5542 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -64,11 +64,15 @@ publish: ifndef AWS_DEFAULT_REGION $(error AWS_DEFAULT_REGION is undefined) endif +ifndef BRANCH_NAME + $(error BRANCH_NAME is undefined) +endif ifndef ELASTIC_LAYER_NAME $(error ELASTIC_LAYER_NAME is undefined) endif @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' @mkdir -p $(AWS_FOLDER) + @mv bin/$(GOARCH).zip bin/$(BRANCH_NAME)-linux-$(GOARCH).zip @aws lambda \ --output json \ publish-layer-version \ @@ -95,4 +99,9 @@ ifndef PREFIX_ARN_FILE endif @gh release list @cat *-$(PREFIX_ARN_FILE) > $(PREFIX_ARN_FILE) - @echo "gh release create $(BRANCH_NAME) -F $(PREFIX_ARN_FILE)" + @echo "gh \ + release \ + create $(BRANCH_NAME) \ + --title '$(BRANCH_NAME)' \ + --notes-file $(PREFIX_ARN_FILE) \ + ./bin/$(BRANCH_NAME)*.zip" From ccc7585a338d9b5fdb0eda6cab26fa55fe3a5fed Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 21:22:19 +0000 Subject: [PATCH 17/41] enable release notes with a draft --- apm-lambda-extension/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 2bcf5542..1fc19b5e 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -99,9 +99,11 @@ ifndef PREFIX_ARN_FILE endif @gh release list @cat *-$(PREFIX_ARN_FILE) > $(PREFIX_ARN_FILE) - @echo "gh \ + @gh \ release \ create $(BRANCH_NAME) \ --title '$(BRANCH_NAME)' \ + --draft \ + --generate-notes \ --notes-file $(PREFIX_ARN_FILE) \ - ./bin/$(BRANCH_NAME)*.zip" + ./bin/$(BRANCH_NAME)*.zip From 5bc534a73253d9fa5e25b6a7717af62eae7657d1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 21:22:50 +0000 Subject: [PATCH 18/41] remove duplication --- apm-lambda-extension/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 1fc19b5e..a63cee29 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -32,9 +32,6 @@ zip: test: go test extension/*.go -v env: -ifndef GOARCH - export GOARCH=amd64 -endif env dist: build test zip From 4d3779291f81fe2ba50480e52a92ad17da2c649b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 21:28:19 +0000 Subject: [PATCH 19/41] makefile: refactor validation in goals --- apm-lambda-extension/Makefile | 60 +++++++++++++++-------------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index a63cee29..c4b51a6c 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -11,18 +11,12 @@ export AWS_FOLDER GOARCH build: GOOS=linux go build -o bin/extensions/apm-lambda-extension main.go chmod +x bin/extensions/apm-lambda-extension -build-and-publish: -ifndef AWS_DEFAULT_REGION - $(error AWS_DEFAULT_REGION is undefined) -endif +build-and-publish: validate-layer-name validate-aws-default-region ifndef AWS_ACCESS_KEY_ID $(error AWS_ACCESS_KEY_ID is undefined) endif ifndef AWS_SECRET_ACCESS_KEY $(error AWS_SECRET_ACCESS_KEY is undefined) -endif -ifndef ELASTIC_LAYER_NAME - $(error ELASTIC_LAYER_NAME is undefined) endif GOARCH=${GOARCH} make build GOARCH=${GOARCH} make zip @@ -47,26 +41,14 @@ get-all-aws-regions: | jq -r '.Regions[].RegionName' > .regions # Publish the given LAYER in all the AWS regions -publish-in-all-aws-regions: get-all-aws-regions -ifndef ELASTIC_LAYER_NAME - $(error ELASTIC_LAYER_NAME is undefined) -endif +publish-in-all-aws-regions: validate-layer-name get-all-aws-regions @while read AWS_DEFAULT_REGION; do \ AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish; \ done <.regions $(MAKE) create-arn-file # Publish the given LAYER in the given AWS region -publish: -ifndef AWS_DEFAULT_REGION - $(error AWS_DEFAULT_REGION is undefined) -endif -ifndef BRANCH_NAME - $(error BRANCH_NAME is undefined) -endif -ifndef ELASTIC_LAYER_NAME - $(error ELASTIC_LAYER_NAME is undefined) -endif +publish: validate-branch-name validate-layer-name validate-aws-default-region @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' @mkdir -p $(AWS_FOLDER) @mv bin/$(GOARCH).zip bin/$(BRANCH_NAME)-linux-$(GOARCH).zip @@ -78,22 +60,10 @@ endif --generate-cli-skeleton > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} # Generate the file with the ARN entries -create-arn-file: -ifndef BRANCH_NAME - $(error BRANCH_NAME is undefined) -endif -ifndef PREFIX_ARN_FILE - $(error PREFIX_ARN_FILE is undefined) -endif +create-arn-file: validate-branch-name validate-prefix-arn-file @../.ci/create-arn-table.sh -release-notes: -ifndef BRANCH_NAME - $(error BRANCH_NAME is undefined) -endif -ifndef PREFIX_ARN_FILE - $(error PREFIX_ARN_FILE is undefined) -endif +release-notes: validate-branch-name validate-prefix-arn-file @gh release list @cat *-$(PREFIX_ARN_FILE) > $(PREFIX_ARN_FILE) @gh \ @@ -104,3 +74,23 @@ endif --generate-notes \ --notes-file $(PREFIX_ARN_FILE) \ ./bin/$(BRANCH_NAME)*.zip + +validate-branch-name: +ifndef BRANCH_NAME + $(error BRANCH_NAME is undefined) +endif + +validate-prefix-arn-file: +ifndef PREFIX_ARN_FILE + $(error PREFIX_ARN_FILE is undefined) +endif + +validate-layer-name: +ifndef ELASTIC_LAYER_NAME + $(error ELASTIC_LAYER_NAME is undefined) +endif + +validate-aws-default-region: +ifndef AWS_DEFAULT_REGION + $(error AWS_DEFAULT_REGION is undefined) +endif From e5b400c1e6d8bbee125eb5ceb9a763df02b0f4ad Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 21:37:02 +0000 Subject: [PATCH 20/41] chore: for testing purposes let's use the forked repo --- apm-lambda-extension/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index c4b51a6c..6529fbe2 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -73,7 +73,8 @@ release-notes: validate-branch-name validate-prefix-arn-file --draft \ --generate-notes \ --notes-file $(PREFIX_ARN_FILE) \ - ./bin/$(BRANCH_NAME)*.zip + ./bin/$(BRANCH_NAME)*.zip \ + --repo v1v/apm-aws-lambda validate-branch-name: ifndef BRANCH_NAME From 254c939166c39d6c540f9b40d34ffec99bf610c7 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 21:52:53 +0000 Subject: [PATCH 21/41] docs: update the automated release process --- apm-lambda-extension/DEVELOPMENT.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/apm-lambda-extension/DEVELOPMENT.md b/apm-lambda-extension/DEVELOPMENT.md index 95bd15d7..f2199203 100644 --- a/apm-lambda-extension/DEVELOPMENT.md +++ b/apm-lambda-extension/DEVELOPMENT.md @@ -1,15 +1,11 @@ ## Releasing -Releasing a version of the Lambda Extension is currently a three step manual process. - -1. Tag the Release -2. Create the Build Artifacts -3. Add a Release via the Github UI +### :robot: Automatically -### Tag the Release +Releasing a version of the Lambda Extension requires a tag release. -First, tag the release via your preferred tagging method. Tagging a release (v0.0.2) via the command line looks something like this. +Tag the release via your preferred tagging method. Tagging a release (v0.0.2) via the command line looks something like this. % git clone git@github.com:elastic/apm-aws-lambda.git # ... @@ -21,7 +17,20 @@ First, tag the release via your preferred tagging method. Tagging a release (v0 To github.com:elastic/apm-aws-lambda.git * [new tag] v0.0.2 -> v0.0.2 +This will trigger a build in the CI that will create the Build Artifacts +and a Release in the Github UI. + +### :thumbsdown: Manually + +Releasing a version of the Lambda Extension is currently a three step manual process. + +1. Tag the Release +2. Create the Build Artifacts +3. Add a Release via the Github UI + +### Tag the Release +See the above section regarding tagging a release. ### Create the Build Artifacts Next, create the build artifacts for the release. These are go binaries of the Lambda Extension, built for both Intel and ARM architectures. From 79315070f64841fb02e20104af7a389ed7849e98 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 22:06:17 +0000 Subject: [PATCH 22/41] make: mv once only --- apm-lambda-extension/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 6529fbe2..fa1b7954 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -42,6 +42,7 @@ get-all-aws-regions: # Publish the given LAYER in all the AWS regions publish-in-all-aws-regions: validate-layer-name get-all-aws-regions + @mv ./bin/$(GOARCH).zip bin/$(BRANCH_NAME)-linux-$(GOARCH).zip @while read AWS_DEFAULT_REGION; do \ AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish; \ done <.regions @@ -51,7 +52,6 @@ publish-in-all-aws-regions: validate-layer-name get-all-aws-regions publish: validate-branch-name validate-layer-name validate-aws-default-region @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' @mkdir -p $(AWS_FOLDER) - @mv bin/$(GOARCH).zip bin/$(BRANCH_NAME)-linux-$(GOARCH).zip @aws lambda \ --output json \ publish-layer-version \ From f8d1173aa3f85440b731250648bd8773c98e5613 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 22:14:11 +0000 Subject: [PATCH 23/41] ci: GOARCH is not defined when publishing but needed Let's delegate this to the Go installer --- .ci/Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index fcbc6bb5..8f950736 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -133,10 +133,12 @@ pipeline { steps { withGithubNotify(context: "Publish-${PLATFORM}") { withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { - dir("${BASE_DIR}/apm-lambda-extension"){ - cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') - stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") - stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") + withGoEnv(){ + dir("${BASE_DIR}/apm-lambda-extension"){ + cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') + stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") + stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") + } } } } From d051cbb85d895e6f90503ccc0e1d476666856a4e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 13 Jan 2022 22:14:51 +0000 Subject: [PATCH 24/41] ci: use the workspace after the build stage --- .ci/Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 8f950736..4236197d 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -79,8 +79,6 @@ pipeline { stage('Test') { steps { withGithubNotify(context: "Test-${GO_VERSION}-${PLATFORM}") { - deleteDir() - unstash 'source' withGoEnv(){ dir("${BASE_DIR}/apm-lambda-extension"){ goTestJUnit(options: '-v ./...', output: 'junit-report.xml') From 323d95985f849a3f5ffcbe60b05949e587b14ec4 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 14 Jan 2022 09:02:37 +0000 Subject: [PATCH 25/41] Revert "ci: GOARCH is not defined when publishing but needed" This reverts commit f8d1173aa3f85440b731250648bd8773c98e5613. --- .ci/Jenkinsfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 4236197d..b360c828 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -131,12 +131,10 @@ pipeline { steps { withGithubNotify(context: "Publish-${PLATFORM}") { withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { - withGoEnv(){ - dir("${BASE_DIR}/apm-lambda-extension"){ - cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') - stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") - stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") - } + dir("${BASE_DIR}/apm-lambda-extension"){ + cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') + stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") + stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") } } } From d9933976a073e6e2e171f041c6fd4ba5c0f5eac1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 14 Jan 2022 09:04:14 +0000 Subject: [PATCH 26/41] make: dist is the one in charge to prepare the arch binary --- apm-lambda-extension/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index fa1b7954..1a72255c 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -27,7 +27,8 @@ test: go test extension/*.go -v env: env -dist: build test zip +dist: validate-branch-name build test zip + @cp ./bin/$(GOARCH).zip bin/$(BRANCH_NAME)-linux-$(GOARCH).zip # List all the AWS regions get-all-aws-regions: @@ -42,7 +43,6 @@ get-all-aws-regions: # Publish the given LAYER in all the AWS regions publish-in-all-aws-regions: validate-layer-name get-all-aws-regions - @mv ./bin/$(GOARCH).zip bin/$(BRANCH_NAME)-linux-$(GOARCH).zip @while read AWS_DEFAULT_REGION; do \ AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish; \ done <.regions From cb676e701da9ef67877565ad240da8a3c43863e5 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 14 Jan 2022 09:37:27 +0000 Subject: [PATCH 27/41] debug: for testing purposes --- .ci/create-arn-table.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh index 0feb0aa5..d6e50f83 100755 --- a/.ci/create-arn-table.sh +++ b/.ci/create-arn-table.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -eo pipefail +set -exo pipefail # # Create the AWS ARN table given the below environment variables: From fd7f8578d560a8fef1e44480559c4a59f9918f92 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 14 Jan 2022 10:17:55 +0000 Subject: [PATCH 28/41] ci: create-are requires the goarch --- .ci/Jenkinsfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index b360c828..635d0045 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -130,11 +130,13 @@ pipeline { stage('Publish') { steps { withGithubNotify(context: "Publish-${PLATFORM}") { - withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { - dir("${BASE_DIR}/apm-lambda-extension"){ - cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') - stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") - stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") + withGoEnv(){ + withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { + dir("${BASE_DIR}/apm-lambda-extension"){ + cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') + stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") + stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") + } } } } From 5e563f42073536e792a6975261a16319742966b1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 14 Jan 2022 10:18:01 +0000 Subject: [PATCH 29/41] debug: for testing purposes --- apm-lambda-extension/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 1a72255c..545d9406 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -65,7 +65,7 @@ create-arn-file: validate-branch-name validate-prefix-arn-file release-notes: validate-branch-name validate-prefix-arn-file @gh release list - @cat *-$(PREFIX_ARN_FILE) > $(PREFIX_ARN_FILE) + cat *-$(PREFIX_ARN_FILE) > $(PREFIX_ARN_FILE) @gh \ release \ create $(BRANCH_NAME) \ From 5deb100f167de2a45ff14aaad15359506a0df760 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 21:23:54 +0000 Subject: [PATCH 30/41] publish goal does not require branch validation --- apm-lambda-extension/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 545d9406..fbb3d6e9 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -49,7 +49,7 @@ publish-in-all-aws-regions: validate-layer-name get-all-aws-regions $(MAKE) create-arn-file # Publish the given LAYER in the given AWS region -publish: validate-branch-name validate-layer-name validate-aws-default-region +publish: validate-layer-name validate-aws-default-region @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' @mkdir -p $(AWS_FOLDER) @aws lambda \ From 19c9bc24f3a636f08ee9e05105bc8b70f49768c7 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 21:33:02 +0000 Subject: [PATCH 31/41] Add license and description fields --- apm-lambda-extension/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index fbb3d6e9..a38dd5fb 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -56,6 +56,8 @@ publish: validate-layer-name validate-aws-default-region --output json \ publish-layer-version \ --layer-name "${ELASTIC_LAYER_NAME}" \ + --description "AWS Lambda Extension Layer for Elastic APM" \ + --license "Apache-2.0" \ --zip-file "fileb://./bin/extension.zip" \ --generate-cli-skeleton > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} From 900d51fad883bb479e7e2b64e4279dc591ad2054 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 21:35:20 +0000 Subject: [PATCH 32/41] use the LayerVersionArn field --- .ci/create-arn-table.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh index d6e50f83..e105e4dc 100755 --- a/.ci/create-arn-table.sh +++ b/.ci/create-arn-table.sh @@ -16,7 +16,7 @@ set -exo pipefail echo '|------|----|---|' for f in $(ls "${AWS_FOLDER}"); do # TODO: identify what field to be used. - echo "|${f}|${GOARCH}|$(cat $AWS_FOLDER/${f} | jq -r .LayerName)|" + echo "|${f}|${GOARCH}|$(cat $AWS_FOLDER/${f} | jq -r .LayerVersionArn)|" done echo '' } > ${GOARCH}-${PREFIX_ARN_FILE} From a8a54419d33c9894940d28eeb5185ee8a927d5e8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 21:45:59 +0000 Subject: [PATCH 33/41] Use a different arch --- .ci/create-arn-table.sh | 8 ++++---- apm-lambda-extension/Makefile | 9 ++++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh index e105e4dc..34a87097 100755 --- a/.ci/create-arn-table.sh +++ b/.ci/create-arn-table.sh @@ -5,18 +5,18 @@ set -exo pipefail # Create the AWS ARN table given the below environment variables: # # - AWS_FOLDER - that's the location of the publish-layer-version output for each region -# - GOARCH - that's the supported architecture. +# - ARCHITECTURE - that's the supported architecture. # - PREFIX_ARN_FILE - that's the output file. # { - echo "### ARCH: ${GOARCH}" + echo "### ARCH: ${ARCHITECTURE}" echo '' echo '|Region|Arch|ARN|' echo '|------|----|---|' for f in $(ls "${AWS_FOLDER}"); do # TODO: identify what field to be used. - echo "|${f}|${GOARCH}|$(cat $AWS_FOLDER/${f} | jq -r .LayerVersionArn)|" + echo "|${f}|${ARCHITECTURE}|$(cat $AWS_FOLDER/${f} | jq -r .LayerVersionArn)|" done echo '' -} > ${GOARCH}-${PREFIX_ARN_FILE} +} > ${ARCHITECTURE}-${PREFIX_ARN_FILE} diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index a38dd5fb..c2e3c7fb 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -6,7 +6,14 @@ ifndef GOARCH GOARCH=amd64 endif -export AWS_FOLDER GOARCH +# Transform GOARCH into the architecture of the extension layer +ifeq ($(GOARCH),amd64) + ARCHITECTURE=x86_64 +else + ARCHITECTURE=arm64 +endif + +export AWS_FOLDER GOARCH ARCHITECTURE build: GOOS=linux go build -o bin/extensions/apm-lambda-extension main.go From 669105b0eafb6c3e390b722df54cffa25993c346 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 21:46:17 +0000 Subject: [PATCH 34/41] unrequired validation --- apm-lambda-extension/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index c2e3c7fb..762aec44 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -69,7 +69,7 @@ publish: validate-layer-name validate-aws-default-region --generate-cli-skeleton > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} # Generate the file with the ARN entries -create-arn-file: validate-branch-name validate-prefix-arn-file +create-arn-file: validate-prefix-arn-file @../.ci/create-arn-table.sh release-notes: validate-branch-name validate-prefix-arn-file From b93bc140b1aa9fe4b9ab6a1efe90ca1334433520 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 21:48:17 +0000 Subject: [PATCH 35/41] use suffix since it's the correct value --- .ci/Jenkinsfile | 4 ++-- .ci/create-arn-table.sh | 4 ++-- apm-lambda-extension/Makefile | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 635d0045..ffbeca52 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -8,7 +8,7 @@ pipeline { REPO = 'apm-aws-lambda' BASE_DIR = "src/github.com/elastic/${env.REPO}" PIPELINE_LOG_LEVEL = 'INFO' - PREFIX_ARN_FILE = 'arn-file.md' + SUFFIX_ARN_FILE = 'arn-file.md' ELASTIC_LAYER_NAME = 'apm-lambda-extension' } options { @@ -134,7 +134,7 @@ pipeline { withAWSEnv(secret: 'secret/observability-team/ci/service-account/apm-aws-lambda', forceInstallation: true, version: '2.4.10') { dir("${BASE_DIR}/apm-lambda-extension"){ cmd(label: 'make publish-in-all-aws-regions', script: 'make publish-in-all-aws-regions') - stash(includes: "*${PREFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") + stash(includes: "*${SUFFIX_ARN_FILE}", name: "arn-${isArm() ? 'arm' : 'amd'}") stash(includes: "bin/${BRANCH_NAME}-*.zip", name: "dist-${isArm() ? 'arm' : 'amd'}") } } diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh index 34a87097..cffba9e7 100755 --- a/.ci/create-arn-table.sh +++ b/.ci/create-arn-table.sh @@ -6,7 +6,7 @@ set -exo pipefail # # - AWS_FOLDER - that's the location of the publish-layer-version output for each region # - ARCHITECTURE - that's the supported architecture. -# - PREFIX_ARN_FILE - that's the output file. +# - SUFFIX_ARN_FILE - that's the output file. # { @@ -19,4 +19,4 @@ set -exo pipefail echo "|${f}|${ARCHITECTURE}|$(cat $AWS_FOLDER/${f} | jq -r .LayerVersionArn)|" done echo '' -} > ${ARCHITECTURE}-${PREFIX_ARN_FILE} +} > ${ARCHITECTURE}-${SUFFIX_ARN_FILE} diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 762aec44..db1321f7 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -69,19 +69,19 @@ publish: validate-layer-name validate-aws-default-region --generate-cli-skeleton > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} # Generate the file with the ARN entries -create-arn-file: validate-prefix-arn-file +create-arn-file: validate-suffix-arn-file @../.ci/create-arn-table.sh -release-notes: validate-branch-name validate-prefix-arn-file +release-notes: validate-branch-name validate-suffix-arn-file @gh release list - cat *-$(PREFIX_ARN_FILE) > $(PREFIX_ARN_FILE) + cat *-$(SUFFIX_ARN_FILE) > $(SUFFIX_ARN_FILE) @gh \ release \ create $(BRANCH_NAME) \ --title '$(BRANCH_NAME)' \ --draft \ --generate-notes \ - --notes-file $(PREFIX_ARN_FILE) \ + --notes-file $(SUFFIX_ARN_FILE) \ ./bin/$(BRANCH_NAME)*.zip \ --repo v1v/apm-aws-lambda @@ -90,9 +90,9 @@ ifndef BRANCH_NAME $(error BRANCH_NAME is undefined) endif -validate-prefix-arn-file: -ifndef PREFIX_ARN_FILE - $(error PREFIX_ARN_FILE is undefined) +validate-suffix-arn-file: +ifndef SUFFIX_ARN_FILE + $(error SUFFIX_ARN_FILE is undefined) endif validate-layer-name: From 63a925e65d36a03509f2c39c8bec29441548e4af Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 22:53:47 +0000 Subject: [PATCH 36/41] enable when tag is created --- .ci/Jenkinsfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index ffbeca52..877aadaa 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -97,10 +97,9 @@ pipeline { } stage('Release') { options { skipDefaultCheckout() } - // TODO: for testing purposes - //when { - // tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP' - //} + when { + tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP' + } stages { stage('BuildPublish') { failFast false From 65a7124eb5e5df7ec6ba5b7aec077663968a63ca Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 22:55:02 +0000 Subject: [PATCH 37/41] dry-run disabled --- apm-lambda-extension/Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index db1321f7..5b4189ff 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -65,8 +65,7 @@ publish: validate-layer-name validate-aws-default-region --layer-name "${ELASTIC_LAYER_NAME}" \ --description "AWS Lambda Extension Layer for Elastic APM" \ --license "Apache-2.0" \ - --zip-file "fileb://./bin/extension.zip" \ - --generate-cli-skeleton > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} + --zip-file "fileb://./bin/extension.zip" > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} # Generate the file with the ARN entries create-arn-file: validate-suffix-arn-file @@ -79,11 +78,9 @@ release-notes: validate-branch-name validate-suffix-arn-file release \ create $(BRANCH_NAME) \ --title '$(BRANCH_NAME)' \ - --draft \ --generate-notes \ --notes-file $(SUFFIX_ARN_FILE) \ - ./bin/$(BRANCH_NAME)*.zip \ - --repo v1v/apm-aws-lambda + ./bin/$(BRANCH_NAME)*.zip validate-branch-name: ifndef BRANCH_NAME From f19713558a46b956f807f65f1c5a800b116512d6 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 22:55:35 +0000 Subject: [PATCH 38/41] no more debugging --- .ci/create-arn-table.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/create-arn-table.sh b/.ci/create-arn-table.sh index cffba9e7..65ccae60 100755 --- a/.ci/create-arn-table.sh +++ b/.ci/create-arn-table.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -exo pipefail +set -eo pipefail # # Create the AWS ARN table given the below environment variables: From f5d444820faf95a88192f33490f8a2e083597651 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 22:58:09 +0000 Subject: [PATCH 39/41] aws: no all regions support the aws lambda publising A misleading error when running the publish - PublishLayerVersion operation: The security token included in the request is invalid. --- apm-lambda-extension/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 5b4189ff..def7eaad 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -43,7 +43,6 @@ get-all-aws-regions: ec2 \ describe-regions \ --region us-east-1 \ - --all-regions \ --output json \ --no-cli-pager \ | jq -r '.Regions[].RegionName' > .regions From 05919ed35e0711775392cf79a4743266bc5b3d3f Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 26 Jan 2022 23:01:57 +0000 Subject: [PATCH 40/41] enable --compatible-architectures --- apm-lambda-extension/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index def7eaad..091b1907 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -64,6 +64,7 @@ publish: validate-layer-name validate-aws-default-region --layer-name "${ELASTIC_LAYER_NAME}" \ --description "AWS Lambda Extension Layer for Elastic APM" \ --license "Apache-2.0" \ + --compatible-architectures "$(ARCHITECTURE)" \ --zip-file "fileb://./bin/extension.zip" > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} # Generate the file with the ARN entries From 1b1d08e43e0e3f16074103cad68493a7bdf0f84e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 27 Jan 2022 20:00:37 +0000 Subject: [PATCH 41/41] keep compatibility with the apm-lambda-extension/cli/build-and-publish.js --- apm-lambda-extension/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apm-lambda-extension/Makefile b/apm-lambda-extension/Makefile index 091b1907..7744b64b 100644 --- a/apm-lambda-extension/Makefile +++ b/apm-lambda-extension/Makefile @@ -49,15 +49,15 @@ get-all-aws-regions: # Publish the given LAYER in all the AWS regions publish-in-all-aws-regions: validate-layer-name get-all-aws-regions + @mkdir -p $(AWS_FOLDER) @while read AWS_DEFAULT_REGION; do \ - AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish; \ + echo "publish '$(ELASTIC_LAYER_NAME)' in $${AWS_DEFAULT_REGION}"; \ + AWS_DEFAULT_REGION="$${AWS_DEFAULT_REGION}" ELASTIC_LAYER_NAME=$(ELASTIC_LAYER_NAME) $(MAKE) publish > $(AWS_FOLDER)/$${AWS_DEFAULT_REGION}; \ done <.regions $(MAKE) create-arn-file # Publish the given LAYER in the given AWS region publish: validate-layer-name validate-aws-default-region - @echo 'publish "${ELASTIC_LAYER_NAME}" in ${AWS_DEFAULT_REGION}' - @mkdir -p $(AWS_FOLDER) @aws lambda \ --output json \ publish-layer-version \ @@ -65,7 +65,7 @@ publish: validate-layer-name validate-aws-default-region --description "AWS Lambda Extension Layer for Elastic APM" \ --license "Apache-2.0" \ --compatible-architectures "$(ARCHITECTURE)" \ - --zip-file "fileb://./bin/extension.zip" > $(AWS_FOLDER)/${AWS_DEFAULT_REGION} + --zip-file "fileb://./bin/extension.zip" # Generate the file with the ARN entries create-arn-file: validate-suffix-arn-file