From 13c8026e4df9bfd0e75b152b25d64fc2f22ec7db Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Sat, 23 Oct 2021 15:15:28 +0200 Subject: [PATCH 1/3] build: setup preview builds for dev-app Sets up preview builds for the dev-app. Whenever the `dev-app preview` label is applied to pull requests, a Github action will build the dev-app using RBE and deploy it to a preview channel within a Firebase project. The deployment and building is split up into two individual workflows to guarantee a secure exeuction of these steps. This follows the concept as outlined in https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. In the future, we can try extracting some of this logic into a common tool in the dev-infra repository.. allowing preview builds to be used for other things, or in other repositories as well (or switching AIO away from the rather-complicated docker preview build setup). --- .circleci/config.yml | 2 +- .github/actions/yarn-install/action.yml | 18 ++++++ .github/workflows/build-dev-app.yml | 50 +++++++++++++++++ .github/workflows/deploy-dev-app.yml | 52 ++++++++++++++++++ {.circleci => scripts/bazel}/gcp_token | Bin scripts/bazel/setup-remote-execution.sh | 28 ++++++++++ .../circleci/bazel/setup-remote-execution.sh | 22 -------- scripts/github/fetch-workflow-artifact.js | 40 ++++++++++++++ 8 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 .github/actions/yarn-install/action.yml create mode 100644 .github/workflows/build-dev-app.yml create mode 100644 .github/workflows/deploy-dev-app.yml rename {.circleci => scripts/bazel}/gcp_token (100%) create mode 100755 scripts/bazel/setup-remote-execution.sh delete mode 100755 scripts/circleci/bazel/setup-remote-execution.sh create mode 100755 scripts/github/fetch-workflow-artifact.js diff --git a/.circleci/config.yml b/.circleci/config.yml index a399d0415696..0024b90e3fe8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,7 +103,7 @@ var_15: &ignore_presubmit_branch_filter var_16: &setup_bazel_remote_execution run: name: "Setup bazel RBE remote execution" - command: ./scripts/circleci/bazel/setup-remote-execution.sh + command: ./scripts/bazel/setup-remote-execution.sh # Sets up the bazel binary globally. We don't want to access bazel through Yarn and NodeJS # because it could mean that the Bazel child process only has access to limited memory. diff --git a/.github/actions/yarn-install/action.yml b/.github/actions/yarn-install/action.yml new file mode 100644 index 000000000000..25e654d125e6 --- /dev/null +++ b/.github/actions/yarn-install/action.yml @@ -0,0 +1,18 @@ +name: "Installing Yarn dependencies" +description: "Installs the dependencies using Yarn" + +runs: + using: "composite" + steps: + - uses: actions/cache@v2 + with: + path: | + **/node_modules + # Cache key. Whenever the postinstall patches change, the cache needs to be invalidated. + # If just the `yarn.lock` file changes, the most recent cache can be restored though. + # See: https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#example-using-the-cache-action. + key: v2-${{hashFiles('tools/postinstall/apply-patches.js')}}-${{hashFiles('yarn.lock')}} + restore-keys: v2-${{hashFiles('tools/postinstall/apply-patches.js')}}- + + - run: yarn install --frozen-lockfile --non-interactive + shell: bash diff --git a/.github/workflows/build-dev-app.yml b/.github/workflows/build-dev-app.yml new file mode 100644 index 000000000000..6e0337d207c3 --- /dev/null +++ b/.github/workflows/build-dev-app.yml @@ -0,0 +1,50 @@ +# This workflow builds the dev-app for pull requests when a certain label is applied. +# The actual deployment happens as part of a dedicated second workflow to avoid security +# issues where the building would otherwise occur in an authorized context where secrets +# could be leaked. More details can be found here: + +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. + +name: Build dev-app for deployment + +on: + pull_request: + types: [synchronize, labeled] + +jobs: + dev-app-build: + runs-on: ubuntu-latest + # We only want to build and deploy the dev-app if the `dev-app preview` label has been + # added, or if the label is already applied and new changes have been made in the PR. + if: | + (github.event.action == 'labeled' && github.event.label.name == 'dev-app preview') || + (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dev-app preview')) + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/yarn-install + + - run: ./scripts/bazel/setup-remote-execution.sh + env: + GCP_DECRYPT_TOKEN: angular + + # Build the web package. Note that we also need to make the Github environment + # variables available so that the RBE is configured. + - name: Building dev-app + run: | + source ${GITHUB_ENV} + bazel build //src/dev-app:web_package --symlink_prefix=dist/ + + # Prepare the workflow artifact that is available for the deploy workflow. We store the pull + # request number and SHA in a file that can be read by the deploy workflow. This is necessary + # so that the deploy workflow can create a comment on the PR that triggered the deploy. + - run: | + mkdir -p dist/devapp + cp -R dist/bin/src/dev-app/web_package/* dist/devapp + echo ${{github.event.pull_request.number}} > dist/devapp/pr_number + echo ${{github.event.pull_request.head.sha}} > dist/devapp/pr_sha + + # Upload the generated dev-app archive. + - uses: actions/upload-artifact@v2 + with: + name: devapp + path: dist/devapp diff --git a/.github/workflows/deploy-dev-app.yml b/.github/workflows/deploy-dev-app.yml new file mode 100644 index 000000000000..d422dad15f58 --- /dev/null +++ b/.github/workflows/deploy-dev-app.yml @@ -0,0 +1,52 @@ +# This workflow runs whenever the dev-app build workflow has completed. Deployment happens +# as part of a dedicated second workflow to avoid security issues where the building would +# otherwise occur in an authorized context where secrets could be leaked. +# +# More details can be found here: +# https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. + +name: Deploying dev-app to firebase previews + +on: + workflow_run: + workflows: [Build dev-app for deployment] + types: [completed] + +jobs: + deploy-dev-app: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - uses: actions/checkout@v2 + - uses: ./.github/actions/yarn-install + + - name: 'Download artifact from build job' + run: | + ./scripts/github/fetch-workflow-artifact.js ${{secrets.GITHUB_TOKEN}} \ + ${{github.event.workflow_run.id}} devapp > devapp.zip + + - name: Extracting workflow artifact into Firebase public directory. + run: | + mkdir -p dist/dev-app-web-pkg + unzip devapp.zip -d dist/dev-app-web-pkg + + - name: Extracting pull request from extracted workflow artifact. + id: pr_info + run: | + echo "::set-output name=number::$(cat ./dist/dev-app-web-pkg/pr_number)" + echo "::set-output name=sha::$(cat ./dist/dev-app-web-pkg/pr_sha)" + + - uses: FirebaseExtended/action-hosting-deploy@v0 + id: deploy + with: + repoToken: '${{secrets.GITHUB_TOKEN}}' + firebaseServiceAccount: '${{secrets.FIREBASE_PREVIEW_SERVICE_TOKEN}}' + expires: 20d + projectId: angular-components-test + channelId: pr-${{steps.pr_info.outputs.number}}-${{steps.pr_info.outputs.sha}} + + - uses: marocchino/sticky-pull-request-comment@v2 + with: + message: | + Deployed dev-app to: ${{ steps.deploy.outputs.details_url }} + number: ${{ steps.pr_info.outputs.number }} diff --git a/.circleci/gcp_token b/scripts/bazel/gcp_token similarity index 100% rename from .circleci/gcp_token rename to scripts/bazel/gcp_token diff --git a/scripts/bazel/setup-remote-execution.sh b/scripts/bazel/setup-remote-execution.sh new file mode 100755 index 000000000000..3c279fbe4356 --- /dev/null +++ b/scripts/bazel/setup-remote-execution.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# The script should immediately exit if any command in the script fails. +set -e + +if [[ -z "${GCP_DECRYPT_TOKEN}" ]]; then + echo "Please specify the \"GCP_DECRYPT_TOKEN\" environment variable when setting up remote " \ + "execution" + exit 1 +fi + +# Decode the GCP token that is needed to authenticate the Bazel remote execution. +openssl aes-256-cbc -d -in scripts/bazel/gcp_token -md md5 -k ${GCP_DECRYPT_TOKEN} \ + -out $HOME/.gcp_credentials + +# Set the "GOOGLE_APPLICATION_CREDENTIALS" environment variable. It should point to the GCP credentials +# file. Bazel will then automatically picks up the credentials from that variable. +# https://github.com/bazelbuild/bazel/blob/master/third_party/grpc/include/grpc/grpc_security.h#L134-L137 +if [[ ! -z "${BASH_ENV}" ]]; then + # CircleCI uses the `BASH_ENV` variable for environment variables. + echo "export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.gcp_credentials" >> ${BASH_ENV} +elif [[ ! -z "${GITHUB_ENV}" ]]; then + # Github actions use the `GITHUB_ENV` variable for environment variables. + echo "GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.gcp_credentials" >> ${GITHUB_ENV} +fi + +# Update the project Bazel configuration to always use remote execution. +echo "build --config=remote" >> .bazelrc diff --git a/scripts/circleci/bazel/setup-remote-execution.sh b/scripts/circleci/bazel/setup-remote-execution.sh deleted file mode 100755 index dfe9d2713609..000000000000 --- a/scripts/circleci/bazel/setup-remote-execution.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# The script should immediately exit if any command in the script fails. -set -e - -if [[ -z "${GCP_DECRYPT_TOKEN}" ]]; then - echo "Please specify the \"GCP_DECRYPT_TOKEN\" environment variable when setting up remote " \ - "execution" - exit 1 -fi - -# Decode the GCP token that is needed to authenticate the Bazel remote execution. -openssl aes-256-cbc -d -in .circleci/gcp_token -md md5 -k ${GCP_DECRYPT_TOKEN} \ - -out $HOME/.gcp_credentials - -# Export the "GOOGLE_APPLICATION_CREDENTIALS" variable that should refer to the GCP credentials -# file. Bazel automatically picks up the credentials from that variable. -# https://github.com/bazelbuild/bazel/blob/master/third_party/grpc/include/grpc/grpc_security.h#L134-L137 -echo "export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp_credentials" >> $BASH_ENV - -# Update the CircleCI Bazel configuration to always use remote execution. -echo "build --config=remote" >> .circleci/bazel.rc diff --git a/scripts/github/fetch-workflow-artifact.js b/scripts/github/fetch-workflow-artifact.js new file mode 100755 index 000000000000..71885d8ca183 --- /dev/null +++ b/scripts/github/fetch-workflow-artifact.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +/** + * Fetches a specified artifact by name from the given workflow and writes + * the downloaded zip file to the stdout. + * + * Command line usage: + * ./fetch-workflow-artifact.js + */ + +const {Octokit} = require('@octokit/rest'); + +async function main() { + const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/', 2); + const [token, workflowId, artifactName] = process.argv.slice(2); + const github = new Octokit({auth: token}); + const artifacts = await github.actions.listWorkflowRunArtifacts({ + owner, + repo, + run_id: workflowId, + }); + + const matchArtifact = artifacts.data.artifacts.filter( + artifact => artifact.name === artifactName, + )[0]; + + const download = await github.actions.downloadArtifact({ + owner, + repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + + process.stdout.write(Buffer.from(download.data)); +} + +main().catch(e => { + console.error(e); + process.exitCode = 1; +}); From b29a33d9a27e2f6b06455321e308324c4010548f Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Mon, 25 Oct 2021 15:02:39 +0200 Subject: [PATCH 2/3] fixup! build: setup preview builds for dev-app Address feedback --- .github/workflows/deploy-dev-app.yml | 4 ++-- ...flow-artifact.js => fetch-workflow-artifact.mjs} | 13 +++++-------- 2 files changed, 7 insertions(+), 10 deletions(-) rename scripts/github/{fetch-workflow-artifact.js => fetch-workflow-artifact.mjs} (77%) diff --git a/.github/workflows/deploy-dev-app.yml b/.github/workflows/deploy-dev-app.yml index d422dad15f58..db1ecc7ecc38 100644 --- a/.github/workflows/deploy-dev-app.yml +++ b/.github/workflows/deploy-dev-app.yml @@ -5,7 +5,7 @@ # More details can be found here: # https://securitylab.github.com/research/github-actions-preventing-pwn-requests/. -name: Deploying dev-app to firebase previews +name: Deploying dev-app to Firebase previews on: workflow_run: @@ -22,7 +22,7 @@ jobs: - name: 'Download artifact from build job' run: | - ./scripts/github/fetch-workflow-artifact.js ${{secrets.GITHUB_TOKEN}} \ + ./scripts/github/fetch-workflow-artifact.mjs ${{secrets.GITHUB_TOKEN}} \ ${{github.event.workflow_run.id}} devapp > devapp.zip - name: Extracting workflow artifact into Firebase public directory. diff --git a/scripts/github/fetch-workflow-artifact.js b/scripts/github/fetch-workflow-artifact.mjs similarity index 77% rename from scripts/github/fetch-workflow-artifact.js rename to scripts/github/fetch-workflow-artifact.mjs index 71885d8ca183..81efbf952c05 100755 --- a/scripts/github/fetch-workflow-artifact.js +++ b/scripts/github/fetch-workflow-artifact.mjs @@ -8,21 +8,21 @@ * ./fetch-workflow-artifact.js */ -const {Octokit} = require('@octokit/rest'); +import octokit from '@octokit/rest'; async function main() { const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/', 2); const [token, workflowId, artifactName] = process.argv.slice(2); - const github = new Octokit({auth: token}); + const github = new octokit.Octokit({auth: token}); const artifacts = await github.actions.listWorkflowRunArtifacts({ owner, repo, run_id: workflowId, }); - const matchArtifact = artifacts.data.artifacts.filter( + const matchArtifact = artifacts.data.artifacts.find( artifact => artifact.name === artifactName, - )[0]; + ); const download = await github.actions.downloadArtifact({ owner, @@ -34,7 +34,4 @@ async function main() { process.stdout.write(Buffer.from(download.data)); } -main().catch(e => { - console.error(e); - process.exitCode = 1; -}); +await main(); From 2f6ee6de1613452c3b4d0df61a0d28b83ee8f1a5 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Tue, 26 Oct 2021 22:14:16 +0200 Subject: [PATCH 3/3] fixup! build: setup preview builds for dev-app Update old links --- scripts/bazel/setup-remote-execution.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/bazel/setup-remote-execution.sh b/scripts/bazel/setup-remote-execution.sh index 3c279fbe4356..ebf6ed1d9d03 100755 --- a/scripts/bazel/setup-remote-execution.sh +++ b/scripts/bazel/setup-remote-execution.sh @@ -15,7 +15,8 @@ openssl aes-256-cbc -d -in scripts/bazel/gcp_token -md md5 -k ${GCP_DECRYPT_TOKE # Set the "GOOGLE_APPLICATION_CREDENTIALS" environment variable. It should point to the GCP credentials # file. Bazel will then automatically picks up the credentials from that variable. -# https://github.com/bazelbuild/bazel/blob/master/third_party/grpc/include/grpc/grpc_security.h#L134-L137 +# https://docs.bazel.build/versions/main/command-line-reference.html#flag--google_default_credentials +# https://cloud.google.com/docs/authentication/production. if [[ ! -z "${BASH_ENV}" ]]; then # CircleCI uses the `BASH_ENV` variable for environment variables. echo "export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/.gcp_credentials" >> ${BASH_ENV}