From 653daa665875249d1f935bd70468b620ae9800d9 Mon Sep 17 00:00:00 2001 From: Azfaar Qureshi Date: Thu, 8 Oct 2020 13:27:40 -0400 Subject: [PATCH 1/3] Added integration and integration-config-db jobs to Github Actions workflow Signed-off-by: Azfaar Qureshi Signed-off-by: Shovnik Bhattacharya adding table to README and removing vestigial lines from workflow Signed-off-by: Azfaar Qureshi Added integration and integration-configs-db jobs Signed-off-by: Azfaar Qureshi reading from cortexproject Signed-off-by: Azfaar Qureshi Made Step Naming Consistent Signed-off-by: Shovnik Bhattacharya read tag only if they exist on the push event updating quay image removing old README changing to v2 Addressed changes requested in PR 1/3 --- .github/workflows-doc.md | 4 -- .github/workflows/test-build-deploy.yml | 95 ++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/.github/workflows-doc.md b/.github/workflows-doc.md index 8384e6014cd..d38fa97c53e 100644 --- a/.github/workflows-doc.md +++ b/.github/workflows-doc.md @@ -62,10 +62,6 @@ As of October 2020, GitHub Actions do not persist between different jobs in the |-------------------------------|-----------|---------------------------------------------|-----------------------------| | website public | build | deploy_website | share data between jobs | | Docker Images | build | deploy, integration, integrations-config-db | share data between jobs | -| Frontend Protobuf | build | | long term storage | -| Caching Index Client Protobuf | build | | long term storage | -| Ring Protobuf | build | | long term storage | -| Rules Protobuf | build | | long term storage | *Note:* Docker Images are zipped before uploading as a workaround. The images contain characters that are illegal in the upload-artifact action. ```yaml diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index bbf0a4a6c94..622649d7937 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -13,6 +13,8 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v2 + # Commands in the Makefile are hardcoded with an assumed file structure of the CI container + # Symlink ensures paths specified in previous commands don’t break - name: Sym Link Expected Path to Workspace run: | mkdir -p /go/src/github.com/cortexproject/cortex @@ -80,10 +82,97 @@ jobs: mkdir /tmp/images ln -s /tmp/images ./docker-images make BUILD_IN_CONTAINER=false save-images - - name: Zip Images - run: tar -zcvf images.tar.gz /tmp/images - - name: Upload Images Artifact + - name: Create Docker Images Archive + run: tar -cvf images.tar.gz /tmp/images + - name: Upload Docker Images Artifact uses: actions/upload-artifact@v2 with: name: Docker Images path: ./images.tar.gz + + integration: + needs: build + runs-on: ubuntu-16.04 + steps: + - name: Install Docker Client + run: | + set -x + VER="17.03.0-ce" + curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz + tar -xz -C /tmp -f /tmp/docker-$VER.tgz + sudo mv /tmp/docker/* /usr/bin + - name: Upgrade golang + run: | + cd /tmp + wget https://dl.google.com/go/go1.14.9.linux-amd64.tar.gz + tar -zxvf go1.14.9.linux-amd64.tar.gz + sudo rm -fr /usr/local/go + sudo mv /tmp/go /usr/local/go + cd - + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Sym Link Expected Path to Workspace + run: | + sudo mkdir -p /go/src/github.com/cortexproject/cortex + sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex + - name: Download Docker Images Artifacts + uses: actions/download-artifact@v2 + with: + name: Docker Images + - name: Create Docker Images Archive + run: tar -xvf images.tar.gz -C / + - name: Load Docker Images + run: | + ln -s /tmp/images ./docker-images + make BUILD_IN_CONTAINER=false load-images + - name: Preload Images + # We download docker images used by integration tests so that all images are available + # locally and the download time doesn't account in the test execution time, which is subject + # to a timeout + run: | + docker pull minio/minio:RELEASE.2019-12-30T05-45-39Z + docker pull amazon/dynamodb-local:1.11.477 + docker pull consul:0.9 + docker pull gcr.io/etcd-development/etcd:v3.4.7 + docker pull quay.io/cortexproject/cortex:v0.6.0 + docker pull quay.io/cortexproject/cortex:v0.7.0 + docker pull quay.io/cortexproject/cortex:v1.0.0 + docker pull quay.io/cortexproject/cortex:v1.1.0 + docker pull shopify/bigtable-emulator:0.1.0 + docker pull rinscy/cassandra:3.11.0 + docker pull memcached:1.6.1 + docker pull bouncestorage/swift-aio:55ba4331 + - name: Integration Tests + run: | + export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}" + export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:${TAG:-$(./tools/image-tag)}" + export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex" + echo "Running integration tests with image: $CORTEX_IMAGE" + go test -tags=requires_docker -timeout 1200s -v -count=1 ./integration/... + env: + IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }} + TAG: ${{ github.event.push.tag_name }} + integration-configs-db: + needs: build + runs-on: ubuntu-16.04 + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Install Docker Client + run: | + set -x + VER="17.03.0-ce" + curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz + tar -xz -C /tmp -f /tmp/docker-$VER.tgz + sudo mv /tmp/docker/* /usr/bin + - name: Download Docker Images Artifact + uses: actions/download-artifact@v2 + with: + name: Docker Images + - name: Extract Docker Images Archive + run: tar -xvf images.tar.gz -C / + - name: Run Integration Configs Tests + run: | + touch build-image/.uptodate + MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations + make BUILD_IMAGE=quay.io/cortexproject/build-image:upgrade-build-image-debian-491e60715-WIP TTY='' configs-integration-test From 778797e0b6dc2e0ab03e87f2f581dcd5b2fecdf3 Mon Sep 17 00:00:00 2001 From: Shovnik Bhattacharya Date: Thu, 15 Oct 2020 13:39:31 -0400 Subject: [PATCH 2/3] Made changes suggested by pracucci in review Signed-off-by: Shovnik Bhattacharya --- .github/workflows/test-build-deploy.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 622649d7937..9911d19ab51 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -83,12 +83,12 @@ jobs: ln -s /tmp/images ./docker-images make BUILD_IN_CONTAINER=false save-images - name: Create Docker Images Archive - run: tar -cvf images.tar.gz /tmp/images + run: tar -cvf images.tar /tmp/images - name: Upload Docker Images Artifact uses: actions/upload-artifact@v2 with: name: Docker Images - path: ./images.tar.gz + path: ./images.tar integration: needs: build @@ -119,8 +119,8 @@ jobs: uses: actions/download-artifact@v2 with: name: Docker Images - - name: Create Docker Images Archive - run: tar -xvf images.tar.gz -C / + - name: Extract Docker Images Archive + run: tar -xvf images.tar -C / - name: Load Docker Images run: | ln -s /tmp/images ./docker-images @@ -134,10 +134,11 @@ jobs: docker pull amazon/dynamodb-local:1.11.477 docker pull consul:0.9 docker pull gcr.io/etcd-development/etcd:v3.4.7 - docker pull quay.io/cortexproject/cortex:v0.6.0 - docker pull quay.io/cortexproject/cortex:v0.7.0 docker pull quay.io/cortexproject/cortex:v1.0.0 docker pull quay.io/cortexproject/cortex:v1.1.0 + docker pull quay.io/cortexproject/cortex:v1.2.0 + docker pull quay.io/cortexproject/cortex:v1.3.0 + docker pull quay.io/cortexproject/cortex:v1.4.0 docker pull shopify/bigtable-emulator:0.1.0 docker pull rinscy/cassandra:3.11.0 docker pull memcached:1.6.1 @@ -148,7 +149,7 @@ jobs: export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:${TAG:-$(./tools/image-tag)}" export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex" echo "Running integration tests with image: $CORTEX_IMAGE" - go test -tags=requires_docker -timeout 1200s -v -count=1 ./integration/... + go test -tags=requires_docker -timeout 1800s -v -count=1 ./integration/... env: IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }} TAG: ${{ github.event.push.tag_name }} @@ -170,7 +171,7 @@ jobs: with: name: Docker Images - name: Extract Docker Images Archive - run: tar -xvf images.tar.gz -C / + run: tar -xvf images.tar -C / - name: Run Integration Configs Tests run: | touch build-image/.uptodate From 5934d49276fd10aa19a5260e29ea2e8ef1586f45 Mon Sep 17 00:00:00 2001 From: Shovnik Bhattacharya Date: Thu, 15 Oct 2020 14:02:33 -0400 Subject: [PATCH 3/3] Moved docker installation to script Signed-off-by: Shovnik Bhattacharya --- .github/workflows/scripts/install-docker.sh | 7 ++++++ .github/workflows/test-build-deploy.yml | 25 +++++---------------- 2 files changed, 13 insertions(+), 19 deletions(-) create mode 100755 .github/workflows/scripts/install-docker.sh diff --git a/.github/workflows/scripts/install-docker.sh b/.github/workflows/scripts/install-docker.sh new file mode 100755 index 00000000000..45f845a428c --- /dev/null +++ b/.github/workflows/scripts/install-docker.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x +VER="17.03.0-ce" +curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz +tar -xz -C /tmp -f /tmp/docker-$VER.tgz +mv /tmp/docker/* /usr/bin diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 9911d19ab51..0547e11d133 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -5,6 +5,7 @@ on: tags: - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions pull_request: + jobs: lint: runs-on: ubuntu-latest @@ -59,12 +60,7 @@ jobs: - name: Checkout Repo uses: actions/checkout@v2 - name: Install Docker Client - run: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - mv /tmp/docker/* /usr/bin + run: ./.github/workflows/scripts/install-docker.sh - name: Sym Link Expected Path to Workspace run: | mkdir -p /go/src/github.com/cortexproject/cortex @@ -94,13 +90,6 @@ jobs: needs: build runs-on: ubuntu-16.04 steps: - - name: Install Docker Client - run: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - sudo mv /tmp/docker/* /usr/bin - name: Upgrade golang run: | cd /tmp @@ -111,6 +100,8 @@ jobs: cd - - name: Checkout Repo uses: actions/checkout@v2 + - name: Install Docker Client + run: sudo ./.github/workflows/scripts/install-docker.sh - name: Sym Link Expected Path to Workspace run: | sudo mkdir -p /go/src/github.com/cortexproject/cortex @@ -153,6 +144,7 @@ jobs: env: IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }} TAG: ${{ github.event.push.tag_name }} + integration-configs-db: needs: build runs-on: ubuntu-16.04 @@ -160,12 +152,7 @@ jobs: - name: Checkout Repo uses: actions/checkout@v2 - name: Install Docker Client - run: | - set -x - VER="17.03.0-ce" - curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - sudo mv /tmp/docker/* /usr/bin + run: sudo ./.github/workflows/scripts/install-docker.sh - name: Download Docker Images Artifact uses: actions/download-artifact@v2 with: