From f1898f7f050dd40ab9cd2c565674bfbe465487e8 Mon Sep 17 00:00:00 2001 From: DevOps Date: Tue, 30 Jan 2024 15:17:04 +0530 Subject: [PATCH 1/6] Add conductor server pipeline --- .github/workflows/workflow.yaml | 151 ++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .github/workflows/workflow.yaml diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml new file mode 100644 index 0000000000..7a959c00e2 --- /dev/null +++ b/.github/workflows/workflow.yaml @@ -0,0 +1,151 @@ +name: Static Hosting for DOCS + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose env + options: + - dev + - prd + +env: + SERVICE_NAME: conductor + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECR_REPOSITORY=$PROJECT_PREFIX-conductor-server" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Building and Pushing Image to ECR + build-push-image: + name: Build and Push Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + MIGRATE_IMAGE: ${{ steps.image.outputs.MIGRATE_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download S3 file + run: | + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties .docker/server/config/conductor-server.properties + + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.0.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }} + with: + context: . + file: ./Dockerfile + push: true + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + - name: image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ github.sha }}" >> $GITHUB_OUTPUT + echo "MIGRATE_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.MIGRATE_ECR_REPOSITORY }}:${{ github.sha }}" >> $GITHUB_OUTPUT + + \ No newline at end of file From 6950d6f468455cfddcf5fddd205bdba4a50bd848 Mon Sep 17 00:00:00 2001 From: DevOps Date: Tue, 30 Jan 2024 15:17:04 +0530 Subject: [PATCH 2/6] Add conductor server pipeline --- .github/workflows/workflow.yaml | 151 ++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .github/workflows/workflow.yaml diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml new file mode 100644 index 0000000000..09825da986 --- /dev/null +++ b/.github/workflows/workflow.yaml @@ -0,0 +1,151 @@ +name: Deploy OSS Conductor + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose env + options: + - dev + - prd + +env: + SERVICE_NAME: conductor + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECR_REPOSITORY=$PROJECT_PREFIX-conductor-server" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Building and Pushing Image to ECR + build-push-image: + name: Build and Push Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + MIGRATE_IMAGE: ${{ steps.image.outputs.MIGRATE_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download S3 file + run: | + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties .docker/server/config/conductor-server.properties + + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.0.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }} + with: + context: . + file: ./Dockerfile + push: true + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + - name: image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ github.sha }}" >> $GITHUB_OUTPUT + echo "MIGRATE_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.MIGRATE_ECR_REPOSITORY }}:${{ github.sha }}" >> $GITHUB_OUTPUT + + \ No newline at end of file From 04e646ef59131f5e31c94901a46da49dc983af3b Mon Sep 17 00:00:00 2001 From: DevOps Date: Tue, 30 Jan 2024 17:48:02 +0530 Subject: [PATCH 3/6] Add image tag --- .github/workflows/workflow.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index b0b0624778..296d87ceeb 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -27,7 +27,6 @@ jobs: outputs: AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} ENV: ${{ steps.vars.outputs.ENV }} - IMAGE_TAG: ${{ steps.vars.outputs.TAG }} PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} @@ -41,7 +40,6 @@ jobs: run: | BRANCH="${GITHUB_REF#refs/heads/}" ENV=${{ github.event.inputs.environment }} - TAG=${{ github.event.inputs.tag }} echo $BRANCH if [ -z "$ENV" ] @@ -103,6 +101,7 @@ jobs: PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + IMAGE_TAG: ${{ github.event.inputs.tag }} outputs: ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} From 2d973e0b51ecb8bea45cc45e72adc45c2a5b52e3 Mon Sep 17 00:00:00 2001 From: DevOps Date: Wed, 31 Jan 2024 11:13:18 +0530 Subject: [PATCH 4/6] Add conductor UI CI --- .github/workflows/ci.yml | 104 ---------------- .github/workflows/generate_gh_pages.yml | 18 --- .github/workflows/publish.yml | 40 ------- .github/workflows/release_draft.yml | 20 ---- .github/workflows/workflow.yaml | 150 ++++++++++++------------ 5 files changed, 77 insertions(+), 255 deletions(-) delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/generate_gh_pages.yml delete mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/release_draft.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index a25c97dd2e..0000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: CI - -on: [ push, pull_request ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - name: Gradle wrapper validation - uses: gradle/wrapper-validation-action@v1 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Cache SonarCloud packages - uses: actions/cache@v3 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - name: Build with Gradle - if: github.ref != 'refs/heads/main' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - ./gradlew build --scan - - name: Build and Publish snapshot - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - run: | - echo "Running build for commit ${{ github.sha }}" - ./gradlew build - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() - with: - report_paths: '**/build/test-results/test/TEST-*.xml' - - name: Upload build artifacts - uses: actions/upload-artifact@v3 - with: - name: build-artifacts - path: '**/build/reports' - - name: Store Buildscan URL - uses: actions/upload-artifact@v3 - with: - name: build-scan - path: 'buildscan.log' - build-ui: - runs-on: ubuntu-latest - container: cypress/browsers:node14.17.6-chrome100-ff98 - defaults: - run: - working-directory: ui - steps: - - uses: actions/checkout@v3 - - - name: Install Dependencies - run: yarn install - - - name: Build UI - run: yarn run build - - - name: Run E2E Tests - uses: cypress-io/github-action@v4 - with: - working-directory: ui - install: false - start: yarn run serve-build - wait-on: 'http://localhost:5000' - - - name: Run Component Tests - uses: cypress-io/github-action@v4 - with: - working-directory: ui - install: false - component: true - - - name: Archive test screenshots - uses: actions/upload-artifact@v2 - if: failure() - with: - name: cypress-screenshots - path: ui/cypress/screenshots - - - name: Archive test videos - uses: actions/upload-artifact@v2 - if: always() - with: - name: cypress-videos - path: ui/cypress/videos - diff --git a/.github/workflows/generate_gh_pages.yml b/.github/workflows/generate_gh_pages.yml deleted file mode 100644 index 8c429e1b8e..0000000000 --- a/.github/workflows/generate_gh_pages.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Publish docs via GitHub Pages -on: - workflow_dispatch - -jobs: - build: - name: Deploy docs - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v2 - - - name: Deploy docs - uses: mhausenblas/mkdocs-deploy-gh-pages@master - env: - GITHUB_TOKEN: ${{ secrets.DOCSITE_TOKEN }} - CONFIG_FILE: mkdocs.yml - REQUIREMENTS: requirements.txt diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 51b514ffa3..0000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Publish Conductor OSS toMaven Central -on: - release: - types: - - released - - prereleased - -permissions: - contents: read - -jobs: - publish: - runs-on: ubuntu-latest - name: Gradle Build and Publish - steps: - - uses: actions/checkout@v3 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Publish release - run: | - export VERSION="${{github.ref_name}}" - export PUBLISH_VERSION=`echo ${VERSION:1}` - echo Publishing version $PUBLISH_VERSION - ./gradlew publish -Pversion=$PUBLISH_VERSION -Pusername=${{ secrets.SONATYPE_USERNAME }} -Ppassword=${{ secrets.SONATYPE_PASSWORD }} - env: - ORG_GRADLE_PROJECT_signingKeyId: ${{ secrets.SIGNING_KEY_ID }} - ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SIGNING_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/release_draft.yml b/.github/workflows/release_draft.yml deleted file mode 100644 index 2f185417d7..0000000000 --- a/.github/workflows/release_draft.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Release Drafter - -on: - push: - branches: - - main - -permissions: - contents: read - -jobs: - update_release_draft: - permissions: - contents: write # for release-drafter/release-drafter to create a github release - pull-requests: write # for release-drafter/release-drafter to add label to PR - runs-on: ubuntu-latest - steps: - - uses: release-drafter/release-drafter@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 296d87ceeb..b38e969a3f 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -87,7 +87,7 @@ jobs: # Building and Pushing Conductor Server Image to ECR build-push-image: - name: Build and Push Image + name: Build and Push Server Image runs-on: 'ubuntu-latest' timeout-minutes: 20 permissions: @@ -105,7 +105,6 @@ jobs: outputs: ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} - MIGRATE_IMAGE: ${{ steps.image.outputs.MIGRATE_IMAGE }} steps: - name: "Checkout repository" @@ -127,7 +126,7 @@ jobs: - name: Download S3 file run: | - aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties .docker/server/config/conductor-server.properties + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties ./docker/server/config/conductor-server.properties - name: Amazon ECR Login id: login-ecr @@ -152,74 +151,79 @@ jobs: run: | echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT - # # Building and Pushing Conductor UI Image to ECR - # build-push-image-ui: - # name: Build and Push Image - # runs-on: 'ubuntu-latest' - # timeout-minutes: 20 - # permissions: - # id-token: write - # pull-requests: write - # contents: read - # needs: prepare-env - # env: - # AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} - # ENV: ${{ needs.prepare-env.outputs.ENV }} - # PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} - # ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} - # ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} - # outputs: - # ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} - # APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} - # MIGRATE_IMAGE: ${{ steps.image.outputs.MIGRATE_IMAGE }} - - # steps: - # - name: "Checkout repository" - # uses: actions/checkout@v4 - # - - # # Add support for more platforms with QEMU (optional) - # # https://github.com/docker/setup-qemu-action - # name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - # - - # name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 + # Building and Pushing Conductor UI Image to ECR + build-push-ui-image: + name: Build and Push UI Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY_UI}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - # - name: Configure AWS credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # role-to-assume: ${{ secrets[env.AWS_ROLE] }} - # aws-region: ${{ env.AWS_REGION }} - - # - name: Download S3 file - # run: | - # aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties .docker/server/config/conductor-server.properties - - - # - name: Amazon ECR Login - # id: login-ecr - # uses: aws-actions/amazon-ecr-login@v1.7.0 - - # - name: Build and push to Amazon ECR - # id: build - # uses: docker/build-push-action@v5.0.0 - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # IMAGE_TAG: ${{ github.sha }} - # with: - # context: . - # file: ./Dockerfile - # push: true - # platforms: linux/amd64 - # tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - # - name: image name - # id: image - # env: - # ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # IMAGE_TAG: ${{ github.sha }} - # run: | - # echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ github.sha }}" >> $GITHUB_OUTPUT - - - - \ No newline at end of file + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download S3 file + run: | + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-ui/.env ./ui/.env + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Node 18 + uses: actions/setup-node@v4 + with: + node-version: 18 + + + - name: Yarn Build + run: | + cd ui/ + yarn install && yarn build + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.0.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: ./ui/Dockerfile + push: true + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT \ No newline at end of file From 45801c48ce1a887bfb486f81876fce0661683b2c Mon Sep 17 00:00:00 2001 From: DevOps Date: Wed, 31 Jan 2024 14:12:35 +0530 Subject: [PATCH 5/6] Separate pipeline for UI & Server --- .github/workflows/workflow-server.yaml | 153 ++++++++++++++++++ .../{workflow.yaml => workflow-ui.yaml} | 78 +-------- 2 files changed, 161 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/workflow-server.yaml rename .github/workflows/{workflow.yaml => workflow-ui.yaml} (68%) diff --git a/.github/workflows/workflow-server.yaml b/.github/workflows/workflow-server.yaml new file mode 100644 index 0000000000..05c8c4e334 --- /dev/null +++ b/.github/workflows/workflow-server.yaml @@ -0,0 +1,153 @@ +name: Build & Publish OSS Conductor Server + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) + +env: + SERVICE_NAME: conductor + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ECR_REPOSITORY_UI: ${{ steps.set_env.outputs.ECR_REPOSITORY_UI }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECR_REPOSITORY=$PROJECT_PREFIX-conductor-server" >> $GITHUB_OUTPUT + echo "ECR_REPOSITORY_UI=$PROJECT_PREFIX-conductor-ui" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Building and Pushing Conductor Server Image to ECR + build-push-image: + name: Build and Push Server Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download S3 file + run: | + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties ./docker/server/config/conductor-server.properties + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.0.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: ./Dockerfile + push: true + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT + diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow-ui.yaml similarity index 68% rename from .github/workflows/workflow.yaml rename to .github/workflows/workflow-ui.yaml index b38e969a3f..dcf11bbe37 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow-ui.yaml @@ -1,4 +1,4 @@ -name: Build & Publish OSS Conductor +name: Build & Publish OSS Conductor UI on: workflow_dispatch: @@ -32,6 +32,7 @@ jobs: ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} ECR_REPOSITORY_UI: ${{ steps.set_env.outputs.ECR_REPOSITORY_UI }} ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + DEFAULT_CONF: ${{ steps.vars.outputs.DEFAULT_CONF }} SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} steps: @@ -63,11 +64,13 @@ jobs: then echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-prd.conf" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-dev.conf" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT else echo "Branch not configured!" @@ -84,72 +87,6 @@ jobs: echo "ECR_REPOSITORY_UI=$PROJECT_PREFIX-conductor-ui" >> $GITHUB_OUTPUT echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY - - # Building and Pushing Conductor Server Image to ECR - build-push-image: - name: Build and Push Server Image - runs-on: 'ubuntu-latest' - timeout-minutes: 20 - permissions: - id-token: write - pull-requests: write - contents: read - needs: prepare-env - env: - AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} - ENV: ${{ needs.prepare-env.outputs.ENV }} - PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} - ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} - ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} - IMAGE_TAG: ${{ github.event.inputs.tag }} - outputs: - ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} - APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} - - steps: - - name: "Checkout repository" - uses: actions/checkout@v4 - - - # Add support for more platforms with QEMU (optional) - # https://github.com/docker/setup-qemu-action - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets[env.AWS_ROLE] }} - aws-region: ${{ env.AWS_REGION }} - - - name: Download S3 file - run: | - aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties ./docker/server/config/conductor-server.properties - - - name: Amazon ECR Login - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1.7.0 - - - name: Build and push to Amazon ECR - id: build - uses: docker/build-push-action@v5.0.0 - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - with: - context: . - file: ./Dockerfile - push: true - platforms: linux/amd64 - tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - - - name: image name - id: image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - run: | - echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT # Building and Pushing Conductor UI Image to ECR build-push-ui-image: @@ -167,6 +104,7 @@ jobs: PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY_UI}} ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + DEFAULT_CONF: ${{needs.prepare-env.outputs.DEFAULT_CONF}} IMAGE_TAG: ${{ github.event.inputs.tag }} outputs: ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} @@ -198,15 +136,15 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v1.7.0 - - name: Node 18 + - name: Setup Node uses: actions/setup-node@v4 with: node-version: 18 - - name: Yarn Build run: | cd ui/ + mv ./${{ env.DEFAULT_CONF }} ./default.conf yarn install && yarn build - name: Build and push to Amazon ECR @@ -215,7 +153,7 @@ jobs: env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} with: - context: . + context: ./ui/ file: ./ui/Dockerfile push: true platforms: linux/amd64 From f2948c78d404df47c29755f44c0f495e6ab7c55c Mon Sep 17 00:00:00 2001 From: Vigneshwar Date: Wed, 6 Mar 2024 14:30:26 +0530 Subject: [PATCH 6/6] feat: Add support for http error status code --- .../java/com/netflix/conductor/tasks/http/HttpTask.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java index d85fa24840..8a0145a25f 100644 --- a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java +++ b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java @@ -24,6 +24,7 @@ import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.util.MultiValueMap; +import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -175,13 +176,17 @@ protected HttpResponse httpCall(Input input) throws Exception { if (responseEntity.getStatusCode().is2xxSuccessful() && responseEntity.hasBody()) { response.body = extractBody(responseEntity.getBody()); } - response.statusCode = responseEntity.getStatusCodeValue(); response.reasonPhrase = HttpStatus.valueOf(responseEntity.getStatusCode().value()).getReasonPhrase(); response.headers = responseEntity.getHeaders(); return response; - } catch (RestClientException ex) { + } catch (HttpClientErrorException ex) { + response.headers = ex.getResponseHeaders(); + response.statusCode = ex.getStatusCode().value(); + response.reasonPhrase = ex.getStatusText(); + return response; + } catch (RestClientException ex) { LOGGER.error( String.format( "Got unexpected http response - uri: %s, vipAddress: %s",