From fef815c6c57c89efd8c50a44df1a8eddc6865233 Mon Sep 17 00:00:00 2001 From: Aswin Dev Date: Mon, 29 Jan 2024 08:57:45 +0530 Subject: [PATCH 01/43] feat: production deployment changes --- Dockerfile | 40 +++++++++++++++++++++++++ core/build.gradle | 2 ++ dependencies.gradle | 2 +- server/build.gradle | 2 ++ ui/Dockerfile | 3 ++ ui/default-dev.conf | 69 +++++++++++++++++++++++++++++++++++++++++++ ui/default-local.conf | 50 +++++++++++++++++++++++++++++++ ui/default-prd.conf | 69 +++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 ui/Dockerfile create mode 100644 ui/default-dev.conf create mode 100644 ui/default-local.conf create mode 100644 ui/default-prd.conf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..935be7dd78 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# +# conductor:server - Netflix conductor server +# + +# =========================================================================================================== +# 0. Builder stage +# =========================================================================================================== +FROM eclipse-temurin:17-jdk-focal AS builder + +LABEL maintainer="Netflix OSS " + +# Copy the project directly onto the image +COPY . /conductor +WORKDIR /conductor + +# Build the server on run +RUN ./gradlew build -x test --stacktrace + +# =========================================================================================================== +# 1. Bin stage +# =========================================================================================================== +FROM eclipse-temurin:17-jre-focal + +LABEL maintainer="Netflix OSS " + +# Make app folders +RUN mkdir -p /app/config /app/logs /app/libs + +# Copy the compiled output to new image +COPY --from=builder /conductor/docker/server/bin /app +COPY --from=builder /conductor/docker/server/config /app/config +COPY --from=builder /conductor/server/build/libs/*boot*.jar /app/libs/conductor-server.jar + +# Copy the files for the server into the app folders +RUN chmod +x /app/startup.sh + +HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1 + +CMD [ "/app/startup.sh" ] +ENTRYPOINT [ "/bin/sh"] diff --git a/core/build.gradle b/core/build.gradle index beef8f7a25..065a9fe26a 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -43,6 +43,8 @@ dependencies { implementation "org.openjdk.nashorn:nashorn-core:15.4" + implementation "com.netflix.spectator:spectator-reg-metrics3:${version_spectator}" + // JAXB is not bundled with Java 11, dependencies added explicitly // These are needed by Apache BVAL implementation "jakarta.xml.bind:jakarta.xml.bind-api:${revJAXB}" diff --git a/dependencies.gradle b/dependencies.gradle index ed1c08fa10..32407c30da 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -67,5 +67,5 @@ ext { revNatsStreaming = '2.6.5' revNats = '2.15.6' revStan = '2.2.3' - + version_spectator='0.60.0' } diff --git a/server/build.gradle b/server/build.gradle index dc9dd8303e..55d1ec2111 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -70,7 +70,9 @@ dependencies { implementation "io.orkes.queues:orkes-conductor-queues:${revOrkesQueues}" implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${revSpringDoc}" + runtimeOnly group: 'com.netflix.conductor', name: 'conductor-postgres-persistence', version: '3.9.1' + implementation "com.netflix.spectator:spectator-reg-metrics3:${version_spectator}" runtimeOnly "org.glassfish.jaxb:jaxb-runtime:${revJAXB}" diff --git a/ui/Dockerfile b/ui/Dockerfile new file mode 100644 index 0000000000..240bdb712e --- /dev/null +++ b/ui/Dockerfile @@ -0,0 +1,3 @@ +FROM nginx +COPY default.conf /etc/nginx/conf.d/default.conf +COPY build/ /usr/share/nginx/html \ No newline at end of file diff --git a/ui/default-dev.conf b/ui/default-dev.conf new file mode 100644 index 0000000000..f37c665194 --- /dev/null +++ b/ui/default-dev.conf @@ -0,0 +1,69 @@ +map $http_x_forwarded_for $allow { + default 0; + "103.138.236.18" 1; + "103.181.238.106" 1; + "103.142.30.151" 1; + "61.2.142.186" 1; +} + +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + + if ($allow != 1) { + return 401; + } + + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-dev-mb-svc-conductor-server.sirn-dev-mb.local:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-dev-mb-svc-conductor-server.sirn-dev-mb.local:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-dev-mb-svc-conductor-server.sirn-dev-mb.local:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } +} \ No newline at end of file diff --git a/ui/default-local.conf b/ui/default-local.conf new file mode 100644 index 0000000000..fa8f0871d0 --- /dev/null +++ b/ui/default-local.conf @@ -0,0 +1,50 @@ +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } +} \ No newline at end of file diff --git a/ui/default-prd.conf b/ui/default-prd.conf new file mode 100644 index 0000000000..bd2c370c57 --- /dev/null +++ b/ui/default-prd.conf @@ -0,0 +1,69 @@ +map $http_x_forwarded_for $allow { + default 0; + "103.138.236.18" 1; + "103.181.238.106" 1; + "103.142.30.151" 1; + "61.2.142.186" 1; +} + +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + + if ($allow != 1) { + return 401; + } + + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-prd-mb-svc-conductor-server.sirn-prd-mb.local:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-prd-mb-svc-conductor-server.sirn-prd-mb.local:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-prd-mb-svc-conductor-server.sirn-prd-mb.local:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } +} \ No newline at end of file From fd1ee28ae30ec820f48c8b08a8305e7f5df335be Mon Sep 17 00:00:00 2001 From: Raveena Date: Tue, 6 Feb 2024 23:59:26 +0530 Subject: [PATCH 02/43] Add CI & CD for Server & UI --- .github/actions/deploy-ecs/action.yaml | 60 +++++++++ .github/workflows/README.md | 34 +++++ .github/workflows/cd-server.yaml | 150 +++++++++++++++++++++ .github/workflows/cd-ui.yaml | 149 +++++++++++++++++++++ .github/workflows/ci-server-.yaml | 154 ++++++++++++++++++++++ .github/workflows/ci-ui.yaml | 168 ++++++++++++++++++++++++ .github/workflows/ci.yml | 104 --------------- .github/workflows/generate_gh_pages.yml | 18 --- .github/workflows/publish.yml | 40 ------ .github/workflows/release_draft.yml | 20 --- 10 files changed, 715 insertions(+), 182 deletions(-) create mode 100644 .github/actions/deploy-ecs/action.yaml create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/cd-server.yaml create mode 100644 .github/workflows/cd-ui.yaml create mode 100644 .github/workflows/ci-server-.yaml create mode 100644 .github/workflows/ci-ui.yaml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/generate_gh_pages.yml delete mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/release_draft.yml diff --git a/.github/actions/deploy-ecs/action.yaml b/.github/actions/deploy-ecs/action.yaml new file mode 100644 index 0000000000..649e6775a1 --- /dev/null +++ b/.github/actions/deploy-ecs/action.yaml @@ -0,0 +1,60 @@ +name: "Deploy to ECS" +description: "Deploy new image to given ECS service by updating task definition file" +inputs: + aws-role: + required: true + description: "AWS ROLE" + aws-region: + required: true + description: "AWS REGION" + task-definition: + required: true + description: "TASK DEFINITION" + container-name: + required: true + description: "CONTAINER NAME" + ecs-service: + required: true + description: "ECS SERVICE" + ecs-cluster: + required: true + description: "ECS CLUSTER" + image: + required: true + description: "APP IMAGE" +runs: + using: "composite" + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ inputs.aws-region }} + + - name: Download task definition + run: | + aws ecs describe-task-definition --task-definition ${{ inputs.task-definition }} \ + --query taskDefinition > task-definition.json + shell: bash + + - name: Fill in the new image ID in the Amazon ECS task definition + id: task_def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: ${{ inputs.container-name }} + image: ${{ inputs.image }} + + - name: Remove unwanted fields from task definition + id: task_def_cleanup + shell: bash + run: | + jq 'del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy)' ${{ steps.task_def.outputs.task-definition }} > updated-task-definition.json + + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: updated-task-definition.json + service: ${{ inputs.ecs-service }} + cluster: ${{ inputs.ecs-cluster }} + wait-for-service-stability: true diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000000..f592f390e4 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,34 @@ +# Conductor CI/CD Workflow + +This repository contains workflow files for implementing Continuous Integration (CI) and Continuous Deployment (CD) processes separately for Conductor UI and server components. The workflow is designed to deploy to both development (dev) and production (prd) environments on AWS (ECS). + +## Workflow Overview + +The CI/CD workflow is triggered manually & comprises two main components: + +1. **Conductor UI CI/CD:** + - Workflow file: `.github/workflows/ci-ui.yaml` + `.github/workflows/cd-ui.yaml` + - These workflows handle the CI & CD process for Conductor UI. + +2. **Conductor Server CI/CD:** + - Workflow file: `.github/workflows/ci-server.yaml` + `.github/workflows/cd-server.yaml` + - These workflows handle the CI & CD process for Conductor server. + +## Deployment Strategy + +- **Branches:** + - The `production` branch is considered the master branch for all deployments. + - All deployments to both development and production environments are triggered from the `production` branch. + +- **Input Variables:** + - The workflow takes the following input variables: + 1. **Branch:** Specifies the branch to be deployed (e.g., `production`). + 2. **Environment:** Specifies the deployment environment (e.g., `dev` or `prd`). + 3. **Tag:** Specifies the version to be deployed. This version is used for tagging the Docker image. + +## Versioning and Docker Image Tagging + +The version provided as an input variable is crucial for versioning and tagging Docker images. The workflow utilizes this version to tag the Docker image before deploying to the AWS Elastic Container Registry (ECR). During ECS deployment, this tagged image is fetched from ECR. + diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml new file mode 100644 index 0000000000..addde195ee --- /dev/null +++ b/.github/workflows/cd-server.yaml @@ -0,0 +1,150 @@ +name: Deploy Conductor Server + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) + +env: + SERVICE_NAME: conductor-server + AWS_REGION: "ap-south-1" + + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECS_SERVICE: ${{ steps.set_env.outputs.ECS_SERVICE }} + TASK_DEFINITION: ${{ steps.set_env.outputs.TASK_DEFINITION }} + CONTAINER_NAME: ${{ steps.set_env.outputs.CONTAINER_NAME }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECS_SERVICE=$PROJECT_PREFIX-svc-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "TASK_DEFINITION=$PROJECT_PREFIX-td-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "CONTAINER_NAME=$PROJECT_PREFIX-cntr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Deploy Conductor UI Image to ECS + deploy-ui-image: + name: Deploy UI Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + ECS_CLUSTER: ${{ needs.prepare-env.outputs.ECS_CLUSTER }} + ECS_SERVICE: ${{ needs.prepare-env.outputs.ECS_SERVICE }} + TASK_DEFINITION: ${{ needs.prepare-env.outputs.TASK_DEFINITION }} + CONTAINER_NAME: ${{ needs.prepare-env.outputs.CONTAINER_NAME }} + + steps: + - name: Checkout code from action + uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Check if image tag exists in ECR + id: check-image-existence + run: | + if aws ecr describe-images --repository-name "${{ env.ECR_REPOSITORY }}" --region "${{ env.AWS_REGION }}" --image-ids imageTag="${{ env.IMAGE_TAG }}" 2>&1 | grep -q "imageTag"; then + echo "Image tag $IMAGE_TAG exists in ECR" + else + echo "Error: Image tag $IMAGE_TAG does not exist in ECR" + exit 1 + fi + + - name: Deploy backend + id: deploy_backend + uses: ./.github/actions/deploy-ecs + env: + APP_IMAGE: ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + with: + aws-region : ${{ env.AWS_REGION }} + aws-role: ${{ secrets[env.AWS_ROLE] }} + task-definition: ${{ env.TASK_DEFINITION }} + container-name: ${{ env.CONTAINER_NAME }} + ecs-service: ${{ env.ECS_SERVICE }} + ecs-cluster: ${{ env.ECS_CLUSTER }} + image: ${{ env.APP_IMAGE }} \ No newline at end of file diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml new file mode 100644 index 0000000000..5b095f8d14 --- /dev/null +++ b/.github/workflows/cd-ui.yaml @@ -0,0 +1,149 @@ +name: Deploy Conductor UI + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) + +env: + SERVICE_NAME: conductor-ui + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECS_SERVICE: ${{ steps.set_env.outputs.ECS_SERVICE }} + TASK_DEFINITION: ${{ steps.set_env.outputs.TASK_DEFINITION }} + CONTAINER_NAME: ${{ steps.set_env.outputs.CONTAINER_NAME }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECS_SERVICE=$PROJECT_PREFIX-svc-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "TASK_DEFINITION=$PROJECT_PREFIX-td-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "CONTAINER_NAME=$PROJECT_PREFIX-cntr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Deploy Conductor UI Image to ECS + deploy-ui-image: + name: Deploy UI Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + ECS_CLUSTER: ${{ needs.prepare-env.outputs.ECS_CLUSTER }} + ECS_SERVICE: ${{ needs.prepare-env.outputs.ECS_SERVICE }} + TASK_DEFINITION: ${{ needs.prepare-env.outputs.TASK_DEFINITION }} + CONTAINER_NAME: ${{ needs.prepare-env.outputs.CONTAINER_NAME }} + + steps: + - name: Checkout code from action + uses: actions/checkout@v2 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Check if image tag exists in ECR + id: check-image-existence + run: | + if aws ecr describe-images --repository-name "${{ env.ECR_REPOSITORY }}" --region "${{ env.AWS_REGION }}" --image-ids imageTag="${{ env.IMAGE_TAG }}" 2>&1 | grep -q "imageTag"; then + echo "Image tag $IMAGE_TAG exists in ECR" + else + echo "Error: Image tag $IMAGE_TAG does not exist in ECR" + exit 1 + fi + + - name: Deploy backend + id: deploy_backend + uses: ./.github/actions/deploy-ecs + env: + APP_IMAGE: ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + with: + aws-region : ${{ env.AWS_REGION }} + aws-role: ${{ secrets[env.AWS_ROLE] }} + task-definition: ${{ env.TASK_DEFINITION }} + container-name: ${{ env.CONTAINER_NAME }} + ecs-service: ${{ env.ECS_SERVICE }} + ecs-cluster: ${{ env.ECS_CLUSTER }} + image: ${{ env.APP_IMAGE }} \ No newline at end of file diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml new file mode 100644 index 0000000000..4cdb9587db --- /dev/null +++ b/.github/workflows/ci-server-.yaml @@ -0,0 +1,154 @@ +name: Build & Publish Conductor Server + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) + +env: + SERVICE_NAME: conductor-server + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Building and Pushing Conductor Server Image to ECR + build-push-image: + name: Build and Push Server Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download S3 file + run: | + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties ./docker/server/config/conductor-server.properties + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.1.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: . + file: ./Dockerfile + push: true + provenance: false + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: Image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT + diff --git a/.github/workflows/ci-ui.yaml b/.github/workflows/ci-ui.yaml new file mode 100644 index 0000000000..48f32bb25a --- /dev/null +++ b/.github/workflows/ci-ui.yaml @@ -0,0 +1,168 @@ +name: Build & Publish Conductor UI + +on: + workflow_dispatch: + inputs: + Environment: + required: true + type: choice + description: Choose aws env + options: + - dev + - prd + Tag: + required: true + type: string + description: Provide tag (Eg:v3.14.0) + +env: + SERVICE_NAME: conductor-ui + AWS_REGION: "ap-south-1" + +jobs: + prepare-env: + name: Prepare Env + runs-on: 'ubuntu-latest' + timeout-minutes: 2 + outputs: + AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} + ENV: ${{ steps.vars.outputs.ENV }} + PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} + ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} + ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} + DEFAULT_CONF: ${{ steps.vars.outputs.DEFAULT_CONF }} + SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + + steps: + - id: vars + shell: bash + run: | + BRANCH="${GITHUB_REF#refs/heads/}" + ENV=${{ github.event.inputs.environment }} + IMAGE_TAG=${{ github.event.inputs.tag }} + echo $BRANCH + + if [ -z "$ENV" ] + then + case $BRANCH in + "dev") + ENV="dev" + ;; + "stg") + ENV="stg" + ;; + "main") + ENV="prd" + ;; + *) + echo "ENV not configured" && exit 1 + ;; + esac + fi + if [[ $ENV == 'prd' && $BRANCH == 'production' ]] + then + echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-prd.conf" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'dev' ] + then + echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-dev.conf" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + else + echo "Branch not configured!" + exit 1 + fi + echo "ENV=$ENV" >> $GITHUB_OUTPUT + echo ":rocket: Environment - $ENV " >> $GITHUB_STEP_SUMMARY + echo ":label: Image Tag - $IMAGE_TAG " >> $GITHUB_STEP_SUMMARY + - name: set variables + id: set_env + run: | + PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} + echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT + echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY + + # Building and Pushing Conductor UI Image to ECR + build-push-ui-image: + name: Build and Push UI Image + runs-on: 'ubuntu-latest' + timeout-minutes: 20 + permissions: + id-token: write + pull-requests: write + contents: read + needs: prepare-env + env: + AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} + ENV: ${{ needs.prepare-env.outputs.ENV }} + PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} + ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} + ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} + DEFAULT_CONF: ${{needs.prepare-env.outputs.DEFAULT_CONF}} + IMAGE_TAG: ${{ github.event.inputs.tag }} + outputs: + ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} + APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} + + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: ${{ env.AWS_REGION }} + + - name: Download S3 file + run: | + aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-ui/.env ./ui/.env + + - name: Amazon ECR Login + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1.7.0 + + - name: Set Up Node + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Yarn Build + run: | + cd ui/ + mv ./${{ env.DEFAULT_CONF }} ./default.conf + yarn install && yarn build + + - name: Build and push to Amazon ECR + id: build + uses: docker/build-push-action@v5.1.0 + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + with: + context: ./ui/ + file: ./ui/Dockerfile + push: true + provenance: false + platforms: linux/amd64 + tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: Image name + id: image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + run: | + echo "APP_IMAGE=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index a25c97dd2e..0000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: CI - -on: [ push, pull_request ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - - name: Gradle wrapper validation - uses: gradle/wrapper-validation-action@v1 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Cache SonarCloud packages - uses: actions/cache@v3 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - name: Build with Gradle - if: github.ref != 'refs/heads/main' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - ./gradlew build --scan - - name: Build and Publish snapshot - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - run: | - echo "Running build for commit ${{ github.sha }}" - ./gradlew build - - name: Publish Test Report - uses: mikepenz/action-junit-report@v3 - if: always() - with: - report_paths: '**/build/test-results/test/TEST-*.xml' - - name: Upload build artifacts - uses: actions/upload-artifact@v3 - with: - name: build-artifacts - path: '**/build/reports' - - name: Store Buildscan URL - uses: actions/upload-artifact@v3 - with: - name: build-scan - path: 'buildscan.log' - build-ui: - runs-on: ubuntu-latest - container: cypress/browsers:node14.17.6-chrome100-ff98 - defaults: - run: - working-directory: ui - steps: - - uses: actions/checkout@v3 - - - name: Install Dependencies - run: yarn install - - - name: Build UI - run: yarn run build - - - name: Run E2E Tests - uses: cypress-io/github-action@v4 - with: - working-directory: ui - install: false - start: yarn run serve-build - wait-on: 'http://localhost:5000' - - - name: Run Component Tests - uses: cypress-io/github-action@v4 - with: - working-directory: ui - install: false - component: true - - - name: Archive test screenshots - uses: actions/upload-artifact@v2 - if: failure() - with: - name: cypress-screenshots - path: ui/cypress/screenshots - - - name: Archive test videos - uses: actions/upload-artifact@v2 - if: always() - with: - name: cypress-videos - path: ui/cypress/videos - diff --git a/.github/workflows/generate_gh_pages.yml b/.github/workflows/generate_gh_pages.yml deleted file mode 100644 index 8c429e1b8e..0000000000 --- a/.github/workflows/generate_gh_pages.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Publish docs via GitHub Pages -on: - workflow_dispatch - -jobs: - build: - name: Deploy docs - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v2 - - - name: Deploy docs - uses: mhausenblas/mkdocs-deploy-gh-pages@master - env: - GITHUB_TOKEN: ${{ secrets.DOCSITE_TOKEN }} - CONFIG_FILE: mkdocs.yml - REQUIREMENTS: requirements.txt diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 51b514ffa3..0000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Publish Conductor OSS toMaven Central -on: - release: - types: - - released - - prereleased - -permissions: - contents: read - -jobs: - publish: - runs-on: ubuntu-latest - name: Gradle Build and Publish - steps: - - uses: actions/checkout@v3 - - name: Set up Zulu JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - name: Cache Gradle packages - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Publish release - run: | - export VERSION="${{github.ref_name}}" - export PUBLISH_VERSION=`echo ${VERSION:1}` - echo Publishing version $PUBLISH_VERSION - ./gradlew publish -Pversion=$PUBLISH_VERSION -Pusername=${{ secrets.SONATYPE_USERNAME }} -Ppassword=${{ secrets.SONATYPE_PASSWORD }} - env: - ORG_GRADLE_PROJECT_signingKeyId: ${{ secrets.SIGNING_KEY_ID }} - ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SIGNING_KEY }} - ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/release_draft.yml b/.github/workflows/release_draft.yml deleted file mode 100644 index 2f185417d7..0000000000 --- a/.github/workflows/release_draft.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Release Drafter - -on: - push: - branches: - - main - -permissions: - contents: read - -jobs: - update_release_draft: - permissions: - contents: write # for release-drafter/release-drafter to create a github release - pull-requests: write # for release-drafter/release-drafter to add label to PR - runs-on: ubuntu-latest - steps: - - uses: release-drafter/release-drafter@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0d0503875ff6980b883930ea26970b9d2cbefed0 Mon Sep 17 00:00:00 2001 From: Raveena Date: Mon, 12 Feb 2024 16:05:53 +0530 Subject: [PATCH 03/43] cicd for stg --- .github/workflows/cd-server.yaml | 8 +++++++- .github/workflows/cd-ui.yaml | 8 +++++++- .github/workflows/ci-server-.yaml | 6 ++++++ .github/workflows/ci-ui.yaml | 8 +++++++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml index addde195ee..090a375e6d 100644 --- a/.github/workflows/cd-server.yaml +++ b/.github/workflows/cd-server.yaml @@ -9,6 +9,7 @@ on: description: Choose aws env options: - dev + - stg - prd Tag: required: true @@ -66,7 +67,12 @@ jobs: then echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT - echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml index 5b095f8d14..3609423632 100644 --- a/.github/workflows/cd-ui.yaml +++ b/.github/workflows/cd-ui.yaml @@ -9,6 +9,7 @@ on: description: Choose aws env options: - dev + - stg - prd Tag: required: true @@ -65,7 +66,12 @@ jobs: then echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT - echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml index 4cdb9587db..0fcb600762 100644 --- a/.github/workflows/ci-server-.yaml +++ b/.github/workflows/ci-server-.yaml @@ -9,6 +9,7 @@ on: description: Choose aws env options: - dev + - stg - prd Tag: required: true @@ -64,6 +65,11 @@ jobs: echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci-ui.yaml b/.github/workflows/ci-ui.yaml index 48f32bb25a..00b58625b9 100644 --- a/.github/workflows/ci-ui.yaml +++ b/.github/workflows/ci-ui.yaml @@ -9,6 +9,7 @@ on: description: Choose aws env options: - dev + - stg - prd Tag: required: true @@ -65,7 +66,12 @@ jobs: echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT echo "DEFAULT_CONF=default-prd.conf" >> $GITHUB_OUTPUT - echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + elif [ $ENV == 'stg' ] + then + echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT + echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT From ae1db175ea2d0d3dbbc14848c9568538ca368a0b Mon Sep 17 00:00:00 2001 From: Raveena Date: Mon, 12 Feb 2024 18:29:48 +0530 Subject: [PATCH 04/43] corrected server cd yaml --- .github/workflows/cd-server.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml index 090a375e6d..1b4bba94b4 100644 --- a/.github/workflows/cd-server.yaml +++ b/.github/workflows/cd-server.yaml @@ -97,8 +97,8 @@ jobs: echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY # Deploy Conductor UI Image to ECS - deploy-ui-image: - name: Deploy UI Image + deploy-server-image: + name: Deploy Server Image runs-on: 'ubuntu-latest' timeout-minutes: 20 permissions: From e074bfea2a4cfbaed29bebd44a2bd1d6af645fa4 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Tue, 20 Feb 2024 09:38:51 +0530 Subject: [PATCH 05/43] feat: Add migration scripts to add initial workflows, tasks and event handler --- ...e_execution_workflow_and_event_handler.sql | 109 ++++++++++++++++++ .../V11__siren_webhook_task_and_workflow.sql | 67 +++++++++++ .../V9__siren_send_notification_task.sql | 27 +++++ 3 files changed, 203 insertions(+) create mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql create mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql create mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql new file mode 100644 index 0000000000..ff9a20f4f5 --- /dev/null +++ b/postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql @@ -0,0 +1,109 @@ +-- Enable workflow status listener in all workflow definitions +UPDATE public.meta_workflow_def +SET json_data = jsonb_set( + json_data::jsonb, + '{workflowStatusListenerEnabled}', + 'true'::jsonb +)::text; + +INSERT INTO + public.meta_task_def ("name", json_data) +VALUES +( + 'finalize_workflow_execution_task', + json_build_object( + 'createTime' ,EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, + 'createdBy' ,'', + 'accessPolicy', jsonb '{}', + 'name' ,'finalize_workflow_execution_task', + 'description' ,'Finalize Workflow Execution Task', + 'retryCount' ,5, + 'timeoutSeconds' ,3600, + 'inputKeys' ,jsonb '[]', + 'outputKeys' ,jsonb '[]', + 'timeoutPolicy' ,'TIME_OUT_WF', + 'retryLogic' ,'EXPONENTIAL_BACKOFF', + 'retryDelaySeconds' ,10, + 'responseTimeoutSeconds' ,600, + 'inputTemplate', jsonb '{}', + 'rateLimitPerFrequency' ,0, + 'rateLimitFrequencyInSeconds' ,1, + 'ownerEmail' ,'admin@sirenapp.io', + 'backoffScaleFactor' ,1 + ) + ); + +INSERT INTO + public.meta_workflow_def ("name", "version", latest_version, json_data) +VALUES( + 'finalize_workflow_execution', + 1, + 1, + json_build_object( + 'createTime',EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, + 'accessPolicy',jsonb '{}', + 'name','finalize_workflow_execution', + 'description','Workflow for finalizing workflow execution', + 'version',1, + 'tasks', + json_build_array(json_build_object( + 'name','finalize_workflow_execution_task', + 'taskReferenceName','finalize_workflow_execution_task_ref', + 'inputParameters', + json_build_object( + 'status',E'\u0024{workflow.input.status}', + 'externalExecutionId',E'\u0024{workflow.input.workflowId}' + ), + 'type','SIMPLE', + 'startDelay',0, + 'optional',false, + 'asyncComplete',false, + 'permissive',false + )), + 'inputParameters',jsonb '[]', + 'outputParameters',jsonb '{}', + 'schemaVersion',2, + 'restartable',true, + 'workflowStatusListenerEnabled',false, + 'ownerEmail','admin@sirenapp.io', + 'timeoutPolicy','ALERT_ONLY', + 'timeoutSeconds',0, + 'variables',jsonb '{}', + 'inputTemplate',jsonb '{}' + ) + ); + +INSERT INTO + public.meta_event_handler (id, "name", "event", active, json_data) +VALUES( + 1, + 'finalize_workflow_execution_event_handler', + 'conductor:finalize_workflow_execution_event', + true, + json_build_object( + 'name','finalize_workflow_execution_event_handler', + 'event','conductor:finalize_workflow_execution_event', + 'actions', + json_build_array( + json_build_object('action','start_workflow', + 'start_workflow', + json_build_object( + 'name','finalize_workflow_execution', + 'input',json_build_object( + 'workflowType',E'\u0024{workflowType}', + 'version',E'\u0024{version}', + 'workflowId',E'\u0024{workflowId}', + 'correlationId',E'\u0024{correlationId}', + 'status',E'\u0024{status}', + 'input',E'\u0024{input}', + 'output',E'\u0024{output}', + 'reasonForIncompletion',E'\u0024{reasonForIncompletion}', + 'executionTime',E'\u0024{executionTime}', + 'event',E'\u0024{event}') + ) + ) + ), + 'active',true + ) + ); + diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql new file mode 100644 index 0000000000..1db15ac536 --- /dev/null +++ b/postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql @@ -0,0 +1,67 @@ +INSERT INTO + public.meta_task_def ("name", json_data) +VALUES +( + 'send_webhook_task', + json_build_object( + 'createTime' ,EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, + 'createdBy' ,'', + 'accessPolicy', jsonb '{}', + 'name' ,'send_webhook_task', + 'description' ,'Send Webhook Task', + 'retryCount' ,5, + 'timeoutSeconds' ,3600, + 'inputKeys' ,jsonb '[]', + 'outputKeys' ,jsonb '[]', + 'timeoutPolicy' ,'TIME_OUT_WF', + 'retryLogic' ,'EXPONENTIAL_BACKOFF', + 'retryDelaySeconds' ,10, + 'responseTimeoutSeconds' ,600, + 'inputTemplate', jsonb '{}', + 'rateLimitPerFrequency' ,0, + 'rateLimitFrequencyInSeconds' ,1, + 'ownerEmail' ,'admin@sirenapp.io', + 'backoffScaleFactor' ,1 + ) + ); + +INSERT INTO public.meta_workflow_def +( "name", "version", latest_version, json_data) +VALUES( + 'webhook_workflow', + 1, + 1, + json_build_object( + 'createTime',EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, + 'accessPolicy', jsonb '{}', + 'name','webhook_workflow', + 'description','Workflow for sending webhook', + 'version',1, + 'tasks', + json_build_array( + json_build_object( + 'name','send_webhook_task', + 'taskReferenceName','send_webhook_task_ref', + 'inputParameters', + json_build_object( + 'notificationAuditId',E'\u0024{workflow.input.notificationAuditId}' + ), + 'type','SIMPLE', + 'startDelay',0, + 'optional',true, + 'asyncComplete',false, + 'permissive',false + )), + 'inputParameters', jsonb '[]', + 'outputParameters',jsonb '{}', + 'schemaVersion',2, + 'restartable',true, + 'workflowStatusListenerEnabled',false, + 'ownerEmail','admin@sirenapp.io', + 'timeoutPolicy','ALERT_ONLY', + 'timeoutSeconds',0, + 'variables',jsonb '{}', + 'inputTemplate',jsonb '{}' + ) + ); + diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql new file mode 100644 index 0000000000..13efbc1db0 --- /dev/null +++ b/postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql @@ -0,0 +1,27 @@ +INSERT INTO + public.meta_task_def ("name", json_data) +VALUES +( + 'send_notification_task', + json_build_object( + 'createTime', EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, + 'createdBy' ,'', + 'accessPolicy', jsonb '{}', + 'name' ,'send_notification_task', + 'description' ,'Send Notification Task', + 'retryCount' ,5, + 'timeoutSeconds' ,3600, + 'inputKeys' , jsonb '[]', + 'outputKeys' , jsonb '[]', + 'timeoutPolicy' ,'TIME_OUT_WF', + 'retryLogic' ,'EXPONENTIAL_BACKOFF', + 'retryDelaySeconds' ,10, + 'responseTimeoutSeconds' ,600, + 'inputTemplate', jsonb '{}', + 'rateLimitPerFrequency' ,0, + 'rateLimitFrequencyInSeconds' ,1, + 'ownerEmail' ,'admin@sirenapp.io', + 'backoffScaleFactor' ,1 + ) + ); + \ No newline at end of file From 59a56b505d45e15442f7737353846e3c4ec25a5b Mon Sep 17 00:00:00 2001 From: Raveena Date: Tue, 20 Feb 2024 17:03:37 +0530 Subject: [PATCH 06/43] Add stg related config for UI --- .github/workflows/ci-ui.yaml | 1 + ui/default-stg.conf | 69 ++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 ui/default-stg.conf diff --git a/.github/workflows/ci-ui.yaml b/.github/workflows/ci-ui.yaml index 00b58625b9..5e7ffebf18 100644 --- a/.github/workflows/ci-ui.yaml +++ b/.github/workflows/ci-ui.yaml @@ -71,6 +71,7 @@ jobs: then echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT + echo "DEFAULT_CONF=default-stg.conf" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then diff --git a/ui/default-stg.conf b/ui/default-stg.conf new file mode 100644 index 0000000000..e057354998 --- /dev/null +++ b/ui/default-stg.conf @@ -0,0 +1,69 @@ +map $http_x_forwarded_for $allow { + default 0; + "103.138.236.18" 1; + "103.181.238.106" 1; + "103.142.30.151" 1; + "61.2.142.186" 1; +} + +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + + if ($allow != 1) { + return 401; + } + + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-stg-mb-svc-conductor-server.sirn-stg-mb.local:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-stg-mb-svc-conductor-server.sirn-stg-mb.local:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://sirn-stg-mb-svc-conductor-server.sirn-stg-mb.local:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } +} \ No newline at end of file From d215ff9d3afac8a1b977358404a14d158dbd4fb5 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Tue, 20 Feb 2024 20:02:31 +0530 Subject: [PATCH 07/43] feat: Create initial workflows and tasks using eventlistener --- ...e_execution_workflow_and_event_handler.sql | 109 --------------- .../V11__siren_webhook_task_and_workflow.sql | 67 --------- .../V9__siren_send_notification_task.sql | 27 ---- .../rest/startup/SirenInitializer.java | 128 ++++++++++++++++++ ...finalizeWorkflowExecutionEventHandler.json | 26 ++++ .../tasks/finalizeWorkflowExecutionTask.json | 21 +++ .../siren/tasks/sendNotificationTask.json | 21 +++ .../siren/tasks/sendWebhookTask.json | 21 +++ .../sirenFinalizeExecutionWorkflow.json | 31 +++++ .../siren/workflows/sirenWebhookWorkflow.json | 30 ++++ 10 files changed, 278 insertions(+), 203 deletions(-) delete mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql delete mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql delete mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql create mode 100644 rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java create mode 100644 rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json create mode 100644 rest/src/main/resources/siren/tasks/finalizeWorkflowExecutionTask.json create mode 100644 rest/src/main/resources/siren/tasks/sendNotificationTask.json create mode 100644 rest/src/main/resources/siren/tasks/sendWebhookTask.json create mode 100644 rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json create mode 100644 rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql deleted file mode 100644 index ff9a20f4f5..0000000000 --- a/postgres-persistence/src/main/resources/db/migration_postgres/V10__siren_finalize_execution_workflow_and_event_handler.sql +++ /dev/null @@ -1,109 +0,0 @@ --- Enable workflow status listener in all workflow definitions -UPDATE public.meta_workflow_def -SET json_data = jsonb_set( - json_data::jsonb, - '{workflowStatusListenerEnabled}', - 'true'::jsonb -)::text; - -INSERT INTO - public.meta_task_def ("name", json_data) -VALUES -( - 'finalize_workflow_execution_task', - json_build_object( - 'createTime' ,EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, - 'createdBy' ,'', - 'accessPolicy', jsonb '{}', - 'name' ,'finalize_workflow_execution_task', - 'description' ,'Finalize Workflow Execution Task', - 'retryCount' ,5, - 'timeoutSeconds' ,3600, - 'inputKeys' ,jsonb '[]', - 'outputKeys' ,jsonb '[]', - 'timeoutPolicy' ,'TIME_OUT_WF', - 'retryLogic' ,'EXPONENTIAL_BACKOFF', - 'retryDelaySeconds' ,10, - 'responseTimeoutSeconds' ,600, - 'inputTemplate', jsonb '{}', - 'rateLimitPerFrequency' ,0, - 'rateLimitFrequencyInSeconds' ,1, - 'ownerEmail' ,'admin@sirenapp.io', - 'backoffScaleFactor' ,1 - ) - ); - -INSERT INTO - public.meta_workflow_def ("name", "version", latest_version, json_data) -VALUES( - 'finalize_workflow_execution', - 1, - 1, - json_build_object( - 'createTime',EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, - 'accessPolicy',jsonb '{}', - 'name','finalize_workflow_execution', - 'description','Workflow for finalizing workflow execution', - 'version',1, - 'tasks', - json_build_array(json_build_object( - 'name','finalize_workflow_execution_task', - 'taskReferenceName','finalize_workflow_execution_task_ref', - 'inputParameters', - json_build_object( - 'status',E'\u0024{workflow.input.status}', - 'externalExecutionId',E'\u0024{workflow.input.workflowId}' - ), - 'type','SIMPLE', - 'startDelay',0, - 'optional',false, - 'asyncComplete',false, - 'permissive',false - )), - 'inputParameters',jsonb '[]', - 'outputParameters',jsonb '{}', - 'schemaVersion',2, - 'restartable',true, - 'workflowStatusListenerEnabled',false, - 'ownerEmail','admin@sirenapp.io', - 'timeoutPolicy','ALERT_ONLY', - 'timeoutSeconds',0, - 'variables',jsonb '{}', - 'inputTemplate',jsonb '{}' - ) - ); - -INSERT INTO - public.meta_event_handler (id, "name", "event", active, json_data) -VALUES( - 1, - 'finalize_workflow_execution_event_handler', - 'conductor:finalize_workflow_execution_event', - true, - json_build_object( - 'name','finalize_workflow_execution_event_handler', - 'event','conductor:finalize_workflow_execution_event', - 'actions', - json_build_array( - json_build_object('action','start_workflow', - 'start_workflow', - json_build_object( - 'name','finalize_workflow_execution', - 'input',json_build_object( - 'workflowType',E'\u0024{workflowType}', - 'version',E'\u0024{version}', - 'workflowId',E'\u0024{workflowId}', - 'correlationId',E'\u0024{correlationId}', - 'status',E'\u0024{status}', - 'input',E'\u0024{input}', - 'output',E'\u0024{output}', - 'reasonForIncompletion',E'\u0024{reasonForIncompletion}', - 'executionTime',E'\u0024{executionTime}', - 'event',E'\u0024{event}') - ) - ) - ), - 'active',true - ) - ); - diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql deleted file mode 100644 index 1db15ac536..0000000000 --- a/postgres-persistence/src/main/resources/db/migration_postgres/V11__siren_webhook_task_and_workflow.sql +++ /dev/null @@ -1,67 +0,0 @@ -INSERT INTO - public.meta_task_def ("name", json_data) -VALUES -( - 'send_webhook_task', - json_build_object( - 'createTime' ,EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, - 'createdBy' ,'', - 'accessPolicy', jsonb '{}', - 'name' ,'send_webhook_task', - 'description' ,'Send Webhook Task', - 'retryCount' ,5, - 'timeoutSeconds' ,3600, - 'inputKeys' ,jsonb '[]', - 'outputKeys' ,jsonb '[]', - 'timeoutPolicy' ,'TIME_OUT_WF', - 'retryLogic' ,'EXPONENTIAL_BACKOFF', - 'retryDelaySeconds' ,10, - 'responseTimeoutSeconds' ,600, - 'inputTemplate', jsonb '{}', - 'rateLimitPerFrequency' ,0, - 'rateLimitFrequencyInSeconds' ,1, - 'ownerEmail' ,'admin@sirenapp.io', - 'backoffScaleFactor' ,1 - ) - ); - -INSERT INTO public.meta_workflow_def -( "name", "version", latest_version, json_data) -VALUES( - 'webhook_workflow', - 1, - 1, - json_build_object( - 'createTime',EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, - 'accessPolicy', jsonb '{}', - 'name','webhook_workflow', - 'description','Workflow for sending webhook', - 'version',1, - 'tasks', - json_build_array( - json_build_object( - 'name','send_webhook_task', - 'taskReferenceName','send_webhook_task_ref', - 'inputParameters', - json_build_object( - 'notificationAuditId',E'\u0024{workflow.input.notificationAuditId}' - ), - 'type','SIMPLE', - 'startDelay',0, - 'optional',true, - 'asyncComplete',false, - 'permissive',false - )), - 'inputParameters', jsonb '[]', - 'outputParameters',jsonb '{}', - 'schemaVersion',2, - 'restartable',true, - 'workflowStatusListenerEnabled',false, - 'ownerEmail','admin@sirenapp.io', - 'timeoutPolicy','ALERT_ONLY', - 'timeoutSeconds',0, - 'variables',jsonb '{}', - 'inputTemplate',jsonb '{}' - ) - ); - diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql deleted file mode 100644 index 13efbc1db0..0000000000 --- a/postgres-persistence/src/main/resources/db/migration_postgres/V9__siren_send_notification_task.sql +++ /dev/null @@ -1,27 +0,0 @@ -INSERT INTO - public.meta_task_def ("name", json_data) -VALUES -( - 'send_notification_task', - json_build_object( - 'createTime', EXTRACT(epoch FROM CURRENT_TIMESTAMP)::bigint * 1000, - 'createdBy' ,'', - 'accessPolicy', jsonb '{}', - 'name' ,'send_notification_task', - 'description' ,'Send Notification Task', - 'retryCount' ,5, - 'timeoutSeconds' ,3600, - 'inputKeys' , jsonb '[]', - 'outputKeys' , jsonb '[]', - 'timeoutPolicy' ,'TIME_OUT_WF', - 'retryLogic' ,'EXPONENTIAL_BACKOFF', - 'retryDelaySeconds' ,10, - 'responseTimeoutSeconds' ,600, - 'inputTemplate', jsonb '{}', - 'rateLimitPerFrequency' ,0, - 'rateLimitFrequencyInSeconds' ,1, - 'ownerEmail' ,'admin@sirenapp.io', - 'backoffScaleFactor' ,1 - ) - ); - \ No newline at end of file diff --git a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java new file mode 100644 index 0000000000..09c7d3ccde --- /dev/null +++ b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java @@ -0,0 +1,128 @@ +package com.netflix.conductor.rest.startup; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.context.event.EventListener; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.stereotype.Component; +import org.springframework.util.FileCopyUtils; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import static org.springframework.http.HttpHeaders.CONTENT_TYPE; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; + +@Component +public class SirenInitializer { + + private static final Logger LOGGER = LoggerFactory.getLogger(SirenInitializer.class); + + private final RestTemplate restTemplate; + + @Value("${server.port:8080}") + private int port; + + @Value("classpath:./siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json") + private Resource finalizeWorkflowExecutionEventHandler; + + @Value("classpath:./siren/tasks/finalizeWorkflowExecutionTask.json") + private Resource finalizeWorkflowExecutionTask; + + @Value("classpath:./siren/tasks/sendNotificationTask.json") + private Resource sendNotificationTask; + + @Value("classpath:./siren/tasks/sendWebhookTask.json") + private Resource sendWebhookTask; + + @Value("classpath:./siren/workflows/sirenFinalizeExecutionWorkflow.json") + private Resource sirenFinalizeExecutionWorkflow; + + @Value("classpath:./siren/workflows/sirenWebhookWorkflow.json") + private Resource sirenWebhookWorkflow; + + public SirenInitializer(RestTemplateBuilder restTemplateBuilder) { + this.restTemplate = restTemplateBuilder.build(); + } + + @EventListener(ApplicationReadyEvent.class) + public void setupSirenResources() { + LOGGER.info("Loading siren resources"); + createSirenResources(); + } + + private void createSirenResources() { + MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.add(CONTENT_TYPE, APPLICATION_JSON_VALUE); + createWorkflow(sirenFinalizeExecutionWorkflow, headers, "sirenFinalizeExecutionWorkflow"); + createWorkflow(sirenWebhookWorkflow, headers, "sirenWebhookWorkflow"); + LOGGER.info("Siren workflows are created"); + + createTask(sendNotificationTask, headers, "sendNotificationTask"); + createTask(sendWebhookTask, headers, "sendWebhookTask"); + createTask(finalizeWorkflowExecutionTask, headers, "finalizeWorkflowExecutionTask"); + LOGGER.info("Siren tasks are created"); + + createEventHandler( + finalizeWorkflowExecutionEventHandler, + headers, + "finalizeWorkflowExecutionEventHandler"); + LOGGER.info("Siren event handlers are created"); + } + + private void createWorkflow( + Resource resource, MultiValueMap headers, String workflowName) { + try { + HttpEntity request = new HttpEntity<>(readToString(resource), headers); + restTemplate.postForEntity(url("/api/metadata/workflow"), request, Map.class); + } catch (RestClientException e) { + LOGGER.info("Skipping create {} ", workflowName); + e.printStackTrace(); + } + } + + private void createTask( + Resource resource, MultiValueMap headers, String taskName) { + try { + HttpEntity request = new HttpEntity<>(readToString(resource), headers); + restTemplate.postForEntity(url("/api/metadata/taskdefs"), request, Map.class); + } catch (RestClientException e) { + LOGGER.info("Skipping create {} ", taskName); + e.printStackTrace(); + } + } + + private void createEventHandler( + Resource resource, MultiValueMap headers, String eventHandlerName) { + try { + HttpEntity request = new HttpEntity<>(readToString(resource), headers); + restTemplate.postForEntity(url("/api/event"), request, Map.class); + } catch (RestClientException e) { + LOGGER.info("Skipping create {} ", eventHandlerName); + e.printStackTrace(); + } + } + + private String readToString(Resource resource) { + try { + return FileCopyUtils.copyToString(new InputStreamReader(resource.getInputStream())); + } catch (IOException e) { + LOGGER.error("Error while loading siren resources", e); + throw new RuntimeException("Error reading resources", e); + } + } + + private String url(String path) { + // TODO replace with url + return "http://localhost:" + port + path; + } +} diff --git a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json new file mode 100644 index 0000000000..a640a28c5e --- /dev/null +++ b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json @@ -0,0 +1,26 @@ +{ + "name": "finalize_workflow_execution_event_handler", + "event": "conductor:finalize_workflow_execution_event", + "actions": [ + { + "action": "start_workflow", + "start_workflow": { + "name": "finalize_workflow_execution", + "input": { + "workflowType": "${workflowType}", + "version": "${version}", + "workflowId": "${workflowId}", + "correlationId": "${correlationId}", + "status": "${status}", + "input": "${input}", + "output": "${output}", + "reasonForIncompletion": "${reasonForIncompletion}", + "executionTime": "${executionTime}", + "event": "${event}" + } + }, + "expandInlineJSON": false + } + ], + "active": true +} \ No newline at end of file diff --git a/rest/src/main/resources/siren/tasks/finalizeWorkflowExecutionTask.json b/rest/src/main/resources/siren/tasks/finalizeWorkflowExecutionTask.json new file mode 100644 index 0000000000..eb19e7a45e --- /dev/null +++ b/rest/src/main/resources/siren/tasks/finalizeWorkflowExecutionTask.json @@ -0,0 +1,21 @@ +[ + { + "createdBy": "", + "accessPolicy": {}, + "name": "finalize_workflow_execution_task", + "description": "Finalize Workflow Execution Task", + "retryCount": 5, + "timeoutSeconds": 3600, + "inputKeys": [], + "outputKeys": [], + "timeoutPolicy": "TIME_OUT_WF", + "retryLogic": "EXPONENTIAL_BACKOFF", + "retryDelaySeconds": 10, + "responseTimeoutSeconds": 600, + "inputTemplate": {}, + "rateLimitPerFrequency": 0, + "rateLimitFrequencyInSeconds": 1, + "ownerEmail": "admin@sirenapp.io", + "backoffScaleFactor": 1 + } +] diff --git a/rest/src/main/resources/siren/tasks/sendNotificationTask.json b/rest/src/main/resources/siren/tasks/sendNotificationTask.json new file mode 100644 index 0000000000..4d331f70d5 --- /dev/null +++ b/rest/src/main/resources/siren/tasks/sendNotificationTask.json @@ -0,0 +1,21 @@ +[ + { + "createdBy": "", + "accessPolicy": {}, + "name": "send_notification_task", + "description": "Send Notification Task", + "retryCount": 5, + "timeoutSeconds": 3600, + "inputKeys": [], + "outputKeys": [], + "timeoutPolicy": "TIME_OUT_WF", + "retryLogic": "EXPONENTIAL_BACKOFF", + "retryDelaySeconds": 10, + "responseTimeoutSeconds": 600, + "inputTemplate": {}, + "rateLimitPerFrequency": 0, + "rateLimitFrequencyInSeconds": 1, + "ownerEmail": "admin@sirenapp.io", + "backoffScaleFactor": 1 + } +] diff --git a/rest/src/main/resources/siren/tasks/sendWebhookTask.json b/rest/src/main/resources/siren/tasks/sendWebhookTask.json new file mode 100644 index 0000000000..1878c2aaa0 --- /dev/null +++ b/rest/src/main/resources/siren/tasks/sendWebhookTask.json @@ -0,0 +1,21 @@ +[ + { + "createdBy": "", + "accessPolicy": {}, + "name": "send_webhook_task", + "description": "Send Webhook Task", + "retryCount": 5, + "timeoutSeconds": 3600, + "inputKeys": [], + "outputKeys": [], + "timeoutPolicy": "TIME_OUT_WF", + "retryLogic": "EXPONENTIAL_BACKOFF", + "retryDelaySeconds": 10, + "responseTimeoutSeconds": 600, + "inputTemplate": {}, + "rateLimitPerFrequency": 0, + "rateLimitFrequencyInSeconds": 1, + "ownerEmail": "admin@sirenapp.io", + "backoffScaleFactor": 1 + } +] diff --git a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json new file mode 100644 index 0000000000..5a2ed11608 --- /dev/null +++ b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json @@ -0,0 +1,31 @@ +{ + "accessPolicy": {}, + "name": "siren_finalize_execution_workflow", + "description": "Edit or extend this sample workflow. Set the workflow name to get started", + "version": 1, + "tasks": [ + { + "name": "finalize_workflow_execution_task", + "taskReferenceName": "finalize_workflow_execution_task_ref", + "inputParameters": { + "status": "${workflow.input.status}", + "externalExecutionId": "${workflow.input.workflowId}" + }, + "type": "SIMPLE", + "startDelay": 0, + "optional": false, + "asyncComplete": false, + "permissive": false + } + ], + "inputParameters": [], + "outputParameters": {}, + "schemaVersion": 2, + "restartable": true, + "workflowStatusListenerEnabled": false, + "ownerEmail": "example@email.com", + "timeoutPolicy": "ALERT_ONLY", + "timeoutSeconds": 0, + "variables": {}, + "inputTemplate": {} +} \ No newline at end of file diff --git a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json new file mode 100644 index 0000000000..a72a229b09 --- /dev/null +++ b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json @@ -0,0 +1,30 @@ +{ + "accessPolicy": {}, + "name": "siren_webhook_workflow", + "description": "Workflow for sending webhook", + "version": 1, + "tasks": [ + { + "name": "send_webhook_task", + "taskReferenceName": "send_webhook_task_ref", + "inputParameters": { + "notificationAuditId": "${workflow.input.notificationAuditId}" + }, + "type": "SIMPLE", + "startDelay": 0, + "optional": true, + "asyncComplete": false, + "permissive": false + } + ], + "inputParameters": [], + "outputParameters": {}, + "schemaVersion": 2, + "restartable": true, + "workflowStatusListenerEnabled": false, + "ownerEmail": "admin@sirenapp.io", + "timeoutPolicy": "ALERT_ONLY", + "timeoutSeconds": 0, + "variables": {}, + "inputTemplate": {} +} \ No newline at end of file From 15b71f40964a8e8342bfb017953572e3adefbb8b Mon Sep 17 00:00:00 2001 From: Muhammad Date: Tue, 20 Feb 2024 20:05:42 +0530 Subject: [PATCH 08/43] refactor: Add new line at EOF --- .../eventhandlers/finalizeWorkflowExecutionEventHandler.json | 2 +- .../siren/workflows/sirenFinalizeExecutionWorkflow.json | 2 +- .../main/resources/siren/workflows/sirenWebhookWorkflow.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json index a640a28c5e..4872db183b 100644 --- a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json +++ b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json @@ -23,4 +23,4 @@ } ], "active": true -} \ No newline at end of file +} diff --git a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json index 5a2ed11608..f4db2ccbfb 100644 --- a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json @@ -28,4 +28,4 @@ "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} -} \ No newline at end of file +} diff --git a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json index a72a229b09..3096211317 100644 --- a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json @@ -27,4 +27,4 @@ "timeoutSeconds": 0, "variables": {}, "inputTemplate": {} -} \ No newline at end of file +} From 12e613ffe9345393d071965b062de685e3e3b897 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Wed, 21 Feb 2024 09:49:07 +0530 Subject: [PATCH 09/43] feat: Handle errors while creating already existing resources --- .../rest/startup/SirenInitializer.java | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java index 09c7d3ccde..71d964ed8e 100644 --- a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java +++ b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java @@ -1,3 +1,15 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ package com.netflix.conductor.rest.startup; import java.io.IOException; @@ -24,13 +36,13 @@ @Component public class SirenInitializer { - + private static final String ALREADY_EXISTS_KEYWORD = "already exists"; private static final Logger LOGGER = LoggerFactory.getLogger(SirenInitializer.class); private final RestTemplate restTemplate; - @Value("${server.port:8080}") - private int port; + @Value("${server.url:http://localhost:8080}") + private String url; @Value("classpath:./siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json") private Resource finalizeWorkflowExecutionEventHandler; @@ -63,52 +75,52 @@ public void setupSirenResources() { private void createSirenResources() { MultiValueMap headers = new LinkedMultiValueMap<>(); headers.add(CONTENT_TYPE, APPLICATION_JSON_VALUE); - createWorkflow(sirenFinalizeExecutionWorkflow, headers, "sirenFinalizeExecutionWorkflow"); - createWorkflow(sirenWebhookWorkflow, headers, "sirenWebhookWorkflow"); + createWorkflow(sirenFinalizeExecutionWorkflow, headers); + createWorkflow(sirenWebhookWorkflow, headers); LOGGER.info("Siren workflows are created"); - createTask(sendNotificationTask, headers, "sendNotificationTask"); - createTask(sendWebhookTask, headers, "sendWebhookTask"); - createTask(finalizeWorkflowExecutionTask, headers, "finalizeWorkflowExecutionTask"); + createTask(sendNotificationTask, headers); + createTask(sendWebhookTask, headers); + createTask(finalizeWorkflowExecutionTask, headers); LOGGER.info("Siren tasks are created"); - createEventHandler( - finalizeWorkflowExecutionEventHandler, - headers, - "finalizeWorkflowExecutionEventHandler"); + createEventHandler(finalizeWorkflowExecutionEventHandler, headers); LOGGER.info("Siren event handlers are created"); } - private void createWorkflow( - Resource resource, MultiValueMap headers, String workflowName) { + private void createWorkflow(Resource resource, MultiValueMap headers) { try { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url("/api/metadata/workflow"), request, Map.class); + restTemplate.postForEntity(url + "/api/metadata/workflow", request, Map.class); } catch (RestClientException e) { - LOGGER.info("Skipping create {} ", workflowName); - e.printStackTrace(); + handleException(e); } } - private void createTask( - Resource resource, MultiValueMap headers, String taskName) { + private void createTask(Resource resource, MultiValueMap headers) { try { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url("/api/metadata/taskdefs"), request, Map.class); + restTemplate.postForEntity(url + "/api/metadata/taskdefs", request, Map.class); } catch (RestClientException e) { - LOGGER.info("Skipping create {} ", taskName); - e.printStackTrace(); + handleException(e); } } - private void createEventHandler( - Resource resource, MultiValueMap headers, String eventHandlerName) { + private void createEventHandler(Resource resource, MultiValueMap headers) { try { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url("/api/event"), request, Map.class); + restTemplate.postForEntity(url + "/api/event", request, Map.class); } catch (RestClientException e) { - LOGGER.info("Skipping create {} ", eventHandlerName); - e.printStackTrace(); + handleException(e); + } + } + + private void handleException(RestClientException e) { + if (e.getMessage().contains(ALREADY_EXISTS_KEYWORD)) { + LOGGER.info("Skipping creation: {}", e.getMessage()); + } else { + LOGGER.error("Error while creation ", e); + throw e; } } @@ -120,9 +132,4 @@ private String readToString(Resource resource) { throw new RuntimeException("Error reading resources", e); } } - - private String url(String path) { - // TODO replace with url - return "http://localhost:" + port + path; - } } From a63195f6a67aec6af0e03c507a817b68be6c1f3a Mon Sep 17 00:00:00 2001 From: Muhammad Date: Wed, 21 Feb 2024 10:05:48 +0530 Subject: [PATCH 10/43] refactor: Refactor update task method --- .../conductor/rest/startup/SirenInitializer.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java index 71d964ed8e..6456077bf7 100644 --- a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java +++ b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java @@ -79,9 +79,9 @@ private void createSirenResources() { createWorkflow(sirenWebhookWorkflow, headers); LOGGER.info("Siren workflows are created"); - createTask(sendNotificationTask, headers); - createTask(sendWebhookTask, headers); - createTask(finalizeWorkflowExecutionTask, headers); + updateTask(sendNotificationTask, headers); + updateTask(sendWebhookTask, headers); + updateTask(finalizeWorkflowExecutionTask, headers); LOGGER.info("Siren tasks are created"); createEventHandler(finalizeWorkflowExecutionEventHandler, headers); @@ -97,13 +97,9 @@ private void createWorkflow(Resource resource, MultiValueMap hea } } - private void createTask(Resource resource, MultiValueMap headers) { - try { - HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url + "/api/metadata/taskdefs", request, Map.class); - } catch (RestClientException e) { - handleException(e); - } + private void updateTask(Resource resource, MultiValueMap headers) { + HttpEntity request = new HttpEntity<>(readToString(resource), headers); + restTemplate.postForEntity(url + "/api/metadata/taskdefs", request, Map.class); } private void createEventHandler(Resource resource, MultiValueMap headers) { From 4d7924eaffecd12b9220afd89b635321cbd69a36 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Wed, 21 Feb 2024 10:08:07 +0530 Subject: [PATCH 11/43] feat: Update finalize execution workflow description --- .../siren/workflows/sirenFinalizeExecutionWorkflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json index f4db2ccbfb..c9494465ba 100644 --- a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json @@ -1,7 +1,7 @@ { "accessPolicy": {}, "name": "siren_finalize_execution_workflow", - "description": "Edit or extend this sample workflow. Set the workflow name to get started", + "description": "Workflow for finalizing workflow execution", "version": 1, "tasks": [ { From bf571b270f6d0592fe5df9a090f603503c8f3170 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Wed, 21 Feb 2024 11:40:23 +0530 Subject: [PATCH 12/43] feat: Update api url --- .../netflix/conductor/rest/startup/SirenInitializer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java index 6456077bf7..ec15526456 100644 --- a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java +++ b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java @@ -91,7 +91,7 @@ private void createSirenResources() { private void createWorkflow(Resource resource, MultiValueMap headers) { try { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url + "/api/metadata/workflow", request, Map.class); + restTemplate.postForEntity(url + "/api/metadata/workflow/", request, Map.class); } catch (RestClientException e) { handleException(e); } @@ -99,13 +99,13 @@ private void createWorkflow(Resource resource, MultiValueMap hea private void updateTask(Resource resource, MultiValueMap headers) { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url + "/api/metadata/taskdefs", request, Map.class); + restTemplate.postForEntity(url + "/api/metadata/taskdefs/", request, Map.class); } private void createEventHandler(Resource resource, MultiValueMap headers) { try { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url + "/api/event", request, Map.class); + restTemplate.postForEntity(url + "/api/event/", request, Map.class); } catch (RestClientException e) { handleException(e); } From 9933a0b34e514876e263ae4618dae3f7e9ce3a4a Mon Sep 17 00:00:00 2001 From: Sreeraj Rajeendran Date: Thu, 29 Feb 2024 18:44:48 +0530 Subject: [PATCH 13/43] fix: Initialize Siren resources as per property value * Without the property value check, build test was failing during integration test --- .../conductor/rest/startup/SirenInitializer.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java index ec15526456..f3a5a50970 100644 --- a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java +++ b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java @@ -41,6 +41,9 @@ public class SirenInitializer { private final RestTemplate restTemplate; + @Value("${loadSirenResources:false}") + private boolean loadSirenResources; + @Value("${server.url:http://localhost:8080}") private String url; @@ -68,8 +71,10 @@ public SirenInitializer(RestTemplateBuilder restTemplateBuilder) { @EventListener(ApplicationReadyEvent.class) public void setupSirenResources() { - LOGGER.info("Loading siren resources"); - createSirenResources(); + if (loadSirenResources) { + LOGGER.info("Loading siren resources"); + createSirenResources(); + } } private void createSirenResources() { @@ -91,7 +96,7 @@ private void createSirenResources() { private void createWorkflow(Resource resource, MultiValueMap headers) { try { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url + "/api/metadata/workflow/", request, Map.class); + restTemplate.postForEntity(url + "/api/metadata/workflow", request, Map.class); } catch (RestClientException e) { handleException(e); } @@ -99,13 +104,13 @@ private void createWorkflow(Resource resource, MultiValueMap hea private void updateTask(Resource resource, MultiValueMap headers) { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url + "/api/metadata/taskdefs/", request, Map.class); + restTemplate.postForEntity(url + "/api/metadata/taskdefs", request, Map.class); } private void createEventHandler(Resource resource, MultiValueMap headers) { try { HttpEntity request = new HttpEntity<>(readToString(resource), headers); - restTemplate.postForEntity(url + "/api/event/", request, Map.class); + restTemplate.postForEntity(url + "/api/event", request, Map.class); } catch (RestClientException e) { handleException(e); } From ac791a55f504d915293b683d28dec50e35d2d16c Mon Sep 17 00:00:00 2001 From: Muhammad Date: Mon, 4 Mar 2024 11:43:06 +0530 Subject: [PATCH 14/43] feat: Add traceId to sirenWebhookWorkflow --- .../main/resources/siren/workflows/sirenWebhookWorkflow.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json index 3096211317..368f988e9a 100644 --- a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json @@ -8,7 +8,8 @@ "name": "send_webhook_task", "taskReferenceName": "send_webhook_task_ref", "inputParameters": { - "notificationAuditId": "${workflow.input.notificationAuditId}" + "notificationAuditId": "${workflow.input.notificationAuditId}", + "traceId":"${workflow.input.traceId}" }, "type": "SIMPLE", "startDelay": 0, From dcf2aff93f2455292d2c11afde3436cec4674f32 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Mon, 4 Mar 2024 11:50:04 +0530 Subject: [PATCH 15/43] fix: Correct workflow name in event handler --- .../eventhandlers/finalizeWorkflowExecutionEventHandler.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json index 4872db183b..5d5ab7b733 100644 --- a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json +++ b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json @@ -5,7 +5,7 @@ { "action": "start_workflow", "start_workflow": { - "name": "finalize_workflow_execution", + "name": "siren_finalize_execution_workflow", "input": { "workflowType": "${workflowType}", "version": "${version}", From 175dcb898b3289fb3834b3ae7754ed7febb2ecbe Mon Sep 17 00:00:00 2001 From: Muhammad Date: Tue, 5 Mar 2024 12:18:45 +0530 Subject: [PATCH 16/43] feat: Add workflow input to finalize_workflow_execution_task input-params --- .../eventhandlers/finalizeWorkflowExecutionEventHandler.json | 4 ++-- .../siren/workflows/sirenFinalizeExecutionWorkflow.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json index 5d5ab7b733..10a446ccec 100644 --- a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json +++ b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json @@ -12,14 +12,14 @@ "workflowId": "${workflowId}", "correlationId": "${correlationId}", "status": "${status}", - "input": "${input}", + "workflowInput": "${input}", "output": "${output}", "reasonForIncompletion": "${reasonForIncompletion}", "executionTime": "${executionTime}", "event": "${event}" } }, - "expandInlineJSON": false + "expandInlineJSON": true } ], "active": true diff --git a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json index c9494465ba..fcaaddfee1 100644 --- a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json @@ -9,7 +9,8 @@ "taskReferenceName": "finalize_workflow_execution_task_ref", "inputParameters": { "status": "${workflow.input.status}", - "externalExecutionId": "${workflow.input.workflowId}" + "externalExecutionId": "${workflow.input.workflowId}", + "input": "${workflow.input.workflowInput}" }, "type": "SIMPLE", "startDelay": 0, From 1ed26e8b88d383b63318137f8c085aa2f9f21ee2 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Wed, 6 Mar 2024 11:18:30 +0530 Subject: [PATCH 17/43] feat: Remove traceId param from siren webhook workflow --- .../main/resources/siren/workflows/sirenWebhookWorkflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json index 368f988e9a..450505f423 100644 --- a/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenWebhookWorkflow.json @@ -9,7 +9,7 @@ "taskReferenceName": "send_webhook_task_ref", "inputParameters": { "notificationAuditId": "${workflow.input.notificationAuditId}", - "traceId":"${workflow.input.traceId}" + "workflowExecutionId": "${workflow.input.workflowExecutionId}" }, "type": "SIMPLE", "startDelay": 0, From 0938446d293491e41cc4030f8d9cef3352f81124 Mon Sep 17 00:00:00 2001 From: Muhammad Date: Wed, 6 Mar 2024 16:55:08 +0530 Subject: [PATCH 18/43] feat: Update owner-email of finalize workflow --- .../siren/workflows/sirenFinalizeExecutionWorkflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json index fcaaddfee1..cfaf4141fa 100644 --- a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json @@ -24,7 +24,7 @@ "schemaVersion": 2, "restartable": true, "workflowStatusListenerEnabled": false, - "ownerEmail": "example@email.com", + "ownerEmail": "admin@sirenapp.io", "timeoutPolicy": "ALERT_ONLY", "timeoutSeconds": 0, "variables": {}, From bee1be32e33e1ae03a3c1aceefcf607b0bdf8c38 Mon Sep 17 00:00:00 2001 From: Vigneshwar Date: Mon, 25 Mar 2024 09:24:37 +0530 Subject: [PATCH 19/43] Http failure handling (#7) --- .../com/netflix/conductor/tasks/http/HttpTask.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java index d85fa24840..942ae234a1 100644 --- a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java +++ b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +25,7 @@ import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.util.MultiValueMap; +import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -117,7 +119,9 @@ public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor execu if (response.body != null) { task.setReasonForIncompletion(response.body.toString()); } else { - task.setReasonForIncompletion("No response from the remote service"); + task.setReasonForIncompletion( + Objects.requireNonNullElse( + response.reasonPhrase, "No response from the remote service")); } task.setStatus(TaskModel.Status.FAILED); } @@ -175,12 +179,16 @@ protected HttpResponse httpCall(Input input) throws Exception { if (responseEntity.getStatusCode().is2xxSuccessful() && responseEntity.hasBody()) { response.body = extractBody(responseEntity.getBody()); } - response.statusCode = responseEntity.getStatusCodeValue(); response.reasonPhrase = HttpStatus.valueOf(responseEntity.getStatusCode().value()).getReasonPhrase(); response.headers = responseEntity.getHeaders(); return response; + } catch (HttpClientErrorException ex) { + response.headers = ex.getResponseHeaders(); + response.statusCode = ex.getStatusCode().value(); + response.reasonPhrase = ex.getStatusText(); + return response; } catch (RestClientException ex) { LOGGER.error( String.format( From 9c60da063cc7e0ad2bc36189f74da170363b1761 Mon Sep 17 00:00:00 2001 From: Sherif-kv <150251195+Sherif-kv@users.noreply.github.com> Date: Tue, 26 Mar 2024 12:06:09 +0530 Subject: [PATCH 20/43] fix: 5XX series issues (#8) --- .../com/netflix/conductor/tasks/http/HttpTask.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java index 942ae234a1..efa5bb2f63 100644 --- a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java +++ b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java @@ -25,7 +25,7 @@ import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.util.MultiValueMap; -import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpStatusCodeException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -120,8 +120,12 @@ public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor execu task.setReasonForIncompletion(response.body.toString()); } else { task.setReasonForIncompletion( - Objects.requireNonNullElse( - response.reasonPhrase, "No response from the remote service")); + String.format( + "%d: %s", + response.statusCode, + Objects.requireNonNullElse( + response.reasonPhrase, + "No response from the remote service"))); } task.setStatus(TaskModel.Status.FAILED); } @@ -184,7 +188,7 @@ protected HttpResponse httpCall(Input input) throws Exception { HttpStatus.valueOf(responseEntity.getStatusCode().value()).getReasonPhrase(); response.headers = responseEntity.getHeaders(); return response; - } catch (HttpClientErrorException ex) { + } catch (HttpStatusCodeException ex) { response.headers = ex.getResponseHeaders(); response.statusCode = ex.getStatusCode().value(); response.reasonPhrase = ex.getStatusText(); From 7bd459b3d78761e204c88172aac8149e735b9444 Mon Sep 17 00:00:00 2001 From: Sherif-kv <150251195+Sherif-kv@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:38:34 +0530 Subject: [PATCH 21/43] chrom: http 1XX response (#9) --- .../main/java/com/netflix/conductor/tasks/http/HttpTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java index efa5bb2f63..b9016c993a 100644 --- a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java +++ b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java @@ -109,7 +109,7 @@ public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor execu response.statusCode, response.body, task.getTaskId()); - if (response.statusCode > 199 && response.statusCode < 300) { + if (response.statusCode >= 100 && response.statusCode < 300) { if (isAsyncComplete(task)) { task.setStatus(TaskModel.Status.IN_PROGRESS); } else { From 7d4c2c7ce34aef11aa315d2f4b4dcdc0d466e3da Mon Sep 17 00:00:00 2001 From: Sherif-kv <150251195+Sherif-kv@users.noreply.github.com> Date: Tue, 26 Mar 2024 15:57:46 +0530 Subject: [PATCH 22/43] Revert "chrom: http 1XX response (#9)" (#10) This reverts commit 7bd459b3d78761e204c88172aac8149e735b9444. --- .../main/java/com/netflix/conductor/tasks/http/HttpTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java index b9016c993a..efa5bb2f63 100644 --- a/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java +++ b/http-task/src/main/java/com/netflix/conductor/tasks/http/HttpTask.java @@ -109,7 +109,7 @@ public void start(WorkflowModel workflow, TaskModel task, WorkflowExecutor execu response.statusCode, response.body, task.getTaskId()); - if (response.statusCode >= 100 && response.statusCode < 300) { + if (response.statusCode > 199 && response.statusCode < 300) { if (isAsyncComplete(task)) { task.setStatus(TaskModel.Status.IN_PROGRESS); } else { From 367b1f071a035fd7b9ccd6f638410ef36f5c2f36 Mon Sep 17 00:00:00 2001 From: Muhammad <112552552+muhammad-keyvalue@users.noreply.github.com> Date: Thu, 18 Apr 2024 00:36:28 +0530 Subject: [PATCH 23/43] feat: Add campaign workflow definition (#11) --- .../rest/startup/SirenInitializer.java | 4 ++ .../workflows/sirenCampaignWorkflow.json | 57 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json diff --git a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java index f3a5a50970..7a3fc396b6 100644 --- a/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java +++ b/rest/src/main/java/com/netflix/conductor/rest/startup/SirenInitializer.java @@ -65,6 +65,9 @@ public class SirenInitializer { @Value("classpath:./siren/workflows/sirenWebhookWorkflow.json") private Resource sirenWebhookWorkflow; + @Value("classpath:./siren/workflows/sirenCampaignWorkflow.json") + private Resource sirenCampaignWorkflow; + public SirenInitializer(RestTemplateBuilder restTemplateBuilder) { this.restTemplate = restTemplateBuilder.build(); } @@ -82,6 +85,7 @@ private void createSirenResources() { headers.add(CONTENT_TYPE, APPLICATION_JSON_VALUE); createWorkflow(sirenFinalizeExecutionWorkflow, headers); createWorkflow(sirenWebhookWorkflow, headers); + createWorkflow(sirenCampaignWorkflow, headers); LOGGER.info("Siren workflows are created"); updateTask(sendNotificationTask, headers); diff --git a/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json b/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json new file mode 100644 index 0000000000..b0246432ff --- /dev/null +++ b/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json @@ -0,0 +1,57 @@ +{ + "accessPolicy": {}, + "name": "siren_campaign_workflow", + "description": "Workflow for executing campaigns", + "version": 1, + "tasks": [ + { + "name": "template_fork", + "taskReferenceName": "temp_pSzeL9pM3AMtpm7hD-WcA_template_fork", + "inputParameters": {}, + "type": "FORK_JOIN", + "forkTasks": [ + [ + { + "name": "send_notification_task", + "taskReferenceName": "prov_j7ooKIGve0iG2n2tH-1Ds_send_notification_task", + "inputParameters": { + "workflowExecutionId": "${workflow.input.workflowExecutionId}", + "providerIntegrationId": "${workflow.input.providerIntegrationId}", + "templateId": "${workflow.input.templateId}", + "notifyVariables": "${workflow.input.notifyVariables}", + "templateVariables": "${workflow.input.templateVariables}" + }, + "type": "SIMPLE", + "startDelay": 0, + "optional": false, + "asyncComplete": false, + "permissive": false + } + ] + ], + "startDelay": 0, + "optional": false, + "asyncComplete": false, + "permissive": false + }, + { + "name": "template_join", + "taskReferenceName": "temp_pSzeL9pM3AMtpm7hD-WcA_template_join", + "inputParameters": {}, + "type": "JOIN", + "startDelay": 0, + "optional": false, + "asyncComplete": false, + "permissive": false + } + ], + "inputParameters": [], + "outputParameters": {}, + "schemaVersion": 2, + "restartable": true, + "workflowStatusListenerEnabled": false, + "ownerEmail": "admin@sirenapp.io", + "timeoutSeconds": 0, + "variables": {}, + "inputTemplate": {} +} \ No newline at end of file From 53ba2171fc300e6bd7beb8f10b625e5871b0eb82 Mon Sep 17 00:00:00 2001 From: Muhammad <112552552+muhammad-keyvalue@users.noreply.github.com> Date: Sat, 27 Apr 2024 17:14:45 +0530 Subject: [PATCH 24/43] feat: Update siren finalize eventhandler, finalize workflow and campaign workflow (#13) --- .../eventhandlers/finalizeWorkflowExecutionEventHandler.json | 1 - .../main/resources/siren/workflows/sirenCampaignWorkflow.json | 4 ++-- .../siren/workflows/sirenFinalizeExecutionWorkflow.json | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json index 10a446ccec..d14435c114 100644 --- a/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json +++ b/rest/src/main/resources/siren/eventhandlers/finalizeWorkflowExecutionEventHandler.json @@ -12,7 +12,6 @@ "workflowId": "${workflowId}", "correlationId": "${correlationId}", "status": "${status}", - "workflowInput": "${input}", "output": "${output}", "reasonForIncompletion": "${reasonForIncompletion}", "executionTime": "${executionTime}", diff --git a/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json b/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json index b0246432ff..aa5b54afef 100644 --- a/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenCampaignWorkflow.json @@ -23,7 +23,7 @@ }, "type": "SIMPLE", "startDelay": 0, - "optional": false, + "optional": true, "asyncComplete": false, "permissive": false } @@ -49,7 +49,7 @@ "outputParameters": {}, "schemaVersion": 2, "restartable": true, - "workflowStatusListenerEnabled": false, + "workflowStatusListenerEnabled": true, "ownerEmail": "admin@sirenapp.io", "timeoutSeconds": 0, "variables": {}, diff --git a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json index cfaf4141fa..3c0373cb7c 100644 --- a/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json +++ b/rest/src/main/resources/siren/workflows/sirenFinalizeExecutionWorkflow.json @@ -9,8 +9,7 @@ "taskReferenceName": "finalize_workflow_execution_task_ref", "inputParameters": { "status": "${workflow.input.status}", - "externalExecutionId": "${workflow.input.workflowId}", - "input": "${workflow.input.workflowInput}" + "externalExecutionId": "${workflow.input.workflowId}" }, "type": "SIMPLE", "startDelay": 0, From 8012938b5c95b409c0aef53b13b5249b8145d6db Mon Sep 17 00:00:00 2001 From: Arjun Shibu <110371027+arjun-keyvalue@users.noreply.github.com> Date: Wed, 8 May 2024 12:20:58 +0530 Subject: [PATCH 25/43] fix: Increase email attachment size to support minimum 10mb (#12) --- .../netflix/conductor/s3/config/S3Configuration.java | 8 +++++++- .../com/netflix/conductor/s3/config/S3Properties.java | 11 +++++++++++ server/build.gradle | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java index b14d79395a..7dce773d12 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java @@ -41,6 +41,12 @@ public ExternalPayloadStorage s3ExternalPayloadStorage( matchIfMissing = true) @Bean public AmazonS3 amazonS3(S3Properties properties) { - return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); + return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); +// TODO: Add localstack support to test locally +// return AmazonS3ClientBuilder.standard() +// .withEndpointConfiguration( +// new AwsClientBuilder.EndpointConfiguration( +// properties.getEndpoint(), properties.getRegion())) +// .build(); } } diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java index 9c41b4a107..b853446fdd 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java @@ -54,4 +54,15 @@ public String getRegion() { public void setRegion(String region) { this.region = region; } + +// TODO: Add localstack support to test locally +// private String endpoint = "http://s3.localhost.localstack.cloud:4566"; +// +// public String getEndpoint() { +// return endpoint; +// } +// +// public void setEndpoint(String endpoint) { +// this.endpoint = endpoint; +// } } diff --git a/server/build.gradle b/server/build.gradle index 55d1ec2111..a387b10d9b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -73,6 +73,7 @@ dependencies { runtimeOnly group: 'com.netflix.conductor', name: 'conductor-postgres-persistence', version: '3.9.1' implementation "com.netflix.spectator:spectator-reg-metrics3:${version_spectator}" + implementation 'javax.xml.bind:jaxb-api:2.3.1' runtimeOnly "org.glassfish.jaxb:jaxb-runtime:${revJAXB}" From bc834a1e8939a43b024990cdf2bb2b913637b232 Mon Sep 17 00:00:00 2001 From: Sherif-kv <150251195+Sherif-kv@users.noreply.github.com> Date: Wed, 8 May 2024 13:46:29 +0530 Subject: [PATCH 26/43] fix: spotless (#14) --- .../conductor/s3/config/S3Configuration.java | 14 ++++++------- .../conductor/s3/config/S3Properties.java | 20 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java index 7dce773d12..106f61664d 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java @@ -41,12 +41,12 @@ public ExternalPayloadStorage s3ExternalPayloadStorage( matchIfMissing = true) @Bean public AmazonS3 amazonS3(S3Properties properties) { - return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); -// TODO: Add localstack support to test locally -// return AmazonS3ClientBuilder.standard() -// .withEndpointConfiguration( -// new AwsClientBuilder.EndpointConfiguration( -// properties.getEndpoint(), properties.getRegion())) -// .build(); + return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); + // TODO: Add localstack support to test locally + // return AmazonS3ClientBuilder.standard() + // .withEndpointConfiguration( + // new AwsClientBuilder.EndpointConfiguration( + // properties.getEndpoint(), properties.getRegion())) + // .build(); } } diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java index b853446fdd..f399cd0d7d 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java @@ -55,14 +55,14 @@ public void setRegion(String region) { this.region = region; } -// TODO: Add localstack support to test locally -// private String endpoint = "http://s3.localhost.localstack.cloud:4566"; -// -// public String getEndpoint() { -// return endpoint; -// } -// -// public void setEndpoint(String endpoint) { -// this.endpoint = endpoint; -// } + // TODO: Add localstack support to test locally + // private String endpoint = "http://s3.localhost.localstack.cloud:4566"; + // + // public String getEndpoint() { + // return endpoint; + // } + // + // public void setEndpoint(String endpoint) { + // this.endpoint = endpoint; + // } } From a0c0152fecbcdab8c5ae1e4dea3765c8f741939c Mon Sep 17 00:00:00 2001 From: Sherif-kv <150251195+Sherif-kv@users.noreply.github.com> Date: Tue, 14 May 2024 09:59:35 +0530 Subject: [PATCH 27/43] debug_external_bucket (#15) --- .../com/netflix/conductor/s3/storage/S3PayloadStorage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java index 19ac68d27d..c9391bf284 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java @@ -132,7 +132,8 @@ public void upload(String path, InputStream payload, long payloadSize) { } catch (SdkClientException e) { String msg = String.format( - "Error uploading to S3 - path:%s, payloadSize: %d", path, payloadSize); + "Error uploading to S3 - path:%s, payloadSize: %d, bucketName: %s", + path, payloadSize, bucketName); LOGGER.error(msg, e); throw new TransientException(msg, e); } From 714f1c160f80941208f0143a70e69c3e08048eb9 Mon Sep 17 00:00:00 2001 From: Sherif-kv <150251195+Sherif-kv@users.noreply.github.com> Date: Tue, 14 May 2024 17:30:59 +0530 Subject: [PATCH 28/43] chore: stack trace (#16) --- .../java/com/netflix/conductor/s3/storage/S3PayloadStorage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java index c9391bf284..4a70ac033e 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/storage/S3PayloadStorage.java @@ -134,6 +134,7 @@ public void upload(String path, InputStream payload, long payloadSize) { String.format( "Error uploading to S3 - path:%s, payloadSize: %d, bucketName: %s", path, payloadSize, bucketName); + e.printStackTrace(); LOGGER.error(msg, e); throw new TransientException(msg, e); } From f85619d63169517686c7a58385fa3a160518a32d Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Fri, 24 May 2024 15:41:25 +0530 Subject: [PATCH 29/43] fix: ci file for pushing image to public ecr --- .github/workflows/ci-server-.yaml | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml index 0fcb600762..95f9f35be8 100644 --- a/.github/workflows/ci-server-.yaml +++ b/.github/workflows/ci-server-.yaml @@ -33,6 +33,7 @@ jobs: ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} + PUBLIC_CONDUCTOR_ECR: ${{ steps.vars.outputs.PUBLIC_CONDUCTOR_ECR}} steps: - id: vars @@ -49,6 +50,9 @@ jobs: "dev") ENV="dev" ;; + "add/ci-public-ecr-push") + ENV="dev" + ;; "stg") ENV="stg" ;; @@ -75,6 +79,7 @@ jobs: echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "PUBLIC_CONDUCTOR_ECR=sandbox-conductor-server" >> $GITHUB_OUTPUT else echo "Branch not configured!" exit 1 @@ -108,6 +113,7 @@ jobs: ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} ENVIRONMENT_BUCKET: ${{needs.prepare-env.outputs.ENVIRONMENT_BUCKET}} IMAGE_TAG: ${{ github.event.inputs.tag }} + PUBLIC_CONDUCTOR_ECR: ${{needs.prepare-env.outputs.PUBLIC_CONDUCTOR_ECR}} outputs: ECR_REPO: ${{ steps.build.outputs.ECR_REPO }} APP_IMAGE: ${{ steps.image.outputs.APP_IMAGE }} @@ -150,6 +156,38 @@ jobs: provenance: false platforms: linux/amd64 tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + - name: Login to Amazon ECR Public + if: ${{ env.ENV == 'dev'}} + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + - name: Push image to Public ECR + if: ${{ env.ENV == 'dev'}} + id: push-to-public-ecr + env: + PVT_ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + PUBLIC_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} + PUBLIC_REGISTRY_ALIAS: kvsiren-${{ env.ENV }} + IMAGE_TAG: ${{ github.sha }} + run: | + docker images + docker tag ${{ env.PVT_ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} + docker push ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} + + # - name: Build and push to Public ECR + # if: ${{ env.ENV == 'dev'}} + # id: build + # uses: docker/build-push-action@v5.1.0 + # env: + # ECR_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} + # with: + # context: . + # file: ./Dockerfile + # push: true + # provenance: false + # platforms: linux/amd64 + # tags: ${{ env.ECR_REGISTRY }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} - name: Image name id: image From c25072de2f9d1f4adc031d543ecfb461b8f2a924 Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Fri, 24 May 2024 16:07:21 +0530 Subject: [PATCH 30/43] fix: ci with region change for public ecr login --- .github/workflows/ci-server-.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml index 95f9f35be8..50ec16313c 100644 --- a/.github/workflows/ci-server-.yaml +++ b/.github/workflows/ci-server-.yaml @@ -156,12 +156,20 @@ jobs: provenance: false platforms: linux/amd64 tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets[env.AWS_ROLE] }} + aws-region: us-east-1 + - name: Login to Amazon ECR Public if: ${{ env.ENV == 'dev'}} id: login-ecr-public uses: aws-actions/amazon-ecr-login@v2 with: registry-type: public + - name: Push image to Public ECR if: ${{ env.ENV == 'dev'}} id: push-to-public-ecr From 123f3ecaf619147a2346783f825da59f9e8e03f0 Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Fri, 24 May 2024 16:39:26 +0530 Subject: [PATCH 31/43] add: docker build action for public ecr --- .github/workflows/ci-server-.yaml | 49 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml index 50ec16313c..3c1d7727e6 100644 --- a/.github/workflows/ci-server-.yaml +++ b/.github/workflows/ci-server-.yaml @@ -157,7 +157,8 @@ jobs: platforms: linux/amd64 tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - - name: Configure AWS credentials + - name: Configure AWS credentials for Public ECR + if: ${{ env.ENV == 'dev'}} uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets[env.AWS_ROLE] }} @@ -169,33 +170,33 @@ jobs: uses: aws-actions/amazon-ecr-login@v2 with: registry-type: public + + # - name: Push image to Public ECR + # if: ${{ env.ENV == 'dev'}} + # id: push-to-public-ecr + # env: + # PVT_ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # PUBLIC_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} + # PUBLIC_REGISTRY_ALIAS: kvsiren-${{ env.ENV }} + # run: | + # docker images + # docker tag ${{ env.PVT_ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} + # docker push ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} - - name: Push image to Public ECR + - name: Build and push to Public ECR if: ${{ env.ENV == 'dev'}} - id: push-to-public-ecr + id: build + uses: docker/build-push-action@v5.1.0 env: - PVT_ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} PUBLIC_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} - PUBLIC_REGISTRY_ALIAS: kvsiren-${{ env.ENV }} - IMAGE_TAG: ${{ github.sha }} - run: | - docker images - docker tag ${{ env.PVT_ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} - docker push ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} - - # - name: Build and push to Public ECR - # if: ${{ env.ENV == 'dev'}} - # id: build - # uses: docker/build-push-action@v5.1.0 - # env: - # ECR_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} - # with: - # context: . - # file: ./Dockerfile - # push: true - # provenance: false - # platforms: linux/amd64 - # tags: ${{ env.ECR_REGISTRY }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} + PUBLIC_REGISTRY_ALIAS: kvsiren-${{ env.ENV }} + with: + context: . + file: ./Dockerfile + push: true + provenance: false + platforms: linux/amd64 + tags: ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} - name: Image name id: image From 541325059d17110706cc5f41036ac56e31c9d70d Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Fri, 24 May 2024 16:41:32 +0530 Subject: [PATCH 32/43] fix: build step name for public ecr --- .github/workflows/ci-server-.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml index 3c1d7727e6..a93044c853 100644 --- a/.github/workflows/ci-server-.yaml +++ b/.github/workflows/ci-server-.yaml @@ -185,7 +185,7 @@ jobs: - name: Build and push to Public ECR if: ${{ env.ENV == 'dev'}} - id: build + id: build-public uses: docker/build-push-action@v5.1.0 env: PUBLIC_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} From 6dc13a4a9d1bf38e7efa4770c6934f675b3f7761 Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Fri, 24 May 2024 16:55:23 +0530 Subject: [PATCH 33/43] fox: ci trigger for conductor server --- .github/workflows/ci-server-.yaml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml index a93044c853..52d570d7e0 100644 --- a/.github/workflows/ci-server-.yaml +++ b/.github/workflows/ci-server-.yaml @@ -49,10 +49,7 @@ jobs: case $BRANCH in "dev") ENV="dev" - ;; - "add/ci-public-ecr-push") - ENV="dev" - ;; + ;; "stg") ENV="stg" ;; @@ -169,19 +166,7 @@ jobs: id: login-ecr-public uses: aws-actions/amazon-ecr-login@v2 with: - registry-type: public - - # - name: Push image to Public ECR - # if: ${{ env.ENV == 'dev'}} - # id: push-to-public-ecr - # env: - # PVT_ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # PUBLIC_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} - # PUBLIC_REGISTRY_ALIAS: kvsiren-${{ env.ENV }} - # run: | - # docker images - # docker tag ${{ env.PVT_ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} - # docker push ${{ env.PUBLIC_REGISTRY }}/${{ env.PUBLIC_REGISTRY_ALIAS }}/${{ env.PUBLIC_CONDUCTOR_ECR }}:${{ env.IMAGE_TAG }} + registry-type: public - name: Build and push to Public ECR if: ${{ env.ENV == 'dev'}} From f7e3d8e0b4632e4bd340e089ad6be84c4c35db2c Mon Sep 17 00:00:00 2001 From: Sherif-kv <150251195+Sherif-kv@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:19:54 +0530 Subject: [PATCH 34/43] Update action deployment verison --- .github/actions/deploy-ecs/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/deploy-ecs/action.yaml b/.github/actions/deploy-ecs/action.yaml index 649e6775a1..39097023e0 100644 --- a/.github/actions/deploy-ecs/action.yaml +++ b/.github/actions/deploy-ecs/action.yaml @@ -52,7 +52,7 @@ runs: jq 'del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy)' ${{ steps.task_def.outputs.task-definition }} > updated-task-definition.json - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: updated-task-definition.json service: ${{ inputs.ecs-service }} From bc8ec6dd014374c9ffa812a6474bcf9c03118c66 Mon Sep 17 00:00:00 2001 From: gayathridevii <145091116+gayathridevii@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:44:42 +0530 Subject: [PATCH 35/43] Add eks deployment pipeline (#21) * add eks deployment pipeline for conductor server * add eks deployment step seperately instead of composite actions * fix in declaring account id variable * remove the steps for ecr login * made change in environemnt bucket format * made change in service name * typo fix in ecr repository * correcting ecr repository format * ecr repository was not properly passed from previous job output, fix * make ecr repository as github outputs --- .github/workflows/cd-server.yaml | 108 ++++++++++++++---------------- .github/workflows/ci-server-.yaml | 6 -- 2 files changed, 49 insertions(+), 65 deletions(-) diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml index 1b4bba94b4..295d3c4c47 100644 --- a/.github/workflows/cd-server.yaml +++ b/.github/workflows/cd-server.yaml @@ -15,28 +15,31 @@ on: required: true type: string description: Provide tag (Eg:v3.14.0) - +permissions: + id-token: write + contents: write + packages: read + actions: read env: - SERVICE_NAME: conductor-server + SERVICE_NAME: conductor AWS_REGION: "ap-south-1" - + HELM_CHART_NAME: "application-helm-chart" jobs: prepare-env: name: Prepare Env - runs-on: 'ubuntu-latest' + runs-on: "ubuntu-latest" timeout-minutes: 2 outputs: AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} ENV: ${{ steps.vars.outputs.ENV }} PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} - ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} - ECS_SERVICE: ${{ steps.set_env.outputs.ECS_SERVICE }} - TASK_DEFINITION: ${{ steps.set_env.outputs.TASK_DEFINITION }} - CONTAINER_NAME: ${{ steps.set_env.outputs.CONTAINER_NAME }} + K8S_CLUSTER: ${{ steps.set_env.outputs.K8S_CLUSTER }} ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} - + AWS_ACCOUNT_ID: ${{ steps.vars.outputs.AWS_ACCOUNT_ID}} + steps: - id: vars shell: bash @@ -45,7 +48,7 @@ jobs: ENV=${{ github.event.inputs.environment }} IMAGE_TAG=${{ github.event.inputs.tag }} echo $BRANCH - + if [ -z "$ENV" ] then case $BRANCH in @@ -67,17 +70,20 @@ jobs: then echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT - echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT elif [ $ENV == 'stg' ] then echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT - echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT else echo "Branch not configured!" exit 1 @@ -89,34 +95,27 @@ jobs: id: set_env run: | PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} - echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT - echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT - echo "ECS_SERVICE=$PROJECT_PREFIX-svc-$SERVICE_NAME" >> $GITHUB_OUTPUT - echo "TASK_DEFINITION=$PROJECT_PREFIX-td-$SERVICE_NAME" >> $GITHUB_OUTPUT - echo "CONTAINER_NAME=$PROJECT_PREFIX-cntr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=$PROJECT_PREFIX-prime" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY - + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-conductor-server" >> $GITHUB_OUTPUT + # Deploy Conductor UI Image to ECS - deploy-server-image: - name: Deploy Server Image - runs-on: 'ubuntu-latest' - timeout-minutes: 20 + deploy-to-k8s: + name: Deploy to k8s + runs-on: ubuntu-latest + container: + image: public.ecr.aws/kvsiren-dev/pipeline/helm-deploy:latest + timeout-minutes: 15 permissions: id-token: write pull-requests: write contents: read - needs: prepare-env + needs: + - prepare-env env: - AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} - ENV: ${{ needs.prepare-env.outputs.ENV }} - PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} - ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} - IMAGE_TAG: ${{ github.event.inputs.tag }} - ECS_CLUSTER: ${{ needs.prepare-env.outputs.ECS_CLUSTER }} - ECS_SERVICE: ${{ needs.prepare-env.outputs.ECS_SERVICE }} - TASK_DEFINITION: ${{ needs.prepare-env.outputs.TASK_DEFINITION }} - CONTAINER_NAME: ${{ needs.prepare-env.outputs.CONTAINER_NAME }} - + AWS_ACCOUNT_ID: ${{ needs.prepare-env.outputs.AWS_ACCOUNT_ID }} + ECR_REPOSITORY: ${{ needs.prepare-env.outputs.ECR_REPOSITORY }} steps: - name: Checkout code from action uses: actions/checkout@v2 @@ -124,33 +123,24 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets[env.AWS_ROLE] }} + role-to-assume: arn:aws:iam::${{ vars[env.AWS_ACCOUNT_ID] }}:role/github-actions aws-region: ${{ env.AWS_REGION }} - - name: Amazon ECR Login - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1.7.0 + - name: Deploy to Kubernetes + shell: bash + run: | + aws eks update-kubeconfig --name ${{ needs.prepare-env.outputs.K8S_CLUSTER }} + aws s3 cp s3://${{ needs.prepare-env.outputs.ENVIRONMENT_BUCKET }}/helm/${{ env.SERVICE_NAME }}/values.yaml ./values.yaml + cat ./values.yaml + aws ecr get-login-password --region ${{ env.AWS_REGION }} | helm registry login --username AWS --password-stdin ${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com - - name: Check if image tag exists in ECR - id: check-image-existence - run: | - if aws ecr describe-images --repository-name "${{ env.ECR_REPOSITORY }}" --region "${{ env.AWS_REGION }}" --image-ids imageTag="${{ env.IMAGE_TAG }}" 2>&1 | grep -q "imageTag"; then - echo "Image tag $IMAGE_TAG exists in ECR" - else - echo "Error: Image tag $IMAGE_TAG does not exist in ECR" - exit 1 - fi + # Construct base Helm command + HELM_CMD="helm upgrade --install ${{ env.SERVICE_NAME }} oci://${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.HELM_CHART_NAME }} \ + --namespace ${{ needs.prepare-env.outputs.ENV }} \ + --values values.yaml \ + --set default.image.repository='${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}' \ + --set default.image.tag='${{ github.event.inputs.tag }}'" - - name: Deploy backend - id: deploy_backend - uses: ./.github/actions/deploy-ecs - env: - APP_IMAGE: ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - with: - aws-region : ${{ env.AWS_REGION }} - aws-role: ${{ secrets[env.AWS_ROLE] }} - task-definition: ${{ env.TASK_DEFINITION }} - container-name: ${{ env.CONTAINER_NAME }} - ecs-service: ${{ env.ECS_SERVICE }} - ecs-cluster: ${{ env.ECS_CLUSTER }} - image: ${{ env.APP_IMAGE }} \ No newline at end of file + # Run the Helm command + echo "Running: $HELM_CMD" + eval $HELM_CMD diff --git a/.github/workflows/ci-server-.yaml b/.github/workflows/ci-server-.yaml index 52d570d7e0..4cce9dae38 100644 --- a/.github/workflows/ci-server-.yaml +++ b/.github/workflows/ci-server-.yaml @@ -29,7 +29,6 @@ jobs: AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} ENV: ${{ steps.vars.outputs.ENV }} PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} - ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} @@ -88,7 +87,6 @@ jobs: id: set_env run: | PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} - echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY @@ -133,10 +131,6 @@ jobs: role-to-assume: ${{ secrets[env.AWS_ROLE] }} aws-region: ${{ env.AWS_REGION }} - - name: Download S3 file - run: | - aws s3 cp s3://${PROJECT_PREFIX}-s3-environment/conductor-server/conductor-server.properties ./docker/server/config/conductor-server.properties - - name: Amazon ECR Login id: login-ecr uses: aws-actions/amazon-ecr-login@v1.7.0 From d15753081c1a93f0ac33bf5c0ad545e2a43113f6 Mon Sep 17 00:00:00 2001 From: gayathridevii <145091116+gayathridevii@users.noreply.github.com> Date: Tue, 25 Mar 2025 11:21:19 +0530 Subject: [PATCH 36/43] changed ci cd pipeline for conductor ui (#22) * changed ci cd pipeline for conductor ui * removed whitelisting of ips from nginx config --- .github/workflows/cd-ui.yaml | 99 ++++++++++++++++-------------------- ui/default-dev.conf | 18 ++----- ui/default-prd.conf | 18 ++----- ui/default-stg.conf | 18 ++----- 4 files changed, 54 insertions(+), 99 deletions(-) diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml index 3609423632..29b6861b76 100644 --- a/.github/workflows/cd-ui.yaml +++ b/.github/workflows/cd-ui.yaml @@ -15,10 +15,15 @@ on: required: true type: string description: Provide tag (Eg:v3.14.0) - +permissions: + id-token: write + contents: write + packages: read + actions: read env: SERVICE_NAME: conductor-ui AWS_REGION: "ap-south-1" + HELM_CHART_NAME: "application-helm-chart" jobs: prepare-env: @@ -29,13 +34,12 @@ jobs: AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} ENV: ${{ steps.vars.outputs.ENV }} PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} - ECS_CLUSTER: ${{ steps.set_env.outputs.ECS_CLUSTER }} - ECS_SERVICE: ${{ steps.set_env.outputs.ECS_SERVICE }} - TASK_DEFINITION: ${{ steps.set_env.outputs.TASK_DEFINITION }} - CONTAINER_NAME: ${{ steps.set_env.outputs.CONTAINER_NAME }} + K8S_CLUSTER: ${{ steps.set_env.outputs.K8S_CLUSTER }} ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} + ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} - + AWS_ACCOUNT_ID: ${{ steps.vars.outputs.AWS_ACCOUNT_ID}} + steps: - id: vars shell: bash @@ -44,7 +48,7 @@ jobs: ENV=${{ github.event.inputs.environment }} IMAGE_TAG=${{ github.event.inputs.tag }} echo $BRANCH - + if [ -z "$ENV" ] then case $BRANCH in @@ -67,16 +71,19 @@ jobs: echo "AWS_ROLE=PRD_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT elif [ $ENV == 'stg' ] then echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT else echo "Branch not configured!" exit 1 @@ -88,34 +95,27 @@ jobs: id: set_env run: | PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} - echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-$SERVICE_NAME" >> $GITHUB_OUTPUT - echo "ECS_CLUSTER=$PROJECT_PREFIX-ecs-cluster" >> $GITHUB_OUTPUT - echo "ECS_SERVICE=$PROJECT_PREFIX-svc-$SERVICE_NAME" >> $GITHUB_OUTPUT - echo "TASK_DEFINITION=$PROJECT_PREFIX-td-$SERVICE_NAME" >> $GITHUB_OUTPUT - echo "CONTAINER_NAME=$PROJECT_PREFIX-cntr-$SERVICE_NAME" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=$PROJECT_PREFIX-prime" >> $GITHUB_OUTPUT + echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY - + echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-conductor-ui" >> $GITHUB_OUTPUT + # Deploy Conductor UI Image to ECS - deploy-ui-image: - name: Deploy UI Image - runs-on: 'ubuntu-latest' - timeout-minutes: 20 + deploy-to-k8s: + name: Deploy to k8s + runs-on: ubuntu-latest + container: + image: public.ecr.aws/kvsiren-dev/pipeline/helm-deploy:latest + timeout-minutes: 15 permissions: id-token: write pull-requests: write contents: read - needs: prepare-env + needs: + - prepare-env env: - AWS_ROLE: ${{ needs.prepare-env.outputs.AWS_ROLE }} - ENV: ${{ needs.prepare-env.outputs.ENV }} - PROJECT_PREFIX: ${{needs.prepare-env.outputs.PROJECT_PREFIX}} - ECR_REPOSITORY: ${{needs.prepare-env.outputs.ECR_REPOSITORY}} - IMAGE_TAG: ${{ github.event.inputs.tag }} - ECS_CLUSTER: ${{ needs.prepare-env.outputs.ECS_CLUSTER }} - ECS_SERVICE: ${{ needs.prepare-env.outputs.ECS_SERVICE }} - TASK_DEFINITION: ${{ needs.prepare-env.outputs.TASK_DEFINITION }} - CONTAINER_NAME: ${{ needs.prepare-env.outputs.CONTAINER_NAME }} - + AWS_ACCOUNT_ID: ${{ needs.prepare-env.outputs.AWS_ACCOUNT_ID }} + ECR_REPOSITORY: ${{ needs.prepare-env.outputs.ECR_REPOSITORY }} steps: - name: Checkout code from action uses: actions/checkout@v2 @@ -123,33 +123,24 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets[env.AWS_ROLE] }} + role-to-assume: arn:aws:iam::${{ vars[env.AWS_ACCOUNT_ID] }}:role/github-actions aws-region: ${{ env.AWS_REGION }} - - name: Amazon ECR Login - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1.7.0 + - name: Deploy to Kubernetes + shell: bash + run: | + aws eks update-kubeconfig --name ${{ needs.prepare-env.outputs.K8S_CLUSTER }} + aws s3 cp s3://${{ needs.prepare-env.outputs.ENVIRONMENT_BUCKET }}/helm/${{ env.SERVICE_NAME }}/values.yaml ./values.yaml + cat ./values.yaml + aws ecr get-login-password --region ${{ env.AWS_REGION }} | helm registry login --username AWS --password-stdin ${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com - - name: Check if image tag exists in ECR - id: check-image-existence - run: | - if aws ecr describe-images --repository-name "${{ env.ECR_REPOSITORY }}" --region "${{ env.AWS_REGION }}" --image-ids imageTag="${{ env.IMAGE_TAG }}" 2>&1 | grep -q "imageTag"; then - echo "Image tag $IMAGE_TAG exists in ECR" - else - echo "Error: Image tag $IMAGE_TAG does not exist in ECR" - exit 1 - fi + # Construct base Helm command + HELM_CMD="helm upgrade --install ${{ env.SERVICE_NAME }} oci://${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.HELM_CHART_NAME }} \ + --namespace ${{ needs.prepare-env.outputs.ENV }} \ + --values values.yaml \ + --set default.image.repository='${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ env.ECR_REPOSITORY }}' \ + --set default.image.tag='${{ github.event.inputs.tag }}'" - - name: Deploy backend - id: deploy_backend - uses: ./.github/actions/deploy-ecs - env: - APP_IMAGE: ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} - with: - aws-region : ${{ env.AWS_REGION }} - aws-role: ${{ secrets[env.AWS_ROLE] }} - task-definition: ${{ env.TASK_DEFINITION }} - container-name: ${{ env.CONTAINER_NAME }} - ecs-service: ${{ env.ECS_SERVICE }} - ecs-cluster: ${{ env.ECS_CLUSTER }} - image: ${{ env.APP_IMAGE }} \ No newline at end of file + # Run the Helm command + echo "Running: $HELM_CMD" + eval $HELM_CMD diff --git a/ui/default-dev.conf b/ui/default-dev.conf index f37c665194..180f83dee3 100644 --- a/ui/default-dev.conf +++ b/ui/default-dev.conf @@ -1,10 +1,3 @@ -map $http_x_forwarded_for $allow { - default 0; - "103.138.236.18" 1; - "103.181.238.106" 1; - "103.142.30.151" 1; - "61.2.142.186" 1; -} server { listen 5000; @@ -12,11 +5,6 @@ server { server_tokens off; location / { - - if ($allow != 1) { - return 401; - } - add_header Referrer-Policy "strict-origin"; add_header X-Frame-Options "SAMEORIGIN"; add_header X-Content-Type-Options "nosniff"; @@ -32,7 +20,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-dev-mb-svc-conductor-server.sirn-dev-mb.local:8080/api; + proxy_pass http://conductor.dev.svc.cluster.local:8080/api; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; @@ -43,7 +31,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-dev-mb-svc-conductor-server.sirn-dev-mb.local:8080/actuator; + proxy_pass http://conductor.dev.svc.cluster.local:8080/actuator; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; @@ -54,7 +42,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-dev-mb-svc-conductor-server.sirn-dev-mb.local:8080/swagger-ui; + proxy_pass http://conductor.dev.svc.cluster.local:8080/swagger-ui; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; diff --git a/ui/default-prd.conf b/ui/default-prd.conf index bd2c370c57..256c0cce2c 100644 --- a/ui/default-prd.conf +++ b/ui/default-prd.conf @@ -1,10 +1,3 @@ -map $http_x_forwarded_for $allow { - default 0; - "103.138.236.18" 1; - "103.181.238.106" 1; - "103.142.30.151" 1; - "61.2.142.186" 1; -} server { listen 5000; @@ -12,11 +5,6 @@ server { server_tokens off; location / { - - if ($allow != 1) { - return 401; - } - add_header Referrer-Policy "strict-origin"; add_header X-Frame-Options "SAMEORIGIN"; add_header X-Content-Type-Options "nosniff"; @@ -32,7 +20,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-prd-mb-svc-conductor-server.sirn-prd-mb.local:8080/api; + proxy_pass http://conductor.prd.svc.cluster.local:8080/api; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; @@ -43,7 +31,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-prd-mb-svc-conductor-server.sirn-prd-mb.local:8080/actuator; + proxy_pass http://conductor.prd.svc.cluster.local:8080/actuator; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; @@ -54,7 +42,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-prd-mb-svc-conductor-server.sirn-prd-mb.local:8080/swagger-ui; + proxy_pass http://conductor.prd.svc.cluster.local:8080/swagger-ui; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; diff --git a/ui/default-stg.conf b/ui/default-stg.conf index e057354998..f88374776e 100644 --- a/ui/default-stg.conf +++ b/ui/default-stg.conf @@ -1,10 +1,3 @@ -map $http_x_forwarded_for $allow { - default 0; - "103.138.236.18" 1; - "103.181.238.106" 1; - "103.142.30.151" 1; - "61.2.142.186" 1; -} server { listen 5000; @@ -12,11 +5,6 @@ server { server_tokens off; location / { - - if ($allow != 1) { - return 401; - } - add_header Referrer-Policy "strict-origin"; add_header X-Frame-Options "SAMEORIGIN"; add_header X-Content-Type-Options "nosniff"; @@ -32,7 +20,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-stg-mb-svc-conductor-server.sirn-stg-mb.local:8080/api; + proxy_pass http://conductor.stg.svc.cluster.local:8080/api; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; @@ -43,7 +31,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-stg-mb-svc-conductor-server.sirn-stg-mb.local:8080/actuator; + proxy_pass http://conductor.stg.svc.cluster.local:8080/actuator; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; @@ -54,7 +42,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-NginX-Proxy true; - proxy_pass http://sirn-stg-mb-svc-conductor-server.sirn-stg-mb.local:8080/swagger-ui; + proxy_pass http://conductor.stg.svc.cluster.local:8080/swagger-ui; proxy_ssl_session_reuse off; proxy_set_header Host $http_host; proxy_cache_bypass $http_upgrade; From 7b41ca08242fa099514ebbb0b8feea472f5a5118 Mon Sep 17 00:00:00 2001 From: gayathridevii <145091116+gayathridevii@users.noreply.github.com> Date: Tue, 1 Apr 2025 18:16:46 +0530 Subject: [PATCH 37/43] minor fix in eks cluster variable (#23) --- .github/workflows/cd-server.yaml | 8 +++++--- .github/workflows/cd-ui.yaml | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml index 295d3c4c47..ebcced8dc0 100644 --- a/.github/workflows/cd-server.yaml +++ b/.github/workflows/cd-server.yaml @@ -34,7 +34,7 @@ jobs: AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} ENV: ${{ steps.vars.outputs.ENV }} PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} - K8S_CLUSTER: ${{ steps.set_env.outputs.K8S_CLUSTER }} + K8S_CLUSTER: ${{ steps.vars.outputs.K8S_CLUSTER }} ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} @@ -72,18 +72,21 @@ jobs: echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT echo "AWS_ACCOUNT_ID=PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-prd-mb-prime" >> $GITHUB_OUTPUT elif [ $ENV == 'stg' ] then echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT - echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT else echo "Branch not configured!" exit 1 @@ -95,7 +98,6 @@ jobs: id: set_env run: | PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} - echo "K8S_CLUSTER=$PROJECT_PREFIX-prime" >> $GITHUB_OUTPUT echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-conductor-server" >> $GITHUB_OUTPUT diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml index 29b6861b76..7877ad6c2f 100644 --- a/.github/workflows/cd-ui.yaml +++ b/.github/workflows/cd-ui.yaml @@ -34,7 +34,7 @@ jobs: AWS_ROLE: ${{ steps.vars.outputs.AWS_ROLE }} ENV: ${{ steps.vars.outputs.ENV }} PROJECT_PREFIX: ${{ steps.vars.outputs.PROJECT_PREFIX }} - K8S_CLUSTER: ${{ steps.set_env.outputs.K8S_CLUSTER }} + K8S_CLUSTER: ${{ steps.vars.outputs.K8S_CLUSTER }} ECR_REPOSITORY: ${{ steps.set_env.outputs.ECR_REPOSITORY }} ENVIRONMENT_BUCKET: ${{ steps.set_env.outputs.ENVIRONMENT_BUCKET }} SLACK_WEBHOOK_URL: ${{ steps.vars.outputs.SLACK_WEBHOOK_URL }} @@ -72,18 +72,21 @@ jobs: echo "PROJECT_PREFIX=sirn-prd-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=PRD_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT echo "AWS_ACCOUNT_ID=PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-prd-mb-prime" >> $GITHUB_OUTPUT elif [ $ENV == 'stg' ] then echo "AWS_ROLE=STG_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-stg-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT elif [ $ENV == 'dev' ] then echo "AWS_ROLE=DEV_AWS_ROLE" >> $GITHUB_OUTPUT echo "PROJECT_PREFIX=sirn-dev-mb" >> $GITHUB_OUTPUT echo "SLACK_WEBHOOK_URL=DEV_SLACK_WEBHOOK_URL" >> $GITHUB_OUTPUT echo "AWS_ACCOUNT_ID=NON_PRD_AWS_ACCOUNT_ID" >> $GITHUB_OUTPUT + echo "K8S_CLUSTER=sirn-dev-mb-prime" >> $GITHUB_OUTPUT else echo "Branch not configured!" exit 1 @@ -95,7 +98,6 @@ jobs: id: set_env run: | PROJECT_PREFIX=${{ steps.vars.outputs.PROJECT_PREFIX }} - echo "K8S_CLUSTER=$PROJECT_PREFIX-prime" >> $GITHUB_OUTPUT echo "ENVIRONMENT_BUCKET=$PROJECT_PREFIX-s3-environment" >> $GITHUB_OUTPUT echo ":seedling: Branch:${GITHUB_REF#refs/heads/}" >> $GITHUB_STEP_SUMMARY echo "ECR_REPOSITORY=$PROJECT_PREFIX-ecr-conductor-ui" >> $GITHUB_OUTPUT From 27fa7f901cbf816072041d532887c947cf1507f9 Mon Sep 17 00:00:00 2001 From: Anandu Date: Wed, 2 Apr 2025 15:33:46 +0530 Subject: [PATCH 38/43] fix: Add web identity setup in s3 config (#20) * fix: Add support for web identity in s3 config * fix: Change to default properties file --- awss3-storage/build.gradle | 3 +++ .../com/netflix/conductor/s3/config/S3Configuration.java | 6 ++++++ .../java/com/netflix/conductor/s3/config/S3Properties.java | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/awss3-storage/build.gradle b/awss3-storage/build.gradle index 57e9d4fc3a..350ae8bd97 100644 --- a/awss3-storage/build.gradle +++ b/awss3-storage/build.gradle @@ -18,4 +18,7 @@ dependencies { implementation "com.amazonaws:aws-java-sdk-s3:${revAwsSdk}" implementation "org.apache.commons:commons-lang3" + implementation 'software.amazon.awssdk:s3:2.20.146' + implementation 'software.amazon.awssdk:sts:2.20.146' + implementation 'com.amazonaws:aws-java-sdk-sts:1.12.782' } diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java index 106f61664d..44dc8a43ed 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Configuration.java @@ -21,6 +21,7 @@ import com.netflix.conductor.core.utils.IDGenerator; import com.netflix.conductor.s3.storage.S3PayloadStorage; +import com.amazonaws.auth.WebIdentityTokenCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -41,6 +42,11 @@ public ExternalPayloadStorage s3ExternalPayloadStorage( matchIfMissing = true) @Bean public AmazonS3 amazonS3(S3Properties properties) { + if (properties.getWebIdentity()) { + return AmazonS3ClientBuilder.standard() + .withCredentials(WebIdentityTokenCredentialsProvider.builder().build()) + .build(); + } return AmazonS3ClientBuilder.standard().withRegion(properties.getRegion()).build(); // TODO: Add localstack support to test locally // return AmazonS3ClientBuilder.standard() diff --git a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java index f399cd0d7d..df28f94c30 100644 --- a/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java +++ b/awss3-storage/src/main/java/com/netflix/conductor/s3/config/S3Properties.java @@ -24,6 +24,8 @@ public class S3Properties { /** The s3 bucket name where the payloads will be stored */ private String bucketName = "conductor_payloads"; + private Boolean webIdentity = false; + /** The time (in seconds) for which the signed url will be valid */ @DurationUnit(ChronoUnit.SECONDS) private Duration signedUrlExpirationDuration = Duration.ofSeconds(5); @@ -55,6 +57,9 @@ public void setRegion(String region) { this.region = region; } + public Boolean getWebIdentity() { + return webIdentity; + } // TODO: Add localstack support to test locally // private String endpoint = "http://s3.localhost.localstack.cloud:4566"; // From 18fc70725a401984b8709450c8d8a692be48adc3 Mon Sep 17 00:00:00 2001 From: Anandu Date: Wed, 2 Apr 2025 15:34:08 +0530 Subject: [PATCH 39/43] chore: Add procedure migrations (#25) * chore: Add procedure migrations * fix: Remove notification archive --- .../V9__archival_procedure.sql | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V9__archival_procedure.sql diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V9__archival_procedure.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V9__archival_procedure.sql new file mode 100644 index 0000000000..83ae483c42 --- /dev/null +++ b/postgres-persistence/src/main/resources/db/migration_postgres/V9__archival_procedure.sql @@ -0,0 +1,55 @@ +CREATE OR REPLACE PROCEDURE public.conductor_archive(IN archival_date date) + LANGUAGE plpgsql +AS $procedure$ +BEGIN + + --CREATING TEMP TABLE FOR TASK IDs + CREATE TEMP TABLE temp_task_ids ON COMMIT DROP AS + SELECT task_id FROM task WHERE created_on < archival_date; + + ALTER TABLE temp_task_ids ADD PRIMARY KEY (task_id); + ANALYZE temp_task_ids; + + --CREATING TEMP TABLE FOR WORKFLOW IDs + CREATE TEMP TABLE temp_workflow_ids ON COMMIT DROP AS + SELECT workflow_id FROM workflow WHERE created_on < archival_date; + + ALTER TABLE temp_workflow_ids ADD PRIMARY KEY (workflow_id); + ANALYZE temp_workflow_ids; + + --CREATING TEMP TABLE FOR temp_workflow_def_to_workflow IDs + CREATE TEMP TABLE temp_workflow_def_to_workflow_ids ON COMMIT DROP AS + SELECT wdt.workflow_id + FROM workflow_def_to_workflow wdt + JOIN temp_workflow_ids tw ON tw.workflow_id = wdt.workflow_id; + + ALTER TABLE temp_workflow_def_to_workflow_ids ADD PRIMARY KEY (workflow_id); + ANALYZE temp_workflow_def_to_workflow_ids; + + --CREATING TEMP TABLES FOR workflow_to_task IDs + CREATE TEMP TABLE temp_workflow_to_task_ids ON COMMIT DROP AS + SELECT w.task_id + FROM workflow_to_task w + JOIN temp_task_ids t ON t.task_id = w.task_id; + + ALTER TABLE temp_workflow_to_task_ids ADD PRIMARY KEY(task_id); + ANALYZE temp_workflow_to_task_ids; + + --CREATING TEMP TABLES FOR task_scheduled IDs + CREATE TEMP TABLE temp_task_scheduled_ids ON COMMIT DROP AS + SELECT ts.task_id + FROM task_scheduled ts + JOIN temp_task_ids t ON t.task_id = ts.task_id; + + ALTER TABLE temp_task_scheduled_ids ADD PRIMARY KEY(task_id); + ANALYZE temp_task_scheduled_ids; + + DELETE FROM task t USING temp_task_ids tti WHERE t.task_id = tti.task_id; + DELETE FROM workflow w USING temp_workflow_ids t WHERE w.workflow_id = t.workflow_id; + DELETE FROM workflow_def_to_workflow w USING temp_workflow_def_to_workflow_ids t WHERE w.workflow_id = t.workflow_id; + DELETE FROM workflow_to_task w USING temp_workflow_to_task_ids t WHERE w.task_id = t.task_id; + DELETE FROM task_scheduled t USING temp_task_scheduled_ids tts WHERE t.task_id = tts.task_id; + DELETE FROM event_execution WHERE created_on < archival_date; +END; +$procedure$ +; From c36114403db4bab8aa496a7e6a895ac2c32f7444 Mon Sep 17 00:00:00 2001 From: Gayathridevi S Date: Tue, 8 Apr 2025 16:45:26 +0530 Subject: [PATCH 40/43] fetch values from infra repo --- .github/workflows/cd-server.yaml | 18 ++++++++++++++++-- .github/workflows/cd-ui.yaml | 18 ++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml index ebcced8dc0..41e9387dec 100644 --- a/.github/workflows/cd-server.yaml +++ b/.github/workflows/cd-server.yaml @@ -122,6 +122,22 @@ jobs: - name: Checkout code from action uses: actions/checkout@v2 + - name: Checkout values.yaml from siren-infra + uses: actions/checkout@v4 + with: + repository: KeyvalueSoftwareSystems/siren-infra + ref: main + token: ${{inputs.PAT}} + sparse-checkout: | + k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml + sparse-checkout-cone-mode: false + + - name: Rename values.yaml for Helm + shell: bash + run: | + cp k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml ./values.yaml + cat ./values.yaml + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -132,8 +148,6 @@ jobs: shell: bash run: | aws eks update-kubeconfig --name ${{ needs.prepare-env.outputs.K8S_CLUSTER }} - aws s3 cp s3://${{ needs.prepare-env.outputs.ENVIRONMENT_BUCKET }}/helm/${{ env.SERVICE_NAME }}/values.yaml ./values.yaml - cat ./values.yaml aws ecr get-login-password --region ${{ env.AWS_REGION }} | helm registry login --username AWS --password-stdin ${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com # Construct base Helm command diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml index 7877ad6c2f..f99b20c87e 100644 --- a/.github/workflows/cd-ui.yaml +++ b/.github/workflows/cd-ui.yaml @@ -122,6 +122,22 @@ jobs: - name: Checkout code from action uses: actions/checkout@v2 + - name: Checkout values.yaml from siren-infra + uses: actions/checkout@v4 + with: + repository: KeyvalueSoftwareSystems/siren-infra + ref: main + token: ${{inputs.PAT}} + sparse-checkout: | + k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml + sparse-checkout-cone-mode: false + + - name: Rename values.yaml for Helm + shell: bash + run: | + cp k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml ./values.yaml + cat ./values.yaml + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -132,8 +148,6 @@ jobs: shell: bash run: | aws eks update-kubeconfig --name ${{ needs.prepare-env.outputs.K8S_CLUSTER }} - aws s3 cp s3://${{ needs.prepare-env.outputs.ENVIRONMENT_BUCKET }}/helm/${{ env.SERVICE_NAME }}/values.yaml ./values.yaml - cat ./values.yaml aws ecr get-login-password --region ${{ env.AWS_REGION }} | helm registry login --username AWS --password-stdin ${{ vars[env.AWS_ACCOUNT_ID] }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com # Construct base Helm command From 9a084ad4c389c6e533b0d082a0304f53f7510f01 Mon Sep 17 00:00:00 2001 From: Gayathridevi S Date: Tue, 8 Apr 2025 16:55:52 +0530 Subject: [PATCH 41/43] fix in fetching variables --- .github/workflows/cd-server.yaml | 6 +++--- .github/workflows/cd-ui.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cd-server.yaml b/.github/workflows/cd-server.yaml index 41e9387dec..14895943f4 100644 --- a/.github/workflows/cd-server.yaml +++ b/.github/workflows/cd-server.yaml @@ -127,15 +127,15 @@ jobs: with: repository: KeyvalueSoftwareSystems/siren-infra ref: main - token: ${{inputs.PAT}} + token: ${{secrets.SIREN_PAT}} sparse-checkout: | - k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml + k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml sparse-checkout-cone-mode: false - name: Rename values.yaml for Helm shell: bash run: | - cp k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml ./values.yaml + cp k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml ./values.yaml cat ./values.yaml - name: Configure AWS credentials diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml index f99b20c87e..dcae81a6ee 100644 --- a/.github/workflows/cd-ui.yaml +++ b/.github/workflows/cd-ui.yaml @@ -127,15 +127,15 @@ jobs: with: repository: KeyvalueSoftwareSystems/siren-infra ref: main - token: ${{inputs.PAT}} + token: ${{secrets.SIREN_PAT}} sparse-checkout: | - k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml + k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml sparse-checkout-cone-mode: false - name: Rename values.yaml for Helm shell: bash run: | - cp k8s/siren-services/${{ inputs.SERVICE_NAME }}/${{ inputs.K8S_NAMESPACE }}-values.yaml ./values.yaml + cp k8s/siren-services/${{ env.SERVICE_NAME }}${{ needs.prepare-env.outputs.ENV }}-values.yaml ./values.yaml cat ./values.yaml - name: Configure AWS credentials From 89cacf84c303d4a7914072c69ebef84c64280f4c Mon Sep 17 00:00:00 2001 From: Gayathridevi S Date: Tue, 8 Apr 2025 17:09:28 +0530 Subject: [PATCH 42/43] typo fix in values path --- .github/workflows/cd-ui.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd-ui.yaml b/.github/workflows/cd-ui.yaml index dcae81a6ee..c5d84fc2eb 100644 --- a/.github/workflows/cd-ui.yaml +++ b/.github/workflows/cd-ui.yaml @@ -135,7 +135,7 @@ jobs: - name: Rename values.yaml for Helm shell: bash run: | - cp k8s/siren-services/${{ env.SERVICE_NAME }}${{ needs.prepare-env.outputs.ENV }}-values.yaml ./values.yaml + cp k8s/siren-services/${{ env.SERVICE_NAME }}/${{ needs.prepare-env.outputs.ENV }}-values.yaml ./values.yaml cat ./values.yaml - name: Configure AWS credentials From b82fb140f969399eaa36673094fd607def32e705 Mon Sep 17 00:00:00 2001 From: Anandu Date: Thu, 16 Oct 2025 14:41:14 +0530 Subject: [PATCH 43/43] chore: Update archival procedure (#29) --- .../V10_update_archival_procedure.sql | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 postgres-persistence/src/main/resources/db/migration_postgres/V10_update_archival_procedure.sql diff --git a/postgres-persistence/src/main/resources/db/migration_postgres/V10_update_archival_procedure.sql b/postgres-persistence/src/main/resources/db/migration_postgres/V10_update_archival_procedure.sql new file mode 100644 index 0000000000..fdb67f93ad --- /dev/null +++ b/postgres-persistence/src/main/resources/db/migration_postgres/V10_update_archival_procedure.sql @@ -0,0 +1,109 @@ + +CREATE TABLE IF NOT EXISTS archival_logs ( + id SERIAL PRIMARY KEY, + log_time TIMESTAMP NOT NULL DEFAULT now(), + log_message TEXT NOT NULL, + archival_date DATE NOT NULL +); + +CREATE OR REPLACE PROCEDURE public.conductor_archive(IN archival_date date) + LANGUAGE plpgsql +AS $procedure$ +DECLARE + deleted_workflows INT := 0; + deleted_wf_def_links INT := 0; + deleted_wf_to_task INT := 0; + deleted_tasks INT := 0; + deleted_task_scheduled INT := 0; + deleted_workflow_index INT := 0; + deleted_task_index INT := 0; + total_deleted INT := 0; + log_message TEXT; +BEGIN + -- Step 1: Collect workflow IDs eligible for deletion + CREATE TEMP TABLE temp_workflows_to_delete ON COMMIT DROP AS + SELECT workflow_id + FROM workflow + WHERE created_on < archival_date + AND (json_data::jsonb ->> 'status') IN ('COMPLETED', 'FAILED', 'TIMED_OUT', 'TERMINATED'); + + ALTER TABLE temp_workflows_to_delete ADD PRIMARY KEY (workflow_id); + ANALYZE temp_workflows_to_delete; + + -- Step 2: Cascade deletes + + -- workflow_def_to_workflow + DELETE FROM workflow_def_to_workflow wdw + USING temp_workflows_to_delete tw + WHERE wdw.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_wf_def_links = ROW_COUNT; + + -- workflow_index + DELETE FROM workflow_index wi + USING temp_workflows_to_delete tw + WHERE wi.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_workflow_index = ROW_COUNT; + + -- workflow_to_task + CREATE TEMP TABLE temp_tasks_to_delete ON COMMIT DROP AS + SELECT wt.task_id + FROM workflow_to_task wt + JOIN temp_workflows_to_delete tw ON wt.workflow_id = tw.workflow_id; + + ALTER TABLE temp_tasks_to_delete ADD PRIMARY KEY (task_id); + ANALYZE temp_tasks_to_delete; + + DELETE FROM workflow_to_task wt + USING temp_workflows_to_delete tw + WHERE wt.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_wf_to_task = ROW_COUNT; + + -- task_scheduled + DELETE FROM task_scheduled ts + USING temp_tasks_to_delete tt + WHERE ts.task_id = tt.task_id; + GET DIAGNOSTICS deleted_task_scheduled = ROW_COUNT; + + -- task_index + DELETE FROM task_index ti + USING temp_tasks_to_delete tt + WHERE ti.task_id = tt.task_id; + GET DIAGNOSTICS deleted_task_index = ROW_COUNT; + + -- task + DELETE FROM task t + USING temp_tasks_to_delete tt + WHERE t.task_id = tt.task_id; + GET DIAGNOSTICS deleted_tasks = ROW_COUNT; + + -- workflow + DELETE FROM workflow w + USING temp_workflows_to_delete tw + WHERE w.workflow_id = tw.workflow_id; + GET DIAGNOSTICS deleted_workflows = ROW_COUNT; + + -- Step 3: Logging + total_deleted := deleted_workflows + deleted_wf_def_links + deleted_workflow_index + + deleted_wf_to_task + deleted_tasks + deleted_task_scheduled + deleted_task_index; + + log_message := 'Cleanup completed successfully for COMPLETED, FAILED, TIMED_OUT, and TERMINATED workflows before ' || archival_date || '. ' || + 'Total deleted: ' || total_deleted || ' | Breakdown: ' || + 'workflow: ' || deleted_workflows || ', ' || + 'workflow_def_to_workflow: ' || deleted_wf_def_links || ', ' || + 'workflow_index: ' || deleted_workflow_index || ', ' || + 'workflow_to_task: ' || deleted_wf_to_task || ', ' || + 'task: ' || deleted_tasks || ', ' || + 'task_scheduled: ' || deleted_task_scheduled || ', ' || + 'task_index: ' || deleted_task_index; + + INSERT INTO archival_logs(log_message, archival_date) + VALUES (log_message, archival_date); + +EXCEPTION + WHEN OTHERS THEN + INSERT INTO archival_logs(log_message, archival_date) + VALUES ('Error in cleanup_completed_workflows: ' || SQLERRM, archival_date); + RAISE; +END; +$procedure$ +;