From 42c06e8548e073464f29c9e8a9bc091e749c86a7 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Fri, 12 Apr 2024 21:35:11 -0700 Subject: [PATCH 1/6] Intrapipeline Queueing (#16) * documentation: update comments * feat: add internal queue --- .circleci/test-deploy.yml | 17 ++++-- src/commands/block_execution.yml | 4 +- src/commands/block_internal.yml | 23 ++++++++ src/jobs/internal-queue.yml | 24 ++++++++ src/jobs/queue.yml | 2 +- src/scripts/internal-queue.sh | 95 ++++++++++++++++++++++++++++++++ src/scripts/test.sh | 21 +++++++ 7 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 src/commands/block_internal.yml create mode 100644 src/jobs/internal-queue.yml create mode 100644 src/scripts/internal-queue.sh create mode 100755 src/scripts/test.sh diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml index f4c88e9..4279bb0 100644 --- a/.circleci/test-deploy.yml +++ b/.circleci/test-deploy.yml @@ -8,23 +8,30 @@ filters: &filters only: /.*/ workflows: - test-deploy: + test-internal-queue: + jobs: + - workflow-queue/internal-queue: + filters: + tags: + ignore: /.*/ + context: orb-publishing + test-queue: jobs: - - orb-tools/pack: - filters: *filters - workflow-queue/queue: filters: tags: ignore: /.*/ context: orb-publishing - requires: [orb-tools/pack] + test-deploy: + jobs: + - orb-tools/pack: + filters: *filters - orb-tools/publish: orb-name: promiseofcake/workflow-queue vcs-type: << pipeline.project.type >> pub-type: production requires: - orb-tools/pack - - workflow-queue/queue context: orb-publishing filters: branches: diff --git a/src/commands/block_execution.yml b/src/commands/block_execution.yml index a7b55a7..b9975de 100755 --- a/src/commands/block_execution.yml +++ b/src/commands/block_execution.yml @@ -1,5 +1,5 @@ description: > - This command is executed and blocks further execution until the necessary criteria is met. + This command is the execution portion to determine that only a single global workflow is running at a point in time. parameters: debug: @@ -32,7 +32,7 @@ parameters: steps: - run: - name: Block execution until workflow is at the front of the line + name: Block execution until the current workflow is at the front of the line environment: CONFIG_DEBUG_ENABLED: "<< parameters.debug >>" CONFIG_TIME: "<< parameters.time >>" diff --git a/src/commands/block_internal.yml b/src/commands/block_internal.yml new file mode 100644 index 0000000..e87d053 --- /dev/null +++ b/src/commands/block_internal.yml @@ -0,0 +1,23 @@ +description: > + This command blocks execution of parallel pipeline workflows until necessary criteria is met. + +parameters: + debug: + type: boolean + default: false + description: "If enabled, DEBUG messages will be logged." + confidence: + type: string + default: "1" + description: > + Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending", + but not yet active. This number indicates the threshold for API returning no previous pending pipelines. + Default is one confirmation, increase if you see issues. + +steps: + - run: + name: Block additional pipeline work executions until all other workflows are complete + environment: + CONFIG_DEBUG_ENABLED: "<< parameters.debug >>" + CONFIG_CONFIDENCE: "<< parameters.confidence >>" + command: <> diff --git a/src/jobs/internal-queue.yml b/src/jobs/internal-queue.yml new file mode 100644 index 0000000..5dacc54 --- /dev/null +++ b/src/jobs/internal-queue.yml @@ -0,0 +1,24 @@ +description: > + This job prevents a workflow within a given pipeline from running until all previous workflows have completed. + +docker: + - image: cimg/base:stable +resource_class: small + +parameters: + debug: + type: boolean + default: false + description: "If enabled, DEBUG messages will be logged." + confidence: + type: string + default: "1" + description: > + Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending", + but not yet active. This number indicates the threshold for API returning no previous pending pipelines. + Default is one confirmation, increase if you see issues. + +steps: + - block_internal: + debug: <> + confidence: <> diff --git a/src/jobs/queue.yml b/src/jobs/queue.yml index cc346f7..0811a5f 100755 --- a/src/jobs/queue.yml +++ b/src/jobs/queue.yml @@ -1,5 +1,5 @@ description: > - This job is executed and blocks further execution until the necessary criteria is met. + This job ensures only a single global workflow is running at a point in time. docker: - image: cimg/base:stable diff --git a/src/scripts/internal-queue.sh b/src/scripts/internal-queue.sh new file mode 100644 index 0000000..08862af --- /dev/null +++ b/src/scripts/internal-queue.sh @@ -0,0 +1,95 @@ +#!/bin/bash +tmp=${TMP_DIR:-/tmp} +tmp="/tmp" +workflows_file="${tmp}/workflow_status.json" + +# logger command for debugging +debug() { + if [ "${CONFIG_DEBUG_ENABLED}" == "1" ]; then + echo "DEBUG: ${*}" + fi +} + +# ensure we have the required variables present to execute +load_variables(){ + # just confirm our required variables are present + : "${CIRCLE_WORKFLOW_ID:?"Required Env Variable not found!"}" + : "${CIRCLE_PIPELINE_ID:?"Required Env Variable not found!"}" + : "${CIRCLE_PROJECT_USERNAME:?"Required Env Variable not found!"}" + : "${CIRCLE_PROJECT_REPONAME:?"Required Env Variable not found!"}" + # Only needed for private projects + if [ -z "${CIRCLECI_API_TOKEN}" ]; then + echo "CIRCLECI_API_TOKEN not set. Private projects will be inaccessible." + else + fetch "https://circleci.com/api/v2/me" "/tmp/me.cci" + me=$(jq -e '.id' /tmp/me.cci) + echo "Using API key for user: ${me}" + fi +} + +# helper function to perform HTTP requests via curl +fetch(){ + url=$1 + target=$2 + method=${3:-GET} + debug "Performing API ${method} Call to ${url} to ${target}" + + http_response=$(curl -s -X "${method}" -H "Circle-Token: ${CIRCLECI_API_TOKEN}" -H "Content-Type: application/json" -o "${target}" -w "%{http_code}" "${url}") + if [ "${http_response}" != "200" ]; then + echo "ERROR: Server returned error code: ${http_response}" + debug "${target}" + exit 1 + else + debug "API Success" + fi +} + +# fetch all workflows within the current pipeline +fetch_pipeline_workflows(){ + debug "Fetching workflow information for pipeline: ${CIRCLE_PIPELINE_ID}" + pipeline_detail=${tmp}/pipeline-${CIRCLE_PIPELINE_ID}.json + fetch "https://circleci.com/api/v2/pipeline/${CIRCLE_PIPELINE_ID}/workflow" "${pipeline_detail}" + debug "Pipeline's details: $(jq -r '.' "${pipeline_detail}")" + # fetch all workflows that are not this workflow + jq -s "[.[].items[] | select((.id != \"${CIRCLE_WORKFLOW_ID}\") and ((.status == \"running\") or (.status == \"created\")))]" "${pipeline_detail}" > ${workflows_file} +} + +# load all the data necessary to compare build executions +update_comparables(){ + fetch_pipeline_workflows + + running_workflows=$(jq length ${workflows_file}) + debug "Running workflows: ${running_workflows}" +} + +load_variables +echo "This build will block until all previous builds complete." +wait_time=0 +loop_time=11 + +# queue loop +confidence=0 +while true; do + update_comparables + + # if we have no running workflows, check confidence, and move to front of line. + if [[ "${running_workflows}" -eq 0 ]] ; then + if [ $confidence -lt "${CONFIG_CONFIDENCE}" ]; then + # To grow confidence, we check again with a delay. + confidence=$((confidence+1)) + echo "API shows no running pipeline workflows, but it is possible a previous workflow has pending jobs not yet visible in API." + echo "Rerunning check ${confidence}/${CONFIG_CONFIDENCE}" + else + echo "Front of the line, WooHoo!, Build continuing" + break + fi + else + # If we fail, reset confidence + confidence=0 + echo "This workflow (${CIRCLE_WORKFLOW_ID}) is queued, waiting for ${running_workflows} pipeline workflows to complete." + echo "Total Queue time: ${wait_time} seconds." + fi + + sleep $loop_time + wait_time=$(( loop_time + wait_time )) +done diff --git a/src/scripts/test.sh b/src/scripts/test.sh new file mode 100755 index 0000000..e913c83 --- /dev/null +++ b/src/scripts/test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# shellcheck disable=all + +# This script is used to test the scripts in the src/scripts directory +TMP_DIR=`mktemp -d` + +# job config +CONFIG_DEBUG_ENABLED=1 +CONFIG_TIME=10 +CONFIG_DONT_QUIT=1 +CONFIG_ONLY_ON_BRANCH=* +CONFIG_CONFIDENCE=1 + +# test values +CIRCLE_PIPELINE_ID=4e1ffd3b-c260-43db-a66c-8766eaf7fc88 +CIRCLE_WORKFLOW_ID=476fa7ff-7534-440f-9a65-a2779170d344 +CIRCLE_PROJECT_USERNAME=promiseofcake +CIRCLE_PROJECT_REPONAME=circleci-workflow-queue +CIRCLECI_API_TOKEN=${CIRCLECI_USER_TOKEN} + +source ./internal-queue.sh From a33c5d0959b4f31d9609b4b1f82f429e9c3ce2f4 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Fri, 12 Apr 2024 22:22:38 -0700 Subject: [PATCH 2/6] Job, Command, Renames / Recontextulization (#18) * feat: renames * feat: add ability to ignore specific workflows * fix: formatting * fix: shellcheck * fix: job name * fix: shellcheck --- .circleci/test-deploy.yml | 8 +-- Makefile | 5 ++ .../{block_execution.yml => global_block.yml} | 7 ++- ...{block_internal.yml => pipeline_block.yml} | 2 +- src/jobs/{queue.yml => global-queue.yml} | 25 ++++++---- ...{internal-queue.yml => pipeline-queue.yml} | 2 +- src/scripts/{queue.sh => global-queue.sh} | 50 +++++++++++-------- .../{internal-queue.sh => pipeline-queue.sh} | 0 src/scripts/test.sh | 9 ++-- src/test/fixture.json | 28 +++++++++++ 10 files changed, 94 insertions(+), 42 deletions(-) create mode 100644 Makefile rename src/commands/{block_execution.yml => global_block.yml} (85%) rename src/commands/{block_internal.yml => pipeline_block.yml} (93%) rename src/jobs/{queue.yml => global-queue.yml} (58%) rename src/jobs/{internal-queue.yml => pipeline-queue.yml} (97%) rename src/scripts/{queue.sh => global-queue.sh} (80%) rename src/scripts/{internal-queue.sh => pipeline-queue.sh} (100%) create mode 100644 src/test/fixture.json diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml index 4279bb0..e586665 100644 --- a/.circleci/test-deploy.yml +++ b/.circleci/test-deploy.yml @@ -8,16 +8,16 @@ filters: &filters only: /.*/ workflows: - test-internal-queue: + test-global-queue: jobs: - - workflow-queue/internal-queue: + - workflow-queue/global-queue: filters: tags: ignore: /.*/ context: orb-publishing - test-queue: + test-pipeline-queue: jobs: - - workflow-queue/queue: + - workflow-queue/pipeline-queue: filters: tags: ignore: /.*/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3e2f59c --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +test-global-queue: + ./src/scripts/test.sh global-queue.sh + +test-pipeline-queue: + ./src/scripts/test.sh pipeline-queue.sh diff --git a/src/commands/block_execution.yml b/src/commands/global_block.yml similarity index 85% rename from src/commands/block_execution.yml rename to src/commands/global_block.yml index b9975de..300ea35 100755 --- a/src/commands/block_execution.yml +++ b/src/commands/global_block.yml @@ -25,6 +25,10 @@ parameters: Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending", but not yet active. This number indicates the threshold for API returning no previous pending pipelines. Default is one confirmation, increase if you see issues. + ignored-workflows: + type: string + default: "" + description: "Comma separated list of workflow names to ignore as blocking workflows." include-on-hold: type: boolean default: false @@ -39,5 +43,6 @@ steps: CONFIG_DONT_QUIT: "<< parameters.dont-quit >>" CONFIG_ONLY_ON_BRANCH: "<< parameters.only-on-branch >>" CONFIG_CONFIDENCE: "<< parameters.confidence >>" + CONFIG_IGNORED_WORKFLOWS: " << parameters.ignored-workflows >>" CONFIG_INCLUDE_ON_HOLD: " << parameters.include-on-hold >>" - command: <> + command: <> diff --git a/src/commands/block_internal.yml b/src/commands/pipeline_block.yml similarity index 93% rename from src/commands/block_internal.yml rename to src/commands/pipeline_block.yml index e87d053..3234fc3 100644 --- a/src/commands/block_internal.yml +++ b/src/commands/pipeline_block.yml @@ -20,4 +20,4 @@ steps: environment: CONFIG_DEBUG_ENABLED: "<< parameters.debug >>" CONFIG_CONFIDENCE: "<< parameters.confidence >>" - command: <> + command: <> diff --git a/src/jobs/queue.yml b/src/jobs/global-queue.yml similarity index 58% rename from src/jobs/queue.yml rename to src/jobs/global-queue.yml index 0811a5f..bcb1d1b 100755 --- a/src/jobs/queue.yml +++ b/src/jobs/global-queue.yml @@ -1,5 +1,5 @@ description: > - This job ensures only a single global workflow is running at a point in time. + This job ensures only a single global workflow is running at a given point in time. docker: - image: cimg/base:stable @@ -9,7 +9,7 @@ parameters: debug: type: boolean default: false - description: "If enabled, DEBUG messages will be logged." + description: "If enabled, debug messages will be logged." time: type: string default: "10" @@ -17,7 +17,7 @@ parameters: dont-quit: type: boolean default: false - description: "Force job through once time expires instead of failing." + description: "Force job through once time (above) expires instead of failing." only-on-branch: type: string default: "*" @@ -28,17 +28,22 @@ parameters: description: > Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending", but not yet active. This number indicates the threshold for API returning no previous pending pipelines. - Default is one confirmation, increase if you see issues. + Default is 1 confirmation, increase if you see issues. + ignored-workflows: + type: string + default: "" + description: "Comma separated list of workflow names to ignore as blocking workflows." include-on-hold: type: boolean default: false description: Consider on-hold workflows waiting for approval as running and include them in the queue. steps: - - block_execution: + - global_block: debug: << parameters.debug >> - time: <> - dont-quit: <> - only-on-branch: <> - confidence: <> - include-on-hold: <> + time: << parameters.time >> + dont-quit: << parameters.dont-quit >> + only-on-branch: << parameters.only-on-branch >> + confidence: << parameters.confidence >> + ignored-workflows: << parameters.ignored-workflows >> + include-on-hold: << parameters.include-on-hold >> diff --git a/src/jobs/internal-queue.yml b/src/jobs/pipeline-queue.yml similarity index 97% rename from src/jobs/internal-queue.yml rename to src/jobs/pipeline-queue.yml index 5dacc54..b2b64b5 100644 --- a/src/jobs/internal-queue.yml +++ b/src/jobs/pipeline-queue.yml @@ -19,6 +19,6 @@ parameters: Default is one confirmation, increase if you see issues. steps: - - block_internal: + - pipeline_block: debug: <> confidence: <> diff --git a/src/scripts/queue.sh b/src/scripts/global-queue.sh similarity index 80% rename from src/scripts/queue.sh rename to src/scripts/global-queue.sh index d1bb409..a479c85 100644 --- a/src/scripts/queue.sh +++ b/src/scripts/global-queue.sh @@ -1,7 +1,6 @@ #!/bin/bash - -tmp="/tmp" -pipeline_file="${tmp}/pipeline_status.json" +tmp=${TMP_DIR:-/tmp} +pipelines_file="${tmp}/pipeline_status.json" workflows_file="${tmp}/workflow_status.json" # logger command for debugging @@ -13,18 +12,18 @@ debug() { # ensure we have the required variables present to execute load_variables(){ - # just confirm our required variables are present : "${CIRCLE_WORKFLOW_ID:?"Required Env Variable not found!"}" : "${CIRCLE_PROJECT_USERNAME:?"Required Env Variable not found!"}" : "${CIRCLE_PROJECT_REPONAME:?"Required Env Variable not found!"}" : "${CIRCLE_REPOSITORY_URL:?"Required Env Variable not found!"}" : "${CIRCLE_JOB:?"Required Env Variable not found!"}" - # Only needed for private projects + + # required for private projects if [ -z "${CIRCLECI_API_TOKEN}" ]; then echo "CIRCLECI_API_TOKEN not set. Private projects will be inaccessible." else - fetch "https://circleci.com/api/v2/me" "/tmp/me.cci" - me=$(jq -e '.id' /tmp/me.cci) + fetch "https://circleci.com/api/v2/me" "${tmp}/me.cci" + me=$(jq -e '.id' "${tmp}/me.cci") echo "Using API key for user: ${me}" fi } @@ -34,15 +33,15 @@ fetch(){ url=$1 target=$2 method=${3:-GET} - debug "Performing API ${method} Call to ${url} to ${target}" + debug "api call: ${method} ${url} > ${target}" http_response=$(curl -s -X "${method}" -H "Circle-Token: ${CIRCLECI_API_TOKEN}" -H "Content-Type: application/json" -o "${target}" -w "%{http_code}" "${url}") if [ "${http_response}" != "200" ]; then - echo "ERROR: Server returned error code: ${http_response}" + echo "ERROR: api-call: server returned error code: ${http_response}" debug "${target}" exit 1 else - debug "API Success" + debug "api call: success" fi } @@ -52,32 +51,41 @@ fetch_pipelines(){ echo "Only blocking execution if running previous workflows on branch: ${CIRCLE_BRANCH}" pipelines_api_url_template="https://circleci.com/api/v2/project/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pipeline?branch=${CIRCLE_BRANCH}" - debug "Fetching piplines: ${pipelines_api_url_template} > ${pipeline_file}" - fetch "${pipelines_api_url_template}" "${pipeline_file}" + debug "Fetching piplines for: ${CIRCLE_BRANCH}" + fetch "${pipelines_api_url_template}" "${pipelines_file}" } -# fetch all running or created workflows for a given pipeline +# iterate over all pipelines, and fetch workflow information fetch_pipeline_workflows(){ - for pipeline in $(jq -r ".items[] | .id //empty" ${pipeline_file} | uniq) + for pipeline in $(jq -r ".items[] | .id //empty" "${pipelines_file}" | uniq) do - debug "Fetching workflow information for pipeline: ${pipeline}" + debug "Fetching workflow metadata for pipeline: ${pipeline}" pipeline_detail=${tmp}/pipeline-${pipeline}.json fetch "https://circleci.com/api/v2/pipeline/${pipeline}/workflow" "${pipeline_detail}" created_at=$(jq -r '.items[] | .created_at' "${pipeline_detail}") - debug "Pipeline's workflow was created at: ${created_at}" + debug "Pipeline:'s workflow was created at: ${created_at}" done + + # filter out any workflows that are not active if [ "${CONFIG_INCLUDE_ON_HOLD}" = "1" ]; then active_statuses="$(printf '%s' '["running","created","on_hold"]')" else active_statuses="$(printf '%s' '["running","created"]')" fi - jq -s "[.[].items[] | select([.status] | inside(${active_statuses}))]" ${tmp}/pipeline-*.json > ${workflows_file} + + # filter out any workflows that match the ignored list + ignored_workflows="[]" + if [ -z "${CONFIG_IGNORED_WORKFLOWS}" ]; then + ignored_workflows=$(printf '"%s"' "${CONFIG_IGNORED_WORKFLOWS}" | jq 'split(",")') + fi + + jq -s "[.[].items[] | select(([.name] | inside(${ignored_workflows}) | not) and ([.status] | inside(${active_statuses})))]" "${tmp}"/pipeline-*.json > "${workflows_file}" } # parse workflows to fetch parmeters about this current running workflow load_current_workflow_values(){ - my_commit_time=$(jq ".[] | select (.id == \"${CIRCLE_WORKFLOW_ID}\").created_at" ${workflows_file}) - my_workflow_id=$(jq ".[] | select (.id == \"${CIRCLE_WORKFLOW_ID}\").id" ${workflows_file}) + my_commit_time=$(jq ".[] | select (.id == \"${CIRCLE_WORKFLOW_ID}\").created_at" "${workflows_file}") + my_workflow_id=$(jq ".[] | select (.id == \"${CIRCLE_WORKFLOW_ID}\").id" "${workflows_file}") } # load all the data necessary to compare build executions @@ -89,8 +97,8 @@ update_comparables(){ load_current_workflow_values echo "This job will block until no previous workflows have *any* workflows running." - oldest_running_workflow_id=$(jq '. | sort_by(.created_at) | .[0].id' ${workflows_file}) - oldest_commit_time=$(jq '. | sort_by(.created_at) | .[0].created_at' ${workflows_file}) + oldest_running_workflow_id=$(jq '. | sort_by(.created_at) | .[0].id' "${workflows_file}") + oldest_commit_time=$(jq '. | sort_by(.created_at) | .[0].created_at' "${workflows_file}") if [ -z "${oldest_commit_time}" ] || [ -z "${oldest_running_workflow_id}" ]; then echo "ERROR: API Error - unable to load previous workflow timings. File a bug" exit 1 diff --git a/src/scripts/internal-queue.sh b/src/scripts/pipeline-queue.sh similarity index 100% rename from src/scripts/internal-queue.sh rename to src/scripts/pipeline-queue.sh diff --git a/src/scripts/test.sh b/src/scripts/test.sh index e913c83..e910af4 100755 --- a/src/scripts/test.sh +++ b/src/scripts/test.sh @@ -1,21 +1,22 @@ #!/usr/bin/env bash # shellcheck disable=all +# This script is used to test the queuing scripts in the src/scripts directory +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# This script is used to test the scripts in the src/scripts directory TMP_DIR=`mktemp -d` -# job config +# default cci job config CONFIG_DEBUG_ENABLED=1 CONFIG_TIME=10 CONFIG_DONT_QUIT=1 CONFIG_ONLY_ON_BRANCH=* CONFIG_CONFIDENCE=1 -# test values +# local test values CIRCLE_PIPELINE_ID=4e1ffd3b-c260-43db-a66c-8766eaf7fc88 CIRCLE_WORKFLOW_ID=476fa7ff-7534-440f-9a65-a2779170d344 CIRCLE_PROJECT_USERNAME=promiseofcake CIRCLE_PROJECT_REPONAME=circleci-workflow-queue CIRCLECI_API_TOKEN=${CIRCLECI_USER_TOKEN} -source ./internal-queue.sh +source ${SCRIPT_DIR}/$@ diff --git a/src/test/fixture.json b/src/test/fixture.json new file mode 100644 index 0000000..dfcf74a --- /dev/null +++ b/src/test/fixture.json @@ -0,0 +1,28 @@ +{ + "next_page_token": null, + "items": [ + { + "pipeline_id": "29422b30-ac5f-4232-b543-d1e6319310e1", + "id": "62badd1b-ed00-49fe-89bd-d6c0e452a426", + "name": "build", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "status": "running", + "started_by": "0989fcb2-c8d2-43ed-bfdd-3ff63487e121", + "pipeline_number": 60988, + "created_at": "2024-04-12T22:58:31Z", + "stopped_at": null + }, + { + "pipeline_id": "29422b30-ac5f-4232-b543-d1e6319310e1", + "id": "43c70590-ceb7-472d-9169-5f5061c9e625", + "name": "setup", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "tag": "setup", + "status": "running", + "started_by": "0989fcb2-c8d2-43ed-bfdd-3ff63487e121", + "pipeline_number": 60988, + "created_at": "2024-04-12T22:58:14Z", + "stopped_at": "2024-04-12T22:58:28Z" + } + ] +} From b9763ce50a8b33b8c504e8d44bdc19f11b85ab48 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Mon, 15 Apr 2024 21:08:13 -0700 Subject: [PATCH 3/6] Validate Queue Behavior (#19) * feat: ensure we can validate the correct ordering of jobs * fix: rename to the correct pipeline to ignore * debugging * fix: debug * walk back validation --- .circleci/test-deploy.yml | 35 +++++++++++++---- src/commands/global_block.yml | 2 +- src/scripts/global-queue.sh | 6 ++- src/scripts/pipeline-queue.sh | 9 ++--- src/test/fixture.json | 28 -------------- test/workflow-fixture.json | 72 +++++++++++++++++++++++++++++++++++ 6 files changed, 109 insertions(+), 43 deletions(-) delete mode 100644 src/test/fixture.json create mode 100644 test/workflow-fixture.json diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml index e586665..e30fc12 100644 --- a/.circleci/test-deploy.yml +++ b/.circleci/test-deploy.yml @@ -7,32 +7,51 @@ filters: &filters tags: only: /.*/ +jobs: + sleep: + docker: + - image: cimg/base:edge + parameters: + time: + type: integer + steps: + - run: echo "sleeping << parameters.time >>s" && sleep << parameters.time >> + workflows: + test-pipeline-a: + jobs: + - workflow-queue/pipeline-queue: + context: orb-publishing + debug: true + test-pipeline-b: + jobs: + - sleep: + time: 5 + test-pipeline-c: + jobs: + - sleep: + time: 20 + test-global-queue: jobs: - workflow-queue/global-queue: - filters: - tags: - ignore: /.*/ context: orb-publishing - test-pipeline-queue: - jobs: - - workflow-queue/pipeline-queue: + debug: true + ignored-workflows: "test-pipeline-a" # given we are in one pipeline, don't block on ourselves filters: tags: ignore: /.*/ - context: orb-publishing test-deploy: jobs: - orb-tools/pack: filters: *filters - orb-tools/publish: + context: orb-publishing orb-name: promiseofcake/workflow-queue vcs-type: << pipeline.project.type >> pub-type: production requires: - orb-tools/pack - context: orb-publishing filters: branches: ignore: /.*/ diff --git a/src/commands/global_block.yml b/src/commands/global_block.yml index 300ea35..e1da939 100755 --- a/src/commands/global_block.yml +++ b/src/commands/global_block.yml @@ -43,6 +43,6 @@ steps: CONFIG_DONT_QUIT: "<< parameters.dont-quit >>" CONFIG_ONLY_ON_BRANCH: "<< parameters.only-on-branch >>" CONFIG_CONFIDENCE: "<< parameters.confidence >>" - CONFIG_IGNORED_WORKFLOWS: " << parameters.ignored-workflows >>" + CONFIG_IGNORED_WORKFLOWS: "<< parameters.ignored-workflows >>" CONFIG_INCLUDE_ON_HOLD: " << parameters.include-on-hold >>" command: <> diff --git a/src/scripts/global-queue.sh b/src/scripts/global-queue.sh index a479c85..769f9c1 100644 --- a/src/scripts/global-queue.sh +++ b/src/scripts/global-queue.sh @@ -73,12 +73,16 @@ fetch_pipeline_workflows(){ active_statuses="$(printf '%s' '["running","created"]')" fi + debug "filtering on statuses: ${active_statuses}" + # filter out any workflows that match the ignored list ignored_workflows="[]" - if [ -z "${CONFIG_IGNORED_WORKFLOWS}" ]; then + if [ -n "${CONFIG_IGNORED_WORKFLOWS}" ]; then ignored_workflows=$(printf '"%s"' "${CONFIG_IGNORED_WORKFLOWS}" | jq 'split(",")') fi + debug "ignoring workflows: ${ignored_workflows}" + jq -s "[.[].items[] | select(([.name] | inside(${ignored_workflows}) | not) and ([.status] | inside(${active_statuses})))]" "${tmp}"/pipeline-*.json > "${workflows_file}" } diff --git a/src/scripts/pipeline-queue.sh b/src/scripts/pipeline-queue.sh index 08862af..6c3834e 100644 --- a/src/scripts/pipeline-queue.sh +++ b/src/scripts/pipeline-queue.sh @@ -1,6 +1,5 @@ #!/bin/bash tmp=${TMP_DIR:-/tmp} -tmp="/tmp" workflows_file="${tmp}/workflow_status.json" # logger command for debugging @@ -21,8 +20,8 @@ load_variables(){ if [ -z "${CIRCLECI_API_TOKEN}" ]; then echo "CIRCLECI_API_TOKEN not set. Private projects will be inaccessible." else - fetch "https://circleci.com/api/v2/me" "/tmp/me.cci" - me=$(jq -e '.id' /tmp/me.cci) + fetch "https://circleci.com/api/v2/me" "${tmp}/me.cci" + me=$(jq -e '.id' "${tmp}/me.cci") echo "Using API key for user: ${me}" fi } @@ -51,14 +50,14 @@ fetch_pipeline_workflows(){ fetch "https://circleci.com/api/v2/pipeline/${CIRCLE_PIPELINE_ID}/workflow" "${pipeline_detail}" debug "Pipeline's details: $(jq -r '.' "${pipeline_detail}")" # fetch all workflows that are not this workflow - jq -s "[.[].items[] | select((.id != \"${CIRCLE_WORKFLOW_ID}\") and ((.status == \"running\") or (.status == \"created\")))]" "${pipeline_detail}" > ${workflows_file} + jq -s "[.[].items[] | select((.id != \"${CIRCLE_WORKFLOW_ID}\") and ((.status == \"running\") or (.status == \"created\")))]" "${pipeline_detail}" > "${workflows_file}" } # load all the data necessary to compare build executions update_comparables(){ fetch_pipeline_workflows - running_workflows=$(jq length ${workflows_file}) + running_workflows=$(jq length "${workflows_file}") debug "Running workflows: ${running_workflows}" } diff --git a/src/test/fixture.json b/src/test/fixture.json deleted file mode 100644 index dfcf74a..0000000 --- a/src/test/fixture.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "next_page_token": null, - "items": [ - { - "pipeline_id": "29422b30-ac5f-4232-b543-d1e6319310e1", - "id": "62badd1b-ed00-49fe-89bd-d6c0e452a426", - "name": "build", - "project_slug": "gh/promiseofcake/circleci-workflow-queue", - "status": "running", - "started_by": "0989fcb2-c8d2-43ed-bfdd-3ff63487e121", - "pipeline_number": 60988, - "created_at": "2024-04-12T22:58:31Z", - "stopped_at": null - }, - { - "pipeline_id": "29422b30-ac5f-4232-b543-d1e6319310e1", - "id": "43c70590-ceb7-472d-9169-5f5061c9e625", - "name": "setup", - "project_slug": "gh/promiseofcake/circleci-workflow-queue", - "tag": "setup", - "status": "running", - "started_by": "0989fcb2-c8d2-43ed-bfdd-3ff63487e121", - "pipeline_number": 60988, - "created_at": "2024-04-12T22:58:14Z", - "stopped_at": "2024-04-12T22:58:28Z" - } - ] -} diff --git a/test/workflow-fixture.json b/test/workflow-fixture.json new file mode 100644 index 0000000..b588320 --- /dev/null +++ b/test/workflow-fixture.json @@ -0,0 +1,72 @@ +{ + "next_page_token": null, + "items": [ + { + "pipeline_id": "c77bf329-c78f-4591-a71f-8541759fb55e", + "id": "5d707409-4828-47bd-9a59-0e349f164d02", + "name": "test-deploy", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "status": "success", + "started_by": "df121a07-9607-43e1-8d85-b100e1a6c0a2", + "pipeline_number": 169, + "created_at": "2024-04-16T00:11:30Z", + "stopped_at": "2024-04-16T00:11:40Z" + }, + { + "pipeline_id": "c77bf329-c78f-4591-a71f-8541759fb55e", + "id": "4967e8f6-a4ad-4b32-87e4-445f2d6d912f", + "name": "test-global-queue", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "status": "success", + "started_by": "df121a07-9607-43e1-8d85-b100e1a6c0a2", + "pipeline_number": 169, + "created_at": "2024-04-16T00:11:30Z", + "stopped_at": "2024-04-16T00:12:11Z" + }, + { + "pipeline_id": "c77bf329-c78f-4591-a71f-8541759fb55e", + "id": "9258267c-bfaa-4e6a-bbb2-31ac5bce5bd4", + "name": "test-pipeline-c", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "status": "success", + "started_by": "df121a07-9607-43e1-8d85-b100e1a6c0a2", + "pipeline_number": 169, + "created_at": "2024-04-16T00:11:30Z", + "stopped_at": "2024-04-16T00:12:05Z" + }, + { + "pipeline_id": "c77bf329-c78f-4591-a71f-8541759fb55e", + "id": "633d6ddf-8ed3-45c5-a376-eba934bf8533", + "name": "test-pipeline-b", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "status": "success", + "started_by": "df121a07-9607-43e1-8d85-b100e1a6c0a2", + "pipeline_number": 169, + "created_at": "2024-04-16T00:11:29Z", + "stopped_at": "2024-04-16T00:11:49Z" + }, + { + "pipeline_id": "c77bf329-c78f-4591-a71f-8541759fb55e", + "id": "b332b406-2860-4354-be89-2b363301fdd4", + "name": "test-pipeline-a", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "status": "failed", + "started_by": "df121a07-9607-43e1-8d85-b100e1a6c0a2", + "pipeline_number": 169, + "created_at": "2024-04-16T00:11:29Z", + "stopped_at": "2024-04-16T00:12:39Z" + }, + { + "pipeline_id": "c77bf329-c78f-4591-a71f-8541759fb55e", + "id": "502d6e3e-8ede-4d8f-b553-87005bfc5271", + "name": "lint-pack", + "project_slug": "gh/promiseofcake/circleci-workflow-queue", + "tag": "setup", + "status": "success", + "started_by": "df121a07-9607-43e1-8d85-b100e1a6c0a2", + "pipeline_number": 169, + "created_at": "2024-04-16T00:10:55Z", + "stopped_at": "2024-04-16T00:11:30Z" + } + ] +} From 4466452f853360fa4fd2a9ab4462fd31ca8836f3 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Mon, 15 Apr 2024 21:21:10 -0700 Subject: [PATCH 4/6] feat: update documentation --- README.md | 6 ++++-- src/README.md | 26 -------------------------- src/commands/global_block.yml | 18 +++++++++--------- src/commands/pipeline_block.yml | 10 +++++----- src/examples/example.yml | 2 +- src/jobs/global-queue.yml | 16 ++++++++-------- test_workflow.json | 16 ---------------- 7 files changed, 27 insertions(+), 67 deletions(-) delete mode 100644 src/README.md delete mode 100644 test_workflow.json diff --git a/README.md b/README.md index f10888f..8533da0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,11 @@ ## Introduction -Forked from and updated to reduce the use-cases, and migrate to the CircleCI V2 API +Originally forked from and updated to reduce the use-cases, and migrate to the CircleCI V2 API -The purpose of this Orb is to add a concept of a queue to specific branch workflow tasks in CircleCi. The main use-case is to isolate a set of changes to ensure that one set of a thing is running at one time. Think of smoke-tests against a nonproduction environment as a promotion gate. +The purpose of this Orb is to add a concept of a queue to specific branch's workflow tasks in CircleCi. The main use-case is to isolate a set of changes to ensure that one set of a thing is running at one time. Think of smoke-tests against a nonproduction environment as a promotion gate. + +Additional use-cases are for queueing workflows within a given pipeline (a feature missing today from CircleCi). ## Configuration Requirements diff --git a/src/README.md b/src/README.md deleted file mode 100644 index b8e1015..0000000 --- a/src/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Orb Source - -Orbs are shipped as individual `orb.yml` files, however, to make development easier, it is possible to author an orb in _unpacked_ form, which can be _packed_ with the CircleCI CLI and published. - -The default `.circleci/config.yml` file contains the configuration code needed to automatically pack, test, and deploy any changes made to the contents of the orb source in this directory. - -## @orb.yml - -This is the entry point for our orb "tree", which becomes our `orb.yml` file later. - -Within the `@orb.yml` we generally specify 4 configuration keys - -**Keys** - -1. **version** - Specify version 2.1 for orb-compatible configuration `version: 2.1` -2. **description** - Give your orb a description. Shown within the CLI and orb registry -3. **display** - Specify the `home_url` referencing documentation or product URL, and `source_url` linking to the orb's source repository. -4. **orbs** - (optional) Some orbs may depend on other orbs. Import them here. - -## See: - - [Orb Author Intro](https://circleci.com/docs/2.0/orb-author-intro/#section=configuration) - - [Reusable Configuration](https://circleci.com/docs/2.0/reusing-config) diff --git a/src/commands/global_block.yml b/src/commands/global_block.yml index e1da939..0e158da 100755 --- a/src/commands/global_block.yml +++ b/src/commands/global_block.yml @@ -1,19 +1,19 @@ description: > - This command is the execution portion to determine that only a single global workflow is running at a point in time. - + When used, this command blocks the assigned workflow from running until all previous (global) workflows have completed. + This ensures only one instance of a given workflow is running at a time across all defined branches. parameters: debug: type: boolean default: false - description: "If enabled, DEBUG messages will be logged." + description: "When enabled, additional debug logging with be output." time: type: string default: "10" - description: "Minutes to wait before giving up." + description: "Number of minutes to wait for a lock before giving up." dont-quit: type: boolean default: false - description: "Force job through once time expires instead of failing." + description: "If true, forces the job through once time expires instead of failing." only-on-branch: type: string default: "*" @@ -22,17 +22,17 @@ parameters: type: string default: "1" description: > - Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending", + Due to concurrency issues, the number of times should we requery the pipeline list to ensure previous jobs are "pending", but not yet active. This number indicates the threshold for API returning no previous pending pipelines. - Default is one confirmation, increase if you see issues. + Default is `1` confirmation, increase if you see issues. ignored-workflows: type: string default: "" - description: "Comma separated list of workflow names to ignore as blocking workflows." + description: Comma separated list of workflow names to ignore as blocking workflows for the global queue. include-on-hold: type: boolean default: false - description: Consider on-hold workflows waiting for approval as running and include them in the queue. + description: Consider `on-hold`` workflows waiting for approval as running and include them in the queue. steps: - run: diff --git a/src/commands/pipeline_block.yml b/src/commands/pipeline_block.yml index 3234fc3..517f56f 100644 --- a/src/commands/pipeline_block.yml +++ b/src/commands/pipeline_block.yml @@ -1,22 +1,22 @@ description: > - This command blocks execution of parallel pipeline workflows until necessary criteria is met. + This command blocks execution of a workflow within the context of a given pipeline. parameters: debug: type: boolean default: false - description: "If enabled, DEBUG messages will be logged." + description: "When enabled, additional debug logging with be output." confidence: type: string default: "1" description: > - Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending", + Due to concurrency issues, the number of times should we requery the pipeline list to ensure previous jobs are "pending", but not yet active. This number indicates the threshold for API returning no previous pending pipelines. - Default is one confirmation, increase if you see issues. + Default is `1` confirmation, increase if you see issues. steps: - run: - name: Block additional pipeline work executions until all other workflows are complete + name: Blocking execution until the current workflow is the last to run in the given pipeline. environment: CONFIG_DEBUG_ENABLED: "<< parameters.debug >>" CONFIG_CONFIDENCE: "<< parameters.confidence >>" diff --git a/src/examples/example.yml b/src/examples/example.yml index 3f7a2c3..d911b38 100755 --- a/src/examples/example.yml +++ b/src/examples/example.yml @@ -3,7 +3,7 @@ description: > usage: version: 2.1 orbs: - workflow-queue: promiseofcake/workflow-queue@1 + workflow-queue: promiseofcake/workflow-queue@2 workflows: example: jobs: diff --git a/src/jobs/global-queue.yml b/src/jobs/global-queue.yml index bcb1d1b..4cadcf5 100755 --- a/src/jobs/global-queue.yml +++ b/src/jobs/global-queue.yml @@ -1,5 +1,5 @@ description: > - This job ensures only a single global workflow is running at a given point in time. + This job ensures only a single defined global workflow is running at a given point in time. docker: - image: cimg/base:stable @@ -9,15 +9,15 @@ parameters: debug: type: boolean default: false - description: "If enabled, debug messages will be logged." + description: "When enabled, additional debug logging with be output." time: type: string default: "10" - description: "Minutes to wait before giving up." + description: "Number of minutes to wait for a lock before giving up." dont-quit: type: boolean default: false - description: "Force job through once time (above) expires instead of failing." + description: "If true, forces the job through once time expires instead of failing." only-on-branch: type: string default: "*" @@ -26,17 +26,17 @@ parameters: type: string default: "1" description: > - Due to concurrency issues, how many times should we requery the pipeline list to ensure previous jobs are "pending", + Due to concurrency issues, the number of times should we requery the pipeline list to ensure previous jobs are "pending", but not yet active. This number indicates the threshold for API returning no previous pending pipelines. - Default is 1 confirmation, increase if you see issues. + Default is `1` confirmation, increase if you see issues. ignored-workflows: type: string default: "" - description: "Comma separated list of workflow names to ignore as blocking workflows." + description: Comma separated list of workflow names to ignore as blocking workflows for the global queue. include-on-hold: type: boolean default: false - description: Consider on-hold workflows waiting for approval as running and include them in the queue. + description: Consider `on-hold`` workflows waiting for approval as running and include them in the queue. steps: - global_block: diff --git a/test_workflow.json b/test_workflow.json deleted file mode 100644 index ddfd282..0000000 --- a/test_workflow.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "next_page_token": null, - "items": [ - { - "pipeline_id": "49287a8d-969e-4bba-850a-051f18078b2c", - "id": "574f4ff2-eadb-48ca-a082-8241c000d5bc", - "name": "workflow", - "project_slug": "gh/boxfortinc/level99-nimbus", - "status": "on_hold", - "started_by": "f471266e-c8b8-4237-8564-36e5537ad72d", - "pipeline_number": 1719, - "created_at": "2023-10-17T17:38:27Z", - "stopped_at": null - } - ] -} From e131089ad6334782edfa529bdd768951cb61d093 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Tue, 16 Apr 2024 15:47:37 -0700 Subject: [PATCH 5/6] Update src/jobs/global-queue.yml Co-authored-by: Andy Horner <114095421+andyhorner-chime@users.noreply.github.com> --- src/jobs/global-queue.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/global-queue.yml b/src/jobs/global-queue.yml index 4cadcf5..3040dd1 100755 --- a/src/jobs/global-queue.yml +++ b/src/jobs/global-queue.yml @@ -36,7 +36,7 @@ parameters: include-on-hold: type: boolean default: false - description: Consider `on-hold`` workflows waiting for approval as running and include them in the queue. + description: Consider `on-hold` workflows waiting for approval as running and include them in the queue. steps: - global_block: From 933a202977f532677ff63ab69d622bca28b1ceb1 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Tue, 16 Apr 2024 15:47:42 -0700 Subject: [PATCH 6/6] Update src/commands/global_block.yml Co-authored-by: Andy Horner <114095421+andyhorner-chime@users.noreply.github.com> --- src/commands/global_block.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/global_block.yml b/src/commands/global_block.yml index 0e158da..291a008 100755 --- a/src/commands/global_block.yml +++ b/src/commands/global_block.yml @@ -32,7 +32,7 @@ parameters: include-on-hold: type: boolean default: false - description: Consider `on-hold`` workflows waiting for approval as running and include them in the queue. + description: Consider `on-hold` workflows waiting for approval as running and include them in the queue. steps: - run: