From 2d0772968272c23bd9ab4019db2a08d32a6b3046 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 16 Jan 2023 08:48:31 -0300 Subject: [PATCH 1/7] more improvements for the crate publishing pipeline --- .gitlab-ci.yml | 11 +--------- scripts/ci/gitlab/pipeline/publish.yml | 30 ++++++++++++-------------- scripts/ci/gitlab/pipeline/test.yml | 6 +++++- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecc27d016a811..2d2677344ca72 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,16 +229,6 @@ variables: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" -.crate-publishing-pipeline: - rules: - - if: $CI_COMMIT_REF_NAME != "master" - when: never - -.scheduled-crate-publishing-pipeline: - rules: - - !reference [.crate-publishing-pipeline, rules] - - if: $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" - .crates-publishing-template: stage: test extends: .docker-env @@ -254,6 +244,7 @@ variables: CRATESIO_API: https://crates.io/api CRATESIO_CRATES_OWNER: parity-crate-owner GH_API: https://api.github.com + GH_URL: https://github.com REPO: substrate REPO_OWNER: paritytech SPUB_TMP: artifacts diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index d1a7514d1707b..d1965cb5b6d43 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -189,18 +189,18 @@ publish-draft-release: # the releases of one another. Use resource_group to ensure that at most one instance of this job # is running at any given time. resource_group: crates-publishing - variables: - # crates.io rate limits crates publishing by 1 per minute, so a delay needs to be inserted - # slightly higher than that after publishing each crate. The value is specified in seconds. - SPUB_AFTER_PUBLISH_DELAY: 64 - # We might have to publish lots of crates at a time. Given the 1 minute delay introduced above and - # taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, that would equate - # to roughly 202 minutes of delay, or 3h and 22 minutes. As such, the job needs to have a much - # higher timeout than average. + # crates.io currently rate limits crate publishing at 1 per minute: + # https://github.com/paritytech/release-engineering/issues/123#issuecomment-1335509748 + # Taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, in the worst + # case, due to the rate limits alone, we'd have to wait through at least 202 minutes of delay. + # Taking into account also the verification steps and extra synchronization delays after + # publishing the crate, the job needs to have a much higher timeout than average. timeout: 9h # A custom publishing environment is used for us to be able to set up protected secrets # specifically for it environment: publish-crates + variables: + PUBLISH_CRATES_BRANCH: master script: - rusty-cachier snapshot create - git clone @@ -211,15 +211,13 @@ publish-draft-release: - rusty-cachier cache upload publish-crates: - extends: - - .publish-crates-template - - .scheduled-crate-publishing-pipeline - needs: - - job: publish-crates-locally - artifacts: false + extends: .publish-crates-template + rules: + - if: $CI_COMMIT_REF_NAME == "$PUBLISH_CRATES_BRANCH" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" publish-crates-manual: extends: .publish-crates-template - rules: !reference [.crate-publishing-pipeline, rules] + rules: + - if: $CI_COMMIT_REF_NAME == "$PUBLISH_CRATES_BRANCH" when: manual - allow_failure: true + interruptible: false diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 466019e42e3e1..60937e6d38640 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -14,7 +14,7 @@ find-fail-ci-phrase: script: - set +e - rg --line-number --hidden --type rust --glob '!{.git,target}' "$ASSERT_REGEX" .; exit_status=$? - - if [ $exit_status -eq 0 ]; then + - if [ $exit_status -eq 0 ]; then echo "$ASSERT_REGEX was found, exiting with 1"; exit 1; else @@ -420,6 +420,10 @@ publish-crates-locally: extends: - .test-refs - .crates-publishing-template + # When lots of crates are taken into account (for example on master where all crates are tested) + # the job might take a long time, as evidenced by: + # https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2269364 + timeout: 4h script: - rusty-cachier snapshot create - git clone From 50fd8e7a87131296462abaaead280cfff6cb3ebb Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 16 Jan 2023 09:08:09 -0300 Subject: [PATCH 2/7] move default definitions to the publish-crates script --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d2677344ca72..3e6573060b822 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -241,10 +241,7 @@ variables: paths: - artifacts/ variables: - CRATESIO_API: https://crates.io/api CRATESIO_CRATES_OWNER: parity-crate-owner - GH_API: https://api.github.com - GH_URL: https://github.com REPO: substrate REPO_OWNER: paritytech SPUB_TMP: artifacts From fe3f93c5264292a6e6fcfb68ac59e94663d79aa8 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 16 Jan 2023 09:57:17 -0300 Subject: [PATCH 3/7] add script to check the crate publishing pipeline at the start --- .gitlab-ci.yml | 32 +++++++++++++++++++++----- scripts/ci/gitlab/pipeline/publish.yml | 8 ++----- scripts/ci/gitlab/pipeline/test.yml | 1 + 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e6573060b822..5cbbacac54d73 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,9 +229,15 @@ variables: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" +.crates-publishing-pipeline: + variables: + CRATES_PUBLISHING_REF: master + CRATES_PUBLISHING_PIPELINE: automatic-crate-publishing + .crates-publishing-template: - stage: test - extends: .docker-env + extends: + - .docker-env + - .crates-publishing-variables # collect artifacts even on failure so that we know how the crates were generated (they'll be # generated to the artifacts folder according to SPUB_TMP further down) artifacts: @@ -241,10 +247,10 @@ variables: paths: - artifacts/ variables: - CRATESIO_CRATES_OWNER: parity-crate-owner - REPO: substrate - REPO_OWNER: paritytech - SPUB_TMP: artifacts + CRATESIO_CRATES_OWNER: parity-crate-owner + REPO: substrate + REPO_OWNER: paritytech + SPUB_TMP: artifacts #### stage: .pre @@ -262,6 +268,20 @@ skip-if-draft: - ./scripts/ci/gitlab/skip_if_draft.sh allow_failure: true +check-crates-publishing-pipeline: + extends: + - .kubernetes-env + - .crates-publishing-variables + stage: .pre + rules: + - if: $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" + script: + - git clone + --depth 1 + --branch "$RELENG_SCRIPTS_BRANCH" + https://github.com/paritytech/releng-scripts.git + - ONLY_CHECK_PIPELINE=true ./releng-scripts/publish-crates + include: # check jobs - scripts/ci/gitlab/pipeline/check.yml diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index d1965cb5b6d43..dade753bc7de8 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -185,6 +185,8 @@ publish-draft-release: .publish-crates-template: stage: publish extends: .crates-publishing-template + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "$CRATES_PUBLISHING_REF" && $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite # the releases of one another. Use resource_group to ensure that at most one instance of this job # is running at any given time. @@ -199,8 +201,6 @@ publish-draft-release: # A custom publishing environment is used for us to be able to set up protected secrets # specifically for it environment: publish-crates - variables: - PUBLISH_CRATES_BRANCH: master script: - rusty-cachier snapshot create - git clone @@ -212,12 +212,8 @@ publish-draft-release: publish-crates: extends: .publish-crates-template - rules: - - if: $CI_COMMIT_REF_NAME == "$PUBLISH_CRATES_BRANCH" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" publish-crates-manual: extends: .publish-crates-template - rules: - - if: $CI_COMMIT_REF_NAME == "$PUBLISH_CRATES_BRANCH" when: manual interruptible: false diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 60937e6d38640..f00857ffa9935 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -417,6 +417,7 @@ cargo-check-each-crate: parallel: 2 publish-crates-locally: + stage: test extends: - .test-refs - .crates-publishing-template From 114a6bd6e1d34079186af5ed96e53ec65f0c2a84 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 16 Jan 2023 09:58:50 -0300 Subject: [PATCH 4/7] fix yaml references --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cbbacac54d73..a79d90530f50f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -237,7 +237,7 @@ variables: .crates-publishing-template: extends: - .docker-env - - .crates-publishing-variables + - .crates-publishing-pipeline # collect artifacts even on failure so that we know how the crates were generated (they'll be # generated to the artifacts folder according to SPUB_TMP further down) artifacts: @@ -271,7 +271,7 @@ skip-if-draft: check-crates-publishing-pipeline: extends: - .kubernetes-env - - .crates-publishing-variables + - .crates-publishing-pipeline stage: .pre rules: - if: $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" From f070cedcc5d097a193287c77b5f2d5bd6541bbde Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 16 Jan 2023 10:10:53 -0300 Subject: [PATCH 5/7] move more variables to .crates-publishing-pipeline --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a79d90530f50f..15a86e1c06072 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,6 +233,9 @@ variables: variables: CRATES_PUBLISHING_REF: master CRATES_PUBLISHING_PIPELINE: automatic-crate-publishing + CRATESIO_CRATES_OWNER: parity-crate-owner + REPO: substrate + REPO_OWNER: paritytech .crates-publishing-template: extends: @@ -247,9 +250,6 @@ variables: paths: - artifacts/ variables: - CRATESIO_CRATES_OWNER: parity-crate-owner - REPO: substrate - REPO_OWNER: paritytech SPUB_TMP: artifacts #### stage: .pre From 06881dd326dad724a231ba3ae3b26f38a11d72f3 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 16 Jan 2023 10:31:14 -0300 Subject: [PATCH 6/7] separate .crates-publishing-pipeline from .crates-publishing-variables --- .gitlab-ci.yml | 12 ++++++++---- scripts/ci/gitlab/pipeline/publish.yml | 6 +++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15a86e1c06072..0e3d25395a75b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,18 +229,22 @@ variables: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" -.crates-publishing-pipeline: +.crates-publishing-variables: variables: - CRATES_PUBLISHING_REF: master CRATES_PUBLISHING_PIPELINE: automatic-crate-publishing CRATESIO_CRATES_OWNER: parity-crate-owner REPO: substrate REPO_OWNER: paritytech +.crates-publishing-pipeline: + extends: .crates-publishing-variables + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "master" && $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" + .crates-publishing-template: extends: - .docker-env - - .crates-publishing-pipeline + - .crates-publishing-variables # collect artifacts even on failure so that we know how the crates were generated (they'll be # generated to the artifacts folder according to SPUB_TMP further down) artifacts: @@ -269,10 +273,10 @@ skip-if-draft: allow_failure: true check-crates-publishing-pipeline: + stage: .pre extends: - .kubernetes-env - .crates-publishing-pipeline - stage: .pre rules: - if: $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" script: diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index dade753bc7de8..4eafb0a055399 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -184,9 +184,9 @@ publish-draft-release: .publish-crates-template: stage: publish - extends: .crates-publishing-template - rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "$CRATES_PUBLISHING_REF" && $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" + extends: + - .crates-publishing-template + - .crates-publishing-pipeline # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite # the releases of one another. Use resource_group to ensure that at most one instance of this job # is running at any given time. From 83224e6acf95f31badceea5ed8bac424657d2b30 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Mon, 16 Jan 2023 10:33:15 -0300 Subject: [PATCH 7/7] clean up redundant and unused code --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e3d25395a75b..ae25a2c5953f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -231,7 +231,6 @@ variables: .crates-publishing-variables: variables: - CRATES_PUBLISHING_PIPELINE: automatic-crate-publishing CRATESIO_CRATES_OWNER: parity-crate-owner REPO: substrate REPO_OWNER: paritytech @@ -239,7 +238,7 @@ variables: .crates-publishing-pipeline: extends: .crates-publishing-variables rules: - - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "master" && $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" + - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "master" && $PIPELINE == "automatic-crate-publishing" .crates-publishing-template: extends: @@ -277,8 +276,6 @@ check-crates-publishing-pipeline: extends: - .kubernetes-env - .crates-publishing-pipeline - rules: - - if: $PIPELINE == "$CRATES_PUBLISHING_PIPELINE" script: - git clone --depth 1