diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecc27d016a811..ae25a2c5953f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -229,19 +229,21 @@ variables: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" -.crate-publishing-pipeline: - rules: - - if: $CI_COMMIT_REF_NAME != "master" - when: never +.crates-publishing-variables: + variables: + CRATESIO_CRATES_OWNER: parity-crate-owner + REPO: substrate + REPO_OWNER: paritytech -.scheduled-crate-publishing-pipeline: +.crates-publishing-pipeline: + extends: .crates-publishing-variables rules: - - !reference [.crate-publishing-pipeline, rules] - - if: $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing" + - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "master" && $PIPELINE == "automatic-crate-publishing" .crates-publishing-template: - stage: test - extends: .docker-env + extends: + - .docker-env + - .crates-publishing-variables # collect artifacts even on failure so that we know how the crates were generated (they'll be # generated to the artifacts folder according to SPUB_TMP further down) artifacts: @@ -251,12 +253,7 @@ variables: paths: - artifacts/ variables: - CRATESIO_API: https://crates.io/api - CRATESIO_CRATES_OWNER: parity-crate-owner - GH_API: https://api.github.com - REPO: substrate - REPO_OWNER: paritytech - SPUB_TMP: artifacts + SPUB_TMP: artifacts #### stage: .pre @@ -274,6 +271,18 @@ skip-if-draft: - ./scripts/ci/gitlab/skip_if_draft.sh allow_failure: true +check-crates-publishing-pipeline: + stage: .pre + extends: + - .kubernetes-env + - .crates-publishing-pipeline + script: + - git clone + --depth 1 + --branch "$RELENG_SCRIPTS_BRANCH" + https://github.com/paritytech/releng-scripts.git + - ONLY_CHECK_PIPELINE=true ./releng-scripts/publish-crates + include: # check jobs - scripts/ci/gitlab/pipeline/check.yml diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index d1a7514d1707b..4eafb0a055399 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -184,19 +184,19 @@ publish-draft-release: .publish-crates-template: stage: publish - extends: .crates-publishing-template + extends: + - .crates-publishing-template + - .crates-publishing-pipeline # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite # the releases of one another. Use resource_group to ensure that at most one instance of this job # is running at any given time. resource_group: crates-publishing - variables: - # crates.io rate limits crates publishing by 1 per minute, so a delay needs to be inserted - # slightly higher than that after publishing each crate. The value is specified in seconds. - SPUB_AFTER_PUBLISH_DELAY: 64 - # We might have to publish lots of crates at a time. Given the 1 minute delay introduced above and - # taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, that would equate - # to roughly 202 minutes of delay, or 3h and 22 minutes. As such, the job needs to have a much - # higher timeout than average. + # crates.io currently rate limits crate publishing at 1 per minute: + # https://github.com/paritytech/release-engineering/issues/123#issuecomment-1335509748 + # Taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, in the worst + # case, due to the rate limits alone, we'd have to wait through at least 202 minutes of delay. + # Taking into account also the verification steps and extra synchronization delays after + # publishing the crate, the job needs to have a much higher timeout than average. timeout: 9h # A custom publishing environment is used for us to be able to set up protected secrets # specifically for it @@ -211,15 +211,9 @@ publish-draft-release: - rusty-cachier cache upload publish-crates: - extends: - - .publish-crates-template - - .scheduled-crate-publishing-pipeline - needs: - - job: publish-crates-locally - artifacts: false + extends: .publish-crates-template publish-crates-manual: extends: .publish-crates-template - rules: !reference [.crate-publishing-pipeline, rules] when: manual - allow_failure: true + interruptible: false diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 466019e42e3e1..f00857ffa9935 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -14,7 +14,7 @@ find-fail-ci-phrase: script: - set +e - rg --line-number --hidden --type rust --glob '!{.git,target}' "$ASSERT_REGEX" .; exit_status=$? - - if [ $exit_status -eq 0 ]; then + - if [ $exit_status -eq 0 ]; then echo "$ASSERT_REGEX was found, exiting with 1"; exit 1; else @@ -417,9 +417,14 @@ cargo-check-each-crate: parallel: 2 publish-crates-locally: + stage: test extends: - .test-refs - .crates-publishing-template + # When lots of crates are taken into account (for example on master where all crates are tested) + # the job might take a long time, as evidenced by: + # https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2269364 + timeout: 4h script: - rusty-cachier snapshot create - git clone