From 7a2bcf36aa95c2d726cb3721a19ddac7481ebe92 Mon Sep 17 00:00:00 2001 From: Mira Ressel Date: Tue, 14 Mar 2023 20:30:32 +0100 Subject: [PATCH 1/4] timestamp gitlab ci job outputs Based on previous work by @alvicsam in #13047. --- .gitlab-ci.yml | 6 ++++++ scripts/ci/gitlab/timestamp.sh | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100755 scripts/ci/gitlab/timestamp.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94fb3c608b771..5a9003739dfd1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,9 +110,14 @@ variables: before_script: - if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi +.timestamp: + before_script: + - source scripts/ci/gitlab/timestamp.sh + .kubernetes-env: image: "${CI_IMAGE}" before_script: + - !reference [.timestamp, before_script] - !reference [.job-switcher, before_script] - !reference [.prepare-env, before_script] tags: @@ -141,6 +146,7 @@ variables: .docker-env: image: "${CI_IMAGE}" before_script: + - !reference [.timestamp, before_script] - !reference [.job-switcher, before_script] - !reference [.prepare-env, before_script] - !reference [.rust-info-script, script] diff --git a/scripts/ci/gitlab/timestamp.sh b/scripts/ci/gitlab/timestamp.sh new file mode 100755 index 0000000000000..cd6caf947dad1 --- /dev/null +++ b/scripts/ci/gitlab/timestamp.sh @@ -0,0 +1,21 @@ +# https://gist.github.com/altaua/477285db40181ef1793bbed5bae32c6b +# based on https://gist.github.com/jstine35/e0fc0e06ec06d74bc3ebd67585bf2a1d + +s_datestamp() { + TZ=UTC + eof="" + while [[ -z "${eof}" ]]; do + IFS="" read -r line || eof=1 + printf '[%(%F %T)T] %s\n' -1 "${line}" + done +} + +cleanup() { + exec >/dev/null 2>&1 + wait "${TIMESTAMP_PID}" +} + +exec > >(s_datestamp) 2>&1 + +TIMESTAMP_PID=$! +trap cleanup EXIT From b32a61befc6ba284bd1645f1554774ef1fd6afce Mon Sep 17 00:00:00 2001 From: Mira Ressel Date: Wed, 15 Mar 2023 12:13:31 +0100 Subject: [PATCH 2/4] inline timestamp script Some of our jobs don't check out the substrate repo. --- .gitlab-ci.yml | 5 +---- scripts/ci/gitlab/timestamp.sh | 21 --------------------- scripts/ci/gitlab/timestamp.yml | 27 +++++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 25 deletions(-) delete mode 100755 scripts/ci/gitlab/timestamp.sh create mode 100644 scripts/ci/gitlab/timestamp.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5a9003739dfd1..49c291b587a1c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,10 +110,6 @@ variables: before_script: - if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi -.timestamp: - before_script: - - source scripts/ci/gitlab/timestamp.sh - .kubernetes-env: image: "${CI_IMAGE}" before_script: @@ -316,6 +312,7 @@ include: - local: scripts/ci/gitlab/default-pipeline.yml rules: - if: $PIPELINE != "automatic-crate-publishing" + - scripts/ci/gitlab/timestamp.yml #### stage: notify diff --git a/scripts/ci/gitlab/timestamp.sh b/scripts/ci/gitlab/timestamp.sh deleted file mode 100755 index cd6caf947dad1..0000000000000 --- a/scripts/ci/gitlab/timestamp.sh +++ /dev/null @@ -1,21 +0,0 @@ -# https://gist.github.com/altaua/477285db40181ef1793bbed5bae32c6b -# based on https://gist.github.com/jstine35/e0fc0e06ec06d74bc3ebd67585bf2a1d - -s_datestamp() { - TZ=UTC - eof="" - while [[ -z "${eof}" ]]; do - IFS="" read -r line || eof=1 - printf '[%(%F %T)T] %s\n' -1 "${line}" - done -} - -cleanup() { - exec >/dev/null 2>&1 - wait "${TIMESTAMP_PID}" -} - -exec > >(s_datestamp) 2>&1 - -TIMESTAMP_PID=$! -trap cleanup EXIT diff --git a/scripts/ci/gitlab/timestamp.yml b/scripts/ci/gitlab/timestamp.yml new file mode 100644 index 0000000000000..f21d26d660854 --- /dev/null +++ b/scripts/ci/gitlab/timestamp.yml @@ -0,0 +1,27 @@ +.timestamp: + before_script: + - | + echo "Install timestamp handler" + eval ' + # https://gist.github.com/altaua/477285db40181ef1793bbed5bae32c6b + # based on https://gist.github.com/jstine35/e0fc0e06ec06d74bc3ebd67585bf2a1d + + s_datestamp() { + TZ=UTC + eof="" + while [[ -z "${eof}" ]]; do + IFS="" read -r line || eof=1 + printf "[%(%F %T)T] %s\n" -1 "${line}" + done + } + + cleanup() { + exec >/dev/null 2>&1 + wait "${TIMESTAMP_PID}" + } + + exec > >(s_datestamp) 2>&1 + + TIMESTAMP_PID=$! + trap cleanup EXIT + ' From 7a9b58732721bc1fd641a46742b8e441f714fa63 Mon Sep 17 00:00:00 2001 From: Mira Ressel Date: Wed, 15 Mar 2023 12:35:20 +0100 Subject: [PATCH 3/4] include .timestamp in pipelines overriding the default before_script Still not including it in the zombienet jobs, they have their own timestamping anyway. --- scripts/ci/gitlab/pipeline/build.yml | 3 +++ scripts/ci/gitlab/pipeline/test.yml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index 02f25a02a2c22..0a36599c70e24 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -62,6 +62,7 @@ build-linux-substrate: - job: test-linux-stable artifacts: false before_script: + - !reference [.timestamp, before_script] - !reference [.job-switcher, before_script] - mkdir -p ./artifacts/substrate/ - !reference [.rusty-cachier, before_script] @@ -95,6 +96,7 @@ build-linux-substrate: # this variable gets overriden by "rusty-cachier environment inject", use the value as default CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target" before_script: + - !reference [.timestamp, before_script] - !reference [.job-switcher, before_script] - mkdir -p ./artifacts/subkey - !reference [.rusty-cachier, before_script] @@ -120,6 +122,7 @@ build-subkey-macos: # duplicating before_script & script sections from .build-subkey hidden job # to overwrite rusty-cachier integration as it doesn't work on macos before_script: + # skip timestamp script, the osx bash doesn't support printf %()T - !reference [.job-switcher, before_script] - mkdir -p ./artifacts/subkey script: diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 49dbb194fb651..fd031d9aa56c3 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -83,6 +83,7 @@ cargo-check-benches: - .collect-artifacts - .pipeline-stopper-artifacts before_script: + - !reference [.timestamp, before_script] # perform rusty-cachier operations before any further modifications to the git repo to make cargo feel cheated not so much - !reference [.rust-info-script, script] - !reference [.job-switcher, before_script] @@ -145,7 +146,8 @@ node-bench-regression-guard: artifacts: true variables: CI_IMAGE: "paritytech/node-bench-regression-guard:latest" - before_script: [""] + before_script: + - !reference [.timestamp, before_script] script: - echo "------- IMPORTANT -------" - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job" @@ -419,6 +421,7 @@ cargo-check-each-crate-macos: - .collect-artifacts - .pipeline-stopper-artifacts before_script: + # skip timestamp script, the osx bash doesn't support printf %()T - !reference [.job-switcher, before_script] - !reference [.rust-info-script, script] - !reference [.pipeline-stopper-vars, script] From 8299e22d36d80e21edae80358859d262aad4a936 Mon Sep 17 00:00:00 2001 From: Mira Ressel Date: Wed, 15 Mar 2023 15:05:28 +0100 Subject: [PATCH 4/4] move timestamp.yml to shared pipeline repo https://gitlab.parity.io/parity/infrastructure/ci_cd/shared --- .gitlab-ci.yml | 3 ++- scripts/ci/gitlab/timestamp.yml | 27 --------------------------- 2 files changed, 2 insertions(+), 28 deletions(-) delete mode 100644 scripts/ci/gitlab/timestamp.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49c291b587a1c..44f98db3bf185 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -312,7 +312,8 @@ include: - local: scripts/ci/gitlab/default-pipeline.yml rules: - if: $PIPELINE != "automatic-crate-publishing" - - scripts/ci/gitlab/timestamp.yml + - project: parity/infrastructure/ci_cd/shared + file: /common/timestamp.yml #### stage: notify diff --git a/scripts/ci/gitlab/timestamp.yml b/scripts/ci/gitlab/timestamp.yml deleted file mode 100644 index f21d26d660854..0000000000000 --- a/scripts/ci/gitlab/timestamp.yml +++ /dev/null @@ -1,27 +0,0 @@ -.timestamp: - before_script: - - | - echo "Install timestamp handler" - eval ' - # https://gist.github.com/altaua/477285db40181ef1793bbed5bae32c6b - # based on https://gist.github.com/jstine35/e0fc0e06ec06d74bc3ebd67585bf2a1d - - s_datestamp() { - TZ=UTC - eof="" - while [[ -z "${eof}" ]]; do - IFS="" read -r line || eof=1 - printf "[%(%F %T)T] %s\n" -1 "${line}" - done - } - - cleanup() { - exec >/dev/null 2>&1 - wait "${TIMESTAMP_PID}" - } - - exec > >(s_datestamp) 2>&1 - - TIMESTAMP_PID=$! - trap cleanup EXIT - '