diff --git a/.github/labeler.yml b/.github/labeler.yml index adee762bfc..14e9a95d30 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,6 +1,9 @@ -# Make sure any dependency changes are benchmarked (only changes to the locks -# make a material difference - changes to the Conda YAML files are not -# benchmarked). +# benchmark_this automatically triggers the benchmark workflow when added by +# a user. No triggering happens when GitHub Actions adds the label (this +# avoids security vulnerabilities), so alternative triggers for the below +# files are therefore included in workflows/benchmarks_run.yml. Automatic +# labelling is still included here to make it easier to search pull requests, +# and to reinforce the culture of using this label. benchmark_this: - changed-files: - - any-glob-to-any-file: 'requirements/locks/*.lock' + - any-glob-to-any-file: ['requirements/locks/*.lock', "setup.py"] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 597c413457..34bc59182c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,3 +7,8 @@ --- [Consult Iris pull request check list]( https://scitools-iris.readthedocs.io/en/latest/developers_guide/contributing_pull_request_checklist.html) + +--- +Add any of the below labels to trigger actions on this PR: + +- https://github.com/SciTools/iris/labels/benchmark_this diff --git a/.github/workflows/benchmarks_run.yml b/.github/workflows/benchmarks_run.yml index 69d753e4cc..626db393db 100644 --- a/.github/workflows/benchmarks_run.yml +++ b/.github/workflows/benchmarks_run.yml @@ -16,15 +16,46 @@ on: required: false type: string pull_request: - types: [labeled] + # Add the `labeled` type to the default list. + types: [labeled, opened, synchronize, reopened] jobs: + pre-checks: + runs-on: ubuntu-latest + if: github.repository == 'SciTools/iris' + outputs: + overnight: ${{ steps.overnight.outputs.check }} + branch: ${{ steps.branch.outputs.check }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - id: files-changed + uses: marceloprado/has-changed-path@df1b7a3161b8fb9fd8c90403c66a9e66dfde50cb + with: + # SEE ALSO .github/labeler.yml . + paths: requirements/locks/*.lock setup.py + - id: overnight + if: github.event_name != 'pull_request' + run: echo "check=true" >> "$GITHUB_OUTPUT" + - id: branch + if: > + github.event_name == 'pull_request' + && + ( + steps.files-changed.outputs.changed == 'true' + || + github.event.label.name == 'benchmark_this' + ) + run: echo "check=true" >> "$GITHUB_OUTPUT" + + benchmark: - if: > - github.repository == 'SciTools/iris' && - (github.event_name != 'pull_request' || - github.event.label.name == 'benchmark_this') runs-on: ubuntu-latest + needs: pre-checks + if: > + needs.pre-checks.outputs.overnight == 'true' || + needs.pre-checks.outputs.branch == 'true' env: IRIS_TEST_DATA_LOC_PATH: benchmarks @@ -42,7 +73,7 @@ jobs: - name: Install ASV & Nox run: | - pip install "asv!=0.6.2" nox + pip install asv nox - name: Cache environment directories id: cache-env-dir @@ -76,7 +107,7 @@ jobs: echo "OVERRIDE_TEST_DATA_REPOSITORY=${GITHUB_WORKSPACE}/${IRIS_TEST_DATA_PATH}/test_data" >> $GITHUB_ENV - name: Benchmark this pull request - if: ${{ github.event.label.name == 'benchmark_this' }} + if: needs.pre-checks.outputs.branch == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} @@ -85,7 +116,7 @@ jobs: - name: Run overnight benchmarks id: overnight - if: ${{ github.event_name != 'pull_request' }} + if: needs.pre-checks.outputs.overnight == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 4184629448..51983ec04d 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -6,6 +6,10 @@ "environment_type": "conda-delegated", "show_commit_url": "https://github.com/scitools/iris/commit/", "branches": ["upstream/main"], + "build_command": [ + "python setup.py build", + "python -mpip wheel --no-deps -w {build_cache_dir} {build_dir}" + ], "benchmark_dir": "./benchmarks", "env_dir": ".asv/env", diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index 10dc5f469a..69f58c4f5f 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -171,7 +171,7 @@ def _gh_create_reports(commit_sha: str, results_full: str, results_shifts: str) performance_report = dedent( ( """ - ### Performance Benchmark Report: {commit_sha} + # :stopwatch: Performance Benchmark Report: {commit_sha}
Performance shifts diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index ab5d18d3eb..8cc5bcdeaa 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -72,11 +72,10 @@ This document explains the changes made to Iris for this release 💼 Internal =========== -#. `@trexfeathers`_ used the `Pull Request Labeler Github action`_ to add the - ``benchmark_this`` label (:ref:`more info `) to - pull requests that modify ``requirements/locks/*.lock`` files - ensuring - that we know whether dependency changes will affect performance. - (:pull:`5763`) +#. `@trexfeathers`_ setup automatic benchmarking on pull requests that modify + files likely to affect performance or performance testing. Such pull + requests are also labelled using the `Pull Request Labeler Github action`_ + to increase visibility. (:pull:`5763`, :pull:`5776`) .. comment