diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index b3145fbdf1..b0f98c04ac 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -32,8 +32,7 @@ # Common ASV arguments for all run_types except `custom`. ASV_HARNESS = ( - "run {posargs} --attribute rounds=4 --interleave-rounds --strict " - "--show-stderr" + "run {posargs} --attribute rounds=4 --interleave-rounds --show-stderr" ) @@ -501,14 +500,19 @@ def csperf( asv_command = ( ASV_HARNESS.format(posargs=commit_range) + f" --bench={run_type}" ) - # C/SPerf benchmarks are much bigger than the CI ones: - # Don't fail the whole run if memory blows on 1 benchmark. - asv_command = asv_command.replace(" --strict", "") + # Only do a single round. asv_command = shlex.split( re.sub(r"rounds=\d", "rounds=1", asv_command) ) - _subprocess_runner([*asv_command, *args.asv_args], asv=True) + try: + _subprocess_runner([*asv_command, *args.asv_args], asv=True) + except subprocess.CalledProcessError as err: + # C/SPerf benchmarks are much bigger than the CI ones: + # Don't fail the whole run if memory blows on 1 benchmark. + # ASV produces return code of 2 if the run includes crashes. + if err.returncode != 2: + raise asv_command = shlex.split( f"publish {commit_range} --html-dir={publish_subdir}" diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 2de3dc1ced..66a48f7b46 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -79,6 +79,9 @@ This document explains the changes made to Iris for this release working properly. (Main pull request: :pull:`5437`, more detail: :pull:`5430`, :pull:`5431`, :pull:`5432`, :pull:`5434`, :pull:`5436`) +#. `@trexfeathers`_ adapted benchmarking to work with ASV ``>=v0.6`` by no + longer using the ``--strict`` argument. (:pull:`5496`) + .. comment Whatsnew author names (@github name) in alphabetical order. Note that,