move benches to subdirectory

Author: davidhewitt

.github/workflows/gh-pages.yml

@@ -60,7 +60,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
+      - uses: actions/setup-python@v4
       - uses: dtolnay/rust-toolchain@stable
 
       - uses: actions/cache@v3
@@ -74,8 +74,10 @@ jobs:
 
       - name: Run benchmarks
         run: |
-          for bench in call dict gil list pyclass pyobject set tuple; do
-            cargo bench --features hashbrown --bench "bench_$bench" -- --output-format bencher | tee -a output.txt
+          python -m pip install --upgrade pip && pip install nox
+          for bench in pyo3-benches/benches/*.rs; do
+            bench_name=$(basename "$bench" .rs)
+            nox -s bench -- --bench "$bench_name" -- --output-format bencher | tee -a output.txt
           done
 
       # Download previous benchmark result from cache (if exists)

Cargo.toml

@@ -44,7 +44,6 @@ serde = { version = "1.0", optional = true }
 [dev-dependencies]
 assert_approx_eq = "1.1.0"
 chrono = { version = "0.4" }
-criterion = "0.3.5"
 # Required for "and $N others" normalization
 trybuild = ">=1.0.70"
 rustversion = "1.0"
@@ -114,65 +113,6 @@ full = [
     "rust_decimal",
 ]
 
-[[bench]]
-name = "bench_any"
-harness = false
-
-[[bench]]
-name = "bench_call"
-harness = false
-
-[[bench]]
-name = "bench_err"
-harness = false
-
-[[bench]]
-name = "bench_decimal"
-harness = false
-required-features = ["rust_decimal"]
-
-[[bench]]
-name = "bench_dict"
-harness = false
-
-[[bench]]
-name = "bench_frompyobject"
-harness = false
-required-features = ["macros"]
-
-[[bench]]
-name = "bench_gil"
-harness = false
-
-[[bench]]
-name = "bench_list"
-harness = false
-
-[[bench]]
-name = "bench_pyclass"
-harness = false
-required-features = ["macros"]
-
-[[bench]]
-name = "bench_pyobject"
-harness = false
-
-[[bench]]
-name = "bench_set"
-harness = false
-
-[[bench]]
-name = "bench_tuple"
-harness = false
-
-[[bench]]
-name = "bench_intern"
-harness = false
-
-[[bench]]
-name = "bench_extract"
-harness = false
-
 [workspace]
 members = [
     "pyo3-ffi",

Contributing.md

@@ -165,9 +165,9 @@ CI tests both the most recent stable Rust version and the minimum supported Rust
 
 PyO3 has two sets of benchmarks for evaluating some aspects of its performance. The benchmark suite is currently very small - please open PRs with new benchmarks if you're interested in helping to expand it!
 
-First, there are Rust-based benchmarks located in the `benches` subdirectory. As long as you have a nightly rust compiler available on your system, you can run these benchmarks with:
+First, there are Rust-based benchmarks located in the `pyo3-benches` subdirectory. You can run these benchmarks with:
 
-    cargo +nightly bench
+    nox -s bench
 
 Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests).
 

noxfile.py

@@ -75,7 +75,7 @@ def fmt(session: nox.Session):
 @nox.session(name="fmt-rust", venv_backend="none")
 def fmt_rust(session: nox.Session):
     _run_cargo(session, "fmt", "--all", "--check")
-    _run_cargo(session, "fmt", *_FFI_CHECK, "--all", "--check")
+    _run_cargo(session, "fmt", _FFI_CHECK, "--all", "--check")
 
 
 @nox.session(name="fmt-py")
@@ -86,7 +86,7 @@ def fmt_py(session: nox.Session):
 
 @nox.session(name="clippy", venv_backend="none")
 def clippy(session: nox.Session) -> bool:
-    if not _clippy(session):
+    if not _clippy(session) and _clippy_additional_workspaces(session):
         session.error("one or more jobs failed")
 
 
@@ -110,6 +110,33 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool:
     return success
 
 
+def _clippy_additional_workspaces(session: nox.Session) -> bool:
+    # pyo3-benches and pyo3-ffi-check are in isolated workspaces so that their
+    # dependencies do not interact with MSRV
+
+    success = True
+    try:
+        _run_cargo(session, "clippy", _BENCHES)
+    except Exception:
+        success = False
+
+    # Run pyo3-ffi-check only on when not cross-compiling, because it needs to
+    # have Python headers to feed to bindgen which gets messy when cross-compiling.
+    target = os.environ.get("CARGO_BUILD_TARGET")
+    if target is None or _get_rust_default_target() == target:
+        try:
+            _build_docs_for_ffi_check(session)
+            _run_cargo(session, "clippy", _FFI_CHECK, "--workspace", "--all-targets")
+        except Exception:
+            success = False
+    return success
+
+
+@nox.session(venv_backend="none")
+def bench(session: nox.Session) -> bool:
+    _run_cargo(session, "bench", _BENCHES, *session.posargs)
+
+
 @nox.session(name="clippy-all", venv_backend="none")
 def clippy_all(session: nox.Session) -> None:
     success = True
@@ -119,6 +146,7 @@ def _clippy_with_config(env: Dict[str, str]) -> None:
         success &= _clippy(session, env=env)
 
     _for_all_version_configs(session, _clippy_with_config)
+    success &= _clippy_additional_workspaces(session)
 
     if not success:
         session.error("one or more jobs failed")
@@ -376,7 +404,7 @@ def address_sanitizer(session: nox.Session):
         "test",
         "--release",
         "-Zbuild-std",
-        f"--target={_get_rust_target()}",
+        f"--target={_get_rust_default_target()}",
         "--",
         "--test-threads=1",
         env={
@@ -533,9 +561,13 @@ def load_pkg_versions():
 
 @nox.session(name="ffi-check")
 def ffi_check(session: nox.Session):
-    _run_cargo(session, "doc", *_FFI_CHECK, "-p", "pyo3-ffi", "--no-deps")
-    _run_cargo(session, "clippy", "--workspace", "--all-targets", *_FFI_CHECK)
-    _run_cargo(session, "run", *_FFI_CHECK)
+    _build_docs_for_ffi_check(session)
+    _run_cargo(session, "run", _FFI_CHECK)
+
+
+def _build_docs_for_ffi_check(session: nox.Session) -> None:
+    # pyo3-ffi-check needs to scrape docs of pyo3-ffi
+    _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps")
 
 
 @lru_cache()
@@ -554,7 +586,7 @@ def _get_rust_version() -> Tuple[int, int, int, List[str]]:
             return (*map(int, version_number.split(".")), extra)
 
 
-def _get_rust_target() -> str:
+def _get_rust_default_target() -> str:
     for line in _get_rust_info():
         if line.startswith(_HOST_LINE_START):
             return line[len(_HOST_LINE_START) :].strip()
@@ -692,4 +724,5 @@ def _job_with_config(implementation, version) -> bool:
             _job_with_config("PyPy", version)
 
 
-_FFI_CHECK = ("--manifest-path", "pyo3-ffi-check/Cargo.toml")
+_BENCHES = "--manifest-path=pyo3-benches/Cargo.toml"
+_FFI_CHECK = "--manifest-path=pyo3-ffi-check/Cargo.toml"

pyo3-benches/Cargo.toml

@@ -0,0 +1,80 @@
+[package]
+name = "pyo3-benches"
+version = "0.1.0"
+description = "In-tree benchmarks for the PyO3 project"
+authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
+edition = "2021"
+publish = false
+
+[dependencies]
+pyo3 = { path = "../", features = ["auto-initialize"] }
+
+[dev-dependencies]
+criterion = "0.3.5"
+
+[[bench]]
+name = "bench_any"
+harness = false
+
+[[bench]]
+name = "bench_call"
+harness = false
+
+[[bench]]
+name = "bench_comparisons"
+harness = false
+
+[[bench]]
+name = "bench_err"
+harness = false
+
+[[bench]]
+name = "bench_decimal"
+harness = false
+required-features = ["pyo3/rust_decimal"]
+
+[[bench]]
+name = "bench_dict"
+harness = false
+required-features = ["pyo3/hashbrown"]
+
+[[bench]]
+name = "bench_frompyobject"
+harness = false
+required-features = ["pyo3/macros"]
+
+[[bench]]
+name = "bench_gil"
+harness = false
+
+[[bench]]
+name = "bench_list"
+harness = false
+
+[[bench]]
+name = "bench_pyclass"
+harness = false
+required-features = ["pyo3/macros"]
+
+[[bench]]
+name = "bench_pyobject"
+harness = false
+
+[[bench]]
+name = "bench_set"
+harness = false
+required-features = ["pyo3/hashbrown"]
+
+[[bench]]
+name = "bench_tuple"
+harness = false
+
+[[bench]]
+name = "bench_intern"
+harness = false
+
+[[bench]]
+name = "bench_extract"
+harness = false
+
+[workspace]

benches/bench_any.rs renamed to pyo3-benches/benches/bench_any.rs

@@ -0,0 +1,70 @@
+use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+
+use pyo3::{prelude::*, pyclass::CompareOp, Python};
+
+#[pyclass]
+struct OrderedDunderMethods(i64);
+
+#[pymethods]
+impl OrderedDunderMethods {
+    fn __lt__(&self, other: &Self) -> bool {
+        self.0 < other.0
+    }
+
+    fn __le__(&self, other: &Self) -> bool {
+        self.0 <= other.0
+    }
+
+    fn __eq__(&self, other: &Self) -> bool {
+        self.0 == other.0
+    }
+
+    fn __ne__(&self, other: &Self) -> bool {
+        self.0 != other.0
+    }
+
+    fn __gt__(&self, other: &Self) -> bool {
+        self.0 > other.0
+    }
+
+    fn __ge__(&self, other: &Self) -> bool {
+        self.0 >= other.0
+    }
+}
+
+#[pyclass]
+#[derive(PartialEq, Eq, PartialOrd, Ord)]
+struct OrderedRichcmp(i64);
+
+#[pymethods]
+impl OrderedRichcmp {
+    fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool {
+        op.matches(self.cmp(other))
+    }
+}
+
+fn bench_ordered_dunder_methods(b: &mut Bencher<'_>) {
+    Python::with_gil(|py| {
+        let obj1 = Py::new(py, OrderedDunderMethods(0)).unwrap().into_ref(py);
+        let obj2 = Py::new(py, OrderedDunderMethods(1)).unwrap().into_ref(py);
+
+        b.iter(|| obj2.gt(obj1).unwrap());
+    });
+}
+
+fn bench_ordered_richcmp(b: &mut Bencher<'_>) {
+    Python::with_gil(|py| {
+        let obj1 = Py::new(py, OrderedRichcmp(0)).unwrap().into_ref(py);
+        let obj2 = Py::new(py, OrderedRichcmp(1)).unwrap().into_ref(py);
+
+        b.iter(|| obj2.gt(obj1).unwrap());
+    });
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("ordered_dunder_methods", bench_ordered_dunder_methods);
+    c.bench_function("ordered_richcmp", bench_ordered_richcmp);
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);