feat(criterion_compat): fork criterion and add walltime support

.github/workflows/ci.yml

@@ -85,6 +85,7 @@ jobs:
 
       - run: cargo codspeed build -p codspeed-divan-compat
       - run: cargo codspeed build -p codspeed-divan-compat-examples
+      - run: cargo codspeed build -p codspeed-criterion-compat
 
       - name: Run the benchmarks
         uses: CodSpeedHQ/action@main

.gitmodules

@@ -1,4 +1,4 @@
 
 [submodule "crates/divan_compat/examples/src/the_algorithms/Rust"]
 	path = crates/divan_compat/examples/src/the_algorithms/Rust
-	url = git@github.com:TheAlgorithms/Rust.git
+	url = https://github.com/TheAlgorithms/Rust.git

Cargo.toml

@@ -2,9 +2,11 @@
 members = [
   "crates/codspeed",
   "crates/bencher_compat",
-  "crates/criterion_compat",
   "crates/cargo-codspeed",
 
+  "crates/criterion_compat",
+  "crates/criterion_compat/criterion_fork",
+
   "crates/divan_compat",
   "crates/divan_compat/macros",
   "crates/divan_compat/divan_fork",
@@ -16,3 +18,4 @@ resolver = "2"
 itertools = "0.14.0"
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.138"
+tempfile = "3.7.0"

crates/cargo-codspeed/src/run.rs

@@ -209,6 +209,15 @@ fn aggregate_raw_walltime_data(workspace_root: &Path) -> Result<()> {
         return Ok(());
     }
 
+    for bench in &walltime_benchmarks {
+        if bench.is_invalid() {
+            eprintln!(
+                "Warning: Benchmark {} was possibly optimized away",
+                bench.name()
+            );
+        }
+    }
+
     let results_folder = std::env::var("CODSPEED_PROFILE_FOLDER")
         .map(PathBuf::from)
         .unwrap_or_else(|_| workspace_root.join("target/codspeed/profiles"))

crates/cargo-codspeed/src/walltime_results.rs

@@ -41,6 +41,16 @@ pub struct WalltimeBenchmark {
     stats: BenchmarkStats,
 }
 
+impl WalltimeBenchmark {
+    pub(crate) fn is_invalid(&self) -> bool {
+        self.stats.min_ns < f64::EPSILON
+    }
+
+    pub(crate) fn name(&self) -> &str {
+        &self.metadata.name
+    }
+}
+
 impl From<RawWallTimeData> for WalltimeBenchmark {
     fn from(value: RawWallTimeData) -> Self {
         let times_ns: Vec<f64> = value.times_ns.iter().map(|&t| t as f64).collect();

crates/codspeed/Cargo.toml

@@ -29,4 +29,4 @@ name = "native"
 harness = false
 
 [dev-dependencies]
-tempfile = "3.7.0"
+tempfile = { workspace = true }

crates/criterion_compat/Cargo.toml

@@ -17,7 +17,7 @@ categories = [
 ]
 keywords = ["codspeed", "benchmark", "criterion"]
 [dependencies]
-criterion = { version = "0.5.1", default-features = false }
+criterion = { package = "codspeed-criterion-compat-walltime", path = "./criterion_fork", version = "0.5.1", default-features = false }
 codspeed = { path = "../codspeed", version = "=2.8.1" }
 colored = "2.1.0"
 

crates/criterion_compat/benches/criterion_integration/mod.rs

@@ -1,6 +1,6 @@
 pub mod compare_functions;
 pub mod custom_measurement;
-pub mod external_process;
+// pub mod external_process;
 pub mod iter_with_large_drop;
 pub mod iter_with_large_setup;
 pub mod iter_with_setup;

crates/criterion_compat/benches/criterion_integration_main.rs

@@ -4,7 +4,7 @@ mod criterion_integration;
 
 criterion_main! {
     criterion_integration::compare_functions::fibonaccis,
-    criterion_integration::external_process::benches,
+    // criterion_integration::external_process::benches, FIXME: Currently doesn't work
     criterion_integration::iter_with_large_drop::benches,
     criterion_integration::iter_with_large_setup::benches,
     criterion_integration::iter_with_setup::benches,

crates/criterion_compat/criterion_fork/.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+indent_style = space

crates/criterion_compat/criterion_fork/.github/workflows/audit.yml

@@ -0,0 +1,14 @@
+name: Security audit
+on:
+  push:
+    paths:
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+jobs:
+  security_audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

crates/criterion_compat/criterion_fork/.github/workflows/ci.yaml

@@ -0,0 +1,113 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+      - version-0.4
+
+name: tests
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+          - 1.64.0 # MSRV
+          - nightly
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions-rs/toolchain@v1
+        name: Setup rust toolchain
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+          components: rustfmt, clippy
+
+      - uses: Swatinem/rust-cache@v2
+        name: Load dependencies from cache
+
+      - uses: actions-rs/cargo@v1
+        name: Build with stable features
+        with:
+          command: build
+          args: --features stable
+
+      - uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust == 'nightly' }}
+        name: Build with unstable features
+        with:
+          command: build
+          args: --all-features
+
+      - uses: actions-rs/cargo@v1
+        name: Build with minimal features
+        with:
+          command: build
+          args: --no-default-features
+
+      - uses: actions-rs/cargo@v1
+        name: Test with stable features
+        with:
+          command: test
+          args: --features stable
+
+      - uses: actions-rs/cargo@v1
+        name: Test with minimal features
+        with:
+          command: test
+          args: --no-default-features
+
+      - uses: actions-rs/cargo@v1
+        name: Check for non-standard formatting
+        if: ${{ matrix.rust == 'stable' }}
+        with:
+          command: fmt
+          args: --all -- --check
+
+      - uses: actions-rs/cargo@v1
+        name: Check for clippy hints
+        if: ${{ matrix.rust == 'stable' }}
+        with:
+          command: clippy
+          args: -- -D warnings
+
+      - name: Test run targeting WASI
+        run: |
+          curl https://wasmtime.dev/install.sh -sSf | bash
+          source ~/.bashrc
+          export PATH=$HOME/.wasmtime/bin/:$PATH
+          cargo install cargo-wasi
+          cargo wasi bench --no-default-features -- --test
+
+  nextest-compat:
+    name: Check compatibility with nextest
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: ${{ matrix.rust }}
+          override: true
+          components: rustfmt, clippy
+
+      - uses: Swatinem/rust-cache@v1
+
+      - uses: taiki-e/install-action@nextest
+
+      - run: ci/nextest-compat.sh