add CI perf-bench job to run short filtered benchmarks on PRs

Author: KeelMatrix

.github/workflows/ci.yml

@@ -61,6 +61,49 @@ jobs:
         uses: github/codeql-action/autobuild@v3
       - name: Analyze with CodeQL
         uses: github/codeql-action/analyze@v3
+  perf-bench:
+    if: (github.event_name == 'pull_request' || github.ref == 'refs/heads/main') && github.actor != 'dependabot[bot]'
+    needs: build
+    runs-on: windows-latest
+    timeout-minutes: 9
+    strategy:
+      fail-fast: false
+      matrix:
+        shard:
+          # original pieces, but 1 per shard
+          - { name: jwt-core,      filter: "*Jwt*ciOi*" }
+          - { name: phone-long,    filter: "*Phone*0958*" }
+          - { name: email-simple,  filter: "*Email*(Sample:*[email protected]*)" }
+          - { name: email-long,    filter: "*Email*(Sample:*@domain.co.uk*)" }
+  
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-dotnet@v4
+        with: { dotnet-version: '8.0.x' }
+  
+      - name: Run perf benchmarks (${{ matrix.shard.name }})
+        shell: pwsh
+        run: >
+          pwsh -NoProfile -File bench/Run-Benchmarks.ps1
+          -Job Short
+          -Filter "${{ matrix.shard.filter }}"
+          -MaxStdevPct 12
+          -Ci
+          -CoolDownSec 0
+  
+      - name: Upload benchmark artifacts (${{ matrix.shard.name }})
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-benchmarks-${{ matrix.shard.name }}
+          path: artifacts/benchmarks
+          if-no-files-found: error
+  
+      - name: Publish benchmark summary to PR (${{ matrix.shard.name }})
+        shell: pwsh
+        run: |
+          $f = Get-ChildItem "artifacts/benchmarks/*/SUMMARY.md" -ErrorAction SilentlyContinue |
+               Sort-Object LastWriteTime -Descending | Select-Object -First 1
+          if ($f) { Get-Content $f.FullName | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 }
 
 # --- Optional companion gates (disabled by default) ---
 # - name: Unrecorded HTTP Call gate

bench/BENCHMARKS.md

@@ -29,6 +29,16 @@ This will:
   pwsh -NoProfile -File bench/Run-Benchmarks.ps1
   ```
 
+- **Run everything from the RedactionBench class:**
+
+  ```powershell
+  pwsh -NoProfile -File bench/Run-Benchmarks.ps1 `
+  -Filter "*RedactionBench.*" `
+  -Job Default `
+  -Framework net8.0 `
+  -CoolDownSec 0
+  ```
+
 - **Run only PhoneRedactor benchmarks:**
 
   ```powershell

bench/KeelMatrix.QueryWatch.Redaction.Benchmarks/CiAwareConfig.cs

@@ -0,0 +1,17 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Exporters.Json;
+
+namespace KeelMatrix.QueryWatch.Redaction.Benchmarks {
+    internal sealed class CiAwareConfig : ManualConfig {
+        public CiAwareConfig() {
+            AddLogger([.. DefaultConfig.Instance.GetLoggers()]);
+            AddColumnProvider([.. DefaultConfig.Instance.GetColumnProviders()]);
+            AddDiagnoser([.. DefaultConfig.Instance.GetDiagnosers()]);
+
+            bool isCi = Environment.GetEnvironmentVariable("CI") is not null;
+            if (!isCi) {
+                AddExporter(JsonExporter.FullCompressed);
+            }
+        }
+    }
+}

bench/KeelMatrix.QueryWatch.Redaction.Benchmarks/Program.cs

@@ -4,7 +4,7 @@
 namespace KeelMatrix.QueryWatch.Redaction.Benchmarks {
     public static class Program {
         public static void Main(string[] args) =>
-            BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+            BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new CiAwareConfig());
     }
 
     [MemoryDiagnoser]
@@ -31,10 +31,20 @@ public class RedactionBench {
         public string Jwt() => _jwt.Redact(Sample);
 
         public static IEnumerable<string> SampleCases() {
-            yield return "Contact me at [email protected] or +1-415-555-2671. Token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.";
-            yield return "noise-no-pii-" + new string('x', 32); // ~32B filler
-            yield return "email test [email protected] " + new string('y', 1024) + " +44 20 7946 0958"; // larger line with phone
-            yield return "jwt " + new string('z', 4096) + " eyJhbGciOi"; // big line w/ jwt-like token
+            // Email-only samples
+            yield return "simple [email protected]";
+            yield return "very.long.email.address_" + new string('x', 128) + "@domain.co.uk";
+
+            // Phone-only
+            yield return "+1-415-555-2671 is my number";
+            yield return "call me at +44 20 7946 0958";
+
+            // JWT-only
+            yield return "jwt eyJhbGciOi" + new string('z', 2048);
+
+            // Pure noise
+            yield return "noise-no-pii-" + new string('x', 1024);
         }
+
     }
 }

bench/Run-Benchmarks.ps1

@@ -54,6 +54,7 @@ param(
   [string]$ArtifactsRoot = 'artifacts/benchmarks',
   [double]$MaxStdevPct = 0,
   [switch]$Ci,
+  [int]$CoolDownSec = 480,
   [switch]$DryRun
 )
 
@@ -114,13 +115,19 @@ foreach($proj in $benchProjects){
 
   # BenchmarkDotNet common CLI args (supported by BenchmarkSwitcher):
   #   --list, --filter, --runtimes, --artifacts, --exporters, --join, --job
-  # Multiple exporters: pass them as separate values to a **single** --exporters option.
+  # Exporters: on CI we need JSON for machines + GitHub for PRs + CSV for gates.
+  # Locally, CiAwareConfig already adds JsonExporter.FullCompressed, so we skip 'json'
+  # to avoid the "already present" warning and ask for the human-friendly extras.
+  $exporters = $Ci ? @('json','github','csv') : @('github','markdown','html','csv')
   $bdnArgs = @(
     '--filter', "$Filter",
-    '--exporters', 'json', 'markdown',
     '--artifacts', "$projOut",
     '--join'
   )
+  
+  # append exporters as separate tokens to avoid nested arrays or a single invalid value
+  $bdnArgs = $bdnArgs + @('--exporters') + $exporters
+  
   if($Job -ne 'Default'){ $bdnArgs += @('--job', $Job) }
 
   # dotnet run to bench project
@@ -132,9 +139,10 @@ foreach($proj in $benchProjects){
   ) + $bdnArgs
 
   Write-Host ("dotnet " + ($cmd -join ' '))
-  $proc = Start-Process -FilePath 'dotnet' -ArgumentList $cmd -PassThru -Wait -NoNewWindow
-  if($proc.ExitCode -ne 0){
-    Write-Error "Benchmark run failed for $projName with exit code $($proc.ExitCode)."
+  & dotnet @cmd
+  $exitCode = $LASTEXITCODE
+  if ($exitCode -ne 0) {
+    Write-Error "dotnet run failed ($exitCode) for $projName"
     $failed++
     Pop-Location
     continue
@@ -230,6 +238,15 @@ if($Ci){
   Write-Host "CI Summary written to $summaryPath"
 }
 
+# ---- Cooldown: CI-aware (skip in CI or when set to 0) ----
+# Note: "Cooling down" is just an intentional pause after a benchmark run on our local machine to let the CPU/fans/OS settle so the next local run isn’t biased by turbo/thermal effects and hot caches. It helps reduce noise between back-to-back local runs and gives us a few minutes to glance at artifacts. In CI we skip it (-Ci -CoolDownSec 0) because there’s only one short run and wall-clock is precious.
+if ($CoolDownSec -gt 0 -and -not $Ci) {
+  Write-Host "Cooling down for $CoolDownSec seconds (local run). The job is complete. You can glance at artifacts while you wait..."
+  Start-Sleep -Seconds $CoolDownSec
+} else {
+  Write-Host "Skipping cooldown (CI or CoolDownSec=0)."
+}
+
 # Exit code logic
 if($failed -gt 0){
   Write-Error "$failed project(s) failed to run."