feat(benchmark): split pytest-benchmark runners

Author: BobTheBuidler

.github/workflows/benchmark.yaml

@@ -26,6 +26,10 @@ jobs:
   benchmark:
     name: Run Benchmarks
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v5
       - name: Set up Python
@@ -66,25 +70,67 @@ jobs:
         run: |
           rm -r faster_web3
           rm -r faster_ens
-          
-      - name: Run Pytest Benchmark & Save Output
-        run: pytest --benchmark-only --benchmark-json=benchmark.json benchmarks/
+
+      - name: Collect and split test nodeids for sharding
+        id: split
+        env:
+          SHARD: ${{ matrix.shard }}
+          SHARD_COUNT: 4
+        run: |
+          pytest --collect-only -q benchmarks/ > nodeids.txt
+          total=$(wc -l < nodeids.txt)
+          per_shard=$(( (total + SHARD_COUNT - 1) / SHARD_COUNT ))
+          start=$(( (SHARD - 1) * per_shard + 1 ))
+          end=$(( SHARD * per_shard ))
+          sed -n "${start},${end}p" nodeids.txt > shard_nodeids.txt
+          if [ ! -s shard_nodeids.txt ]; then
+            echo "No tests for this shard" > shard_nodeids.txt
+          fi
+
+      - name: Run Pytest Benchmark & Save Output (sharded)
+        if: ${{ hashFiles('shard_nodeids.txt') != '' && hashFiles('shard_nodeids.txt') != 'No tests for this shard' }}
+        run: |
+          ids=$(paste -sd ' or ' shard_nodeids.txt)
+          if [ -s shard_nodeids.txt ] && [ "$ids" != "No tests for this shard" ]; then
+            pytest --benchmark-only --benchmark-json=benchmark.json benchmarks/ -k "$ids"
+          else
+            echo "No tests for this shard"
+          fi
         
       - name: Upload Pytest Benchmark Results
+        uses: actions/upload-artifact@v5
+        with:
+          name: pytest-benchmark-results-shard-${{ matrix.shard }}
+          path: benchmark.json
+
+  join-benchmark-results:
+    name: Join Benchmark Results
+    runs-on: ubuntu-latest
+    needs: benchmark
+    steps:
+      - uses: actions/checkout@v5
+      - name: Download all shard benchmark results
+        uses: actions/download-artifact@v6
+        with:
+          pattern: pytest-benchmark-results-shard-*
+          path: shards
+      - name: Merge benchmark.json files
+        run: python scripts/merge_pytest_benchmark_json.py
+      - name: Upload Merged Benchmark Results
         uses: actions/upload-artifact@v5
         with:
           name: pytest-benchmark-results
           path: benchmark.json
-          
+
       - name: Parse Pytest Benchmark Output
         run: python scripts/benchmark/parse_benchmark_output.py benchmark.json pytest_benchmark_results.json
-        
+
       - name: Compare Pytest Benchmark Results
         run: python scripts/benchmark/compare_benchmark_results.py pytest_benchmark_results.json pytest_benchmark_diff.json
-        
+
       - name: Generate Markdown Benchmark Results
         run: python scripts/benchmark/generate_benchmark_markdown.py
-        
+
       - name: Commit and Push Markdown Benchmark Results
         continue-on-error: true
         run: |
@@ -101,19 +147,19 @@ jobs:
           fi
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          
+
       - name: Upload Markdown Benchmark Results
         uses: actions/upload-artifact@v5
         with:
           name: markdown-benchmark-results
           path: benchmarks/results/*.md
-          
+
       - name: Upload Pytest Benchmark Diff
         uses: actions/upload-artifact@v5
         with:
           name: pytest-benchmark-diff
           path: pytest_benchmark_diff.json
-          
+
       - name: Post Pytest Benchmark Diff as PR Comment
         uses: actions/github-script@v8
         with:
@@ -160,9 +206,6 @@ jobs:
               body
             });
         if: github.event_name == 'pull_request'
-      
-      - name: Save ccache cache
-        uses: actions/cache/save@v4
-        with:
-          path: ~/.cache/ccache
-          key: ccache-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('**/*.h', '**/*.c', '**/*.py') }}
+
+      - name: Cleanup Shard Artifacts
+        run: rm -rf shards

faster_web3/scripts/merge_pytest_benchmark_json.py

@@ -0,0 +1,42 @@
+"""
+merge_pytest_benchmark_json.py
+
+Merges multiple pytest-benchmark JSON result files (from sharded CI runs) into a single benchmark.json file.
+- Scans for all benchmark.json files in the given search path (default: shards/**/benchmark.json).
+- Combines all 'benchmarks' entries into a single report, preserving the pytest-benchmark format.
+- Intended for use in CI workflows where benchmarks are run in parallel shards and results need to be aggregated.
+
+Usage:
+    python faster_web3/scripts/merge_pytest_benchmark_json.py
+
+Arguments:
+    --output_path: Path to write the merged benchmark JSON (default: benchmark.json)
+    --search_path: Glob pattern to find input benchmark.json files (default: shards/**/benchmark.json)
+"""
+
+import json
+import glob
+import os
+
+def merge_benchmark_json_files(output_path="benchmark.json", search_path="shards/**/benchmark.json"):
+    files = sorted(glob.glob(search_path, recursive=True))
+    benchmarks = []
+    for f in files:
+        with open(f) as fh:
+            data = json.load(fh)
+            if 'benchmarks' in data:
+                benchmarks.extend(data['benchmarks'])
+            else:
+                benchmarks.append(data)
+    if benchmarks and isinstance(benchmarks[0], dict) and 'benchmarks' in benchmarks[0]:
+        # If each file is a full pytest-benchmark report
+        merged = benchmarks[0]
+        for b in benchmarks[1:]:
+            merged['benchmarks'].extend(b['benchmarks'])
+    else:
+        merged = {'benchmarks': benchmarks}
+    with open(output_path, 'w') as out:
+        json.dump(merged, out)
+
+if __name__ == "__main__":
+    merge_benchmark_json_files()