feat: benchmark suite

.github/actions/benchmark-setup/action.yml

@@ -0,0 +1,34 @@
+name: "Benchmark Setup"
+description: "Common setup steps for benchmark jobs"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Foundry
+      shell: bash
+      env:
+        FOUNDRY_DIR: ${{ github.workspace }}/.foundry
+      run: |
+        ./.github/scripts/setup-foundryup.sh
+        echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: "24"
+
+    - name: Install hyperfine
+      shell: bash
+      run: |
+        curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz
+        sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/
+        rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu
+
+    - name: Download benchmark binary
+      uses: actions/download-artifact@v4
+      with:
+        name: foundry-bench
+
+    - name: Make binary executable
+      shell: bash
+      run: chmod +x foundry-bench

.github/scripts/combine-benchmarks.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+set -euo pipefail
+
+# Script to combine individual benchmark results into LATEST.md
+# Usage: ./combine-benchmarks.sh <output_dir>
+
+OUTPUT_DIR="${1:-benches}"
+
+# Create output directory if it doesn't exist
+mkdir -p "$OUTPUT_DIR"
+
+# Define the benchmark files and their section names
+declare -A BENCHMARK_FILES=(
+    ["forge_test_bench.md"]="Forge Test"
+    ["forge_build_bench.md"]="Forge Build" 
+    ["forge_coverage_bench.md"]="Forge Coverage"
+)
+
+# Function to extract a specific section from a benchmark file
+extract_section() {
+    local file=$1
+    local section=$2
+    local in_section=0
+
+    while IFS= read -r line; do
+        if [[ "$line" =~ ^##[[:space:]]+"$section" ]]; then
+            in_section=1
+            echo "$line"
+        elif [[ $in_section -eq 1 && "$line" =~ ^##[[:space:]] && ! "$line" =~ ^##[[:space:]]+"$section" ]]; then
+            break
+        elif [[ $in_section -eq 1 ]]; then
+            echo "$line"
+        fi
+    done < "$file"
+}
+
+# Function to extract summary info (repos and versions) from a file
+extract_summary_info() {
+    local file=$1
+    local in_summary=0
+    local in_repos=0
+    local in_versions=0
+
+    while IFS= read -r line; do
+        # Check for Summary section
+        if [[ "$line" =~ ^##[[:space:]]+Summary ]]; then
+            in_summary=1
+            continue
+        fi
+
+        # Check for Repositories Tested subsection
+        if [[ $in_summary -eq 1 && "$line" =~ ^###[[:space:]]+Repositories[[:space:]]+Tested ]]; then
+            in_repos=1
+            echo "### Repositories Tested"
+            echo
+            continue
+        fi
+
+        # Check for Foundry Versions subsection
+        if [[ $in_summary -eq 1 && "$line" =~ ^###[[:space:]]+Foundry[[:space:]]+Versions ]]; then
+            in_repos=0
+            in_versions=1
+            echo "### Foundry Versions"
+            echo
+            continue
+        fi
+
+        # End of summary section
+        if [[ $in_summary -eq 1 && "$line" =~ ^##[[:space:]] && ! "$line" =~ ^##[[:space:]]+Summary ]]; then
+            break
+        fi
+
+        # Output repo or version lines
+        if [[ ($in_repos -eq 1 || $in_versions -eq 1) && -n "$line" ]]; then
+            echo "$line"
+        fi
+    done < "$file"
+}
+
+# Function to extract benchmark table from a section
+extract_benchmark_table() {
+    local file=$1
+    local section=$2
+    local in_section=0
+    local found_table=0
+
+    while IFS= read -r line; do
+        if [[ "$line" =~ ^##[[:space:]]+"$section" ]]; then
+            in_section=1
+            continue
+        elif [[ $in_section -eq 1 && "$line" =~ ^##[[:space:]] && ! "$line" =~ ^##[[:space:]]+"$section" ]]; then
+            break
+        elif [[ $in_section -eq 1 ]]; then
+            # Skip empty lines before table
+            if [[ -z "$line" && $found_table -eq 0 ]]; then
+                continue
+            fi
+            # Detect table start
+            if [[ "$line" =~ ^\|[[:space:]]*Repository ]]; then
+                found_table=1
+            fi
+            # Output table lines
+            if [[ $found_table -eq 1 && -n "$line" ]]; then
+                echo "$line"
+            fi
+        fi
+    done < "$file"
+}
+
+# Function to extract system information
+extract_system_info() {
+    local file=$1
+    # Extract from System Information to end of file (EOF)
+    awk '/^## System Information/ { found=1; next } found { print }' "$file"
+}
+
+# Start building LATEST.md
+cat > "$OUTPUT_DIR/LATEST.md" << EOF
+# 📊 Foundry Benchmark Results
+
+**Generated at**: $(date -u '+%Y-%m-%d %H:%M:%S UTC')
+
+EOF
+
+# Process each benchmark file
+FIRST_FILE=1
+SYSTEM_INFO=""
+
+for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do
+    if [ -f "$OUTPUT_DIR/$bench_file" ]; then
+        echo "Processing $bench_file..."
+
+        # Get the section name
+        case "$bench_file" in
+            "forge_test_bench.md")
+                SECTION_NAME="Forge Test"
+                ;;
+            "forge_build_bench.md")
+                SECTION_NAME="Forge Build"
+                ;;
+            "forge_coverage_bench.md")
+                SECTION_NAME="Forge Coverage"
+                ;;
+        esac
+
+        # Add section header
+        echo "## $SECTION_NAME" >> "$OUTPUT_DIR/LATEST.md"
+        echo >> "$OUTPUT_DIR/LATEST.md"
+
+        # Add summary info (repos and versions)
+        extract_summary_info "$OUTPUT_DIR/$bench_file" >> "$OUTPUT_DIR/LATEST.md"
+        echo >> "$OUTPUT_DIR/LATEST.md"
+
+        # Handle different benchmark types
+        if [[ "$bench_file" == "forge_test_bench.md" ]]; then
+            # Extract both Forge Test and Forge Fuzz Test tables
+            extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Test" >> "$OUTPUT_DIR/LATEST.md"
+
+            # Check if Forge Fuzz Test section exists
+            if grep -q "^## Forge Fuzz Test" "$OUTPUT_DIR/$bench_file"; then
+                echo >> "$OUTPUT_DIR/LATEST.md"
+                echo "## Forge Fuzz Test" >> "$OUTPUT_DIR/LATEST.md"
+                echo >> "$OUTPUT_DIR/LATEST.md"
+                extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Fuzz Test" >> "$OUTPUT_DIR/LATEST.md"
+            fi
+        elif [[ "$bench_file" == "forge_build_bench.md" ]]; then
+            # Extract No Cache table
+            echo "### No Cache" >> "$OUTPUT_DIR/LATEST.md"
+            echo >> "$OUTPUT_DIR/LATEST.md"
+            extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Build (No Cache)" >> "$OUTPUT_DIR/LATEST.md"
+            echo >> "$OUTPUT_DIR/LATEST.md"
+
+            # Extract With Cache table
+            echo "### With Cache" >> "$OUTPUT_DIR/LATEST.md"
+            echo >> "$OUTPUT_DIR/LATEST.md"
+            extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Build (With Cache)" >> "$OUTPUT_DIR/LATEST.md"
+        else
+            # Extract the benchmark table for other types
+            extract_benchmark_table "$OUTPUT_DIR/$bench_file" "$SECTION_NAME" >> "$OUTPUT_DIR/LATEST.md"
+        fi
+
+        echo >> "$OUTPUT_DIR/LATEST.md"
+
+        # Extract system info from first file only
+        if [[ $FIRST_FILE -eq 1 ]]; then
+            SYSTEM_INFO=$(extract_system_info "$OUTPUT_DIR/$bench_file")
+            FIRST_FILE=0
+        fi
+    else
+        echo "Warning: $bench_file not found, skipping..."
+    fi
+done
+
+# Add system information at the end
+if [[ -n "$SYSTEM_INFO" ]]; then
+    echo "## System Information" >> "$OUTPUT_DIR/LATEST.md"
+    echo >> "$OUTPUT_DIR/LATEST.md"
+    echo "$SYSTEM_INFO" >> "$OUTPUT_DIR/LATEST.md"
+fi
+
+echo "Successfully combined benchmark results into $OUTPUT_DIR/LATEST.md"

.github/scripts/commit-and-read-benchmarks.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+set -euo pipefail
+
+# Script to commit benchmark results and read them for GitHub Actions output
+# Usage: ./commit-and-read-benchmarks.sh <output_dir> <github_event_name> <github_repository>
+
+OUTPUT_DIR="${1:-benches}"
+GITHUB_EVENT_NAME="${2:-pull_request}"
+GITHUB_REPOSITORY="${3:-}"
+
+# Global variable for branch name
+BRANCH_NAME=""
+
+# Function to commit benchmark results
+commit_results() {
+    echo "Configuring git..."
+    git config --local user.email "[email protected]"
+    git config --local user.name "GitHub Action"
+
+    # For PR runs, fetch and checkout the PR branch to ensure we're up to date
+    if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ -n "${GITHUB_HEAD_REF:-}" ]; then
+        echo "Fetching latest changes for PR branch: $GITHUB_HEAD_REF"
+        git fetch origin "$GITHUB_HEAD_REF"
+        git checkout -B "$GITHUB_HEAD_REF" "origin/$GITHUB_HEAD_REF"
+    fi
+
+    echo "Adding benchmark file..."
+    git add "$OUTPUT_DIR/LATEST.md"
+
+    if git diff --staged --quiet; then
+        echo "No changes to commit"
+    else
+        echo "Committing benchmark results..."
+        git commit -m "chore(\`benches\`): update benchmark results
+
+🤖 Generated with [Foundry Benchmarks](https://github.com/${GITHUB_REPOSITORY}/actions)
+
+Co-Authored-By: github-actions <[email protected]>"
+
+        echo "Pushing to repository..."
+        if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then
+            # For manual runs, we're on a new branch
+            git push origin "$BRANCH_NAME"
+        elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then
+            # For PR runs, push to the PR branch
+            if [ -n "${GITHUB_HEAD_REF:-}" ]; then
+                echo "Pushing to PR branch: $GITHUB_HEAD_REF"
+                git push origin "$GITHUB_HEAD_REF"
+            else
+                echo "Error: GITHUB_HEAD_REF not set for pull_request event"
+                exit 1
+            fi
+        else
+            # This workflow should only run on workflow_dispatch or pull_request
+            echo "Error: Unexpected event type: $GITHUB_EVENT_NAME"
+            echo "This workflow only supports 'workflow_dispatch' and 'pull_request' events"
+            exit 1
+        fi
+        echo "Successfully pushed benchmark results"
+    fi
+}
+
+# Function to read benchmark results and output for GitHub Actions
+read_results() {
+    if [ -f "$OUTPUT_DIR/LATEST.md" ]; then
+        echo "Reading benchmark results..."
+
+        # Output full results
+        {
+            echo 'results<<EOF'
+            cat "$OUTPUT_DIR/LATEST.md"
+            echo 'EOF'
+        } >> "$GITHUB_OUTPUT"
+
+        # Format results for PR comment
+        echo "Formatting results for PR comment..."
+        FORMATTED_COMMENT=$("$(dirname "$0")/format-pr-comment.sh" "$OUTPUT_DIR/LATEST.md")
+
+        {
+            echo 'pr_comment<<EOF'
+            echo "$FORMATTED_COMMENT"
+            echo 'EOF'
+        } >> "$GITHUB_OUTPUT"
+
+        echo "Successfully read and formatted benchmark results"
+    else
+        echo 'results=No benchmark results found.' >> "$GITHUB_OUTPUT"
+        echo 'pr_comment=No benchmark results found.' >> "$GITHUB_OUTPUT"
+        echo "Warning: No benchmark results found at $OUTPUT_DIR/LATEST.md"
+    fi
+}
+
+# Main execution
+echo "Starting benchmark results processing..."
+
+# Create new branch for manual runs
+if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then
+    echo "Manual workflow run detected, creating new branch..."
+    BRANCH_NAME="benchmarks/results-$(date +%Y%m%d-%H%M%S)"
+    git checkout -b "$BRANCH_NAME"
+    echo "Created branch: $BRANCH_NAME"
+
+    # Output branch name for later use
+    echo "branch_name=$BRANCH_NAME" >> "$GITHUB_OUTPUT"
+fi
+
+# Always commit benchmark results
+echo "Committing benchmark results..."
+commit_results
+
+# Always read results for output
+read_results
+
+echo "Benchmark results processing complete"

.github/scripts/format-pr-comment.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -euo pipefail
+
+# Script to format benchmark results for PR comment
+# Usage: ./format-pr-comment.sh <benchmark_results_file>
+
+RESULTS_FILE="${1:-}"
+
+if [ -z "$RESULTS_FILE" ] || [ ! -f "$RESULTS_FILE" ]; then
+    echo "Error: Benchmark results file not provided or does not exist"
+    exit 1
+fi
+
+# Read the file content
+CONTENT=$(cat "$RESULTS_FILE")
+
+# Find where "## Forge Build" starts and split the content
+# Extract everything before "## Forge Build"
+BEFORE_FORGE_BUILD=$(echo "$CONTENT" | awk '/^## Forge Build$/ {exit} {print}')
+
+# Extract everything from "## Forge Build" onwards
+FROM_FORGE_BUILD=$(echo "$CONTENT" | awk '/^## Forge Build$/ {found=1} found {print}')
+
+# Output the formatted comment with dropdown
+cat << EOF
+${BEFORE_FORGE_BUILD}
+
+<details>
+<summary>📈 View all benchmark results</summary>
+
+${FROM_FORGE_BUILD}
+
+</details>
+EOF