From 4818f3dde68ff3bd452fcd03d1b505a33f4d2c69 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 10 Jun 2025 16:55:36 +0200 Subject: [PATCH 01/74] Add Foundry multi-version benchmarking suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Automated benchmarking across multiple Foundry versions using hyperfine - Supports stable, nightly, and specific version tags (e.g., v1.0.0) - Benchmarks 5 major Foundry projects: account, v4-core, solady, morpho-blue, spark-psm - Tests forge test, forge build (no cache), and forge build (with cache) - Generates comparison tables in markdown format - Uses foundryup for version management - Exports JSON data for detailed analysis 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- benchmark.sh | 506 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 506 insertions(+) create mode 100755 benchmark.sh diff --git a/benchmark.sh b/benchmark.sh new file mode 100755 index 0000000000000..542cd5ae66c3b --- /dev/null +++ b/benchmark.sh @@ -0,0 +1,506 @@ +#!/bin/bash + +# Foundry Multi-Version Benchmarking Suite using hyperfine +# This script benchmarks forge test and forge build commands across multiple repositories +# and multiple Foundry versions for comprehensive performance comparison + +set -e + +# Configuration +BENCHMARK_DIR="./benchmark_repos" +RESULTS_DIR="./benchmark_results" +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +RESULTS_FILE="${RESULTS_DIR}/foundry_multi_version_benchmark_${TIMESTAMP}.md" +JSON_RESULTS_DIR="${RESULTS_DIR}/json_${TIMESTAMP}" + +# Foundry versions to benchmark (can be modified via command line) +DEFAULT_FOUNDRY_VERSIONS=("stable" "nightly" "v1.0.0") +FOUNDRY_VERSIONS=("${FOUNDRY_VERSIONS[@]:-${DEFAULT_FOUNDRY_VERSIONS[@]}}") + +# Repository configurations +REPO_NAMES=( + "account" + "v4-core" + "solady" + "morpho-blue" + "spark-psm" +) + +REPO_URLS=( + "https://github.com/ithacaxyz/account" + "https://github.com/Uniswap/v4-core" + "https://github.com/Vectorized/solady" + "https://github.com/morpho-org/morpho-blue" + "https://github.com/sparkdotfi/spark-psm" +) + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Helper functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Install foundryup if not present +install_foundryup() { + if ! command -v foundryup &> /dev/null; then + log_info "Installing foundryup..." + curl -L https://foundry.paradigm.xyz | bash + # Source the bashrc/profile to get foundryup in PATH + export PATH="$HOME/.foundry/bin:$PATH" + fi +} + +# Install a specific Foundry version +install_foundry_version() { + local version=$1 + log_info "Installing Foundry version: $version" + + case "$version" in + "stable"|"nightly") + foundryup --install "$version" || { + log_error "Failed to install Foundry $version" + return 1 + } + ;; + v*) + foundryup --install "$version" || { + log_error "Failed to install Foundry $version" + return 1 + } + ;; + *) + log_error "Unsupported version format: $version" + return 1 + ;; + esac + + # Verify installation + local installed_version=$(forge --version | head -n1 || echo "unknown") + log_success "Installed Foundry: $installed_version" +} + +# Check if required tools are installed +check_dependencies() { + local missing_deps=() + + if ! command -v hyperfine &> /dev/null; then + missing_deps+=("hyperfine") + fi + + if ! command -v git &> /dev/null; then + missing_deps+=("git") + fi + + if ! command -v curl &> /dev/null; then + missing_deps+=("curl") + fi + + if [ ${#missing_deps[@]} -ne 0 ]; then + log_error "Missing required dependencies: ${missing_deps[*]}" + log_info "Install hyperfine: https://github.com/sharkdp/hyperfine#installation" + exit 1 + fi + + # Install foundryup if needed + install_foundryup +} + +# Setup directories +setup_directories() { + log_info "Setting up benchmark directories..." + mkdir -p "$BENCHMARK_DIR" + mkdir -p "$RESULTS_DIR" + mkdir -p "$JSON_RESULTS_DIR" +} + +# Clone or update repository +clone_or_update_repo() { + local name=$1 + local url=$2 + local repo_dir="${BENCHMARK_DIR}/${name}" + + if [ -d "$repo_dir" ]; then + log_info "Updating existing repository: $name" + cd "$repo_dir" + git pull origin main 2>/dev/null || git pull origin master 2>/dev/null || true + cd - > /dev/null + else + log_info "Cloning repository: $name" + git clone "$url" "$repo_dir" + fi +} + +# Install dependencies for a repository +install_dependencies() { + local repo_dir=$1 + local repo_name=$2 + + log_info "Installing dependencies for $repo_name..." + cd "$repo_dir" + + # Install forge dependencies + if [ -f "foundry.toml" ] || [ -f "forge.toml" ]; then + forge install 2>/dev/null || true + fi + + # Install npm dependencies if package.json exists + if [ -f "package.json" ]; then + if command -v npm &> /dev/null; then + npm install 2>/dev/null || true + fi + fi + + cd - > /dev/null +} + +# Run benchmarks for a single repository across all Foundry versions +benchmark_repository() { + local repo_name=$1 + local repo_dir="${BENCHMARK_DIR}/${repo_name}" + + log_info "Benchmarking repository: $repo_name across ${#FOUNDRY_VERSIONS[@]} Foundry versions" + + if [ ! -d "$repo_dir" ]; then + log_error "Repository directory not found: $repo_dir" + return 1 + fi + + cd "$repo_dir" + + # Check if it's a valid Foundry project + if [ ! -f "foundry.toml" ] && [ ! -f "forge.toml" ]; then + log_warn "No foundry.toml or forge.toml found in $repo_name, skipping..." + cd - > /dev/null + return 0 + fi + + # Benchmark each Foundry version + for version in "${FOUNDRY_VERSIONS[@]}"; do + log_info "Benchmarking $repo_name with Foundry $version" + + # Install the specific version + install_foundry_version "$version" || { + log_warn "Failed to install Foundry $version, skipping..." + continue + } + + # Clean version string for filenames (remove 'v' prefix, replace '.' with '_') + local clean_version="${version//v/}" + clean_version="${clean_version//\./_}" + + local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" + mkdir -p "$version_results_dir" + + # Benchmark 1: forge test + log_info "Running 'forge test' benchmark for $repo_name (Foundry $version)..." + hyperfine \ + --runs 3 \ + --warmup 1 \ + --export-json "${version_results_dir}/test_results.json" \ + "forge test" \ + 2>/dev/null || log_warn "forge test benchmark failed for $repo_name (Foundry $version)" + + # Benchmark 2: forge build (no cache) + log_info "Running 'forge build' (no cache) benchmark for $repo_name (Foundry $version)..." + hyperfine \ + --runs 3 \ + --cleanup 'forge clean' \ + --export-json "${version_results_dir}/build_no_cache_results.json" \ + "forge build" \ + 2>/dev/null || log_warn "forge build (no cache) benchmark failed for $repo_name (Foundry $version)" + + # Benchmark 3: forge build (with cache) + log_info "Running 'forge build' (with cache) benchmark for $repo_name (Foundry $version)..." + # First build to populate cache + forge build > /dev/null 2>&1 || true + hyperfine \ + --runs 5 \ + --warmup 1 \ + --export-json "${version_results_dir}/build_with_cache_results.json" \ + "forge build" \ + 2>/dev/null || log_warn "forge build (with cache) benchmark failed for $repo_name (Foundry $version)" + + # Store version info for this benchmark + forge --version | head -n1 > "${version_results_dir}/forge_version.txt" 2>/dev/null || echo "unknown" > "${version_results_dir}/forge_version.txt" + done + + cd - > /dev/null + log_success "Completed benchmarking for $repo_name across all versions" +} + +# Extract mean time from JSON result file +extract_mean_time() { + local json_file=$1 + if [ -f "$json_file" ]; then + # Extract mean time in seconds, format to 3 decimal places + python3 -c " +import json, sys +try: + with open('$json_file') as f: + data = json.load(f) + mean_time = data['results'][0]['mean'] + print(f'{mean_time:.3f}') +except: + print('N/A') +" 2>/dev/null || echo "N/A" + else + echo "N/A" + fi +} + +# Get Foundry version string from file +get_forge_version() { + local version_file=$1 + if [ -f "$version_file" ]; then + cat "$version_file" | sed 's/forge //' | sed 's/ (.*//' + else + echo "unknown" + fi +} + +# Compile results into markdown with comparison tables +compile_results() { + log_info "Compiling multi-version benchmark results..." + + cat > "$RESULTS_FILE" << EOF +# Foundry Multi-Version Benchmarking Results + +**Generated on:** $(date) +**Hyperfine Version:** $(hyperfine --version) +**Foundry Versions Tested:** ${FOUNDRY_VERSIONS[*]} +**Repositories Tested:** ${REPO_NAMES[*]} + +## Summary + +This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. +The following benchmarks were performed: + +1. **forge test** - Running the test suite (3 runs, 1 warmup) +2. **forge build (no cache)** - Clean build without cache (3 runs, cache cleaned after each run) +3. **forge build (with cache)** - Build with warm cache (5 runs, 1 warmup) + +--- + +## Performance Comparison Tables + +EOF + + # Create comparison tables for each benchmark type + local benchmark_types=("test" "build_no_cache" "build_with_cache") + local benchmark_names=("forge test" "forge build (no cache)" "forge build (with cache)") + + for i in "${!benchmark_types[@]}"; do + local bench_type="${benchmark_types[$i]}" + local bench_name="${benchmark_names[$i]}" + + echo "### $bench_name" >> "$RESULTS_FILE" + echo "" >> "$RESULTS_FILE" + echo "Times in seconds (lower is better):" >> "$RESULTS_FILE" + echo "" >> "$RESULTS_FILE" + + # Create table header + echo -n "| Project " >> "$RESULTS_FILE" + for version in "${FOUNDRY_VERSIONS[@]}"; do + echo -n "| $version " >> "$RESULTS_FILE" + done + echo "|" >> "$RESULTS_FILE" + + # Create table separator + echo -n "|:---" >> "$RESULTS_FILE" + for version in "${FOUNDRY_VERSIONS[@]}"; do + echo -n "|---:" >> "$RESULTS_FILE" + done + echo "|" >> "$RESULTS_FILE" + + # Add data rows + for repo_name in "${REPO_NAMES[@]}"; do + echo -n "| **$repo_name** " >> "$RESULTS_FILE" + + for version in "${FOUNDRY_VERSIONS[@]}"; do + local clean_version="${version//v/}" + clean_version="${clean_version//\./_}" + local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" + local json_file="${version_results_dir}/${bench_type}_results.json" + + local mean_time=$(extract_mean_time "$json_file") + echo -n "| $mean_time " >> "$RESULTS_FILE" + done + echo "|" >> "$RESULTS_FILE" + done + echo "" >> "$RESULTS_FILE" + echo "" >> "$RESULTS_FILE" + done + + # Add detailed version information + echo "## Foundry Version Details" >> "$RESULTS_FILE" + echo "" >> "$RESULTS_FILE" + + for version in "${FOUNDRY_VERSIONS[@]}"; do + echo "### $version" >> "$RESULTS_FILE" + echo "" >> "$RESULTS_FILE" + + # Find any version file to get the detailed version info + local clean_version="${version//v/}" + clean_version="${clean_version//\./_}" + + for repo_name in "${REPO_NAMES[@]}"; do + local version_file="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}/forge_version.txt" + if [ -f "$version_file" ]; then + echo "\`\`\`" >> "$RESULTS_FILE" + cat "$version_file" >> "$RESULTS_FILE" + echo "\`\`\`" >> "$RESULTS_FILE" + break + fi + done + echo "" >> "$RESULTS_FILE" + done + + # Add notes and system info + cat >> "$RESULTS_FILE" << EOF + +## Notes + +- All benchmarks were run with hyperfine +- **forge test**: 3 runs with 1 warmup per version +- **forge build (no cache)**: 3 runs with cache cleanup after each run +- **forge build (with cache)**: 5 runs with 1 warmup on pre-warmed cache +- Results show mean execution time in seconds +- N/A indicates benchmark failed or data unavailable + +## System Information + +- **OS:** $(uname -s) +- **Architecture:** $(uname -m) +- **Date:** $(date) + +## Raw Data + +Raw JSON benchmark data is available in: \`$JSON_RESULTS_DIR\` + +EOF +} + +# Cleanup temporary files +cleanup() { + # Clean up any temporary files (currently none used in multi-version approach) + log_info "Cleanup completed" +} + +# Main execution +main() { + log_info "Starting Foundry Multi-Version Benchmarking Suite..." + log_info "Testing Foundry versions: ${FOUNDRY_VERSIONS[*]}" + log_info "Testing repositories: ${REPO_NAMES[*]}" + + # Setup + check_dependencies + setup_directories + + # Ensure cleanup on exit + trap cleanup EXIT + + # Clone/update repositories + for i in "${!REPO_NAMES[@]}"; do + clone_or_update_repo "${REPO_NAMES[$i]}" "${REPO_URLS[$i]}" + install_dependencies "${BENCHMARK_DIR}/${REPO_NAMES[$i]}" "${REPO_NAMES[$i]}" + done + + # Run benchmarks across all versions + for repo_name in "${REPO_NAMES[@]}"; do + benchmark_repository "$repo_name" + done + + # Compile results + compile_results + + log_success "Multi-version benchmarking complete!" + log_success "Results saved to: $RESULTS_FILE" + log_success "Raw JSON data saved to: $JSON_RESULTS_DIR" + log_info "You can view the results with: cat $RESULTS_FILE" +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --versions) + shift + if [[ $# -eq 0 ]]; then + log_error "--versions requires a space-separated list of versions" + exit 1 + fi + # Read versions until next flag or end of args + FOUNDRY_VERSIONS=() + while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do + FOUNDRY_VERSIONS+=("$1") + shift + done + ;; + --help|-h) + echo "Foundry Multi-Version Benchmarking Suite" + echo "" + echo "Usage: $0 [OPTIONS]" + echo "" + echo "OPTIONS:" + echo " --help, -h Show this help message" + echo " --version, -v Show version information" + echo " --versions ... Specify Foundry versions to benchmark" + echo " (default: stable nightly v1.0.0)" + echo "" + echo "EXAMPLES:" + echo " $0 # Use default versions" + echo " $0 --versions stable nightly # Benchmark stable and nightly only" + echo " $0 --versions v1.0.0 v1.1.0 v1.2.0 # Benchmark specific versions" + echo "" + echo "This script benchmarks forge test and forge build commands across" + echo "multiple Foundry repositories and versions using hyperfine." + echo "" + echo "Supported version formats:" + echo " - stable, nightly (special tags)" + echo " - v1.0.0, v1.1.0, etc. (specific versions)" + echo "" + echo "The script will:" + echo " 1. Install foundryup if not present" + echo " 2. Clone/update target repositories" + echo " 3. Install each specified Foundry version" + echo " 4. Run benchmarks for each repo with each version" + echo " 5. Generate comparison tables in markdown format" + exit 0 + ;; + --version|-v) + echo "Foundry Multi-Version Benchmarking Suite v2.0.0" + exit 0 + ;; + *) + log_error "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac + done +} + +# Handle command line arguments +if [[ $# -gt 0 ]]; then + parse_args "$@" +fi + +main \ No newline at end of file From f43b5d95ce70a2778c86e3ac43af6961d0267db9 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 11 Jun 2025 18:06:12 +0200 Subject: [PATCH 02/74] Fix benchmark script JSON data extraction and table formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix relative path issue causing JSON files to fail creation - Convert benchmark directories to absolute paths using SCRIPT_DIR - Improve markdown table formatting with proper column names and alignment - Use unified table generation with string concatenation for better formatting - Increase benchmark runs from 3 to 5 for more reliable results - Use --prepare instead of --cleanup for better cache management - Remove stderr suppression to catch hyperfine errors - Update table headers to show units (seconds) for clarity 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- benchmark.sh | 66 +++++++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index 542cd5ae66c3b..a911350402074 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -7,14 +7,15 @@ set -e # Configuration -BENCHMARK_DIR="./benchmark_repos" -RESULTS_DIR="./benchmark_results" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BENCHMARK_DIR="${SCRIPT_DIR}/benchmark_repos" +RESULTS_DIR="${SCRIPT_DIR}/benchmark_results" TIMESTAMP=$(date +"%Y%m%d_%H%M%S") RESULTS_FILE="${RESULTS_DIR}/foundry_multi_version_benchmark_${TIMESTAMP}.md" JSON_RESULTS_DIR="${RESULTS_DIR}/json_${TIMESTAMP}" # Foundry versions to benchmark (can be modified via command line) -DEFAULT_FOUNDRY_VERSIONS=("stable" "nightly" "v1.0.0") +DEFAULT_FOUNDRY_VERSIONS=("stable" "nightly") FOUNDRY_VERSIONS=("${FOUNDRY_VERSIONS[@]:-${DEFAULT_FOUNDRY_VERSIONS[@]}}") # Repository configurations @@ -186,8 +187,8 @@ benchmark_repository() { cd "$repo_dir" # Check if it's a valid Foundry project - if [ ! -f "foundry.toml" ] && [ ! -f "forge.toml" ]; then - log_warn "No foundry.toml or forge.toml found in $repo_name, skipping..." + if [ ! -f "foundry.toml" ]; then + log_warn "No foundry.toml found in $repo_name, skipping..." cd - > /dev/null return 0 fi @@ -212,31 +213,29 @@ benchmark_repository() { # Benchmark 1: forge test log_info "Running 'forge test' benchmark for $repo_name (Foundry $version)..." hyperfine \ - --runs 3 \ + --runs 5 \ + --prepare 'forge build' \ --warmup 1 \ --export-json "${version_results_dir}/test_results.json" \ - "forge test" \ - 2>/dev/null || log_warn "forge test benchmark failed for $repo_name (Foundry $version)" + "forge test" || log_warn "forge test benchmark failed for $repo_name (Foundry $version)" # Benchmark 2: forge build (no cache) log_info "Running 'forge build' (no cache) benchmark for $repo_name (Foundry $version)..." hyperfine \ - --runs 3 \ - --cleanup 'forge clean' \ + --runs 5 \ + --prepare 'forge clean' \ --export-json "${version_results_dir}/build_no_cache_results.json" \ - "forge build" \ - 2>/dev/null || log_warn "forge build (no cache) benchmark failed for $repo_name (Foundry $version)" + "forge build" || log_warn "forge build (no cache) benchmark failed for $repo_name (Foundry $version)" # Benchmark 3: forge build (with cache) log_info "Running 'forge build' (with cache) benchmark for $repo_name (Foundry $version)..." # First build to populate cache - forge build > /dev/null 2>&1 || true hyperfine \ --runs 5 \ + --prepare 'forge build' \ --warmup 1 \ --export-json "${version_results_dir}/build_with_cache_results.json" \ - "forge build" \ - 2>/dev/null || log_warn "forge build (with cache) benchmark failed for $repo_name (Foundry $version)" + "forge build" || log_warn "forge build (with cache) benchmark failed for $repo_name (Foundry $version)" # Store version info for this benchmark forge --version | head -n1 > "${version_results_dir}/forge_version.txt" 2>/dev/null || echo "unknown" > "${version_results_dir}/forge_version.txt" @@ -281,7 +280,7 @@ compile_results() { log_info "Compiling multi-version benchmark results..." cat > "$RESULTS_FILE" << EOF -# Foundry Multi-Version Benchmarking Results +# Forge Benchmarking Results **Generated on:** $(date) **Hyperfine Version:** $(hyperfine --version) @@ -293,8 +292,8 @@ compile_results() { This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. The following benchmarks were performed: -1. **forge test** - Running the test suite (3 runs, 1 warmup) -2. **forge build (no cache)** - Clean build without cache (3 runs, cache cleaned after each run) +1. **forge test** - Running the test suite (5 runs, 1 warmup) +2. **forge build (no cache)** - Clean build without cache (5 runs, cache cleaned after each run) 3. **forge build (with cache)** - Build with warm cache (5 runs, 1 warmup) --- @@ -303,7 +302,7 @@ The following benchmarks were performed: EOF - # Create comparison tables for each benchmark type + # Create unified comparison tables for each benchmark type local benchmark_types=("test" "build_no_cache" "build_with_cache") local benchmark_names=("forge test" "forge build (no cache)" "forge build (with cache)") @@ -313,26 +312,28 @@ EOF echo "### $bench_name" >> "$RESULTS_FILE" echo "" >> "$RESULTS_FILE" - echo "Times in seconds (lower is better):" >> "$RESULTS_FILE" + echo "Mean execution time in seconds (lower is better):" >> "$RESULTS_FILE" echo "" >> "$RESULTS_FILE" - # Create table header - echo -n "| Project " >> "$RESULTS_FILE" + # Create table header with proper column names + local header_row="| Project" for version in "${FOUNDRY_VERSIONS[@]}"; do - echo -n "| $version " >> "$RESULTS_FILE" + header_row+=" | $version (s)" done - echo "|" >> "$RESULTS_FILE" + header_row+=" |" + echo "$header_row" >> "$RESULTS_FILE" - # Create table separator - echo -n "|:---" >> "$RESULTS_FILE" + # Create table separator with proper alignment + local separator_row="|------" for version in "${FOUNDRY_VERSIONS[@]}"; do - echo -n "|---:" >> "$RESULTS_FILE" + separator_row+="|--------:" done - echo "|" >> "$RESULTS_FILE" + separator_row+="|" + echo "$separator_row" >> "$RESULTS_FILE" # Add data rows for repo_name in "${REPO_NAMES[@]}"; do - echo -n "| **$repo_name** " >> "$RESULTS_FILE" + local data_row="| **$repo_name**" for version in "${FOUNDRY_VERSIONS[@]}"; do local clean_version="${version//v/}" @@ -341,9 +342,10 @@ EOF local json_file="${version_results_dir}/${bench_type}_results.json" local mean_time=$(extract_mean_time "$json_file") - echo -n "| $mean_time " >> "$RESULTS_FILE" + data_row+=" | $mean_time" done - echo "|" >> "$RESULTS_FILE" + data_row+=" |" + echo "$data_row" >> "$RESULTS_FILE" done echo "" >> "$RESULTS_FILE" echo "" >> "$RESULTS_FILE" @@ -503,4 +505,4 @@ if [[ $# -gt 0 ]]; then parse_args "$@" fi -main \ No newline at end of file +main From a8e28b1f626fa8d2d636a6ce5e9e58b6266be12a Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 12 Jun 2025 11:43:19 +0200 Subject: [PATCH 03/74] parallel benchmarking --- benchmark.sh | 196 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 129 insertions(+), 67 deletions(-) diff --git a/benchmark.sh b/benchmark.sh index a911350402074..27096d22324a9 100755 --- a/benchmark.sh +++ b/benchmark.sh @@ -15,7 +15,7 @@ RESULTS_FILE="${RESULTS_DIR}/foundry_multi_version_benchmark_${TIMESTAMP}.md" JSON_RESULTS_DIR="${RESULTS_DIR}/json_${TIMESTAMP}" # Foundry versions to benchmark (can be modified via command line) -DEFAULT_FOUNDRY_VERSIONS=("stable" "nightly") +DEFAULT_FOUNDRY_VERSIONS=("stable" "nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae" "nightly") # nightly-ac0 with linter not included in forge build. FOUNDRY_VERSIONS=("${FOUNDRY_VERSIONS[@]:-${DEFAULT_FOUNDRY_VERSIONS[@]}}") # Repository configurations @@ -74,28 +74,16 @@ install_foundry_version() { local version=$1 log_info "Installing Foundry version: $version" - case "$version" in - "stable"|"nightly") - foundryup --install "$version" || { - log_error "Failed to install Foundry $version" - return 1 - } - ;; - v*) - foundryup --install "$version" || { - log_error "Failed to install Foundry $version" - return 1 - } - ;; - *) - log_error "Unsupported version format: $version" - return 1 - ;; - esac - - # Verify installation - local installed_version=$(forge --version | head -n1 || echo "unknown") - log_success "Installed Foundry: $installed_version" + # Let foundryup handle any version format and determine validity + if foundryup --install "$version"; then + # Verify installation + local installed_version=$(forge --version | head -n1 || echo "unknown") + log_success "Installed Foundry: $installed_version" + return 0 + else + log_error "Failed to install Foundry $version" + return 1 + fi } # Check if required tools are installed @@ -172,36 +160,32 @@ install_dependencies() { cd - > /dev/null } -# Run benchmarks for a single repository across all Foundry versions -benchmark_repository() { +# Run benchmarks for a single repository with a specific Foundry version +benchmark_repository_for_version() { local repo_name=$1 + local version=$2 local repo_dir="${BENCHMARK_DIR}/${repo_name}" + local log_prefix="[$repo_name:$version]" - log_info "Benchmarking repository: $repo_name across ${#FOUNDRY_VERSIONS[@]} Foundry versions" - - if [ ! -d "$repo_dir" ]; then - log_error "Repository directory not found: $repo_dir" - return 1 - fi - - cd "$repo_dir" - - # Check if it's a valid Foundry project - if [ ! -f "foundry.toml" ]; then - log_warn "No foundry.toml found in $repo_name, skipping..." - cd - > /dev/null - return 0 - fi + # Create a unique log file for this repo+version combination + local log_file="${JSON_RESULTS_DIR}/${repo_name}_${version//[^a-zA-Z0-9]/_}_benchmark.log" - # Benchmark each Foundry version - for version in "${FOUNDRY_VERSIONS[@]}"; do - log_info "Benchmarking $repo_name with Foundry $version" + { + echo "$(date): Starting benchmark for $repo_name with Foundry $version" - # Install the specific version - install_foundry_version "$version" || { - log_warn "Failed to install Foundry $version, skipping..." - continue - } + if [ ! -d "$repo_dir" ]; then + echo "ERROR: Repository directory not found: $repo_dir" + return 1 + fi + + cd "$repo_dir" + + # Check if it's a valid Foundry project + if [ ! -f "foundry.toml" ]; then + echo "WARN: No foundry.toml found in $repo_name, skipping..." + cd - > /dev/null + return 0 + fi # Clean version string for filenames (remove 'v' prefix, replace '.' with '_') local clean_version="${version//v/}" @@ -210,41 +194,114 @@ benchmark_repository() { local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" mkdir -p "$version_results_dir" + echo "Running benchmarks for $repo_name with Foundry $version..." + # Benchmark 1: forge test - log_info "Running 'forge test' benchmark for $repo_name (Foundry $version)..." - hyperfine \ + echo "Running 'forge test' benchmark..." + if hyperfine \ --runs 5 \ --prepare 'forge build' \ --warmup 1 \ --export-json "${version_results_dir}/test_results.json" \ - "forge test" || log_warn "forge test benchmark failed for $repo_name (Foundry $version)" + "forge test" 2>>"$log_file.error"; then + echo "✓ forge test completed" + else + echo "✗ forge test failed" + fi # Benchmark 2: forge build (no cache) - log_info "Running 'forge build' (no cache) benchmark for $repo_name (Foundry $version)..." - hyperfine \ + echo "Running 'forge build' (no cache) benchmark..." + if hyperfine \ --runs 5 \ --prepare 'forge clean' \ --export-json "${version_results_dir}/build_no_cache_results.json" \ - "forge build" || log_warn "forge build (no cache) benchmark failed for $repo_name (Foundry $version)" + "forge build" 2>>"$log_file.error"; then + echo "✓ forge build (no cache) completed" + else + echo "✗ forge build (no cache) failed" + fi # Benchmark 3: forge build (with cache) - log_info "Running 'forge build' (with cache) benchmark for $repo_name (Foundry $version)..." - # First build to populate cache - hyperfine \ + echo "Running 'forge build' (with cache) benchmark..." + if hyperfine \ --runs 5 \ --prepare 'forge build' \ --warmup 1 \ --export-json "${version_results_dir}/build_with_cache_results.json" \ - "forge build" || log_warn "forge build (with cache) benchmark failed for $repo_name (Foundry $version)" + "forge build" 2>>"$log_file.error"; then + echo "✓ forge build (with cache) completed" + else + echo "✗ forge build (with cache) failed" + fi # Store version info for this benchmark forge --version | head -n1 > "${version_results_dir}/forge_version.txt" 2>/dev/null || echo "unknown" > "${version_results_dir}/forge_version.txt" + + cd - > /dev/null + echo "$(date): Completed benchmark for $repo_name with Foundry $version" + + } > "$log_file" 2>&1 +} + +# Run benchmarks for all repositories in parallel for each Foundry version +benchmark_all_repositories_parallel() { + for version in "${FOUNDRY_VERSIONS[@]}"; do + log_info "Installing Foundry version: $version" + + # Install the specific version once for all repositories + install_foundry_version "$version" || { + log_warn "Failed to install Foundry $version, skipping all repositories for this version..." + continue + } + + log_info "Starting parallel benchmarks for all repositories with Foundry $version" + + # Launch all repositories in parallel + local pids=() + local failed_repos=() + + for repo_name in "${REPO_NAMES[@]}"; do + # Check if repo directory exists and is valid before starting background process + local repo_dir="${BENCHMARK_DIR}/${repo_name}" + if [ ! -d "$repo_dir" ]; then + log_warn "Repository directory not found: $repo_dir, skipping..." + continue + fi + + if [ ! -f "${repo_dir}/foundry.toml" ]; then + log_warn "No foundry.toml found in $repo_name, skipping..." + continue + fi + + log_info "Launching background benchmark for $repo_name..." + benchmark_repository_for_version "$repo_name" "$version" & + local pid=$! + pids+=($pid) + echo "$repo_name:$pid" >> "${JSON_RESULTS_DIR}/parallel_pids_${version//[^a-zA-Z0-9]/_}.txt" + done + + # Wait for all repositories to complete + log_info "Waiting for ${#pids[@]} parallel benchmarks to complete for Foundry $version..." + local completed=0 + local total=${#pids[@]} + + for pid in "${pids[@]}"; do + if wait "$pid"; then + completed=$((completed + 1)) + log_info "Progress: $completed/$total repositories completed for Foundry $version" + else + log_warn "One benchmark process failed (PID: $pid)" + fi + done + + log_success "All repositories completed for Foundry $version ($completed/$total successful)" + + # Show summary of log files created + log_info "Individual benchmark logs available in: ${JSON_RESULTS_DIR}/*_${version//[^a-zA-Z0-9]/_}_benchmark.log" done - - cd - > /dev/null - log_success "Completed benchmarking for $repo_name across all versions" } + # Extract mean time from JSON result file extract_mean_time() { local json_file=$1 @@ -425,10 +482,9 @@ main() { install_dependencies "${BENCHMARK_DIR}/${REPO_NAMES[$i]}" "${REPO_NAMES[$i]}" done - # Run benchmarks across all versions - for repo_name in "${REPO_NAMES[@]}"; do - benchmark_repository "$repo_name" - done + # Run benchmarks in parallel + log_info "Using parallel execution mode" + benchmark_all_repositories_parallel # Compile results compile_results @@ -468,22 +524,28 @@ parse_args() { echo " (default: stable nightly v1.0.0)" echo "" echo "EXAMPLES:" - echo " $0 # Use default versions" + echo " $0 # Use default versions (parallel)" echo " $0 --versions stable nightly # Benchmark stable and nightly only" echo " $0 --versions v1.0.0 v1.1.0 v1.2.0 # Benchmark specific versions" echo "" echo "This script benchmarks forge test and forge build commands across" echo "multiple Foundry repositories and versions using hyperfine." echo "" + echo "EXECUTION MODE:" + echo " - Parallel: Install each Foundry version once, then run all repositories" + echo " in parallel for that version. This provides much better performance." + echo "" echo "Supported version formats:" echo " - stable, nightly (special tags)" echo " - v1.0.0, v1.1.0, etc. (specific versions)" + echo " - nightly- (specific nightly builds)" + echo " - Any format supported by foundryup" echo "" echo "The script will:" echo " 1. Install foundryup if not present" echo " 2. Clone/update target repositories" echo " 3. Install each specified Foundry version" - echo " 4. Run benchmarks for each repo with each version" + echo " 4. Run benchmarks for each repo with each version in parallel" echo " 5. Generate comparison tables in markdown format" exit 0 ;; From 0a569d78cdac519c750e5091fa380482c69c5aeb Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 12 Jun 2025 17:23:51 +0200 Subject: [PATCH 04/74] refac: mv to benches/ dir --- .gitignore | 4 + benches/LATEST.md | 85 ++++++++++++ benchmark.sh => benches/benchmark.sh | 145 ++++++++------------- benches/commands/forge_build_no_cache.sh | 45 +++++++ benches/commands/forge_build_with_cache.sh | 47 +++++++ benches/commands/forge_test.sh | 47 +++++++ benches/repos_and_versions.sh | 45 +++++++ 7 files changed, 327 insertions(+), 91 deletions(-) create mode 100644 benches/LATEST.md rename benchmark.sh => benches/benchmark.sh (79%) create mode 100755 benches/commands/forge_build_no_cache.sh create mode 100755 benches/commands/forge_build_with_cache.sh create mode 100755 benches/commands/forge_test.sh create mode 100755 benches/repos_and_versions.sh diff --git a/.gitignore b/.gitignore index 9297bbbc7d4f2..bb80c9c0da088 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ snapshots/ out.json .idea .vscode + +# Benchmark directories +benches/benchmark_repos/ +benches/benchmark_results/ diff --git a/benches/LATEST.md b/benches/LATEST.md new file mode 100644 index 0000000000000..b212d64452d7b --- /dev/null +++ b/benches/LATEST.md @@ -0,0 +1,85 @@ +# Forge Benchmarking Results + +**Generated on:** Thu 12 Jun 2025 16:57:20 CEST +**Hyperfine Version:** hyperfine 1.19.0 +**Foundry Versions Tested:** stable nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae nightly +**Repositories Tested:** ithacaxyz-account solady + +## Summary + +This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. +The following benchmarks were performed: + +1. **forge test - Running the test suite (5 runs, 1 warmup)** +2. **forge build (no cache) - Clean build without cache (5 runs, cache cleaned after each run)** +3. **forge build (with cache) - Build with warm cache (5 runs, 1 warmup)** + +--- + +## Performance Comparison Tables + +### forge test + +Mean execution time in seconds (lower is better): + +| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | +| --------------------- | ---------: | ---------------------------------------------------: | ----------: | +| **ithacaxyz-account** | 4.662 | 3.738 | 5.588 | +| **solady** | 3.559 | 2.933 | 3.517 | + +### forge build no cache + +Mean execution time in seconds (lower is better): + +| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | +| --------------------- | ---------: | ---------------------------------------------------: | ----------: | +| **ithacaxyz-account** | 10.777 | 10.982 | 10.979 | +| **solady** | 17.486 | 17.139 | 17.509 | + +### forge build with cache + +Mean execution time in seconds (lower is better): + +| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | +| --------------------- | ---------: | ---------------------------------------------------: | ----------: | +| **ithacaxyz-account** | 0.111 | 0.113 | 0.158 | +| **solady** | 0.084 | 0.089 | 0.108 | + +## Foundry Version Details + +### stable + +``` +forge Version: 1.2.3-stable +``` + +### nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae + +``` +forge Version: 1.2.3-nightly +``` + +### nightly + +``` +forge Version: 1.2.3-nightly +``` + +## Notes + +- All benchmarks were run with hyperfine in parallel mode +- **forge test - Running the test suite (5 runs, 1 warmup)** +- **forge build (no cache) - Clean build without cache (5 runs, cache cleaned after each run)** +- **forge build (with cache) - Build with warm cache (5 runs, 1 warmup)** +- Results show mean execution time in seconds +- N/A indicates benchmark failed. + +## System Information + +- **OS:** Darwin +- **Architecture:** arm64 +- **Date:** Thu 12 Jun 2025 16:57:21 CEST + +## Raw Data + +Raw JSON benchmark data is available in: `/Users/yash/dev/paradigm/foundry-rs/foundry/benches/benchmark_results/json_20250612_165120` diff --git a/benchmark.sh b/benches/benchmark.sh similarity index 79% rename from benchmark.sh rename to benches/benchmark.sh index 27096d22324a9..dc71db4606d7a 100755 --- a/benchmark.sh +++ b/benches/benchmark.sh @@ -1,39 +1,29 @@ #!/bin/bash -# Foundry Multi-Version Benchmarking Suite using hyperfine +# Foundry Multi-Version Benchmarking Suite # This script benchmarks forge test and forge build commands across multiple repositories # and multiple Foundry versions for comprehensive performance comparison set -e -# Configuration +# Get the directory where this script is located SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Configuration BENCHMARK_DIR="${SCRIPT_DIR}/benchmark_repos" RESULTS_DIR="${SCRIPT_DIR}/benchmark_results" TIMESTAMP=$(date +"%Y%m%d_%H%M%S") RESULTS_FILE="${RESULTS_DIR}/foundry_multi_version_benchmark_${TIMESTAMP}.md" +LATEST_RESULTS_FILE="${SCRIPT_DIR}/LATEST.md" JSON_RESULTS_DIR="${RESULTS_DIR}/json_${TIMESTAMP}" -# Foundry versions to benchmark (can be modified via command line) -DEFAULT_FOUNDRY_VERSIONS=("stable" "nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae" "nightly") # nightly-ac0 with linter not included in forge build. -FOUNDRY_VERSIONS=("${FOUNDRY_VERSIONS[@]:-${DEFAULT_FOUNDRY_VERSIONS[@]}}") - -# Repository configurations -REPO_NAMES=( - "account" - "v4-core" - "solady" - "morpho-blue" - "spark-psm" -) - -REPO_URLS=( - "https://github.com/ithacaxyz/account" - "https://github.com/Uniswap/v4-core" - "https://github.com/Vectorized/solady" - "https://github.com/morpho-org/morpho-blue" - "https://github.com/sparkdotfi/spark-psm" -) +# Load configuration +source "${SCRIPT_DIR}/repos_and_versions.sh" + +# Load benchmark commands +source "${SCRIPT_DIR}/commands/forge_test.sh" +source "${SCRIPT_DIR}/commands/forge_build_no_cache.sh" +source "${SCRIPT_DIR}/commands/forge_build_with_cache.sh" # Colors for output RED='\033[0;31m' @@ -146,7 +136,7 @@ install_dependencies() { cd "$repo_dir" # Install forge dependencies - if [ -f "foundry.toml" ] || [ -f "forge.toml" ]; then + if [ -f "foundry.toml" ]; then forge install 2>/dev/null || true fi @@ -165,7 +155,6 @@ benchmark_repository_for_version() { local repo_name=$1 local version=$2 local repo_dir="${BENCHMARK_DIR}/${repo_name}" - local log_prefix="[$repo_name:$version]" # Create a unique log file for this repo+version combination local log_file="${JSON_RESULTS_DIR}/${repo_name}_${version//[^a-zA-Z0-9]/_}_benchmark.log" @@ -196,43 +185,19 @@ benchmark_repository_for_version() { echo "Running benchmarks for $repo_name with Foundry $version..." - # Benchmark 1: forge test - echo "Running 'forge test' benchmark..." - if hyperfine \ - --runs 5 \ - --prepare 'forge build' \ - --warmup 1 \ - --export-json "${version_results_dir}/test_results.json" \ - "forge test" 2>>"$log_file.error"; then - echo "✓ forge test completed" - else - echo "✗ forge test failed" - fi - - # Benchmark 2: forge build (no cache) - echo "Running 'forge build' (no cache) benchmark..." - if hyperfine \ - --runs 5 \ - --prepare 'forge clean' \ - --export-json "${version_results_dir}/build_no_cache_results.json" \ - "forge build" 2>>"$log_file.error"; then - echo "✓ forge build (no cache) completed" - else - echo "✗ forge build (no cache) failed" - fi - - # Benchmark 3: forge build (with cache) - echo "Running 'forge build' (with cache) benchmark..." - if hyperfine \ - --runs 5 \ - --prepare 'forge build' \ - --warmup 1 \ - --export-json "${version_results_dir}/build_with_cache_results.json" \ - "forge build" 2>>"$log_file.error"; then - echo "✓ forge build (with cache) completed" - else - echo "✗ forge build (with cache) failed" - fi + # Run all benchmark commands - fail fast if any command fails + benchmark_forge_test "$repo_name" "$version" "$version_results_dir" "$log_file" || { + echo "FATAL: forge test benchmark failed for $repo_name with Foundry $version" >> "$log_file" + exit 1 + } + benchmark_forge_build_no_cache "$repo_name" "$version" "$version_results_dir" "$log_file" || { + echo "FATAL: forge build (no cache) benchmark failed for $repo_name with Foundry $version" >> "$log_file" + exit 1 + } + benchmark_forge_build_with_cache "$repo_name" "$version" "$version_results_dir" "$log_file" || { + echo "FATAL: forge build (with cache) benchmark failed for $repo_name with Foundry $version" >> "$log_file" + exit 1 + } # Store version info for this benchmark forge --version | head -n1 > "${version_results_dir}/forge_version.txt" 2>/dev/null || echo "unknown" > "${version_results_dir}/forge_version.txt" @@ -258,7 +223,6 @@ benchmark_all_repositories_parallel() { # Launch all repositories in parallel local pids=() - local failed_repos=() for repo_name in "${REPO_NAMES[@]}"; do # Check if repo directory exists and is valid before starting background process @@ -290,7 +254,8 @@ benchmark_all_repositories_parallel() { completed=$((completed + 1)) log_info "Progress: $completed/$total repositories completed for Foundry $version" else - log_warn "One benchmark process failed (PID: $pid)" + log_error "Benchmark process failed (PID: $pid) for Foundry $version" + exit 1 fi done @@ -301,7 +266,6 @@ benchmark_all_repositories_parallel() { done } - # Extract mean time from JSON result file extract_mean_time() { local json_file=$1 @@ -334,7 +298,7 @@ get_forge_version() { # Compile results into markdown with comparison tables compile_results() { - log_info "Compiling multi-version benchmark results..." + log_info "Compiling benchmark results..." cat > "$RESULTS_FILE" << EOF # Forge Benchmarking Results @@ -349,9 +313,9 @@ compile_results() { This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. The following benchmarks were performed: -1. **forge test** - Running the test suite (5 runs, 1 warmup) -2. **forge build (no cache)** - Clean build without cache (5 runs, cache cleaned after each run) -3. **forge build (with cache)** - Build with warm cache (5 runs, 1 warmup) +1. **$(get_forge_test_description)** +2. **$(get_forge_build_no_cache_description)** +3. **$(get_forge_build_with_cache_description)** --- @@ -360,12 +324,12 @@ The following benchmarks were performed: EOF # Create unified comparison tables for each benchmark type - local benchmark_types=("test" "build_no_cache" "build_with_cache") - local benchmark_names=("forge test" "forge build (no cache)" "forge build (with cache)") + local benchmark_commands=("forge_test" "forge_build_no_cache" "forge_build_with_cache") - for i in "${!benchmark_types[@]}"; do - local bench_type="${benchmark_types[$i]}" - local bench_name="${benchmark_names[$i]}" + for cmd in "${benchmark_commands[@]}"; do + local bench_name="${cmd//_/ }" + local bench_type=$(get_${cmd}_type) + local json_filename=$(get_${cmd}_json_filename) echo "### $bench_name" >> "$RESULTS_FILE" echo "" >> "$RESULTS_FILE" @@ -396,7 +360,7 @@ EOF local clean_version="${version//v/}" clean_version="${clean_version//\./_}" local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" - local json_file="${version_results_dir}/${bench_type}_results.json" + local json_file="${version_results_dir}/${json_filename}" local mean_time=$(extract_mean_time "$json_file") data_row+=" | $mean_time" @@ -437,10 +401,10 @@ EOF ## Notes -- All benchmarks were run with hyperfine -- **forge test**: 3 runs with 1 warmup per version -- **forge build (no cache)**: 3 runs with cache cleanup after each run -- **forge build (with cache)**: 5 runs with 1 warmup on pre-warmed cache +- All benchmarks were run with hyperfine in parallel mode +- **$(get_forge_test_description)** +- **$(get_forge_build_no_cache_description)** +- **$(get_forge_build_with_cache_description)** - Results show mean execution time in seconds - N/A indicates benchmark failed or data unavailable @@ -455,11 +419,14 @@ EOF Raw JSON benchmark data is available in: \`$JSON_RESULTS_DIR\` EOF + + # Copy to LATEST.md + cp "$RESULTS_FILE" "$LATEST_RESULTS_FILE" + log_success "Latest results also saved to: $LATEST_RESULTS_FILE" } # Cleanup temporary files cleanup() { - # Clean up any temporary files (currently none used in multi-version approach) log_info "Cleanup completed" } @@ -483,16 +450,16 @@ main() { done # Run benchmarks in parallel - log_info "Using parallel execution mode" benchmark_all_repositories_parallel # Compile results compile_results - log_success "Multi-version benchmarking complete!" + log_success "Benchmarking complete!" log_success "Results saved to: $RESULTS_FILE" + log_success "Latest results: $LATEST_RESULTS_FILE" log_success "Raw JSON data saved to: $JSON_RESULTS_DIR" - log_info "You can view the results with: cat $RESULTS_FILE" + log_info "You can view the results with: cat $LATEST_RESULTS_FILE" } # Parse command line arguments @@ -513,7 +480,7 @@ parse_args() { done ;; --help|-h) - echo "Foundry Multi-Version Benchmarking Suite" + echo "Foundry Benchmarking Suite" echo "" echo "Usage: $0 [OPTIONS]" echo "" @@ -521,7 +488,7 @@ parse_args() { echo " --help, -h Show this help message" echo " --version, -v Show version information" echo " --versions ... Specify Foundry versions to benchmark" - echo " (default: stable nightly v1.0.0)" + echo " (default: from repos_and_versions.sh)" echo "" echo "EXAMPLES:" echo " $0 # Use default versions (parallel)" @@ -530,11 +497,7 @@ parse_args() { echo "" echo "This script benchmarks forge test and forge build commands across" echo "multiple Foundry repositories and versions using hyperfine." - echo "" - echo "EXECUTION MODE:" - echo " - Parallel: Install each Foundry version once, then run all repositories" - echo " in parallel for that version. This provides much better performance." - echo "" + echo "Supported version formats:" echo " - stable, nightly (special tags)" echo " - v1.0.0, v1.1.0, etc. (specific versions)" @@ -547,10 +510,10 @@ parse_args() { echo " 3. Install each specified Foundry version" echo " 4. Run benchmarks for each repo with each version in parallel" echo " 5. Generate comparison tables in markdown format" + echo " 6. Save results to LATEST.md" exit 0 ;; --version|-v) - echo "Foundry Multi-Version Benchmarking Suite v2.0.0" exit 0 ;; *) @@ -567,4 +530,4 @@ if [[ $# -gt 0 ]]; then parse_args "$@" fi -main +main \ No newline at end of file diff --git a/benches/commands/forge_build_no_cache.sh b/benches/commands/forge_build_no_cache.sh new file mode 100755 index 0000000000000..756528176ac4e --- /dev/null +++ b/benches/commands/forge_build_no_cache.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Forge Build (No Cache) Benchmark Command +# This file contains the configuration and execution logic for benchmarking 'forge build' with no cache + +# Command configuration +FORGE_BUILD_NO_CACHE_RUNS=5 + +# Benchmark function for forge build (no cache) +benchmark_forge_build_no_cache() { + local repo_name=$1 + local version=$2 + local version_results_dir=$3 + local log_file=$4 + + echo "Running 'forge build' (no cache) benchmark..." >> "$log_file" + + if hyperfine \ + --runs "$FORGE_BUILD_NO_CACHE_RUNS" \ + --prepare 'forge clean' \ + --export-json "${version_results_dir}/build_no_cache_results.json" \ + "forge build" 2>>"$log_file.error"; then + echo "✓ forge build (no cache) completed" >> "$log_file" + return 0 + else + echo "✗ forge build (no cache) failed" >> "$log_file" + echo "FATAL: forge build (no cache) benchmark failed" >> "$log_file" + return 1 + fi +} + +# Get command description for reporting +get_forge_build_no_cache_description() { + echo "forge build (no cache) - Clean build without cache ($FORGE_BUILD_NO_CACHE_RUNS runs, cache cleaned after each run)" +} + +# Get JSON result filename +get_forge_build_no_cache_json_filename() { + echo "build_no_cache_results.json" +} + +# Get benchmark type identifier +get_forge_build_no_cache_type() { + echo "build_no_cache" +} \ No newline at end of file diff --git a/benches/commands/forge_build_with_cache.sh b/benches/commands/forge_build_with_cache.sh new file mode 100755 index 0000000000000..25fab9dbc190f --- /dev/null +++ b/benches/commands/forge_build_with_cache.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Forge Build (With Cache) Benchmark Command +# This file contains the configuration and execution logic for benchmarking 'forge build' with cache + +# Command configuration +FORGE_BUILD_WITH_CACHE_RUNS=5 +FORGE_BUILD_WITH_CACHE_WARMUP=1 + +# Benchmark function for forge build (with cache) +benchmark_forge_build_with_cache() { + local repo_name=$1 + local version=$2 + local version_results_dir=$3 + local log_file=$4 + + echo "Running 'forge build' (with cache) benchmark..." >> "$log_file" + + if hyperfine \ + --runs "$FORGE_BUILD_WITH_CACHE_RUNS" \ + --prepare 'forge build' \ + --warmup "$FORGE_BUILD_WITH_CACHE_WARMUP" \ + --export-json "${version_results_dir}/build_with_cache_results.json" \ + "forge build" 2>>"$log_file.error"; then + echo "✓ forge build (with cache) completed" >> "$log_file" + return 0 + else + echo "✗ forge build (with cache) failed" >> "$log_file" + echo "FATAL: forge build (with cache) benchmark failed" >> "$log_file" + return 1 + fi +} + +# Get command description for reporting +get_forge_build_with_cache_description() { + echo "forge build (with cache) - Build with warm cache ($FORGE_BUILD_WITH_CACHE_RUNS runs, $FORGE_BUILD_WITH_CACHE_WARMUP warmup)" +} + +# Get JSON result filename +get_forge_build_with_cache_json_filename() { + echo "build_with_cache_results.json" +} + +# Get benchmark type identifier +get_forge_build_with_cache_type() { + echo "build_with_cache" +} \ No newline at end of file diff --git a/benches/commands/forge_test.sh b/benches/commands/forge_test.sh new file mode 100755 index 0000000000000..9427d18d3c30c --- /dev/null +++ b/benches/commands/forge_test.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Forge Test Benchmark Command +# This file contains the configuration and execution logic for benchmarking 'forge test' + +# Command configuration +FORGE_TEST_RUNS=5 +FORGE_TEST_WARMUP=1 + +# Benchmark function for forge test +benchmark_forge_test() { + local repo_name=$1 + local version=$2 + local version_results_dir=$3 + local log_file=$4 + + echo "Running 'forge test' benchmark..." >> "$log_file" + + if hyperfine \ + --runs "$FORGE_TEST_RUNS" \ + --prepare 'forge build' \ + --warmup "$FORGE_TEST_WARMUP" \ + --export-json "${version_results_dir}/test_results.json" \ + "forge test" 2>>"$log_file.error"; then + echo "✓ forge test completed" >> "$log_file" + return 0 + else + echo "✗ forge test failed" >> "$log_file" + echo "FATAL: forge test benchmark failed" >> "$log_file" + return 1 + fi +} + +# Get command description for reporting +get_forge_test_description() { + echo "forge test - Running the test suite ($FORGE_TEST_RUNS runs, $FORGE_TEST_WARMUP warmup)" +} + +# Get JSON result filename +get_forge_test_json_filename() { + echo "test_results.json" +} + +# Get benchmark type identifier +get_forge_test_type() { + echo "test" +} \ No newline at end of file diff --git a/benches/repos_and_versions.sh b/benches/repos_and_versions.sh new file mode 100755 index 0000000000000..29f47e14a63a7 --- /dev/null +++ b/benches/repos_and_versions.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Foundry Multi-Version Benchmarking Configuration +# This file contains the configuration for repositories and Foundry versions to benchmark + +# Foundry versions to benchmark +# Supported formats: +# - stable, nightly (special tags) +# - v1.0.0, v1.1.0, etc. (specific versions) +# - nightly- (specific nightly builds) +# - Any format supported by foundryup +FOUNDRY_VERSIONS=( + "stable" + "nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae" + "nightly" +) + +# Repository configurations +# Add new repositories by adding entries to both arrays +REPO_NAMES=( + "ithacaxyz-account" + # "v4-core" + "solady" + # "morpho-blue" + # "spark-psm" +) + +REPO_URLS=( + "https://github.com/ithacaxyz/account" + # "https://github.com/Uniswap/v4-core" + "https://github.com/Vectorized/solady" + # "https://github.com/morpho-org/morpho-blue" + # "https://github.com/sparkdotfi/spark-psm" +) + +# Verify arrays have the same length +if [ ${#REPO_NAMES[@]} -ne ${#REPO_URLS[@]} ]; then + echo "ERROR: REPO_NAMES and REPO_URLS arrays must have the same length" + exit 1 +fi + +# Export variables for use in other scripts +export FOUNDRY_VERSIONS +export REPO_NAMES +export REPO_URLS \ No newline at end of file From 09bfb573c36895689bbed157c7e08e09df58c60f Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 18 Jun 2025 17:18:49 +0100 Subject: [PATCH 05/74] feat: criterion benches --- Cargo.lock | 110 +++++++++++++++++++ Cargo.toml | 1 + benches/Cargo.toml | 34 ++++++ benches/forge_build_no_cache.rs | 33 ++++++ benches/forge_build_with_cache.rs | 36 +++++++ benches/forge_test.rs | 36 +++++++ benches/src/lib.rs | 171 ++++++++++++++++++++++++++++++ 7 files changed, 421 insertions(+) create mode 100644 benches/Cargo.toml create mode 100644 benches/forge_build_no_cache.rs create mode 100644 benches/forge_build_with_cache.rs create mode 100644 benches/forge_test.rs create mode 100644 benches/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 4ebb5c3ed4097..812228968ba4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -929,6 +929,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "annotate-snippets" version = "0.11.5" @@ -2344,6 +2350,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cast" version = "1.2.3" @@ -2486,8 +2498,10 @@ checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -2949,6 +2963,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast 0.3.0", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast 0.3.0", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -4030,6 +4080,22 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foundry-bench" +version = "0.1.0" +dependencies = [ + "chrono", + "criterion", + "eyre", + "foundry-compilers", + "foundry-config", + "foundry-test-utils", + "serde", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "foundry-block-explorers" version = "0.18.0" @@ -6467,6 +6533,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "op-alloy-consensus" version = "0.17.2" @@ -6930,6 +7002,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -9292,6 +9392,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.9.0" diff --git a/Cargo.toml b/Cargo.toml index d635ea33ac0fc..d3d5d99f25178 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "benches/", "crates/anvil/", "crates/anvil/core/", "crates/anvil/rpc/", diff --git a/benches/Cargo.toml b/benches/Cargo.toml new file mode 100644 index 0000000000000..4d13048ad860b --- /dev/null +++ b/benches/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "foundry-bench" +version = "0.1.0" +edition = "2021" + +[[bench]] +name = "forge_test" +path = "forge_test.rs" +harness = false + +[[bench]] +name = "forge_build_no_cache" +path = "forge_build_no_cache.rs" +harness = false + +[[bench]] +name = "forge_build_with_cache" +path = "forge_build_with_cache.rs" +harness = false + +[dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +foundry-test-utils.workspace = true +foundry-config.workspace = true +foundry-compilers = { workspace = true, features = ["project-util"] } +eyre.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true +tokio = { workspace = true, features = ["full"] } +chrono = { version = "0.4", features = ["serde"] } + +[dev-dependencies] +foundry-test-utils.workspace = true \ No newline at end of file diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs new file mode 100644 index 0000000000000..51eb1219ff0e6 --- /dev/null +++ b/benches/forge_build_no_cache.rs @@ -0,0 +1,33 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{install_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; + +fn benchmark_forge_build_no_cache(c: &mut Criterion) { + let mut group = c.benchmark_group("forge-build-no-cache"); + group.sample_size(10); + + for &version in FOUNDRY_VERSIONS { + // Install foundry version once per version + install_foundry_version(version).expect("Failed to install foundry version"); + + for repo_config in BENCHMARK_REPOS { + // Setup: prepare project OUTSIDE benchmark + let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + + // Format: table_name/column_name/row_name + // This creates: forge-build-no-cache/{version}/{repo_name} + let bench_id = BenchmarkId::new(version, repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let output = project.run_forge_build(true).expect("forge build failed"); + black_box(output); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, benchmark_forge_build_no_cache); +criterion_main!(benches); diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs new file mode 100644 index 0000000000000..bc0af1de0fb2c --- /dev/null +++ b/benches/forge_build_with_cache.rs @@ -0,0 +1,36 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{install_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; + +fn benchmark_forge_build_with_cache(c: &mut Criterion) { + let mut group = c.benchmark_group("forge-build-with-cache"); + group.sample_size(10); + + for &version in FOUNDRY_VERSIONS { + // Install foundry version once per version + install_foundry_version(version).expect("Failed to install foundry version"); + + for repo_config in BENCHMARK_REPOS { + // Setup: prepare project OUTSIDE benchmark + let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + + // Prime the cache OUTSIDE benchmark + let _ = project.run_forge_build(false); + + // Format: table_name/column_name/row_name + // This creates: forge-build-with-cache/{version}/{repo_name} + let bench_id = BenchmarkId::new(version, repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let output = project.run_forge_build(false).expect("forge build failed"); + black_box(output); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, benchmark_forge_build_with_cache); +criterion_main!(benches); diff --git a/benches/forge_test.rs b/benches/forge_test.rs new file mode 100644 index 0000000000000..08c46dcbbbe17 --- /dev/null +++ b/benches/forge_test.rs @@ -0,0 +1,36 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{install_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; + +fn benchmark_forge_test(c: &mut Criterion) { + let mut group = c.benchmark_group("forge-test"); + group.sample_size(10); + + for &version in FOUNDRY_VERSIONS { + // Install foundry version once per version + install_foundry_version(version).expect("Failed to install foundry version"); + + for repo_config in BENCHMARK_REPOS { + // Setup: prepare project OUTSIDE benchmark + let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + + // Build the project before running tests + project.run_forge_build(false).expect("forge build failed"); + + // Format: table_name/column_name/row_name + // This creates: forge-test/{version}/{repo_name} + let bench_id = BenchmarkId::new(version, repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let output = project.run_forge_test().expect("forge test failed"); + black_box(output); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, benchmark_forge_test); +criterion_main!(benches); diff --git a/benches/src/lib.rs b/benches/src/lib.rs new file mode 100644 index 0000000000000..2c274690bc663 --- /dev/null +++ b/benches/src/lib.rs @@ -0,0 +1,171 @@ +use eyre::{Result, WrapErr}; +use foundry_compilers::project_util::TempProject; +use foundry_test_utils::util::clone_remote; +use std::{ + path::{Path, PathBuf}, + process::{Command, Output}, +}; + +/// Configuration for repositories to benchmark +#[derive(Debug, Clone)] +pub struct RepoConfig { + pub name: &'static str, + pub org: &'static str, + pub repo: &'static str, + pub rev: &'static str, +} + +/// Available repositories for benchmarking +pub static BENCHMARK_REPOS: &[RepoConfig] = &[ + RepoConfig { name: "account", org: "ithacaxyz", repo: "account", rev: "main" }, + // Temporarily reduced for testing + // RepoConfig { name: "solady", org: "Vectorized", repo: "solady", rev: "main" }, + // RepoConfig { name: "v4-core", org: "Uniswap", repo: "v4-core", rev: "main" }, + // RepoConfig { name: "morpho-blue", org: "morpho-org", repo: "morpho-blue", rev: "main" }, + // RepoConfig { name: "spark-psm", org: "marsfoundation", repo: "spark-psm", rev: "master" }, +]; + +/// Foundry versions to benchmark +pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; + +/// A benchmark project that represents a cloned repository ready for testing +pub struct BenchmarkProject { + pub name: String, + pub temp_project: TempProject, + pub root_path: PathBuf, +} + +impl BenchmarkProject { + /// Set up a benchmark project by cloning the repository + pub fn setup(config: &RepoConfig) -> Result { + let temp_project = + TempProject::dapptools().wrap_err("Failed to create temporary project")?; + + // Get root path before clearing + let root_path = temp_project.root().to_path_buf(); + let root = root_path.to_str().unwrap(); + + // Remove all files in the directory + for entry in std::fs::read_dir(&root_path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + std::fs::remove_dir_all(&path).ok(); + } else { + std::fs::remove_file(&path).ok(); + } + } + + // Clone the repository + let repo_url = format!("https://github.com/{}/{}.git", config.org, config.repo); + clone_remote(&repo_url, root); + + // Checkout specific revision if provided + if !config.rev.is_empty() && config.rev != "main" && config.rev != "master" { + let status = Command::new("git") + .current_dir(root) + .args(["checkout", config.rev]) + .status() + .wrap_err("Failed to checkout revision")?; + + if !status.success() { + eyre::bail!("Git checkout failed for {}", config.name); + } + } + + // Install dependencies + Self::install_dependencies(&root_path)?; + + Ok(BenchmarkProject { name: config.name.to_string(), root_path, temp_project }) + } + + /// Install forge dependencies for the project + fn install_dependencies(root: &Path) -> Result<()> { + // Install forge dependencies if foundry.toml exists + if root.join("foundry.toml").exists() { + let status = Command::new("forge") + .current_dir(root) + .args(["install"]) + .status() + .wrap_err("Failed to run forge install")?; + + if !status.success() { + println!("Warning: forge install failed for {}", root.display()); + } + } + + // Install npm dependencies if package.json exists + if root.join("package.json").exists() { + let status = Command::new("npm") + .current_dir(root) + .args(["install"]) + .status() + .wrap_err("Failed to run npm install")?; + + if !status.success() { + println!("Warning: npm install failed for {}", root.display()); + } + } + + Ok(()) + } + + /// Run forge test command and return the output + pub fn run_forge_test(&self) -> Result { + Command::new("forge") + .current_dir(&self.root_path) + .args(["test"]) + .output() + .wrap_err("Failed to run forge test") + } + + /// Run forge build command and return the output + pub fn run_forge_build(&self, clean_cache: bool) -> Result { + if clean_cache { + // Clean first + let _ = Command::new("forge").current_dir(&self.root_path).args(["clean"]).output(); + } + + Command::new("forge") + .current_dir(&self.root_path) + .args(["build"]) + .output() + .wrap_err("Failed to run forge build") + } + + /// Get the root path of the project + pub fn root(&self) -> &Path { + &self.root_path + } +} + +/// Install a specific foundry version +pub fn install_foundry_version(version: &str) -> Result<()> { + let status = Command::new("foundryup") + .args(["--install", version]) + .status() + .wrap_err("Failed to run foundryup")?; + + if !status.success() { + eyre::bail!("Failed to install foundry version: {}", version); + } + + Ok(()) +} + +/// Get the current forge version +pub fn get_forge_version() -> Result { + let output = Command::new("forge") + .args(["--version"]) + .output() + .wrap_err("Failed to get forge version")?; + + if !output.status.success() { + eyre::bail!("forge --version failed"); + } + + let version = + String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?; + + Ok(version.lines().next().unwrap_or("unknown").to_string()) +} From 8a673c9d7ebb6c4c78fe5b1e9f61cac75b0d68cc Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 18 Jun 2025 17:48:08 +0100 Subject: [PATCH 06/74] fix: install foundry versions at once --- benches/LATEST.md | 45 +++++++++++++-------------- benches/benchmark.sh | 58 ++++++++++++++++++++++++++++++----- benches/repos_and_versions.sh | 1 - 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index b212d64452d7b..f276dc5fd6133 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,8 +1,8 @@ # Forge Benchmarking Results -**Generated on:** Thu 12 Jun 2025 16:57:20 CEST +**Generated on:** Wed 18 Jun 2025 17:46:19 BST **Hyperfine Version:** hyperfine 1.19.0 -**Foundry Versions Tested:** stable nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae nightly +**Foundry Versions Tested:** stable nightly **Repositories Tested:** ithacaxyz-account solady ## Summary @@ -22,28 +22,31 @@ The following benchmarks were performed: Mean execution time in seconds (lower is better): -| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | -| --------------------- | ---------: | ---------------------------------------------------: | ----------: | -| **ithacaxyz-account** | 4.662 | 3.738 | 5.588 | -| **solady** | 3.559 | 2.933 | 3.517 | +| Project | stable (s) | nightly (s) | +|------|--------:|--------:| +| **ithacaxyz-account** | 5.791 | 3.875 | +| **solady** | 3.578 | 2.966 | + ### forge build no cache Mean execution time in seconds (lower is better): -| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | -| --------------------- | ---------: | ---------------------------------------------------: | ----------: | -| **ithacaxyz-account** | 10.777 | 10.982 | 10.979 | -| **solady** | 17.486 | 17.139 | 17.509 | +| Project | stable (s) | nightly (s) | +|------|--------:|--------:| +| **ithacaxyz-account** | 19.079 | 16.177 | +| **solady** | 27.408 | 22.745 | + ### forge build with cache Mean execution time in seconds (lower is better): -| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | -| --------------------- | ---------: | ---------------------------------------------------: | ----------: | -| **ithacaxyz-account** | 0.111 | 0.113 | 0.158 | -| **solady** | 0.084 | 0.089 | 0.108 | +| Project | stable (s) | nightly (s) | +|------|--------:|--------:| +| **ithacaxyz-account** | 0.181 | 0.158 | +| **solady** | 0.091 | 0.103 | + ## Foundry Version Details @@ -53,18 +56,13 @@ Mean execution time in seconds (lower is better): forge Version: 1.2.3-stable ``` -### nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae - -``` -forge Version: 1.2.3-nightly -``` - ### nightly ``` forge Version: 1.2.3-nightly ``` + ## Notes - All benchmarks were run with hyperfine in parallel mode @@ -72,14 +70,15 @@ forge Version: 1.2.3-nightly - **forge build (no cache) - Clean build without cache (5 runs, cache cleaned after each run)** - **forge build (with cache) - Build with warm cache (5 runs, 1 warmup)** - Results show mean execution time in seconds -- N/A indicates benchmark failed. +- N/A indicates benchmark failed or data unavailable ## System Information - **OS:** Darwin - **Architecture:** arm64 -- **Date:** Thu 12 Jun 2025 16:57:21 CEST +- **Date:** Wed 18 Jun 2025 17:46:19 BST ## Raw Data -Raw JSON benchmark data is available in: `/Users/yash/dev/paradigm/foundry-rs/foundry/benches/benchmark_results/json_20250612_165120` +Raw JSON benchmark data is available in: `/Users/yash/dev/paradigm/foundry-rs/foundry/benches/benchmark_results/json_20250618_174101` + diff --git a/benches/benchmark.sh b/benches/benchmark.sh index dc71db4606d7a..c16b8cb97e720 100755 --- a/benches/benchmark.sh +++ b/benches/benchmark.sh @@ -76,6 +76,47 @@ install_foundry_version() { fi } +# Switch to a specific installed Foundry version +use_foundry_version() { + local version=$1 + log_info "Switching to Foundry version: $version" + + if foundryup --use "$version"; then + # Verify switch + local current_version=$(forge --version | head -n1 || echo "unknown") + log_success "Now using Foundry: $current_version" + return 0 + else + log_error "Failed to switch to Foundry $version" + return 1 + fi +} + +# Install all required Foundry versions upfront +install_all_foundry_versions() { + log_info "Installing all required Foundry versions as preprocessing step..." + + local failed_versions=() + + for version in "${FOUNDRY_VERSIONS[@]}"; do + if ! install_foundry_version "$version"; then + failed_versions+=("$version") + fi + done + + if [ ${#failed_versions[@]} -ne 0 ]; then + log_error "Failed to install the following Foundry versions: ${failed_versions[*]}" + log_error "Please check the version names and try again" + exit 1 + fi + + log_success "All Foundry versions installed successfully!" + + # List all installed versions for verification + log_info "Available installed versions:" + foundryup --list || log_warn "Could not list installed versions" +} + # Check if required tools are installed check_dependencies() { local missing_deps=() @@ -211,11 +252,11 @@ benchmark_repository_for_version() { # Run benchmarks for all repositories in parallel for each Foundry version benchmark_all_repositories_parallel() { for version in "${FOUNDRY_VERSIONS[@]}"; do - log_info "Installing Foundry version: $version" + log_info "Switching to Foundry version: $version" - # Install the specific version once for all repositories - install_foundry_version "$version" || { - log_warn "Failed to install Foundry $version, skipping all repositories for this version..." + # Switch to the pre-installed version + use_foundry_version "$version" || { + log_warn "Failed to switch to Foundry $version, skipping all repositories for this version..." continue } @@ -443,6 +484,9 @@ main() { # Ensure cleanup on exit trap cleanup EXIT + # Install all Foundry versions upfront (preprocessing step) + install_all_foundry_versions + # Clone/update repositories for i in "${!REPO_NAMES[@]}"; do clone_or_update_repo "${REPO_NAMES[$i]}" "${REPO_URLS[$i]}" @@ -506,9 +550,9 @@ parse_args() { echo "" echo "The script will:" echo " 1. Install foundryup if not present" - echo " 2. Clone/update target repositories" - echo " 3. Install each specified Foundry version" - echo " 4. Run benchmarks for each repo with each version in parallel" + echo " 2. Install all specified Foundry versions (preprocessing step)" + echo " 3. Clone/update target repositories" + echo " 4. Switch between versions and run benchmarks in parallel" echo " 5. Generate comparison tables in markdown format" echo " 6. Save results to LATEST.md" exit 0 diff --git a/benches/repos_and_versions.sh b/benches/repos_and_versions.sh index 29f47e14a63a7..07da480b05d2d 100755 --- a/benches/repos_and_versions.sh +++ b/benches/repos_and_versions.sh @@ -11,7 +11,6 @@ # - Any format supported by foundryup FOUNDRY_VERSIONS=( "stable" - "nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae" "nightly" ) From c1e52ddda3795d88e56bbfd7d9eb4c66a026fd44 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 18 Jun 2025 18:00:39 +0100 Subject: [PATCH 07/74] nit --- benches/benchmark.sh | 70 ++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/benches/benchmark.sh b/benches/benchmark.sh index c16b8cb97e720..3d1e92aa62d9d 100755 --- a/benches/benchmark.sh +++ b/benches/benchmark.sh @@ -6,6 +6,41 @@ set -e +# Main execution +main() { + log_info "Starting Foundry Multi-Version Benchmarking Suite..." + log_info "Testing Foundry versions: ${FOUNDRY_VERSIONS[*]}" + log_info "Testing repositories: ${REPO_NAMES[*]}" + + # Setup + check_dependencies + setup_directories + + # Ensure cleanup on exit + trap cleanup EXIT + + # Install all Foundry versions upfront + install_all_foundry_versions + + # Clone/update repositories + for i in "${!REPO_NAMES[@]}"; do + clone_or_update_repo "${REPO_NAMES[$i]}" "${REPO_URLS[$i]}" + install_dependencies "${BENCHMARK_DIR}/${REPO_NAMES[$i]}" "${REPO_NAMES[$i]}" + done + + # Run benchmarks in parallel + benchmark_all_repositories_parallel + + # Compile results + compile_results + + log_success "Benchmarking complete!" + log_success "Results saved to: $RESULTS_FILE" + log_success "Latest results: $LATEST_RESULTS_FILE" + log_success "Raw JSON data saved to: $JSON_RESULTS_DIR" + log_info "You can view the results with: cat $LATEST_RESULTS_FILE" +} + # Get the directory where this script is located SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -471,41 +506,6 @@ cleanup() { log_info "Cleanup completed" } -# Main execution -main() { - log_info "Starting Foundry Multi-Version Benchmarking Suite..." - log_info "Testing Foundry versions: ${FOUNDRY_VERSIONS[*]}" - log_info "Testing repositories: ${REPO_NAMES[*]}" - - # Setup - check_dependencies - setup_directories - - # Ensure cleanup on exit - trap cleanup EXIT - - # Install all Foundry versions upfront (preprocessing step) - install_all_foundry_versions - - # Clone/update repositories - for i in "${!REPO_NAMES[@]}"; do - clone_or_update_repo "${REPO_NAMES[$i]}" "${REPO_URLS[$i]}" - install_dependencies "${BENCHMARK_DIR}/${REPO_NAMES[$i]}" "${REPO_NAMES[$i]}" - done - - # Run benchmarks in parallel - benchmark_all_repositories_parallel - - # Compile results - compile_results - - log_success "Benchmarking complete!" - log_success "Results saved to: $RESULTS_FILE" - log_success "Latest results: $LATEST_RESULTS_FILE" - log_success "Raw JSON data saved to: $JSON_RESULTS_DIR" - log_info "You can view the results with: cat $LATEST_RESULTS_FILE" -} - # Parse command line arguments parse_args() { while [[ $# -gt 0 ]]; do From 0dc81870554a6ec2f9f9a3fda180afe47a5e3897 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 25 Jun 2025 16:56:18 +0530 Subject: [PATCH 08/74] - setup benchmark repos in parallel - run forge build in parallet for forge-test bench - switch foundry versions - README specifying prereqs --- Cargo.lock | 1 + benches/Cargo.toml | 1 + benches/README.md | 88 +++++++++++++++++++++++++++++++ benches/forge_build_no_cache.rs | 23 +++++--- benches/forge_build_with_cache.rs | 27 +++++++--- benches/forge_test.rs | 28 ++++++---- benches/src/lib.rs | 22 ++++++-- 7 files changed, 162 insertions(+), 28 deletions(-) create mode 100644 benches/README.md diff --git a/Cargo.lock b/Cargo.lock index 812228968ba4b..98999f6f16ed0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4090,6 +4090,7 @@ dependencies = [ "foundry-compilers", "foundry-config", "foundry-test-utils", + "rayon", "serde", "serde_json", "tempfile", diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 4d13048ad860b..faa0a13a04d63 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -29,6 +29,7 @@ serde_json.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["full"] } chrono = { version = "0.4", features = ["serde"] } +rayon.workspace = true [dev-dependencies] foundry-test-utils.workspace = true \ No newline at end of file diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 0000000000000..8bdfcc9f63ea2 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,88 @@ +# Foundry Benchmarks + +This directory contains performance benchmarks for Foundry commands across multiple repositories and Foundry versions. + +## Prerequisites + +Before running the benchmarks, ensure you have the following installed: + +1. **Rust and Cargo** - Required for building and running the benchmarks + ```bash + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + ``` + +2. **Foundryup** - The Foundry toolchain installer + ```bash + curl -L https://foundry.paradigm.xyz | bash + foundryup + ``` + +3. **Required Foundry Versions** - Install all versions defined in `src/lib.rs` (see `FOUNDRY_VERSIONS`) + ```bash + foundryup --install stable + foundryup --install nightly + # Install any additional versions you add to FOUNDRY_VERSIONS in src/lib.rs + ``` + +4. **Git** - For cloning benchmark repositories + +5. **npm** - Some repositories require npm dependencies + ```bash + # Install Node.js and npm from https://nodejs.org/ + ``` + +## Running Benchmarks + +### Run all benchmarks +```bash +cargo bench +``` + +### Run specific benchmark +```bash +cargo bench forge_test +cargo bench forge_build_no_cache +cargo bench forge_build_with_cache +``` + +### Generate HTML reports +Criterion automatically generates HTML reports in `target/criterion/`. Open the reports in a browser: +```bash +open target/criterion/report/index.html +``` + +## Benchmark Structure + +- `forge_test` - Benchmarks `forge test` command across repos +- `forge_build_no_cache` - Benchmarks `forge build` with clean cache +- `forge_build_with_cache` - Benchmarks `forge build` with existing cache + +## Configuration + +### Repositories +Edit `src/lib.rs` to modify the list of repositories to benchmark: +```rust +pub static BENCHMARK_REPOS: &[RepoConfig] = &[ + RepoConfig { name: "account", org: "ithacaxyz", repo: "account", rev: "main" }, + // Add more repositories here +]; +``` + +### Foundry Versions +Edit `src/lib.rs` to modify the list of Foundry versions: +```rust +pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; +``` + +## Results + +Benchmark results are displayed in the terminal and saved as HTML reports. The reports show: +- Execution time statistics (mean, median, standard deviation) +- Comparison between different Foundry versions +- Performance trends across repositories + +## Troubleshooting + +1. **Foundry version not found**: Ensure the version is installed with `foundryup --install ` +2. **Repository clone fails**: Check network connectivity and repository access +3. **Build failures**: Some repositories may have specific dependencies - check their README files \ No newline at end of file diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs index 51eb1219ff0e6..a84ce5c691b54 100644 --- a/benches/forge_build_no_cache.rs +++ b/benches/forge_build_no_cache.rs @@ -1,18 +1,27 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{install_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; fn benchmark_forge_build_no_cache(c: &mut Criterion) { let mut group = c.benchmark_group("forge-build-no-cache"); group.sample_size(10); - for &version in FOUNDRY_VERSIONS { - // Install foundry version once per version - install_foundry_version(version).expect("Failed to install foundry version"); - - for repo_config in BENCHMARK_REPOS { - // Setup: prepare project OUTSIDE benchmark + // Setup all projects once - clone repos in parallel + let projects: Vec<_> = BENCHMARK_REPOS + .par_iter() + .map(|repo_config| { + // Setup: prepare project (clone repo) let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + (repo_config, project) + }) + .collect(); + + for &version in FOUNDRY_VERSIONS { + // Switch foundry version + switch_foundry_version(version).expect("Failed to switch foundry version"); + // Run benchmarks for each project + for (repo_config, project) in &projects { // Format: table_name/column_name/row_name // This creates: forge-build-no-cache/{version}/{repo_name} let bench_id = BenchmarkId::new(version, repo_config.name); diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs index bc0af1de0fb2c..33f788cdfbfa2 100644 --- a/benches/forge_build_with_cache.rs +++ b/benches/forge_build_with_cache.rs @@ -1,21 +1,32 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{install_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; fn benchmark_forge_build_with_cache(c: &mut Criterion) { let mut group = c.benchmark_group("forge-build-with-cache"); group.sample_size(10); - for &version in FOUNDRY_VERSIONS { - // Install foundry version once per version - install_foundry_version(version).expect("Failed to install foundry version"); - - for repo_config in BENCHMARK_REPOS { - // Setup: prepare project OUTSIDE benchmark + // Setup all projects once - clone repos in parallel + let projects: Vec<_> = BENCHMARK_REPOS + .par_iter() + .map(|repo_config| { + // Setup: prepare project (clone repo) let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + (repo_config, project) + }) + .collect(); + + for &version in FOUNDRY_VERSIONS { + // Switch foundry version once per version + switch_foundry_version(version).expect("Failed to switch foundry version"); - // Prime the cache OUTSIDE benchmark + // Prime the cache for all projects in parallel + projects.par_iter().for_each(|(repo_config, project)| { let _ = project.run_forge_build(false); + }); + // Run benchmarks for each project + for (repo_config, project) in &projects { // Format: table_name/column_name/row_name // This creates: forge-build-with-cache/{version}/{repo_name} let bench_id = BenchmarkId::new(version, repo_config.name); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index 08c46dcbbbe17..fab4d59184232 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -1,21 +1,31 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{install_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; - +use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; fn benchmark_forge_test(c: &mut Criterion) { let mut group = c.benchmark_group("forge-test"); group.sample_size(10); - for &version in FOUNDRY_VERSIONS { - // Install foundry version once per version - install_foundry_version(version).expect("Failed to install foundry version"); - - for repo_config in BENCHMARK_REPOS { - // Setup: prepare project OUTSIDE benchmark + // Setup all projects once - clone repos in parallel + let projects: Vec<_> = BENCHMARK_REPOS + .par_iter() + .map(|repo_config| { + // Setup: prepare project (clone repo) let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + (repo_config, project) + }) + .collect(); + + for &version in FOUNDRY_VERSIONS { + // Switch foundry version once per version + switch_foundry_version(version).expect("Failed to switch foundry version"); - // Build the project before running tests + // Build all projects in parallel for this foundry version + projects.par_iter().for_each(|(_repo_config, project)| { project.run_forge_build(false).expect("forge build failed"); + }); + // Run benchmarks for each project + for (repo_config, project) in &projects { // Format: table_name/column_name/row_name // This creates: forge-test/{version}/{repo_name} let bench_id = BenchmarkId::new(version, repo_config.name); diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 2c274690bc663..8994b06f745be 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -26,6 +26,20 @@ pub static BENCHMARK_REPOS: &[RepoConfig] = &[ ]; /// Foundry versions to benchmark +/// +/// To add more versions for comparison, install them first: +/// ```bash +/// foundryup --install stable +/// foundryup --install nightly +/// foundryup --install v0.2.0 # Example specific version +/// ``` +/// +/// Then add the version strings to this array. Supported formats: +/// - "stable" - Latest stable release +/// - "nightly" - Latest nightly build +/// - "v0.2.0" - Specific version tag +/// - "commit-hash" - Specific commit hash +/// - "nightly-" - Nightly build with specific revision pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; /// A benchmark project that represents a cloned repository ready for testing @@ -139,15 +153,15 @@ impl BenchmarkProject { } } -/// Install a specific foundry version -pub fn install_foundry_version(version: &str) -> Result<()> { +/// Switch to a specific foundry version +pub fn switch_foundry_version(version: &str) -> Result<()> { let status = Command::new("foundryup") - .args(["--install", version]) + .args(["--use", version]) .status() .wrap_err("Failed to run foundryup")?; if !status.success() { - eyre::bail!("Failed to install foundry version: {}", version); + eyre::bail!("Failed to switch to foundry version: {}", version); } Ok(()) From 9f13124fdb05b3652acc0e66d2f9764812c9b41f Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 25 Jun 2025 17:56:58 +0530 Subject: [PATCH 09/74] feat: shell script to run benches --- benches/LATEST.md | 108 ++++++----- benches/forge_build_no_cache.rs | 4 +- benches/forge_build_with_cache.rs | 8 +- benches/forge_test.rs | 4 +- benches/run_benchmarks.sh | 289 ++++++++++++++++++++++++++++++ benches/src/lib.rs | 8 +- 6 files changed, 370 insertions(+), 51 deletions(-) create mode 100755 benches/run_benchmarks.sh diff --git a/benches/LATEST.md b/benches/LATEST.md index b212d64452d7b..bc373cb9d4c5d 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,85 +1,107 @@ -# Forge Benchmarking Results +# Foundry Benchmarking Results -**Generated on:** Thu 12 Jun 2025 16:57:20 CEST -**Hyperfine Version:** hyperfine 1.19.0 -**Foundry Versions Tested:** stable nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae nightly -**Repositories Tested:** ithacaxyz-account solady +**Generated on:** Wed Jun 25 17:43:45 IST 2025 +**Tool:** Criterion.rs with criterion-table +**Foundry Versions Tested:** stable nightly +**Repositories Tested:** account solady v4-core morpho-blue spark-psm ## Summary -This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. +This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects using Criterion.rs for precise performance measurements. + The following benchmarks were performed: -1. **forge test - Running the test suite (5 runs, 1 warmup)** -2. **forge build (no cache) - Clean build without cache (5 runs, cache cleaned after each run)** -3. **forge build (with cache) - Build with warm cache (5 runs, 1 warmup)** +1. **forge-test** - Running the test suite (10 samples each) +2. **forge-build-no-cache** - Clean build without cache (10 samples each) +3. **forge-build-with-cache** - Build with warm cache (10 samples each) --- ## Performance Comparison Tables -### forge test - -Mean execution time in seconds (lower is better): - -| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | -| --------------------- | ---------: | ---------------------------------------------------: | ----------: | -| **ithacaxyz-account** | 4.662 | 3.738 | 5.588 | -| **solady** | 3.559 | 2.933 | 3.517 | +# Benchmarks -### forge build no cache +## Table of Contents -Mean execution time in seconds (lower is better): +- [Benchmark Results](#benchmark-results) + - [forge-build-with-cache](#forge-build-with-cache) -| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | -| --------------------- | ---------: | ---------------------------------------------------: | ----------: | -| **ithacaxyz-account** | 10.777 | 10.982 | 10.979 | -| **solady** | 17.486 | 17.139 | 17.509 | +## Benchmark Results -### forge build with cache +### forge-build-with-cache -Mean execution time in seconds (lower is better): +| | `stable` | `nightly` | +|:--------------|:--------------------------|:--------------------------------- | +| **`account`** | `164.00 ms` (✅ **1.00x**) | `166.34 ms` (✅ **1.01x slower**) | -| Project | stable (s) | nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae (s) | nightly (s) | -| --------------------- | ---------: | ---------------------------------------------------: | ----------: | -| **ithacaxyz-account** | 0.111 | 0.113 | 0.158 | -| **solady** | 0.084 | 0.089 | 0.108 | +--- +Made with [criterion-table](https://github.com/nu11ptr/criterion-table) +[INFO] Getting Foundry version information... ## Foundry Version Details ### stable ``` +foundryup: use - forge Version: 1.2.3-stable +Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f +Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) +Build Profile: maxperf +foundryup: use - cast Version: 1.2.3-stable +Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f +Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) +Build Profile: maxperf +foundryup: use - anvil Version: 1.2.3-stable +Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f +Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) +Build Profile: maxperf +foundryup: use - chisel Version: 1.2.3-stable +Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f +Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) +Build Profile: maxperf forge Version: 1.2.3-stable ``` -### nightly-ac0411d0e3b9632247c9aea9535472eda09a57ae - -``` -forge Version: 1.2.3-nightly -``` - ### nightly ``` +foundryup: use - forge Version: 1.2.3-nightly +Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f +Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) +Build Profile: maxperf +foundryup: use - cast Version: 1.2.3-nightly +Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f +Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) +Build Profile: maxperf +foundryup: use - anvil Version: 1.2.3-nightly +Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f +Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) +Build Profile: maxperf +foundryup: use - chisel Version: 1.2.3-nightly +Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f +Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) +Build Profile: maxperf forge Version: 1.2.3-nightly ``` ## Notes -- All benchmarks were run with hyperfine in parallel mode -- **forge test - Running the test suite (5 runs, 1 warmup)** -- **forge build (no cache) - Clean build without cache (5 runs, cache cleaned after each run)** -- **forge build (with cache) - Build with warm cache (5 runs, 1 warmup)** -- Results show mean execution time in seconds -- N/A indicates benchmark failed. +- All benchmarks use Criterion.rs for statistical analysis +- Each benchmark runs 10 samples by default +- Results show mean execution time with confidence intervals +- Repositories are cloned once and reused across all Foundry versions +- Build and setup operations are parallelized using Rayon +- The first version tested becomes the baseline for comparisons ## System Information - **OS:** Darwin - **Architecture:** arm64 -- **Date:** Thu 12 Jun 2025 16:57:21 CEST +- **Date:** Wed Jun 25 17:43:46 IST 2025 ## Raw Data -Raw JSON benchmark data is available in: `/Users/yash/dev/paradigm/foundry-rs/foundry/benches/benchmark_results/json_20250612_165120` +Detailed benchmark data and HTML reports are available in: +- `target/criterion/` - Individual benchmark reports +- `target/criterion/report/index.html` - Combined HTML report + diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs index a84ce5c691b54..4b4f1b94b125e 100644 --- a/benches/forge_build_no_cache.rs +++ b/benches/forge_build_no_cache.rs @@ -1,10 +1,10 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS, SAMPLE_SIZE}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; fn benchmark_forge_build_no_cache(c: &mut Criterion) { let mut group = c.benchmark_group("forge-build-no-cache"); - group.sample_size(10); + group.sample_size(SAMPLE_SIZE); // Setup all projects once - clone repos in parallel let projects: Vec<_> = BENCHMARK_REPOS diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs index 33f788cdfbfa2..e8df74732eda4 100644 --- a/benches/forge_build_with_cache.rs +++ b/benches/forge_build_with_cache.rs @@ -1,10 +1,12 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use foundry_bench::{ + switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS, SAMPLE_SIZE, +}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; fn benchmark_forge_build_with_cache(c: &mut Criterion) { let mut group = c.benchmark_group("forge-build-with-cache"); - group.sample_size(10); + group.sample_size(SAMPLE_SIZE); // Setup all projects once - clone repos in parallel let projects: Vec<_> = BENCHMARK_REPOS @@ -21,7 +23,7 @@ fn benchmark_forge_build_with_cache(c: &mut Criterion) { switch_foundry_version(version).expect("Failed to switch foundry version"); // Prime the cache for all projects in parallel - projects.par_iter().for_each(|(repo_config, project)| { + projects.par_iter().for_each(|(_repo_config, project)| { let _ = project.run_forge_build(false); }); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index fab4d59184232..6832aee005d0c 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -1,9 +1,9 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS, SAMPLE_SIZE}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; fn benchmark_forge_test(c: &mut Criterion) { let mut group = c.benchmark_group("forge-test"); - group.sample_size(10); + group.sample_size(SAMPLE_SIZE); // Setup all projects once - clone repos in parallel let projects: Vec<_> = BENCHMARK_REPOS diff --git a/benches/run_benchmarks.sh b/benches/run_benchmarks.sh new file mode 100755 index 0000000000000..ed2afc36fa9f2 --- /dev/null +++ b/benches/run_benchmarks.sh @@ -0,0 +1,289 @@ +#!/bin/bash + +# Foundry Benchmark Runner with Criterion Table Output +# This script runs the criterion-based benchmarks and generates a markdown report + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# Check if required tools are installed +check_dependencies() { + if ! command -v criterion-table &> /dev/null; then + log_error "criterion-table is not installed. Please install it with:" + echo "cargo install criterion-table" + exit 1 + fi + + if ! cargo criterion --help &> /dev/null; then + log_error "cargo-criterion is not installed. Please install it with:" + echo "cargo install cargo-criterion" + exit 1 + fi +} + +# Get system information +get_system_info() { + local os_name=$(uname -s) + local arch=$(uname -m) + local date=$(date) + + echo "- **OS:** $os_name" + echo "- **Architecture:** $arch" + echo "- **Date:** $date" +} + + +# Run benchmarks and generate report +run_benchmarks() { + log_info "Running Foundry benchmarks..." + + # Create temp files for each benchmark + local temp_dir=$(mktemp -d) + local forge_test_json="$temp_dir/forge_test.json" + local forge_build_no_cache_json="$temp_dir/forge_build_no_cache.json" + local forge_build_with_cache_json="$temp_dir/forge_build_with_cache.json" + + # Set up output redirection based on verbose flag + local output_redirect="" + if [[ "${VERBOSE:-false}" != "true" ]]; then + output_redirect="2>/dev/null" + fi + + # Run benchmarks in specific order (this determines baseline column) + log_info "Running forge_test benchmark..." + if [[ "${VERBOSE:-false}" == "true" ]]; then + cargo criterion --bench forge_test --message-format=json > "$forge_test_json" || { + log_error "forge_test benchmark failed" + exit 1 + } + else + cargo criterion --bench forge_test --message-format=json > "$forge_test_json" 2>/dev/null || { + log_error "forge_test benchmark failed" + exit 1 + } + fi + + log_info "Running forge_build_no_cache benchmark..." + if [[ "${VERBOSE:-false}" == "true" ]]; then + cargo criterion --bench forge_build_no_cache --message-format=json > "$forge_build_no_cache_json" || { + log_error "forge_build_no_cache benchmark failed" + exit 1 + } + else + cargo criterion --bench forge_build_no_cache --message-format=json > "$forge_build_no_cache_json" 2>/dev/null || { + log_error "forge_build_no_cache benchmark failed" + exit 1 + } + fi + + log_info "Running forge_build_with_cache benchmark..." + if [[ "${VERBOSE:-false}" == "true" ]]; then + cargo criterion --bench forge_build_with_cache --message-format=json > "$forge_build_with_cache_json" || { + log_error "forge_build_with_cache benchmark failed" + exit 1 + } + else + cargo criterion --bench forge_build_with_cache --message-format=json > "$forge_build_with_cache_json" 2>/dev/null || { + log_error "forge_build_with_cache benchmark failed" + exit 1 + } + fi + + # Combine all results and generate markdown + log_info "Generating markdown report with criterion-table..." + + # Generate the final report + generate_report "$temp_dir/tables.md" + + # Cleanup + rm -rf "$temp_dir" + + log_success "Benchmark report generated in LATEST.md" +} + +# Generate the final markdown report +generate_report() { + local tables_file="$1" + local report_file="LATEST.md" + + log_info "Generating final report..." + + # Get current timestamp + local timestamp=$(date) + + # Get repository information and create numbered list with links + local versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"' | tr '\n' ' ') + + # Extract repository info for numbered list + local repo_list="" + local counter=1 + + # Parse the BENCHMARK_REPOS section + while IFS= read -r line; do + if [[ $line =~ RepoConfig.*name:.*\"([^\"]+)\".*org:.*\"([^\"]+)\".*repo:.*\"([^\"]+)\" ]]; then + local name="${BASH_REMATCH[1]}" + local org="${BASH_REMATCH[2]}" + local repo="${BASH_REMATCH[3]}" + repo_list+="$counter. [$name](https://github.com/$org/$repo)\n" + ((counter++)) + fi + done < <(grep -A 20 'pub static BENCHMARK_REPOS' src/lib.rs | grep 'RepoConfig') + + # Write the report + cat > "$report_file" << EOF +# Foundry Benchmarking Results + +**Generated on:** $timestamp +**Foundry Versions Tested:** $versions + +## Repositories Tested + +$(echo -e "$repo_list") + +## Summary + +This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects using Criterion.rs for precise performance measurements. + +The following benchmarks were performed: + +1. **forge-test** - Running the test suite (10 samples each) +2. **forge-build-no-cache** - Clean build without cache (10 samples each) +3. **forge-build-with-cache** - Build with warm cache (10 samples each) + +--- + +## Performance Comparison Tables + +EOF + + # Append the criterion-table generated tables + cat "$tables_file" >> "$report_file" + + # Add notes and system info + cat >> "$report_file" << EOF +## Notes + +- All benchmarks use Criterion.rs for statistical analysis +- Each benchmark runs 10 samples by default +- Results show mean execution time with confidence intervals +- Repositories are cloned once and reused across all Foundry versions +- Build and setup operations are parallelized using Rayon +- The first version tested becomes the baseline for comparisons + +## System Information + +$(get_system_info) + +## Raw Data + +Detailed benchmark data and HTML reports are available in: +- \`target/criterion/\` - Individual benchmark reports + +EOF + + log_success "Report written to $report_file" +} + +# Main function +main() { + log_info "Starting Foundry benchmark suite..." + + # Check dependencies + check_dependencies + + # Run benchmarks and generate report + run_benchmarks + + log_success "Benchmark suite completed successfully!" + echo "" + echo "View the results:" + echo " - Text report: cat LATEST.md" + echo " - HTML report: open target/criterion/report/index.html" +} + +# Help function +show_help() { + cat << EOF +Foundry Benchmark Runner + +This script runs Criterion-based benchmarks for Foundry commands and generates +a markdown report using criterion-table. + +USAGE: + $0 [OPTIONS] + +OPTIONS: + -h, --help Show this help message + -v, --version Show version information + --verbose Show benchmark output (by default output is suppressed) + +REQUIREMENTS: + - criterion-table: cargo install criterion-table + - cargo-criterion: cargo install cargo-criterion + - All Foundry versions defined in src/lib.rs must be installed + +EXAMPLES: + $0 # Run all benchmarks and generate LATEST.md + $0 --verbose # Run benchmarks with full output visible + +The script will: +1. Run forge_test, forge_build_no_cache, and forge_build_with_cache benchmarks +2. Generate comparison tables using criterion-table +3. Include system information and Foundry version details +4. Save the complete report to LATEST.md + +EOF +} + +# Parse command line arguments +VERBOSE=false +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + show_help + exit 0 + ;; + -v|--version) + echo "Foundry Benchmark Runner v1.0.0" + exit 0 + ;; + --verbose) + VERBOSE=true + shift + ;; + *) + log_error "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +main \ No newline at end of file diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 8994b06f745be..5d5814103a6e8 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -17,7 +17,7 @@ pub struct RepoConfig { /// Available repositories for benchmarking pub static BENCHMARK_REPOS: &[RepoConfig] = &[ - RepoConfig { name: "account", org: "ithacaxyz", repo: "account", rev: "main" }, + RepoConfig { name: "ithacaxyz-account", org: "ithacaxyz", repo: "account", rev: "main" }, // Temporarily reduced for testing // RepoConfig { name: "solady", org: "Vectorized", repo: "solady", rev: "main" }, // RepoConfig { name: "v4-core", org: "Uniswap", repo: "v4-core", rev: "main" }, @@ -25,6 +25,12 @@ pub static BENCHMARK_REPOS: &[RepoConfig] = &[ // RepoConfig { name: "spark-psm", org: "marsfoundation", repo: "spark-psm", rev: "master" }, ]; +/// Sample size for benchmark measurements +/// +/// This controls how many times each benchmark is run for statistical analysis. +/// Higher values provide more accurate results but take longer to complete. +pub const SAMPLE_SIZE: usize = 10; + /// Foundry versions to benchmark /// /// To add more versions for comparison, install them first: From 7d1d85a3f4a0ee1b8b00a337e7e4155701c31c28 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 25 Jun 2025 18:34:05 +0530 Subject: [PATCH 10/74] feat: ci workflow, fix script --- .github/workflows/benchmarks.yml | 108 +++++++++++++++++++++++++++++++ benches/LATEST.md | 82 ++++++++--------------- benches/README.md | 24 ++++--- benches/run_benchmarks.sh | 46 ++++++++++++- 4 files changed, 192 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/benchmarks.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000000..d5c6e0cecf501 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,108 @@ +name: Foundry Benchmarks + +on: + workflow_dispatch: + inputs: + pr_number: + description: "PR number to comment on (optional)" + required: false + type: string + +permissions: + contents: write + pull-requests: write + +jobs: + benchmark: + name: Run Foundry Benchmarks + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + ./ + + - name: Install foundryup + run: | + curl -L https://foundry.paradigm.xyz | bash + echo "$HOME/.foundry/bin" >> $GITHUB_PATH + + - name: Install benchmark dependencies + run: | + cargo install cargo-criterion + cargo install criterion-table + + - name: Run benchmarks + working-directory: ./benches + run: | + chmod +x run_benchmarks.sh + ./run_benchmarks.sh + + - name: Commit benchmark results + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add benches/LATEST.md + if git diff --staged --quiet; then + echo "No changes to commit" + else + git commit -m "Update benchmark results + + 🤖 Generated with [Foundry Benchmarks](https://github.com/${{ github.repository }}/actions) + + Co-Authored-By: github-actions " + git push + fi + + - name: Read benchmark results + id: benchmark_results + run: | + if [ -f "benches/LATEST.md" ]; then + { + echo 'results<> $GITHUB_OUTPUT + else + echo 'results=No benchmark results found.' >> $GITHUB_OUTPUT + fi + + - name: Comment on PR + if: github.event.inputs.pr_number != '' + uses: actions/github-script@v7 + with: + script: | + const prNumber = ${{ github.event.inputs.pr_number }}; + const benchmarkResults = `${{ steps.benchmark_results.outputs.results }}`; + + const comment = `## 📊 Foundry Benchmark Results + +
+ Click to view detailed benchmark results + + ${benchmarkResults} + +
+ + --- + + 🤖 This comment was automatically generated by the [Foundry Benchmarks workflow](https://github.com/${{ github.repository }}/actions). + + To run benchmarks manually: Go to [Actions](https://github.com/${{ github.repository }}/actions/workflows/foundry-benchmarks.yml) → "Run workflow"`; + + github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + diff --git a/benches/LATEST.md b/benches/LATEST.md index bc373cb9d4c5d..9d6be5a7291d0 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,9 +1,15 @@ # Foundry Benchmarking Results -**Generated on:** Wed Jun 25 17:43:45 IST 2025 -**Tool:** Criterion.rs with criterion-table +**Generated on:** Wed Jun 25 18:27:00 IST 2025 **Foundry Versions Tested:** stable nightly -**Repositories Tested:** account solady v4-core morpho-blue spark-psm + +## Repositories Tested + +1. [ithacaxyz-account](https://github.com/ithacaxyz/main) +2. [solady](https://github.com/Vectorized/main) +3. [v4-core](https://github.com/Uniswap/main) +4. [morpho-blue](https://github.com/morpho-org/main) +5. [spark-psm](https://github.com/marsfoundation/master) ## Summary @@ -24,66 +30,33 @@ The following benchmarks were performed: ## Table of Contents - [Benchmark Results](#benchmark-results) + - [forge-test](#forge-test) + - [forge-build-no-cache](#forge-build-no-cache) - [forge-build-with-cache](#forge-build-with-cache) ## Benchmark Results +### forge-test + +| | `stable` | `nightly` | +|:------------------------|:-----------------------|:------------------------------ | +| **`ithacaxyz-account`** | `3.73 s` (✅ **1.00x**) | `3.30 s` (✅ **1.13x faster**) | + +### forge-build-no-cache + +| | `stable` | `nightly` | +|:------------------------|:------------------------|:------------------------------- | +| **`ithacaxyz-account`** | `14.32 s` (✅ **1.00x**) | `14.37 s` (✅ **1.00x slower**) | + ### forge-build-with-cache -| | `stable` | `nightly` | -|:--------------|:--------------------------|:--------------------------------- | -| **`account`** | `164.00 ms` (✅ **1.00x**) | `166.34 ms` (✅ **1.01x slower**) | +| | `stable` | `nightly` | +|:------------------------|:--------------------------|:--------------------------------- | +| **`ithacaxyz-account`** | `162.64 ms` (✅ **1.00x**) | `167.49 ms` (✅ **1.03x slower**) | --- Made with [criterion-table](https://github.com/nu11ptr/criterion-table) -[INFO] Getting Foundry version information... -## Foundry Version Details - -### stable - -``` -foundryup: use - forge Version: 1.2.3-stable -Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f -Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) -Build Profile: maxperf -foundryup: use - cast Version: 1.2.3-stable -Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f -Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) -Build Profile: maxperf -foundryup: use - anvil Version: 1.2.3-stable -Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f -Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) -Build Profile: maxperf -foundryup: use - chisel Version: 1.2.3-stable -Commit SHA: a813a2cee7dd4926e7c56fd8a785b54f32e0d10f -Build Timestamp: 2025-06-08T15:42:50.507050000Z (1749397370) -Build Profile: maxperf -forge Version: 1.2.3-stable -``` - -### nightly - -``` -foundryup: use - forge Version: 1.2.3-nightly -Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f -Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) -Build Profile: maxperf -foundryup: use - cast Version: 1.2.3-nightly -Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f -Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) -Build Profile: maxperf -foundryup: use - anvil Version: 1.2.3-nightly -Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f -Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) -Build Profile: maxperf -foundryup: use - chisel Version: 1.2.3-nightly -Commit SHA: b515c90b9be9645b844943fc6d54f2304b83f75f -Build Timestamp: 2025-06-18T06:02:35.553006000Z (1750226555) -Build Profile: maxperf -forge Version: 1.2.3-nightly -``` - ## Notes - All benchmarks use Criterion.rs for statistical analysis @@ -97,11 +70,10 @@ forge Version: 1.2.3-nightly - **OS:** Darwin - **Architecture:** arm64 -- **Date:** Wed Jun 25 17:43:46 IST 2025 +- **Date:** Wed Jun 25 18:27:01 IST 2025 ## Raw Data Detailed benchmark data and HTML reports are available in: - `target/criterion/` - Individual benchmark reports -- `target/criterion/report/index.html` - Combined HTML report diff --git a/benches/README.md b/benches/README.md index 8bdfcc9f63ea2..f410e2bcdcba3 100644 --- a/benches/README.md +++ b/benches/README.md @@ -7,26 +7,21 @@ This directory contains performance benchmarks for Foundry commands across multi Before running the benchmarks, ensure you have the following installed: 1. **Rust and Cargo** - Required for building and running the benchmarks + ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` 2. **Foundryup** - The Foundry toolchain installer + ```bash curl -L https://foundry.paradigm.xyz | bash foundryup ``` -3. **Required Foundry Versions** - Install all versions defined in `src/lib.rs` (see `FOUNDRY_VERSIONS`) - ```bash - foundryup --install stable - foundryup --install nightly - # Install any additional versions you add to FOUNDRY_VERSIONS in src/lib.rs - ``` +3. **Git** - For cloning benchmark repositories -4. **Git** - For cloning benchmark repositories - -5. **npm** - Some repositories require npm dependencies +4. **npm** - Some repositories require npm dependencies ```bash # Install Node.js and npm from https://nodejs.org/ ``` @@ -34,11 +29,13 @@ Before running the benchmarks, ensure you have the following installed: ## Running Benchmarks ### Run all benchmarks + ```bash cargo bench ``` ### Run specific benchmark + ```bash cargo bench forge_test cargo bench forge_build_no_cache @@ -46,7 +43,9 @@ cargo bench forge_build_with_cache ``` ### Generate HTML reports + Criterion automatically generates HTML reports in `target/criterion/`. Open the reports in a browser: + ```bash open target/criterion/report/index.html ``` @@ -60,7 +59,9 @@ open target/criterion/report/index.html ## Configuration ### Repositories + Edit `src/lib.rs` to modify the list of repositories to benchmark: + ```rust pub static BENCHMARK_REPOS: &[RepoConfig] = &[ RepoConfig { name: "account", org: "ithacaxyz", repo: "account", rev: "main" }, @@ -69,7 +70,9 @@ pub static BENCHMARK_REPOS: &[RepoConfig] = &[ ``` ### Foundry Versions + Edit `src/lib.rs` to modify the list of Foundry versions: + ```rust pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; ``` @@ -77,6 +80,7 @@ pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; ## Results Benchmark results are displayed in the terminal and saved as HTML reports. The reports show: + - Execution time statistics (mean, median, standard deviation) - Comparison between different Foundry versions - Performance trends across repositories @@ -85,4 +89,4 @@ Benchmark results are displayed in the terminal and saved as HTML reports. The r 1. **Foundry version not found**: Ensure the version is installed with `foundryup --install ` 2. **Repository clone fails**: Check network connectivity and repository access -3. **Build failures**: Some repositories may have specific dependencies - check their README files \ No newline at end of file +3. **Build failures**: Some repositories may have specific dependencies - check their README files diff --git a/benches/run_benchmarks.sh b/benches/run_benchmarks.sh index ed2afc36fa9f2..1a30506a9fdc8 100755 --- a/benches/run_benchmarks.sh +++ b/benches/run_benchmarks.sh @@ -48,6 +48,41 @@ check_dependencies() { fi } +# Check and install all required Foundry versions +check_and_install_foundry() { + log_info "Checking and installing required Foundry versions..." + + # Read the versions from the Rust source + local versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"') + + # Check if foundryup is available + if ! command -v foundryup &> /dev/null; then + log_error "foundryup not found. Please install Foundry first:" + echo "curl -L https://foundry.paradigm.xyz | bash" + exit 1 + fi + + # Install each version if not already available + for version in $versions; do + log_info "Checking Foundry version: $version" + + # Try to switch to the version to check if it's installed + if foundryup --use "$version" 2>/dev/null; then + log_info "✓ Version $version is already installed" + else + log_info "Installing Foundry version: $version" + if foundryup --install "$version"; then + log_success "✓ Successfully installed version $version" + else + log_error "Failed to install Foundry version: $version" + exit 1 + fi + fi + done + + log_success "All required Foundry versions are available" +} + # Get system information get_system_info() { local os_name=$(uname -s) @@ -118,6 +153,11 @@ run_benchmarks() { # Combine all results and generate markdown log_info "Generating markdown report with criterion-table..." + + if ! cat "$forge_test_json" "$forge_build_no_cache_json" "$forge_build_with_cache_json" | criterion-table > "$temp_dir/tables.md"; then + log_error "criterion-table failed to process benchmark data" + exit 1 + fi # Generate the final report generate_report "$temp_dir/tables.md" @@ -179,8 +219,6 @@ The following benchmarks were performed: --- -## Performance Comparison Tables - EOF # Append the criterion-table generated tables @@ -218,6 +256,9 @@ main() { # Check dependencies check_dependencies + # Check and install required Foundry versions + check_and_install_foundry + # Run benchmarks and generate report run_benchmarks @@ -225,7 +266,6 @@ main() { echo "" echo "View the results:" echo " - Text report: cat LATEST.md" - echo " - HTML report: open target/criterion/report/index.html" } # Help function From b6671063934b3fd60439a79550d6a5dfad4ef655 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 26 Jun 2025 12:45:17 +0530 Subject: [PATCH 11/74] update readme --- benches/README.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/benches/README.md b/benches/README.md index f410e2bcdcba3..b3663831853b1 100644 --- a/benches/README.md +++ b/benches/README.md @@ -22,32 +22,44 @@ Before running the benchmarks, ensure you have the following installed: 3. **Git** - For cloning benchmark repositories 4. **npm** - Some repositories require npm dependencies + ```bash # Install Node.js and npm from https://nodejs.org/ ``` +5. **Benchmark tools** - Required for generating reports + ```bash + cargo install cargo-criterion + cargo install criterion-table + ``` + ## Running Benchmarks -### Run all benchmarks +### Run the complete benchmark suite ```bash -cargo bench +cargo run ``` -### Run specific benchmark +This will: + +1. Check and install required Foundry versions +2. Run all benchmark suites (forge_test, forge_build_no_cache, forge_build_with_cache) +3. Generate comparison tables using criterion-table +4. Create the final LATEST.md report + +### Run individual benchmark suites ```bash -cargo bench forge_test -cargo bench forge_build_no_cache -cargo bench forge_build_with_cache +./run_benchmarks.sh ``` -### Generate HTML reports - -Criterion automatically generates HTML reports in `target/criterion/`. Open the reports in a browser: +### Run specific benchmark ```bash -open target/criterion/report/index.html +cargo criterion --bench forge_test +cargo criterion --bench forge_build_no_cache +cargo criterion --bench forge_build_with_cache ``` ## Benchmark Structure From a3962020938a754b4af39d7edf6114acc976bfa3 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Jun 2025 15:41:45 +0530 Subject: [PATCH 12/74] feat: enhance benchmarking suite with version flexibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `get_benchmark_versions()` helper to read versions from env var - Update all benchmarks to use version helper for consistency - Add `--versions` and `--force-install` flags to shell script - Enable all three benchmarks (forge_test, build_no_cache, build_with_cache) - Improve error handling for corrupted forge installations - Remove complex workarounds in favor of clear error messages The benchmarks now support custom versions via: ./run_benchmarks.sh --versions stable,nightly,v1.2.0 🤖 Generated with Claude Code Co-Authored-By: Claude --- Cargo.lock | 14 ++--- benches/Cargo.toml | 2 +- benches/LATEST.md | 44 ++++++-------- benches/forge_build_no_cache.rs | 13 +++-- benches/forge_build_with_cache.rs | 15 ++--- benches/forge_test.rs | 14 +++-- benches/run_benchmarks.sh | 96 +++++++++++++++++++++---------- benches/src/lib.rs | 66 +++++++++++++-------- 8 files changed, 163 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1b8c84e76151..9a469d70d4a4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -588,7 +588,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", - "itertools 0.13.0", + "itertools 0.14.0", "serde", "serde_json", "thiserror 2.0.12", @@ -4332,7 +4332,7 @@ dependencies = [ "fs_extra", "futures-util", "home", - "itertools 0.13.0", + "itertools 0.14.0", "path-slash", "rand 0.8.5", "rayon", @@ -7263,7 +7263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.103", @@ -8795,7 +8795,7 @@ dependencies = [ "derive_builder", "derive_more 2.0.1", "dunce", - "itertools 0.13.0", + "itertools 0.14.0", "itoa", "lasso", "match_cfg", @@ -8807,7 +8807,7 @@ dependencies = [ "solar-config", "solar-data-structures", "solar-macros", - "thiserror 1.0.69", + "thiserror 2.0.12", "tracing", "unicode-width 0.2.0", ] @@ -8832,7 +8832,7 @@ dependencies = [ "alloy-primitives", "bitflags 2.9.1", "bumpalo", - "itertools 0.13.0", + "itertools 0.14.0", "memchr", "num-bigint", "num-rational", @@ -9143,7 +9143,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.12", "url", "zip", ] diff --git a/benches/Cargo.toml b/benches/Cargo.toml index faa0a13a04d63..81c3c71e3a829 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -32,4 +32,4 @@ chrono = { version = "0.4", features = ["serde"] } rayon.workspace = true [dev-dependencies] -foundry-test-utils.workspace = true \ No newline at end of file +foundry-test-utils.workspace = true diff --git a/benches/LATEST.md b/benches/LATEST.md index 6fde175ce3364..354ca2f054897 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,7 +1,7 @@ # Foundry Benchmarking Results -**Generated on:** Wed Jun 25 18:27:00 IST 2025 -**Foundry Versions Tested:** stable nightly +**Generated on:** Fri 27 Jun 2025 15:16:52 IST +**Foundry Versions Tested:** stable nightly ## Repositories Tested @@ -18,54 +18,46 @@ This report contains comprehensive benchmarking results comparing different Foun The following benchmarks were performed: 1. **forge-test** - Running the test suite (10 samples each) -2. **forge-build-no-cache** - Clean build without cache (10 samples each) +2. **forge-build-no-cache** - Clean build without cache (10 samples each) 3. **forge-build-with-cache** - Build with warm cache (10 samples each) --- -## Performance Comparison Tables - # Benchmarks ## Table of Contents - [Benchmark Results](#benchmark-results) - - [forge-test](#forge-test) - - [forge-build-no-cache](#forge-build-no-cache) - - [forge-build-with-cache](#forge-build-with-cache) + - [forge-build-with-cache](#forge-build-with-cache) ## Benchmark Results -### forge-test - -| | `stable` | `nightly` | -| :---------------------- | :---------------------- | :----------------------------- | -| **`ithacaxyz-account`** | `3.73 s` (✅ **1.00x**) | `3.30 s` (✅ **1.13x faster**) | - -### forge-build-no-cache - -| | `stable` | `nightly` | -| :---------------------- | :----------------------- | :------------------------------ | -| **`ithacaxyz-account`** | `14.32 s` (✅ **1.00x**) | `14.37 s` (✅ **1.00x slower**) | - ### forge-build-with-cache -| | `stable` | `nightly` | -| :---------------------- | :------------------------- | :-------------------------------- | -| **`ithacaxyz-account`** | `162.64 ms` (✅ **1.00x**) | `167.49 ms` (✅ **1.03x slower**) | +| | `stable` | `nightly` | +|:------------------------|:--------------------------|:--------------------------------- | +| **`ithacaxyz-account`** | `166.32 ms` (✅ **1.00x**) | `171.37 ms` (✅ **1.03x slower**) | --- - Made with [criterion-table](https://github.com/nu11ptr/criterion-table) +## Notes + +- All benchmarks use Criterion.rs for statistical analysis +- Each benchmark runs 10 samples by default +- Results show mean execution time with confidence intervals +- Repositories are cloned once and reused across all Foundry versions +- Build and setup operations are parallelized using Rayon +- The first version tested becomes the baseline for comparisons + ## System Information - **OS:** Darwin - **Architecture:** arm64 -- **Date:** Wed Jun 25 18:27:01 IST 2025 +- **Date:** Fri 27 Jun 2025 15:16:52 IST ## Raw Data Detailed benchmark data and HTML reports are available in: - - `target/criterion/` - Individual benchmark reports + diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs index 4b4f1b94b125e..f7b4a2d13f527 100644 --- a/benches/forge_build_no_cache.rs +++ b/benches/forge_build_no_cache.rs @@ -1,5 +1,7 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS, SAMPLE_SIZE}; +use foundry_bench::{ + get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, +}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; fn benchmark_forge_build_no_cache(c: &mut Criterion) { @@ -16,15 +18,18 @@ fn benchmark_forge_build_no_cache(c: &mut Criterion) { }) .collect(); - for &version in FOUNDRY_VERSIONS { + // Get versions from environment variable or default + let versions = get_benchmark_versions(); + + for version in versions { // Switch foundry version - switch_foundry_version(version).expect("Failed to switch foundry version"); + switch_foundry_version(&version).expect("Failed to switch foundry version"); // Run benchmarks for each project for (repo_config, project) in &projects { // Format: table_name/column_name/row_name // This creates: forge-build-no-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(version, repo_config.name); + let bench_id = BenchmarkId::new(&version, repo_config.name); group.bench_function(bench_id, |b| { b.iter(|| { diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs index e8df74732eda4..154dd8a7f6100 100644 --- a/benches/forge_build_with_cache.rs +++ b/benches/forge_build_with_cache.rs @@ -1,6 +1,6 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; use foundry_bench::{ - switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS, SAMPLE_SIZE, + get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, }; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; @@ -12,17 +12,18 @@ fn benchmark_forge_build_with_cache(c: &mut Criterion) { let projects: Vec<_> = BENCHMARK_REPOS .par_iter() .map(|repo_config| { - // Setup: prepare project (clone repo) let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); (repo_config, project) }) .collect(); - for &version in FOUNDRY_VERSIONS { + // Get versions from environment variable or default + let versions = get_benchmark_versions(); + + for version in versions { // Switch foundry version once per version - switch_foundry_version(version).expect("Failed to switch foundry version"); + switch_foundry_version(&version).expect("Failed to switch foundry version"); - // Prime the cache for all projects in parallel projects.par_iter().for_each(|(_repo_config, project)| { let _ = project.run_forge_build(false); }); @@ -31,10 +32,10 @@ fn benchmark_forge_build_with_cache(c: &mut Criterion) { for (repo_config, project) in &projects { // Format: table_name/column_name/row_name // This creates: forge-build-with-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(version, repo_config.name); - + let bench_id = BenchmarkId::new(&version, repo_config.name); group.bench_function(bench_id, |b| { b.iter(|| { + println!("Benching: forge-build-with-cache/{}/{}", version, repo_config.name); let output = project.run_forge_build(false).expect("forge build failed"); black_box(output); }); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index 6832aee005d0c..b12b30a067b47 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -1,6 +1,9 @@ use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, FOUNDRY_VERSIONS, SAMPLE_SIZE}; +use foundry_bench::{ + get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, +}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + fn benchmark_forge_test(c: &mut Criterion) { let mut group = c.benchmark_group("forge-test"); group.sample_size(SAMPLE_SIZE); @@ -15,9 +18,12 @@ fn benchmark_forge_test(c: &mut Criterion) { }) .collect(); - for &version in FOUNDRY_VERSIONS { + // Get versions from environment variable or default + let versions = get_benchmark_versions(); + + for version in versions { // Switch foundry version once per version - switch_foundry_version(version).expect("Failed to switch foundry version"); + switch_foundry_version(&version).expect("Failed to switch foundry version"); // Build all projects in parallel for this foundry version projects.par_iter().for_each(|(_repo_config, project)| { @@ -28,7 +34,7 @@ fn benchmark_forge_test(c: &mut Criterion) { for (repo_config, project) in &projects { // Format: table_name/column_name/row_name // This creates: forge-test/{version}/{repo_name} - let bench_id = BenchmarkId::new(version, repo_config.name); + let bench_id = BenchmarkId::new(&version, repo_config.name); group.bench_function(bench_id, |b| { b.iter(|| { diff --git a/benches/run_benchmarks.sh b/benches/run_benchmarks.sh index 1a30506a9fdc8..b9ddfe378c25d 100755 --- a/benches/run_benchmarks.sh +++ b/benches/run_benchmarks.sh @@ -48,12 +48,23 @@ check_dependencies() { fi } -# Check and install all required Foundry versions -check_and_install_foundry() { - log_info "Checking and installing required Foundry versions..." +# Install Foundry versions if requested +install_foundry_versions() { + if [[ "$FORCE_INSTALL" != "true" ]]; then + return + fi - # Read the versions from the Rust source - local versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"') + local versions + + # Use custom versions if provided, otherwise read from lib.rs + if [[ -n "$CUSTOM_VERSIONS" ]]; then + versions=$(echo "$CUSTOM_VERSIONS" | tr ',' ' ') + log_info "Installing custom Foundry versions: $versions" + else + # Read the versions from the Rust source + versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"') + log_info "Installing default Foundry versions from lib.rs: $versions" + fi # Check if foundryup is available if ! command -v foundryup &> /dev/null; then @@ -62,25 +73,18 @@ check_and_install_foundry() { exit 1 fi - # Install each version if not already available + # Install each version for version in $versions; do - log_info "Checking Foundry version: $version" - - # Try to switch to the version to check if it's installed - if foundryup --use "$version" 2>/dev/null; then - log_info "✓ Version $version is already installed" + log_info "Installing Foundry version: $version" + if foundryup --install "$version"; then + log_success "✓ Successfully installed version $version" else - log_info "Installing Foundry version: $version" - if foundryup --install "$version"; then - log_success "✓ Successfully installed version $version" - else - log_error "Failed to install Foundry version: $version" - exit 1 - fi + log_error "Failed to install Foundry version: $version" + exit 1 fi done - log_success "All required Foundry versions are available" + log_success "All Foundry versions installed successfully" } # Get system information @@ -99,6 +103,12 @@ get_system_info() { run_benchmarks() { log_info "Running Foundry benchmarks..." + # Set environment variable for custom versions if provided + if [[ -n "$CUSTOM_VERSIONS" ]]; then + export FOUNDRY_BENCH_VERSIONS="$CUSTOM_VERSIONS" + log_info "Set FOUNDRY_BENCH_VERSIONS=$CUSTOM_VERSIONS" + fi + # Create temp files for each benchmark local temp_dir=$(mktemp -d) local forge_test_json="$temp_dir/forge_test.json" @@ -179,7 +189,12 @@ generate_report() { local timestamp=$(date) # Get repository information and create numbered list with links - local versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"' | tr '\n' ' ') + local versions + if [[ -n "$CUSTOM_VERSIONS" ]]; then + versions=$(echo "$CUSTOM_VERSIONS" | tr ',' ' ') + else + versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"' | tr '\n' ' ') + fi # Extract repository info for numbered list local repo_list="" @@ -256,8 +271,8 @@ main() { # Check dependencies check_dependencies - # Check and install required Foundry versions - check_and_install_foundry + # Install Foundry versions if --force-install is used + install_foundry_versions # Run benchmarks and generate report run_benchmarks @@ -280,18 +295,25 @@ USAGE: $0 [OPTIONS] OPTIONS: - -h, --help Show this help message - -v, --version Show version information - --verbose Show benchmark output (by default output is suppressed) + -h, --help Show this help message + -v, --version Show version information + --verbose Show benchmark output (by default output is suppressed) + --versions Comma-separated list of Foundry versions to test + (e.g. stable,nightly,v1.2.0) + If not specified, uses versions from src/lib.rs + --force-install Force installation of Foundry versions + By default, assumes versions are already installed REQUIREMENTS: - criterion-table: cargo install criterion-table - cargo-criterion: cargo install cargo-criterion - - All Foundry versions defined in src/lib.rs must be installed + - Foundry versions must be installed (or use --force-install) EXAMPLES: - $0 # Run all benchmarks and generate LATEST.md - $0 --verbose # Run benchmarks with full output visible + $0 # Run with default versions + $0 --verbose # Show full output + $0 --versions stable,nightly # Test specific versions + $0 --versions stable,nightly --force-install # Install and test versions The script will: 1. Run forge_test, forge_build_no_cache, and forge_build_with_cache benchmarks @@ -302,8 +324,12 @@ The script will: EOF } -# Parse command line arguments +# Default values VERBOSE=false +FORCE_INSTALL=false +CUSTOM_VERSIONS="" + +# Parse command line arguments while [[ $# -gt 0 ]]; do case $1 in -h|--help) @@ -318,6 +344,18 @@ while [[ $# -gt 0 ]]; do VERBOSE=true shift ;; + --force-install) + FORCE_INSTALL=true + shift + ;; + --versions) + if [[ -z "$2" ]] || [[ "$2" == --* ]]; then + log_error "--versions requires a comma-separated list of versions" + exit 1 + fi + CUSTOM_VERSIONS="$2" + shift 2 + ;; *) log_error "Unknown option: $1" echo "Use -h or --help for usage information" diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 5d5814103a6e8..b1c74b18c6142 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -2,6 +2,7 @@ use eyre::{Result, WrapErr}; use foundry_compilers::project_util::TempProject; use foundry_test_utils::util::clone_remote; use std::{ + env, path::{Path, PathBuf}, process::{Command, Output}, }; @@ -93,40 +94,32 @@ impl BenchmarkProject { } } - // Install dependencies - Self::install_dependencies(&root_path)?; + // Git submodules are already cloned via --recursive flag + // But npm dependencies still need to be installed + Self::install_npm_dependencies(&root_path)?; + println!(" ✅ Project {} setup complete at {}", config.name, root); Ok(BenchmarkProject { name: config.name.to_string(), root_path, temp_project }) } - /// Install forge dependencies for the project - fn install_dependencies(root: &Path) -> Result<()> { - // Install forge dependencies if foundry.toml exists - if root.join("foundry.toml").exists() { - let status = Command::new("forge") - .current_dir(root) - .args(["install"]) - .status() - .wrap_err("Failed to run forge install")?; - - if !status.success() { - println!("Warning: forge install failed for {}", root.display()); - } - } - - // Install npm dependencies if package.json exists + /// Install npm dependencies if package.json exists + fn install_npm_dependencies(root: &Path) -> Result<()> { if root.join("package.json").exists() { + println!(" 📦 Running npm install..."); let status = Command::new("npm") .current_dir(root) .args(["install"]) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) .status() .wrap_err("Failed to run npm install")?; if !status.success() { - println!("Warning: npm install failed for {}", root.display()); + println!(" ⚠️ Warning: npm install failed with exit code: {:?}", status.code()); + } else { + println!(" ✅ npm install completed successfully"); } } - Ok(()) } @@ -161,15 +154,23 @@ impl BenchmarkProject { /// Switch to a specific foundry version pub fn switch_foundry_version(version: &str) -> Result<()> { - let status = Command::new("foundryup") + let output = Command::new("foundryup") .args(["--use", version]) - .status() + .output() .wrap_err("Failed to run foundryup")?; - if !status.success() { + // Check if the error is about forge --version failing + let stderr = String::from_utf8_lossy(&output.stderr); + if stderr.contains("command failed") && stderr.contains("forge --version") { + eyre::bail!("Foundry binaries maybe corrupted. Please reinstall, please run `foundryup` and install the required versions."); + } + + if !output.status.success() { + eprintln!("foundryup stderr: {}", stderr); eyre::bail!("Failed to switch to foundry version: {}", version); } + println!(" Successfully switched to version: {}", version); Ok(()) } @@ -189,3 +190,22 @@ pub fn get_forge_version() -> Result { Ok(version.lines().next().unwrap_or("unknown").to_string()) } + +/// Get Foundry versions to benchmark from environment variable or default +/// +/// Reads from FOUNDRY_BENCH_VERSIONS environment variable if set, +/// otherwise returns the default versions from FOUNDRY_VERSIONS constant. +/// +/// The environment variable should be a comma-separated list of versions, +/// e.g., "stable,nightly,v1.2.0" +pub fn get_benchmark_versions() -> Vec { + if let Ok(versions_env) = env::var("FOUNDRY_BENCH_VERSIONS") { + versions_env + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() + } else { + FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect() + } +} From fcba24231c4c584c712abb9fbe72308256f14c0f Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Jun 2025 15:58:07 +0530 Subject: [PATCH 13/74] latest bench --- benches/LATEST.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index 354ca2f054897..27fb56662a7d4 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # Foundry Benchmarking Results -**Generated on:** Fri 27 Jun 2025 15:16:52 IST +**Generated on:** Fri 27 Jun 2025 15:51:19 IST **Foundry Versions Tested:** stable nightly ## Repositories Tested @@ -28,15 +28,29 @@ The following benchmarks were performed: ## Table of Contents - [Benchmark Results](#benchmark-results) + - [forge-test](#forge-test) + - [forge-build-no-cache](#forge-build-no-cache) - [forge-build-with-cache](#forge-build-with-cache) ## Benchmark Results +### forge-test + +| | `stable` | `nightly` | +|:------------------------|:-----------------------|:------------------------------ | +| **`ithacaxyz-account`** | `3.75 s` (✅ **1.00x**) | `3.27 s` (✅ **1.15x faster**) | + +### forge-build-no-cache + +| | `stable` | `nightly` | +|:------------------------|:------------------------|:------------------------------- | +| **`ithacaxyz-account`** | `14.23 s` (✅ **1.00x**) | `14.25 s` (✅ **1.00x slower**) | + ### forge-build-with-cache | | `stable` | `nightly` | |:------------------------|:--------------------------|:--------------------------------- | -| **`ithacaxyz-account`** | `166.32 ms` (✅ **1.00x**) | `171.37 ms` (✅ **1.03x slower**) | +| **`ithacaxyz-account`** | `163.53 ms` (✅ **1.00x**) | `168.00 ms` (✅ **1.03x slower**) | --- Made with [criterion-table](https://github.com/nu11ptr/criterion-table) @@ -54,7 +68,7 @@ Made with [criterion-table](https://github.com/nu11ptr/criterion-table) - **OS:** Darwin - **Architecture:** arm64 -- **Date:** Fri 27 Jun 2025 15:16:52 IST +- **Date:** Fri 27 Jun 2025 15:51:19 IST ## Raw Data From 858d8d99daf8d1addaf37c21375bf864d7db0be5 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:05:31 +0530 Subject: [PATCH 14/74] rm notes --- benches/LATEST.md | 40 ++++++++++++++++----------------------- benches/run_benchmarks.sh | 10 +--------- 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index 27fb56662a7d4..a2c4e59b5931b 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,7 +1,7 @@ # Foundry Benchmarking Results **Generated on:** Fri 27 Jun 2025 15:51:19 IST -**Foundry Versions Tested:** stable nightly +**Foundry Versions Tested:** stable nightly ## Repositories Tested @@ -18,7 +18,7 @@ This report contains comprehensive benchmarking results comparing different Foun The following benchmarks were performed: 1. **forge-test** - Running the test suite (10 samples each) -2. **forge-build-no-cache** - Clean build without cache (10 samples each) +2. **forge-build-no-cache** - Clean build without cache (10 samples each) 3. **forge-build-with-cache** - Build with warm cache (10 samples each) --- @@ -28,41 +28,33 @@ The following benchmarks were performed: ## Table of Contents - [Benchmark Results](#benchmark-results) - - [forge-test](#forge-test) - - [forge-build-no-cache](#forge-build-no-cache) - - [forge-build-with-cache](#forge-build-with-cache) + - [forge-test](#forge-test) + - [forge-build-no-cache](#forge-build-no-cache) + - [forge-build-with-cache](#forge-build-with-cache) ## Benchmark Results ### forge-test -| | `stable` | `nightly` | -|:------------------------|:-----------------------|:------------------------------ | -| **`ithacaxyz-account`** | `3.75 s` (✅ **1.00x**) | `3.27 s` (✅ **1.15x faster**) | +| | `stable` | `nightly` | +| :---------------------- | :---------------------- | :----------------------------- | +| **`ithacaxyz-account`** | `3.75 s` (✅ **1.00x**) | `3.27 s` (✅ **1.15x faster**) | ### forge-build-no-cache -| | `stable` | `nightly` | -|:------------------------|:------------------------|:------------------------------- | -| **`ithacaxyz-account`** | `14.23 s` (✅ **1.00x**) | `14.25 s` (✅ **1.00x slower**) | +| | `stable` | `nightly` | +| :---------------------- | :----------------------- | :------------------------------ | +| **`ithacaxyz-account`** | `14.23 s` (✅ **1.00x**) | `14.25 s` (✅ **1.00x slower**) | ### forge-build-with-cache -| | `stable` | `nightly` | -|:------------------------|:--------------------------|:--------------------------------- | -| **`ithacaxyz-account`** | `163.53 ms` (✅ **1.00x**) | `168.00 ms` (✅ **1.03x slower**) | +| | `stable` | `nightly` | +| :---------------------- | :------------------------- | :-------------------------------- | +| **`ithacaxyz-account`** | `163.53 ms` (✅ **1.00x**) | `168.00 ms` (✅ **1.03x slower**) | --- -Made with [criterion-table](https://github.com/nu11ptr/criterion-table) - -## Notes -- All benchmarks use Criterion.rs for statistical analysis -- Each benchmark runs 10 samples by default -- Results show mean execution time with confidence intervals -- Repositories are cloned once and reused across all Foundry versions -- Build and setup operations are parallelized using Rayon -- The first version tested becomes the baseline for comparisons +Made with [criterion-table](https://github.com/nu11ptr/criterion-table) ## System Information @@ -73,5 +65,5 @@ Made with [criterion-table](https://github.com/nu11ptr/criterion-table) ## Raw Data Detailed benchmark data and HTML reports are available in: -- `target/criterion/` - Individual benchmark reports +- `target/criterion/` - Individual benchmark reports diff --git a/benches/run_benchmarks.sh b/benches/run_benchmarks.sh index b9ddfe378c25d..a4b343a0ac226 100755 --- a/benches/run_benchmarks.sh +++ b/benches/run_benchmarks.sh @@ -239,16 +239,8 @@ EOF # Append the criterion-table generated tables cat "$tables_file" >> "$report_file" - # Add notes and system info + # Add system info cat >> "$report_file" << EOF -## Notes - -- All benchmarks use Criterion.rs for statistical analysis -- Each benchmark runs 10 samples by default -- Results show mean execution time with confidence intervals -- Repositories are cloned once and reused across all Foundry versions -- Build and setup operations are parallelized using Rayon -- The first version tested becomes the baseline for comparisons ## System Information From 44835cf9cfd0b95e4029b844d0df925f89492b9e Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:20:10 +0530 Subject: [PATCH 15/74] remove shell based bench suite --- benches/benchmark.sh | 577 --------------------- benches/commands/forge_build_no_cache.sh | 45 -- benches/commands/forge_build_with_cache.sh | 47 -- benches/commands/forge_test.sh | 47 -- benches/repos_and_versions.sh | 44 -- 5 files changed, 760 deletions(-) delete mode 100755 benches/benchmark.sh delete mode 100755 benches/commands/forge_build_no_cache.sh delete mode 100755 benches/commands/forge_build_with_cache.sh delete mode 100755 benches/commands/forge_test.sh delete mode 100755 benches/repos_and_versions.sh diff --git a/benches/benchmark.sh b/benches/benchmark.sh deleted file mode 100755 index 3d1e92aa62d9d..0000000000000 --- a/benches/benchmark.sh +++ /dev/null @@ -1,577 +0,0 @@ -#!/bin/bash - -# Foundry Multi-Version Benchmarking Suite -# This script benchmarks forge test and forge build commands across multiple repositories -# and multiple Foundry versions for comprehensive performance comparison - -set -e - -# Main execution -main() { - log_info "Starting Foundry Multi-Version Benchmarking Suite..." - log_info "Testing Foundry versions: ${FOUNDRY_VERSIONS[*]}" - log_info "Testing repositories: ${REPO_NAMES[*]}" - - # Setup - check_dependencies - setup_directories - - # Ensure cleanup on exit - trap cleanup EXIT - - # Install all Foundry versions upfront - install_all_foundry_versions - - # Clone/update repositories - for i in "${!REPO_NAMES[@]}"; do - clone_or_update_repo "${REPO_NAMES[$i]}" "${REPO_URLS[$i]}" - install_dependencies "${BENCHMARK_DIR}/${REPO_NAMES[$i]}" "${REPO_NAMES[$i]}" - done - - # Run benchmarks in parallel - benchmark_all_repositories_parallel - - # Compile results - compile_results - - log_success "Benchmarking complete!" - log_success "Results saved to: $RESULTS_FILE" - log_success "Latest results: $LATEST_RESULTS_FILE" - log_success "Raw JSON data saved to: $JSON_RESULTS_DIR" - log_info "You can view the results with: cat $LATEST_RESULTS_FILE" -} - -# Get the directory where this script is located -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# Configuration -BENCHMARK_DIR="${SCRIPT_DIR}/benchmark_repos" -RESULTS_DIR="${SCRIPT_DIR}/benchmark_results" -TIMESTAMP=$(date +"%Y%m%d_%H%M%S") -RESULTS_FILE="${RESULTS_DIR}/foundry_multi_version_benchmark_${TIMESTAMP}.md" -LATEST_RESULTS_FILE="${SCRIPT_DIR}/LATEST.md" -JSON_RESULTS_DIR="${RESULTS_DIR}/json_${TIMESTAMP}" - -# Load configuration -source "${SCRIPT_DIR}/repos_and_versions.sh" - -# Load benchmark commands -source "${SCRIPT_DIR}/commands/forge_test.sh" -source "${SCRIPT_DIR}/commands/forge_build_no_cache.sh" -source "${SCRIPT_DIR}/commands/forge_build_with_cache.sh" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Helper functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Install foundryup if not present -install_foundryup() { - if ! command -v foundryup &> /dev/null; then - log_info "Installing foundryup..." - curl -L https://foundry.paradigm.xyz | bash - # Source the bashrc/profile to get foundryup in PATH - export PATH="$HOME/.foundry/bin:$PATH" - fi -} - -# Install a specific Foundry version -install_foundry_version() { - local version=$1 - log_info "Installing Foundry version: $version" - - # Let foundryup handle any version format and determine validity - if foundryup --install "$version"; then - # Verify installation - local installed_version=$(forge --version | head -n1 || echo "unknown") - log_success "Installed Foundry: $installed_version" - return 0 - else - log_error "Failed to install Foundry $version" - return 1 - fi -} - -# Switch to a specific installed Foundry version -use_foundry_version() { - local version=$1 - log_info "Switching to Foundry version: $version" - - if foundryup --use "$version"; then - # Verify switch - local current_version=$(forge --version | head -n1 || echo "unknown") - log_success "Now using Foundry: $current_version" - return 0 - else - log_error "Failed to switch to Foundry $version" - return 1 - fi -} - -# Install all required Foundry versions upfront -install_all_foundry_versions() { - log_info "Installing all required Foundry versions as preprocessing step..." - - local failed_versions=() - - for version in "${FOUNDRY_VERSIONS[@]}"; do - if ! install_foundry_version "$version"; then - failed_versions+=("$version") - fi - done - - if [ ${#failed_versions[@]} -ne 0 ]; then - log_error "Failed to install the following Foundry versions: ${failed_versions[*]}" - log_error "Please check the version names and try again" - exit 1 - fi - - log_success "All Foundry versions installed successfully!" - - # List all installed versions for verification - log_info "Available installed versions:" - foundryup --list || log_warn "Could not list installed versions" -} - -# Check if required tools are installed -check_dependencies() { - local missing_deps=() - - if ! command -v hyperfine &> /dev/null; then - missing_deps+=("hyperfine") - fi - - if ! command -v git &> /dev/null; then - missing_deps+=("git") - fi - - if ! command -v curl &> /dev/null; then - missing_deps+=("curl") - fi - - if [ ${#missing_deps[@]} -ne 0 ]; then - log_error "Missing required dependencies: ${missing_deps[*]}" - log_info "Install hyperfine: https://github.com/sharkdp/hyperfine#installation" - exit 1 - fi - - # Install foundryup if needed - install_foundryup -} - -# Setup directories -setup_directories() { - log_info "Setting up benchmark directories..." - mkdir -p "$BENCHMARK_DIR" - mkdir -p "$RESULTS_DIR" - mkdir -p "$JSON_RESULTS_DIR" -} - -# Clone or update repository -clone_or_update_repo() { - local name=$1 - local url=$2 - local repo_dir="${BENCHMARK_DIR}/${name}" - - if [ -d "$repo_dir" ]; then - log_info "Updating existing repository: $name" - cd "$repo_dir" - git pull origin main 2>/dev/null || git pull origin master 2>/dev/null || true - cd - > /dev/null - else - log_info "Cloning repository: $name" - git clone "$url" "$repo_dir" - fi -} - -# Install dependencies for a repository -install_dependencies() { - local repo_dir=$1 - local repo_name=$2 - - log_info "Installing dependencies for $repo_name..." - cd "$repo_dir" - - # Install forge dependencies - if [ -f "foundry.toml" ]; then - forge install 2>/dev/null || true - fi - - # Install npm dependencies if package.json exists - if [ -f "package.json" ]; then - if command -v npm &> /dev/null; then - npm install 2>/dev/null || true - fi - fi - - cd - > /dev/null -} - -# Run benchmarks for a single repository with a specific Foundry version -benchmark_repository_for_version() { - local repo_name=$1 - local version=$2 - local repo_dir="${BENCHMARK_DIR}/${repo_name}" - - # Create a unique log file for this repo+version combination - local log_file="${JSON_RESULTS_DIR}/${repo_name}_${version//[^a-zA-Z0-9]/_}_benchmark.log" - - { - echo "$(date): Starting benchmark for $repo_name with Foundry $version" - - if [ ! -d "$repo_dir" ]; then - echo "ERROR: Repository directory not found: $repo_dir" - return 1 - fi - - cd "$repo_dir" - - # Check if it's a valid Foundry project - if [ ! -f "foundry.toml" ]; then - echo "WARN: No foundry.toml found in $repo_name, skipping..." - cd - > /dev/null - return 0 - fi - - # Clean version string for filenames (remove 'v' prefix, replace '.' with '_') - local clean_version="${version//v/}" - clean_version="${clean_version//\./_}" - - local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" - mkdir -p "$version_results_dir" - - echo "Running benchmarks for $repo_name with Foundry $version..." - - # Run all benchmark commands - fail fast if any command fails - benchmark_forge_test "$repo_name" "$version" "$version_results_dir" "$log_file" || { - echo "FATAL: forge test benchmark failed for $repo_name with Foundry $version" >> "$log_file" - exit 1 - } - benchmark_forge_build_no_cache "$repo_name" "$version" "$version_results_dir" "$log_file" || { - echo "FATAL: forge build (no cache) benchmark failed for $repo_name with Foundry $version" >> "$log_file" - exit 1 - } - benchmark_forge_build_with_cache "$repo_name" "$version" "$version_results_dir" "$log_file" || { - echo "FATAL: forge build (with cache) benchmark failed for $repo_name with Foundry $version" >> "$log_file" - exit 1 - } - - # Store version info for this benchmark - forge --version | head -n1 > "${version_results_dir}/forge_version.txt" 2>/dev/null || echo "unknown" > "${version_results_dir}/forge_version.txt" - - cd - > /dev/null - echo "$(date): Completed benchmark for $repo_name with Foundry $version" - - } > "$log_file" 2>&1 -} - -# Run benchmarks for all repositories in parallel for each Foundry version -benchmark_all_repositories_parallel() { - for version in "${FOUNDRY_VERSIONS[@]}"; do - log_info "Switching to Foundry version: $version" - - # Switch to the pre-installed version - use_foundry_version "$version" || { - log_warn "Failed to switch to Foundry $version, skipping all repositories for this version..." - continue - } - - log_info "Starting parallel benchmarks for all repositories with Foundry $version" - - # Launch all repositories in parallel - local pids=() - - for repo_name in "${REPO_NAMES[@]}"; do - # Check if repo directory exists and is valid before starting background process - local repo_dir="${BENCHMARK_DIR}/${repo_name}" - if [ ! -d "$repo_dir" ]; then - log_warn "Repository directory not found: $repo_dir, skipping..." - continue - fi - - if [ ! -f "${repo_dir}/foundry.toml" ]; then - log_warn "No foundry.toml found in $repo_name, skipping..." - continue - fi - - log_info "Launching background benchmark for $repo_name..." - benchmark_repository_for_version "$repo_name" "$version" & - local pid=$! - pids+=($pid) - echo "$repo_name:$pid" >> "${JSON_RESULTS_DIR}/parallel_pids_${version//[^a-zA-Z0-9]/_}.txt" - done - - # Wait for all repositories to complete - log_info "Waiting for ${#pids[@]} parallel benchmarks to complete for Foundry $version..." - local completed=0 - local total=${#pids[@]} - - for pid in "${pids[@]}"; do - if wait "$pid"; then - completed=$((completed + 1)) - log_info "Progress: $completed/$total repositories completed for Foundry $version" - else - log_error "Benchmark process failed (PID: $pid) for Foundry $version" - exit 1 - fi - done - - log_success "All repositories completed for Foundry $version ($completed/$total successful)" - - # Show summary of log files created - log_info "Individual benchmark logs available in: ${JSON_RESULTS_DIR}/*_${version//[^a-zA-Z0-9]/_}_benchmark.log" - done -} - -# Extract mean time from JSON result file -extract_mean_time() { - local json_file=$1 - if [ -f "$json_file" ]; then - # Extract mean time in seconds, format to 3 decimal places - python3 -c " -import json, sys -try: - with open('$json_file') as f: - data = json.load(f) - mean_time = data['results'][0]['mean'] - print(f'{mean_time:.3f}') -except: - print('N/A') -" 2>/dev/null || echo "N/A" - else - echo "N/A" - fi -} - -# Get Foundry version string from file -get_forge_version() { - local version_file=$1 - if [ -f "$version_file" ]; then - cat "$version_file" | sed 's/forge //' | sed 's/ (.*//' - else - echo "unknown" - fi -} - -# Compile results into markdown with comparison tables -compile_results() { - log_info "Compiling benchmark results..." - - cat > "$RESULTS_FILE" << EOF -# Forge Benchmarking Results - -**Generated on:** $(date) -**Hyperfine Version:** $(hyperfine --version) -**Foundry Versions Tested:** ${FOUNDRY_VERSIONS[*]} -**Repositories Tested:** ${REPO_NAMES[*]} - -## Summary - -This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. -The following benchmarks were performed: - -1. **$(get_forge_test_description)** -2. **$(get_forge_build_no_cache_description)** -3. **$(get_forge_build_with_cache_description)** - ---- - -## Performance Comparison Tables - -EOF - - # Create unified comparison tables for each benchmark type - local benchmark_commands=("forge_test" "forge_build_no_cache" "forge_build_with_cache") - - for cmd in "${benchmark_commands[@]}"; do - local bench_name="${cmd//_/ }" - local bench_type=$(get_${cmd}_type) - local json_filename=$(get_${cmd}_json_filename) - - echo "### $bench_name" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - echo "Mean execution time in seconds (lower is better):" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - - # Create table header with proper column names - local header_row="| Project" - for version in "${FOUNDRY_VERSIONS[@]}"; do - header_row+=" | $version (s)" - done - header_row+=" |" - echo "$header_row" >> "$RESULTS_FILE" - - # Create table separator with proper alignment - local separator_row="|------" - for version in "${FOUNDRY_VERSIONS[@]}"; do - separator_row+="|--------:" - done - separator_row+="|" - echo "$separator_row" >> "$RESULTS_FILE" - - # Add data rows - for repo_name in "${REPO_NAMES[@]}"; do - local data_row="| **$repo_name**" - - for version in "${FOUNDRY_VERSIONS[@]}"; do - local clean_version="${version//v/}" - clean_version="${clean_version//\./_}" - local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" - local json_file="${version_results_dir}/${json_filename}" - - local mean_time=$(extract_mean_time "$json_file") - data_row+=" | $mean_time" - done - data_row+=" |" - echo "$data_row" >> "$RESULTS_FILE" - done - echo "" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - done - - # Add detailed version information - echo "## Foundry Version Details" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - - for version in "${FOUNDRY_VERSIONS[@]}"; do - echo "### $version" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - - # Find any version file to get the detailed version info - local clean_version="${version//v/}" - clean_version="${clean_version//\./_}" - - for repo_name in "${REPO_NAMES[@]}"; do - local version_file="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}/forge_version.txt" - if [ -f "$version_file" ]; then - echo "\`\`\`" >> "$RESULTS_FILE" - cat "$version_file" >> "$RESULTS_FILE" - echo "\`\`\`" >> "$RESULTS_FILE" - break - fi - done - echo "" >> "$RESULTS_FILE" - done - - # Add notes and system info - cat >> "$RESULTS_FILE" << EOF - -## Notes - -- All benchmarks were run with hyperfine in parallel mode -- **$(get_forge_test_description)** -- **$(get_forge_build_no_cache_description)** -- **$(get_forge_build_with_cache_description)** -- Results show mean execution time in seconds -- N/A indicates benchmark failed or data unavailable - -## System Information - -- **OS:** $(uname -s) -- **Architecture:** $(uname -m) -- **Date:** $(date) - -## Raw Data - -Raw JSON benchmark data is available in: \`$JSON_RESULTS_DIR\` - -EOF - - # Copy to LATEST.md - cp "$RESULTS_FILE" "$LATEST_RESULTS_FILE" - log_success "Latest results also saved to: $LATEST_RESULTS_FILE" -} - -# Cleanup temporary files -cleanup() { - log_info "Cleanup completed" -} - -# Parse command line arguments -parse_args() { - while [[ $# -gt 0 ]]; do - case $1 in - --versions) - shift - if [[ $# -eq 0 ]]; then - log_error "--versions requires a space-separated list of versions" - exit 1 - fi - # Read versions until next flag or end of args - FOUNDRY_VERSIONS=() - while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do - FOUNDRY_VERSIONS+=("$1") - shift - done - ;; - --help|-h) - echo "Foundry Benchmarking Suite" - echo "" - echo "Usage: $0 [OPTIONS]" - echo "" - echo "OPTIONS:" - echo " --help, -h Show this help message" - echo " --version, -v Show version information" - echo " --versions ... Specify Foundry versions to benchmark" - echo " (default: from repos_and_versions.sh)" - echo "" - echo "EXAMPLES:" - echo " $0 # Use default versions (parallel)" - echo " $0 --versions stable nightly # Benchmark stable and nightly only" - echo " $0 --versions v1.0.0 v1.1.0 v1.2.0 # Benchmark specific versions" - echo "" - echo "This script benchmarks forge test and forge build commands across" - echo "multiple Foundry repositories and versions using hyperfine." - - echo "Supported version formats:" - echo " - stable, nightly (special tags)" - echo " - v1.0.0, v1.1.0, etc. (specific versions)" - echo " - nightly- (specific nightly builds)" - echo " - Any format supported by foundryup" - echo "" - echo "The script will:" - echo " 1. Install foundryup if not present" - echo " 2. Install all specified Foundry versions (preprocessing step)" - echo " 3. Clone/update target repositories" - echo " 4. Switch between versions and run benchmarks in parallel" - echo " 5. Generate comparison tables in markdown format" - echo " 6. Save results to LATEST.md" - exit 0 - ;; - --version|-v) - exit 0 - ;; - *) - log_error "Unknown option: $1" - echo "Use --help for usage information" - exit 1 - ;; - esac - done -} - -# Handle command line arguments -if [[ $# -gt 0 ]]; then - parse_args "$@" -fi - -main \ No newline at end of file diff --git a/benches/commands/forge_build_no_cache.sh b/benches/commands/forge_build_no_cache.sh deleted file mode 100755 index 756528176ac4e..0000000000000 --- a/benches/commands/forge_build_no_cache.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Forge Build (No Cache) Benchmark Command -# This file contains the configuration and execution logic for benchmarking 'forge build' with no cache - -# Command configuration -FORGE_BUILD_NO_CACHE_RUNS=5 - -# Benchmark function for forge build (no cache) -benchmark_forge_build_no_cache() { - local repo_name=$1 - local version=$2 - local version_results_dir=$3 - local log_file=$4 - - echo "Running 'forge build' (no cache) benchmark..." >> "$log_file" - - if hyperfine \ - --runs "$FORGE_BUILD_NO_CACHE_RUNS" \ - --prepare 'forge clean' \ - --export-json "${version_results_dir}/build_no_cache_results.json" \ - "forge build" 2>>"$log_file.error"; then - echo "✓ forge build (no cache) completed" >> "$log_file" - return 0 - else - echo "✗ forge build (no cache) failed" >> "$log_file" - echo "FATAL: forge build (no cache) benchmark failed" >> "$log_file" - return 1 - fi -} - -# Get command description for reporting -get_forge_build_no_cache_description() { - echo "forge build (no cache) - Clean build without cache ($FORGE_BUILD_NO_CACHE_RUNS runs, cache cleaned after each run)" -} - -# Get JSON result filename -get_forge_build_no_cache_json_filename() { - echo "build_no_cache_results.json" -} - -# Get benchmark type identifier -get_forge_build_no_cache_type() { - echo "build_no_cache" -} \ No newline at end of file diff --git a/benches/commands/forge_build_with_cache.sh b/benches/commands/forge_build_with_cache.sh deleted file mode 100755 index 25fab9dbc190f..0000000000000 --- a/benches/commands/forge_build_with_cache.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Forge Build (With Cache) Benchmark Command -# This file contains the configuration and execution logic for benchmarking 'forge build' with cache - -# Command configuration -FORGE_BUILD_WITH_CACHE_RUNS=5 -FORGE_BUILD_WITH_CACHE_WARMUP=1 - -# Benchmark function for forge build (with cache) -benchmark_forge_build_with_cache() { - local repo_name=$1 - local version=$2 - local version_results_dir=$3 - local log_file=$4 - - echo "Running 'forge build' (with cache) benchmark..." >> "$log_file" - - if hyperfine \ - --runs "$FORGE_BUILD_WITH_CACHE_RUNS" \ - --prepare 'forge build' \ - --warmup "$FORGE_BUILD_WITH_CACHE_WARMUP" \ - --export-json "${version_results_dir}/build_with_cache_results.json" \ - "forge build" 2>>"$log_file.error"; then - echo "✓ forge build (with cache) completed" >> "$log_file" - return 0 - else - echo "✗ forge build (with cache) failed" >> "$log_file" - echo "FATAL: forge build (with cache) benchmark failed" >> "$log_file" - return 1 - fi -} - -# Get command description for reporting -get_forge_build_with_cache_description() { - echo "forge build (with cache) - Build with warm cache ($FORGE_BUILD_WITH_CACHE_RUNS runs, $FORGE_BUILD_WITH_CACHE_WARMUP warmup)" -} - -# Get JSON result filename -get_forge_build_with_cache_json_filename() { - echo "build_with_cache_results.json" -} - -# Get benchmark type identifier -get_forge_build_with_cache_type() { - echo "build_with_cache" -} \ No newline at end of file diff --git a/benches/commands/forge_test.sh b/benches/commands/forge_test.sh deleted file mode 100755 index 9427d18d3c30c..0000000000000 --- a/benches/commands/forge_test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Forge Test Benchmark Command -# This file contains the configuration and execution logic for benchmarking 'forge test' - -# Command configuration -FORGE_TEST_RUNS=5 -FORGE_TEST_WARMUP=1 - -# Benchmark function for forge test -benchmark_forge_test() { - local repo_name=$1 - local version=$2 - local version_results_dir=$3 - local log_file=$4 - - echo "Running 'forge test' benchmark..." >> "$log_file" - - if hyperfine \ - --runs "$FORGE_TEST_RUNS" \ - --prepare 'forge build' \ - --warmup "$FORGE_TEST_WARMUP" \ - --export-json "${version_results_dir}/test_results.json" \ - "forge test" 2>>"$log_file.error"; then - echo "✓ forge test completed" >> "$log_file" - return 0 - else - echo "✗ forge test failed" >> "$log_file" - echo "FATAL: forge test benchmark failed" >> "$log_file" - return 1 - fi -} - -# Get command description for reporting -get_forge_test_description() { - echo "forge test - Running the test suite ($FORGE_TEST_RUNS runs, $FORGE_TEST_WARMUP warmup)" -} - -# Get JSON result filename -get_forge_test_json_filename() { - echo "test_results.json" -} - -# Get benchmark type identifier -get_forge_test_type() { - echo "test" -} \ No newline at end of file diff --git a/benches/repos_and_versions.sh b/benches/repos_and_versions.sh deleted file mode 100755 index 07da480b05d2d..0000000000000 --- a/benches/repos_and_versions.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Foundry Multi-Version Benchmarking Configuration -# This file contains the configuration for repositories and Foundry versions to benchmark - -# Foundry versions to benchmark -# Supported formats: -# - stable, nightly (special tags) -# - v1.0.0, v1.1.0, etc. (specific versions) -# - nightly- (specific nightly builds) -# - Any format supported by foundryup -FOUNDRY_VERSIONS=( - "stable" - "nightly" -) - -# Repository configurations -# Add new repositories by adding entries to both arrays -REPO_NAMES=( - "ithacaxyz-account" - # "v4-core" - "solady" - # "morpho-blue" - # "spark-psm" -) - -REPO_URLS=( - "https://github.com/ithacaxyz/account" - # "https://github.com/Uniswap/v4-core" - "https://github.com/Vectorized/solady" - # "https://github.com/morpho-org/morpho-blue" - # "https://github.com/sparkdotfi/spark-psm" -) - -# Verify arrays have the same length -if [ ${#REPO_NAMES[@]} -ne ${#REPO_URLS[@]} ]; then - echo "ERROR: REPO_NAMES and REPO_URLS arrays must have the same length" - exit 1 -fi - -# Export variables for use in other scripts -export FOUNDRY_VERSIONS -export REPO_NAMES -export REPO_URLS \ No newline at end of file From 64baae2c04fede1dbf3485484b15f0c8a855ee6a Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 27 Jun 2025 18:53:57 +0530 Subject: [PATCH 16/74] feat: benches using criterion (#10805) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: criterion benches * - setup benchmark repos in parallel - run forge build in parallet for forge-test bench - switch foundry versions - README specifying prereqs * feat: shell script to run benches * feat: ci workflow, fix script * update readme * feat: enhance benchmarking suite with version flexibility - Add `get_benchmark_versions()` helper to read versions from env var - Update all benchmarks to use version helper for consistency - Add `--versions` and `--force-install` flags to shell script - Enable all three benchmarks (forge_test, build_no_cache, build_with_cache) - Improve error handling for corrupted forge installations - Remove complex workarounds in favor of clear error messages The benchmarks now support custom versions via: ./run_benchmarks.sh --versions stable,nightly,v1.2.0 🤖 Generated with Claude Code Co-Authored-By: Claude * latest bench * rm notes * remove shell based bench suite --------- Co-authored-by: Claude --- .github/workflows/benchmarks.yml | 108 ++++ Cargo.lock | 125 ++++- Cargo.toml | 1 + benches/Cargo.toml | 35 ++ benches/LATEST.md | 93 ++-- benches/README.md | 104 ++++ benches/benchmark.sh | 577 --------------------- benches/commands/forge_build_no_cache.sh | 45 -- benches/commands/forge_build_with_cache.sh | 47 -- benches/commands/forge_test.sh | 47 -- benches/forge_build_no_cache.rs | 47 ++ benches/forge_build_with_cache.rs | 50 ++ benches/forge_test.rs | 52 ++ benches/repos_and_versions.sh | 44 -- benches/run_benchmarks.sh | 359 +++++++++++++ benches/src/lib.rs | 211 ++++++++ 16 files changed, 1124 insertions(+), 821 deletions(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 benches/Cargo.toml create mode 100644 benches/README.md delete mode 100755 benches/benchmark.sh delete mode 100755 benches/commands/forge_build_no_cache.sh delete mode 100755 benches/commands/forge_build_with_cache.sh delete mode 100755 benches/commands/forge_test.sh create mode 100644 benches/forge_build_no_cache.rs create mode 100644 benches/forge_build_with_cache.rs create mode 100644 benches/forge_test.rs delete mode 100755 benches/repos_and_versions.sh create mode 100755 benches/run_benchmarks.sh create mode 100644 benches/src/lib.rs diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000000..d5c6e0cecf501 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,108 @@ +name: Foundry Benchmarks + +on: + workflow_dispatch: + inputs: + pr_number: + description: "PR number to comment on (optional)" + required: false + type: string + +permissions: + contents: write + pull-requests: write + +jobs: + benchmark: + name: Run Foundry Benchmarks + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + ./ + + - name: Install foundryup + run: | + curl -L https://foundry.paradigm.xyz | bash + echo "$HOME/.foundry/bin" >> $GITHUB_PATH + + - name: Install benchmark dependencies + run: | + cargo install cargo-criterion + cargo install criterion-table + + - name: Run benchmarks + working-directory: ./benches + run: | + chmod +x run_benchmarks.sh + ./run_benchmarks.sh + + - name: Commit benchmark results + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add benches/LATEST.md + if git diff --staged --quiet; then + echo "No changes to commit" + else + git commit -m "Update benchmark results + + 🤖 Generated with [Foundry Benchmarks](https://github.com/${{ github.repository }}/actions) + + Co-Authored-By: github-actions " + git push + fi + + - name: Read benchmark results + id: benchmark_results + run: | + if [ -f "benches/LATEST.md" ]; then + { + echo 'results<> $GITHUB_OUTPUT + else + echo 'results=No benchmark results found.' >> $GITHUB_OUTPUT + fi + + - name: Comment on PR + if: github.event.inputs.pr_number != '' + uses: actions/github-script@v7 + with: + script: | + const prNumber = ${{ github.event.inputs.pr_number }}; + const benchmarkResults = `${{ steps.benchmark_results.outputs.results }}`; + + const comment = `## 📊 Foundry Benchmark Results + +
+ Click to view detailed benchmark results + + ${benchmarkResults} + +
+ + --- + + 🤖 This comment was automatically generated by the [Foundry Benchmarks workflow](https://github.com/${{ github.repository }}/actions). + + To run benchmarks manually: Go to [Actions](https://github.com/${{ github.repository }}/actions/workflows/foundry-benchmarks.yml) → "Run workflow"`; + + github.rest.issues.createComment({ + issue_number: prNumber, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + diff --git a/Cargo.lock b/Cargo.lock index 5b92cb3acb23a..9a469d70d4a4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -588,7 +588,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", - "itertools 0.13.0", + "itertools 0.14.0", "serde", "serde_json", "thiserror 2.0.12", @@ -946,6 +946,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "annotate-snippets" version = "0.11.5" @@ -2361,6 +2367,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cast" version = "1.2.3" @@ -2503,8 +2515,10 @@ checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -2966,6 +2980,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast 0.3.0", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast 0.3.0", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -4047,6 +4097,23 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foundry-bench" +version = "0.1.0" +dependencies = [ + "chrono", + "criterion", + "eyre", + "foundry-compilers", + "foundry-config", + "foundry-test-utils", + "rayon", + "serde", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "foundry-block-explorers" version = "0.18.0" @@ -4265,7 +4332,7 @@ dependencies = [ "fs_extra", "futures-util", "home", - "itertools 0.13.0", + "itertools 0.14.0", "path-slash", "rand 0.8.5", "rayon", @@ -6483,6 +6550,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "op-alloy-consensus" version = "0.17.2" @@ -6945,6 +7018,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -7162,7 +7263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.103", @@ -8694,7 +8795,7 @@ dependencies = [ "derive_builder", "derive_more 2.0.1", "dunce", - "itertools 0.13.0", + "itertools 0.14.0", "itoa", "lasso", "match_cfg", @@ -8706,7 +8807,7 @@ dependencies = [ "solar-config", "solar-data-structures", "solar-macros", - "thiserror 1.0.69", + "thiserror 2.0.12", "tracing", "unicode-width 0.2.0", ] @@ -8731,7 +8832,7 @@ dependencies = [ "alloy-primitives", "bitflags 2.9.1", "bumpalo", - "itertools 0.13.0", + "itertools 0.14.0", "memchr", "num-bigint", "num-rational", @@ -9042,7 +9143,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.12", "url", "zip", ] @@ -9322,6 +9423,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.9.0" diff --git a/Cargo.toml b/Cargo.toml index 30e2b3d0f1603..e6c5bce09b194 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "benches/", "crates/anvil/", "crates/anvil/core/", "crates/anvil/rpc/", diff --git a/benches/Cargo.toml b/benches/Cargo.toml new file mode 100644 index 0000000000000..81c3c71e3a829 --- /dev/null +++ b/benches/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "foundry-bench" +version = "0.1.0" +edition = "2021" + +[[bench]] +name = "forge_test" +path = "forge_test.rs" +harness = false + +[[bench]] +name = "forge_build_no_cache" +path = "forge_build_no_cache.rs" +harness = false + +[[bench]] +name = "forge_build_with_cache" +path = "forge_build_with_cache.rs" +harness = false + +[dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +foundry-test-utils.workspace = true +foundry-config.workspace = true +foundry-compilers = { workspace = true, features = ["project-util"] } +eyre.workspace = true +serde.workspace = true +serde_json.workspace = true +tempfile.workspace = true +tokio = { workspace = true, features = ["full"] } +chrono = { version = "0.4", features = ["serde"] } +rayon.workspace = true + +[dev-dependencies] +foundry-test-utils.workspace = true diff --git a/benches/LATEST.md b/benches/LATEST.md index f276dc5fd6133..a2c4e59b5931b 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,84 +1,69 @@ -# Forge Benchmarking Results +# Foundry Benchmarking Results -**Generated on:** Wed 18 Jun 2025 17:46:19 BST -**Hyperfine Version:** hyperfine 1.19.0 +**Generated on:** Fri 27 Jun 2025 15:51:19 IST **Foundry Versions Tested:** stable nightly -**Repositories Tested:** ithacaxyz-account solady -## Summary - -This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. -The following benchmarks were performed: - -1. **forge test - Running the test suite (5 runs, 1 warmup)** -2. **forge build (no cache) - Clean build without cache (5 runs, cache cleaned after each run)** -3. **forge build (with cache) - Build with warm cache (5 runs, 1 warmup)** - ---- - -## Performance Comparison Tables +## Repositories Tested -### forge test - -Mean execution time in seconds (lower is better): - -| Project | stable (s) | nightly (s) | -|------|--------:|--------:| -| **ithacaxyz-account** | 5.791 | 3.875 | -| **solady** | 3.578 | 2.966 | +1. [ithacaxyz-account](https://github.com/ithacaxyz/main) +2. [solady](https://github.com/Vectorized/main) +3. [v4-core](https://github.com/Uniswap/main) +4. [morpho-blue](https://github.com/morpho-org/main) +5. [spark-psm](https://github.com/marsfoundation/master) +## Summary -### forge build no cache +This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects using Criterion.rs for precise performance measurements. -Mean execution time in seconds (lower is better): +The following benchmarks were performed: -| Project | stable (s) | nightly (s) | -|------|--------:|--------:| -| **ithacaxyz-account** | 19.079 | 16.177 | -| **solady** | 27.408 | 22.745 | +1. **forge-test** - Running the test suite (10 samples each) +2. **forge-build-no-cache** - Clean build without cache (10 samples each) +3. **forge-build-with-cache** - Build with warm cache (10 samples each) +--- -### forge build with cache +# Benchmarks -Mean execution time in seconds (lower is better): +## Table of Contents -| Project | stable (s) | nightly (s) | -|------|--------:|--------:| -| **ithacaxyz-account** | 0.181 | 0.158 | -| **solady** | 0.091 | 0.103 | +- [Benchmark Results](#benchmark-results) + - [forge-test](#forge-test) + - [forge-build-no-cache](#forge-build-no-cache) + - [forge-build-with-cache](#forge-build-with-cache) +## Benchmark Results -## Foundry Version Details +### forge-test -### stable +| | `stable` | `nightly` | +| :---------------------- | :---------------------- | :----------------------------- | +| **`ithacaxyz-account`** | `3.75 s` (✅ **1.00x**) | `3.27 s` (✅ **1.15x faster**) | -``` -forge Version: 1.2.3-stable -``` +### forge-build-no-cache -### nightly +| | `stable` | `nightly` | +| :---------------------- | :----------------------- | :------------------------------ | +| **`ithacaxyz-account`** | `14.23 s` (✅ **1.00x**) | `14.25 s` (✅ **1.00x slower**) | -``` -forge Version: 1.2.3-nightly -``` +### forge-build-with-cache +| | `stable` | `nightly` | +| :---------------------- | :------------------------- | :-------------------------------- | +| **`ithacaxyz-account`** | `163.53 ms` (✅ **1.00x**) | `168.00 ms` (✅ **1.03x slower**) | -## Notes +--- -- All benchmarks were run with hyperfine in parallel mode -- **forge test - Running the test suite (5 runs, 1 warmup)** -- **forge build (no cache) - Clean build without cache (5 runs, cache cleaned after each run)** -- **forge build (with cache) - Build with warm cache (5 runs, 1 warmup)** -- Results show mean execution time in seconds -- N/A indicates benchmark failed or data unavailable +Made with [criterion-table](https://github.com/nu11ptr/criterion-table) ## System Information - **OS:** Darwin - **Architecture:** arm64 -- **Date:** Wed 18 Jun 2025 17:46:19 BST +- **Date:** Fri 27 Jun 2025 15:51:19 IST ## Raw Data -Raw JSON benchmark data is available in: `/Users/yash/dev/paradigm/foundry-rs/foundry/benches/benchmark_results/json_20250618_174101` +Detailed benchmark data and HTML reports are available in: +- `target/criterion/` - Individual benchmark reports diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 0000000000000..b3663831853b1 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,104 @@ +# Foundry Benchmarks + +This directory contains performance benchmarks for Foundry commands across multiple repositories and Foundry versions. + +## Prerequisites + +Before running the benchmarks, ensure you have the following installed: + +1. **Rust and Cargo** - Required for building and running the benchmarks + + ```bash + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + ``` + +2. **Foundryup** - The Foundry toolchain installer + + ```bash + curl -L https://foundry.paradigm.xyz | bash + foundryup + ``` + +3. **Git** - For cloning benchmark repositories + +4. **npm** - Some repositories require npm dependencies + + ```bash + # Install Node.js and npm from https://nodejs.org/ + ``` + +5. **Benchmark tools** - Required for generating reports + ```bash + cargo install cargo-criterion + cargo install criterion-table + ``` + +## Running Benchmarks + +### Run the complete benchmark suite + +```bash +cargo run +``` + +This will: + +1. Check and install required Foundry versions +2. Run all benchmark suites (forge_test, forge_build_no_cache, forge_build_with_cache) +3. Generate comparison tables using criterion-table +4. Create the final LATEST.md report + +### Run individual benchmark suites + +```bash +./run_benchmarks.sh +``` + +### Run specific benchmark + +```bash +cargo criterion --bench forge_test +cargo criterion --bench forge_build_no_cache +cargo criterion --bench forge_build_with_cache +``` + +## Benchmark Structure + +- `forge_test` - Benchmarks `forge test` command across repos +- `forge_build_no_cache` - Benchmarks `forge build` with clean cache +- `forge_build_with_cache` - Benchmarks `forge build` with existing cache + +## Configuration + +### Repositories + +Edit `src/lib.rs` to modify the list of repositories to benchmark: + +```rust +pub static BENCHMARK_REPOS: &[RepoConfig] = &[ + RepoConfig { name: "account", org: "ithacaxyz", repo: "account", rev: "main" }, + // Add more repositories here +]; +``` + +### Foundry Versions + +Edit `src/lib.rs` to modify the list of Foundry versions: + +```rust +pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; +``` + +## Results + +Benchmark results are displayed in the terminal and saved as HTML reports. The reports show: + +- Execution time statistics (mean, median, standard deviation) +- Comparison between different Foundry versions +- Performance trends across repositories + +## Troubleshooting + +1. **Foundry version not found**: Ensure the version is installed with `foundryup --install ` +2. **Repository clone fails**: Check network connectivity and repository access +3. **Build failures**: Some repositories may have specific dependencies - check their README files diff --git a/benches/benchmark.sh b/benches/benchmark.sh deleted file mode 100755 index 3d1e92aa62d9d..0000000000000 --- a/benches/benchmark.sh +++ /dev/null @@ -1,577 +0,0 @@ -#!/bin/bash - -# Foundry Multi-Version Benchmarking Suite -# This script benchmarks forge test and forge build commands across multiple repositories -# and multiple Foundry versions for comprehensive performance comparison - -set -e - -# Main execution -main() { - log_info "Starting Foundry Multi-Version Benchmarking Suite..." - log_info "Testing Foundry versions: ${FOUNDRY_VERSIONS[*]}" - log_info "Testing repositories: ${REPO_NAMES[*]}" - - # Setup - check_dependencies - setup_directories - - # Ensure cleanup on exit - trap cleanup EXIT - - # Install all Foundry versions upfront - install_all_foundry_versions - - # Clone/update repositories - for i in "${!REPO_NAMES[@]}"; do - clone_or_update_repo "${REPO_NAMES[$i]}" "${REPO_URLS[$i]}" - install_dependencies "${BENCHMARK_DIR}/${REPO_NAMES[$i]}" "${REPO_NAMES[$i]}" - done - - # Run benchmarks in parallel - benchmark_all_repositories_parallel - - # Compile results - compile_results - - log_success "Benchmarking complete!" - log_success "Results saved to: $RESULTS_FILE" - log_success "Latest results: $LATEST_RESULTS_FILE" - log_success "Raw JSON data saved to: $JSON_RESULTS_DIR" - log_info "You can view the results with: cat $LATEST_RESULTS_FILE" -} - -# Get the directory where this script is located -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# Configuration -BENCHMARK_DIR="${SCRIPT_DIR}/benchmark_repos" -RESULTS_DIR="${SCRIPT_DIR}/benchmark_results" -TIMESTAMP=$(date +"%Y%m%d_%H%M%S") -RESULTS_FILE="${RESULTS_DIR}/foundry_multi_version_benchmark_${TIMESTAMP}.md" -LATEST_RESULTS_FILE="${SCRIPT_DIR}/LATEST.md" -JSON_RESULTS_DIR="${RESULTS_DIR}/json_${TIMESTAMP}" - -# Load configuration -source "${SCRIPT_DIR}/repos_and_versions.sh" - -# Load benchmark commands -source "${SCRIPT_DIR}/commands/forge_test.sh" -source "${SCRIPT_DIR}/commands/forge_build_no_cache.sh" -source "${SCRIPT_DIR}/commands/forge_build_with_cache.sh" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Helper functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Install foundryup if not present -install_foundryup() { - if ! command -v foundryup &> /dev/null; then - log_info "Installing foundryup..." - curl -L https://foundry.paradigm.xyz | bash - # Source the bashrc/profile to get foundryup in PATH - export PATH="$HOME/.foundry/bin:$PATH" - fi -} - -# Install a specific Foundry version -install_foundry_version() { - local version=$1 - log_info "Installing Foundry version: $version" - - # Let foundryup handle any version format and determine validity - if foundryup --install "$version"; then - # Verify installation - local installed_version=$(forge --version | head -n1 || echo "unknown") - log_success "Installed Foundry: $installed_version" - return 0 - else - log_error "Failed to install Foundry $version" - return 1 - fi -} - -# Switch to a specific installed Foundry version -use_foundry_version() { - local version=$1 - log_info "Switching to Foundry version: $version" - - if foundryup --use "$version"; then - # Verify switch - local current_version=$(forge --version | head -n1 || echo "unknown") - log_success "Now using Foundry: $current_version" - return 0 - else - log_error "Failed to switch to Foundry $version" - return 1 - fi -} - -# Install all required Foundry versions upfront -install_all_foundry_versions() { - log_info "Installing all required Foundry versions as preprocessing step..." - - local failed_versions=() - - for version in "${FOUNDRY_VERSIONS[@]}"; do - if ! install_foundry_version "$version"; then - failed_versions+=("$version") - fi - done - - if [ ${#failed_versions[@]} -ne 0 ]; then - log_error "Failed to install the following Foundry versions: ${failed_versions[*]}" - log_error "Please check the version names and try again" - exit 1 - fi - - log_success "All Foundry versions installed successfully!" - - # List all installed versions for verification - log_info "Available installed versions:" - foundryup --list || log_warn "Could not list installed versions" -} - -# Check if required tools are installed -check_dependencies() { - local missing_deps=() - - if ! command -v hyperfine &> /dev/null; then - missing_deps+=("hyperfine") - fi - - if ! command -v git &> /dev/null; then - missing_deps+=("git") - fi - - if ! command -v curl &> /dev/null; then - missing_deps+=("curl") - fi - - if [ ${#missing_deps[@]} -ne 0 ]; then - log_error "Missing required dependencies: ${missing_deps[*]}" - log_info "Install hyperfine: https://github.com/sharkdp/hyperfine#installation" - exit 1 - fi - - # Install foundryup if needed - install_foundryup -} - -# Setup directories -setup_directories() { - log_info "Setting up benchmark directories..." - mkdir -p "$BENCHMARK_DIR" - mkdir -p "$RESULTS_DIR" - mkdir -p "$JSON_RESULTS_DIR" -} - -# Clone or update repository -clone_or_update_repo() { - local name=$1 - local url=$2 - local repo_dir="${BENCHMARK_DIR}/${name}" - - if [ -d "$repo_dir" ]; then - log_info "Updating existing repository: $name" - cd "$repo_dir" - git pull origin main 2>/dev/null || git pull origin master 2>/dev/null || true - cd - > /dev/null - else - log_info "Cloning repository: $name" - git clone "$url" "$repo_dir" - fi -} - -# Install dependencies for a repository -install_dependencies() { - local repo_dir=$1 - local repo_name=$2 - - log_info "Installing dependencies for $repo_name..." - cd "$repo_dir" - - # Install forge dependencies - if [ -f "foundry.toml" ]; then - forge install 2>/dev/null || true - fi - - # Install npm dependencies if package.json exists - if [ -f "package.json" ]; then - if command -v npm &> /dev/null; then - npm install 2>/dev/null || true - fi - fi - - cd - > /dev/null -} - -# Run benchmarks for a single repository with a specific Foundry version -benchmark_repository_for_version() { - local repo_name=$1 - local version=$2 - local repo_dir="${BENCHMARK_DIR}/${repo_name}" - - # Create a unique log file for this repo+version combination - local log_file="${JSON_RESULTS_DIR}/${repo_name}_${version//[^a-zA-Z0-9]/_}_benchmark.log" - - { - echo "$(date): Starting benchmark for $repo_name with Foundry $version" - - if [ ! -d "$repo_dir" ]; then - echo "ERROR: Repository directory not found: $repo_dir" - return 1 - fi - - cd "$repo_dir" - - # Check if it's a valid Foundry project - if [ ! -f "foundry.toml" ]; then - echo "WARN: No foundry.toml found in $repo_name, skipping..." - cd - > /dev/null - return 0 - fi - - # Clean version string for filenames (remove 'v' prefix, replace '.' with '_') - local clean_version="${version//v/}" - clean_version="${clean_version//\./_}" - - local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" - mkdir -p "$version_results_dir" - - echo "Running benchmarks for $repo_name with Foundry $version..." - - # Run all benchmark commands - fail fast if any command fails - benchmark_forge_test "$repo_name" "$version" "$version_results_dir" "$log_file" || { - echo "FATAL: forge test benchmark failed for $repo_name with Foundry $version" >> "$log_file" - exit 1 - } - benchmark_forge_build_no_cache "$repo_name" "$version" "$version_results_dir" "$log_file" || { - echo "FATAL: forge build (no cache) benchmark failed for $repo_name with Foundry $version" >> "$log_file" - exit 1 - } - benchmark_forge_build_with_cache "$repo_name" "$version" "$version_results_dir" "$log_file" || { - echo "FATAL: forge build (with cache) benchmark failed for $repo_name with Foundry $version" >> "$log_file" - exit 1 - } - - # Store version info for this benchmark - forge --version | head -n1 > "${version_results_dir}/forge_version.txt" 2>/dev/null || echo "unknown" > "${version_results_dir}/forge_version.txt" - - cd - > /dev/null - echo "$(date): Completed benchmark for $repo_name with Foundry $version" - - } > "$log_file" 2>&1 -} - -# Run benchmarks for all repositories in parallel for each Foundry version -benchmark_all_repositories_parallel() { - for version in "${FOUNDRY_VERSIONS[@]}"; do - log_info "Switching to Foundry version: $version" - - # Switch to the pre-installed version - use_foundry_version "$version" || { - log_warn "Failed to switch to Foundry $version, skipping all repositories for this version..." - continue - } - - log_info "Starting parallel benchmarks for all repositories with Foundry $version" - - # Launch all repositories in parallel - local pids=() - - for repo_name in "${REPO_NAMES[@]}"; do - # Check if repo directory exists and is valid before starting background process - local repo_dir="${BENCHMARK_DIR}/${repo_name}" - if [ ! -d "$repo_dir" ]; then - log_warn "Repository directory not found: $repo_dir, skipping..." - continue - fi - - if [ ! -f "${repo_dir}/foundry.toml" ]; then - log_warn "No foundry.toml found in $repo_name, skipping..." - continue - fi - - log_info "Launching background benchmark for $repo_name..." - benchmark_repository_for_version "$repo_name" "$version" & - local pid=$! - pids+=($pid) - echo "$repo_name:$pid" >> "${JSON_RESULTS_DIR}/parallel_pids_${version//[^a-zA-Z0-9]/_}.txt" - done - - # Wait for all repositories to complete - log_info "Waiting for ${#pids[@]} parallel benchmarks to complete for Foundry $version..." - local completed=0 - local total=${#pids[@]} - - for pid in "${pids[@]}"; do - if wait "$pid"; then - completed=$((completed + 1)) - log_info "Progress: $completed/$total repositories completed for Foundry $version" - else - log_error "Benchmark process failed (PID: $pid) for Foundry $version" - exit 1 - fi - done - - log_success "All repositories completed for Foundry $version ($completed/$total successful)" - - # Show summary of log files created - log_info "Individual benchmark logs available in: ${JSON_RESULTS_DIR}/*_${version//[^a-zA-Z0-9]/_}_benchmark.log" - done -} - -# Extract mean time from JSON result file -extract_mean_time() { - local json_file=$1 - if [ -f "$json_file" ]; then - # Extract mean time in seconds, format to 3 decimal places - python3 -c " -import json, sys -try: - with open('$json_file') as f: - data = json.load(f) - mean_time = data['results'][0]['mean'] - print(f'{mean_time:.3f}') -except: - print('N/A') -" 2>/dev/null || echo "N/A" - else - echo "N/A" - fi -} - -# Get Foundry version string from file -get_forge_version() { - local version_file=$1 - if [ -f "$version_file" ]; then - cat "$version_file" | sed 's/forge //' | sed 's/ (.*//' - else - echo "unknown" - fi -} - -# Compile results into markdown with comparison tables -compile_results() { - log_info "Compiling benchmark results..." - - cat > "$RESULTS_FILE" << EOF -# Forge Benchmarking Results - -**Generated on:** $(date) -**Hyperfine Version:** $(hyperfine --version) -**Foundry Versions Tested:** ${FOUNDRY_VERSIONS[*]} -**Repositories Tested:** ${REPO_NAMES[*]} - -## Summary - -This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects. -The following benchmarks were performed: - -1. **$(get_forge_test_description)** -2. **$(get_forge_build_no_cache_description)** -3. **$(get_forge_build_with_cache_description)** - ---- - -## Performance Comparison Tables - -EOF - - # Create unified comparison tables for each benchmark type - local benchmark_commands=("forge_test" "forge_build_no_cache" "forge_build_with_cache") - - for cmd in "${benchmark_commands[@]}"; do - local bench_name="${cmd//_/ }" - local bench_type=$(get_${cmd}_type) - local json_filename=$(get_${cmd}_json_filename) - - echo "### $bench_name" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - echo "Mean execution time in seconds (lower is better):" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - - # Create table header with proper column names - local header_row="| Project" - for version in "${FOUNDRY_VERSIONS[@]}"; do - header_row+=" | $version (s)" - done - header_row+=" |" - echo "$header_row" >> "$RESULTS_FILE" - - # Create table separator with proper alignment - local separator_row="|------" - for version in "${FOUNDRY_VERSIONS[@]}"; do - separator_row+="|--------:" - done - separator_row+="|" - echo "$separator_row" >> "$RESULTS_FILE" - - # Add data rows - for repo_name in "${REPO_NAMES[@]}"; do - local data_row="| **$repo_name**" - - for version in "${FOUNDRY_VERSIONS[@]}"; do - local clean_version="${version//v/}" - clean_version="${clean_version//\./_}" - local version_results_dir="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}" - local json_file="${version_results_dir}/${json_filename}" - - local mean_time=$(extract_mean_time "$json_file") - data_row+=" | $mean_time" - done - data_row+=" |" - echo "$data_row" >> "$RESULTS_FILE" - done - echo "" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - done - - # Add detailed version information - echo "## Foundry Version Details" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - - for version in "${FOUNDRY_VERSIONS[@]}"; do - echo "### $version" >> "$RESULTS_FILE" - echo "" >> "$RESULTS_FILE" - - # Find any version file to get the detailed version info - local clean_version="${version//v/}" - clean_version="${clean_version//\./_}" - - for repo_name in "${REPO_NAMES[@]}"; do - local version_file="${JSON_RESULTS_DIR}/${repo_name}_${clean_version}/forge_version.txt" - if [ -f "$version_file" ]; then - echo "\`\`\`" >> "$RESULTS_FILE" - cat "$version_file" >> "$RESULTS_FILE" - echo "\`\`\`" >> "$RESULTS_FILE" - break - fi - done - echo "" >> "$RESULTS_FILE" - done - - # Add notes and system info - cat >> "$RESULTS_FILE" << EOF - -## Notes - -- All benchmarks were run with hyperfine in parallel mode -- **$(get_forge_test_description)** -- **$(get_forge_build_no_cache_description)** -- **$(get_forge_build_with_cache_description)** -- Results show mean execution time in seconds -- N/A indicates benchmark failed or data unavailable - -## System Information - -- **OS:** $(uname -s) -- **Architecture:** $(uname -m) -- **Date:** $(date) - -## Raw Data - -Raw JSON benchmark data is available in: \`$JSON_RESULTS_DIR\` - -EOF - - # Copy to LATEST.md - cp "$RESULTS_FILE" "$LATEST_RESULTS_FILE" - log_success "Latest results also saved to: $LATEST_RESULTS_FILE" -} - -# Cleanup temporary files -cleanup() { - log_info "Cleanup completed" -} - -# Parse command line arguments -parse_args() { - while [[ $# -gt 0 ]]; do - case $1 in - --versions) - shift - if [[ $# -eq 0 ]]; then - log_error "--versions requires a space-separated list of versions" - exit 1 - fi - # Read versions until next flag or end of args - FOUNDRY_VERSIONS=() - while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do - FOUNDRY_VERSIONS+=("$1") - shift - done - ;; - --help|-h) - echo "Foundry Benchmarking Suite" - echo "" - echo "Usage: $0 [OPTIONS]" - echo "" - echo "OPTIONS:" - echo " --help, -h Show this help message" - echo " --version, -v Show version information" - echo " --versions ... Specify Foundry versions to benchmark" - echo " (default: from repos_and_versions.sh)" - echo "" - echo "EXAMPLES:" - echo " $0 # Use default versions (parallel)" - echo " $0 --versions stable nightly # Benchmark stable and nightly only" - echo " $0 --versions v1.0.0 v1.1.0 v1.2.0 # Benchmark specific versions" - echo "" - echo "This script benchmarks forge test and forge build commands across" - echo "multiple Foundry repositories and versions using hyperfine." - - echo "Supported version formats:" - echo " - stable, nightly (special tags)" - echo " - v1.0.0, v1.1.0, etc. (specific versions)" - echo " - nightly- (specific nightly builds)" - echo " - Any format supported by foundryup" - echo "" - echo "The script will:" - echo " 1. Install foundryup if not present" - echo " 2. Install all specified Foundry versions (preprocessing step)" - echo " 3. Clone/update target repositories" - echo " 4. Switch between versions and run benchmarks in parallel" - echo " 5. Generate comparison tables in markdown format" - echo " 6. Save results to LATEST.md" - exit 0 - ;; - --version|-v) - exit 0 - ;; - *) - log_error "Unknown option: $1" - echo "Use --help for usage information" - exit 1 - ;; - esac - done -} - -# Handle command line arguments -if [[ $# -gt 0 ]]; then - parse_args "$@" -fi - -main \ No newline at end of file diff --git a/benches/commands/forge_build_no_cache.sh b/benches/commands/forge_build_no_cache.sh deleted file mode 100755 index 756528176ac4e..0000000000000 --- a/benches/commands/forge_build_no_cache.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Forge Build (No Cache) Benchmark Command -# This file contains the configuration and execution logic for benchmarking 'forge build' with no cache - -# Command configuration -FORGE_BUILD_NO_CACHE_RUNS=5 - -# Benchmark function for forge build (no cache) -benchmark_forge_build_no_cache() { - local repo_name=$1 - local version=$2 - local version_results_dir=$3 - local log_file=$4 - - echo "Running 'forge build' (no cache) benchmark..." >> "$log_file" - - if hyperfine \ - --runs "$FORGE_BUILD_NO_CACHE_RUNS" \ - --prepare 'forge clean' \ - --export-json "${version_results_dir}/build_no_cache_results.json" \ - "forge build" 2>>"$log_file.error"; then - echo "✓ forge build (no cache) completed" >> "$log_file" - return 0 - else - echo "✗ forge build (no cache) failed" >> "$log_file" - echo "FATAL: forge build (no cache) benchmark failed" >> "$log_file" - return 1 - fi -} - -# Get command description for reporting -get_forge_build_no_cache_description() { - echo "forge build (no cache) - Clean build without cache ($FORGE_BUILD_NO_CACHE_RUNS runs, cache cleaned after each run)" -} - -# Get JSON result filename -get_forge_build_no_cache_json_filename() { - echo "build_no_cache_results.json" -} - -# Get benchmark type identifier -get_forge_build_no_cache_type() { - echo "build_no_cache" -} \ No newline at end of file diff --git a/benches/commands/forge_build_with_cache.sh b/benches/commands/forge_build_with_cache.sh deleted file mode 100755 index 25fab9dbc190f..0000000000000 --- a/benches/commands/forge_build_with_cache.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Forge Build (With Cache) Benchmark Command -# This file contains the configuration and execution logic for benchmarking 'forge build' with cache - -# Command configuration -FORGE_BUILD_WITH_CACHE_RUNS=5 -FORGE_BUILD_WITH_CACHE_WARMUP=1 - -# Benchmark function for forge build (with cache) -benchmark_forge_build_with_cache() { - local repo_name=$1 - local version=$2 - local version_results_dir=$3 - local log_file=$4 - - echo "Running 'forge build' (with cache) benchmark..." >> "$log_file" - - if hyperfine \ - --runs "$FORGE_BUILD_WITH_CACHE_RUNS" \ - --prepare 'forge build' \ - --warmup "$FORGE_BUILD_WITH_CACHE_WARMUP" \ - --export-json "${version_results_dir}/build_with_cache_results.json" \ - "forge build" 2>>"$log_file.error"; then - echo "✓ forge build (with cache) completed" >> "$log_file" - return 0 - else - echo "✗ forge build (with cache) failed" >> "$log_file" - echo "FATAL: forge build (with cache) benchmark failed" >> "$log_file" - return 1 - fi -} - -# Get command description for reporting -get_forge_build_with_cache_description() { - echo "forge build (with cache) - Build with warm cache ($FORGE_BUILD_WITH_CACHE_RUNS runs, $FORGE_BUILD_WITH_CACHE_WARMUP warmup)" -} - -# Get JSON result filename -get_forge_build_with_cache_json_filename() { - echo "build_with_cache_results.json" -} - -# Get benchmark type identifier -get_forge_build_with_cache_type() { - echo "build_with_cache" -} \ No newline at end of file diff --git a/benches/commands/forge_test.sh b/benches/commands/forge_test.sh deleted file mode 100755 index 9427d18d3c30c..0000000000000 --- a/benches/commands/forge_test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Forge Test Benchmark Command -# This file contains the configuration and execution logic for benchmarking 'forge test' - -# Command configuration -FORGE_TEST_RUNS=5 -FORGE_TEST_WARMUP=1 - -# Benchmark function for forge test -benchmark_forge_test() { - local repo_name=$1 - local version=$2 - local version_results_dir=$3 - local log_file=$4 - - echo "Running 'forge test' benchmark..." >> "$log_file" - - if hyperfine \ - --runs "$FORGE_TEST_RUNS" \ - --prepare 'forge build' \ - --warmup "$FORGE_TEST_WARMUP" \ - --export-json "${version_results_dir}/test_results.json" \ - "forge test" 2>>"$log_file.error"; then - echo "✓ forge test completed" >> "$log_file" - return 0 - else - echo "✗ forge test failed" >> "$log_file" - echo "FATAL: forge test benchmark failed" >> "$log_file" - return 1 - fi -} - -# Get command description for reporting -get_forge_test_description() { - echo "forge test - Running the test suite ($FORGE_TEST_RUNS runs, $FORGE_TEST_WARMUP warmup)" -} - -# Get JSON result filename -get_forge_test_json_filename() { - echo "test_results.json" -} - -# Get benchmark type identifier -get_forge_test_type() { - echo "test" -} \ No newline at end of file diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs new file mode 100644 index 0000000000000..f7b4a2d13f527 --- /dev/null +++ b/benches/forge_build_no_cache.rs @@ -0,0 +1,47 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{ + get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, +}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +fn benchmark_forge_build_no_cache(c: &mut Criterion) { + let mut group = c.benchmark_group("forge-build-no-cache"); + group.sample_size(SAMPLE_SIZE); + + // Setup all projects once - clone repos in parallel + let projects: Vec<_> = BENCHMARK_REPOS + .par_iter() + .map(|repo_config| { + // Setup: prepare project (clone repo) + let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + (repo_config, project) + }) + .collect(); + + // Get versions from environment variable or default + let versions = get_benchmark_versions(); + + for version in versions { + // Switch foundry version + switch_foundry_version(&version).expect("Failed to switch foundry version"); + + // Run benchmarks for each project + for (repo_config, project) in &projects { + // Format: table_name/column_name/row_name + // This creates: forge-build-no-cache/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let output = project.run_forge_build(true).expect("forge build failed"); + black_box(output); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, benchmark_forge_build_no_cache); +criterion_main!(benches); diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs new file mode 100644 index 0000000000000..154dd8a7f6100 --- /dev/null +++ b/benches/forge_build_with_cache.rs @@ -0,0 +1,50 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{ + get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, +}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +fn benchmark_forge_build_with_cache(c: &mut Criterion) { + let mut group = c.benchmark_group("forge-build-with-cache"); + group.sample_size(SAMPLE_SIZE); + + // Setup all projects once - clone repos in parallel + let projects: Vec<_> = BENCHMARK_REPOS + .par_iter() + .map(|repo_config| { + let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + (repo_config, project) + }) + .collect(); + + // Get versions from environment variable or default + let versions = get_benchmark_versions(); + + for version in versions { + // Switch foundry version once per version + switch_foundry_version(&version).expect("Failed to switch foundry version"); + + projects.par_iter().for_each(|(_repo_config, project)| { + let _ = project.run_forge_build(false); + }); + + // Run benchmarks for each project + for (repo_config, project) in &projects { + // Format: table_name/column_name/row_name + // This creates: forge-build-with-cache/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, repo_config.name); + group.bench_function(bench_id, |b| { + b.iter(|| { + println!("Benching: forge-build-with-cache/{}/{}", version, repo_config.name); + let output = project.run_forge_build(false).expect("forge build failed"); + black_box(output); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, benchmark_forge_build_with_cache); +criterion_main!(benches); diff --git a/benches/forge_test.rs b/benches/forge_test.rs new file mode 100644 index 0000000000000..b12b30a067b47 --- /dev/null +++ b/benches/forge_test.rs @@ -0,0 +1,52 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{ + get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, +}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +fn benchmark_forge_test(c: &mut Criterion) { + let mut group = c.benchmark_group("forge-test"); + group.sample_size(SAMPLE_SIZE); + + // Setup all projects once - clone repos in parallel + let projects: Vec<_> = BENCHMARK_REPOS + .par_iter() + .map(|repo_config| { + // Setup: prepare project (clone repo) + let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + (repo_config, project) + }) + .collect(); + + // Get versions from environment variable or default + let versions = get_benchmark_versions(); + + for version in versions { + // Switch foundry version once per version + switch_foundry_version(&version).expect("Failed to switch foundry version"); + + // Build all projects in parallel for this foundry version + projects.par_iter().for_each(|(_repo_config, project)| { + project.run_forge_build(false).expect("forge build failed"); + }); + + // Run benchmarks for each project + for (repo_config, project) in &projects { + // Format: table_name/column_name/row_name + // This creates: forge-test/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let output = project.run_forge_test().expect("forge test failed"); + black_box(output); + }); + }); + } + } + + group.finish(); +} + +criterion_group!(benches, benchmark_forge_test); +criterion_main!(benches); diff --git a/benches/repos_and_versions.sh b/benches/repos_and_versions.sh deleted file mode 100755 index 07da480b05d2d..0000000000000 --- a/benches/repos_and_versions.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Foundry Multi-Version Benchmarking Configuration -# This file contains the configuration for repositories and Foundry versions to benchmark - -# Foundry versions to benchmark -# Supported formats: -# - stable, nightly (special tags) -# - v1.0.0, v1.1.0, etc. (specific versions) -# - nightly- (specific nightly builds) -# - Any format supported by foundryup -FOUNDRY_VERSIONS=( - "stable" - "nightly" -) - -# Repository configurations -# Add new repositories by adding entries to both arrays -REPO_NAMES=( - "ithacaxyz-account" - # "v4-core" - "solady" - # "morpho-blue" - # "spark-psm" -) - -REPO_URLS=( - "https://github.com/ithacaxyz/account" - # "https://github.com/Uniswap/v4-core" - "https://github.com/Vectorized/solady" - # "https://github.com/morpho-org/morpho-blue" - # "https://github.com/sparkdotfi/spark-psm" -) - -# Verify arrays have the same length -if [ ${#REPO_NAMES[@]} -ne ${#REPO_URLS[@]} ]; then - echo "ERROR: REPO_NAMES and REPO_URLS arrays must have the same length" - exit 1 -fi - -# Export variables for use in other scripts -export FOUNDRY_VERSIONS -export REPO_NAMES -export REPO_URLS \ No newline at end of file diff --git a/benches/run_benchmarks.sh b/benches/run_benchmarks.sh new file mode 100755 index 0000000000000..a4b343a0ac226 --- /dev/null +++ b/benches/run_benchmarks.sh @@ -0,0 +1,359 @@ +#!/bin/bash + +# Foundry Benchmark Runner with Criterion Table Output +# This script runs the criterion-based benchmarks and generates a markdown report + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# Check if required tools are installed +check_dependencies() { + if ! command -v criterion-table &> /dev/null; then + log_error "criterion-table is not installed. Please install it with:" + echo "cargo install criterion-table" + exit 1 + fi + + if ! cargo criterion --help &> /dev/null; then + log_error "cargo-criterion is not installed. Please install it with:" + echo "cargo install cargo-criterion" + exit 1 + fi +} + +# Install Foundry versions if requested +install_foundry_versions() { + if [[ "$FORCE_INSTALL" != "true" ]]; then + return + fi + + local versions + + # Use custom versions if provided, otherwise read from lib.rs + if [[ -n "$CUSTOM_VERSIONS" ]]; then + versions=$(echo "$CUSTOM_VERSIONS" | tr ',' ' ') + log_info "Installing custom Foundry versions: $versions" + else + # Read the versions from the Rust source + versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"') + log_info "Installing default Foundry versions from lib.rs: $versions" + fi + + # Check if foundryup is available + if ! command -v foundryup &> /dev/null; then + log_error "foundryup not found. Please install Foundry first:" + echo "curl -L https://foundry.paradigm.xyz | bash" + exit 1 + fi + + # Install each version + for version in $versions; do + log_info "Installing Foundry version: $version" + if foundryup --install "$version"; then + log_success "✓ Successfully installed version $version" + else + log_error "Failed to install Foundry version: $version" + exit 1 + fi + done + + log_success "All Foundry versions installed successfully" +} + +# Get system information +get_system_info() { + local os_name=$(uname -s) + local arch=$(uname -m) + local date=$(date) + + echo "- **OS:** $os_name" + echo "- **Architecture:** $arch" + echo "- **Date:** $date" +} + + +# Run benchmarks and generate report +run_benchmarks() { + log_info "Running Foundry benchmarks..." + + # Set environment variable for custom versions if provided + if [[ -n "$CUSTOM_VERSIONS" ]]; then + export FOUNDRY_BENCH_VERSIONS="$CUSTOM_VERSIONS" + log_info "Set FOUNDRY_BENCH_VERSIONS=$CUSTOM_VERSIONS" + fi + + # Create temp files for each benchmark + local temp_dir=$(mktemp -d) + local forge_test_json="$temp_dir/forge_test.json" + local forge_build_no_cache_json="$temp_dir/forge_build_no_cache.json" + local forge_build_with_cache_json="$temp_dir/forge_build_with_cache.json" + + # Set up output redirection based on verbose flag + local output_redirect="" + if [[ "${VERBOSE:-false}" != "true" ]]; then + output_redirect="2>/dev/null" + fi + + # Run benchmarks in specific order (this determines baseline column) + log_info "Running forge_test benchmark..." + if [[ "${VERBOSE:-false}" == "true" ]]; then + cargo criterion --bench forge_test --message-format=json > "$forge_test_json" || { + log_error "forge_test benchmark failed" + exit 1 + } + else + cargo criterion --bench forge_test --message-format=json > "$forge_test_json" 2>/dev/null || { + log_error "forge_test benchmark failed" + exit 1 + } + fi + + log_info "Running forge_build_no_cache benchmark..." + if [[ "${VERBOSE:-false}" == "true" ]]; then + cargo criterion --bench forge_build_no_cache --message-format=json > "$forge_build_no_cache_json" || { + log_error "forge_build_no_cache benchmark failed" + exit 1 + } + else + cargo criterion --bench forge_build_no_cache --message-format=json > "$forge_build_no_cache_json" 2>/dev/null || { + log_error "forge_build_no_cache benchmark failed" + exit 1 + } + fi + + log_info "Running forge_build_with_cache benchmark..." + if [[ "${VERBOSE:-false}" == "true" ]]; then + cargo criterion --bench forge_build_with_cache --message-format=json > "$forge_build_with_cache_json" || { + log_error "forge_build_with_cache benchmark failed" + exit 1 + } + else + cargo criterion --bench forge_build_with_cache --message-format=json > "$forge_build_with_cache_json" 2>/dev/null || { + log_error "forge_build_with_cache benchmark failed" + exit 1 + } + fi + + # Combine all results and generate markdown + log_info "Generating markdown report with criterion-table..." + + if ! cat "$forge_test_json" "$forge_build_no_cache_json" "$forge_build_with_cache_json" | criterion-table > "$temp_dir/tables.md"; then + log_error "criterion-table failed to process benchmark data" + exit 1 + fi + + # Generate the final report + generate_report "$temp_dir/tables.md" + + # Cleanup + rm -rf "$temp_dir" + + log_success "Benchmark report generated in LATEST.md" +} + +# Generate the final markdown report +generate_report() { + local tables_file="$1" + local report_file="LATEST.md" + + log_info "Generating final report..." + + # Get current timestamp + local timestamp=$(date) + + # Get repository information and create numbered list with links + local versions + if [[ -n "$CUSTOM_VERSIONS" ]]; then + versions=$(echo "$CUSTOM_VERSIONS" | tr ',' ' ') + else + versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"' | tr '\n' ' ') + fi + + # Extract repository info for numbered list + local repo_list="" + local counter=1 + + # Parse the BENCHMARK_REPOS section + while IFS= read -r line; do + if [[ $line =~ RepoConfig.*name:.*\"([^\"]+)\".*org:.*\"([^\"]+)\".*repo:.*\"([^\"]+)\" ]]; then + local name="${BASH_REMATCH[1]}" + local org="${BASH_REMATCH[2]}" + local repo="${BASH_REMATCH[3]}" + repo_list+="$counter. [$name](https://github.com/$org/$repo)\n" + ((counter++)) + fi + done < <(grep -A 20 'pub static BENCHMARK_REPOS' src/lib.rs | grep 'RepoConfig') + + # Write the report + cat > "$report_file" << EOF +# Foundry Benchmarking Results + +**Generated on:** $timestamp +**Foundry Versions Tested:** $versions + +## Repositories Tested + +$(echo -e "$repo_list") + +## Summary + +This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects using Criterion.rs for precise performance measurements. + +The following benchmarks were performed: + +1. **forge-test** - Running the test suite (10 samples each) +2. **forge-build-no-cache** - Clean build without cache (10 samples each) +3. **forge-build-with-cache** - Build with warm cache (10 samples each) + +--- + +EOF + + # Append the criterion-table generated tables + cat "$tables_file" >> "$report_file" + + # Add system info + cat >> "$report_file" << EOF + +## System Information + +$(get_system_info) + +## Raw Data + +Detailed benchmark data and HTML reports are available in: +- \`target/criterion/\` - Individual benchmark reports + +EOF + + log_success "Report written to $report_file" +} + +# Main function +main() { + log_info "Starting Foundry benchmark suite..." + + # Check dependencies + check_dependencies + + # Install Foundry versions if --force-install is used + install_foundry_versions + + # Run benchmarks and generate report + run_benchmarks + + log_success "Benchmark suite completed successfully!" + echo "" + echo "View the results:" + echo " - Text report: cat LATEST.md" +} + +# Help function +show_help() { + cat << EOF +Foundry Benchmark Runner + +This script runs Criterion-based benchmarks for Foundry commands and generates +a markdown report using criterion-table. + +USAGE: + $0 [OPTIONS] + +OPTIONS: + -h, --help Show this help message + -v, --version Show version information + --verbose Show benchmark output (by default output is suppressed) + --versions Comma-separated list of Foundry versions to test + (e.g. stable,nightly,v1.2.0) + If not specified, uses versions from src/lib.rs + --force-install Force installation of Foundry versions + By default, assumes versions are already installed + +REQUIREMENTS: + - criterion-table: cargo install criterion-table + - cargo-criterion: cargo install cargo-criterion + - Foundry versions must be installed (or use --force-install) + +EXAMPLES: + $0 # Run with default versions + $0 --verbose # Show full output + $0 --versions stable,nightly # Test specific versions + $0 --versions stable,nightly --force-install # Install and test versions + +The script will: +1. Run forge_test, forge_build_no_cache, and forge_build_with_cache benchmarks +2. Generate comparison tables using criterion-table +3. Include system information and Foundry version details +4. Save the complete report to LATEST.md + +EOF +} + +# Default values +VERBOSE=false +FORCE_INSTALL=false +CUSTOM_VERSIONS="" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + show_help + exit 0 + ;; + -v|--version) + echo "Foundry Benchmark Runner v1.0.0" + exit 0 + ;; + --verbose) + VERBOSE=true + shift + ;; + --force-install) + FORCE_INSTALL=true + shift + ;; + --versions) + if [[ -z "$2" ]] || [[ "$2" == --* ]]; then + log_error "--versions requires a comma-separated list of versions" + exit 1 + fi + CUSTOM_VERSIONS="$2" + shift 2 + ;; + *) + log_error "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +main \ No newline at end of file diff --git a/benches/src/lib.rs b/benches/src/lib.rs new file mode 100644 index 0000000000000..b1c74b18c6142 --- /dev/null +++ b/benches/src/lib.rs @@ -0,0 +1,211 @@ +use eyre::{Result, WrapErr}; +use foundry_compilers::project_util::TempProject; +use foundry_test_utils::util::clone_remote; +use std::{ + env, + path::{Path, PathBuf}, + process::{Command, Output}, +}; + +/// Configuration for repositories to benchmark +#[derive(Debug, Clone)] +pub struct RepoConfig { + pub name: &'static str, + pub org: &'static str, + pub repo: &'static str, + pub rev: &'static str, +} + +/// Available repositories for benchmarking +pub static BENCHMARK_REPOS: &[RepoConfig] = &[ + RepoConfig { name: "ithacaxyz-account", org: "ithacaxyz", repo: "account", rev: "main" }, + // Temporarily reduced for testing + // RepoConfig { name: "solady", org: "Vectorized", repo: "solady", rev: "main" }, + // RepoConfig { name: "v4-core", org: "Uniswap", repo: "v4-core", rev: "main" }, + // RepoConfig { name: "morpho-blue", org: "morpho-org", repo: "morpho-blue", rev: "main" }, + // RepoConfig { name: "spark-psm", org: "marsfoundation", repo: "spark-psm", rev: "master" }, +]; + +/// Sample size for benchmark measurements +/// +/// This controls how many times each benchmark is run for statistical analysis. +/// Higher values provide more accurate results but take longer to complete. +pub const SAMPLE_SIZE: usize = 10; + +/// Foundry versions to benchmark +/// +/// To add more versions for comparison, install them first: +/// ```bash +/// foundryup --install stable +/// foundryup --install nightly +/// foundryup --install v0.2.0 # Example specific version +/// ``` +/// +/// Then add the version strings to this array. Supported formats: +/// - "stable" - Latest stable release +/// - "nightly" - Latest nightly build +/// - "v0.2.0" - Specific version tag +/// - "commit-hash" - Specific commit hash +/// - "nightly-" - Nightly build with specific revision +pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; + +/// A benchmark project that represents a cloned repository ready for testing +pub struct BenchmarkProject { + pub name: String, + pub temp_project: TempProject, + pub root_path: PathBuf, +} + +impl BenchmarkProject { + /// Set up a benchmark project by cloning the repository + pub fn setup(config: &RepoConfig) -> Result { + let temp_project = + TempProject::dapptools().wrap_err("Failed to create temporary project")?; + + // Get root path before clearing + let root_path = temp_project.root().to_path_buf(); + let root = root_path.to_str().unwrap(); + + // Remove all files in the directory + for entry in std::fs::read_dir(&root_path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + std::fs::remove_dir_all(&path).ok(); + } else { + std::fs::remove_file(&path).ok(); + } + } + + // Clone the repository + let repo_url = format!("https://github.com/{}/{}.git", config.org, config.repo); + clone_remote(&repo_url, root); + + // Checkout specific revision if provided + if !config.rev.is_empty() && config.rev != "main" && config.rev != "master" { + let status = Command::new("git") + .current_dir(root) + .args(["checkout", config.rev]) + .status() + .wrap_err("Failed to checkout revision")?; + + if !status.success() { + eyre::bail!("Git checkout failed for {}", config.name); + } + } + + // Git submodules are already cloned via --recursive flag + // But npm dependencies still need to be installed + Self::install_npm_dependencies(&root_path)?; + + println!(" ✅ Project {} setup complete at {}", config.name, root); + Ok(BenchmarkProject { name: config.name.to_string(), root_path, temp_project }) + } + + /// Install npm dependencies if package.json exists + fn install_npm_dependencies(root: &Path) -> Result<()> { + if root.join("package.json").exists() { + println!(" 📦 Running npm install..."); + let status = Command::new("npm") + .current_dir(root) + .args(["install"]) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()) + .status() + .wrap_err("Failed to run npm install")?; + + if !status.success() { + println!(" ⚠️ Warning: npm install failed with exit code: {:?}", status.code()); + } else { + println!(" ✅ npm install completed successfully"); + } + } + Ok(()) + } + + /// Run forge test command and return the output + pub fn run_forge_test(&self) -> Result { + Command::new("forge") + .current_dir(&self.root_path) + .args(["test"]) + .output() + .wrap_err("Failed to run forge test") + } + + /// Run forge build command and return the output + pub fn run_forge_build(&self, clean_cache: bool) -> Result { + if clean_cache { + // Clean first + let _ = Command::new("forge").current_dir(&self.root_path).args(["clean"]).output(); + } + + Command::new("forge") + .current_dir(&self.root_path) + .args(["build"]) + .output() + .wrap_err("Failed to run forge build") + } + + /// Get the root path of the project + pub fn root(&self) -> &Path { + &self.root_path + } +} + +/// Switch to a specific foundry version +pub fn switch_foundry_version(version: &str) -> Result<()> { + let output = Command::new("foundryup") + .args(["--use", version]) + .output() + .wrap_err("Failed to run foundryup")?; + + // Check if the error is about forge --version failing + let stderr = String::from_utf8_lossy(&output.stderr); + if stderr.contains("command failed") && stderr.contains("forge --version") { + eyre::bail!("Foundry binaries maybe corrupted. Please reinstall, please run `foundryup` and install the required versions."); + } + + if !output.status.success() { + eprintln!("foundryup stderr: {}", stderr); + eyre::bail!("Failed to switch to foundry version: {}", version); + } + + println!(" Successfully switched to version: {}", version); + Ok(()) +} + +/// Get the current forge version +pub fn get_forge_version() -> Result { + let output = Command::new("forge") + .args(["--version"]) + .output() + .wrap_err("Failed to get forge version")?; + + if !output.status.success() { + eyre::bail!("forge --version failed"); + } + + let version = + String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?; + + Ok(version.lines().next().unwrap_or("unknown").to_string()) +} + +/// Get Foundry versions to benchmark from environment variable or default +/// +/// Reads from FOUNDRY_BENCH_VERSIONS environment variable if set, +/// otherwise returns the default versions from FOUNDRY_VERSIONS constant. +/// +/// The environment variable should be a comma-separated list of versions, +/// e.g., "stable,nightly,v1.2.0" +pub fn get_benchmark_versions() -> Vec { + if let Ok(versions_env) = env::var("FOUNDRY_BENCH_VERSIONS") { + versions_env + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() + } else { + FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect() + } +} From 792c5923b83ae6b729f9ec50e762813e171a8385 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 30 Jun 2025 16:34:43 +0530 Subject: [PATCH 17/74] unified benchmarker - * main.rs * forge version is controlled by the bin * parses criterion json to collect results - writes to LATEST.md --- Cargo.lock | 4 + benches/Cargo.toml | 8 + benches/LATEST.md | 75 ++-- benches/forge_build_no_cache.rs | 53 ++- benches/forge_build_with_cache.rs | 51 ++- benches/forge_test.rs | 59 ++-- benches/src/lib.rs | 23 +- benches/src/main.rs | 551 ++++++++++++++++++++++++++++++ 8 files changed, 661 insertions(+), 163 deletions(-) create mode 100644 benches/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 9a469d70d4a4c..5d8cff79f6038 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4102,11 +4102,15 @@ name = "foundry-bench" version = "0.1.0" dependencies = [ "chrono", + "clap", + "color-eyre", "criterion", "eyre", "foundry-compilers", "foundry-config", "foundry-test-utils", + "num_cpus", + "once_cell", "rayon", "serde", "serde_json", diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 81c3c71e3a829..88db146544173 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -3,6 +3,10 @@ name = "foundry-bench" version = "0.1.0" edition = "2021" +[[bin]] +name = "foundry-bench" +path = "src/main.rs" + [[bench]] name = "forge_test" path = "forge_test.rs" @@ -30,6 +34,10 @@ tempfile.workspace = true tokio = { workspace = true, features = ["full"] } chrono = { version = "0.4", features = ["serde"] } rayon.workspace = true +clap = { version = "4.0", features = ["derive"] } +color-eyre.workspace = true +num_cpus = "1.16" +once_cell = "1.19" [dev-dependencies] foundry-test-utils.workspace = true diff --git a/benches/LATEST.md b/benches/LATEST.md index a2c4e59b5931b..31da10bbc401b 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,69 +1,40 @@ -# Foundry Benchmarking Results +# Foundry Benchmark Results -**Generated on:** Fri 27 Jun 2025 15:51:19 IST -**Foundry Versions Tested:** stable nightly - -## Repositories Tested - -1. [ithacaxyz-account](https://github.com/ithacaxyz/main) -2. [solady](https://github.com/Vectorized/main) -3. [v4-core](https://github.com/Uniswap/main) -4. [morpho-blue](https://github.com/morpho-org/main) -5. [spark-psm](https://github.com/marsfoundation/master) +**Date**: 2025-06-30 15:06:48 ## Summary -This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects using Criterion.rs for precise performance measurements. - -The following benchmarks were performed: - -1. **forge-test** - Running the test suite (10 samples each) -2. **forge-build-no-cache** - Clean build without cache (10 samples each) -3. **forge-build-with-cache** - Build with warm cache (10 samples each) - ---- +Benchmarked 2 Foundry versions across 1 repositories. -# Benchmarks +### Repositories Tested -## Table of Contents +1. [ithacaxyz/account](https://github.com/ithacaxyz/account) -- [Benchmark Results](#benchmark-results) - - [forge-test](#forge-test) - - [forge-build-no-cache](#forge-build-no-cache) - - [forge-build-with-cache](#forge-build-with-cache) +### Foundry Versions -## Benchmark Results +- stable +- nightly -### forge-test +## Forge Build Performance (No Cache) -| | `stable` | `nightly` | -| :---------------------- | :---------------------- | :----------------------------- | -| **`ithacaxyz-account`** | `3.75 s` (✅ **1.00x**) | `3.27 s` (✅ **1.15x faster**) | +| Repository | stable | nightly | +| ----------------- | ------- | ------- | +| ithacaxyz-account | 10.96 s | 10.72 s | -### forge-build-no-cache +## Forge Test Performance -| | `stable` | `nightly` | -| :---------------------- | :----------------------- | :------------------------------ | -| **`ithacaxyz-account`** | `14.23 s` (✅ **1.00x**) | `14.25 s` (✅ **1.00x slower**) | +| Repository | stable | nightly | +| ----------------- | ------ | ------- | +| ithacaxyz-account | 4.44 s | 3.94 s | -### forge-build-with-cache +## Forge Build Performance (With Cache) -| | `stable` | `nightly` | -| :---------------------- | :------------------------- | :-------------------------------- | -| **`ithacaxyz-account`** | `163.53 ms` (✅ **1.00x**) | `168.00 ms` (✅ **1.03x slower**) | - ---- - -Made with [criterion-table](https://github.com/nu11ptr/criterion-table) +| Repository | stable | nightly | +| ----------------- | --------- | --------- | +| ithacaxyz-account | 163.50 ms | 164.71 ms | ## System Information -- **OS:** Darwin -- **Architecture:** arm64 -- **Date:** Fri 27 Jun 2025 15:51:19 IST - -## Raw Data - -Detailed benchmark data and HTML reports are available in: - -- `target/criterion/` - Individual benchmark reports +- **OS**: macos +- **CPU**: 8 +- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs index f7b4a2d13f527..173bb5933a29e 100644 --- a/benches/forge_build_no_cache.rs +++ b/benches/forge_build_no_cache.rs @@ -1,43 +1,28 @@ -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{ - get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, -}; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; +use std::env; fn benchmark_forge_build_no_cache(c: &mut Criterion) { let mut group = c.benchmark_group("forge-build-no-cache"); group.sample_size(SAMPLE_SIZE); - // Setup all projects once - clone repos in parallel - let projects: Vec<_> = BENCHMARK_REPOS - .par_iter() - .map(|repo_config| { - // Setup: prepare project (clone repo) - let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); - (repo_config, project) - }) - .collect(); - - // Get versions from environment variable or default - let versions = get_benchmark_versions(); - - for version in versions { - // Switch foundry version - switch_foundry_version(&version).expect("Failed to switch foundry version"); - - // Run benchmarks for each project - for (repo_config, project) in &projects { - // Format: table_name/column_name/row_name - // This creates: forge-build-no-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, repo_config.name); - - group.bench_function(bench_id, |b| { - b.iter(|| { - let output = project.run_forge_build(true).expect("forge build failed"); - black_box(output); - }); + // Get the current version being tested + let version = + env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); + + println!("Running forge-build-no-cache for version: {}", version); + + let projects: Vec<_> = setup_benchmark_repos(); + + for (repo_config, project) in &projects { + // This creates: forge-build-no-cache/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let _output = project.run_forge_build(true).expect("forge build failed"); }); - } + }); } group.finish(); diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs index 154dd8a7f6100..6c92512cd28e9 100644 --- a/benches/forge_build_with_cache.rs +++ b/benches/forge_build_with_cache.rs @@ -1,46 +1,33 @@ -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{ - get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, -}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::env; fn benchmark_forge_build_with_cache(c: &mut Criterion) { let mut group = c.benchmark_group("forge-build-with-cache"); group.sample_size(SAMPLE_SIZE); - // Setup all projects once - clone repos in parallel - let projects: Vec<_> = BENCHMARK_REPOS - .par_iter() - .map(|repo_config| { - let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); - (repo_config, project) - }) - .collect(); + // Get the current version being tested + let version = + env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - // Get versions from environment variable or default - let versions = get_benchmark_versions(); + println!("Running forge-build-with-cache for version: {}", version); - for version in versions { - // Switch foundry version once per version - switch_foundry_version(&version).expect("Failed to switch foundry version"); + let projects: Vec<_> = setup_benchmark_repos(); - projects.par_iter().for_each(|(_repo_config, project)| { - let _ = project.run_forge_build(false); - }); + // Prime the cache by building once + projects.par_iter().for_each(|(_repo_config, project)| { + let _ = project.run_forge_build(false); + }); - // Run benchmarks for each project - for (repo_config, project) in &projects { - // Format: table_name/column_name/row_name - // This creates: forge-build-with-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, repo_config.name); - group.bench_function(bench_id, |b| { - b.iter(|| { - println!("Benching: forge-build-with-cache/{}/{}", version, repo_config.name); - let output = project.run_forge_build(false).expect("forge build failed"); - black_box(output); - }); + for (repo_config, project) in &projects { + // This creates: forge-build-with-cache/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, repo_config.name); + group.bench_function(bench_id, |b| { + b.iter(|| { + let _output = project.run_forge_build(false).expect("forge build failed"); }); - } + }); } group.finish(); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index b12b30a067b47..94ca5dc12a076 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -1,48 +1,33 @@ -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{ - get_benchmark_versions, switch_foundry_version, BenchmarkProject, BENCHMARK_REPOS, SAMPLE_SIZE, -}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::env; fn benchmark_forge_test(c: &mut Criterion) { let mut group = c.benchmark_group("forge-test"); group.sample_size(SAMPLE_SIZE); - // Setup all projects once - clone repos in parallel - let projects: Vec<_> = BENCHMARK_REPOS - .par_iter() - .map(|repo_config| { - // Setup: prepare project (clone repo) - let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); - (repo_config, project) - }) - .collect(); - - // Get versions from environment variable or default - let versions = get_benchmark_versions(); - - for version in versions { - // Switch foundry version once per version - switch_foundry_version(&version).expect("Failed to switch foundry version"); - - // Build all projects in parallel for this foundry version - projects.par_iter().for_each(|(_repo_config, project)| { - project.run_forge_build(false).expect("forge build failed"); - }); + // Get the current version being tested + let version = + env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); + + println!("Running forge-test for version: {}", version); + + let projects = setup_benchmark_repos(); - // Run benchmarks for each project - for (repo_config, project) in &projects { - // Format: table_name/column_name/row_name - // This creates: forge-test/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, repo_config.name); - - group.bench_function(bench_id, |b| { - b.iter(|| { - let output = project.run_forge_test().expect("forge test failed"); - black_box(output); - }); + projects.par_iter().for_each(|(_repo_config, project)| { + project.run_forge_build(false).expect("forge build failed"); + }); + + for (repo_config, project) in &projects { + // This creates: forge-test/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let _output = project.run_forge_test().expect("forge test failed"); }); - } + }); } group.finish(); diff --git a/benches/src/lib.rs b/benches/src/lib.rs index b1c74b18c6142..ab89556ac7446 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -1,6 +1,7 @@ use eyre::{Result, WrapErr}; use foundry_compilers::project_util::TempProject; use foundry_test_utils::util::clone_remote; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ env, path::{Path, PathBuf}, @@ -19,7 +20,6 @@ pub struct RepoConfig { /// Available repositories for benchmarking pub static BENCHMARK_REPOS: &[RepoConfig] = &[ RepoConfig { name: "ithacaxyz-account", org: "ithacaxyz", repo: "account", rev: "main" }, - // Temporarily reduced for testing // RepoConfig { name: "solady", org: "Vectorized", repo: "solady", rev: "main" }, // RepoConfig { name: "v4-core", org: "Uniswap", repo: "v4-core", rev: "main" }, // RepoConfig { name: "morpho-blue", org: "morpho-org", repo: "morpho-blue", rev: "main" }, @@ -192,20 +192,27 @@ pub fn get_forge_version() -> Result { } /// Get Foundry versions to benchmark from environment variable or default -/// +/// /// Reads from FOUNDRY_BENCH_VERSIONS environment variable if set, /// otherwise returns the default versions from FOUNDRY_VERSIONS constant. -/// +/// /// The environment variable should be a comma-separated list of versions, /// e.g., "stable,nightly,v1.2.0" pub fn get_benchmark_versions() -> Vec { if let Ok(versions_env) = env::var("FOUNDRY_BENCH_VERSIONS") { - versions_env - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect() + versions_env.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect() } else { FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect() } } + +/// Setup Repositories for benchmarking +pub fn setup_benchmark_repos() -> Vec<(RepoConfig, BenchmarkProject)> { + BENCHMARK_REPOS + .par_iter() + .map(|repo_config| { + let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); + (repo_config.clone(), project) + }) + .collect() +} diff --git a/benches/src/main.rs b/benches/src/main.rs new file mode 100644 index 0000000000000..6b798c4776008 --- /dev/null +++ b/benches/src/main.rs @@ -0,0 +1,551 @@ +use clap::Parser; +use color_eyre::eyre::{Result, WrapErr}; +use foundry_bench::{get_forge_version, switch_foundry_version, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + fs::File, + io::Write, + path::PathBuf, + process::{Command, Stdio}, +}; + +/// Foundry Benchmark Runner +#[derive(Parser, Debug)] +#[clap( + name = "foundry-bench", + version = "1.0.0", + about = "Run Foundry benchmarks across multiple versions" +)] +struct Cli { + /// Comma-separated list of Foundry versions to test (e.g., stable,nightly,v1.2.0) + #[clap(long, value_delimiter = ',')] + versions: Option>, + + /// Force install Foundry versions + #[clap(long)] + force_install: bool, + + /// Show verbose output + #[clap(long)] + verbose: bool, + + /// Output directory for benchmark results + #[clap(long, default_value = ".")] + output_dir: PathBuf, + + /// Run only specific benchmarks (comma-separated: + /// forge_test,forge_build_no_cache,forge_build_with_cache) + #[clap(long, value_delimiter = ',')] + benchmarks: Option>, +} + +/// Benchmark result from Criterion JSON output +#[derive(Debug, Deserialize, Serialize)] +struct CriterionResult { + reason: String, + id: Option, + report_directory: Option, + iteration_count: Option>, + measured_values: Option>, + unit: Option, + throughput: Option>, + typical: Option, + mean: Option, + median: Option, + slope: Option, + change: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +struct Throughput { + per_iteration: u64, + unit: String, +} + +#[derive(Debug, Deserialize, Serialize)] +struct Estimate { + confidence_interval: ConfidenceInterval, + point_estimate: f64, + standard_error: f64, +} + +#[derive(Debug, Deserialize, Serialize)] +struct ConfidenceInterval { + confidence_level: f64, + lower_bound: f64, + upper_bound: f64, +} + +#[derive(Debug, Deserialize, Serialize)] +struct Change { + mean: Option, + median: Option, + change: Option, // "NoChange", "Improved", or "Regressed" +} + +#[derive(Debug, Deserialize, Serialize)] +struct ChangeEstimate { + estimate: f64, + unit: String, +} + +/// Aggregated benchmark results +#[derive(Debug)] +struct BenchmarkResults { + /// Map of benchmark_name -> version -> repo -> result + data: HashMap>>, + /// Track the baseline version for comparison + baseline_version: Option, +} + +impl BenchmarkResults { + fn new() -> Self { + Self { data: HashMap::new(), baseline_version: None } + } + + fn set_baseline_version(&mut self, version: String) { + self.baseline_version = Some(version); + } + + fn add_result(&mut self, benchmark: &str, version: &str, repo: &str, result: CriterionResult) { + self.data + .entry(benchmark.to_string()) + .or_insert_with(HashMap::new) + .entry(version.to_string()) + .or_insert_with(HashMap::new) + .insert(repo.to_string(), result); + } + + fn generate_markdown(&self, versions: &[String]) -> String { + let mut output = String::new(); + + // Header + output.push_str("# Foundry Benchmark Results\n\n"); + output.push_str(&format!( + "**Date**: {}\n\n", + chrono::Local::now().format("%Y-%m-%d %H:%M:%S") + )); + + // Summary + output.push_str("## Summary\n\n"); + // Count actual repos that have results + let mut repos_with_results = std::collections::HashSet::new(); + for (_, version_data) in &self.data { + for (_, repo_data) in version_data { + for repo_name in repo_data.keys() { + repos_with_results.insert(repo_name.clone()); + } + } + } + + output.push_str(&format!( + "Benchmarked {} Foundry versions across {} repositories.\n\n", + versions.len(), + repos_with_results.len() + )); + + // Repositories tested + output.push_str("### Repositories Tested\n\n"); + for (i, repo) in BENCHMARK_REPOS.iter().enumerate() { + output.push_str(&format!( + "{}. [{}/{}](https://github.com/{}/{})\n", + i + 1, + repo.org, + repo.repo, + repo.org, + repo.repo + )); + } + output.push('\n'); + + // Versions tested + output.push_str("### Foundry Versions\n\n"); + for version in versions { + output.push_str(&format!("- {}\n", version)); + } + output.push('\n'); + + // Results for each benchmark type + for (benchmark_name, version_data) in &self.data { + output.push_str(&format!("## {}\n\n", format_benchmark_name(benchmark_name))); + + // Create table header + output.push_str("| Repository |"); + for version in versions { + output.push_str(&format!(" {} |", version)); + } + output.push('\n'); + + // Table separator + output.push_str("|------------|"); + for _ in versions { + output.push_str("----------|"); + } + output.push('\n'); + + // Table rows + for repo in BENCHMARK_REPOS { + output.push_str(&format!("| {} |", repo.name)); + + let mut values = Vec::new(); + for version in versions { + if let Some(repo_data) = version_data.get(version) { + if let Some(result) = repo_data.get(repo.name) { + if let Some(mean) = &result.mean { + let value = format_duration( + mean.point_estimate, + result.unit.as_deref().unwrap_or("ns"), + ); + output.push_str(&format!(" {} |", value)); + values.push(Some(mean.point_estimate)); + } else { + output.push_str(" N/A |"); + values.push(None); + } + } else { + output.push_str(" N/A |"); + values.push(None); + } + } else { + output.push_str(" N/A |"); + values.push(None); + } + } + + output.push('\n'); + } + output.push('\n'); + } + + // System info + output.push_str("## System Information\n\n"); + output.push_str(&format!("- **OS**: {}\n", std::env::consts::OS)); + output.push_str(&format!("- **CPU**: {}\n", num_cpus::get())); + output.push_str(&format!( + "- **Rustc**: {}\n", + get_rustc_version().unwrap_or_else(|_| "unknown".to_string()) + )); + + output + } +} + +fn format_benchmark_name(name: &str) -> String { + match name { + "forge-test" => "Forge Test Performance", + "forge-build-no-cache" => "Forge Build Performance (No Cache)", + "forge-build-with-cache" => "Forge Build Performance (With Cache)", + _ => name, + } + .to_string() +} + +fn format_duration(nanos: f64, unit: &str) -> String { + match unit { + "ns" => { + if nanos < 1_000.0 { + format!("{:.2} ns", nanos) + } else if nanos < 1_000_000.0 { + format!("{:.2} µs", nanos / 1_000.0) + } else if nanos < 1_000_000_000.0 { + format!("{:.2} ms", nanos / 1_000_000.0) + } else { + format!("{:.2} s", nanos / 1_000_000_000.0) + } + } + _ => format!("{:.2} {}", nanos, unit), + } +} + +fn get_rustc_version() -> Result { + let output = + Command::new("rustc").arg("--version").output().wrap_err("Failed to get rustc version")?; + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +use once_cell::sync::Lazy; +/// Mutex to prevent concurrent foundryup calls +use std::sync::Mutex; + +static FOUNDRY_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + +fn switch_version_safe(version: &str) -> Result<()> { + let _lock = FOUNDRY_LOCK.lock().unwrap(); + switch_foundry_version(version) +} + +fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result> { + println!(" Running {} benchmark...", name); + + // Setup paths + let criterion_dir = PathBuf::from("../target/criterion"); + let dir_name = name.replace('_', "-"); + let group_dir = criterion_dir.join(&dir_name).join(version); + + // Always run fresh benchmarks + println!(" Running fresh benchmark..."); + + // Set environment variable for the current version + std::env::set_var("FOUNDRY_BENCH_CURRENT_VERSION", version); + + // Run the benchmark + let mut cmd = Command::new("cargo"); + cmd.args(&["bench", "--bench", name]); + + if verbose { + cmd.stderr(Stdio::inherit()); + cmd.stdout(Stdio::inherit()); + } else { + cmd.stderr(Stdio::null()); + cmd.stdout(Stdio::null()); + } + + let status = cmd.status().wrap_err("Failed to run benchmark")?; + if !status.success() { + eyre::bail!("Benchmark {} failed", name); + } + + // Now read the results from the criterion output directory + let mut results = Vec::new(); + println!(" Looking for results in: {}", group_dir.display()); + println!(" Directory exists: {}", group_dir.exists()); + if group_dir.exists() { + println!(" Reading directory: {}", group_dir.display()); + for entry in std::fs::read_dir(&group_dir)? { + let entry = entry?; + let path = entry.path(); + println!(" Found entry: {}", path.display()); + if path.is_dir() { + let repo_name = path.file_name().unwrap().to_string_lossy().to_string(); + + // Only process repos that are in BENCHMARK_REPOS + let is_valid_repo = BENCHMARK_REPOS.iter().any(|r| r.name == repo_name); + if !is_valid_repo { + println!(" Skipping unknown repo: {}", repo_name); + continue; + } + + println!(" Processing repo: {}", repo_name); + let benchmark_json = path.join("new/benchmark.json"); + if benchmark_json.exists() { + let content = std::fs::read_to_string(&benchmark_json)?; + if let Ok(_benchmark_data) = serde_json::from_str::(&content) + { + // Create a CriterionResult from the benchmark.json data + let id = format!("{}/{}/{}", dir_name, version, repo_name); + + // Read estimates.json for the mean value + let estimates_json = path.join("new/estimates.json"); + if estimates_json.exists() { + let estimates_content = std::fs::read_to_string(&estimates_json)?; + if let Ok(estimates) = + serde_json::from_str::(&estimates_content) + { + if let Some(mean_obj) = estimates.get("mean") { + let mean_estimate = Estimate { + point_estimate: mean_obj["point_estimate"] + .as_f64() + .unwrap_or(0.0), + standard_error: mean_obj["standard_error"] + .as_f64() + .unwrap_or(0.0), + confidence_interval: ConfidenceInterval { + confidence_level: 0.95, + lower_bound: mean_obj["confidence_interval"] + ["lower_bound"] + .as_f64() + .unwrap_or(0.0), + upper_bound: mean_obj["confidence_interval"] + ["upper_bound"] + .as_f64() + .unwrap_or(0.0), + }, + }; + + // Check for change data + let change_json = path.join("change/estimates.json"); + let change = if change_json.exists() { + let change_content = std::fs::read_to_string(&change_json)?; + if let Ok(change_data) = + serde_json::from_str::( + &change_content, + ) + { + let mean_change = change_data.get("mean").and_then(|m| { + // The change is in decimal format (e.g., 0.03 = 3%) + let decimal = m["point_estimate"].as_f64()?; + Some(ChangeEstimate { + estimate: decimal * 100.0, // Convert to percentage + unit: "%".to_string(), + }) + }); + Some(Change { + mean: mean_change, + median: None, + change: None, + }) + } else { + None + } + } else { + None + }; + + let result = CriterionResult { + reason: "benchmark-complete".to_string(), + id: Some(id.clone()), + report_directory: None, + iteration_count: None, + measured_values: None, + unit: Some("ns".to_string()), + throughput: None, + typical: None, + mean: Some(mean_estimate), + median: None, + slope: None, + change, + }; + + println!(" Found result: {}", id); + results.push(result); + } + } + } + } + } + } + } + } + + println!(" Total results collected: {}", results.len()); + Ok(results) +} + +fn install_foundry_versions(versions: &[String]) -> Result<()> { + println!("Installing Foundry versions..."); + + for version in versions { + println!(" Installing {}...", version); + + let status = Command::new("foundryup") + .args(&["--install", version]) + .status() + .wrap_err("Failed to run foundryup")?; + + if !status.success() { + eyre::bail!("Failed to install Foundry version: {}", version); + } + } + + println!("✅ All versions installed successfully"); + Ok(()) +} + +fn main() -> Result<()> { + color_eyre::install()?; + let cli = Cli::parse(); + + // Determine versions to test + let versions = if let Some(v) = cli.versions { + v + } else { + FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect() + }; + + println!("🚀 Foundry Benchmark Runner"); + println!("📊 Testing versions: {}", versions.join(", ")); + + // Install versions if requested + if cli.force_install { + install_foundry_versions(&versions)?; + } + + // Determine benchmarks to run + let all_benchmarks = vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache"]; + let benchmarks = if let Some(b) = cli.benchmarks { + b.into_iter().filter(|b| all_benchmarks.contains(&b.as_str())).collect() + } else { + // For testing, only run forge_build_with_cache + // vec!["forge_build_with_cache".to_string()] + all_benchmarks.into_iter().map(String::from).collect::>() + }; + + println!("📈 Running benchmarks: {}", benchmarks.join(", ")); + + let mut results = BenchmarkResults::new(); + + // Set the first version as baseline + if let Some(first_version) = versions.first() { + results.set_baseline_version(first_version.clone()); + } + + // Run benchmarks for each version + for version in &versions { + println!("\n🔧 Switching to Foundry version: {}", version); + switch_version_safe(version)?; + + // Verify the switch + let current = get_forge_version()?; + println!(" Current version: {}", current.trim()); + + // Run each benchmark + for benchmark in &benchmarks { + let bench_results = run_benchmark(benchmark, version, cli.verbose)?; + + // Parse and store results + println!(" Processing {} results", bench_results.len()); + for result in bench_results { + if let Some(id) = &result.id { + println!(" Found result: {}", id); + // Parse ID format: benchmark-name/version/repo + let parts: Vec<&str> = id.split('/').collect(); + if parts.len() >= 3 { + let bench_type = parts[0].to_string(); + // Skip parts[1] which is the version (already known) + let repo = parts[2].to_string(); + + // Debug: show change info if present + if let Some(change) = &result.change { + if let Some(mean) = &change.mean { + println!( + " Change from baseline: {:.2}% ({})", + mean.estimate, + change.change.as_ref().unwrap_or(&"Unknown".to_string()) + ); + } + } + + results.add_result(&bench_type, version, &repo, result); + } + } + } + } + } + + // Generate markdown report + println!("\n📝 Generating report..."); + + // Debug: print what we have collected + println!("Collected data structure:"); + for (bench, version_data) in &results.data { + println!(" Benchmark: {}", bench); + for (version, repo_data) in version_data { + println!(" Version: {}", version); + for (repo, _) in repo_data { + println!(" Repo: {}", repo); + } + } + } + + let markdown = results.generate_markdown(&versions); + + let output_path = cli.output_dir.join("LATEST.md"); + let mut file = File::create(&output_path).wrap_err("Failed to create output file")?; + file.write_all(markdown.as_bytes()).wrap_err("Failed to write output file")?; + + println!("✅ Report written to: {}", output_path.display()); + + Ok(()) +} From b18141d450c525533ed506cdf0a44ad32ae3f3be Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 30 Jun 2025 17:28:16 +0530 Subject: [PATCH 18/74] parallel bench --- benches/LATEST.md | 26 ++++++++++++---------- benches/src/lib.rs | 6 +++-- benches/src/main.rs | 53 ++++++++++++++++++++++++++------------------- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index 31da10bbc401b..9147536d83c22 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,37 +1,41 @@ # Foundry Benchmark Results -**Date**: 2025-06-30 15:06:48 +**Date**: 2025-06-30 17:23:42 ## Summary -Benchmarked 2 Foundry versions across 1 repositories. +Benchmarked 2 Foundry versions across 2 repositories. ### Repositories Tested 1. [ithacaxyz/account](https://github.com/ithacaxyz/account) +2. [Vectorized/solady](https://github.com/Vectorized/solady) ### Foundry Versions - stable - nightly -## Forge Build Performance (No Cache) +## Forge Build Performance (With Cache) -| Repository | stable | nightly | -| ----------------- | ------- | ------- | -| ithacaxyz-account | 10.96 s | 10.72 s | +| Repository | stable | nightly | +| ----------------- | --------- | --------- | +| ithacaxyz-account | 227.20 ms | 263.73 ms | +| solady | 148.77 ms | 192.25 ms | ## Forge Test Performance | Repository | stable | nightly | | ----------------- | ------ | ------- | -| ithacaxyz-account | 4.44 s | 3.94 s | +| ithacaxyz-account | 4.88 s | 4.37 s | +| solady | 3.45 s | 3.43 s | -## Forge Build Performance (With Cache) +## Forge Build Performance (No Cache) -| Repository | stable | nightly | -| ----------------- | --------- | --------- | -| ithacaxyz-account | 163.50 ms | 164.71 ms | +| Repository | stable | nightly | +| ----------------- | ------- | ------- | +| ithacaxyz-account | 16.35 s | 13.85 s | +| solady | 15.27 s | 15.12 s | ## System Information diff --git a/benches/src/lib.rs b/benches/src/lib.rs index ab89556ac7446..a75400649dedc 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -20,7 +20,7 @@ pub struct RepoConfig { /// Available repositories for benchmarking pub static BENCHMARK_REPOS: &[RepoConfig] = &[ RepoConfig { name: "ithacaxyz-account", org: "ithacaxyz", repo: "account", rev: "main" }, - // RepoConfig { name: "solady", org: "Vectorized", repo: "solady", rev: "main" }, + RepoConfig { name: "solady", org: "Vectorized", repo: "solady", rev: "main" }, // RepoConfig { name: "v4-core", org: "Uniswap", repo: "v4-core", rev: "main" }, // RepoConfig { name: "morpho-blue", org: "morpho-org", repo: "morpho-blue", rev: "main" }, // RepoConfig { name: "spark-psm", org: "marsfoundation", repo: "spark-psm", rev: "master" }, @@ -162,7 +162,9 @@ pub fn switch_foundry_version(version: &str) -> Result<()> { // Check if the error is about forge --version failing let stderr = String::from_utf8_lossy(&output.stderr); if stderr.contains("command failed") && stderr.contains("forge --version") { - eyre::bail!("Foundry binaries maybe corrupted. Please reinstall, please run `foundryup` and install the required versions."); + eyre::bail!( + "Foundry binaries maybe corrupted. Please reinstall, please run `foundryup` and install the required versions." + ); } if !output.status.success() { diff --git a/benches/src/main.rs b/benches/src/main.rs index 6b798c4776008..f914cddbd7272 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,6 +1,7 @@ use clap::Parser; use color_eyre::eyre::{Result, WrapErr}; -use foundry_bench::{get_forge_version, switch_foundry_version, BENCHMARK_REPOS, FOUNDRY_VERSIONS}; +use foundry_bench::{BENCHMARK_REPOS, FOUNDRY_VERSIONS, get_forge_version, switch_foundry_version}; +use rayon::prelude::*; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, @@ -8,6 +9,7 @@ use std::{ io::Write, path::PathBuf, process::{Command, Stdio}, + sync::Mutex, }; /// Foundry Benchmark Runner @@ -138,7 +140,7 @@ impl BenchmarkResults { } } } - + output.push_str(&format!( "Benchmarked {} Foundry versions across {} repositories.\n\n", versions.len(), @@ -266,9 +268,8 @@ fn get_rustc_version() -> Result { } use once_cell::sync::Lazy; -/// Mutex to prevent concurrent foundryup calls -use std::sync::Mutex; +/// Mutex to prevent concurrent foundryup calls static FOUNDRY_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); fn switch_version_safe(version: &str) -> Result<()> { @@ -277,8 +278,6 @@ fn switch_version_safe(version: &str) -> Result<()> { } fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result> { - println!(" Running {} benchmark...", name); - // Setup paths let criterion_dir = PathBuf::from("../target/criterion"); let dir_name = name.replace('_', "-"); @@ -319,14 +318,14 @@ fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result Result Result<()> { let current = get_forge_version()?; println!(" Current version: {}", current.trim()); - // Run each benchmark - for benchmark in &benchmarks { - let bench_results = run_benchmark(benchmark, version, cli.verbose)?; - - // Parse and store results - println!(" Processing {} results", bench_results.len()); + // Run each benchmark in parallel + let bench_results: Vec<(String, Vec)> = benchmarks + .par_iter() + .map(|benchmark| { + println!(" Running {} benchmark...", benchmark); + let results = run_benchmark(benchmark, version, cli.verbose).unwrap_or_else(|e| { + eprintln!(" Error running benchmark {}: {}", benchmark, e); + Vec::new() + }); + (benchmark.clone(), results) + }) + .collect(); + + // Process results sequentially to maintain order in output + for (benchmark, bench_results) in bench_results { + println!(" Processing {} results for {}", bench_results.len(), benchmark); for result in bench_results { if let Some(id) = &result.id { println!(" Found result: {}", id); From 35d9861b76793a98d91703aed23b37d1b8980d8a Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 12:05:39 +0530 Subject: [PATCH 19/74] refac --- benches/src/criterion_types.rs | 51 +++++++ benches/src/lib.rs | 3 + benches/src/main.rs | 264 ++------------------------------- benches/src/results.rs | 182 +++++++++++++++++++++++ 4 files changed, 252 insertions(+), 248 deletions(-) create mode 100644 benches/src/criterion_types.rs create mode 100644 benches/src/results.rs diff --git a/benches/src/criterion_types.rs b/benches/src/criterion_types.rs new file mode 100644 index 0000000000000..d7ddd05fd1a6e --- /dev/null +++ b/benches/src/criterion_types.rs @@ -0,0 +1,51 @@ +use serde::{Deserialize, Serialize}; + +/// Benchmark result from Criterion JSON output +#[derive(Debug, Deserialize, Serialize)] +pub struct CriterionResult { + pub reason: String, + pub id: Option, + pub report_directory: Option, + pub iteration_count: Option>, + pub measured_values: Option>, + pub unit: Option, + pub throughput: Option>, + pub typical: Option, + pub mean: Option, + pub median: Option, + pub slope: Option, + pub change: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Throughput { + pub per_iteration: u64, + pub unit: String, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Estimate { + pub confidence_interval: ConfidenceInterval, + pub point_estimate: f64, + pub standard_error: f64, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct ConfidenceInterval { + pub confidence_level: f64, + pub lower_bound: f64, + pub upper_bound: f64, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Change { + pub mean: Option, + pub median: Option, + pub change: Option, // "NoChange", "Improved", or "Regressed" +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct ChangeEstimate { + pub estimate: f64, + pub unit: String, +} diff --git a/benches/src/lib.rs b/benches/src/lib.rs index a75400649dedc..e60f4b1ad1afd 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -8,6 +8,9 @@ use std::{ process::{Command, Output}, }; +pub mod criterion_types; +pub mod results; + /// Configuration for repositories to benchmark #[derive(Debug, Clone)] pub struct RepoConfig { diff --git a/benches/src/main.rs b/benches/src/main.rs index f914cddbd7272..d9b8974f4259d 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,10 +1,14 @@ use clap::Parser; use color_eyre::eyre::{Result, WrapErr}; -use foundry_bench::{BENCHMARK_REPOS, FOUNDRY_VERSIONS, get_forge_version, switch_foundry_version}; +use foundry_bench::{ + criterion_types::{Change, ChangeEstimate, ConfidenceInterval, CriterionResult, Estimate}, + get_forge_version, + results::BenchmarkResults, + switch_foundry_version, BENCHMARK_REPOS, FOUNDRY_VERSIONS, +}; +use once_cell::sync::Lazy; use rayon::prelude::*; -use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, fs::File, io::Write, path::PathBuf, @@ -14,11 +18,7 @@ use std::{ /// Foundry Benchmark Runner #[derive(Parser, Debug)] -#[clap( - name = "foundry-bench", - version = "1.0.0", - about = "Run Foundry benchmarks across multiple versions" -)] +#[clap(name = "foundry-bench", about = "Run Foundry benchmarks across multiple versions")] struct Cli { /// Comma-separated list of Foundry versions to test (e.g., stable,nightly,v1.2.0) #[clap(long, value_delimiter = ',')] @@ -42,236 +42,8 @@ struct Cli { benchmarks: Option>, } -/// Benchmark result from Criterion JSON output -#[derive(Debug, Deserialize, Serialize)] -struct CriterionResult { - reason: String, - id: Option, - report_directory: Option, - iteration_count: Option>, - measured_values: Option>, - unit: Option, - throughput: Option>, - typical: Option, - mean: Option, - median: Option, - slope: Option, - change: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -struct Throughput { - per_iteration: u64, - unit: String, -} - -#[derive(Debug, Deserialize, Serialize)] -struct Estimate { - confidence_interval: ConfidenceInterval, - point_estimate: f64, - standard_error: f64, -} - -#[derive(Debug, Deserialize, Serialize)] -struct ConfidenceInterval { - confidence_level: f64, - lower_bound: f64, - upper_bound: f64, -} - -#[derive(Debug, Deserialize, Serialize)] -struct Change { - mean: Option, - median: Option, - change: Option, // "NoChange", "Improved", or "Regressed" -} - -#[derive(Debug, Deserialize, Serialize)] -struct ChangeEstimate { - estimate: f64, - unit: String, -} - -/// Aggregated benchmark results -#[derive(Debug)] -struct BenchmarkResults { - /// Map of benchmark_name -> version -> repo -> result - data: HashMap>>, - /// Track the baseline version for comparison - baseline_version: Option, -} - -impl BenchmarkResults { - fn new() -> Self { - Self { data: HashMap::new(), baseline_version: None } - } - - fn set_baseline_version(&mut self, version: String) { - self.baseline_version = Some(version); - } - - fn add_result(&mut self, benchmark: &str, version: &str, repo: &str, result: CriterionResult) { - self.data - .entry(benchmark.to_string()) - .or_insert_with(HashMap::new) - .entry(version.to_string()) - .or_insert_with(HashMap::new) - .insert(repo.to_string(), result); - } - - fn generate_markdown(&self, versions: &[String]) -> String { - let mut output = String::new(); - - // Header - output.push_str("# Foundry Benchmark Results\n\n"); - output.push_str(&format!( - "**Date**: {}\n\n", - chrono::Local::now().format("%Y-%m-%d %H:%M:%S") - )); - - // Summary - output.push_str("## Summary\n\n"); - // Count actual repos that have results - let mut repos_with_results = std::collections::HashSet::new(); - for (_, version_data) in &self.data { - for (_, repo_data) in version_data { - for repo_name in repo_data.keys() { - repos_with_results.insert(repo_name.clone()); - } - } - } - - output.push_str(&format!( - "Benchmarked {} Foundry versions across {} repositories.\n\n", - versions.len(), - repos_with_results.len() - )); - - // Repositories tested - output.push_str("### Repositories Tested\n\n"); - for (i, repo) in BENCHMARK_REPOS.iter().enumerate() { - output.push_str(&format!( - "{}. [{}/{}](https://github.com/{}/{})\n", - i + 1, - repo.org, - repo.repo, - repo.org, - repo.repo - )); - } - output.push('\n'); - - // Versions tested - output.push_str("### Foundry Versions\n\n"); - for version in versions { - output.push_str(&format!("- {}\n", version)); - } - output.push('\n'); - - // Results for each benchmark type - for (benchmark_name, version_data) in &self.data { - output.push_str(&format!("## {}\n\n", format_benchmark_name(benchmark_name))); - - // Create table header - output.push_str("| Repository |"); - for version in versions { - output.push_str(&format!(" {} |", version)); - } - output.push('\n'); - - // Table separator - output.push_str("|------------|"); - for _ in versions { - output.push_str("----------|"); - } - output.push('\n'); - - // Table rows - for repo in BENCHMARK_REPOS { - output.push_str(&format!("| {} |", repo.name)); - - let mut values = Vec::new(); - for version in versions { - if let Some(repo_data) = version_data.get(version) { - if let Some(result) = repo_data.get(repo.name) { - if let Some(mean) = &result.mean { - let value = format_duration( - mean.point_estimate, - result.unit.as_deref().unwrap_or("ns"), - ); - output.push_str(&format!(" {} |", value)); - values.push(Some(mean.point_estimate)); - } else { - output.push_str(" N/A |"); - values.push(None); - } - } else { - output.push_str(" N/A |"); - values.push(None); - } - } else { - output.push_str(" N/A |"); - values.push(None); - } - } - - output.push('\n'); - } - output.push('\n'); - } - - // System info - output.push_str("## System Information\n\n"); - output.push_str(&format!("- **OS**: {}\n", std::env::consts::OS)); - output.push_str(&format!("- **CPU**: {}\n", num_cpus::get())); - output.push_str(&format!( - "- **Rustc**: {}\n", - get_rustc_version().unwrap_or_else(|_| "unknown".to_string()) - )); - - output - } -} - -fn format_benchmark_name(name: &str) -> String { - match name { - "forge-test" => "Forge Test Performance", - "forge-build-no-cache" => "Forge Build Performance (No Cache)", - "forge-build-with-cache" => "Forge Build Performance (With Cache)", - _ => name, - } - .to_string() -} - -fn format_duration(nanos: f64, unit: &str) -> String { - match unit { - "ns" => { - if nanos < 1_000.0 { - format!("{:.2} ns", nanos) - } else if nanos < 1_000_000.0 { - format!("{:.2} µs", nanos / 1_000.0) - } else if nanos < 1_000_000_000.0 { - format!("{:.2} ms", nanos / 1_000_000.0) - } else { - format!("{:.2} s", nanos / 1_000_000_000.0) - } - } - _ => format!("{:.2} {}", nanos, unit), - } -} - -fn get_rustc_version() -> Result { - let output = - Command::new("rustc").arg("--version").output().wrap_err("Failed to get rustc version")?; - - Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) -} - -use once_cell::sync::Lazy; - /// Mutex to prevent concurrent foundryup calls static FOUNDRY_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); - fn switch_version_safe(version: &str) -> Result<()> { let _lock = FOUNDRY_LOCK.lock().unwrap(); switch_foundry_version(version) @@ -283,9 +55,6 @@ fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result Result Result<()> { }) .collect(); - // Process results sequentially to maintain order in output for (benchmark, bench_results) in bench_results { println!(" Processing {} results for {}", bench_results.len(), benchmark); for result in bench_results { diff --git a/benches/src/results.rs b/benches/src/results.rs new file mode 100644 index 0000000000000..1ce75657b996d --- /dev/null +++ b/benches/src/results.rs @@ -0,0 +1,182 @@ +use crate::{criterion_types::CriterionResult, BENCHMARK_REPOS}; +use color_eyre::eyre::Result; +use std::{collections::HashMap, process::Command}; + +/// Aggregated benchmark results +#[derive(Debug)] +pub struct BenchmarkResults { + /// Map of benchmark_name -> version -> repo -> result + pub data: HashMap>>, + /// Track the baseline version for comparison + pub baseline_version: Option, +} + +impl BenchmarkResults { + pub fn new() -> Self { + Self { data: HashMap::new(), baseline_version: None } + } + + pub fn set_baseline_version(&mut self, version: String) { + self.baseline_version = Some(version); + } + + pub fn add_result( + &mut self, + benchmark: &str, + version: &str, + repo: &str, + result: CriterionResult, + ) { + self.data + .entry(benchmark.to_string()) + .or_insert_with(HashMap::new) + .entry(version.to_string()) + .or_insert_with(HashMap::new) + .insert(repo.to_string(), result); + } + + pub fn generate_markdown(&self, versions: &[String]) -> String { + let mut output = String::new(); + + // Header + output.push_str("# Foundry Benchmark Results\n\n"); + output.push_str(&format!( + "**Date**: {}\n\n", + chrono::Local::now().format("%Y-%m-%d %H:%M:%S") + )); + + // Summary + output.push_str("## Summary\n\n"); + // Count actual repos that have results + let mut repos_with_results = std::collections::HashSet::new(); + for (_, version_data) in &self.data { + for (_, repo_data) in version_data { + for repo_name in repo_data.keys() { + repos_with_results.insert(repo_name.clone()); + } + } + } + output.push_str(&format!( + "Benchmarked {} Foundry versions across {} repositories.\n\n", + versions.len(), + repos_with_results.len() + )); + + // Repositories tested + output.push_str("### Repositories Tested\n\n"); + for (i, repo) in BENCHMARK_REPOS.iter().enumerate() { + output.push_str(&format!( + "{}. [{}/{}](https://github.com/{}/{})\n", + i + 1, + repo.org, + repo.repo, + repo.org, + repo.repo + )); + } + output.push('\n'); + + // Versions tested + output.push_str("### Foundry Versions\n\n"); + for version in versions { + output.push_str(&format!("- {}\n", version)); + } + output.push('\n'); + + // Results for each benchmark type + for (benchmark_name, version_data) in &self.data { + output.push_str(&format!("## {}\n\n", format_benchmark_name(benchmark_name))); + + // Create table header + output.push_str("| Repository |"); + for version in versions { + output.push_str(&format!(" {} |", version)); + } + output.push('\n'); + + // Table separator + output.push_str("|------------|"); + for _ in versions { + output.push_str("----------|"); + } + output.push('\n'); + + // Table rows + for repo in BENCHMARK_REPOS { + output.push_str(&format!("| {} |", repo.name)); + + let mut values = Vec::new(); + for version in versions { + if let Some(repo_data) = version_data.get(version) { + if let Some(result) = repo_data.get(repo.name) { + if let Some(mean) = &result.mean { + let value = format_duration( + mean.point_estimate, + result.unit.as_deref().unwrap_or("ns"), + ); + output.push_str(&format!(" {} |", value)); + values.push(Some(mean.point_estimate)); + } else { + output.push_str(" N/A |"); + values.push(None); + } + } else { + output.push_str(" N/A |"); + values.push(None); + } + } else { + output.push_str(" N/A |"); + values.push(None); + } + } + + output.push('\n'); + } + output.push('\n'); + } + + // System info + output.push_str("## System Information\n\n"); + output.push_str(&format!("- **OS**: {}\n", std::env::consts::OS)); + output.push_str(&format!("- **CPU**: {}\n", num_cpus::get())); + output.push_str(&format!( + "- **Rustc**: {}\n", + get_rustc_version().unwrap_or_else(|_| "unknown".to_string()) + )); + + output + } +} + +pub fn format_benchmark_name(name: &str) -> String { + match name { + "forge-test" => "Forge Test Performance", + "forge-build-no-cache" => "Forge Build Performance (No Cache)", + "forge-build-with-cache" => "Forge Build Performance (With Cache)", + _ => name, + } + .to_string() +} + +pub fn format_duration(nanos: f64, unit: &str) -> String { + match unit { + "ns" => { + if nanos < 1_000.0 { + format!("{:.2} ns", nanos) + } else if nanos < 1_000_000.0 { + format!("{:.2} µs", nanos / 1_000.0) + } else if nanos < 1_000_000_000.0 { + format!("{:.2} ms", nanos / 1_000_000.0) + } else { + format!("{:.2} s", nanos / 1_000_000_000.0) + } + } + _ => format!("{:.2} {}", nanos, unit), + } +} + +pub fn get_rustc_version() -> Result { + let output = Command::new("rustc").arg("--version").output()?; + + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} From 239259077064b7831e357b1efe99b8d0a644ad1d Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 12:24:06 +0530 Subject: [PATCH 20/74] refac benchmark results table generation --- benches/src/results.rs | 140 ++++++++++++++++++++++++++--------------- 1 file changed, 89 insertions(+), 51 deletions(-) diff --git a/benches/src/results.rs b/benches/src/results.rs index 1ce75657b996d..46aad679bfaa6 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -85,54 +85,7 @@ impl BenchmarkResults { // Results for each benchmark type for (benchmark_name, version_data) in &self.data { - output.push_str(&format!("## {}\n\n", format_benchmark_name(benchmark_name))); - - // Create table header - output.push_str("| Repository |"); - for version in versions { - output.push_str(&format!(" {} |", version)); - } - output.push('\n'); - - // Table separator - output.push_str("|------------|"); - for _ in versions { - output.push_str("----------|"); - } - output.push('\n'); - - // Table rows - for repo in BENCHMARK_REPOS { - output.push_str(&format!("| {} |", repo.name)); - - let mut values = Vec::new(); - for version in versions { - if let Some(repo_data) = version_data.get(version) { - if let Some(result) = repo_data.get(repo.name) { - if let Some(mean) = &result.mean { - let value = format_duration( - mean.point_estimate, - result.unit.as_deref().unwrap_or("ns"), - ); - output.push_str(&format!(" {} |", value)); - values.push(Some(mean.point_estimate)); - } else { - output.push_str(" N/A |"); - values.push(None); - } - } else { - output.push_str(" N/A |"); - values.push(None); - } - } else { - output.push_str(" N/A |"); - values.push(None); - } - } - - output.push('\n'); - } - output.push('\n'); + output.push_str(&self.generate_benchmark_table(benchmark_name, version_data, versions)); } // System info @@ -146,13 +99,98 @@ impl BenchmarkResults { output } + + /// Generate a complete markdown table for a single benchmark type + /// + /// This includes the section header, table header, separator, and all rows + fn generate_benchmark_table( + &self, + benchmark_name: &str, + version_data: &HashMap>, + versions: &[String], + ) -> String { + let mut output = String::new(); + + // Section header + output.push_str(&format!("## {}\n\n", format_benchmark_name(benchmark_name))); + + // Create table header + output.push_str("| Repository |"); + for version in versions { + output.push_str(&format!(" {} |", version)); + } + output.push('\n'); + + // Table separator + output.push_str("|------------|"); + for _ in versions { + output.push_str("----------|"); + } + output.push('\n'); + + // Table rows + output.push_str(&generate_table_rows(version_data, versions)); + output.push('\n'); + + output + } +} + +/// Generate table rows for benchmark results +/// +/// This function creates the markdown table rows for each repository, +/// showing the benchmark results for each version. +fn generate_table_rows( + version_data: &HashMap>, + versions: &[String], +) -> String { + let mut output = String::new(); + + for repo in BENCHMARK_REPOS { + output.push_str(&format!("| {} |", repo.name)); + + for version in versions { + let cell_content = get_benchmark_cell_content(version_data, version, repo.name); + output.push_str(&format!(" {} |", cell_content)); + } + + output.push('\n'); + } + + output +} + +/// Get the content for a single benchmark table cell +/// +/// Returns the formatted duration or "N/A" if no data is available. +/// The nested if-let statements handle the following cases: +/// 1. Check if version data exists +/// 2. Check if repository data exists for this version +/// 3. Check if mean estimate exists for this result +fn get_benchmark_cell_content( + version_data: &HashMap>, + version: &str, + repo_name: &str, +) -> String { + // Check if we have data for this version + if let Some(repo_data) = version_data.get(version) { + // Check if we have data for this repository + if let Some(result) = repo_data.get(repo_name) { + // Check if we have a mean value to display + if let Some(mean) = &result.mean { + return format_duration(mean.point_estimate, result.unit.as_deref().unwrap_or("ns")); + } + } + } + + "N/A".to_string() } pub fn format_benchmark_name(name: &str) -> String { match name { - "forge-test" => "Forge Test Performance", - "forge-build-no-cache" => "Forge Build Performance (No Cache)", - "forge-build-with-cache" => "Forge Build Performance (With Cache)", + "forge-test" => "Forge Test", + "forge-build-no-cache" => "Forge Build (No Cache)", + "forge-build-with-cache" => "Forge Build (With Cache)", _ => name, } .to_string() From 4f896ae13c064ebd7aa1065a0e0685f5a403c6ab Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 12:52:31 +0530 Subject: [PATCH 21/74] cleanup main.rs --- benches/src/main.rs | 262 +++++++++++++++++++---------------------- benches/src/results.rs | 2 +- 2 files changed, 125 insertions(+), 139 deletions(-) diff --git a/benches/src/main.rs b/benches/src/main.rs index d9b8974f4259d..792d079cf42d4 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,5 +1,5 @@ use clap::Parser; -use color_eyre::eyre::{Result, WrapErr}; +use eyre::{OptionExt, Result, WrapErr}; use foundry_bench::{ criterion_types::{Change, ChangeEstimate, ConfidenceInterval, CriterionResult, Estimate}, get_forge_version, @@ -32,10 +32,14 @@ struct Cli { #[clap(long)] verbose: bool, - /// Output directory for benchmark results + /// Directory where the aggregated benchmark results will be written. #[clap(long, default_value = ".")] output_dir: PathBuf, + /// Name of the output file (default: LATEST.md) + #[clap(long, default_value = "LATEST.md")] + output_file: String, + /// Run only specific benchmarks (comma-separated: /// forge_test,forge_build_no_cache,forge_build_with_cache) #[clap(long, value_delimiter = ',')] @@ -77,117 +81,120 @@ fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result(&content) { + // Create a CriterionResult from the benchmark.json data + let id = format!("{}/{}/{}", dir_name, version, repo_name); + + // Read estimates.json for the mean value + let estimates_json = path.join("new/estimates.json"); + if !estimates_json.exists() { + eyre::bail!( + "Estimates JSON file does not exist for {}: {}", + repo_name, + estimates_json.display() + ); } - println!(" Processing repo: {}", repo_name); - let benchmark_json = path.join("new/benchmark.json"); - if benchmark_json.exists() { - let content = std::fs::read_to_string(&benchmark_json)?; - if let Ok(_benchmark_data) = serde_json::from_str::(&content) - { - // Create a CriterionResult from the benchmark.json data - let id = format!("{}/{}/{}", dir_name, version, repo_name); - - // Read estimates.json for the mean value - let estimates_json = path.join("new/estimates.json"); - if estimates_json.exists() { - let estimates_content = std::fs::read_to_string(&estimates_json)?; - if let Ok(estimates) = - serde_json::from_str::(&estimates_content) + let estimates_content = std::fs::read_to_string(&estimates_json)?; + if let Ok(estimates) = serde_json::from_str::(&estimates_content) + { + if let Some(mean_obj) = estimates.get("mean") { + let mean_estimate = Estimate { + point_estimate: mean_obj["point_estimate"].as_f64().unwrap_or(0.0), + standard_error: mean_obj["standard_error"].as_f64().unwrap_or(0.0), + confidence_interval: ConfidenceInterval { + confidence_level: 0.95, + lower_bound: mean_obj["confidence_interval"]["lower_bound"] + .as_f64() + .unwrap_or(0.0), + upper_bound: mean_obj["confidence_interval"]["upper_bound"] + .as_f64() + .unwrap_or(0.0), + }, + }; + + // Check for change data + let change_json = path.join("change/estimates.json"); + let change = if change_json.exists() { + let change_content = std::fs::read_to_string(&change_json)?; + if let Ok(change_data) = + serde_json::from_str::(&change_content) { - if let Some(mean_obj) = estimates.get("mean") { - let mean_estimate = Estimate { - point_estimate: mean_obj["point_estimate"] - .as_f64() - .unwrap_or(0.0), - standard_error: mean_obj["standard_error"] - .as_f64() - .unwrap_or(0.0), - confidence_interval: ConfidenceInterval { - confidence_level: 0.95, - lower_bound: mean_obj["confidence_interval"] - ["lower_bound"] - .as_f64() - .unwrap_or(0.0), - upper_bound: mean_obj["confidence_interval"] - ["upper_bound"] - .as_f64() - .unwrap_or(0.0), - }, - }; - - // Check for change data - let change_json = path.join("change/estimates.json"); - let change = if change_json.exists() { - let change_content = std::fs::read_to_string(&change_json)?; - if let Ok(change_data) = - serde_json::from_str::( - &change_content, - ) - { - let mean_change = change_data.get("mean").and_then(|m| { - // The change is in decimal format (e.g., 0.03 = 3%) - let decimal = m["point_estimate"].as_f64()?; - Some(ChangeEstimate { - estimate: decimal * 100.0, // Convert to percentage - unit: "%".to_string(), - }) - }); - Some(Change { - mean: mean_change, - median: None, - change: None, - }) - } else { - None - } - } else { - None - }; - - let result = CriterionResult { - reason: "benchmark-complete".to_string(), - id: Some(id.clone()), - report_directory: None, - iteration_count: None, - measured_values: None, - unit: Some("ns".to_string()), - throughput: None, - typical: None, - mean: Some(mean_estimate), - median: None, - slope: None, - change, - }; - - println!(" Found result: {}", id); - results.push(result); - } + let mean_change = change_data.get("mean").and_then(|m| { + // The change is in decimal format (e.g., 0.03 = 3%) + let decimal = m["point_estimate"].as_f64()?; + Some(ChangeEstimate { + estimate: decimal * 100.0, // Convert to percentage + unit: "%".to_string(), + }) + }); + Some(Change { mean: mean_change, median: None, change: None }) + } else { + None } - } + } else { + None + }; + + let result = CriterionResult { + reason: "benchmark-complete".to_string(), + id: Some(id.clone()), + report_directory: None, + iteration_count: None, + measured_values: None, + unit: Some("ns".to_string()), + throughput: None, + typical: None, + mean: Some(mean_estimate), + median: None, + slope: None, + change, + }; + + println!(" Found result: {}", id); + results.push(result); } } } } } - println!(" Total results collected: {}", results.len()); + println!("Total results collected: {}", results.len()); Ok(results) } @@ -195,7 +202,7 @@ fn install_foundry_versions(versions: &[String]) -> Result<()> { println!("Installing Foundry versions..."); for version in versions { - println!(" Installing {}...", version); + println!("Installing {}...", version); let status = Command::new("foundryup") .args(&["--install", version]) @@ -212,7 +219,6 @@ fn install_foundry_versions(versions: &[String]) -> Result<()> { } fn main() -> Result<()> { - color_eyre::install()?; let cli = Cli::parse(); // Determine versions to test @@ -223,7 +229,7 @@ fn main() -> Result<()> { }; println!("🚀 Foundry Benchmark Runner"); - println!("📊 Testing versions: {}", versions.join(", ")); + println!("Testing versions: {}", versions.join(", ")); // Install versions if requested if cli.force_install { @@ -240,10 +246,9 @@ fn main() -> Result<()> { all_benchmarks.into_iter().map(String::from).collect::>() }; - println!("📈 Running benchmarks: {}", benchmarks.join(", ")); + println!(" Running benchmarks: {}", benchmarks.join(", ")); let mut results = BenchmarkResults::new(); - // Set the first version as baseline if let Some(first_version) = versions.first() { results.set_baseline_version(first_version.clone()); @@ -251,31 +256,27 @@ fn main() -> Result<()> { // Run benchmarks for each version for version in &versions { - println!("\n🔧 Switching to Foundry version: {}", version); + println!("🔧 Switching to Foundry version: {}", version); switch_version_safe(version)?; // Verify the switch let current = get_forge_version()?; - println!(" Current version: {}", current.trim()); + println!("Current version: {}", current.trim()); // Run each benchmark in parallel let bench_results: Vec<(String, Vec)> = benchmarks .par_iter() - .map(|benchmark| { - println!(" Running {} benchmark...", benchmark); - let results = run_benchmark(benchmark, version, cli.verbose).unwrap_or_else(|e| { - eprintln!(" Error running benchmark {}: {}", benchmark, e); - Vec::new() - }); - (benchmark.clone(), results) + .map(|benchmark| -> Result<(String, Vec)> { + println!("Running {} benchmark...", benchmark); + let results = run_benchmark(benchmark, version, cli.verbose)?; + Ok((benchmark.clone(), results)) }) - .collect(); + .collect::>()?; for (benchmark, bench_results) in bench_results { - println!(" Processing {} results for {}", bench_results.len(), benchmark); + println!("Processing {} results for {}", bench_results.len(), benchmark); for result in bench_results { if let Some(id) = &result.id { - println!(" Found result: {}", id); // Parse ID format: benchmark-name/version/repo let parts: Vec<&str> = id.split('/').collect(); if parts.len() >= 3 { @@ -287,7 +288,7 @@ fn main() -> Result<()> { if let Some(change) = &result.change { if let Some(mean) = &change.mean { println!( - " Change from baseline: {:.2}% ({})", + "Change from baseline: {:.2}% ({})", mean.estimate, change.change.as_ref().unwrap_or(&"Unknown".to_string()) ); @@ -302,26 +303,11 @@ fn main() -> Result<()> { } // Generate markdown report - println!("\n📝 Generating report..."); - - // Debug: print what we have collected - println!("Collected data structure:"); - for (bench, version_data) in &results.data { - println!(" Benchmark: {}", bench); - for (version, repo_data) in version_data { - println!(" Version: {}", version); - for (repo, _) in repo_data { - println!(" Repo: {}", repo); - } - } - } - + println!("📝 Generating report..."); let markdown = results.generate_markdown(&versions); - - let output_path = cli.output_dir.join("LATEST.md"); + let output_path = cli.output_dir.join(cli.output_file); let mut file = File::create(&output_path).wrap_err("Failed to create output file")?; file.write_all(markdown.as_bytes()).wrap_err("Failed to write output file")?; - println!("✅ Report written to: {}", output_path.display()); Ok(()) diff --git a/benches/src/results.rs b/benches/src/results.rs index 46aad679bfaa6..5726040f52d8f 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -1,5 +1,5 @@ use crate::{criterion_types::CriterionResult, BENCHMARK_REPOS}; -use color_eyre::eyre::Result; +use eyre::Result; use std::{collections::HashMap, process::Command}; /// Aggregated benchmark results From 7edb40e1bfdea5bcc69b367b15aeaa77d19b5d82 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 12:52:42 +0530 Subject: [PATCH 22/74] rm dep --- Cargo.lock | 1 - benches/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d8cff79f6038..160ae81cc38a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4103,7 +4103,6 @@ version = "0.1.0" dependencies = [ "chrono", "clap", - "color-eyre", "criterion", "eyre", "foundry-compilers", diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 88db146544173..063ddad08ce9f 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -35,7 +35,6 @@ tokio = { workspace = true, features = ["full"] } chrono = { version = "0.4", features = ["serde"] } rayon.workspace = true clap = { version = "4.0", features = ["derive"] } -color-eyre.workspace = true num_cpus = "1.16" once_cell = "1.19" From d0a1525e454aaa8049f6cfa2824a68289a88937b Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:18:53 +0530 Subject: [PATCH 23/74] cleanup main.rs --- benches/src/main.rs | 309 +++++++++++++++++++++++++------------------- 1 file changed, 174 insertions(+), 135 deletions(-) diff --git a/benches/src/main.rs b/benches/src/main.rs index 792d079cf42d4..bd7bdacd64fa2 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -79,145 +79,12 @@ fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result(&content) { - // Create a CriterionResult from the benchmark.json data - let id = format!("{}/{}/{}", dir_name, version, repo_name); - - // Read estimates.json for the mean value - let estimates_json = path.join("new/estimates.json"); - if !estimates_json.exists() { - eyre::bail!( - "Estimates JSON file does not exist for {}: {}", - repo_name, - estimates_json.display() - ); - } - - let estimates_content = std::fs::read_to_string(&estimates_json)?; - if let Ok(estimates) = serde_json::from_str::(&estimates_content) - { - if let Some(mean_obj) = estimates.get("mean") { - let mean_estimate = Estimate { - point_estimate: mean_obj["point_estimate"].as_f64().unwrap_or(0.0), - standard_error: mean_obj["standard_error"].as_f64().unwrap_or(0.0), - confidence_interval: ConfidenceInterval { - confidence_level: 0.95, - lower_bound: mean_obj["confidence_interval"]["lower_bound"] - .as_f64() - .unwrap_or(0.0), - upper_bound: mean_obj["confidence_interval"]["upper_bound"] - .as_f64() - .unwrap_or(0.0), - }, - }; - - // Check for change data - let change_json = path.join("change/estimates.json"); - let change = if change_json.exists() { - let change_content = std::fs::read_to_string(&change_json)?; - if let Ok(change_data) = - serde_json::from_str::(&change_content) - { - let mean_change = change_data.get("mean").and_then(|m| { - // The change is in decimal format (e.g., 0.03 = 3%) - let decimal = m["point_estimate"].as_f64()?; - Some(ChangeEstimate { - estimate: decimal * 100.0, // Convert to percentage - unit: "%".to_string(), - }) - }); - Some(Change { mean: mean_change, median: None, change: None }) - } else { - None - } - } else { - None - }; - - let result = CriterionResult { - reason: "benchmark-complete".to_string(), - id: Some(id.clone()), - report_directory: None, - iteration_count: None, - measured_values: None, - unit: Some("ns".to_string()), - throughput: None, - typical: None, - mean: Some(mean_estimate), - median: None, - slope: None, - change, - }; - - println!(" Found result: {}", id); - results.push(result); - } - } - } - } - } - + // Collect benchmark results from criterion output + let results = collect_benchmark_results(&group_dir, &dir_name, version)?; println!("Total results collected: {}", results.len()); Ok(results) } -fn install_foundry_versions(versions: &[String]) -> Result<()> { - println!("Installing Foundry versions..."); - - for version in versions { - println!("Installing {}...", version); - - let status = Command::new("foundryup") - .args(&["--install", version]) - .status() - .wrap_err("Failed to run foundryup")?; - - if !status.success() { - eyre::bail!("Failed to install Foundry version: {}", version); - } - } - - println!("✅ All versions installed successfully"); - Ok(()) -} - fn main() -> Result<()> { let cli = Cli::parse(); @@ -312,3 +179,175 @@ fn main() -> Result<()> { Ok(()) } + +fn install_foundry_versions(versions: &[String]) -> Result<()> { + println!("Installing Foundry versions..."); + + for version in versions { + println!("Installing {}...", version); + + let status = Command::new("foundryup") + .args(&["--install", version]) + .status() + .wrap_err("Failed to run foundryup")?; + + if !status.success() { + eyre::bail!("Failed to install Foundry version: {}", version); + } + } + + println!("✅ All versions installed successfully"); + Ok(()) +} + +/// Collect benchmark results from Criterion output directory +/// +/// This function reads the Criterion JSON output files and constructs CriterionResult objects. +/// It processes: +/// - benchmark.json for basic benchmark info +/// - estimates.json for mean performance values +/// - change.json for performance change data (if available) +fn collect_benchmark_results( + group_dir: &PathBuf, + benchmark_name: &str, + version: &str, +) -> Result> { + let mut results = Vec::new(); + + println!("Looking for results in: {}", group_dir.display()); + if !group_dir.exists() { + eyre::bail!("Benchmark directory does not exist: {}", group_dir.display()); + } + + // Iterate through each repository directory + for entry in std::fs::read_dir(group_dir)? { + let entry = entry?; + let path = entry.path(); + + if !path.is_dir() { + println!("Skipping non-directory entry: {}", path.display()); + continue; + } + + let repo_name = path + .file_name() + .ok_or_eyre("Failed to get repo_name using path")? + .to_string_lossy() + .to_string(); + + // Only process repos that are in BENCHMARK_REPOS + let is_valid_repo = BENCHMARK_REPOS.iter().any(|r| r.name == repo_name); + if !is_valid_repo { + println!("Skipping unknown repo: {}", repo_name); + continue; + } + + println!("Processing repo: {}", repo_name); + + // Process the benchmark results for this repository + if let Some(result) = process_repo_benchmark(&path, benchmark_name, version, &repo_name)? { + println!("Found result: {}", result.id.as_ref().unwrap()); + results.push(result); + } + } + + Ok(results) +} + +/// Process benchmark results for a single repository +/// +/// Returns Some(CriterionResult) if valid results are found, None otherwise +fn process_repo_benchmark( + repo_path: &PathBuf, + benchmark_name: &str, + version: &str, + repo_name: &str, +) -> Result> { + let benchmark_json = repo_path.join("new/benchmark.json"); + + if !benchmark_json.exists() { + eyre::bail!( + "Benchmark JSON file does not exist for {}: {}", + repo_name, + benchmark_json.display() + ); + } + + // Read and validate benchmark.json + let content = std::fs::read_to_string(&benchmark_json)?; + let _benchmark_data = serde_json::from_str::(&content)?; + + // Create result ID + let id = format!("{}/{}/{}", benchmark_name, version, repo_name); + + // Read estimates for mean value + let mean_estimate = read_mean_estimate(repo_path, repo_name)?; + + // Read change data if available + let change = read_change_data(repo_path)?; + + Ok(Some(CriterionResult { + reason: "benchmark-complete".to_string(), + id: Some(id), + report_directory: None, + iteration_count: None, + measured_values: None, + unit: Some("ns".to_string()), + throughput: None, + typical: None, + mean: Some(mean_estimate), + median: None, + slope: None, + change, + })) +} + +/// Read mean estimate from estimates.json +fn read_mean_estimate(repo_path: &PathBuf, repo_name: &str) -> Result { + let estimates_json = repo_path.join("new/estimates.json"); + if !estimates_json.exists() { + eyre::bail!( + "Estimates JSON file does not exist for {}: {}", + repo_name, + estimates_json.display() + ); + } + + let estimates_content = std::fs::read_to_string(&estimates_json)?; + let estimates = serde_json::from_str::(&estimates_content)?; + + let mean_obj = estimates.get("mean").ok_or_eyre("No mean value found in estimates.json")?; + + Ok(Estimate { + point_estimate: mean_obj["point_estimate"].as_f64().unwrap_or(0.0), + standard_error: mean_obj["standard_error"].as_f64().unwrap_or(0.0), + confidence_interval: ConfidenceInterval { + confidence_level: 0.95, + lower_bound: mean_obj["confidence_interval"]["lower_bound"].as_f64().unwrap_or(0.0), + upper_bound: mean_obj["confidence_interval"]["upper_bound"].as_f64().unwrap_or(0.0), + }, + }) +} + +/// Read change data from change/estimates.json if it exists +fn read_change_data(repo_path: &PathBuf) -> Result> { + let change_json = repo_path.join("change/estimates.json"); + + if !change_json.exists() { + return Ok(None); + } + + let change_content = std::fs::read_to_string(&change_json)?; + let change_data = serde_json::from_str::(&change_content)?; + + let mean_change = change_data.get("mean").and_then(|m| { + // The change is in decimal format (e.g., 0.03 = 3%) + let decimal = m["point_estimate"].as_f64()?; + Some(ChangeEstimate { + estimate: decimal * 100.0, // Convert to percentage + unit: "%".to_string(), + }) + }); + + Ok(Some(Change { mean: mean_change, median: None, change: None })) +} From 03b54fd4a3de36630f077516f425553446f15aee Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:21:25 +0530 Subject: [PATCH 24/74] deser estimate --- benches/src/main.rs | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/benches/src/main.rs b/benches/src/main.rs index bd7bdacd64fa2..b9ed14393dd6c 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -206,7 +206,7 @@ fn install_foundry_versions(versions: &[String]) -> Result<()> { /// It processes: /// - benchmark.json for basic benchmark info /// - estimates.json for mean performance values -/// - change.json for performance change data (if available) +/// - change/estimates.json for performance change data (if available) fn collect_benchmark_results( group_dir: &PathBuf, benchmark_name: &str, @@ -273,16 +273,11 @@ fn process_repo_benchmark( ); } - // Read and validate benchmark.json - let content = std::fs::read_to_string(&benchmark_json)?; - let _benchmark_data = serde_json::from_str::(&content)?; - // Create result ID let id = format!("{}/{}/{}", benchmark_name, version, repo_name); - // Read estimates for mean value + // Read new estimates for mean value let mean_estimate = read_mean_estimate(repo_path, repo_name)?; - // Read change data if available let change = read_change_data(repo_path)?; @@ -315,18 +310,10 @@ fn read_mean_estimate(repo_path: &PathBuf, repo_name: &str) -> Result let estimates_content = std::fs::read_to_string(&estimates_json)?; let estimates = serde_json::from_str::(&estimates_content)?; - let mean_obj = estimates.get("mean").ok_or_eyre("No mean value found in estimates.json")?; - - Ok(Estimate { - point_estimate: mean_obj["point_estimate"].as_f64().unwrap_or(0.0), - standard_error: mean_obj["standard_error"].as_f64().unwrap_or(0.0), - confidence_interval: ConfidenceInterval { - confidence_level: 0.95, - lower_bound: mean_obj["confidence_interval"]["lower_bound"].as_f64().unwrap_or(0.0), - upper_bound: mean_obj["confidence_interval"]["upper_bound"].as_f64().unwrap_or(0.0), - }, - }) + let estimate = serde_json::from_value::(mean_obj.clone()) + .wrap_err("Failed to parse mean estimate from estimates.json")?; + Ok(estimate) } /// Read change data from change/estimates.json if it exists From 6c82d2f02fc2e45408743b3003002f902ec53e63 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:24:51 +0530 Subject: [PATCH 25/74] nit --- benches/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/benches/src/main.rs b/benches/src/main.rs index b9ed14393dd6c..6c932d9f7ddcf 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -140,6 +140,7 @@ fn main() -> Result<()> { }) .collect::>()?; + // Aggregate the results and add them to BenchmarkResults for (benchmark, bench_results) in bench_results { println!("Processing {} results for {}", bench_results.len(), benchmark); for result in bench_results { From ca5f86c7a2bedc0d19ed47f3b8f9f0e693240932 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:31:40 +0530 Subject: [PATCH 26/74] cleanup CriterionResult type --- benches/src/criterion_types.rs | 26 ++++++---------- benches/src/main.rs | 54 ++++++++++++++-------------------- benches/src/results.rs | 6 +--- 3 files changed, 32 insertions(+), 54 deletions(-) diff --git a/benches/src/criterion_types.rs b/benches/src/criterion_types.rs index d7ddd05fd1a6e..3ef888df25f1f 100644 --- a/benches/src/criterion_types.rs +++ b/benches/src/criterion_types.rs @@ -1,26 +1,18 @@ use serde::{Deserialize, Serialize}; /// Benchmark result from Criterion JSON output +/// +/// This is a simplified version containing only the fields we actually use #[derive(Debug, Deserialize, Serialize)] pub struct CriterionResult { - pub reason: String, - pub id: Option, - pub report_directory: Option, - pub iteration_count: Option>, - pub measured_values: Option>, - pub unit: Option, - pub throughput: Option>, - pub typical: Option, - pub mean: Option, - pub median: Option, - pub slope: Option, - pub change: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct Throughput { - pub per_iteration: u64, + /// Unique identifier for the benchmark result (format: benchmark-name/version/repo) + pub id: String, + /// Mean performance estimate + pub mean: Estimate, + /// Unit of measurement (always "ns" for nanoseconds in our case) pub unit: String, + /// Performance change data compared to baseline (if available) + pub change: Option, } #[derive(Debug, Deserialize, Serialize)] diff --git a/benches/src/main.rs b/benches/src/main.rs index 6c932d9f7ddcf..496eae4a2c176 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,7 +1,7 @@ use clap::Parser; use eyre::{OptionExt, Result, WrapErr}; use foundry_bench::{ - criterion_types::{Change, ChangeEstimate, ConfidenceInterval, CriterionResult, Estimate}, + criterion_types::{Change, ChangeEstimate, CriterionResult, Estimate}, get_forge_version, results::BenchmarkResults, switch_foundry_version, BENCHMARK_REPOS, FOUNDRY_VERSIONS, @@ -144,27 +144,25 @@ fn main() -> Result<()> { for (benchmark, bench_results) in bench_results { println!("Processing {} results for {}", bench_results.len(), benchmark); for result in bench_results { - if let Some(id) = &result.id { - // Parse ID format: benchmark-name/version/repo - let parts: Vec<&str> = id.split('/').collect(); - if parts.len() >= 3 { - let bench_type = parts[0].to_string(); - // Skip parts[1] which is the version (already known) - let repo = parts[2].to_string(); - - // Debug: show change info if present - if let Some(change) = &result.change { - if let Some(mean) = &change.mean { - println!( - "Change from baseline: {:.2}% ({})", - mean.estimate, - change.change.as_ref().unwrap_or(&"Unknown".to_string()) - ); - } + // Parse ID format: benchmark-name/version/repo + let parts: Vec<&str> = result.id.split('/').collect(); + if parts.len() >= 3 { + let bench_type = parts[0].to_string(); + // Skip parts[1] which is the version (already known) + let repo = parts[2].to_string(); + + // Debug: show change info if present + if let Some(change) = &result.change { + if let Some(mean) = &change.mean { + println!( + "Change from baseline: {:.2}% ({})", + mean.estimate, + change.change.as_ref().unwrap_or(&"Unknown".to_string()) + ); } - - results.add_result(&bench_type, version, &repo, result); } + + results.add_result(&bench_type, version, &repo, result); } } } @@ -247,7 +245,7 @@ fn collect_benchmark_results( // Process the benchmark results for this repository if let Some(result) = process_repo_benchmark(&path, benchmark_name, version, &repo_name)? { - println!("Found result: {}", result.id.as_ref().unwrap()); + println!("Found result: {}", result.id); results.push(result); } } @@ -283,17 +281,9 @@ fn process_repo_benchmark( let change = read_change_data(repo_path)?; Ok(Some(CriterionResult { - reason: "benchmark-complete".to_string(), - id: Some(id), - report_directory: None, - iteration_count: None, - measured_values: None, - unit: Some("ns".to_string()), - throughput: None, - typical: None, - mean: Some(mean_estimate), - median: None, - slope: None, + id, + mean: mean_estimate, + unit: "ns".to_string(), change, })) } diff --git a/benches/src/results.rs b/benches/src/results.rs index 5726040f52d8f..9c8cc43390281 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -166,7 +166,6 @@ fn generate_table_rows( /// The nested if-let statements handle the following cases: /// 1. Check if version data exists /// 2. Check if repository data exists for this version -/// 3. Check if mean estimate exists for this result fn get_benchmark_cell_content( version_data: &HashMap>, version: &str, @@ -176,10 +175,7 @@ fn get_benchmark_cell_content( if let Some(repo_data) = version_data.get(version) { // Check if we have data for this repository if let Some(result) = repo_data.get(repo_name) { - // Check if we have a mean value to display - if let Some(mean) = &result.mean { - return format_duration(mean.point_estimate, result.unit.as_deref().unwrap_or("ns")); - } + return format_duration(result.mean.point_estimate, &result.unit); } } From 6a010de60cf6d4a236674c418da3ccbe9622a24d Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 15:17:59 +0530 Subject: [PATCH 27/74] feat: specify repos via flag --- benches/forge_build_no_cache.rs | 4 +- benches/forge_build_with_cache.rs | 4 +- benches/forge_test.rs | 2 +- benches/src/lib.rs | 104 ++++++++++++++++++++++++++---- benches/src/main.rs | 56 +++++++++++----- benches/src/results.rs | 16 +++-- 6 files changed, 146 insertions(+), 40 deletions(-) diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs index 173bb5933a29e..e14229a3135df 100644 --- a/benches/forge_build_no_cache.rs +++ b/benches/forge_build_no_cache.rs @@ -12,11 +12,11 @@ fn benchmark_forge_build_no_cache(c: &mut Criterion) { println!("Running forge-build-no-cache for version: {}", version); - let projects: Vec<_> = setup_benchmark_repos(); + let projects = setup_benchmark_repos(); for (repo_config, project) in &projects { // This creates: forge-build-no-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, repo_config.name); + let bench_id = BenchmarkId::new(&version, &repo_config.name); group.bench_function(bench_id, |b| { b.iter(|| { diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs index 6c92512cd28e9..d3c896d5e785d 100644 --- a/benches/forge_build_with_cache.rs +++ b/benches/forge_build_with_cache.rs @@ -13,7 +13,7 @@ fn benchmark_forge_build_with_cache(c: &mut Criterion) { println!("Running forge-build-with-cache for version: {}", version); - let projects: Vec<_> = setup_benchmark_repos(); + let projects = setup_benchmark_repos(); // Prime the cache by building once projects.par_iter().for_each(|(_repo_config, project)| { @@ -22,7 +22,7 @@ fn benchmark_forge_build_with_cache(c: &mut Criterion) { for (repo_config, project) in &projects { // This creates: forge-build-with-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, repo_config.name); + let bench_id = BenchmarkId::new(&version, &repo_config.name); group.bench_function(bench_id, |b| { b.iter(|| { let _output = project.run_forge_build(false).expect("forge build failed"); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index 94ca5dc12a076..0ec89664cf081 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -21,7 +21,7 @@ fn benchmark_forge_test(c: &mut Criterion) { for (repo_config, project) in &projects { // This creates: forge-test/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, repo_config.name); + let bench_id = BenchmarkId::new(&version, &repo_config.name); group.bench_function(bench_id, |b| { b.iter(|| { diff --git a/benches/src/lib.rs b/benches/src/lib.rs index e60f4b1ad1afd..ce2f382912fee 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -1,6 +1,7 @@ use eyre::{Result, WrapErr}; use foundry_compilers::project_util::TempProject; use foundry_test_utils::util::clone_remote; +use once_cell::sync::Lazy; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ env, @@ -14,20 +15,82 @@ pub mod results; /// Configuration for repositories to benchmark #[derive(Debug, Clone)] pub struct RepoConfig { - pub name: &'static str, - pub org: &'static str, - pub repo: &'static str, - pub rev: &'static str, + pub name: String, + pub org: String, + pub repo: String, + pub rev: String, +} + +impl TryFrom<&str> for RepoConfig { + type Error = eyre::Error; + + fn try_from(spec: &str) -> Result { + // Split by ':' first to separate repo path from optional rev + let parts: Vec<&str> = spec.splitn(2, ':').collect(); + let repo_path = parts[0]; + let custom_rev = parts.get(1).map(|&s| s); + + // Now split the repo path by '/' + let path_parts: Vec<&str> = repo_path.split('/').collect(); + if path_parts.len() != 2 { + eyre::bail!("Invalid repo format '{}'. Expected 'org/repo' or 'org/repo:rev'", spec); + } + + let org = path_parts[0]; + let repo = path_parts[1]; + + // Try to find this repo in BENCHMARK_REPOS to get the full config + let existing_config = BENCHMARK_REPOS.iter().find(|r| r.org == org && r.repo == repo); + + let config = if let Some(existing) = existing_config { + // Use existing config but allow custom rev to override + let mut config = existing.clone(); + if let Some(rev) = custom_rev { + config.rev = rev.to_string(); + } + config + } else { + // Create new config with custom rev or default + // Name should follow the format: org-repo (with hyphen) + RepoConfig { + name: format!("{}-{}", org, repo), + org: org.to_string(), + repo: repo.to_string(), + rev: custom_rev.unwrap_or("main").to_string(), + } + }; + + println!("Parsed repo spec '{}' -> {:?}", spec, config); + Ok(config) + } } /// Available repositories for benchmarking -pub static BENCHMARK_REPOS: &[RepoConfig] = &[ - RepoConfig { name: "ithacaxyz-account", org: "ithacaxyz", repo: "account", rev: "main" }, - RepoConfig { name: "solady", org: "Vectorized", repo: "solady", rev: "main" }, - // RepoConfig { name: "v4-core", org: "Uniswap", repo: "v4-core", rev: "main" }, - // RepoConfig { name: "morpho-blue", org: "morpho-org", repo: "morpho-blue", rev: "main" }, - // RepoConfig { name: "spark-psm", org: "marsfoundation", repo: "spark-psm", rev: "master" }, -]; +pub fn default_benchmark_repos() -> Vec { + vec![ + RepoConfig { + name: "ithacaxyz-account".to_string(), + org: "ithacaxyz".to_string(), + repo: "account".to_string(), + rev: "main".to_string(), + }, + RepoConfig { + name: "solady".to_string(), + org: "Vectorized".to_string(), + repo: "solady".to_string(), + rev: "main".to_string(), + }, + // RepoConfig { name: "v4-core".to_string(), org: "Uniswap".to_string(), repo: + // "v4-core".to_string(), rev: "main".to_string() }, RepoConfig { name: + // "morpho-blue".to_string(), org: "morpho-org".to_string(), repo: + // "morpho-blue".to_string(), rev: "main".to_string() }, RepoConfig { name: + // "spark-psm".to_string(), org: "marsfoundation".to_string(), repo: + // "spark-psm".to_string(), rev: "master".to_string() }, + ] +} + +// Keep a lazy static for compatibility +pub static BENCHMARK_REPOS: Lazy> = Lazy::new(|| default_benchmark_repos()); /// Sample size for benchmark measurements /// @@ -88,7 +151,7 @@ impl BenchmarkProject { if !config.rev.is_empty() && config.rev != "main" && config.rev != "master" { let status = Command::new("git") .current_dir(root) - .args(["checkout", config.rev]) + .args(["checkout", &config.rev]) .status() .wrap_err("Failed to checkout revision")?; @@ -213,7 +276,22 @@ pub fn get_benchmark_versions() -> Vec { /// Setup Repositories for benchmarking pub fn setup_benchmark_repos() -> Vec<(RepoConfig, BenchmarkProject)> { - BENCHMARK_REPOS + // Check for FOUNDRY_BENCH_REPOS environment variable + let repos = if let Ok(repos_env) = env::var("FOUNDRY_BENCH_REPOS") { + // Parse repo specs from the environment variable + // Format should be: "org1/repo1,org2/repo2" + repos_env + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|spec| RepoConfig::try_from(spec)) + .collect::>>() + .expect("Failed to parse FOUNDRY_BENCH_REPOS") + } else { + BENCHMARK_REPOS.clone() + }; + + repos .par_iter() .map(|repo_config| { let project = BenchmarkProject::setup(repo_config).expect("Failed to setup project"); diff --git a/benches/src/main.rs b/benches/src/main.rs index 496eae4a2c176..d14d40f6c21fa 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -4,7 +4,7 @@ use foundry_bench::{ criterion_types::{Change, ChangeEstimate, CriterionResult, Estimate}, get_forge_version, results::BenchmarkResults, - switch_foundry_version, BENCHMARK_REPOS, FOUNDRY_VERSIONS, + switch_foundry_version, RepoConfig, BENCHMARK_REPOS, FOUNDRY_VERSIONS, }; use once_cell::sync::Lazy; use rayon::prelude::*; @@ -44,6 +44,11 @@ struct Cli { /// forge_test,forge_build_no_cache,forge_build_with_cache) #[clap(long, value_delimiter = ',')] benchmarks: Option>, + + /// Run only on specific repositories (comma-separated in org/repo[:rev] format: + /// ithacaxyz/account,Vectorized/solady:main,foundry-rs/foundry:v1.0.0) + #[clap(long, value_delimiter = ',')] + repos: Option>, } /// Mutex to prevent concurrent foundryup calls @@ -53,15 +58,25 @@ fn switch_version_safe(version: &str) -> Result<()> { switch_foundry_version(version) } -fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result> { +fn run_benchmark( + name: &str, + version: &str, + repos: &[RepoConfig], + verbose: bool, +) -> Result> { // Setup paths - let criterion_dir = PathBuf::from("../target/criterion"); + let criterion_dir = PathBuf::from("target/criterion"); let dir_name = name.replace('_', "-"); let group_dir = criterion_dir.join(&dir_name).join(version); // Set environment variable for the current version std::env::set_var("FOUNDRY_BENCH_CURRENT_VERSION", version); + // Set environment variable for the repos to benchmark + // Use org/repo format for proper parsing in benchmarks + let repo_specs: Vec = repos.iter().map(|r| format!("{}/{}", r.org, r.repo)).collect(); + std::env::set_var("FOUNDRY_BENCH_REPOS", repo_specs.join(",")); + // Run the benchmark let mut cmd = Command::new("cargo"); cmd.args(&["bench", "--bench", name]); @@ -80,7 +95,7 @@ fn run_benchmark(name: &str, version: &str, verbose: bool) -> Result Result<()> { FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect() }; + // Determine repos to test + let repos = if let Some(repo_specs) = cli.repos { + repo_specs + .iter() + .map(|spec| RepoConfig::try_from(spec.as_str())) + .collect::>>()? + } else { + BENCHMARK_REPOS.clone() + }; + println!("🚀 Foundry Benchmark Runner"); - println!("Testing versions: {}", versions.join(", ")); + println!("Running with versions: {}", versions.join(", ")); + println!( + "Running on repos: {}", + repos.iter().map(|r| format!("{}/{}", r.org, r.repo)).collect::>().join(", ") + ); // Install versions if requested if cli.force_install { @@ -135,7 +164,7 @@ fn main() -> Result<()> { .par_iter() .map(|benchmark| -> Result<(String, Vec)> { println!("Running {} benchmark...", benchmark); - let results = run_benchmark(benchmark, version, cli.verbose)?; + let results = run_benchmark(benchmark, version, &repos, cli.verbose)?; Ok((benchmark.clone(), results)) }) .collect::>()?; @@ -170,7 +199,7 @@ fn main() -> Result<()> { // Generate markdown report println!("📝 Generating report..."); - let markdown = results.generate_markdown(&versions); + let markdown = results.generate_markdown(&versions, &repos); let output_path = cli.output_dir.join(cli.output_file); let mut file = File::create(&output_path).wrap_err("Failed to create output file")?; file.write_all(markdown.as_bytes()).wrap_err("Failed to write output file")?; @@ -179,6 +208,7 @@ fn main() -> Result<()> { Ok(()) } + fn install_foundry_versions(versions: &[String]) -> Result<()> { println!("Installing Foundry versions..."); @@ -210,6 +240,7 @@ fn collect_benchmark_results( group_dir: &PathBuf, benchmark_name: &str, version: &str, + repos: &[RepoConfig], ) -> Result> { let mut results = Vec::new(); @@ -234,8 +265,8 @@ fn collect_benchmark_results( .to_string_lossy() .to_string(); - // Only process repos that are in BENCHMARK_REPOS - let is_valid_repo = BENCHMARK_REPOS.iter().any(|r| r.name == repo_name); + // Only process repos that are in the specified repos list + let is_valid_repo = repos.iter().any(|r| r.name == repo_name); if !is_valid_repo { println!("Skipping unknown repo: {}", repo_name); continue; @@ -280,12 +311,7 @@ fn process_repo_benchmark( // Read change data if available let change = read_change_data(repo_path)?; - Ok(Some(CriterionResult { - id, - mean: mean_estimate, - unit: "ns".to_string(), - change, - })) + Ok(Some(CriterionResult { id, mean: mean_estimate, unit: "ns".to_string(), change })) } /// Read mean estimate from estimates.json diff --git a/benches/src/results.rs b/benches/src/results.rs index 9c8cc43390281..7ab52633eab80 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -1,4 +1,4 @@ -use crate::{criterion_types::CriterionResult, BENCHMARK_REPOS}; +use crate::{criterion_types::CriterionResult, RepoConfig}; use eyre::Result; use std::{collections::HashMap, process::Command}; @@ -35,7 +35,7 @@ impl BenchmarkResults { .insert(repo.to_string(), result); } - pub fn generate_markdown(&self, versions: &[String]) -> String { + pub fn generate_markdown(&self, versions: &[String], repos: &[RepoConfig]) -> String { let mut output = String::new(); // Header @@ -64,7 +64,7 @@ impl BenchmarkResults { // Repositories tested output.push_str("### Repositories Tested\n\n"); - for (i, repo) in BENCHMARK_REPOS.iter().enumerate() { + for (i, repo) in repos.iter().enumerate() { output.push_str(&format!( "{}. [{}/{}](https://github.com/{}/{})\n", i + 1, @@ -85,7 +85,7 @@ impl BenchmarkResults { // Results for each benchmark type for (benchmark_name, version_data) in &self.data { - output.push_str(&self.generate_benchmark_table(benchmark_name, version_data, versions)); + output.push_str(&self.generate_benchmark_table(benchmark_name, version_data, versions, repos)); } // System info @@ -108,6 +108,7 @@ impl BenchmarkResults { benchmark_name: &str, version_data: &HashMap>, versions: &[String], + repos: &[RepoConfig], ) -> String { let mut output = String::new(); @@ -129,7 +130,7 @@ impl BenchmarkResults { output.push('\n'); // Table rows - output.push_str(&generate_table_rows(version_data, versions)); + output.push_str(&generate_table_rows(version_data, versions, repos)); output.push('\n'); output @@ -143,14 +144,15 @@ impl BenchmarkResults { fn generate_table_rows( version_data: &HashMap>, versions: &[String], + repos: &[RepoConfig], ) -> String { let mut output = String::new(); - for repo in BENCHMARK_REPOS { + for repo in repos { output.push_str(&format!("| {} |", repo.name)); for version in versions { - let cell_content = get_benchmark_cell_content(version_data, version, repo.name); + let cell_content = get_benchmark_cell_content(version_data, version, &repo.name); output.push_str(&format!(" {} |", cell_content)); } From 0a58ecc979bf279ae059da4e07307f5c2704819d Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 15:32:28 +0530 Subject: [PATCH 28/74] nits --- benches/forge_build_no_cache.rs | 2 +- benches/forge_build_with_cache.rs | 2 +- benches/forge_test.rs | 2 +- benches/src/lib.rs | 14 +++++++------- benches/src/main.rs | 24 +++++++++++------------- benches/src/results.rs | 29 +++++++++++++++++------------ 6 files changed, 38 insertions(+), 35 deletions(-) diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs index e14229a3135df..3dd0e536f747f 100644 --- a/benches/forge_build_no_cache.rs +++ b/benches/forge_build_no_cache.rs @@ -10,7 +10,7 @@ fn benchmark_forge_build_no_cache(c: &mut Criterion) { let version = env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - println!("Running forge-build-no-cache for version: {}", version); + println!("Running forge-build-no-cache for version: {version}"); let projects = setup_benchmark_repos(); diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs index d3c896d5e785d..5a8691c2a33e0 100644 --- a/benches/forge_build_with_cache.rs +++ b/benches/forge_build_with_cache.rs @@ -11,7 +11,7 @@ fn benchmark_forge_build_with_cache(c: &mut Criterion) { let version = env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - println!("Running forge-build-with-cache for version: {}", version); + println!("Running forge-build-with-cache for version: {version}"); let projects = setup_benchmark_repos(); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index 0ec89664cf081..e3205d02ac335 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -11,7 +11,7 @@ fn benchmark_forge_test(c: &mut Criterion) { let version = env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - println!("Running forge-test for version: {}", version); + println!("Running forge-test for version: {version}"); let projects = setup_benchmark_repos(); diff --git a/benches/src/lib.rs b/benches/src/lib.rs index f175490fb9c37..f0ecffcf3c8d8 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -28,7 +28,7 @@ impl TryFrom<&str> for RepoConfig { // Split by ':' first to separate repo path from optional rev let parts: Vec<&str> = spec.splitn(2, ':').collect(); let repo_path = parts[0]; - let custom_rev = parts.get(1).map(|&s| s); + let custom_rev = parts.get(1).copied(); // Now split the repo path by '/' let path_parts: Vec<&str> = repo_path.split('/').collect(); @@ -53,14 +53,14 @@ impl TryFrom<&str> for RepoConfig { // Create new config with custom rev or default // Name should follow the format: org-repo (with hyphen) RepoConfig { - name: format!("{}-{}", org, repo), + name: format!("{org}-{repo}"), org: org.to_string(), repo: repo.to_string(), rev: custom_rev.unwrap_or("main").to_string(), } }; - println!("Parsed repo spec '{}' -> {:?}", spec, config); + println!("Parsed repo spec '{spec}' -> {config:?}"); Ok(config) } } @@ -90,7 +90,7 @@ pub fn default_benchmark_repos() -> Vec { } // Keep a lazy static for compatibility -pub static BENCHMARK_REPOS: Lazy> = Lazy::new(|| default_benchmark_repos()); +pub static BENCHMARK_REPOS: Lazy> = Lazy::new(default_benchmark_repos); /// Sample size for benchmark measurements /// @@ -234,11 +234,11 @@ pub fn switch_foundry_version(version: &str) -> Result<()> { } if !output.status.success() { - eprintln!("foundryup stderr: {}", stderr); + eprintln!("foundryup stderr: {stderr}"); eyre::bail!("Failed to switch to foundry version: {}", version); } - println!(" Successfully switched to version: {}", version); + println!(" Successfully switched to version: {version}"); Ok(()) } @@ -284,7 +284,7 @@ pub fn setup_benchmark_repos() -> Vec<(RepoConfig, BenchmarkProject)> { .split(',') .map(|s| s.trim()) .filter(|s| !s.is_empty()) - .map(|spec| RepoConfig::try_from(spec)) + .map(RepoConfig::try_from) .collect::>>() .expect("Failed to parse FOUNDRY_BENCH_REPOS") } else { diff --git a/benches/src/main.rs b/benches/src/main.rs index d14d40f6c21fa..143cfb3e05ab2 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -11,7 +11,7 @@ use rayon::prelude::*; use std::{ fs::File, io::Write, - path::PathBuf, + path::{Path, PathBuf}, process::{Command, Stdio}, sync::Mutex, }; @@ -79,7 +79,7 @@ fn run_benchmark( // Run the benchmark let mut cmd = Command::new("cargo"); - cmd.args(&["bench", "--bench", name]); + cmd.args(["bench", "--bench", name]); if verbose { cmd.stderr(Stdio::inherit()); @@ -152,7 +152,7 @@ fn main() -> Result<()> { // Run benchmarks for each version for version in &versions { - println!("🔧 Switching to Foundry version: {}", version); + println!("🔧 Switching to Foundry version: {version}"); switch_version_safe(version)?; // Verify the switch @@ -163,7 +163,7 @@ fn main() -> Result<()> { let bench_results: Vec<(String, Vec)> = benchmarks .par_iter() .map(|benchmark| -> Result<(String, Vec)> { - println!("Running {} benchmark...", benchmark); + println!("Running {benchmark} benchmark..."); let results = run_benchmark(benchmark, version, &repos, cli.verbose)?; Ok((benchmark.clone(), results)) }) @@ -208,15 +208,14 @@ fn main() -> Result<()> { Ok(()) } - fn install_foundry_versions(versions: &[String]) -> Result<()> { println!("Installing Foundry versions..."); for version in versions { - println!("Installing {}...", version); + println!("Installing {version}..."); let status = Command::new("foundryup") - .args(&["--install", version]) + .args(["--install", version]) .status() .wrap_err("Failed to run foundryup")?; @@ -268,15 +267,14 @@ fn collect_benchmark_results( // Only process repos that are in the specified repos list let is_valid_repo = repos.iter().any(|r| r.name == repo_name); if !is_valid_repo { - println!("Skipping unknown repo: {}", repo_name); + println!("Skipping unknown repo: {repo_name}"); continue; } - println!("Processing repo: {}", repo_name); + println!("Processing repo: {repo_name}"); // Process the benchmark results for this repository if let Some(result) = process_repo_benchmark(&path, benchmark_name, version, &repo_name)? { - println!("Found result: {}", result.id); results.push(result); } } @@ -304,7 +302,7 @@ fn process_repo_benchmark( } // Create result ID - let id = format!("{}/{}/{}", benchmark_name, version, repo_name); + let id = format!("{benchmark_name}/{version}/{repo_name}"); // Read new estimates for mean value let mean_estimate = read_mean_estimate(repo_path, repo_name)?; @@ -315,7 +313,7 @@ fn process_repo_benchmark( } /// Read mean estimate from estimates.json -fn read_mean_estimate(repo_path: &PathBuf, repo_name: &str) -> Result { +fn read_mean_estimate(repo_path: &Path, repo_name: &str) -> Result { let estimates_json = repo_path.join("new/estimates.json"); if !estimates_json.exists() { eyre::bail!( @@ -334,7 +332,7 @@ fn read_mean_estimate(repo_path: &PathBuf, repo_name: &str) -> Result } /// Read change data from change/estimates.json if it exists -fn read_change_data(repo_path: &PathBuf) -> Result> { +fn read_change_data(repo_path: &Path) -> Result> { let change_json = repo_path.join("change/estimates.json"); if !change_json.exists() { diff --git a/benches/src/results.rs b/benches/src/results.rs index 7ab52633eab80..d36ff3dd460ae 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -3,7 +3,7 @@ use eyre::Result; use std::{collections::HashMap, process::Command}; /// Aggregated benchmark results -#[derive(Debug)] +#[derive(Debug, Default)] pub struct BenchmarkResults { /// Map of benchmark_name -> version -> repo -> result pub data: HashMap>>, @@ -13,7 +13,7 @@ pub struct BenchmarkResults { impl BenchmarkResults { pub fn new() -> Self { - Self { data: HashMap::new(), baseline_version: None } + Self::default() } pub fn set_baseline_version(&mut self, version: String) { @@ -29,9 +29,9 @@ impl BenchmarkResults { ) { self.data .entry(benchmark.to_string()) - .or_insert_with(HashMap::new) + .or_default() .entry(version.to_string()) - .or_insert_with(HashMap::new) + .or_default() .insert(repo.to_string(), result); } @@ -49,8 +49,8 @@ impl BenchmarkResults { output.push_str("## Summary\n\n"); // Count actual repos that have results let mut repos_with_results = std::collections::HashSet::new(); - for (_, version_data) in &self.data { - for (_, repo_data) in version_data { + for version_data in self.data.values() { + for repo_data in version_data.values() { for repo_name in repo_data.keys() { repos_with_results.insert(repo_name.clone()); } @@ -79,13 +79,18 @@ impl BenchmarkResults { // Versions tested output.push_str("### Foundry Versions\n\n"); for version in versions { - output.push_str(&format!("- {}\n", version)); + output.push_str(&format!("- {version}\n")); } output.push('\n'); // Results for each benchmark type for (benchmark_name, version_data) in &self.data { - output.push_str(&self.generate_benchmark_table(benchmark_name, version_data, versions, repos)); + output.push_str(&self.generate_benchmark_table( + benchmark_name, + version_data, + versions, + repos, + )); } // System info @@ -118,7 +123,7 @@ impl BenchmarkResults { // Create table header output.push_str("| Repository |"); for version in versions { - output.push_str(&format!(" {} |", version)); + output.push_str(&format!(" {version} |")); } output.push('\n'); @@ -153,7 +158,7 @@ fn generate_table_rows( for version in versions { let cell_content = get_benchmark_cell_content(version_data, version, &repo.name); - output.push_str(&format!(" {} |", cell_content)); + output.push_str(&format!(" {cell_content} |")); } output.push('\n'); @@ -198,7 +203,7 @@ pub fn format_duration(nanos: f64, unit: &str) -> String { match unit { "ns" => { if nanos < 1_000.0 { - format!("{:.2} ns", nanos) + format!("{nanos:.2} ns") } else if nanos < 1_000_000.0 { format!("{:.2} µs", nanos / 1_000.0) } else if nanos < 1_000_000_000.0 { @@ -207,7 +212,7 @@ pub fn format_duration(nanos: f64, unit: &str) -> String { format!("{:.2} s", nanos / 1_000_000_000.0) } } - _ => format!("{:.2} {}", nanos, unit), + _ => format!("{nanos:.2} {unit}"), } } From c395527c6e60910fd638ae39c03228830521896f Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 15:43:12 +0530 Subject: [PATCH 29/74] update bench ci and README --- .github/workflows/benchmarks.yml | 47 ++++++++++++--- benches/README.md | 100 ++++++++++++++++++++++++------- 2 files changed, 115 insertions(+), 32 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d5c6e0cecf501..aceb45815477d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -7,6 +7,19 @@ on: description: "PR number to comment on (optional)" required: false type: string + versions: + description: "Comma-separated list of Foundry versions to benchmark (e.g., stable,nightly,v1.0.0)" + required: false + type: string + default: "stable,nightly" + repos: + description: "Comma-separated list of repos to benchmark (e.g., ithacaxyz/account:main,Vectorized/solady)" + required: false + type: string + benchmarks: + description: "Comma-separated list of benchmarks to run (forge_test,forge_build_no_cache,forge_build_with_cache)" + required: false + type: string permissions: contents: write @@ -36,16 +49,33 @@ jobs: curl -L https://foundry.paradigm.xyz | bash echo "$HOME/.foundry/bin" >> $GITHUB_PATH - - name: Install benchmark dependencies - run: | - cargo install cargo-criterion - cargo install criterion-table + - name: Build benchmark binary + run: cargo build --release --bin foundry-bench - name: Run benchmarks - working-directory: ./benches run: | - chmod +x run_benchmarks.sh - ./run_benchmarks.sh + VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" + REPOS="${{ github.event.inputs.repos }}" + BENCHMARKS="${{ github.event.inputs.benchmarks }}" + + # Build the command with force-install for CI + CMD="./target/release/foundry-bench --output-dir ./benches --force-install" + + # Add versions + CMD="$CMD --versions $VERSIONS" + + # Add repos if specified + if [ -n "$REPOS" ]; then + CMD="$CMD --repos $REPOS" + fi + + # Add benchmarks if specified + if [ -n "$BENCHMARKS" ]; then + CMD="$CMD --benchmarks $BENCHMARKS" + fi + + echo "Running: $CMD" + $CMD - name: Commit benchmark results run: | @@ -55,7 +85,7 @@ jobs: if git diff --staged --quiet; then echo "No changes to commit" else - git commit -m "Update benchmark results + git commit -m "chore(`benches`): update benchmark results 🤖 Generated with [Foundry Benchmarks](https://github.com/${{ github.repository }}/actions) @@ -105,4 +135,3 @@ jobs: repo: context.repo.repo, body: comment }); - diff --git a/benches/README.md b/benches/README.md index b3663831853b1..dd73ae4211b60 100644 --- a/benches/README.md +++ b/benches/README.md @@ -27,39 +27,71 @@ Before running the benchmarks, ensure you have the following installed: # Install Node.js and npm from https://nodejs.org/ ``` -5. **Benchmark tools** - Required for generating reports - ```bash - cargo install cargo-criterion - cargo install criterion-table - ``` - ## Running Benchmarks -### Run the complete benchmark suite +### Using the Benchmark Binary + +Build the benchmark runner: ```bash -cargo run +cargo build --release --bin foundry-bench ``` -This will: +#### Run with default settings -1. Check and install required Foundry versions -2. Run all benchmark suites (forge_test, forge_build_no_cache, forge_build_with_cache) -3. Generate comparison tables using criterion-table -4. Create the final LATEST.md report +```bash +# Run all benchmarks on default repos with stable and nightly versions +cargo run --release --bin foundry-bench -- --versions stable,nightly +``` -### Run individual benchmark suites +#### Run with custom configurations ```bash -./run_benchmarks.sh +# Bench specific versions +cargo run --release --bin foundry-bench -- --versions stable,nightly,v1.0.0 + +# Run on specific repositories. Default rev for the repo is "main" +cargo run --release --bin foundry-bench -- --repos ithacaxyz/account,Vectorized/solady + +# Test specific repository with custom revision +cargo run --release --bin foundry-bench -- --repos ithacaxyz/account:main,Vectorized/solady:v0.0.123 + +# Run only specific benchmarks +cargo run --release --bin foundry-bench -- --benchmarks forge_build_with_cache,forge_test + +# Combine options +cargo run --release --bin foundry-bench -- \ + --versions stable,nightly \ + --repos ithacaxyz/account \ + --benchmarks forge_build_with_cache + +# Force install Foundry versions +cargo run --release --bin foundry-bench -- --force-install + +# Verbose output to see criterion logs +cargo run --release --bin foundry-bench -- --verbose + +# Output to specific directory +cargo run --release --bin foundry-bench -- --output-dir ./results --output-file LATEST_RESULTS.md ``` -### Run specific benchmark +#### Command-line Options + +- `--versions ` - Comma-separated list of Foundry versions (default: stable,nightly) +- `--repos ` - Comma-separated list of repos in org/repo[:rev] format +- `--benchmarks ` - Comma-separated list of benchmarks to run +- `--force-install` - Force installation of Foundry versions +- `--verbose` - Show detailed benchmark output +- `--output-dir ` - Directory for output files (default: current directory) +- `--output-file ` - Name of the output file (default: LATEST.md) + +### Run individual Criterion benchmarks ```bash -cargo criterion --bench forge_test -cargo criterion --bench forge_build_no_cache -cargo criterion --bench forge_build_with_cache +# Run specific benchmark with Criterion +cargo bench --bench forge_test +cargo bench --bench forge_build_no_cache +cargo bench --bench forge_build_with_cache ``` ## Benchmark Structure @@ -91,11 +123,33 @@ pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; ## Results -Benchmark results are displayed in the terminal and saved as HTML reports. The reports show: +Benchmark results are saved to `LATEST.md` (or custom output directory). The report includes: + +- Summary of versions and repositories tested +- Performance comparison tables for each benchmark type +- Execution time statistics for each repository/version combination +- System information (OS, CPU, Rust version) + +Results are also stored in Criterion's format in `target/criterion/` for detailed analysis. + +## GitHub Actions Integration + +The benchmarks can be run automatically via GitHub Actions: + +1. Go to the [Actions tab](../../actions/workflows/benchmarks.yml) +2. Click on "Foundry Benchmarks" workflow +3. Click "Run workflow" +4. Configure options: + - PR number (optional) - Add benchmark results as a comment to a PR + - Versions - Foundry versions to test (default: stable,nightly) + - Repos - Custom repositories to benchmark + - Benchmarks - Specific benchmarks to run + +The workflow will: -- Execution time statistics (mean, median, standard deviation) -- Comparison between different Foundry versions -- Performance trends across repositories +- Build and run the benchmark binary +- Commit results to `benches/LATEST.md` +- Optionally comment on a PR with the results ## Troubleshooting From ad5ef6ecaa097c899af121cc7508d016ba0ece22 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:51:44 +0530 Subject: [PATCH 30/74] bench fuzz tests --- .github/workflows/benchmarks.yml | 2 +- benches/Cargo.toml | 5 + benches/FUZZ_BENCH.md | 30 +++ benches/README.md | 5 + benches/forge_fuzz_test.rs | 37 ++++ benches/run_benchmarks.sh | 359 ------------------------------- benches/src/lib.rs | 10 + benches/src/main.rs | 12 +- benches/src/results.rs | 1 + test-forge-regex | 1 + 10 files changed, 97 insertions(+), 365 deletions(-) create mode 100644 benches/FUZZ_BENCH.md create mode 100644 benches/forge_fuzz_test.rs delete mode 100755 benches/run_benchmarks.sh create mode 160000 test-forge-regex diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index aceb45815477d..7d6b7087b964b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -17,7 +17,7 @@ on: required: false type: string benchmarks: - description: "Comma-separated list of benchmarks to run (forge_test,forge_build_no_cache,forge_build_with_cache)" + description: "Comma-separated list of benchmarks to run (forge_test,forge_build_no_cache,forge_build_with_cache,forge_fuzz_test)" required: false type: string diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 063ddad08ce9f..4facb5cf2575c 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -22,6 +22,11 @@ name = "forge_build_with_cache" path = "forge_build_with_cache.rs" harness = false +[[bench]] +name = "forge_fuzz_test" +path = "forge_fuzz_test.rs" +harness = false + [dependencies] criterion = { version = "0.5", features = ["html_reports"] } foundry-test-utils.workspace = true diff --git a/benches/FUZZ_BENCH.md b/benches/FUZZ_BENCH.md new file mode 100644 index 0000000000000..46a51b95b4178 --- /dev/null +++ b/benches/FUZZ_BENCH.md @@ -0,0 +1,30 @@ +# Foundry Benchmark Results + +**Date**: 2025-07-01 16:41:48 + +## Summary + +Benchmarked 2 Foundry versions across 2 repositories. + +### Repositories Tested + +1. [ithacaxyz/account](https://github.com/ithacaxyz/account) +2. [Vectorized/solady](https://github.com/Vectorized/solady) + +### Foundry Versions + +- stable +- nightly + +## Forge Fuzz Test + +| Repository | stable | nightly | +| ----------------- | ------ | ------- | +| ithacaxyz-account | 4.34 s | 3.69 s | +| solady | 3.68 s | 2.92 s | + +## System Information + +- **OS**: macos +- **CPU**: 8 +- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) diff --git a/benches/README.md b/benches/README.md index dd73ae4211b60..13b572fb08fd5 100644 --- a/benches/README.md +++ b/benches/README.md @@ -59,6 +59,9 @@ cargo run --release --bin foundry-bench -- --repos ithacaxyz/account:main,Vector # Run only specific benchmarks cargo run --release --bin foundry-bench -- --benchmarks forge_build_with_cache,forge_test +# Run only fuzz tests +cargo run --release --bin foundry-bench -- --benchmarks forge_fuzz_test + # Combine options cargo run --release --bin foundry-bench -- \ --versions stable,nightly \ @@ -92,6 +95,7 @@ cargo run --release --bin foundry-bench -- --output-dir ./results --output-file cargo bench --bench forge_test cargo bench --bench forge_build_no_cache cargo bench --bench forge_build_with_cache +cargo bench --bench forge_fuzz_test ``` ## Benchmark Structure @@ -99,6 +103,7 @@ cargo bench --bench forge_build_with_cache - `forge_test` - Benchmarks `forge test` command across repos - `forge_build_no_cache` - Benchmarks `forge build` with clean cache - `forge_build_with_cache` - Benchmarks `forge build` with existing cache +- `forge_fuzz_test` - Benchmarks `forge test` with only fuzz tests (tests with parameters) ## Configuration diff --git a/benches/forge_fuzz_test.rs b/benches/forge_fuzz_test.rs new file mode 100644 index 0000000000000..ce8a5c6fb8ee0 --- /dev/null +++ b/benches/forge_fuzz_test.rs @@ -0,0 +1,37 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::{get_benchmark_versions, setup_benchmark_repos}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::env; + +pub fn forge_fuzz_test_benchmark(c: &mut Criterion) { + // Get the version to test from environment variable + let version = + env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); + + let mut group = c.benchmark_group("forge-fuzz-test"); + group.sample_size(foundry_bench::SAMPLE_SIZE as usize); + + println!("Running forge-fuzz-test for version: {}", version); + + let projects = setup_benchmark_repos(); + + projects.par_iter().for_each(|(_repo_config, project)| { + project.run_forge_build(false).expect("forge build failed"); + }); + + for (repo_config, project) in &projects { + // This creates: forge-fuzz-test/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, &repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let _output = project.run_fuzz_tests().expect("forge fuzz test failed"); + }); + }); + } + + group.finish(); +} + +criterion_group!(benches, forge_fuzz_test_benchmark); +criterion_main!(benches); diff --git a/benches/run_benchmarks.sh b/benches/run_benchmarks.sh deleted file mode 100755 index a4b343a0ac226..0000000000000 --- a/benches/run_benchmarks.sh +++ /dev/null @@ -1,359 +0,0 @@ -#!/bin/bash - -# Foundry Benchmark Runner with Criterion Table Output -# This script runs the criterion-based benchmarks and generates a markdown report - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warn() { - echo -e "${YELLOW}[WARN]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# Script directory -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR" - -# Check if required tools are installed -check_dependencies() { - if ! command -v criterion-table &> /dev/null; then - log_error "criterion-table is not installed. Please install it with:" - echo "cargo install criterion-table" - exit 1 - fi - - if ! cargo criterion --help &> /dev/null; then - log_error "cargo-criterion is not installed. Please install it with:" - echo "cargo install cargo-criterion" - exit 1 - fi -} - -# Install Foundry versions if requested -install_foundry_versions() { - if [[ "$FORCE_INSTALL" != "true" ]]; then - return - fi - - local versions - - # Use custom versions if provided, otherwise read from lib.rs - if [[ -n "$CUSTOM_VERSIONS" ]]; then - versions=$(echo "$CUSTOM_VERSIONS" | tr ',' ' ') - log_info "Installing custom Foundry versions: $versions" - else - # Read the versions from the Rust source - versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"') - log_info "Installing default Foundry versions from lib.rs: $versions" - fi - - # Check if foundryup is available - if ! command -v foundryup &> /dev/null; then - log_error "foundryup not found. Please install Foundry first:" - echo "curl -L https://foundry.paradigm.xyz | bash" - exit 1 - fi - - # Install each version - for version in $versions; do - log_info "Installing Foundry version: $version" - if foundryup --install "$version"; then - log_success "✓ Successfully installed version $version" - else - log_error "Failed to install Foundry version: $version" - exit 1 - fi - done - - log_success "All Foundry versions installed successfully" -} - -# Get system information -get_system_info() { - local os_name=$(uname -s) - local arch=$(uname -m) - local date=$(date) - - echo "- **OS:** $os_name" - echo "- **Architecture:** $arch" - echo "- **Date:** $date" -} - - -# Run benchmarks and generate report -run_benchmarks() { - log_info "Running Foundry benchmarks..." - - # Set environment variable for custom versions if provided - if [[ -n "$CUSTOM_VERSIONS" ]]; then - export FOUNDRY_BENCH_VERSIONS="$CUSTOM_VERSIONS" - log_info "Set FOUNDRY_BENCH_VERSIONS=$CUSTOM_VERSIONS" - fi - - # Create temp files for each benchmark - local temp_dir=$(mktemp -d) - local forge_test_json="$temp_dir/forge_test.json" - local forge_build_no_cache_json="$temp_dir/forge_build_no_cache.json" - local forge_build_with_cache_json="$temp_dir/forge_build_with_cache.json" - - # Set up output redirection based on verbose flag - local output_redirect="" - if [[ "${VERBOSE:-false}" != "true" ]]; then - output_redirect="2>/dev/null" - fi - - # Run benchmarks in specific order (this determines baseline column) - log_info "Running forge_test benchmark..." - if [[ "${VERBOSE:-false}" == "true" ]]; then - cargo criterion --bench forge_test --message-format=json > "$forge_test_json" || { - log_error "forge_test benchmark failed" - exit 1 - } - else - cargo criterion --bench forge_test --message-format=json > "$forge_test_json" 2>/dev/null || { - log_error "forge_test benchmark failed" - exit 1 - } - fi - - log_info "Running forge_build_no_cache benchmark..." - if [[ "${VERBOSE:-false}" == "true" ]]; then - cargo criterion --bench forge_build_no_cache --message-format=json > "$forge_build_no_cache_json" || { - log_error "forge_build_no_cache benchmark failed" - exit 1 - } - else - cargo criterion --bench forge_build_no_cache --message-format=json > "$forge_build_no_cache_json" 2>/dev/null || { - log_error "forge_build_no_cache benchmark failed" - exit 1 - } - fi - - log_info "Running forge_build_with_cache benchmark..." - if [[ "${VERBOSE:-false}" == "true" ]]; then - cargo criterion --bench forge_build_with_cache --message-format=json > "$forge_build_with_cache_json" || { - log_error "forge_build_with_cache benchmark failed" - exit 1 - } - else - cargo criterion --bench forge_build_with_cache --message-format=json > "$forge_build_with_cache_json" 2>/dev/null || { - log_error "forge_build_with_cache benchmark failed" - exit 1 - } - fi - - # Combine all results and generate markdown - log_info "Generating markdown report with criterion-table..." - - if ! cat "$forge_test_json" "$forge_build_no_cache_json" "$forge_build_with_cache_json" | criterion-table > "$temp_dir/tables.md"; then - log_error "criterion-table failed to process benchmark data" - exit 1 - fi - - # Generate the final report - generate_report "$temp_dir/tables.md" - - # Cleanup - rm -rf "$temp_dir" - - log_success "Benchmark report generated in LATEST.md" -} - -# Generate the final markdown report -generate_report() { - local tables_file="$1" - local report_file="LATEST.md" - - log_info "Generating final report..." - - # Get current timestamp - local timestamp=$(date) - - # Get repository information and create numbered list with links - local versions - if [[ -n "$CUSTOM_VERSIONS" ]]; then - versions=$(echo "$CUSTOM_VERSIONS" | tr ',' ' ') - else - versions=$(grep -A 10 'pub static FOUNDRY_VERSIONS' src/lib.rs | grep -o '"[^"]*"' | tr -d '"' | tr '\n' ' ') - fi - - # Extract repository info for numbered list - local repo_list="" - local counter=1 - - # Parse the BENCHMARK_REPOS section - while IFS= read -r line; do - if [[ $line =~ RepoConfig.*name:.*\"([^\"]+)\".*org:.*\"([^\"]+)\".*repo:.*\"([^\"]+)\" ]]; then - local name="${BASH_REMATCH[1]}" - local org="${BASH_REMATCH[2]}" - local repo="${BASH_REMATCH[3]}" - repo_list+="$counter. [$name](https://github.com/$org/$repo)\n" - ((counter++)) - fi - done < <(grep -A 20 'pub static BENCHMARK_REPOS' src/lib.rs | grep 'RepoConfig') - - # Write the report - cat > "$report_file" << EOF -# Foundry Benchmarking Results - -**Generated on:** $timestamp -**Foundry Versions Tested:** $versions - -## Repositories Tested - -$(echo -e "$repo_list") - -## Summary - -This report contains comprehensive benchmarking results comparing different Foundry versions across multiple projects using Criterion.rs for precise performance measurements. - -The following benchmarks were performed: - -1. **forge-test** - Running the test suite (10 samples each) -2. **forge-build-no-cache** - Clean build without cache (10 samples each) -3. **forge-build-with-cache** - Build with warm cache (10 samples each) - ---- - -EOF - - # Append the criterion-table generated tables - cat "$tables_file" >> "$report_file" - - # Add system info - cat >> "$report_file" << EOF - -## System Information - -$(get_system_info) - -## Raw Data - -Detailed benchmark data and HTML reports are available in: -- \`target/criterion/\` - Individual benchmark reports - -EOF - - log_success "Report written to $report_file" -} - -# Main function -main() { - log_info "Starting Foundry benchmark suite..." - - # Check dependencies - check_dependencies - - # Install Foundry versions if --force-install is used - install_foundry_versions - - # Run benchmarks and generate report - run_benchmarks - - log_success "Benchmark suite completed successfully!" - echo "" - echo "View the results:" - echo " - Text report: cat LATEST.md" -} - -# Help function -show_help() { - cat << EOF -Foundry Benchmark Runner - -This script runs Criterion-based benchmarks for Foundry commands and generates -a markdown report using criterion-table. - -USAGE: - $0 [OPTIONS] - -OPTIONS: - -h, --help Show this help message - -v, --version Show version information - --verbose Show benchmark output (by default output is suppressed) - --versions Comma-separated list of Foundry versions to test - (e.g. stable,nightly,v1.2.0) - If not specified, uses versions from src/lib.rs - --force-install Force installation of Foundry versions - By default, assumes versions are already installed - -REQUIREMENTS: - - criterion-table: cargo install criterion-table - - cargo-criterion: cargo install cargo-criterion - - Foundry versions must be installed (or use --force-install) - -EXAMPLES: - $0 # Run with default versions - $0 --verbose # Show full output - $0 --versions stable,nightly # Test specific versions - $0 --versions stable,nightly --force-install # Install and test versions - -The script will: -1. Run forge_test, forge_build_no_cache, and forge_build_with_cache benchmarks -2. Generate comparison tables using criterion-table -3. Include system information and Foundry version details -4. Save the complete report to LATEST.md - -EOF -} - -# Default values -VERBOSE=false -FORCE_INSTALL=false -CUSTOM_VERSIONS="" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case $1 in - -h|--help) - show_help - exit 0 - ;; - -v|--version) - echo "Foundry Benchmark Runner v1.0.0" - exit 0 - ;; - --verbose) - VERBOSE=true - shift - ;; - --force-install) - FORCE_INSTALL=true - shift - ;; - --versions) - if [[ -z "$2" ]] || [[ "$2" == --* ]]; then - log_error "--versions requires a comma-separated list of versions" - exit 1 - fi - CUSTOM_VERSIONS="$2" - shift 2 - ;; - *) - log_error "Unknown option: $1" - echo "Use -h or --help for usage information" - exit 1 - ;; - esac -done - -main \ No newline at end of file diff --git a/benches/src/lib.rs b/benches/src/lib.rs index f0ecffcf3c8d8..7836ccad7bdd9 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -216,6 +216,16 @@ impl BenchmarkProject { pub fn root(&self) -> &Path { &self.root_path } + + /// Run forge test with fuzz tests only (tests with parameters) + pub fn run_fuzz_tests(&self) -> Result { + // Use shell to properly handle the regex pattern + Command::new("sh") + .current_dir(&self.root_path) + .args(["-c", r#"forge test --match-test "test[^(]*\([^)]+\)""#]) + .output() + .wrap_err("Failed to run forge fuzz tests") + } } /// Switch to a specific foundry version diff --git a/benches/src/main.rs b/benches/src/main.rs index 143cfb3e05ab2..e0202b8fa1754 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -41,7 +41,7 @@ struct Cli { output_file: String, /// Run only specific benchmarks (comma-separated: - /// forge_test,forge_build_no_cache,forge_build_with_cache) + /// forge_test,forge_build_no_cache,forge_build_with_cache,forge_fuzz_test) #[clap(long, value_delimiter = ',')] benchmarks: Option>, @@ -133,13 +133,15 @@ fn main() -> Result<()> { } // Determine benchmarks to run - let all_benchmarks = vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache"]; + let all_benchmarks = vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test"]; let benchmarks = if let Some(b) = cli.benchmarks { b.into_iter().filter(|b| all_benchmarks.contains(&b.as_str())).collect() } else { - // For testing, only run forge_build_with_cache - // vec!["forge_build_with_cache".to_string()] - all_benchmarks.into_iter().map(String::from).collect::>() + // Default: run all benchmarks except fuzz tests (which can be slow) + vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache"] + .into_iter() + .map(String::from) + .collect::>() }; println!(" Running benchmarks: {}", benchmarks.join(", ")); diff --git a/benches/src/results.rs b/benches/src/results.rs index d36ff3dd460ae..8943b42a94fca 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -194,6 +194,7 @@ pub fn format_benchmark_name(name: &str) -> String { "forge-test" => "Forge Test", "forge-build-no-cache" => "Forge Build (No Cache)", "forge-build-with-cache" => "Forge Build (With Cache)", + "forge-fuzz-test" => "Forge Fuzz Test", _ => name, } .to_string() diff --git a/test-forge-regex b/test-forge-regex new file mode 160000 index 0000000000000..c3decafda6d31 --- /dev/null +++ b/test-forge-regex @@ -0,0 +1 @@ +Subproject commit c3decafda6d313fa159ce2a50a352467af7fa58b From afcf3ed18724caf8bc83cd11644b16bb5cbcce39 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:52:37 +0530 Subject: [PATCH 31/74] fmt --- benches/src/criterion_types.rs | 2 +- benches/src/main.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benches/src/criterion_types.rs b/benches/src/criterion_types.rs index 3ef888df25f1f..eb3919ed994d5 100644 --- a/benches/src/criterion_types.rs +++ b/benches/src/criterion_types.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; /// Benchmark result from Criterion JSON output -/// +/// /// This is a simplified version containing only the fields we actually use #[derive(Debug, Deserialize, Serialize)] pub struct CriterionResult { diff --git a/benches/src/main.rs b/benches/src/main.rs index e0202b8fa1754..936571f42970b 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -133,7 +133,8 @@ fn main() -> Result<()> { } // Determine benchmarks to run - let all_benchmarks = vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test"]; + let all_benchmarks = + vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test"]; let benchmarks = if let Some(b) = cli.benchmarks { b.into_iter().filter(|b| all_benchmarks.contains(&b.as_str())).collect() } else { From ed82d8a88a8aa9d80511db3ee11dde54346f3503 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:53:28 +0530 Subject: [PATCH 32/74] license --- benches/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 4facb5cf2575c..273990534d1e2 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -2,6 +2,7 @@ name = "foundry-bench" version = "0.1.0" edition = "2021" +license = "Apache-2.0 OR MIT" [[bin]] name = "foundry-bench" From 4dac5c35c4d71fab9ad576d4a610d2360f84ab25 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 17:33:15 +0530 Subject: [PATCH 33/74] coverage bench --- benches/COVERAGE_BENCH.md | 27 +++++++++++++++++++++++++++ benches/Cargo.toml | 5 +++++ benches/README.md | 5 +++++ benches/forge_coverage.rs | 33 +++++++++++++++++++++++++++++++++ benches/src/lib.rs | 9 +++++++++ benches/src/main.rs | 6 +++--- benches/src/results.rs | 1 + 7 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 benches/COVERAGE_BENCH.md create mode 100644 benches/forge_coverage.rs diff --git a/benches/COVERAGE_BENCH.md b/benches/COVERAGE_BENCH.md new file mode 100644 index 0000000000000..3f5c6869a36df --- /dev/null +++ b/benches/COVERAGE_BENCH.md @@ -0,0 +1,27 @@ +# Foundry Benchmark Results + +**Date**: 2025-07-01 17:32:13 + +## Summary + +Benchmarked 1 Foundry versions across 1 repositories. + +### Repositories Tested + +1. [ithacaxyz/account](https://github.com/ithacaxyz/account) + +### Foundry Versions + +- nightly + +## Forge Coverage + +| Repository | nightly | +| ----------------- | ------- | +| ithacaxyz-account | 25.54 s | + +## System Information + +- **OS**: macos +- **CPU**: 8 +- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 273990534d1e2..aa1d5b0845328 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -28,6 +28,11 @@ name = "forge_fuzz_test" path = "forge_fuzz_test.rs" harness = false +[[bench]] +name = "forge_coverage" +path = "forge_coverage.rs" +harness = false + [dependencies] criterion = { version = "0.5", features = ["html_reports"] } foundry-test-utils.workspace = true diff --git a/benches/README.md b/benches/README.md index 13b572fb08fd5..41a9344657554 100644 --- a/benches/README.md +++ b/benches/README.md @@ -62,6 +62,9 @@ cargo run --release --bin foundry-bench -- --benchmarks forge_build_with_cache,f # Run only fuzz tests cargo run --release --bin foundry-bench -- --benchmarks forge_fuzz_test +# Run coverage benchmark +cargo run --release --bin foundry-bench -- --benchmarks forge_coverage + # Combine options cargo run --release --bin foundry-bench -- \ --versions stable,nightly \ @@ -96,6 +99,7 @@ cargo bench --bench forge_test cargo bench --bench forge_build_no_cache cargo bench --bench forge_build_with_cache cargo bench --bench forge_fuzz_test +cargo bench --bench forge_coverage ``` ## Benchmark Structure @@ -104,6 +108,7 @@ cargo bench --bench forge_fuzz_test - `forge_build_no_cache` - Benchmarks `forge build` with clean cache - `forge_build_with_cache` - Benchmarks `forge build` with existing cache - `forge_fuzz_test` - Benchmarks `forge test` with only fuzz tests (tests with parameters) +- `forge_coverage` - Benchmarks `forge coverage --ir-minimum` command across repos ## Configuration diff --git a/benches/forge_coverage.rs b/benches/forge_coverage.rs new file mode 100644 index 0000000000000..b587082377545 --- /dev/null +++ b/benches/forge_coverage.rs @@ -0,0 +1,33 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use foundry_bench::setup_benchmark_repos; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use std::env; + +pub fn forge_coverage_benchmark(c: &mut Criterion) { + // Get the version to test from environment variable + let version = + env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); + + let mut group = c.benchmark_group("forge-coverage"); + group.sample_size(foundry_bench::SAMPLE_SIZE as usize); + + println!("Running forge-coverage for version: {}", version); + + let projects = setup_benchmark_repos(); + + for (repo_config, project) in &projects { + // This creates: forge-coverage/{version}/{repo_name} + let bench_id = BenchmarkId::new(&version, &repo_config.name); + + group.bench_function(bench_id, |b| { + b.iter(|| { + let _output = project.run_forge_coverage().expect("forge coverage failed"); + }); + }); + } + + group.finish(); +} + +criterion_group!(benches, forge_coverage_benchmark); +criterion_main!(benches); \ No newline at end of file diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 7836ccad7bdd9..d60e1eb86c793 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -226,6 +226,15 @@ impl BenchmarkProject { .output() .wrap_err("Failed to run forge fuzz tests") } + + /// Run forge coverage command with --ir-minimum flag + pub fn run_forge_coverage(&self) -> Result { + Command::new("forge") + .current_dir(&self.root_path) + .args(["coverage", "--ir-minimum"]) + .output() + .wrap_err("Failed to run forge coverage") + } } /// Switch to a specific foundry version diff --git a/benches/src/main.rs b/benches/src/main.rs index 936571f42970b..9a96bcd51cfd4 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -41,7 +41,7 @@ struct Cli { output_file: String, /// Run only specific benchmarks (comma-separated: - /// forge_test,forge_build_no_cache,forge_build_with_cache,forge_fuzz_test) + /// forge_test,forge_build_no_cache,forge_build_with_cache,forge_fuzz_test,forge_coverage) #[clap(long, value_delimiter = ',')] benchmarks: Option>, @@ -134,11 +134,11 @@ fn main() -> Result<()> { // Determine benchmarks to run let all_benchmarks = - vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test"]; + vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test", "forge_coverage"]; let benchmarks = if let Some(b) = cli.benchmarks { b.into_iter().filter(|b| all_benchmarks.contains(&b.as_str())).collect() } else { - // Default: run all benchmarks except fuzz tests (which can be slow) + // Default: run all benchmarks except fuzz tests and coverage (which can be slow) vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache"] .into_iter() .map(String::from) diff --git a/benches/src/results.rs b/benches/src/results.rs index 8943b42a94fca..90eb770840374 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -195,6 +195,7 @@ pub fn format_benchmark_name(name: &str) -> String { "forge-build-no-cache" => "Forge Build (No Cache)", "forge-build-with-cache" => "Forge Build (With Cache)", "forge-fuzz-test" => "Forge Fuzz Test", + "forge-coverage" => "Forge Coverage", _ => name, } .to_string() From 38873709cccccc007c9a9819d98aaa42cb3f2997 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 17:50:20 +0530 Subject: [PATCH 34/74] nits --- benches/forge_coverage.rs | 2 +- benches/src/lib.rs | 2 +- benches/src/main.rs | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/benches/forge_coverage.rs b/benches/forge_coverage.rs index b587082377545..06438cef9b4e8 100644 --- a/benches/forge_coverage.rs +++ b/benches/forge_coverage.rs @@ -30,4 +30,4 @@ pub fn forge_coverage_benchmark(c: &mut Criterion) { } criterion_group!(benches, forge_coverage_benchmark); -criterion_main!(benches); \ No newline at end of file +criterion_main!(benches); diff --git a/benches/src/lib.rs b/benches/src/lib.rs index d60e1eb86c793..7b3243dbf3874 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -112,7 +112,7 @@ pub const SAMPLE_SIZE: usize = 10; /// - "nightly" - Latest nightly build /// - "v0.2.0" - Specific version tag /// - "commit-hash" - Specific commit hash -/// - "nightly-" - Nightly build with specific revision +/// - "nightly-rev" - Nightly build with specific revision pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; /// A benchmark project that represents a cloned repository ready for testing diff --git a/benches/src/main.rs b/benches/src/main.rs index 9a96bcd51cfd4..6f77cfe8aae58 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -133,8 +133,13 @@ fn main() -> Result<()> { } // Determine benchmarks to run - let all_benchmarks = - vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test", "forge_coverage"]; + let all_benchmarks = vec![ + "forge_test", + "forge_build_no_cache", + "forge_build_with_cache", + "forge_fuzz_test", + "forge_coverage", + ]; let benchmarks = if let Some(b) = cli.benchmarks { b.into_iter().filter(|b| all_benchmarks.contains(&b.as_str())).collect() } else { From cbd17d86c6a63041109984e57f3a530825964e10 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 18:31:42 +0530 Subject: [PATCH 35/74] clippy --- Cargo.lock | 1 + benches/Cargo.toml | 1 + benches/forge_build_no_cache.rs | 3 +- benches/forge_build_with_cache.rs | 3 +- benches/forge_coverage.rs | 6 +-- benches/forge_fuzz_test.rs | 7 ++-- benches/forge_test.rs | 3 +- benches/src/lib.rs | 21 +++++++---- benches/src/main.rs | 61 +++++++++++++++++-------------- 9 files changed, 62 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 595e9878a2bab..31539da0db89d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4108,6 +4108,7 @@ dependencies = [ "clap", "criterion", "eyre", + "foundry-common", "foundry-compilers", "foundry-config", "foundry-test-utils", diff --git a/benches/Cargo.toml b/benches/Cargo.toml index aa1d5b0845328..74b18419e7211 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -37,6 +37,7 @@ harness = false criterion = { version = "0.5", features = ["html_reports"] } foundry-test-utils.workspace = true foundry-config.workspace = true +foundry-common.workspace = true foundry-compilers = { workspace = true, features = ["project-util"] } eyre.workspace = true serde.workspace = true diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs index 3dd0e536f747f..28771becec489 100644 --- a/benches/forge_build_no_cache.rs +++ b/benches/forge_build_no_cache.rs @@ -1,5 +1,6 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; +use foundry_common::sh_println; use std::env; fn benchmark_forge_build_no_cache(c: &mut Criterion) { @@ -10,7 +11,7 @@ fn benchmark_forge_build_no_cache(c: &mut Criterion) { let version = env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - println!("Running forge-build-no-cache for version: {version}"); + let _ = sh_println!("Running forge-build-no-cache for version: {version}"); let projects = setup_benchmark_repos(); diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs index 5a8691c2a33e0..ac299b3ecd59e 100644 --- a/benches/forge_build_with_cache.rs +++ b/benches/forge_build_with_cache.rs @@ -1,5 +1,6 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; +use foundry_common::sh_println; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::env; @@ -11,7 +12,7 @@ fn benchmark_forge_build_with_cache(c: &mut Criterion) { let version = env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - println!("Running forge-build-with-cache for version: {version}"); + let _ = sh_println!("Running forge-build-with-cache for version: {version}"); let projects = setup_benchmark_repos(); diff --git a/benches/forge_coverage.rs b/benches/forge_coverage.rs index 06438cef9b4e8..d2f589ed8c5c0 100644 --- a/benches/forge_coverage.rs +++ b/benches/forge_coverage.rs @@ -1,6 +1,6 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use foundry_bench::setup_benchmark_repos; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use foundry_common::sh_println; use std::env; pub fn forge_coverage_benchmark(c: &mut Criterion) { @@ -9,9 +9,9 @@ pub fn forge_coverage_benchmark(c: &mut Criterion) { env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); let mut group = c.benchmark_group("forge-coverage"); - group.sample_size(foundry_bench::SAMPLE_SIZE as usize); + group.sample_size(foundry_bench::SAMPLE_SIZE); - println!("Running forge-coverage for version: {}", version); + let _ = sh_println!("Running forge-coverage for version: {}", version); let projects = setup_benchmark_repos(); diff --git a/benches/forge_fuzz_test.rs b/benches/forge_fuzz_test.rs index ce8a5c6fb8ee0..ee3d80b0cdad6 100644 --- a/benches/forge_fuzz_test.rs +++ b/benches/forge_fuzz_test.rs @@ -1,5 +1,6 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{get_benchmark_versions, setup_benchmark_repos}; +use foundry_bench::setup_benchmark_repos; +use foundry_common::sh_println; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::env; @@ -9,9 +10,9 @@ pub fn forge_fuzz_test_benchmark(c: &mut Criterion) { env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); let mut group = c.benchmark_group("forge-fuzz-test"); - group.sample_size(foundry_bench::SAMPLE_SIZE as usize); + group.sample_size(foundry_bench::SAMPLE_SIZE); - println!("Running forge-fuzz-test for version: {}", version); + sh_println!("Running forge-fuzz-test for version: {}", version); let projects = setup_benchmark_repos(); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index e3205d02ac335..75a42b5debd27 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -1,5 +1,6 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; +use foundry_common::sh_println; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::env; @@ -11,7 +12,7 @@ fn benchmark_forge_test(c: &mut Criterion) { let version = env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - println!("Running forge-test for version: {version}"); + sh_println!("Running forge-test for version: {version}"); let projects = setup_benchmark_repos(); diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 7b3243dbf3874..e5d67ef9470c0 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -1,4 +1,5 @@ use eyre::{Result, WrapErr}; +use foundry_common::{sh_eprintln, sh_println}; use foundry_compilers::project_util::TempProject; use foundry_test_utils::util::clone_remote; use once_cell::sync::Lazy; @@ -60,7 +61,7 @@ impl TryFrom<&str> for RepoConfig { } }; - println!("Parsed repo spec '{spec}' -> {config:?}"); + sh_println!("Parsed repo spec '{spec}' -> {config:?}"); Ok(config) } } @@ -124,6 +125,7 @@ pub struct BenchmarkProject { impl BenchmarkProject { /// Set up a benchmark project by cloning the repository + #[allow(unused_must_use)] pub fn setup(config: &RepoConfig) -> Result { let temp_project = TempProject::dapptools().wrap_err("Failed to create temporary project")?; @@ -164,14 +166,15 @@ impl BenchmarkProject { // But npm dependencies still need to be installed Self::install_npm_dependencies(&root_path)?; - println!(" ✅ Project {} setup complete at {}", config.name, root); + sh_println!(" ✅ Project {} setup complete at {}", config.name, root); Ok(BenchmarkProject { name: config.name.to_string(), root_path, temp_project }) } /// Install npm dependencies if package.json exists + #[allow(unused_must_use)] fn install_npm_dependencies(root: &Path) -> Result<()> { if root.join("package.json").exists() { - println!(" 📦 Running npm install..."); + sh_println!(" 📦 Running npm install..."); let status = Command::new("npm") .current_dir(root) .args(["install"]) @@ -181,9 +184,12 @@ impl BenchmarkProject { .wrap_err("Failed to run npm install")?; if !status.success() { - println!(" ⚠️ Warning: npm install failed with exit code: {:?}", status.code()); + sh_println!( + " ⚠️ Warning: npm install failed with exit code: {:?}", + status.code() + ); } else { - println!(" ✅ npm install completed successfully"); + sh_println!(" ✅ npm install completed successfully"); } } Ok(()) @@ -238,6 +244,7 @@ impl BenchmarkProject { } /// Switch to a specific foundry version +#[allow(unused_must_use)] pub fn switch_foundry_version(version: &str) -> Result<()> { let output = Command::new("foundryup") .args(["--use", version]) @@ -253,11 +260,11 @@ pub fn switch_foundry_version(version: &str) -> Result<()> { } if !output.status.success() { - eprintln!("foundryup stderr: {stderr}"); + sh_eprintln!("foundryup stderr: {stderr}"); eyre::bail!("Failed to switch to foundry version: {}", version); } - println!(" Successfully switched to version: {version}"); + sh_println!(" Successfully switched to version: {version}"); Ok(()) } diff --git a/benches/src/main.rs b/benches/src/main.rs index 6f77cfe8aae58..3230923abecb3 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -6,6 +6,7 @@ use foundry_bench::{ results::BenchmarkResults, switch_foundry_version, RepoConfig, BENCHMARK_REPOS, FOUNDRY_VERSIONS, }; +use foundry_common::sh_println; use once_cell::sync::Lazy; use rayon::prelude::*; use std::{ @@ -16,6 +17,14 @@ use std::{ sync::Mutex, }; +const ALL_BENCHMARKS: [&str; 5] = [ + "forge_test", + "forge_build_no_cache", + "forge_build_with_cache", + "forge_fuzz_test", + "forge_coverage", +]; + /// Foundry Benchmark Runner #[derive(Parser, Debug)] #[clap(name = "foundry-bench", about = "Run Foundry benchmarks across multiple versions")] @@ -96,7 +105,7 @@ fn run_benchmark( // Collect benchmark results from criterion output let results = collect_benchmark_results(&group_dir, &dir_name, version, repos)?; - println!("Total results collected: {}", results.len()); + sh_println!("Total results collected: {}", results.len()); Ok(results) } @@ -120,9 +129,9 @@ fn main() -> Result<()> { BENCHMARK_REPOS.clone() }; - println!("🚀 Foundry Benchmark Runner"); - println!("Running with versions: {}", versions.join(", ")); - println!( + sh_println!("🚀 Foundry Benchmark Runner"); + sh_println!("Running with versions: {}", versions.join(", ")); + sh_println!( "Running on repos: {}", repos.iter().map(|r| format!("{}/{}", r.org, r.repo)).collect::>().join(", ") ); @@ -133,15 +142,9 @@ fn main() -> Result<()> { } // Determine benchmarks to run - let all_benchmarks = vec![ - "forge_test", - "forge_build_no_cache", - "forge_build_with_cache", - "forge_fuzz_test", - "forge_coverage", - ]; + let benchmarks = if let Some(b) = cli.benchmarks { - b.into_iter().filter(|b| all_benchmarks.contains(&b.as_str())).collect() + b.into_iter().filter(|b| ALL_BENCHMARKS.contains(&b.as_str())).collect() } else { // Default: run all benchmarks except fuzz tests and coverage (which can be slow) vec!["forge_test", "forge_build_no_cache", "forge_build_with_cache"] @@ -150,7 +153,7 @@ fn main() -> Result<()> { .collect::>() }; - println!(" Running benchmarks: {}", benchmarks.join(", ")); + sh_println!(" Running benchmarks: {}", benchmarks.join(", ")); let mut results = BenchmarkResults::new(); // Set the first version as baseline @@ -160,18 +163,18 @@ fn main() -> Result<()> { // Run benchmarks for each version for version in &versions { - println!("🔧 Switching to Foundry version: {version}"); + sh_println!("🔧 Switching to Foundry version: {version}"); switch_version_safe(version)?; // Verify the switch let current = get_forge_version()?; - println!("Current version: {}", current.trim()); + sh_println!("Current version: {}", current.trim()); // Run each benchmark in parallel let bench_results: Vec<(String, Vec)> = benchmarks .par_iter() .map(|benchmark| -> Result<(String, Vec)> { - println!("Running {benchmark} benchmark..."); + sh_println!("Running {benchmark} benchmark..."); let results = run_benchmark(benchmark, version, &repos, cli.verbose)?; Ok((benchmark.clone(), results)) }) @@ -179,7 +182,7 @@ fn main() -> Result<()> { // Aggregate the results and add them to BenchmarkResults for (benchmark, bench_results) in bench_results { - println!("Processing {} results for {}", bench_results.len(), benchmark); + sh_println!("Processing {} results for {}", bench_results.len(), benchmark); for result in bench_results { // Parse ID format: benchmark-name/version/repo let parts: Vec<&str> = result.id.split('/').collect(); @@ -191,7 +194,7 @@ fn main() -> Result<()> { // Debug: show change info if present if let Some(change) = &result.change { if let Some(mean) = &change.mean { - println!( + sh_println!( "Change from baseline: {:.2}% ({})", mean.estimate, change.change.as_ref().unwrap_or(&"Unknown".to_string()) @@ -206,21 +209,22 @@ fn main() -> Result<()> { } // Generate markdown report - println!("📝 Generating report..."); + sh_println!("📝 Generating report..."); let markdown = results.generate_markdown(&versions, &repos); let output_path = cli.output_dir.join(cli.output_file); let mut file = File::create(&output_path).wrap_err("Failed to create output file")?; file.write_all(markdown.as_bytes()).wrap_err("Failed to write output file")?; - println!("✅ Report written to: {}", output_path.display()); + sh_println!("✅ Report written to: {}", output_path.display()); Ok(()) } +#[allow(unused_must_use)] fn install_foundry_versions(versions: &[String]) -> Result<()> { - println!("Installing Foundry versions..."); + sh_println!("Installing Foundry versions..."); for version in versions { - println!("Installing {version}..."); + sh_println!("Installing {version}..."); let status = Command::new("foundryup") .args(["--install", version]) @@ -232,7 +236,7 @@ fn install_foundry_versions(versions: &[String]) -> Result<()> { } } - println!("✅ All versions installed successfully"); + sh_println!("✅ All versions installed successfully"); Ok(()) } @@ -243,6 +247,7 @@ fn install_foundry_versions(versions: &[String]) -> Result<()> { /// - benchmark.json for basic benchmark info /// - estimates.json for mean performance values /// - change/estimates.json for performance change data (if available) +#[allow(unused_must_use)] fn collect_benchmark_results( group_dir: &PathBuf, benchmark_name: &str, @@ -251,7 +256,7 @@ fn collect_benchmark_results( ) -> Result> { let mut results = Vec::new(); - println!("Looking for results in: {}", group_dir.display()); + sh_println!("Looking for results in: {}", group_dir.display()); if !group_dir.exists() { eyre::bail!("Benchmark directory does not exist: {}", group_dir.display()); } @@ -262,7 +267,7 @@ fn collect_benchmark_results( let path = entry.path(); if !path.is_dir() { - println!("Skipping non-directory entry: {}", path.display()); + sh_println!("Skipping non-directory entry: {}", path.display()); continue; } @@ -275,11 +280,11 @@ fn collect_benchmark_results( // Only process repos that are in the specified repos list let is_valid_repo = repos.iter().any(|r| r.name == repo_name); if !is_valid_repo { - println!("Skipping unknown repo: {repo_name}"); + sh_println!("Skipping unknown repo: {repo_name}"); continue; } - println!("Processing repo: {repo_name}"); + sh_println!("Processing repo: {repo_name}"); // Process the benchmark results for this repository if let Some(result) = process_repo_benchmark(&path, benchmark_name, version, &repo_name)? { @@ -294,7 +299,7 @@ fn collect_benchmark_results( /// /// Returns Some(CriterionResult) if valid results are found, None otherwise fn process_repo_benchmark( - repo_path: &PathBuf, + repo_path: &Path, benchmark_name: &str, version: &str, repo_name: &str, From 4fa231563ff0320cbb52d7b6d202b58948bdd4dc Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 1 Jul 2025 18:47:29 +0530 Subject: [PATCH 36/74] clippy --- benches/forge_fuzz_test.rs | 2 +- benches/forge_test.rs | 2 +- benches/src/lib.rs | 2 +- benches/src/main.rs | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/benches/forge_fuzz_test.rs b/benches/forge_fuzz_test.rs index ee3d80b0cdad6..c49fa284ebb21 100644 --- a/benches/forge_fuzz_test.rs +++ b/benches/forge_fuzz_test.rs @@ -12,7 +12,7 @@ pub fn forge_fuzz_test_benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("forge-fuzz-test"); group.sample_size(foundry_bench::SAMPLE_SIZE); - sh_println!("Running forge-fuzz-test for version: {}", version); + let _ = sh_println!("Running forge-fuzz-test for version: {}", version); let projects = setup_benchmark_repos(); diff --git a/benches/forge_test.rs b/benches/forge_test.rs index 75a42b5debd27..2a4f5911c38b8 100644 --- a/benches/forge_test.rs +++ b/benches/forge_test.rs @@ -12,7 +12,7 @@ fn benchmark_forge_test(c: &mut Criterion) { let version = env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - sh_println!("Running forge-test for version: {version}"); + let _ = sh_println!("Running forge-test for version: {version}"); let projects = setup_benchmark_repos(); diff --git a/benches/src/lib.rs b/benches/src/lib.rs index e5d67ef9470c0..653d45ee451c8 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -61,7 +61,7 @@ impl TryFrom<&str> for RepoConfig { } }; - sh_println!("Parsed repo spec '{spec}' -> {config:?}"); + let _ = sh_println!("Parsed repo spec '{spec}' -> {config:?}"); Ok(config) } } diff --git a/benches/src/main.rs b/benches/src/main.rs index 3230923abecb3..5bed6589b526f 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -105,10 +105,11 @@ fn run_benchmark( // Collect benchmark results from criterion output let results = collect_benchmark_results(&group_dir, &dir_name, version, repos)?; - sh_println!("Total results collected: {}", results.len()); + let _ = sh_println!("Total results collected: {}", results.len()); Ok(results) } +#[allow(unused_must_use)] fn main() -> Result<()> { let cli = Cli::parse(); From afc236b0f0c9498a3687aa98df9dd38cb60021d4 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 2 Jul 2025 12:20:55 +0530 Subject: [PATCH 37/74] separate benches into different jobs in CI --- .github/workflows/benchmarks.yml | 180 +++++++++++++++++++++++++++---- 1 file changed, 158 insertions(+), 22 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7d6b7087b964b..37ba3656df536 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -16,20 +16,16 @@ on: description: "Comma-separated list of repos to benchmark (e.g., ithacaxyz/account:main,Vectorized/solady)" required: false type: string - benchmarks: - description: "Comma-separated list of benchmarks to run (forge_test,forge_build_no_cache,forge_build_with_cache,forge_fuzz_test)" - required: false - type: string + default: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22" permissions: contents: write pull-requests: write jobs: - benchmark: - name: Run Foundry Benchmarks + forge-test: + name: Run forge_test and forge_fuzz_test benchmarks runs-on: ubuntu-latest - timeout-minutes: 60 steps: - name: Checkout repository @@ -52,40 +48,180 @@ jobs: - name: Build benchmark binary run: cargo build --release --bin foundry-bench - - name: Run benchmarks + - name: Run forge_test and forge_fuzz_test benchmarks run: | - VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" + VERSIONS="${{ github.event.inputs.versions }}" REPOS="${{ github.event.inputs.repos }}" - BENCHMARKS="${{ github.event.inputs.benchmarks }}" # Build the command with force-install for CI CMD="./target/release/foundry-bench --output-dir ./benches --force-install" + CMD="$CMD --versions $VERSIONS" + CMD="$CMD --repos $REPOS" + CMD="$CMD --benchmarks forge_test,forge_fuzz_test" + CMD="$CMD --output-file forge_test_results.md" + + echo "Running: $CMD" + $CMD + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: forge-test-results + path: benches/forge_test_results.md + + forge-build: + name: Run forge_build benchmarks (with and without cache) + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + ./ - # Add versions + - name: Install foundryup + run: | + curl -L https://foundry.paradigm.xyz | bash + echo "$HOME/.foundry/bin" >> $GITHUB_PATH + + - name: Build benchmark binary + run: cargo build --release --bin foundry-bench + + - name: Run forge_build benchmarks (with and without cache) + run: | + VERSIONS="${{ github.event.inputs.versions }}" + REPOS="${{ github.event.inputs.repos }}" + + # Build the command with force-install for CI + CMD="./target/release/foundry-bench --output-dir ./benches --force-install" CMD="$CMD --versions $VERSIONS" + CMD="$CMD --repos $REPOS" + CMD="$CMD --benchmarks forge_build_with_cache,forge_build_no_cache" + CMD="$CMD --output-file forge_build_results.md" - # Add repos if specified - if [ -n "$REPOS" ]; then - CMD="$CMD --repos $REPOS" - fi + echo "Running: $CMD" + $CMD - # Add benchmarks if specified - if [ -n "$BENCHMARKS" ]; then - CMD="$CMD --benchmarks $BENCHMARKS" - fi + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: forge-build-results + path: benches/forge_build_results.md + + forge-coverage: + name: Run forge_coverage benchmark + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + ./ + + - name: Install foundryup + run: | + curl -L https://foundry.paradigm.xyz | bash + echo "$HOME/.foundry/bin" >> $GITHUB_PATH + + - name: Build benchmark binary + run: cargo build --release --bin foundry-bench + + - name: Run forge_coverage benchmark + run: | + VERSIONS="${{ github.event.inputs.versions }}" + REPOS="${{ github.event.inputs.repos }}" + + # Build the command with force-install for CI + CMD="./target/release/foundry-bench --output-dir ./benches --force-install" + CMD="$CMD --versions $VERSIONS" + CMD="$CMD --repos $REPOS" + CMD="$CMD --benchmarks forge_coverage" + CMD="$CMD --output-file forge_coverage_results.md" echo "Running: $CMD" $CMD + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: forge-coverage-results + path: benches/forge_coverage_results.md + + collect-results: + name: Collect and commit results + runs-on: ubuntu-latest + needs: [forge-test, forge-build, forge-coverage] + if: always() + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: benches/ + + - name: Combine benchmark results + run: | + # Create combined results file + echo "# Foundry Benchmark Results" > benches/LATEST.md + echo "" >> benches/LATEST.md + echo "Generated on: $(date -u)" >> benches/LATEST.md + echo "" >> benches/LATEST.md + + # Add forge_test results if available + if [ -f "benches/forge-test-results/forge_test_results.md" ]; then + echo "## Forge Test and Fuzz Test Results" >> benches/LATEST.md + echo "" >> benches/LATEST.md + tail -n +2 benches/forge-test-results/forge_test_results.md >> benches/LATEST.md + echo "" >> benches/LATEST.md + fi + + # Add forge_build results if available + if [ -f "benches/forge-build-results/forge_build_results.md" ]; then + echo "## Forge Build Results" >> benches/LATEST.md + echo "" >> benches/LATEST.md + tail -n +2 benches/forge-build-results/forge_build_results.md >> benches/LATEST.md + echo "" >> benches/LATEST.md + fi + + # Add forge_coverage results if available + if [ -f "benches/forge-coverage-results/forge_coverage_results.md" ]; then + echo "## Forge Coverage Results" >> benches/LATEST.md + echo "" >> benches/LATEST.md + tail -n +2 benches/forge-coverage-results/forge_coverage_results.md >> benches/LATEST.md + echo "" >> benches/LATEST.md + fi + + # Clean up artifact directories + rm -rf benches/forge-test-results + rm -rf benches/forge-build-results + rm -rf benches/forge-coverage-results + - name: Commit benchmark results run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git add benches/LATEST.md + git add benches/LATEST.md benches/forge_test_results.md benches/forge_build_results.md benches/forge_coverage_results.md 2>/dev/null || true if git diff --staged --quiet; then echo "No changes to commit" else - git commit -m "chore(`benches`): update benchmark results + git commit -m "chore(\`benches\`): update benchmark results 🤖 Generated with [Foundry Benchmarks](https://github.com/${{ github.repository }}/actions) @@ -127,7 +263,7 @@ jobs: 🤖 This comment was automatically generated by the [Foundry Benchmarks workflow](https://github.com/${{ github.repository }}/actions). - To run benchmarks manually: Go to [Actions](https://github.com/${{ github.repository }}/actions/workflows/foundry-benchmarks.yml) → "Run workflow"`; + To run benchmarks manually: Go to [Actions](https://github.com/${{ github.repository }}/actions/workflows/benchmarks.yml) → "Run workflow"`; github.rest.issues.createComment({ issue_number: prNumber, From 3c1c8cbbfcc4fc933636c57924d57a02c3fa5298 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 3 Jul 2025 13:03:34 +0530 Subject: [PATCH 38/74] remove criterion --- benches/forge_build_no_cache.rs | 33 ------------------------ benches/forge_build_with_cache.rs | 38 --------------------------- benches/forge_coverage.rs | 33 ------------------------ benches/forge_fuzz_test.rs | 38 --------------------------- benches/forge_test.rs | 38 --------------------------- benches/src/criterion_types.rs | 43 ------------------------------- 6 files changed, 223 deletions(-) delete mode 100644 benches/forge_build_no_cache.rs delete mode 100644 benches/forge_build_with_cache.rs delete mode 100644 benches/forge_coverage.rs delete mode 100644 benches/forge_fuzz_test.rs delete mode 100644 benches/forge_test.rs delete mode 100644 benches/src/criterion_types.rs diff --git a/benches/forge_build_no_cache.rs b/benches/forge_build_no_cache.rs deleted file mode 100644 index 28771becec489..0000000000000 --- a/benches/forge_build_no_cache.rs +++ /dev/null @@ -1,33 +0,0 @@ -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; -use foundry_common::sh_println; -use std::env; - -fn benchmark_forge_build_no_cache(c: &mut Criterion) { - let mut group = c.benchmark_group("forge-build-no-cache"); - group.sample_size(SAMPLE_SIZE); - - // Get the current version being tested - let version = - env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - - let _ = sh_println!("Running forge-build-no-cache for version: {version}"); - - let projects = setup_benchmark_repos(); - - for (repo_config, project) in &projects { - // This creates: forge-build-no-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, &repo_config.name); - - group.bench_function(bench_id, |b| { - b.iter(|| { - let _output = project.run_forge_build(true).expect("forge build failed"); - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, benchmark_forge_build_no_cache); -criterion_main!(benches); diff --git a/benches/forge_build_with_cache.rs b/benches/forge_build_with_cache.rs deleted file mode 100644 index ac299b3ecd59e..0000000000000 --- a/benches/forge_build_with_cache.rs +++ /dev/null @@ -1,38 +0,0 @@ -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; -use foundry_common::sh_println; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use std::env; - -fn benchmark_forge_build_with_cache(c: &mut Criterion) { - let mut group = c.benchmark_group("forge-build-with-cache"); - group.sample_size(SAMPLE_SIZE); - - // Get the current version being tested - let version = - env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - - let _ = sh_println!("Running forge-build-with-cache for version: {version}"); - - let projects = setup_benchmark_repos(); - - // Prime the cache by building once - projects.par_iter().for_each(|(_repo_config, project)| { - let _ = project.run_forge_build(false); - }); - - for (repo_config, project) in &projects { - // This creates: forge-build-with-cache/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, &repo_config.name); - group.bench_function(bench_id, |b| { - b.iter(|| { - let _output = project.run_forge_build(false).expect("forge build failed"); - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, benchmark_forge_build_with_cache); -criterion_main!(benches); diff --git a/benches/forge_coverage.rs b/benches/forge_coverage.rs deleted file mode 100644 index d2f589ed8c5c0..0000000000000 --- a/benches/forge_coverage.rs +++ /dev/null @@ -1,33 +0,0 @@ -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::setup_benchmark_repos; -use foundry_common::sh_println; -use std::env; - -pub fn forge_coverage_benchmark(c: &mut Criterion) { - // Get the version to test from environment variable - let version = - env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - - let mut group = c.benchmark_group("forge-coverage"); - group.sample_size(foundry_bench::SAMPLE_SIZE); - - let _ = sh_println!("Running forge-coverage for version: {}", version); - - let projects = setup_benchmark_repos(); - - for (repo_config, project) in &projects { - // This creates: forge-coverage/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, &repo_config.name); - - group.bench_function(bench_id, |b| { - b.iter(|| { - let _output = project.run_forge_coverage().expect("forge coverage failed"); - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, forge_coverage_benchmark); -criterion_main!(benches); diff --git a/benches/forge_fuzz_test.rs b/benches/forge_fuzz_test.rs deleted file mode 100644 index c49fa284ebb21..0000000000000 --- a/benches/forge_fuzz_test.rs +++ /dev/null @@ -1,38 +0,0 @@ -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::setup_benchmark_repos; -use foundry_common::sh_println; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use std::env; - -pub fn forge_fuzz_test_benchmark(c: &mut Criterion) { - // Get the version to test from environment variable - let version = - env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - - let mut group = c.benchmark_group("forge-fuzz-test"); - group.sample_size(foundry_bench::SAMPLE_SIZE); - - let _ = sh_println!("Running forge-fuzz-test for version: {}", version); - - let projects = setup_benchmark_repos(); - - projects.par_iter().for_each(|(_repo_config, project)| { - project.run_forge_build(false).expect("forge build failed"); - }); - - for (repo_config, project) in &projects { - // This creates: forge-fuzz-test/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, &repo_config.name); - - group.bench_function(bench_id, |b| { - b.iter(|| { - let _output = project.run_fuzz_tests().expect("forge fuzz test failed"); - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, forge_fuzz_test_benchmark); -criterion_main!(benches); diff --git a/benches/forge_test.rs b/benches/forge_test.rs deleted file mode 100644 index 2a4f5911c38b8..0000000000000 --- a/benches/forge_test.rs +++ /dev/null @@ -1,38 +0,0 @@ -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use foundry_bench::{setup_benchmark_repos, SAMPLE_SIZE}; -use foundry_common::sh_println; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use std::env; - -fn benchmark_forge_test(c: &mut Criterion) { - let mut group = c.benchmark_group("forge-test"); - group.sample_size(SAMPLE_SIZE); - - // Get the current version being tested - let version = - env::var("FOUNDRY_BENCH_CURRENT_VERSION").unwrap_or_else(|_| "unknown".to_string()); - - let _ = sh_println!("Running forge-test for version: {version}"); - - let projects = setup_benchmark_repos(); - - projects.par_iter().for_each(|(_repo_config, project)| { - project.run_forge_build(false).expect("forge build failed"); - }); - - for (repo_config, project) in &projects { - // This creates: forge-test/{version}/{repo_name} - let bench_id = BenchmarkId::new(&version, &repo_config.name); - - group.bench_function(bench_id, |b| { - b.iter(|| { - let _output = project.run_forge_test().expect("forge test failed"); - }); - }); - } - - group.finish(); -} - -criterion_group!(benches, benchmark_forge_test); -criterion_main!(benches); diff --git a/benches/src/criterion_types.rs b/benches/src/criterion_types.rs deleted file mode 100644 index eb3919ed994d5..0000000000000 --- a/benches/src/criterion_types.rs +++ /dev/null @@ -1,43 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Benchmark result from Criterion JSON output -/// -/// This is a simplified version containing only the fields we actually use -#[derive(Debug, Deserialize, Serialize)] -pub struct CriterionResult { - /// Unique identifier for the benchmark result (format: benchmark-name/version/repo) - pub id: String, - /// Mean performance estimate - pub mean: Estimate, - /// Unit of measurement (always "ns" for nanoseconds in our case) - pub unit: String, - /// Performance change data compared to baseline (if available) - pub change: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct Estimate { - pub confidence_interval: ConfidenceInterval, - pub point_estimate: f64, - pub standard_error: f64, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct ConfidenceInterval { - pub confidence_level: f64, - pub lower_bound: f64, - pub upper_bound: f64, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct Change { - pub mean: Option, - pub median: Option, - pub change: Option, // "NoChange", "Improved", or "Regressed" -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct ChangeEstimate { - pub estimate: f64, - pub unit: String, -} From 51d10b556cb8880feb67bf15285ebaa9d063ac6a Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 3 Jul 2025 19:17:40 +0530 Subject: [PATCH 39/74] feat: hyperfine setup in foundry-bench --- Cargo.lock | 94 +------------- benches/Cargo.toml | 27 +--- benches/src/lib.rs | 176 +++++++++++++++++++------- benches/src/main.rs | 281 ++++++++++------------------------------- benches/src/results.rs | 76 +++++++---- 5 files changed, 245 insertions(+), 409 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31539da0db89d..0dc424be9efdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -947,12 +947,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "annotate-snippets" version = "0.11.5" @@ -2368,12 +2362,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cast" version = "1.2.3" @@ -2982,42 +2970,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast 0.3.0", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast 0.3.0", - "itertools 0.10.5", -] - [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -4106,7 +4058,7 @@ version = "0.1.0" dependencies = [ "chrono", "clap", - "criterion", + "color-eyre", "eyre", "foundry-common", "foundry-compilers", @@ -6562,12 +6514,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "op-alloy-consensus" version = "0.17.2" @@ -7030,34 +6976,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "portable-atomic" version = "1.11.1" @@ -9435,16 +9353,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.9.0" diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 74b18419e7211..e9dfce24b6880 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -8,38 +8,13 @@ license = "Apache-2.0 OR MIT" name = "foundry-bench" path = "src/main.rs" -[[bench]] -name = "forge_test" -path = "forge_test.rs" -harness = false - -[[bench]] -name = "forge_build_no_cache" -path = "forge_build_no_cache.rs" -harness = false - -[[bench]] -name = "forge_build_with_cache" -path = "forge_build_with_cache.rs" -harness = false - -[[bench]] -name = "forge_fuzz_test" -path = "forge_fuzz_test.rs" -harness = false - -[[bench]] -name = "forge_coverage" -path = "forge_coverage.rs" -harness = false - [dependencies] -criterion = { version = "0.5", features = ["html_reports"] } foundry-test-utils.workspace = true foundry-config.workspace = true foundry-common.workspace = true foundry-compilers = { workspace = true, features = ["project-util"] } eyre.workspace = true +color-eyre.workspace = true serde.workspace = true serde_json.workspace = true tempfile.workspace = true diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 653d45ee451c8..d2e790a3d6f1e 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -1,3 +1,4 @@ +use crate::results::{HyperfineOutput, HyperfineResult}; use eyre::{Result, WrapErr}; use foundry_common::{sh_eprintln, sh_println}; use foundry_compilers::project_util::TempProject; @@ -7,12 +8,14 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ env, path::{Path, PathBuf}, - process::{Command, Output}, + process::Command, }; -pub mod criterion_types; pub mod results; +/// Default number of runs for benchmarks +pub const RUNS: u32 = 5; + /// Configuration for repositories to benchmark #[derive(Debug, Clone)] pub struct RepoConfig { @@ -81,23 +84,12 @@ pub fn default_benchmark_repos() -> Vec { repo: "solady".to_string(), rev: "main".to_string(), }, - // RepoConfig { name: "v4-core".to_string(), org: "Uniswap".to_string(), repo: - // "v4-core".to_string(), rev: "main".to_string() }, RepoConfig { name: - // "morpho-blue".to_string(), org: "morpho-org".to_string(), repo: - // "morpho-blue".to_string(), rev: "main".to_string() }, RepoConfig { name: - // "spark-psm".to_string(), org: "marsfoundation".to_string(), repo: - // "spark-psm".to_string(), rev: "master".to_string() }, ] } // Keep a lazy static for compatibility pub static BENCHMARK_REPOS: Lazy> = Lazy::new(default_benchmark_repos); -/// Sample size for benchmark measurements -/// -/// This controls how many times each benchmark is run for statistical analysis. -/// Higher values provide more accurate results but take longer to complete. -pub const SAMPLE_SIZE: usize = 10; /// Foundry versions to benchmark /// @@ -195,27 +187,124 @@ impl BenchmarkProject { Ok(()) } - /// Run forge test command and return the output - pub fn run_forge_test(&self) -> Result { - Command::new("forge") + /// Run a command with hyperfine and return the results + /// + /// # Arguments + /// * `benchmark_name` - Name of the benchmark for organizing output + /// * `version` - Foundry version being benchmarked + /// * `command` - The command to benchmark + /// * `runs` - Number of runs to perform + /// * `setup` - Optional setup command to run before the benchmark series (e.g., "forge build") + /// * `conclude` - Optional conclude command to run after each timing run (e.g., cleanup) + /// * `verbose` - Whether to show command output + /// + /// # Hyperfine flags used: + /// * `--runs` - Number of timing runs + /// * `--setup` - Execute before the benchmark series (not before each run) + /// * `--conclude` - Execute after each timing run + /// * `--export-json` - Export results to JSON for parsing + /// * `--shell=bash` - Use bash for shell command execution + /// * `--show-output` - Show command output (when verbose) + fn hyperfine( + &self, + benchmark_name: &str, + version: &str, + command: &str, + runs: u32, + setup: Option<&str>, + conclude: Option<&str>, + verbose: bool, + ) -> Result { + // Create structured temp directory for JSON output + // Format: ////.json + let temp_dir = std::env::temp_dir(); + let json_dir = temp_dir + .join("foundry-bench") + .join(benchmark_name) + .join(version) + .join(&self.name); + std::fs::create_dir_all(&json_dir)?; + + let json_path = json_dir.join(format!("{}.json", benchmark_name)); + + // Build hyperfine command + let mut hyperfine_cmd = Command::new("hyperfine"); + hyperfine_cmd .current_dir(&self.root_path) - .args(["test"]) - .output() - .wrap_err("Failed to run forge test") - } + .arg("--runs") + .arg(runs.to_string()) + .arg("--export-json") + .arg(&json_path) + .arg("--shell=bash"); + + // Add optional setup command + if let Some(setup_cmd) = setup { + hyperfine_cmd.arg("--setup").arg(setup_cmd); + } - /// Run forge build command and return the output - pub fn run_forge_build(&self, clean_cache: bool) -> Result { - if clean_cache { - // Clean first - let _ = Command::new("forge").current_dir(&self.root_path).args(["clean"]).output(); + // Add optional conclude command + if let Some(conclude_cmd) = conclude { + hyperfine_cmd.arg("--conclude").arg(conclude_cmd); } - Command::new("forge") - .current_dir(&self.root_path) - .args(["build"]) - .output() - .wrap_err("Failed to run forge build") + if verbose { + hyperfine_cmd.arg("--show-output"); + hyperfine_cmd.stderr(std::process::Stdio::inherit()); + hyperfine_cmd.stdout(std::process::Stdio::inherit()); + } + + // Add the benchmark command last + hyperfine_cmd.arg(command); + + let status = hyperfine_cmd.status().wrap_err("Failed to run hyperfine")?; + if !status.success() { + eyre::bail!("Hyperfine failed for command: {}", command); + } + + // Read and parse the JSON output + let json_content = std::fs::read_to_string(json_path)?; + let output: HyperfineOutput = serde_json::from_str(&json_content)?; + + // Extract the first result (we only run one command at a time) + output.results.into_iter().next().ok_or_else(|| eyre::eyre!("No results from hyperfine")) + } + + /// Benchmark forge test + pub fn bench_forge_test(&self, version: &str, runs: u32, verbose: bool) -> Result { + // Build before running tests + self.hyperfine("forge_test", version, "forge test", runs, Some("forge build"), None, verbose) + } + + /// Benchmark forge build with cache + pub fn bench_forge_build_with_cache(&self, version: &str, runs: u32, verbose: bool) -> Result { + // No setup needed, uses existing cache + self.hyperfine("forge_build_with_cache", version, "forge build", runs, None, None, verbose) + } + + /// Benchmark forge build without cache + pub fn bench_forge_build_no_cache(&self, version: &str, runs: u32, verbose: bool) -> Result { + // Clean before the benchmark series + self.hyperfine("forge_build_no_cache", version, "forge build", runs, Some("forge clean"), None, verbose) + } + + /// Benchmark forge fuzz tests + pub fn bench_forge_fuzz_test(&self, version: &str, runs: u32, verbose: bool) -> Result { + // Build before running fuzz tests + self.hyperfine( + "forge_fuzz_test", + version, + r#"forge test --match-test "test[^(]*\([^)]+\)""#, + runs, + Some("forge build"), + None, + verbose, + ) + } + + /// Benchmark forge coverage + pub fn bench_forge_coverage(&self, version: &str, runs: u32, verbose: bool) -> Result { + // No setup needed, forge coverage builds internally + self.hyperfine("forge_coverage", version, "forge coverage", runs, None, None, verbose) } /// Get the root path of the project @@ -223,23 +312,16 @@ impl BenchmarkProject { &self.root_path } - /// Run forge test with fuzz tests only (tests with parameters) - pub fn run_fuzz_tests(&self) -> Result { - // Use shell to properly handle the regex pattern - Command::new("sh") - .current_dir(&self.root_path) - .args(["-c", r#"forge test --match-test "test[^(]*\([^)]+\)""#]) - .output() - .wrap_err("Failed to run forge fuzz tests") - } - - /// Run forge coverage command with --ir-minimum flag - pub fn run_forge_coverage(&self) -> Result { - Command::new("forge") - .current_dir(&self.root_path) - .args(["coverage", "--ir-minimum"]) - .output() - .wrap_err("Failed to run forge coverage") + /// Run a specific benchmark by name + pub fn run(&self, benchmark: &str, version: &str, runs: u32, verbose: bool) -> Result { + match benchmark { + "forge_test" => self.bench_forge_test(version, runs, verbose), + "forge_build_no_cache" => self.bench_forge_build_no_cache(version, runs, verbose), + "forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose), + "forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose), + "forge_coverage" => self.bench_forge_coverage(version, runs, verbose), + _ => eyre::bail!("Unknown benchmark: {}", benchmark), + } } } diff --git a/benches/src/main.rs b/benches/src/main.rs index 5bed6589b526f..13bf7fca2e99d 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,21 +1,14 @@ use clap::Parser; -use eyre::{OptionExt, Result, WrapErr}; +use eyre::{Result, WrapErr}; use foundry_bench::{ - criterion_types::{Change, ChangeEstimate, CriterionResult, Estimate}, get_forge_version, - results::BenchmarkResults, - switch_foundry_version, RepoConfig, BENCHMARK_REPOS, FOUNDRY_VERSIONS, + results::{BenchmarkResults, HyperfineResult}, + switch_foundry_version, RepoConfig, BENCHMARK_REPOS, FOUNDRY_VERSIONS, RUNS, }; use foundry_common::sh_println; use once_cell::sync::Lazy; use rayon::prelude::*; -use std::{ - fs::File, - io::Write, - path::{Path, PathBuf}, - process::{Command, Stdio}, - sync::Mutex, -}; +use std::{fs::File, io::Write, path::PathBuf, process::Command, sync::Mutex}; const ALL_BENCHMARKS: [&str; 5] = [ "forge_test", @@ -67,52 +60,17 @@ fn switch_version_safe(version: &str) -> Result<()> { switch_foundry_version(version) } -fn run_benchmark( - name: &str, - version: &str, - repos: &[RepoConfig], - verbose: bool, -) -> Result> { - // Setup paths - let criterion_dir = PathBuf::from("target/criterion"); - let dir_name = name.replace('_', "-"); - let group_dir = criterion_dir.join(&dir_name).join(version); - - // Set environment variable for the current version - std::env::set_var("FOUNDRY_BENCH_CURRENT_VERSION", version); - - // Set environment variable for the repos to benchmark - // Use org/repo format for proper parsing in benchmarks - let repo_specs: Vec = repos.iter().map(|r| format!("{}/{}", r.org, r.repo)).collect(); - std::env::set_var("FOUNDRY_BENCH_REPOS", repo_specs.join(",")); - - // Run the benchmark - let mut cmd = Command::new("cargo"); - cmd.args(["bench", "--bench", name]); - - if verbose { - cmd.stderr(Stdio::inherit()); - cmd.stdout(Stdio::inherit()); - } else { - cmd.stderr(Stdio::null()); - cmd.stdout(Stdio::null()); - } - - let status = cmd.status().wrap_err("Failed to run benchmark")?; - if !status.success() { - eyre::bail!("Benchmark {} failed", name); - } - - // Collect benchmark results from criterion output - let results = collect_benchmark_results(&group_dir, &dir_name, version, repos)?; - let _ = sh_println!("Total results collected: {}", results.len()); - Ok(results) -} - #[allow(unused_must_use)] fn main() -> Result<()> { + color_eyre::install()?; let cli = Cli::parse(); + // Check if hyperfine is installed + let hyperfine_check = Command::new("hyperfine").arg("--version").output(); + if hyperfine_check.is_err() || !hyperfine_check.unwrap().status.success() { + eyre::bail!("hyperfine is not installed. Please install it first: https://github.com/sharkdp/hyperfine"); + } + // Determine versions to test let versions = if let Some(v) = cli.versions { v @@ -120,8 +78,9 @@ fn main() -> Result<()> { FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect() }; - // Determine repos to test - let repos = if let Some(repo_specs) = cli.repos { + + // Get repo configurations + let repos = if let Some(repo_specs) = cli.repos.clone() { repo_specs .iter() .map(|spec| RepoConfig::try_from(spec.as_str())) @@ -143,7 +102,6 @@ fn main() -> Result<()> { } // Determine benchmarks to run - let benchmarks = if let Some(b) = cli.benchmarks { b.into_iter().filter(|b| ALL_BENCHMARKS.contains(&b.as_str())).collect() } else { @@ -154,7 +112,7 @@ fn main() -> Result<()> { .collect::>() }; - sh_println!(" Running benchmarks: {}", benchmarks.join(", ")); + sh_println!("Running benchmarks: {}", benchmarks.join(", ")); let mut results = BenchmarkResults::new(); // Set the first version as baseline @@ -171,41 +129,59 @@ fn main() -> Result<()> { let current = get_forge_version()?; sh_println!("Current version: {}", current.trim()); - // Run each benchmark in parallel - let bench_results: Vec<(String, Vec)> = benchmarks - .par_iter() - .map(|benchmark| -> Result<(String, Vec)> { - sh_println!("Running {benchmark} benchmark..."); - let results = run_benchmark(benchmark, version, &repos, cli.verbose)?; - Ok((benchmark.clone(), results)) + // Create a list of all benchmark tasks + let benchmark_tasks: Vec<_> = repos + .iter() + .flat_map(|repo| { + benchmarks.iter().map(move |benchmark| (repo.clone(), benchmark.clone())) }) - .collect::>()?; - - // Aggregate the results and add them to BenchmarkResults - for (benchmark, bench_results) in bench_results { - sh_println!("Processing {} results for {}", bench_results.len(), benchmark); - for result in bench_results { - // Parse ID format: benchmark-name/version/repo - let parts: Vec<&str> = result.id.split('/').collect(); - if parts.len() >= 3 { - let bench_type = parts[0].to_string(); - // Skip parts[1] which is the version (already known) - let repo = parts[2].to_string(); - - // Debug: show change info if present - if let Some(change) = &result.change { - if let Some(mean) = &change.mean { - sh_println!( - "Change from baseline: {:.2}% ({})", - mean.estimate, - change.change.as_ref().unwrap_or(&"Unknown".to_string()) - ); - } - } + .collect(); - results.add_result(&bench_type, version, &repo, result); + sh_println!("Running {} benchmark tasks in parallel...", benchmark_tasks.len()); + + // Run all benchmarks in parallel + let version_results = benchmark_tasks + .par_iter() + .map(|(repo_config, benchmark)| -> Result<(String, String, HyperfineResult)> { + sh_println!("Setting up {}/{} for {}", repo_config.org, repo_config.repo, benchmark); + + // Setup a fresh project for this specific benchmark + let project = foundry_bench::BenchmarkProject::setup(&repo_config) + .wrap_err(format!("Failed to setup project for {}/{}", repo_config.org, repo_config.repo))?; + + sh_println!("Running {} on {}/{}", benchmark, repo_config.org, repo_config.repo); + + // Determine runs based on benchmark type + let runs = match benchmark.as_str() { + "forge_coverage" => 1, // Coverage runs only once as an exception + _ => RUNS, // Use default RUNS constant for all other benchmarks + }; + + // Run the appropriate benchmark + let result = project.run(benchmark, version, runs, cli.verbose); + + match result { + Ok(hyperfine_result) => { + sh_println!( + " {} on {}/{}: {:.3}s ± {:.3}s", + benchmark, + repo_config.org, + repo_config.repo, + hyperfine_result.mean, + hyperfine_result.stddev + ); + Ok((repo_config.name.clone(), benchmark.clone(), hyperfine_result)) + } + Err(e) => { + eyre::bail!("Benchmark {} failed for {}/{}: {}", benchmark, repo_config.org, repo_config.repo, e); + } } - } + }) + .collect::>>()?; + + // Add all collected results to the main results structure + for (repo_name, benchmark, hyperfine_result) in version_results { + results.add_result(&benchmark, version, &repo_name, hyperfine_result); } } @@ -240,130 +216,3 @@ fn install_foundry_versions(versions: &[String]) -> Result<()> { sh_println!("✅ All versions installed successfully"); Ok(()) } - -/// Collect benchmark results from Criterion output directory -/// -/// This function reads the Criterion JSON output files and constructs CriterionResult objects. -/// It processes: -/// - benchmark.json for basic benchmark info -/// - estimates.json for mean performance values -/// - change/estimates.json for performance change data (if available) -#[allow(unused_must_use)] -fn collect_benchmark_results( - group_dir: &PathBuf, - benchmark_name: &str, - version: &str, - repos: &[RepoConfig], -) -> Result> { - let mut results = Vec::new(); - - sh_println!("Looking for results in: {}", group_dir.display()); - if !group_dir.exists() { - eyre::bail!("Benchmark directory does not exist: {}", group_dir.display()); - } - - // Iterate through each repository directory - for entry in std::fs::read_dir(group_dir)? { - let entry = entry?; - let path = entry.path(); - - if !path.is_dir() { - sh_println!("Skipping non-directory entry: {}", path.display()); - continue; - } - - let repo_name = path - .file_name() - .ok_or_eyre("Failed to get repo_name using path")? - .to_string_lossy() - .to_string(); - - // Only process repos that are in the specified repos list - let is_valid_repo = repos.iter().any(|r| r.name == repo_name); - if !is_valid_repo { - sh_println!("Skipping unknown repo: {repo_name}"); - continue; - } - - sh_println!("Processing repo: {repo_name}"); - - // Process the benchmark results for this repository - if let Some(result) = process_repo_benchmark(&path, benchmark_name, version, &repo_name)? { - results.push(result); - } - } - - Ok(results) -} - -/// Process benchmark results for a single repository -/// -/// Returns Some(CriterionResult) if valid results are found, None otherwise -fn process_repo_benchmark( - repo_path: &Path, - benchmark_name: &str, - version: &str, - repo_name: &str, -) -> Result> { - let benchmark_json = repo_path.join("new/benchmark.json"); - - if !benchmark_json.exists() { - eyre::bail!( - "Benchmark JSON file does not exist for {}: {}", - repo_name, - benchmark_json.display() - ); - } - - // Create result ID - let id = format!("{benchmark_name}/{version}/{repo_name}"); - - // Read new estimates for mean value - let mean_estimate = read_mean_estimate(repo_path, repo_name)?; - // Read change data if available - let change = read_change_data(repo_path)?; - - Ok(Some(CriterionResult { id, mean: mean_estimate, unit: "ns".to_string(), change })) -} - -/// Read mean estimate from estimates.json -fn read_mean_estimate(repo_path: &Path, repo_name: &str) -> Result { - let estimates_json = repo_path.join("new/estimates.json"); - if !estimates_json.exists() { - eyre::bail!( - "Estimates JSON file does not exist for {}: {}", - repo_name, - estimates_json.display() - ); - } - - let estimates_content = std::fs::read_to_string(&estimates_json)?; - let estimates = serde_json::from_str::(&estimates_content)?; - let mean_obj = estimates.get("mean").ok_or_eyre("No mean value found in estimates.json")?; - let estimate = serde_json::from_value::(mean_obj.clone()) - .wrap_err("Failed to parse mean estimate from estimates.json")?; - Ok(estimate) -} - -/// Read change data from change/estimates.json if it exists -fn read_change_data(repo_path: &Path) -> Result> { - let change_json = repo_path.join("change/estimates.json"); - - if !change_json.exists() { - return Ok(None); - } - - let change_content = std::fs::read_to_string(&change_json)?; - let change_data = serde_json::from_str::(&change_content)?; - - let mean_change = change_data.get("mean").and_then(|m| { - // The change is in decimal format (e.g., 0.03 = 3%) - let decimal = m["point_estimate"].as_f64()?; - Some(ChangeEstimate { - estimate: decimal * 100.0, // Convert to percentage - unit: "%".to_string(), - }) - }); - - Ok(Some(Change { mean: mean_change, median: None, change: None })) -} diff --git a/benches/src/results.rs b/benches/src/results.rs index 90eb770840374..73d26b37662d7 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -1,12 +1,37 @@ -use crate::{criterion_types::CriterionResult, RepoConfig}; +use crate::RepoConfig; use eyre::Result; +use serde::{Deserialize, Serialize}; use std::{collections::HashMap, process::Command}; +/// Hyperfine benchmark result +#[derive(Debug, Deserialize, Serialize)] +pub struct HyperfineResult { + pub command: String, + pub mean: f64, + pub stddev: f64, + pub median: f64, + pub user: f64, + pub system: f64, + pub min: f64, + pub max: f64, + pub times: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub exit_codes: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub parameters: Option>, +} + +/// Hyperfine JSON output format +#[derive(Debug, Deserialize, Serialize)] +pub struct HyperfineOutput { + pub results: Vec, +} + /// Aggregated benchmark results #[derive(Debug, Default)] pub struct BenchmarkResults { /// Map of benchmark_name -> version -> repo -> result - pub data: HashMap>>, + pub data: HashMap>>, /// Track the baseline version for comparison pub baseline_version: Option, } @@ -25,7 +50,7 @@ impl BenchmarkResults { benchmark: &str, version: &str, repo: &str, - result: CriterionResult, + result: HyperfineResult, ) { self.data .entry(benchmark.to_string()) @@ -111,7 +136,7 @@ impl BenchmarkResults { fn generate_benchmark_table( &self, benchmark_name: &str, - version_data: &HashMap>, + version_data: &HashMap>, versions: &[String], repos: &[RepoConfig], ) -> String { @@ -147,7 +172,7 @@ impl BenchmarkResults { /// This function creates the markdown table rows for each repository, /// showing the benchmark results for each version. fn generate_table_rows( - version_data: &HashMap>, + version_data: &HashMap>, versions: &[String], repos: &[RepoConfig], ) -> String { @@ -174,7 +199,7 @@ fn generate_table_rows( /// 1. Check if version data exists /// 2. Check if repository data exists for this version fn get_benchmark_cell_content( - version_data: &HashMap>, + version_data: &HashMap>, version: &str, repo_name: &str, ) -> String { @@ -182,7 +207,7 @@ fn get_benchmark_cell_content( if let Some(repo_data) = version_data.get(version) { // Check if we have data for this repository if let Some(result) = repo_data.get(repo_name) { - return format_duration(result.mean.point_estimate, &result.unit); + return format_duration_seconds(result.mean); } } @@ -191,30 +216,27 @@ fn get_benchmark_cell_content( pub fn format_benchmark_name(name: &str) -> String { match name { - "forge-test" => "Forge Test", - "forge-build-no-cache" => "Forge Build (No Cache)", - "forge-build-with-cache" => "Forge Build (With Cache)", - "forge-fuzz-test" => "Forge Fuzz Test", - "forge-coverage" => "Forge Coverage", + "forge_test" => "Forge Test", + "forge_build_no_cache" => "Forge Build (No Cache)", + "forge_build_with_cache" => "Forge Build (With Cache)", + "forge_fuzz_test" => "Forge Fuzz Test", + "forge_coverage" => "Forge Coverage", _ => name, } .to_string() } -pub fn format_duration(nanos: f64, unit: &str) -> String { - match unit { - "ns" => { - if nanos < 1_000.0 { - format!("{nanos:.2} ns") - } else if nanos < 1_000_000.0 { - format!("{:.2} µs", nanos / 1_000.0) - } else if nanos < 1_000_000_000.0 { - format!("{:.2} ms", nanos / 1_000_000.0) - } else { - format!("{:.2} s", nanos / 1_000_000_000.0) - } - } - _ => format!("{nanos:.2} {unit}"), +pub fn format_duration_seconds(seconds: f64) -> String { + if seconds < 0.001 { + format!("{:.2} ms", seconds * 1000.0) + } else if seconds < 1.0 { + format!("{:.3} s", seconds) + } else if seconds < 60.0 { + format!("{:.2} s", seconds) + } else { + let minutes = (seconds / 60.0).floor(); + let remaining_seconds = seconds % 60.0; + format!("{:.0}m {:.1}s", minutes, remaining_seconds) } } @@ -222,4 +244,4 @@ pub fn get_rustc_version() -> Result { let output = Command::new("rustc").arg("--version").output()?; Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) -} +} \ No newline at end of file From a79409dcce7bdf153c5fdd9d051ea8985feaaae6 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Thu, 3 Jul 2025 19:33:53 +0530 Subject: [PATCH 40/74] forge version details: hash and date --- .github/workflows/benchmarks.yml | 213 ++++++++----------------------- LATEST.md | 28 ++++ benches/src/lib.rs | 33 +++++ benches/src/main.rs | 13 +- benches/src/results.rs | 12 +- 5 files changed, 133 insertions(+), 166 deletions(-) create mode 100644 LATEST.md diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 37ba3656df536..8ef0069e2c77a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,6 +1,8 @@ name: Foundry Benchmarks on: + pull_request: + types: [opened, synchronize, reopened] workflow_dispatch: inputs: pr_number: @@ -23,8 +25,8 @@ permissions: pull-requests: write jobs: - forge-test: - name: Run forge_test and forge_fuzz_test benchmarks + benchmark: + name: Run Foundry Benchmarks runs-on: ubuntu-latest steps: @@ -40,184 +42,75 @@ jobs: workspaces: | ./ - - name: Install foundryup + - name: Install Foundry run: | curl -L https://foundry.paradigm.xyz | bash echo "$HOME/.foundry/bin" >> $GITHUB_PATH - - name: Build benchmark binary - run: cargo build --release --bin foundry-bench - - - name: Run forge_test and forge_fuzz_test benchmarks - run: | - VERSIONS="${{ github.event.inputs.versions }}" - REPOS="${{ github.event.inputs.repos }}" - - # Build the command with force-install for CI - CMD="./target/release/foundry-bench --output-dir ./benches --force-install" - CMD="$CMD --versions $VERSIONS" - CMD="$CMD --repos $REPOS" - CMD="$CMD --benchmarks forge_test,forge_fuzz_test" - CMD="$CMD --output-file forge_test_results.md" - - echo "Running: $CMD" - $CMD - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: forge-test-results - path: benches/forge_test_results.md - - forge-build: - name: Run forge_build benchmarks (with and without cache) - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - ./ - - - name: Install foundryup + - name: Install hyperfine run: | - curl -L https://foundry.paradigm.xyz | bash - echo "$HOME/.foundry/bin" >> $GITHUB_PATH + wget https://github.com/sharkdp/hyperfine/releases/download/v1.18.0/hyperfine_1.18.0_amd64.deb + sudo dpkg -i hyperfine_1.18.0_amd64.deb - name: Build benchmark binary run: cargo build --release --bin foundry-bench - - name: Run forge_build benchmarks (with and without cache) + - name: Run forge test benchmark run: | - VERSIONS="${{ github.event.inputs.versions }}" - REPOS="${{ github.event.inputs.repos }}" - - # Build the command with force-install for CI - CMD="./target/release/foundry-bench --output-dir ./benches --force-install" - CMD="$CMD --versions $VERSIONS" - CMD="$CMD --repos $REPOS" - CMD="$CMD --benchmarks forge_build_with_cache,forge_build_no_cache" - CMD="$CMD --output-file forge_build_results.md" - - echo "Running: $CMD" - $CMD - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: forge-build-results - path: benches/forge_build_results.md - - forge-coverage: - name: Run forge_coverage benchmark - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - ./ - - - name: Install foundryup + VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" + REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" + + ./target/release/foundry-bench --output-dir ./benches --force-install \ + --versions $VERSIONS \ + --repos $REPOS \ + --benchmarks forge_test \ + --output-file forge_test.md + + - name: Run forge build (no cache) benchmark run: | - curl -L https://foundry.paradigm.xyz | bash - echo "$HOME/.foundry/bin" >> $GITHUB_PATH - - - name: Build benchmark binary - run: cargo build --release --bin foundry-bench - - - name: Run forge_coverage benchmark + VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" + REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" + + ./target/release/foundry-bench --output-dir ./benches --force-install \ + --versions $VERSIONS \ + --repos $REPOS \ + --benchmarks forge_build_no_cache \ + --output-file forge_build_no_cache.md + + - name: Run forge build (with cache) benchmark run: | - VERSIONS="${{ github.event.inputs.versions }}" - REPOS="${{ github.event.inputs.repos }}" - - # Build the command with force-install for CI - CMD="./target/release/foundry-bench --output-dir ./benches --force-install" - CMD="$CMD --versions $VERSIONS" - CMD="$CMD --repos $REPOS" - CMD="$CMD --benchmarks forge_coverage" - CMD="$CMD --output-file forge_coverage_results.md" - - echo "Running: $CMD" - $CMD - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: forge-coverage-results - path: benches/forge_coverage_results.md - - collect-results: - name: Collect and commit results - runs-on: ubuntu-latest - needs: [forge-test, forge-build, forge-coverage] - if: always() - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - path: benches/ + VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" + REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" + + ./target/release/foundry-bench --output-dir ./benches --force-install \ + --versions $VERSIONS \ + --repos $REPOS \ + --benchmarks forge_build_with_cache \ + --output-file forge_build_with_cache.md - name: Combine benchmark results run: | - # Create combined results file + # Combine all benchmark results into LATEST.md echo "# Foundry Benchmark Results" > benches/LATEST.md echo "" >> benches/LATEST.md - echo "Generated on: $(date -u)" >> benches/LATEST.md + echo "Generated at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> benches/LATEST.md echo "" >> benches/LATEST.md - - # Add forge_test results if available - if [ -f "benches/forge-test-results/forge_test_results.md" ]; then - echo "## Forge Test and Fuzz Test Results" >> benches/LATEST.md - echo "" >> benches/LATEST.md - tail -n +2 benches/forge-test-results/forge_test_results.md >> benches/LATEST.md - echo "" >> benches/LATEST.md - fi - - # Add forge_build results if available - if [ -f "benches/forge-build-results/forge_build_results.md" ]; then - echo "## Forge Build Results" >> benches/LATEST.md - echo "" >> benches/LATEST.md - tail -n +2 benches/forge-build-results/forge_build_results.md >> benches/LATEST.md - echo "" >> benches/LATEST.md - fi - - # Add forge_coverage results if available - if [ -f "benches/forge-coverage-results/forge_coverage_results.md" ]; then - echo "## Forge Coverage Results" >> benches/LATEST.md - echo "" >> benches/LATEST.md - tail -n +2 benches/forge-coverage-results/forge_coverage_results.md >> benches/LATEST.md - echo "" >> benches/LATEST.md - fi - - # Clean up artifact directories - rm -rf benches/forge-test-results - rm -rf benches/forge-build-results - rm -rf benches/forge-coverage-results + + # Add each benchmark result if it exists + for bench in forge_test forge_build_no_cache forge_build_with_cache; do + if [ -f "benches/${bench}.md" ]; then + # Skip the header from individual files and append content + tail -n +2 "benches/${bench}.md" >> benches/LATEST.md + echo "" >> benches/LATEST.md + fi + done - name: Commit benchmark results + if: github.event_name != 'pull_request' run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git add benches/LATEST.md benches/forge_test_results.md benches/forge_build_results.md benches/forge_coverage_results.md 2>/dev/null || true + git add benches/LATEST.md benches/forge_test.md benches/forge_build_no_cache.md benches/forge_build_with_cache.md if git diff --staged --quiet; then echo "No changes to commit" else @@ -243,11 +136,11 @@ jobs: fi - name: Comment on PR - if: github.event.inputs.pr_number != '' + if: github.event.inputs.pr_number != '' || github.event_name == 'pull_request' uses: actions/github-script@v7 with: script: | - const prNumber = ${{ github.event.inputs.pr_number }}; + const prNumber = ${{ github.event.inputs.pr_number || github.event.pull_request.number }}; const benchmarkResults = `${{ steps.benchmark_results.outputs.results }}`; const comment = `## 📊 Foundry Benchmark Results @@ -270,4 +163,4 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, body: comment - }); + }); \ No newline at end of file diff --git a/LATEST.md b/LATEST.md new file mode 100644 index 0000000000000..24f57afaaa928 --- /dev/null +++ b/LATEST.md @@ -0,0 +1,28 @@ +# Foundry Benchmark Results + +**Date**: 2025-07-03 19:31:14 + +## Summary + +Benchmarked 2 Foundry versions across 1 repositories. + +### Repositories Tested + +1. [ithacaxyz/account](https://github.com/ithacaxyz/account) + +### Foundry Versions + +- **stable**: forge Version: 1.2.3-stable +- **nightly**: forge Version: 1.2.3-nightly + +## Forge Test + +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 3.87 s | 3.41 s | + +## System Information + +- **OS**: macos +- **CPU**: 8 +- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) diff --git a/benches/src/lib.rs b/benches/src/lib.rs index d2e790a3d6f1e..b82ede08ad73d 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -367,6 +367,39 @@ pub fn get_forge_version() -> Result { Ok(version.lines().next().unwrap_or("unknown").to_string()) } +/// Get the full forge version details including commit hash and date +pub fn get_forge_version_details() -> Result { + let output = Command::new("forge") + .args(["--version"]) + .output() + .wrap_err("Failed to get forge version")?; + + if !output.status.success() { + eyre::bail!("forge --version failed"); + } + + let full_output = + String::from_utf8(output.stdout).wrap_err("Invalid UTF-8 in forge version output")?; + + // Extract relevant lines and format them + let lines: Vec<&str> = full_output.lines().collect(); + if lines.len() >= 3 { + // Extract version, commit, and timestamp + let version = lines[0].trim(); + let commit = lines[1].trim().replace("Commit SHA: ", ""); + let timestamp = lines[2].trim().replace("Build Timestamp: ", ""); + + // Format as: "forge 1.2.3-nightly (51650ea 2025-06-27)" + let short_commit = &commit[..7]; // First 7 chars of commit hash + let date = timestamp.split('T').next().unwrap_or(×tamp); + + Ok(format!("{} ({} {})", version, short_commit, date)) + } else { + // Fallback to just the first line if format is unexpected + Ok(lines.first().unwrap_or(&"unknown").to_string()) + } +} + /// Get Foundry versions to benchmark from environment variable or default /// /// Reads from FOUNDRY_BENCH_VERSIONS environment variable if set, diff --git a/benches/src/main.rs b/benches/src/main.rs index 13bf7fca2e99d..ea78ee4532954 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,14 +1,14 @@ use clap::Parser; use eyre::{Result, WrapErr}; use foundry_bench::{ - get_forge_version, + get_forge_version, get_forge_version_details, results::{BenchmarkResults, HyperfineResult}, switch_foundry_version, RepoConfig, BENCHMARK_REPOS, FOUNDRY_VERSIONS, RUNS, }; use foundry_common::sh_println; use once_cell::sync::Lazy; use rayon::prelude::*; -use std::{fs::File, io::Write, path::PathBuf, process::Command, sync::Mutex}; +use std::{fs, path::PathBuf, process::Command, sync::Mutex}; const ALL_BENCHMARKS: [&str; 5] = [ "forge_test", @@ -125,9 +125,13 @@ fn main() -> Result<()> { sh_println!("🔧 Switching to Foundry version: {version}"); switch_version_safe(version)?; - // Verify the switch + // Verify the switch and capture full version details let current = get_forge_version()?; sh_println!("Current version: {}", current.trim()); + + // Get and store the full version details with commit hash and date + let version_details = get_forge_version_details()?; + results.add_version_details(version, version_details); // Create a list of all benchmark tasks let benchmark_tasks: Vec<_> = repos @@ -189,8 +193,7 @@ fn main() -> Result<()> { sh_println!("📝 Generating report..."); let markdown = results.generate_markdown(&versions, &repos); let output_path = cli.output_dir.join(cli.output_file); - let mut file = File::create(&output_path).wrap_err("Failed to create output file")?; - file.write_all(markdown.as_bytes()).wrap_err("Failed to write output file")?; + fs::write(&output_path, markdown).wrap_err("Failed to write output file")?; sh_println!("✅ Report written to: {}", output_path.display()); Ok(()) diff --git a/benches/src/results.rs b/benches/src/results.rs index 73d26b37662d7..5daf5446a3a9a 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -34,6 +34,8 @@ pub struct BenchmarkResults { pub data: HashMap>>, /// Track the baseline version for comparison pub baseline_version: Option, + /// Map of version name -> full version details + pub version_details: HashMap, } impl BenchmarkResults { @@ -60,6 +62,10 @@ impl BenchmarkResults { .insert(repo.to_string(), result); } + pub fn add_version_details(&mut self, version: &str, details: String) { + self.version_details.insert(version.to_string(), details); + } + pub fn generate_markdown(&self, versions: &[String], repos: &[RepoConfig]) -> String { let mut output = String::new(); @@ -104,7 +110,11 @@ impl BenchmarkResults { // Versions tested output.push_str("### Foundry Versions\n\n"); for version in versions { - output.push_str(&format!("- {version}\n")); + if let Some(details) = self.version_details.get(version) { + output.push_str(&format!("- **{version}**: {}\n", details.trim())); + } else { + output.push_str(&format!("- {version}\n")); + } } output.push('\n'); From 354a8fed68740209090947c76258d83fc7bc4bb2 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 4 Jul 2025 11:50:01 +0530 Subject: [PATCH 41/74] run benches again - run cov with --ir-min --- .github/workflows/benchmarks.yml | 10 +++++----- LATEST.md | 8 ++++---- benches/COVERAGE_BENCH.md | 13 +++++++------ benches/FUZZ_BENCH.md | 10 +++++----- benches/LATEST.md | 32 ++++++++++++++++---------------- benches/src/lib.rs | 4 +++- benches/src/main.rs | 29 ++++++++++++++++++++--------- benches/src/results.rs | 2 +- crates/test-utils/src/util.rs | 2 +- test_coverage | 1 + 10 files changed, 63 insertions(+), 48 deletions(-) create mode 160000 test_coverage diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 8ef0069e2c77a..11cedf5f154cd 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -59,7 +59,7 @@ jobs: run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" - + ./target/release/foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ @@ -70,7 +70,7 @@ jobs: run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" - + ./target/release/foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ @@ -81,7 +81,7 @@ jobs: run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" - + ./target/release/foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ @@ -95,7 +95,7 @@ jobs: echo "" >> benches/LATEST.md echo "Generated at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> benches/LATEST.md echo "" >> benches/LATEST.md - + # Add each benchmark result if it exists for bench in forge_test forge_build_no_cache forge_build_with_cache; do if [ -f "benches/${bench}.md" ]; then @@ -163,4 +163,4 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, body: comment - }); \ No newline at end of file + }); diff --git a/LATEST.md b/LATEST.md index 24f57afaaa928..e542d7def2948 100644 --- a/LATEST.md +++ b/LATEST.md @@ -1,6 +1,6 @@ # Foundry Benchmark Results -**Date**: 2025-07-03 19:31:14 +**Date**: 2025-07-03 19:43:36 ## Summary @@ -12,14 +12,14 @@ Benchmarked 2 Foundry versions across 1 repositories. ### Foundry Versions -- **stable**: forge Version: 1.2.3-stable -- **nightly**: forge Version: 1.2.3-nightly +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) ## Forge Test | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 3.87 s | 3.41 s | +| ithacaxyz-account | 3.70 s | 3.17 s | ## System Information diff --git a/benches/COVERAGE_BENCH.md b/benches/COVERAGE_BENCH.md index 3f5c6869a36df..815e25ca3da65 100644 --- a/benches/COVERAGE_BENCH.md +++ b/benches/COVERAGE_BENCH.md @@ -1,10 +1,10 @@ # Foundry Benchmark Results -**Date**: 2025-07-01 17:32:13 +**Date**: 2025-07-04 11:37:27 ## Summary -Benchmarked 1 Foundry versions across 1 repositories. +Benchmarked 2 Foundry versions across 1 repositories. ### Repositories Tested @@ -12,13 +12,14 @@ Benchmarked 1 Foundry versions across 1 repositories. ### Foundry Versions -- nightly +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) ## Forge Coverage -| Repository | nightly | -| ----------------- | ------- | -| ithacaxyz-account | 25.54 s | +| Repository | stable | nightly | +| ----------------- | ------- | ------- | +| ithacaxyz-account | 26.44 s | 32.34 s | ## System Information diff --git a/benches/FUZZ_BENCH.md b/benches/FUZZ_BENCH.md index 46a51b95b4178..9fb08b1c4caee 100644 --- a/benches/FUZZ_BENCH.md +++ b/benches/FUZZ_BENCH.md @@ -1,6 +1,6 @@ # Foundry Benchmark Results -**Date**: 2025-07-01 16:41:48 +**Date**: 2025-07-04 11:05:43 ## Summary @@ -13,15 +13,15 @@ Benchmarked 2 Foundry versions across 2 repositories. ### Foundry Versions -- stable -- nightly +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) ## Forge Fuzz Test | Repository | stable | nightly | | ----------------- | ------ | ------- | -| ithacaxyz-account | 4.34 s | 3.69 s | -| solady | 3.68 s | 2.92 s | +| ithacaxyz-account | 3.62 s | 3.31 s | +| solady | 2.82 s | 2.65 s | ## System Information diff --git a/benches/LATEST.md b/benches/LATEST.md index 9147536d83c22..d7c65883a9e4f 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # Foundry Benchmark Results -**Date**: 2025-06-30 17:23:42 +**Date**: 2025-07-04 11:14:10 ## Summary @@ -13,29 +13,29 @@ Benchmarked 2 Foundry versions across 2 repositories. ### Foundry Versions -- stable -- nightly +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) -## Forge Build Performance (With Cache) +## Forge Test -| Repository | stable | nightly | -| ----------------- | --------- | --------- | -| ithacaxyz-account | 227.20 ms | 263.73 ms | -| solady | 148.77 ms | 192.25 ms | +| Repository | stable | nightly | +| ----------------- | ------ | ------- | +| ithacaxyz-account | 4.07 s | 4.39 s | +| solady | 3.93 s | 5.26 s | -## Forge Test Performance +## Forge Build (With Cache) | Repository | stable | nightly | | ----------------- | ------ | ------- | -| ithacaxyz-account | 4.88 s | 4.37 s | -| solady | 3.45 s | 3.43 s | +| ithacaxyz-account | 2.02 s | 4.96 s | +| solady | 3.22 s | 3.66 s | -## Forge Build Performance (No Cache) +## Forge Build (No Cache) -| Repository | stable | nightly | -| ----------------- | ------- | ------- | -| ithacaxyz-account | 16.35 s | 13.85 s | -| solady | 15.27 s | 15.12 s | +| Repository | stable | nightly | +| ----------------- | ------ | ------- | +| ithacaxyz-account | 3.54 s | 3.18 s | +| solady | 5.36 s | 3.71 s | ## System Information diff --git a/benches/src/lib.rs b/benches/src/lib.rs index b82ede08ad73d..103741abbe95d 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -269,6 +269,7 @@ impl BenchmarkProject { output.results.into_iter().next().ok_or_else(|| eyre::eyre!("No results from hyperfine")) } + /// Benchmark forge test pub fn bench_forge_test(&self, version: &str, runs: u32, verbose: bool) -> Result { // Build before running tests @@ -304,7 +305,8 @@ impl BenchmarkProject { /// Benchmark forge coverage pub fn bench_forge_coverage(&self, version: &str, runs: u32, verbose: bool) -> Result { // No setup needed, forge coverage builds internally - self.hyperfine("forge_coverage", version, "forge coverage", runs, None, None, verbose) + // Use --ir-minimum to avoid "Stack too deep" errors + self.hyperfine("forge_coverage", version, "forge coverage --ir-minimum", runs, None, None, verbose) } /// Get the root path of the project diff --git a/benches/src/main.rs b/benches/src/main.rs index ea78ee4532954..e1f2545bbb296 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -78,7 +78,6 @@ fn main() -> Result<()> { FOUNDRY_VERSIONS.iter().map(|&s| s.to_string()).collect() }; - // Get repo configurations let repos = if let Some(repo_specs) = cli.repos.clone() { repo_specs @@ -128,7 +127,7 @@ fn main() -> Result<()> { // Verify the switch and capture full version details let current = get_forge_version()?; sh_println!("Current version: {}", current.trim()); - + // Get and store the full version details with commit hash and date let version_details = get_forge_version_details()?; results.add_version_details(version, version_details); @@ -147,12 +146,18 @@ fn main() -> Result<()> { let version_results = benchmark_tasks .par_iter() .map(|(repo_config, benchmark)| -> Result<(String, String, HyperfineResult)> { - sh_println!("Setting up {}/{} for {}", repo_config.org, repo_config.repo, benchmark); - + sh_println!( + "Setting up {}/{} for {}", + repo_config.org, + repo_config.repo, + benchmark + ); + // Setup a fresh project for this specific benchmark - let project = foundry_bench::BenchmarkProject::setup(&repo_config) - .wrap_err(format!("Failed to setup project for {}/{}", repo_config.org, repo_config.repo))?; - + let project = foundry_bench::BenchmarkProject::setup(&repo_config).wrap_err( + format!("Failed to setup project for {}/{}", repo_config.org, repo_config.repo), + )?; + sh_println!("Running {} on {}/{}", benchmark, repo_config.org, repo_config.repo); // Determine runs based on benchmark type @@ -172,12 +177,18 @@ fn main() -> Result<()> { repo_config.org, repo_config.repo, hyperfine_result.mean, - hyperfine_result.stddev + hyperfine_result.stddev.unwrap_or(0.0) ); Ok((repo_config.name.clone(), benchmark.clone(), hyperfine_result)) } Err(e) => { - eyre::bail!("Benchmark {} failed for {}/{}: {}", benchmark, repo_config.org, repo_config.repo, e); + eyre::bail!( + "Benchmark {} failed for {}/{}: {}", + benchmark, + repo_config.org, + repo_config.repo, + e + ); } } }) diff --git a/benches/src/results.rs b/benches/src/results.rs index 5daf5446a3a9a..2eaccee4e6ce4 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -8,7 +8,7 @@ use std::{collections::HashMap, process::Command}; pub struct HyperfineResult { pub command: String, pub mean: f64, - pub stddev: f64, + pub stddev: Option, pub median: f64, pub user: f64, pub system: f64, diff --git a/crates/test-utils/src/util.rs b/crates/test-utils/src/util.rs index 8016add1c4820..51cda6a19e90e 100644 --- a/crates/test-utils/src/util.rs +++ b/crates/test-utils/src/util.rs @@ -316,7 +316,7 @@ pub fn initialize(target: &Path) { /// Clones a remote repository into the specified directory. Panics if the command fails. pub fn clone_remote(repo_url: &str, target_dir: &str) { let mut cmd = Command::new("git"); - cmd.args(["clone", "--no-tags", "--recursive", "--shallow-submodules"]); + cmd.args(["clone", "--recursive", "--shallow-submodules"]); cmd.args([repo_url, target_dir]); println!("{cmd:?}"); let status = cmd.status().unwrap(); diff --git a/test_coverage b/test_coverage new file mode 160000 index 0000000000000..fcad3c1724559 --- /dev/null +++ b/test_coverage @@ -0,0 +1 @@ +Subproject commit fcad3c1724559f49b06fe3e4910fd9f19546d423 From 3666090340f60c5c3619f88d5089264ba1dedc1f Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 4 Jul 2025 12:07:27 +0530 Subject: [PATCH 42/74] del --- test-forge-regex | 1 - test_coverage | 1 - 2 files changed, 2 deletions(-) delete mode 160000 test-forge-regex delete mode 160000 test_coverage diff --git a/test-forge-regex b/test-forge-regex deleted file mode 160000 index c3decafda6d31..0000000000000 --- a/test-forge-regex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c3decafda6d313fa159ce2a50a352467af7fa58b diff --git a/test_coverage b/test_coverage deleted file mode 160000 index fcad3c1724559..0000000000000 --- a/test_coverage +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fcad3c1724559f49b06fe3e4910fd9f19546d423 From fcd2d82fd122f202ba47825631ef5d86f5a35ae1 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:00:24 +0530 Subject: [PATCH 43/74] bench in separate ci jobs --- .github/workflows/benchmarks.yml | 158 +++++++++++++++++++++++++++---- 1 file changed, 137 insertions(+), 21 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 11cedf5f154cd..b41975d3b853a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -25,10 +25,9 @@ permissions: pull-requests: write jobs: - benchmark: - name: Run Foundry Benchmarks + setup: + name: Setup and Build runs-on: ubuntu-latest - steps: - name: Checkout repository uses: actions/checkout@v4 @@ -49,44 +48,161 @@ jobs: - name: Install hyperfine run: | - wget https://github.com/sharkdp/hyperfine/releases/download/v1.18.0/hyperfine_1.18.0_amd64.deb - sudo dpkg -i hyperfine_1.18.0_amd64.deb + wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb - name: Build benchmark binary run: cargo build --release --bin foundry-bench - - name: Run forge test benchmark + - name: Upload benchmark binary + uses: actions/upload-artifact@v4 + with: + name: foundry-bench + path: target/release/foundry-bench + + forge-test-bench: + name: Forge Test Benchmarks + needs: setup + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Foundry + run: | + curl -L https://foundry.paradigm.xyz | bash + echo "$HOME/.foundry/bin" >> $GITHUB_PATH + + - name: Install hyperfine + run: | + wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb + + - name: Download benchmark binary + uses: actions/download-artifact@v4 + with: + name: foundry-bench + + - name: Make binary executable + run: chmod +x foundry-bench + + - name: Run forge test benchmarks run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" - ./target/release/foundry-bench --output-dir ./benches --force-install \ + ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ - --benchmarks forge_test \ - --output-file forge_test.md + --benchmarks forge_test,forge_fuzz_test \ + --output-file forge_test_bench.md + + - name: Upload test benchmark results + uses: actions/upload-artifact@v4 + with: + name: forge-test-results + path: benches/forge_test_bench.md + + forge-build-bench: + name: Forge Build Benchmarks + needs: setup + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Foundry + run: | + curl -L https://foundry.paradigm.xyz | bash + echo "$HOME/.foundry/bin" >> $GITHUB_PATH + + - name: Install hyperfine + run: | + wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb + + - name: Download benchmark binary + uses: actions/download-artifact@v4 + with: + name: foundry-bench - - name: Run forge build (no cache) benchmark + - name: Make binary executable + run: chmod +x foundry-bench + + - name: Run forge build benchmarks run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" - ./target/release/foundry-bench --output-dir ./benches --force-install \ + ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ - --benchmarks forge_build_no_cache \ - --output-file forge_build_no_cache.md + --benchmarks forge_build_no_cache,forge_build_with_cache \ + --output-file forge_build_bench.md - - name: Run forge build (with cache) benchmark + - name: Upload build benchmark results + uses: actions/upload-artifact@v4 + with: + name: forge-build-results + path: benches/forge_build_bench.md + + forge-coverage-bench: + name: Forge Coverage Benchmarks + needs: setup + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Foundry + run: | + curl -L https://foundry.paradigm.xyz | bash + echo "$HOME/.foundry/bin" >> $GITHUB_PATH + + - name: Install hyperfine + run: | + wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb + + - name: Download benchmark binary + uses: actions/download-artifact@v4 + with: + name: foundry-bench + + - name: Make binary executable + run: chmod +x foundry-bench + + - name: Run forge coverage benchmarks run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" - REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" + # Coverage only runs on ithacaxyz/account:v0.3.2 - ./target/release/foundry-bench --output-dir ./benches --force-install \ + ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ - --repos $REPOS \ - --benchmarks forge_build_with_cache \ - --output-file forge_build_with_cache.md + --repos ithacaxyz/account:v0.3.2 \ + --benchmarks forge_coverage \ + --output-file forge_coverage_bench.md + + - name: Upload coverage benchmark results + uses: actions/upload-artifact@v4 + with: + name: forge-coverage-results + path: benches/forge_coverage_bench.md + + combine-results: + name: Combine and Publish Results + needs: [forge-test-bench, forge-build-bench, forge-coverage-bench] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download all benchmark results + uses: actions/download-artifact@v4 + with: + pattern: forge-*-results + path: benches/ + merge-multiple: true - name: Combine benchmark results run: | @@ -97,7 +213,7 @@ jobs: echo "" >> benches/LATEST.md # Add each benchmark result if it exists - for bench in forge_test forge_build_no_cache forge_build_with_cache; do + for bench in forge_test_bench forge_build_bench forge_coverage_bench; do if [ -f "benches/${bench}.md" ]; then # Skip the header from individual files and append content tail -n +2 "benches/${bench}.md" >> benches/LATEST.md @@ -110,7 +226,7 @@ jobs: run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - git add benches/LATEST.md benches/forge_test.md benches/forge_build_no_cache.md benches/forge_build_with_cache.md + git add benches/LATEST.md benches/forge_test_bench.md benches/forge_build_bench.md benches/forge_coverage_bench.md if git diff --staged --quiet; then echo "No changes to commit" else From 51225d6d4a1269505dd7358a41e506e2e7448726 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:09:30 +0530 Subject: [PATCH 44/74] move combine bench results logic to scripts --- .github/scripts/combine-benchmarks.sh | 39 ++++++++ .github/scripts/commit-and-read-benchmarks.sh | 71 ++++++++++++++ .github/workflows/benchmarks.yml | 48 +--------- benches/src/lib.rs | 92 ++++++++++++++----- benches/src/results.rs | 2 +- 5 files changed, 186 insertions(+), 66 deletions(-) create mode 100755 .github/scripts/combine-benchmarks.sh create mode 100755 .github/scripts/commit-and-read-benchmarks.sh diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh new file mode 100755 index 0000000000000..821d4239ebfd9 --- /dev/null +++ b/.github/scripts/combine-benchmarks.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euo pipefail + +# Script to combine individual benchmark results into LATEST.md +# Usage: ./combine-benchmarks.sh + +OUTPUT_DIR="${1:-benches}" + +# Create output directory if it doesn't exist +mkdir -p "$OUTPUT_DIR" + +# Start building LATEST.md +cat > "$OUTPUT_DIR/LATEST.md" << EOF +# Foundry Benchmark Results + +Generated at: $(date -u '+%Y-%m-%d %H:%M:%S UTC') + +EOF + +# Define the benchmark files to combine in order +BENCHMARK_FILES=( + "forge_test_bench.md" + "forge_build_bench.md" + "forge_coverage_bench.md" +) + +# Add each benchmark result if it exists +for bench_file in "${BENCHMARK_FILES[@]}"; do + if [ -f "$OUTPUT_DIR/$bench_file" ]; then + echo "Adding $bench_file to combined results..." + # Skip the header from individual files (first line) and append content + tail -n +2 "$OUTPUT_DIR/$bench_file" >> "$OUTPUT_DIR/LATEST.md" + echo "" >> "$OUTPUT_DIR/LATEST.md" + else + echo "Warning: $bench_file not found, skipping..." + fi +done + +echo "Successfully combined benchmark results into $OUTPUT_DIR/LATEST.md" \ No newline at end of file diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh new file mode 100755 index 0000000000000..2de978b139859 --- /dev/null +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -euo pipefail + +# Script to commit benchmark results and read them for GitHub Actions output +# Usage: ./commit-and-read-benchmarks.sh + +OUTPUT_DIR="${1:-benches}" +GITHUB_EVENT_NAME="${2:-pull_request}" +GITHUB_REPOSITORY="${3:-}" + +# Function to commit benchmark results +commit_results() { + echo "Configuring git..." + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + echo "Adding benchmark files..." + git add "$OUTPUT_DIR/LATEST.md" \ + "$OUTPUT_DIR/forge_test_bench.md" \ + "$OUTPUT_DIR/forge_build_bench.md" \ + "$OUTPUT_DIR/forge_coverage_bench.md" || { + echo "Warning: Some benchmark files may not exist" + } + + if git diff --staged --quiet; then + echo "No changes to commit" + else + echo "Committing benchmark results..." + git commit -m "chore(\`benches\`): update benchmark results + +🤖 Generated with [Foundry Benchmarks](https://github.com/${GITHUB_REPOSITORY}/actions) + +Co-Authored-By: github-actions " + + echo "Pushing to repository..." + git push + echo "Successfully pushed benchmark results" + fi +} + +# Function to read benchmark results and output for GitHub Actions +read_results() { + if [ -f "$OUTPUT_DIR/LATEST.md" ]; then + echo "Reading benchmark results..." + { + echo 'results<> "$GITHUB_OUTPUT" + echo "Successfully read benchmark results" + else + echo 'results=No benchmark results found.' >> "$GITHUB_OUTPUT" + echo "Warning: No benchmark results found at $OUTPUT_DIR/LATEST.md" + fi +} + +# Main execution +echo "Starting benchmark results processing..." + +# Only commit if not a pull request +if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then + echo "Event is not a pull request, proceeding with commit..." + commit_results +else + echo "Event is a pull request, skipping commit" +fi + +# Always read results for output +read_results + +echo "Benchmark results processing complete" \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b41975d3b853a..7e1acd6629b25 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -205,51 +205,11 @@ jobs: merge-multiple: true - name: Combine benchmark results - run: | - # Combine all benchmark results into LATEST.md - echo "# Foundry Benchmark Results" > benches/LATEST.md - echo "" >> benches/LATEST.md - echo "Generated at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> benches/LATEST.md - echo "" >> benches/LATEST.md - - # Add each benchmark result if it exists - for bench in forge_test_bench forge_build_bench forge_coverage_bench; do - if [ -f "benches/${bench}.md" ]; then - # Skip the header from individual files and append content - tail -n +2 "benches/${bench}.md" >> benches/LATEST.md - echo "" >> benches/LATEST.md - fi - done - - - name: Commit benchmark results - if: github.event_name != 'pull_request' - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add benches/LATEST.md benches/forge_test_bench.md benches/forge_build_bench.md benches/forge_coverage_bench.md - if git diff --staged --quiet; then - echo "No changes to commit" - else - git commit -m "chore(\`benches\`): update benchmark results - - 🤖 Generated with [Foundry Benchmarks](https://github.com/${{ github.repository }}/actions) - - Co-Authored-By: github-actions " - git push - fi - - - name: Read benchmark results + run: ./.github/scripts/combine-benchmarks.sh benches + + - name: Commit and read benchmark results id: benchmark_results - run: | - if [ -f "benches/LATEST.md" ]; then - { - echo 'results<> $GITHUB_OUTPUT - else - echo 'results=No benchmark results found.' >> $GITHUB_OUTPUT - fi + run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}" - name: Comment on PR if: github.event.inputs.pr_number != '' || github.event_name == 'pull_request' diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 103741abbe95d..00075e88cffb7 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -90,7 +90,6 @@ pub fn default_benchmark_repos() -> Vec { // Keep a lazy static for compatibility pub static BENCHMARK_REPOS: Lazy> = Lazy::new(default_benchmark_repos); - /// Foundry versions to benchmark /// /// To add more versions for comparison, install them first: @@ -188,7 +187,7 @@ impl BenchmarkProject { } /// Run a command with hyperfine and return the results - /// + /// /// # Arguments /// * `benchmark_name` - Name of the benchmark for organizing output /// * `version` - Foundry version being benchmarked @@ -197,7 +196,7 @@ impl BenchmarkProject { /// * `setup` - Optional setup command to run before the benchmark series (e.g., "forge build") /// * `conclude` - Optional conclude command to run after each timing run (e.g., cleanup) /// * `verbose` - Whether to show command output - /// + /// /// # Hyperfine flags used: /// * `--runs` - Number of timing runs /// * `--setup` - Execute before the benchmark series (not before each run) @@ -218,13 +217,10 @@ impl BenchmarkProject { // Create structured temp directory for JSON output // Format: ////.json let temp_dir = std::env::temp_dir(); - let json_dir = temp_dir - .join("foundry-bench") - .join(benchmark_name) - .join(version) - .join(&self.name); + let json_dir = + temp_dir.join("foundry-bench").join(benchmark_name).join(version).join(&self.name); std::fs::create_dir_all(&json_dir)?; - + let json_path = json_dir.join(format!("{}.json", benchmark_name)); // Build hyperfine command @@ -269,27 +265,62 @@ impl BenchmarkProject { output.results.into_iter().next().ok_or_else(|| eyre::eyre!("No results from hyperfine")) } - /// Benchmark forge test - pub fn bench_forge_test(&self, version: &str, runs: u32, verbose: bool) -> Result { + pub fn bench_forge_test( + &self, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { // Build before running tests - self.hyperfine("forge_test", version, "forge test", runs, Some("forge build"), None, verbose) + self.hyperfine( + "forge_test", + version, + "forge test", + runs, + Some("forge build"), + None, + verbose, + ) } /// Benchmark forge build with cache - pub fn bench_forge_build_with_cache(&self, version: &str, runs: u32, verbose: bool) -> Result { + pub fn bench_forge_build_with_cache( + &self, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { // No setup needed, uses existing cache self.hyperfine("forge_build_with_cache", version, "forge build", runs, None, None, verbose) } /// Benchmark forge build without cache - pub fn bench_forge_build_no_cache(&self, version: &str, runs: u32, verbose: bool) -> Result { + pub fn bench_forge_build_no_cache( + &self, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { // Clean before the benchmark series - self.hyperfine("forge_build_no_cache", version, "forge build", runs, Some("forge clean"), None, verbose) + self.hyperfine( + "forge_build_no_cache", + version, + "forge build", + runs, + Some("forge clean"), + None, + verbose, + ) } /// Benchmark forge fuzz tests - pub fn bench_forge_fuzz_test(&self, version: &str, runs: u32, verbose: bool) -> Result { + pub fn bench_forge_fuzz_test( + &self, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { // Build before running fuzz tests self.hyperfine( "forge_fuzz_test", @@ -303,10 +334,23 @@ impl BenchmarkProject { } /// Benchmark forge coverage - pub fn bench_forge_coverage(&self, version: &str, runs: u32, verbose: bool) -> Result { + pub fn bench_forge_coverage( + &self, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { // No setup needed, forge coverage builds internally // Use --ir-minimum to avoid "Stack too deep" errors - self.hyperfine("forge_coverage", version, "forge coverage --ir-minimum", runs, None, None, verbose) + self.hyperfine( + "forge_coverage", + version, + "forge coverage --ir-minimum", + runs, + None, + None, + verbose, + ) } /// Get the root path of the project @@ -315,7 +359,13 @@ impl BenchmarkProject { } /// Run a specific benchmark by name - pub fn run(&self, benchmark: &str, version: &str, runs: u32, verbose: bool) -> Result { + pub fn run( + &self, + benchmark: &str, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { match benchmark { "forge_test" => self.bench_forge_test(version, runs, verbose), "forge_build_no_cache" => self.bench_forge_build_no_cache(version, runs, verbose), @@ -390,11 +440,11 @@ pub fn get_forge_version_details() -> Result { let version = lines[0].trim(); let commit = lines[1].trim().replace("Commit SHA: ", ""); let timestamp = lines[2].trim().replace("Build Timestamp: ", ""); - + // Format as: "forge 1.2.3-nightly (51650ea 2025-06-27)" let short_commit = &commit[..7]; // First 7 chars of commit hash let date = timestamp.split('T').next().unwrap_or(×tamp); - + Ok(format!("{} ({} {})", version, short_commit, date)) } else { // Fallback to just the first line if format is unexpected diff --git a/benches/src/results.rs b/benches/src/results.rs index 2eaccee4e6ce4..6f9dee72ab5f5 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -254,4 +254,4 @@ pub fn get_rustc_version() -> Result { let output = Command::new("rustc").arg("--version").output()?; Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) -} \ No newline at end of file +} From f23b52bb97dc99d85da0dac418bb424d9d217eec Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:14:39 +0530 Subject: [PATCH 45/74] setup foundryup in ci --- .github/scripts/setup-foundryup.sh | 47 ++++++++++++++++++++++++++++++ .github/workflows/benchmarks.yml | 16 +++++----- 2 files changed, 55 insertions(+), 8 deletions(-) create mode 100755 .github/scripts/setup-foundryup.sh diff --git a/.github/scripts/setup-foundryup.sh b/.github/scripts/setup-foundryup.sh new file mode 100755 index 0000000000000..1979f6220d56b --- /dev/null +++ b/.github/scripts/setup-foundryup.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -euo pipefail + +# Script to setup foundryup in CI environment +# This ensures foundryup is available in PATH for the benchmark binary + +echo "Setting up foundryup..." + +# Check if foundryup script exists in the repo +if [ ! -f "foundryup/foundryup" ]; then + echo "Error: foundryup/foundryup script not found in repository" + exit 1 +fi + +# Copy foundryup to a location in PATH +echo "Copying foundryup to /usr/local/bin..." +sudo cp foundryup/foundryup /usr/local/bin/foundryup +sudo chmod +x /usr/local/bin/foundryup + +# Verify foundryup is accessible +if ! command -v foundryup &> /dev/null; then + echo "Error: foundryup not found in PATH after installation" + exit 1 +fi + +echo "foundryup is now available at: $(which foundryup)" + +# Create foundry directories +echo "Creating foundry directories..." +mkdir -p "$HOME/.foundry/bin" +mkdir -p "$HOME/.foundry/versions" + +# Export PATH for current session +export PATH="$HOME/.foundry/bin:$PATH" + +# Run foundryup to install default version +echo "Installing default foundry version..." +foundryup + +# Verify installation +if command -v forge &> /dev/null; then + echo "Forge installed successfully: $(forge --version)" +else + echo "Warning: forge not found in PATH after installation" +fi + +echo "Foundry setup complete!" \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7e1acd6629b25..5250869aca8f8 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -41,9 +41,9 @@ jobs: workspaces: | ./ - - name: Install Foundry + - name: Setup Foundry run: | - curl -L https://foundry.paradigm.xyz | bash + ./.github/scripts/setup-foundryup.sh echo "$HOME/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine @@ -68,9 +68,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Foundry + - name: Setup Foundry run: | - curl -L https://foundry.paradigm.xyz | bash + ./.github/scripts/setup-foundryup.sh echo "$HOME/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine @@ -111,9 +111,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Foundry + - name: Setup Foundry run: | - curl -L https://foundry.paradigm.xyz | bash + ./.github/scripts/setup-foundryup.sh echo "$HOME/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine @@ -154,9 +154,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install Foundry + - name: Setup Foundry run: | - curl -L https://foundry.paradigm.xyz | bash + ./.github/scripts/setup-foundryup.sh echo "$HOME/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine From 09a174d95d880d850089b4b1088acb68963a3d5f Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 4 Jul 2025 13:29:25 +0530 Subject: [PATCH 46/74] setup foundryup fix --- .github/scripts/setup-foundryup.sh | 13 ++++++++++--- .github/workflows/benchmarks.yml | 22 ++++++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/scripts/setup-foundryup.sh b/.github/scripts/setup-foundryup.sh index 1979f6220d56b..4ac30d64ea72d 100755 --- a/.github/scripts/setup-foundryup.sh +++ b/.github/scripts/setup-foundryup.sh @@ -27,11 +27,18 @@ echo "foundryup is now available at: $(which foundryup)" # Create foundry directories echo "Creating foundry directories..." -mkdir -p "$HOME/.foundry/bin" -mkdir -p "$HOME/.foundry/versions" + +# Use FOUNDRY_DIR if set, otherwise default to $HOME/.foundry +FOUNDRY_DIR="${FOUNDRY_DIR:-$HOME/.foundry}" +echo "Using FOUNDRY_DIR: $FOUNDRY_DIR" + +# Create all necessary directories +mkdir -p "$FOUNDRY_DIR/bin" +mkdir -p "$FOUNDRY_DIR/versions" +mkdir -p "$FOUNDRY_DIR/share/man/man1" # Export PATH for current session -export PATH="$HOME/.foundry/bin:$PATH" +export PATH="$FOUNDRY_DIR/bin:$PATH" # Run foundryup to install default version echo "Installing default foundry version..." diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 5250869aca8f8..132e1f27c84cf 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -42,9 +42,11 @@ jobs: ./ - name: Setup Foundry + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | ./.github/scripts/setup-foundryup.sh - echo "$HOME/.foundry/bin" >> $GITHUB_PATH + echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine run: | @@ -69,9 +71,11 @@ jobs: uses: actions/checkout@v4 - name: Setup Foundry + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | ./.github/scripts/setup-foundryup.sh - echo "$HOME/.foundry/bin" >> $GITHUB_PATH + echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine run: | @@ -87,6 +91,8 @@ jobs: run: chmod +x foundry-bench - name: Run forge test benchmarks + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" @@ -112,9 +118,11 @@ jobs: uses: actions/checkout@v4 - name: Setup Foundry + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | ./.github/scripts/setup-foundryup.sh - echo "$HOME/.foundry/bin" >> $GITHUB_PATH + echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine run: | @@ -130,6 +138,8 @@ jobs: run: chmod +x foundry-bench - name: Run forge build benchmarks + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" @@ -155,9 +165,11 @@ jobs: uses: actions/checkout@v4 - name: Setup Foundry + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | ./.github/scripts/setup-foundryup.sh - echo "$HOME/.foundry/bin" >> $GITHUB_PATH + echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - name: Install hyperfine run: | @@ -173,6 +185,8 @@ jobs: run: chmod +x foundry-bench - name: Run forge coverage benchmarks + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" # Coverage only runs on ithacaxyz/account:v0.3.2 From 3bbc762fd9fe6a650f9a142169c9bb412360e53d Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 4 Jul 2025 15:21:28 +0530 Subject: [PATCH 47/74] clippy --- benches/src/lib.rs | 5 +++-- benches/src/main.rs | 2 +- benches/src/results.rs | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 00075e88cffb7..35a6415d55ccf 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -204,6 +204,7 @@ impl BenchmarkProject { /// * `--export-json` - Export results to JSON for parsing /// * `--shell=bash` - Use bash for shell command execution /// * `--show-output` - Show command output (when verbose) + #[allow(clippy::too_many_arguments)] fn hyperfine( &self, benchmark_name: &str, @@ -221,7 +222,7 @@ impl BenchmarkProject { temp_dir.join("foundry-bench").join(benchmark_name).join(version).join(&self.name); std::fs::create_dir_all(&json_dir)?; - let json_path = json_dir.join(format!("{}.json", benchmark_name)); + let json_path = json_dir.join(format!("{benchmark_name}.json")); // Build hyperfine command let mut hyperfine_cmd = Command::new("hyperfine"); @@ -445,7 +446,7 @@ pub fn get_forge_version_details() -> Result { let short_commit = &commit[..7]; // First 7 chars of commit hash let date = timestamp.split('T').next().unwrap_or(×tamp); - Ok(format!("{} ({} {})", version, short_commit, date)) + Ok(format!("{version} ({short_commit} {date})")) } else { // Fallback to just the first line if format is unexpected Ok(lines.first().unwrap_or(&"unknown").to_string()) diff --git a/benches/src/main.rs b/benches/src/main.rs index e1f2545bbb296..218fcf79b52b9 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -154,7 +154,7 @@ fn main() -> Result<()> { ); // Setup a fresh project for this specific benchmark - let project = foundry_bench::BenchmarkProject::setup(&repo_config).wrap_err( + let project = foundry_bench::BenchmarkProject::setup(repo_config).wrap_err( format!("Failed to setup project for {}/{}", repo_config.org, repo_config.repo), )?; diff --git a/benches/src/results.rs b/benches/src/results.rs index 6f9dee72ab5f5..14b180083d63b 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -240,13 +240,13 @@ pub fn format_duration_seconds(seconds: f64) -> String { if seconds < 0.001 { format!("{:.2} ms", seconds * 1000.0) } else if seconds < 1.0 { - format!("{:.3} s", seconds) + format!("{seconds:.3} s") } else if seconds < 60.0 { - format!("{:.2} s", seconds) + format!("{seconds:.2} s") } else { let minutes = (seconds / 60.0).floor(); let remaining_seconds = seconds % 60.0; - format!("{:.0}m {:.1}s", minutes, remaining_seconds) + format!("{minutes:.0}m {remaining_seconds:.1}s") } } From 7335a6ed0c439b3e2a3c1f3eddb658033a99d2fb Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:06:14 +0530 Subject: [PATCH 48/74] ci: run on foundry-runner --- .github/workflows/benchmarks.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 132e1f27c84cf..c9a433c067b6c 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -27,7 +27,7 @@ permissions: jobs: setup: name: Setup and Build - runs-on: ubuntu-latest + runs-on: foundry-runner steps: - name: Checkout repository uses: actions/checkout@v4 @@ -65,7 +65,7 @@ jobs: forge-test-bench: name: Forge Test Benchmarks needs: setup - runs-on: ubuntu-latest + runs-on: foundry-runner steps: - name: Checkout repository uses: actions/checkout@v4 @@ -112,7 +112,7 @@ jobs: forge-build-bench: name: Forge Build Benchmarks needs: setup - runs-on: ubuntu-latest + runs-on: foundry-runner steps: - name: Checkout repository uses: actions/checkout@v4 @@ -159,7 +159,7 @@ jobs: forge-coverage-bench: name: Forge Coverage Benchmarks needs: setup - runs-on: ubuntu-latest + runs-on: foundry-runner steps: - name: Checkout repository uses: actions/checkout@v4 @@ -206,7 +206,7 @@ jobs: combine-results: name: Combine and Publish Results needs: [forge-test-bench, forge-build-bench, forge-coverage-bench] - runs-on: ubuntu-latest + runs-on: foundry-runner steps: - name: Checkout repository uses: actions/checkout@v4 From fcbaddc441b69477cba41403b671fa13d87f2ccb Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:31:37 +0530 Subject: [PATCH 49/74] ci: don't use wget --- .github/workflows/benchmarks.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c9a433c067b6c..96b04be4c791a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -50,8 +50,8 @@ jobs: - name: Install hyperfine run: | - wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb + curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - name: Build benchmark binary run: cargo build --release --bin foundry-bench @@ -79,8 +79,8 @@ jobs: - name: Install hyperfine run: | - wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb + curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - name: Download benchmark binary uses: actions/download-artifact@v4 @@ -126,8 +126,8 @@ jobs: - name: Install hyperfine run: | - wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb + curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - name: Download benchmark binary uses: actions/download-artifact@v4 @@ -173,8 +173,8 @@ jobs: - name: Install hyperfine run: | - wget https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb + curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - name: Download benchmark binary uses: actions/download-artifact@v4 From 34e0c7efee67531f7a1e39ad62f7a776f311f430 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:46:33 +0530 Subject: [PATCH 50/74] ci: add build essential --- .github/workflows/benchmarks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 96b04be4c791a..92d7c4766da5c 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -32,6 +32,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential pkg-config + - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable From 0d3aa3bb36ad27391d6b1b2b8d10234fbfcffe6d Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 7 Jul 2025 17:43:06 +0530 Subject: [PATCH 51/74] ci: nodejs and npm --- .github/workflows/benchmarks.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 92d7c4766da5c..905e73485b305 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -35,7 +35,7 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y build-essential pkg-config + sudo apt-get install -y build-essential pkg-config nodejs npm - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -82,11 +82,6 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - name: Install hyperfine - run: | - curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - - name: Download benchmark binary uses: actions/download-artifact@v4 with: @@ -129,11 +124,6 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - name: Install hyperfine - run: | - curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - - name: Download benchmark binary uses: actions/download-artifact@v4 with: @@ -176,11 +166,6 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - name: Install hyperfine - run: | - curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - - name: Download benchmark binary uses: actions/download-artifact@v4 with: From 822852948cddeac80684226979670d4771478014 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 7 Jul 2025 18:00:47 +0530 Subject: [PATCH 52/74] install hyperfine for each job --- .github/workflows/benchmarks.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 905e73485b305..79bfa11cf0b16 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -53,11 +53,6 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - name: Install hyperfine - run: | - curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y - - name: Build benchmark binary run: cargo build --release --bin foundry-bench @@ -82,6 +77,11 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH + - name: Install hyperfine + run: | + curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y + - name: Download benchmark binary uses: actions/download-artifact@v4 with: @@ -124,6 +124,11 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH + - name: Install hyperfine + run: | + curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y + - name: Download benchmark binary uses: actions/download-artifact@v4 with: @@ -166,6 +171,11 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH + - name: Install hyperfine + run: | + curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb + sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y + - name: Download benchmark binary uses: actions/download-artifact@v4 with: From eef0a5f207a1ecbdbcdf5c44332457f521fde44e Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Mon, 7 Jul 2025 18:08:01 +0530 Subject: [PATCH 53/74] fix --- .github/workflows/benchmarks.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 79bfa11cf0b16..98f5d915fdcc0 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -35,7 +35,7 @@ jobs: - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y build-essential pkg-config nodejs npm + sudo apt-get install -y build-essential pkg-config - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable @@ -77,8 +77,9 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - name: Install hyperfine + - name: Install dependencies run: | + sudo apt-get install -y nodejs npm curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y @@ -124,8 +125,9 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - name: Install hyperfine + - name: Install dependencies run: | + sudo apt-get install -y nodejs npm curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y @@ -171,8 +173,9 @@ jobs: ./.github/scripts/setup-foundryup.sh echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - name: Install hyperfine + - name: Install dependencies run: | + sudo apt-get install -y nodejs npm curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y From 3a7fab78d022c5a8c8885271a09bb3675f999ce8 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 8 Jul 2025 11:15:27 +0530 Subject: [PATCH 54/74] install deps script --- .github/scripts/install-dependencies.sh | 53 +++++++++++++++++++++++++ .github/workflows/benchmarks.yml | 18 ++++----- 2 files changed, 62 insertions(+), 9 deletions(-) create mode 100755 .github/scripts/install-dependencies.sh diff --git a/.github/scripts/install-dependencies.sh b/.github/scripts/install-dependencies.sh new file mode 100755 index 0000000000000..07cbe010871e9 --- /dev/null +++ b/.github/scripts/install-dependencies.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -euo pipefail + +# Script to install all dependencies needed for benchmarking +# This includes Node.js/npm (via nvm) and hyperfine + +echo "Installing benchmark dependencies..." + +# Install Node.js and npm using nvm +echo "=== Installing Node.js and npm ===" + +# Download and install nvm +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash + +# Load nvm without restarting the shell +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + +# Download and install Node.js +echo "Installing Node.js 24..." +nvm install 24 + +# Use Node.js 24 +nvm use 24 + +# Verify the Node.js version +echo "Node.js version: $(node -v)" +echo "Current nvm version: $(nvm current)" + +# Verify npm version +echo "npm version: $(npm -v)" + +# Export PATH for the current session +export PATH="$HOME/.nvm/versions/node/v24.3.0/bin:$PATH" + +echo "Node.js path: $(which node)" +echo "npm path: $(which npm)" + +# Install hyperfine +echo "" +echo "=== Installing hyperfine ===" + +# Download and extract hyperfine binary +curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz +sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/ +rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu + +# Verify hyperfine installation +echo "hyperfine version: $(hyperfine --version)" +echo "hyperfine path: $(which hyperfine)" + +echo "" +echo "All dependencies installed successfully!" \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 98f5d915fdcc0..b505686760acc 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -79,9 +79,9 @@ jobs: - name: Install dependencies run: | - sudo apt-get install -y nodejs npm - curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y + ./.github/scripts/install-dependencies.sh + # Add nvm and node to PATH for subsequent steps + echo "$HOME/.nvm/versions/node/v24.3.0/bin" >> $GITHUB_PATH - name: Download benchmark binary uses: actions/download-artifact@v4 @@ -127,9 +127,9 @@ jobs: - name: Install dependencies run: | - sudo apt-get install -y nodejs npm - curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y + ./.github/scripts/install-dependencies.sh + # Add nvm and node to PATH for subsequent steps + echo "$HOME/.nvm/versions/node/v24.3.0/bin" >> $GITHUB_PATH - name: Download benchmark binary uses: actions/download-artifact@v4 @@ -175,9 +175,9 @@ jobs: - name: Install dependencies run: | - sudo apt-get install -y nodejs npm - curl -L -o hyperfine_1.19.0_amd64.deb https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine_1.19.0_amd64.deb - sudo dpkg -i hyperfine_1.19.0_amd64.deb || sudo apt-get install -f -y + ./.github/scripts/install-dependencies.sh + # Add nvm and node to PATH for subsequent steps + echo "$HOME/.nvm/versions/node/v24.3.0/bin" >> $GITHUB_PATH - name: Download benchmark binary uses: actions/download-artifact@v4 From 2b327cda25e6fd2ab872fd1431b93e5c6dc71d21 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:02:31 +0530 Subject: [PATCH 55/74] add benchmark-setup, using setup-node action, remove redundant files --- .github/actions/benchmark-setup.yml | 37 ++++++++ .github/scripts/commit-and-read-benchmarks.sh | 9 +- .github/scripts/install-dependencies.sh | 53 ----------- .github/workflows/benchmarks.yml | 88 ++++--------------- LATEST.md | 28 ------ benches/COVERAGE_BENCH.md | 28 ------ benches/FUZZ_BENCH.md | 30 ------- 7 files changed, 54 insertions(+), 219 deletions(-) create mode 100644 .github/actions/benchmark-setup.yml delete mode 100755 .github/scripts/install-dependencies.sh delete mode 100644 LATEST.md delete mode 100644 benches/COVERAGE_BENCH.md delete mode 100644 benches/FUZZ_BENCH.md diff --git a/.github/actions/benchmark-setup.yml b/.github/actions/benchmark-setup.yml new file mode 100644 index 0000000000000..5926035e2da1c --- /dev/null +++ b/.github/actions/benchmark-setup.yml @@ -0,0 +1,37 @@ +name: "Benchmark Setup" +description: "Common setup steps for benchmark jobs" + +runs: + using: "composite" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Foundry + shell: bash + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry + run: | + ./.github/scripts/setup-foundryup.sh + echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Install hyperfine + shell: bash + run: | + curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz + sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/ + rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu + + - name: Download benchmark binary + uses: actions/download-artifact@v4 + with: + name: foundry-bench + + - name: Make binary executable + shell: bash + run: chmod +x foundry-bench diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh index 2de978b139859..962ff11918721 100755 --- a/.github/scripts/commit-and-read-benchmarks.sh +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -14,13 +14,8 @@ commit_results() { git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - echo "Adding benchmark files..." - git add "$OUTPUT_DIR/LATEST.md" \ - "$OUTPUT_DIR/forge_test_bench.md" \ - "$OUTPUT_DIR/forge_build_bench.md" \ - "$OUTPUT_DIR/forge_coverage_bench.md" || { - echo "Warning: Some benchmark files may not exist" - } + echo "Adding benchmark file..." + git add "$OUTPUT_DIR/LATEST.md" if git diff --staged --quiet; then echo "No changes to commit" diff --git a/.github/scripts/install-dependencies.sh b/.github/scripts/install-dependencies.sh deleted file mode 100755 index 07cbe010871e9..0000000000000 --- a/.github/scripts/install-dependencies.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Script to install all dependencies needed for benchmarking -# This includes Node.js/npm (via nvm) and hyperfine - -echo "Installing benchmark dependencies..." - -# Install Node.js and npm using nvm -echo "=== Installing Node.js and npm ===" - -# Download and install nvm -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash - -# Load nvm without restarting the shell -export NVM_DIR="$HOME/.nvm" -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - -# Download and install Node.js -echo "Installing Node.js 24..." -nvm install 24 - -# Use Node.js 24 -nvm use 24 - -# Verify the Node.js version -echo "Node.js version: $(node -v)" -echo "Current nvm version: $(nvm current)" - -# Verify npm version -echo "npm version: $(npm -v)" - -# Export PATH for the current session -export PATH="$HOME/.nvm/versions/node/v24.3.0/bin:$PATH" - -echo "Node.js path: $(which node)" -echo "npm path: $(which npm)" - -# Install hyperfine -echo "" -echo "=== Installing hyperfine ===" - -# Download and extract hyperfine binary -curl -L https://github.com/sharkdp/hyperfine/releases/download/v1.19.0/hyperfine-v1.19.0-x86_64-unknown-linux-gnu.tar.gz | tar xz -sudo mv hyperfine-v1.19.0-x86_64-unknown-linux-gnu/hyperfine /usr/local/bin/ -rm -rf hyperfine-v1.19.0-x86_64-unknown-linux-gnu - -# Verify hyperfine installation -echo "hyperfine version: $(hyperfine --version)" -echo "hyperfine path: $(which hyperfine)" - -echo "" -echo "All dependencies installed successfully!" \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b505686760acc..08594c8469e4b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -18,12 +18,17 @@ on: description: "Comma-separated list of repos to benchmark (e.g., ithacaxyz/account:main,Vectorized/solady)" required: false type: string - default: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22" + default: "${{ env.DEFAULT_REPOS }}" permissions: contents: write pull-requests: write +env: + ITHACAXYZ_ACCOUNT: "ithacaxyz/account:v0.3.2" + VECTORIZED_SOLADY: "Vectorized/solady:v0.1.22" + DEFAULT_REPOS: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22" + jobs: setup: name: Setup and Build @@ -67,36 +72,15 @@ jobs: needs: setup runs-on: foundry-runner steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Foundry - env: - FOUNDRY_DIR: ${{ github.workspace }}/.foundry - run: | - ./.github/scripts/setup-foundryup.sh - echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - - name: Install dependencies - run: | - ./.github/scripts/install-dependencies.sh - # Add nvm and node to PATH for subsequent steps - echo "$HOME/.nvm/versions/node/v24.3.0/bin" >> $GITHUB_PATH - - - name: Download benchmark binary - uses: actions/download-artifact@v4 - with: - name: foundry-bench - - - name: Make binary executable - run: chmod +x foundry-bench + - name: Benchmark setup + uses: ./.github/actions/benchmark-setup.yml - name: Run forge test benchmarks env: FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" - REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" + REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}" ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ @@ -115,36 +99,15 @@ jobs: needs: setup runs-on: foundry-runner steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Foundry - env: - FOUNDRY_DIR: ${{ github.workspace }}/.foundry - run: | - ./.github/scripts/setup-foundryup.sh - echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - - name: Install dependencies - run: | - ./.github/scripts/install-dependencies.sh - # Add nvm and node to PATH for subsequent steps - echo "$HOME/.nvm/versions/node/v24.3.0/bin" >> $GITHUB_PATH - - - name: Download benchmark binary - uses: actions/download-artifact@v4 - with: - name: foundry-bench - - - name: Make binary executable - run: chmod +x foundry-bench + - name: Benchmark setup + uses: ./.github/actions/benchmark-setup.yml - name: Run forge build benchmarks env: FOUNDRY_DIR: ${{ github.workspace }}/.foundry run: | VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" - REPOS="${{ github.event.inputs.repos || 'ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22' }}" + REPOS="${{ github.event.inputs.repos || env.DEFAULT_REPOS }}" ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ @@ -163,29 +126,8 @@ jobs: needs: setup runs-on: foundry-runner steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Foundry - env: - FOUNDRY_DIR: ${{ github.workspace }}/.foundry - run: | - ./.github/scripts/setup-foundryup.sh - echo "${{ github.workspace }}/.foundry/bin" >> $GITHUB_PATH - - - name: Install dependencies - run: | - ./.github/scripts/install-dependencies.sh - # Add nvm and node to PATH for subsequent steps - echo "$HOME/.nvm/versions/node/v24.3.0/bin" >> $GITHUB_PATH - - - name: Download benchmark binary - uses: actions/download-artifact@v4 - with: - name: foundry-bench - - - name: Make binary executable - run: chmod +x foundry-bench + - name: Benchmark setup + uses: ./.github/actions/benchmark-setup.yml - name: Run forge coverage benchmarks env: @@ -196,7 +138,7 @@ jobs: ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ - --repos ithacaxyz/account:v0.3.2 \ + --repos ${{ env.ITHACAXYZ_ACCOUNT }} \ --benchmarks forge_coverage \ --output-file forge_coverage_bench.md diff --git a/LATEST.md b/LATEST.md deleted file mode 100644 index e542d7def2948..0000000000000 --- a/LATEST.md +++ /dev/null @@ -1,28 +0,0 @@ -# Foundry Benchmark Results - -**Date**: 2025-07-03 19:43:36 - -## Summary - -Benchmarked 2 Foundry versions across 1 repositories. - -### Repositories Tested - -1. [ithacaxyz/account](https://github.com/ithacaxyz/account) - -### Foundry Versions - -- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) - -## Forge Test - -| Repository | stable | nightly | -|------------|----------|----------| -| ithacaxyz-account | 3.70 s | 3.17 s | - -## System Information - -- **OS**: macos -- **CPU**: 8 -- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) diff --git a/benches/COVERAGE_BENCH.md b/benches/COVERAGE_BENCH.md deleted file mode 100644 index 815e25ca3da65..0000000000000 --- a/benches/COVERAGE_BENCH.md +++ /dev/null @@ -1,28 +0,0 @@ -# Foundry Benchmark Results - -**Date**: 2025-07-04 11:37:27 - -## Summary - -Benchmarked 2 Foundry versions across 1 repositories. - -### Repositories Tested - -1. [ithacaxyz/account](https://github.com/ithacaxyz/account) - -### Foundry Versions - -- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) - -## Forge Coverage - -| Repository | stable | nightly | -| ----------------- | ------- | ------- | -| ithacaxyz-account | 26.44 s | 32.34 s | - -## System Information - -- **OS**: macos -- **CPU**: 8 -- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) diff --git a/benches/FUZZ_BENCH.md b/benches/FUZZ_BENCH.md deleted file mode 100644 index 9fb08b1c4caee..0000000000000 --- a/benches/FUZZ_BENCH.md +++ /dev/null @@ -1,30 +0,0 @@ -# Foundry Benchmark Results - -**Date**: 2025-07-04 11:05:43 - -## Summary - -Benchmarked 2 Foundry versions across 2 repositories. - -### Repositories Tested - -1. [ithacaxyz/account](https://github.com/ithacaxyz/account) -2. [Vectorized/solady](https://github.com/Vectorized/solady) - -### Foundry Versions - -- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) - -## Forge Fuzz Test - -| Repository | stable | nightly | -| ----------------- | ------ | ------- | -| ithacaxyz-account | 3.62 s | 3.31 s | -| solady | 2.82 s | 2.65 s | - -## System Information - -- **OS**: macos -- **CPU**: 8 -- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) From 541cd481eaea904ebab4523c29f9e0efa674d1a8 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:06:42 +0530 Subject: [PATCH 56/74] fix --- .github/workflows/benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 08594c8469e4b..01931c18886e0 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -18,7 +18,7 @@ on: description: "Comma-separated list of repos to benchmark (e.g., ithacaxyz/account:main,Vectorized/solady)" required: false type: string - default: "${{ env.DEFAULT_REPOS }}" + default: "ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22" permissions: contents: write From 834be0115a0d22f4d9b2fc65d4a2aee20567a67e Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:29:17 +0530 Subject: [PATCH 57/74] fix --- .../{benchmark-setup.yml => benchmark-setup/action.yml} | 0 .github/workflows/benchmarks.yml | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename .github/actions/{benchmark-setup.yml => benchmark-setup/action.yml} (100%) diff --git a/.github/actions/benchmark-setup.yml b/.github/actions/benchmark-setup/action.yml similarity index 100% rename from .github/actions/benchmark-setup.yml rename to .github/actions/benchmark-setup/action.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 01931c18886e0..9e78b22d62aca 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -73,7 +73,7 @@ jobs: runs-on: foundry-runner steps: - name: Benchmark setup - uses: ./.github/actions/benchmark-setup.yml + uses: ./.github/actions/benchmark-setup - name: Run forge test benchmarks env: @@ -100,7 +100,7 @@ jobs: runs-on: foundry-runner steps: - name: Benchmark setup - uses: ./.github/actions/benchmark-setup.yml + uses: ./.github/actions/benchmark-setup - name: Run forge build benchmarks env: @@ -127,7 +127,7 @@ jobs: runs-on: foundry-runner steps: - name: Benchmark setup - uses: ./.github/actions/benchmark-setup.yml + uses: ./.github/actions/benchmark-setup - name: Run forge coverage benchmarks env: From f31472188216e68e4029dbd829d384348a87b134 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:42:01 +0530 Subject: [PATCH 58/74] checkout repo --- .github/actions/benchmark-setup/action.yml | 3 --- .github/workflows/benchmarks.yml | 9 +++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/actions/benchmark-setup/action.yml b/.github/actions/benchmark-setup/action.yml index 5926035e2da1c..8278e12adaacd 100644 --- a/.github/actions/benchmark-setup/action.yml +++ b/.github/actions/benchmark-setup/action.yml @@ -4,9 +4,6 @@ description: "Common setup steps for benchmark jobs" runs: using: "composite" steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Setup Foundry shell: bash env: diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9e78b22d62aca..9320c7f1a65ff 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -72,6 +72,9 @@ jobs: needs: setup runs-on: foundry-runner steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Benchmark setup uses: ./.github/actions/benchmark-setup @@ -99,6 +102,9 @@ jobs: needs: setup runs-on: foundry-runner steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Benchmark setup uses: ./.github/actions/benchmark-setup @@ -126,6 +132,9 @@ jobs: needs: setup runs-on: foundry-runner steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Benchmark setup uses: ./.github/actions/benchmark-setup From 51efad330408a75536d86917924ce4bb6ab5254f Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 9 Jul 2025 11:50:18 +0530 Subject: [PATCH 59/74] nits --- .gitignore | 1 + benches/src/results.rs | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index eae7b2c0ade6f..64af508ada43c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ CLAUDE.md node_modules dist bin +_ \ No newline at end of file diff --git a/benches/src/results.rs b/benches/src/results.rs index 14b180083d63b..3bc42685a42d5 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -214,11 +214,11 @@ fn get_benchmark_cell_content( repo_name: &str, ) -> String { // Check if we have data for this version - if let Some(repo_data) = version_data.get(version) { - // Check if we have data for this repository - if let Some(result) = repo_data.get(repo_name) { - return format_duration_seconds(result.mean); - } + if let Some(repo_data) = version_data.get(version) && + // Check if we have data for this repository + let Some(result) = repo_data.get(repo_name) + { + return format_duration_seconds(result.mean); } "N/A".to_string() From bdca555d3e0d659a8bcf248f68d71fee1f4207ed Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 9 Jul 2025 11:55:55 +0530 Subject: [PATCH 60/74] nit --- benches/Cargo.toml | 2 +- benches/src/main.rs | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index e9dfce24b6880..3e4c4a955b1a2 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "foundry-bench" version = "0.1.0" -edition = "2021" +edition = "2024" license = "Apache-2.0 OR MIT" [[bin]] diff --git a/benches/src/main.rs b/benches/src/main.rs index 218fcf79b52b9..276160d86eb1f 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -1,9 +1,10 @@ use clap::Parser; use eyre::{Result, WrapErr}; use foundry_bench::{ - get_forge_version, get_forge_version_details, + BENCHMARK_REPOS, FOUNDRY_VERSIONS, RUNS, RepoConfig, get_forge_version, + get_forge_version_details, results::{BenchmarkResults, HyperfineResult}, - switch_foundry_version, RepoConfig, BENCHMARK_REPOS, FOUNDRY_VERSIONS, RUNS, + switch_foundry_version, }; use foundry_common::sh_println; use once_cell::sync::Lazy; @@ -68,7 +69,9 @@ fn main() -> Result<()> { // Check if hyperfine is installed let hyperfine_check = Command::new("hyperfine").arg("--version").output(); if hyperfine_check.is_err() || !hyperfine_check.unwrap().status.success() { - eyre::bail!("hyperfine is not installed. Please install it first: https://github.com/sharkdp/hyperfine"); + eyre::bail!( + "hyperfine is not installed. Please install it first: https://github.com/sharkdp/hyperfine" + ); } // Determine versions to test From 236da6bf9eaef42e96cfb642ffb1927a2930cf87 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 10:19:36 +0530 Subject: [PATCH 61/74] fix --- benches/README.md | 114 +++++++++++++++----------------------------- benches/src/lib.rs | 9 ++-- benches/src/main.rs | 8 +--- 3 files changed, 46 insertions(+), 85 deletions(-) diff --git a/benches/README.md b/benches/README.md index 41a9344657554..fee254a18067b 100644 --- a/benches/README.md +++ b/benches/README.md @@ -6,7 +6,7 @@ This directory contains performance benchmarks for Foundry commands across multi Before running the benchmarks, ensure you have the following installed: -1. **Rust and Cargo** - Required for building and running the benchmarks +1. **Rust and Cargo** - Required for building the benchmark binary ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh @@ -21,87 +21,78 @@ Before running the benchmarks, ensure you have the following installed: 3. **Git** - For cloning benchmark repositories -4. **npm** - Some repositories require npm dependencies +4. [**Hyperfine**](https://github.com/sharkdp/hyperfine/blob/master/README.md) - The benchmarking tool used by foundry-bench - ```bash - # Install Node.js and npm from https://nodejs.org/ - ``` +5. **Node.js and npm** - Some repositories require npm dependencies ## Running Benchmarks -### Using the Benchmark Binary - -Build the benchmark runner: +Build and install the benchmark runner: ```bash cargo build --release --bin foundry-bench ``` +To install `foundry-bench` to your PATH: + +```bash +cd benches && cargo install --path . --bin foundry-bench +``` + #### Run with default settings ```bash # Run all benchmarks on default repos with stable and nightly versions -cargo run --release --bin foundry-bench -- --versions stable,nightly +foundry-bench --versions stable,nightly ``` #### Run with custom configurations ```bash # Bench specific versions -cargo run --release --bin foundry-bench -- --versions stable,nightly,v1.0.0 +foundry-bench --versions stable,nightly,v1.0.0 # Run on specific repositories. Default rev for the repo is "main" -cargo run --release --bin foundry-bench -- --repos ithacaxyz/account,Vectorized/solady +foundry-bench --repos ithacaxyz/account,Vectorized/solady # Test specific repository with custom revision -cargo run --release --bin foundry-bench -- --repos ithacaxyz/account:main,Vectorized/solady:v0.0.123 +foundry-bench --repos ithacaxyz/account:main,Vectorized/solady:v0.0.123 # Run only specific benchmarks -cargo run --release --bin foundry-bench -- --benchmarks forge_build_with_cache,forge_test +foundry-bench --benchmarks forge_build_with_cache,forge_test # Run only fuzz tests -cargo run --release --bin foundry-bench -- --benchmarks forge_fuzz_test +foundry-bench --benchmarks forge_fuzz_test # Run coverage benchmark -cargo run --release --bin foundry-bench -- --benchmarks forge_coverage +foundry-bench --benchmarks forge_coverage # Combine options -cargo run --release --bin foundry-bench -- \ +foundry-bench \ --versions stable,nightly \ --repos ithacaxyz/account \ --benchmarks forge_build_with_cache # Force install Foundry versions -cargo run --release --bin foundry-bench -- --force-install +foundry-bench --force-install -# Verbose output to see criterion logs -cargo run --release --bin foundry-bench -- --verbose +# Verbose output to see hyperfine logs +foundry-bench --verbose # Output to specific directory -cargo run --release --bin foundry-bench -- --output-dir ./results --output-file LATEST_RESULTS.md +foundry-bench --output-dir ./results --output-file LATEST_RESULTS.md ``` #### Command-line Options - `--versions ` - Comma-separated list of Foundry versions (default: stable,nightly) -- `--repos ` - Comma-separated list of repos in org/repo[:rev] format +- `--repos ` - Comma-separated list of repos in org/repo[:rev] format (default: ithacaxyz/account:v0.3.2,Vectorized/solady:v0.1.22) - `--benchmarks ` - Comma-separated list of benchmarks to run - `--force-install` - Force installation of Foundry versions - `--verbose` - Show detailed benchmark output -- `--output-dir ` - Directory for output files (default: current directory) +- `--output-dir ` - Directory for output files (default: benches) - `--output-file ` - Name of the output file (default: LATEST.md) -### Run individual Criterion benchmarks - -```bash -# Run specific benchmark with Criterion -cargo bench --bench forge_test -cargo bench --bench forge_build_no_cache -cargo bench --bench forge_build_with_cache -cargo bench --bench forge_fuzz_test -cargo bench --bench forge_coverage -``` - ## Benchmark Structure - `forge_test` - Benchmarks `forge test` command across repos @@ -112,57 +103,30 @@ cargo bench --bench forge_coverage ## Configuration -### Repositories +The benchmark binary uses command-line arguments to configure which repositories and versions to test. The default repositories are: -Edit `src/lib.rs` to modify the list of repositories to benchmark: +- `ithacaxyz/account:v0.3.2` +- `Vectorized/solady:v0.1.22` -```rust -pub static BENCHMARK_REPOS: &[RepoConfig] = &[ - RepoConfig { name: "account", org: "ithacaxyz", repo: "account", rev: "main" }, - // Add more repositories here -]; -``` - -### Foundry Versions - -Edit `src/lib.rs` to modify the list of Foundry versions: - -```rust -pub static FOUNDRY_VERSIONS: &[&str] = &["stable", "nightly"]; -``` +You can override these using the `--repos` flag with the format `org/repo[:rev]`. ## Results -Benchmark results are saved to `LATEST.md` (or custom output directory). The report includes: +Benchmark results are saved to `benches/LATEST.md` (or custom output file specified with `--output-file`). The report includes: - Summary of versions and repositories tested -- Performance comparison tables for each benchmark type -- Execution time statistics for each repository/version combination -- System information (OS, CPU, Rust version) - -Results are also stored in Criterion's format in `target/criterion/` for detailed analysis. - -## GitHub Actions Integration - -The benchmarks can be run automatically via GitHub Actions: - -1. Go to the [Actions tab](../../actions/workflows/benchmarks.yml) -2. Click on "Foundry Benchmarks" workflow -3. Click "Run workflow" -4. Configure options: - - PR number (optional) - Add benchmark results as a comment to a PR - - Versions - Foundry versions to test (default: stable,nightly) - - Repos - Custom repositories to benchmark - - Benchmarks - Specific benchmarks to run - -The workflow will: - -- Build and run the benchmark binary -- Commit results to `benches/LATEST.md` -- Optionally comment on a PR with the results +- Performance comparison tables for each benchmark type showing: + - Mean execution time + - Min/Max times + - Standard deviation + - Relative performance comparison between versions +- System information (OS, CPU cores) +- Detailed hyperfine benchmark results in JSON format ## Troubleshooting -1. **Foundry version not found**: Ensure the version is installed with `foundryup --install ` +1. **Foundry version not found**: Use `--force-install` flag or manually install with `foundryup --install ` 2. **Repository clone fails**: Check network connectivity and repository access 3. **Build failures**: Some repositories may have specific dependencies - check their README files +4. **Hyperfine not found**: Install hyperfine using the instructions in Prerequisites +5. **npm/Node.js errors**: Ensure Node.js and npm are installed for repositories that require them diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 35a6415d55ccf..aa9c6ad318c54 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -9,6 +9,7 @@ use std::{ env, path::{Path, PathBuf}, process::Command, + str::FromStr, }; pub mod results; @@ -25,10 +26,10 @@ pub struct RepoConfig { pub rev: String, } -impl TryFrom<&str> for RepoConfig { - type Error = eyre::Error; +impl FromStr for RepoConfig { + type Err = eyre::Error; - fn try_from(spec: &str) -> Result { + fn from_str(spec: &str) -> Result { // Split by ':' first to separate repo path from optional rev let parts: Vec<&str> = spec.splitn(2, ':').collect(); let repo_path = parts[0]; @@ -478,7 +479,7 @@ pub fn setup_benchmark_repos() -> Vec<(RepoConfig, BenchmarkProject)> { .split(',') .map(|s| s.trim()) .filter(|s| !s.is_empty()) - .map(RepoConfig::try_from) + .map(|s| s.parse::()) .collect::>>() .expect("Failed to parse FOUNDRY_BENCH_REPOS") } else { diff --git a/benches/src/main.rs b/benches/src/main.rs index 276160d86eb1f..dac843058feb2 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -7,7 +7,6 @@ use foundry_bench::{ switch_foundry_version, }; use foundry_common::sh_println; -use once_cell::sync::Lazy; use rayon::prelude::*; use std::{fs, path::PathBuf, process::Command, sync::Mutex}; @@ -55,7 +54,7 @@ struct Cli { } /// Mutex to prevent concurrent foundryup calls -static FOUNDRY_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); +static FOUNDRY_LOCK: Mutex<()> = Mutex::new(()); fn switch_version_safe(version: &str) -> Result<()> { let _lock = FOUNDRY_LOCK.lock().unwrap(); switch_foundry_version(version) @@ -83,10 +82,7 @@ fn main() -> Result<()> { // Get repo configurations let repos = if let Some(repo_specs) = cli.repos.clone() { - repo_specs - .iter() - .map(|spec| RepoConfig::try_from(spec.as_str())) - .collect::>>()? + repo_specs.iter().map(|spec| spec.parse::()).collect::>>()? } else { BENCHMARK_REPOS.clone() }; From 88e45c7f0976fc54732cda28c68356813b7a5f33 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 10:29:48 +0530 Subject: [PATCH 62/74] show forge test result in top comment --- .github/scripts/commit-and-read-benchmarks.sh | 12 ++++++++++++ .github/workflows/benchmarks.yml | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh index 962ff11918721..15861970df0c7 100755 --- a/.github/scripts/commit-and-read-benchmarks.sh +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -37,6 +37,17 @@ Co-Authored-By: github-actions " read_results() { if [ -f "$OUTPUT_DIR/LATEST.md" ]; then echo "Reading benchmark results..." + + # Extract Forge Test results summary + echo "Extracting Forge Test summary..." + { + echo 'forge_test_summary<> "$GITHUB_OUTPUT" + + # Output full results { echo 'results<> "$GITHUB_OUTPUT" + echo 'forge_test_summary=No Forge Test results found.' >> "$GITHUB_OUTPUT" echo "Warning: No benchmark results found at $OUTPUT_DIR/LATEST.md" fi } diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9320c7f1a65ff..5ed1ff2ebe7ab 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -185,12 +185,15 @@ jobs: with: script: | const prNumber = ${{ github.event.inputs.pr_number || github.event.pull_request.number }}; + const forgeTestSummary = `${{ steps.benchmark_results.outputs.forge_test_summary }}`; const benchmarkResults = `${{ steps.benchmark_results.outputs.results }}`; const comment = `## 📊 Foundry Benchmark Results + ${forgeTestSummary} +
- Click to view detailed benchmark results + 📈 View all benchmark results ${benchmarkResults} From 24ce0a285d660e14d16a7a87987176e2310fe8db Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 10:33:52 +0530 Subject: [PATCH 63/74] force foundry install --- benches/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/src/main.rs b/benches/src/main.rs index dac843058feb2..f9fb27d5ac04a 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -217,7 +217,7 @@ fn install_foundry_versions(versions: &[String]) -> Result<()> { sh_println!("Installing {version}..."); let status = Command::new("foundryup") - .args(["--install", version]) + .args(["--install", version, "--force"]) .status() .wrap_err("Failed to run foundryup")?; From 1f4cfdff12f052a3d6a023df2c49cd9a9205abab Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 10:56:17 +0530 Subject: [PATCH 64/74] fix bench comment aggregation --- .github/scripts/commit-and-read-benchmarks.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh index 15861970df0c7..7e95a0539aaf9 100755 --- a/.github/scripts/commit-and-read-benchmarks.sh +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -43,7 +43,8 @@ read_results() { { echo 'forge_test_summary<> "$GITHUB_OUTPUT" From 178f3c5e81dc4013191184f4a8eec690988a6029 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 10:57:18 +0530 Subject: [PATCH 65/74] nit --- crates/anvil/src/eth/backend/mem/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/anvil/src/eth/backend/mem/mod.rs b/crates/anvil/src/eth/backend/mem/mod.rs index 16b533332c9db..5b775838d625a 100644 --- a/crates/anvil/src/eth/backend/mem/mod.rs +++ b/crates/anvil/src/eth/backend/mem/mod.rs @@ -1885,7 +1885,7 @@ impl Backend { GethDebugBuiltInTracerType::CallTracer => { let call_config = tracer_config .into_call_config() - .map_err(|e| (RpcError::invalid_params(e.to_string())))?; + .map_err(|e| RpcError::invalid_params(e.to_string()))?; let mut inspector = self.build_inspector().with_tracing_config( TracingInspectorConfig::from_geth_call_config(&call_config), From c47af230dadfe135e50dde90b4b42f3234938282 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:52:15 +0530 Subject: [PATCH 66/74] fix --- .github/scripts/combine-benchmarks.sh | 178 ++++++++++++++++-- .github/scripts/commit-and-read-benchmarks.sh | 23 +-- .github/scripts/format-pr-comment.sh | 34 ++++ .github/workflows/benchmarks.yml | 14 +- 4 files changed, 212 insertions(+), 37 deletions(-) create mode 100755 .github/scripts/format-pr-comment.sh diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh index 821d4239ebfd9..90ec7bea49530 100755 --- a/.github/scripts/combine-benchmarks.sh +++ b/.github/scripts/combine-benchmarks.sh @@ -9,31 +9,181 @@ OUTPUT_DIR="${1:-benches}" # Create output directory if it doesn't exist mkdir -p "$OUTPUT_DIR" +# Define the benchmark files and their section names +declare -A BENCHMARK_FILES=( + ["forge_test_bench.md"]="Forge Test" + ["forge_build_bench.md"]="Forge Build" + ["forge_coverage_bench.md"]="Forge Coverage" +) + +# Function to extract a specific section from a benchmark file +extract_section() { + local file=$1 + local section=$2 + local in_section=0 + + while IFS= read -r line; do + if [[ "$line" =~ ^##[[:space:]]+"$section" ]]; then + in_section=1 + echo "$line" + elif [[ $in_section -eq 1 && "$line" =~ ^##[[:space:]] && ! "$line" =~ ^##[[:space:]]+"$section" ]]; then + break + elif [[ $in_section -eq 1 ]]; then + echo "$line" + fi + done < "$file" +} + +# Function to extract summary info (repos and versions) from a file +extract_summary_info() { + local file=$1 + local in_summary=0 + local in_repos=0 + local in_versions=0 + + while IFS= read -r line; do + # Check for Summary section + if [[ "$line" =~ ^##[[:space:]]+Summary ]]; then + in_summary=1 + continue + fi + + # Check for Repositories Tested subsection + if [[ $in_summary -eq 1 && "$line" =~ ^###[[:space:]]+Repositories[[:space:]]+Tested ]]; then + in_repos=1 + echo "### Repositories Tested" + echo + continue + fi + + # Check for Foundry Versions subsection + if [[ $in_summary -eq 1 && "$line" =~ ^###[[:space:]]+Foundry[[:space:]]+Versions ]]; then + in_repos=0 + in_versions=1 + echo "### Foundry Versions" + echo + continue + fi + + # End of summary section + if [[ $in_summary -eq 1 && "$line" =~ ^##[[:space:]] && ! "$line" =~ ^##[[:space:]]+Summary ]]; then + break + fi + + # Output repo or version lines + if [[ ($in_repos -eq 1 || $in_versions -eq 1) && -n "$line" ]]; then + echo "$line" + fi + done < "$file" +} + +# Function to extract benchmark table from a section +extract_benchmark_table() { + local file=$1 + local section=$2 + local in_section=0 + local found_table=0 + + while IFS= read -r line; do + if [[ "$line" =~ ^##[[:space:]]+"$section" ]]; then + in_section=1 + continue + elif [[ $in_section -eq 1 && "$line" =~ ^##[[:space:]] && ! "$line" =~ ^##[[:space:]]+"$section" ]]; then + break + elif [[ $in_section -eq 1 ]]; then + # Skip empty lines before table + if [[ -z "$line" && $found_table -eq 0 ]]; then + continue + fi + # Detect table start + if [[ "$line" =~ ^\|[[:space:]]*Repository ]]; then + found_table=1 + fi + # Output table lines + if [[ $found_table -eq 1 && -n "$line" ]]; then + echo "$line" + fi + fi + done < "$file" +} + +# Function to extract system information +extract_system_info() { + local file=$1 + awk '/^## System Information/,/^$/ { if (!/^## System Information/ && !/^$/) print }' "$file" +} + # Start building LATEST.md cat > "$OUTPUT_DIR/LATEST.md" << EOF -# Foundry Benchmark Results +# 📊 Foundry Benchmark Results -Generated at: $(date -u '+%Y-%m-%d %H:%M:%S UTC') +**Generated at**: $(date -u '+%Y-%m-%d %H:%M:%S UTC') EOF -# Define the benchmark files to combine in order -BENCHMARK_FILES=( - "forge_test_bench.md" - "forge_build_bench.md" - "forge_coverage_bench.md" -) +# Process each benchmark file +FIRST_FILE=1 +SYSTEM_INFO="" -# Add each benchmark result if it exists -for bench_file in "${BENCHMARK_FILES[@]}"; do +for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do if [ -f "$OUTPUT_DIR/$bench_file" ]; then - echo "Adding $bench_file to combined results..." - # Skip the header from individual files (first line) and append content - tail -n +2 "$OUTPUT_DIR/$bench_file" >> "$OUTPUT_DIR/LATEST.md" - echo "" >> "$OUTPUT_DIR/LATEST.md" + echo "Processing $bench_file..." + + # Get the section name + case "$bench_file" in + "forge_test_bench.md") + SECTION_NAME="Forge Test" + ;; + "forge_build_bench.md") + SECTION_NAME="Forge Build" + ;; + "forge_coverage_bench.md") + SECTION_NAME="Forge Coverage" + ;; + esac + + # Add section header + echo "## $SECTION_NAME" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + + # Add summary info (repos and versions) + extract_summary_info "$OUTPUT_DIR/$bench_file" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + + # For build benchmarks, add both sub-sections + if [[ "$bench_file" == "forge_build_bench.md" ]]; then + # Extract No Cache table + echo "### No Cache" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Build (No Cache)" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + + # Extract With Cache table + echo "### With Cache" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Build (With Cache)" >> "$OUTPUT_DIR/LATEST.md" + else + # Extract the benchmark table for other types + extract_benchmark_table "$OUTPUT_DIR/$bench_file" "$SECTION_NAME" >> "$OUTPUT_DIR/LATEST.md" + fi + + echo >> "$OUTPUT_DIR/LATEST.md" + + # Extract system info from first file only + if [[ $FIRST_FILE -eq 1 ]]; then + SYSTEM_INFO=$(extract_system_info "$OUTPUT_DIR/$bench_file") + FIRST_FILE=0 + fi else echo "Warning: $bench_file not found, skipping..." fi done +# Add system information at the end +if [[ -n "$SYSTEM_INFO" ]]; then + echo "## System Information" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + echo "$SYSTEM_INFO" >> "$OUTPUT_DIR/LATEST.md" +fi + echo "Successfully combined benchmark results into $OUTPUT_DIR/LATEST.md" \ No newline at end of file diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh index 7e95a0539aaf9..20eedbbfe0617 100755 --- a/.github/scripts/commit-and-read-benchmarks.sh +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -38,26 +38,27 @@ read_results() { if [ -f "$OUTPUT_DIR/LATEST.md" ]; then echo "Reading benchmark results..." - # Extract Forge Test results summary - echo "Extracting Forge Test summary..." + # Output full results { - echo 'forge_test_summary<> "$GITHUB_OUTPUT" - # Output full results + # Format results for PR comment + echo "Formatting results for PR comment..." + FORMATTED_COMMENT=$("$(dirname "$0")/format-pr-comment.sh" "$OUTPUT_DIR/LATEST.md") + { - echo 'results<> "$GITHUB_OUTPUT" - echo "Successfully read benchmark results" + + echo "Successfully read and formatted benchmark results" else echo 'results=No benchmark results found.' >> "$GITHUB_OUTPUT" - echo 'forge_test_summary=No Forge Test results found.' >> "$GITHUB_OUTPUT" + echo 'pr_comment=No benchmark results found.' >> "$GITHUB_OUTPUT" echo "Warning: No benchmark results found at $OUTPUT_DIR/LATEST.md" fi } diff --git a/.github/scripts/format-pr-comment.sh b/.github/scripts/format-pr-comment.sh new file mode 100755 index 0000000000000..74ad095464aba --- /dev/null +++ b/.github/scripts/format-pr-comment.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +# Script to format benchmark results for PR comment +# Usage: ./format-pr-comment.sh + +RESULTS_FILE="${1:-}" + +if [ -z "$RESULTS_FILE" ] || [ ! -f "$RESULTS_FILE" ]; then + echo "Error: Benchmark results file not provided or does not exist" + exit 1 +fi + +# Read the file content +CONTENT=$(cat "$RESULTS_FILE") + +# Find where "## Forge Build" starts and split the content +# Extract everything before "## Forge Build" +BEFORE_FORGE_BUILD=$(echo "$CONTENT" | awk '/^## Forge Build$/ {exit} {print}') + +# Extract everything from "## Forge Build" onwards +FROM_FORGE_BUILD=$(echo "$CONTENT" | awk '/^## Forge Build$/ {found=1} found {print}') + +# Output the formatted comment with dropdown +cat << EOF +${BEFORE_FORGE_BUILD} + +
+📈 View all benchmark results + +${FROM_FORGE_BUILD} + +
+EOF \ No newline at end of file diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 5ed1ff2ebe7ab..5db0be106fddc 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -185,19 +185,9 @@ jobs: with: script: | const prNumber = ${{ github.event.inputs.pr_number || github.event.pull_request.number }}; - const forgeTestSummary = `${{ steps.benchmark_results.outputs.forge_test_summary }}`; - const benchmarkResults = `${{ steps.benchmark_results.outputs.results }}`; + const prComment = `${{ steps.benchmark_results.outputs.pr_comment }}`; - const comment = `## 📊 Foundry Benchmark Results - - ${forgeTestSummary} - -
- 📈 View all benchmark results - - ${benchmarkResults} - -
+ const comment = `${prComment} --- From a2252fca4e70d52e21d68c4e3ae1b03c7ebfd266 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 14:12:17 +0530 Subject: [PATCH 67/74] feat: create PR for manual runs, else commit in the PR itself. --- .github/scripts/commit-and-read-benchmarks.sh | 19 +++++++---- .github/workflows/benchmarks.yml | 34 +++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh index 20eedbbfe0617..ad0693ae9ca50 100755 --- a/.github/scripts/commit-and-read-benchmarks.sh +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -66,14 +66,21 @@ read_results() { # Main execution echo "Starting benchmark results processing..." -# Only commit if not a pull request -if [ "$GITHUB_EVENT_NAME" != "pull_request" ]; then - echo "Event is not a pull request, proceeding with commit..." - commit_results -else - echo "Event is a pull request, skipping commit" +# Create new branch for manual runs +if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + echo "Manual workflow run detected, creating new branch..." + BRANCH_NAME="benchmarks/results-$(date +%Y%m%d-%H%M%S)" + git checkout -b "$BRANCH_NAME" + echo "Created branch: $BRANCH_NAME" + + # Output branch name for later use + echo "branch_name=$BRANCH_NAME" >> "$GITHUB_OUTPUT" fi +# Always commit benchmark results +echo "Committing benchmark results..." +commit_results + # Always read results for output read_results diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 5db0be106fddc..8c19da8303609 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -179,6 +179,40 @@ jobs: id: benchmark_results run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}" + - name: Push branch for manual runs + if: github.event_name == 'workflow_dispatch' + run: | + git push origin "${{ steps.benchmark_results.outputs.branch_name }}" + echo "Pushed branch: ${{ steps.benchmark_results.outputs.branch_name }}" + + - name: Create PR for manual runs + if: github.event_name == 'workflow_dispatch' + uses: actions/github-script@v7 + with: + script: | + const branchName = '${{ steps.benchmark_results.outputs.branch_name }}'; + const prComment = `${{ steps.benchmark_results.outputs.pr_comment }}`; + + // Create the pull request + const { data: pr } = await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: 'chore(bench): update benchmark results', + head: branchName, + base: 'master', + body: `## Benchmark Results Update + + This PR contains the latest benchmark results from a manual workflow run. + + ${prComment} + + --- + + 🤖 This PR was automatically generated by the [Foundry Benchmarks workflow](https://github.com/${{ github.repository }}/actions).` + }); + + console.log(`Created PR #${pr.number}: ${pr.html_url}`); + - name: Comment on PR if: github.event.inputs.pr_number != '' || github.event_name == 'pull_request' uses: actions/github-script@v7 From 609ba817da9a86b71193b2bd63b32c57a380d24c Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 15:00:43 +0530 Subject: [PATCH 68/74] fix --- .github/scripts/commit-and-read-benchmarks.sh | 23 ++++++++++++++++++- .github/workflows/benchmarks.yml | 2 ++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh index ad0693ae9ca50..b0a28c809a63e 100755 --- a/.github/scripts/commit-and-read-benchmarks.sh +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -8,6 +8,9 @@ OUTPUT_DIR="${1:-benches}" GITHUB_EVENT_NAME="${2:-pull_request}" GITHUB_REPOSITORY="${3:-}" +# Global variable for branch name +BRANCH_NAME="" + # Function to commit benchmark results commit_results() { echo "Configuring git..." @@ -28,7 +31,25 @@ commit_results() { Co-Authored-By: github-actions " echo "Pushing to repository..." - git push + if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then + # For manual runs, we're on a new branch + git push origin "$BRANCH_NAME" + elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + # For PR runs, we need to push to the PR branch + # GitHub Actions provides the branch name in GITHUB_HEAD_REF + if [ -n "${GITHUB_HEAD_REF:-}" ]; then + echo "Pushing to PR branch: $GITHUB_HEAD_REF" + git push origin "HEAD:refs/heads/$GITHUB_HEAD_REF" + else + echo "Error: GITHUB_HEAD_REF not set for pull_request event" + exit 1 + fi + else + # This workflow should only run on workflow_dispatch or pull_request + echo "Error: Unexpected event type: $GITHUB_EVENT_NAME" + echo "This workflow only supports 'workflow_dispatch' and 'pull_request' events" + exit 1 + fi echo "Successfully pushed benchmark results" fi } diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 8c19da8303609..d6f6c24ea945b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -177,6 +177,8 @@ jobs: - name: Commit and read benchmark results id: benchmark_results + env: + GITHUB_HEAD_REF: ${{ github.head_ref }} run: ./.github/scripts/commit-and-read-benchmarks.sh benches "${{ github.event_name }}" "${{ github.repository }}" - name: Push branch for manual runs From 42a553a4817e679472d30caab52a8dd66794df91 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 15:45:34 +0530 Subject: [PATCH 69/74] fetch and pull --- .github/scripts/commit-and-read-benchmarks.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/scripts/commit-and-read-benchmarks.sh b/.github/scripts/commit-and-read-benchmarks.sh index b0a28c809a63e..358b53a73155a 100755 --- a/.github/scripts/commit-and-read-benchmarks.sh +++ b/.github/scripts/commit-and-read-benchmarks.sh @@ -17,6 +17,13 @@ commit_results() { git config --local user.email "action@github.com" git config --local user.name "GitHub Action" + # For PR runs, fetch and checkout the PR branch to ensure we're up to date + if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ -n "${GITHUB_HEAD_REF:-}" ]; then + echo "Fetching latest changes for PR branch: $GITHUB_HEAD_REF" + git fetch origin "$GITHUB_HEAD_REF" + git checkout -B "$GITHUB_HEAD_REF" "origin/$GITHUB_HEAD_REF" + fi + echo "Adding benchmark file..." git add "$OUTPUT_DIR/LATEST.md" @@ -35,11 +42,10 @@ Co-Authored-By: github-actions " # For manual runs, we're on a new branch git push origin "$BRANCH_NAME" elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then - # For PR runs, we need to push to the PR branch - # GitHub Actions provides the branch name in GITHUB_HEAD_REF + # For PR runs, push to the PR branch if [ -n "${GITHUB_HEAD_REF:-}" ]; then echo "Pushing to PR branch: $GITHUB_HEAD_REF" - git push origin "HEAD:refs/heads/$GITHUB_HEAD_REF" + git push origin "$GITHUB_HEAD_REF" else echo "Error: GITHUB_HEAD_REF not set for pull_request event" exit 1 From c4b707f5181ca9646255912a0ca03d9b424d2610 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 11 Jul 2025 10:28:35 +0000 Subject: [PATCH 70/74] chore(`benches`): update benchmark results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Foundry Benchmarks](https://github.com/foundry-rs/foundry/actions) Co-Authored-By: github-actions --- benches/LATEST.md | 65 +++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index d7c65883a9e4f..afe33b1579d92 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,44 +1,59 @@ -# Foundry Benchmark Results +# 📊 Foundry Benchmark Results -**Date**: 2025-07-04 11:14:10 +**Generated at**: 2025-07-11 10:28:32 UTC -## Summary - -Benchmarked 2 Foundry versions across 2 repositories. +## Forge Test ### Repositories Tested 1. [ithacaxyz/account](https://github.com/ithacaxyz/account) 2. [Vectorized/solady](https://github.com/Vectorized/solady) +### Foundry Versions + +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) + +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 7.46 s | 6.13 s | +| solady | 7.84 s | 6.55 s | + +## Forge Build + +### Repositories Tested +1. [ithacaxyz/account](https://github.com/ithacaxyz/account) +2. [Vectorized/solady](https://github.com/Vectorized/solady) ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (51650ea 2025-06-27) +- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) -## Forge Test +### No Cache -| Repository | stable | nightly | -| ----------------- | ------ | ------- | -| ithacaxyz-account | 4.07 s | 4.39 s | -| solady | 3.93 s | 5.26 s | +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 2.61 s | 2.58 s | +| solady | 4.06 s | 3.99 s | -## Forge Build (With Cache) +### With Cache -| Repository | stable | nightly | -| ----------------- | ------ | ------- | -| ithacaxyz-account | 2.02 s | 4.96 s | -| solady | 3.22 s | 3.66 s | +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 2.63 s | 2.63 s | +| solady | 4.02 s | 4.09 s | -## Forge Build (No Cache) +## Forge Coverage + +### Repositories Tested -| Repository | stable | nightly | -| ----------------- | ------ | ------- | -| ithacaxyz-account | 3.54 s | 3.18 s | -| solady | 5.36 s | 3.71 s | +1. [ithacaxyz/account](https://github.com/ithacaxyz/account) +### Foundry Versions + +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) -## System Information +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 21.52 s | 21.63 s | -- **OS**: macos -- **CPU**: 8 -- **Rustc**: rustc 1.89.0-nightly (d97326eab 2025-05-15) From 747211060e1235c73d9a08bc65055240a44cd759 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 16:08:27 +0530 Subject: [PATCH 71/74] fix --- .github/scripts/combine-benchmarks.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh index 90ec7bea49530..ccccb3fd53fcf 100755 --- a/.github/scripts/combine-benchmarks.sh +++ b/.github/scripts/combine-benchmarks.sh @@ -110,7 +110,8 @@ extract_benchmark_table() { # Function to extract system information extract_system_info() { local file=$1 - awk '/^## System Information/,/^$/ { if (!/^## System Information/ && !/^$/) print }' "$file" + # Extract from System Information to end of file (EOF) + awk '/^## System Information/ { found=1; next } found { print }' "$file" } # Start building LATEST.md @@ -150,8 +151,19 @@ for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_b extract_summary_info "$OUTPUT_DIR/$bench_file" >> "$OUTPUT_DIR/LATEST.md" echo >> "$OUTPUT_DIR/LATEST.md" - # For build benchmarks, add both sub-sections - if [[ "$bench_file" == "forge_build_bench.md" ]]; then + # Handle different benchmark types + if [[ "$bench_file" == "forge_test_bench.md" ]]; then + # Extract both Forge Test and Forge Fuzz Test tables + extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Test" >> "$OUTPUT_DIR/LATEST.md" + + # Check if Forge Fuzz Test section exists + if grep -q "^## Forge Fuzz Test" "$OUTPUT_DIR/$bench_file"; then + echo >> "$OUTPUT_DIR/LATEST.md" + echo "## Forge Fuzz Test" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Fuzz Test" >> "$OUTPUT_DIR/LATEST.md" + fi + elif [[ "$bench_file" == "forge_build_bench.md" ]]; then # Extract No Cache table echo "### No Cache" >> "$OUTPUT_DIR/LATEST.md" echo >> "$OUTPUT_DIR/LATEST.md" From 4acd4f7198f77091fd439e44451d71bcc98b7a2f Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 11 Jul 2025 10:50:01 +0000 Subject: [PATCH 72/74] chore(`benches`): update benchmark results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Foundry Benchmarks](https://github.com/foundry-rs/foundry/actions) Co-Authored-By: github-actions --- benches/LATEST.md | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index afe33b1579d92..1d2fe4571f31c 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # 📊 Foundry Benchmark Results -**Generated at**: 2025-07-11 10:28:32 UTC +**Generated at**: 2025-07-11 10:49:57 UTC ## Forge Test @@ -15,8 +15,15 @@ | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 7.46 s | 6.13 s | -| solady | 7.84 s | 6.55 s | +| ithacaxyz-account | 7.21 s | 5.94 s | +| solady | 7.88 s | 6.57 s | + +## Forge Fuzz Test + +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 7.47 s | 6.13 s | +| solady | 7.84 s | 6.84 s | ## Forge Build @@ -33,15 +40,15 @@ | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.61 s | 2.58 s | -| solady | 4.06 s | 3.99 s | +| ithacaxyz-account | 2.60 s | 2.66 s | +| solady | 3.93 s | 4.01 s | ### With Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.63 s | 2.63 s | -| solady | 4.02 s | 4.09 s | +| ithacaxyz-account | 2.62 s | 2.57 s | +| solady | 4.04 s | 4.00 s | ## Forge Coverage @@ -55,5 +62,11 @@ | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 21.52 s | 21.63 s | +| ithacaxyz-account | 21.42 s | 21.46 s | + +## System Information + +- **OS**: linux +- **CPU**: 8 +- **Rustc**: unknown From 81cef2355884c648c98dbf92364cd3aaabaf7dcf Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 11 Jul 2025 11:48:24 +0000 Subject: [PATCH 73/74] chore(`benches`): update benchmark results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Foundry Benchmarks](https://github.com/foundry-rs/foundry/actions) Co-Authored-By: github-actions --- benches/LATEST.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index 1d2fe4571f31c..e4f600ad96626 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # 📊 Foundry Benchmark Results -**Generated at**: 2025-07-11 10:49:57 UTC +**Generated at**: 2025-07-11 11:48:21 UTC ## Forge Test @@ -15,15 +15,15 @@ | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 7.21 s | 5.94 s | -| solady | 7.88 s | 6.57 s | +| ithacaxyz-account | 6.86 s | 6.03 s | +| solady | 7.70 s | 6.94 s | ## Forge Fuzz Test | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 7.47 s | 6.13 s | -| solady | 7.84 s | 6.84 s | +| ithacaxyz-account | 6.65 s | 6.28 s | +| solady | 7.87 s | 6.98 s | ## Forge Build @@ -40,15 +40,15 @@ | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.60 s | 2.66 s | -| solady | 3.93 s | 4.01 s | +| ithacaxyz-account | 2.57 s | 2.61 s | +| solady | 3.95 s | 4.08 s | ### With Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.62 s | 2.57 s | -| solady | 4.04 s | 4.00 s | +| ithacaxyz-account | 2.58 s | 2.60 s | +| solady | 4.04 s | 4.07 s | ## Forge Coverage @@ -62,7 +62,7 @@ | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 21.42 s | 21.46 s | +| ithacaxyz-account | 21.49 s | 20.85 s | ## System Information From 8de85492f6024f29493d05090bf2d596d73f28a5 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 11 Jul 2025 17:33:59 +0530 Subject: [PATCH 74/74] don't run on PRs --- .github/workflows/benchmarks.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d6f6c24ea945b..c664680b31daa 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,8 +1,6 @@ name: Foundry Benchmarks on: - pull_request: - types: [opened, synchronize, reopened] workflow_dispatch: inputs: pr_number: @@ -194,7 +192,7 @@ jobs: script: | const branchName = '${{ steps.benchmark_results.outputs.branch_name }}'; const prComment = `${{ steps.benchmark_results.outputs.pr_comment }}`; - + // Create the pull request const { data: pr } = await github.rest.pulls.create({ owner: context.repo.owner,