diff --git a/package.json b/package.json index c550e20f..a70dbcb9 100644 --- a/package.json +++ b/package.json @@ -56,9 +56,9 @@ "test:browser:ui": "playwright test --ui", "test:autobahn": "cd test/autobahn && ./run-wstest.js", "bench": "vitest bench --run --config vitest.bench.config.mjs", - "bench:baseline": "node test/benchmark/track-performance.mjs save", - "bench:compare": "node test/benchmark/track-performance.mjs compare", - "bench:check": "node test/benchmark/track-performance.mjs check", + "bench:baseline": "pnpm run bench -- --outputJson test/benchmark/baseline.json", + "bench:compare": "pnpm run bench -- --compare test/benchmark/baseline.json", + "bench:check": "pnpm run bench -- --compare test/benchmark/baseline.json", "lint": "eslint lib/**/*.js test/**/*.js", "lint:fix": "eslint lib/**/*.js test/**/*.js --fix" }, diff --git a/test/benchmark/README.md b/test/benchmark/README.md index f20b8361..c0ba582b 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -1,6 +1,6 @@ # WebSocket-Node Performance Benchmarks -This directory contains performance benchmarks for critical WebSocket operations. +This directory contains performance benchmarks for critical WebSocket operations using Vitest's built-in benchmarking functionality. ## Running Benchmarks @@ -8,10 +8,18 @@ This directory contains performance benchmarks for critical WebSocket operations # Run all benchmarks pnpm run bench -# Compare with previous results +# Save current results as baseline +pnpm run bench:baseline + +# Compare with baseline (shows ⇑/⇓ indicators) pnpm run bench:compare + +# Check for regressions (exits with error on performance drops) +pnpm run bench:check ``` +Note: `bench:check` is the same as `bench:compare` but is intended for CI environments where you want the build to fail on performance regressions. + ## Benchmark Suites ### Frame Operations (`frame-operations.bench.mjs`) @@ -46,15 +54,35 @@ Benchmarks output operations per second (hz) and timing statistics: ## Performance Baselines -These benchmarks establish baseline performance for regression detection: -1. Frame serialization should maintain 4M+ ops/sec for small frames -2. Connection operations should maintain 25K+ ops/sec -3. Large message handling (64KB) should not degrade significantly +Baseline results are stored in `baseline.json` using Vitest's JSON format. When running `bench:compare` or `bench:check`, Vitest automatically compares current results against the baseline and shows: +- `[1.05x] ⇑` for improvements (faster) +- `[0.95x] ⇓` for regressions (slower) +- Baseline values for reference + +Expected performance ranges: +1. Frame serialization: 3-4.5M ops/sec +2. Message sending: 100K-900K ops/sec (varies by size) +3. Ping/Pong: 1.5-2M ops/sec +4. Connection creation: 30K ops/sec + +## Benchmark Structure + +Each operation is in its own `describe` block to prevent Vitest from treating them as alternative implementations for comparison. This structure ensures each operation is measured independently: + +```javascript +describe('Send Ping Frame', () => { + bench('send ping frame', () => { + sharedConnection.ping(); + }); +}); +``` ## Adding New Benchmarks When adding benchmarks: 1. Pre-allocate buffers and data outside the benchmark loop -2. Use descriptive test names with size information -3. Focus on operations that directly impact production performance -4. Avoid testing implementation details +2. Create shared connections at module scope (not inside benchmark functions) +3. Use descriptive test names with size information +4. Put each unique operation in its own `describe` block +5. Focus on operations that directly impact production performance +6. Avoid testing implementation details diff --git a/test/benchmark/baseline.json b/test/benchmark/baseline.json index 1da79efb..9d27fc91 100644 --- a/test/benchmark/baseline.json +++ b/test/benchmark/baseline.json @@ -1,19 +1,294 @@ { - "timestamp": "2025-10-06T17:27:59.641Z", - "results": { - "WebSocketConnection Performance 7359ms": { - "create connection instance": 28847.63, - "send small UTF-8 message": 914160.4, - "send medium UTF-8 message (1KB)": 108086.21, - "send binary message (1KB)": 222918.81, - "send ping frame": 2375454.67, - "send pong frame": 1935975.19 + "files": [ + { + "filepath": "/home/ubuntu/code/websocket-node/test/benchmark/connection-operations.bench.mjs", + "groups": [ + { + "fullName": "test/benchmark/connection-operations.bench.mjs > Connection Creation", + "benchmarks": [ + { + "id": "347648886_0_0", + "sampleCount": 14950, + "name": "create connection instance", + "rank": 1, + "rme": 5.721365271168026, + "totalTime": 500.017038999993, + "min": 0.02091599999994287, + "max": 5.860308999999916, + "hz": 29898.981102522404, + "period": 0.03344595578595271, + "mean": 0.03344595578595271, + "variance": 0.014250024909513178, + "sd": 0.1193734681975571, + "sem": 0.0009763088259937304, + "df": 14949, + "critical": 1.96, + "moe": 0.0019135652989477115, + "p75": 0.035286000000041895, + "p99": 0.07499600000005557, + "p995": 0.0863120000001345, + "p999": 0.40689399999996567 + } + ] + }, + { + "fullName": "test/benchmark/connection-operations.bench.mjs > Send Small UTF-8 Message", + "benchmarks": [ + { + "id": "347648886_1_0", + "sampleCount": 433879, + "name": "send small UTF-8 message", + "rank": 1, + "rme": 11.43139905071693, + "totalTime": 506.6114810000274, + "min": 0.00028599999996004044, + "max": 9.61037800000031, + "hz": 856433.4135174811, + "period": 0.0011676330981679856, + "mean": 0.0011676330981679856, + "variance": 0.0020121856762223642, + "sd": 0.04485739265965382, + "sem": 0.000068100407601955, + "df": 433878, + "critical": 1.96, + "moe": 0.00013347679889983178, + "p75": 0.0007550000000264845, + "p99": 0.0038449999997283157, + "p995": 0.01220100000000457, + "p999": 0.02408300000001873 + } + ] + }, + { + "fullName": "test/benchmark/connection-operations.bench.mjs > Send Medium UTF-8 Message (1KB)", + "benchmarks": [ + { + "id": "347648886_2_0", + "sampleCount": 48440, + "name": "send medium UTF-8 message (1KB)", + "rank": 1, + "rme": 4.570568475755207, + "totalTime": 500.00057599997626, + "min": 0.0008000000002539309, + "max": 5.454282000000148, + "hz": 96879.88839437316, + "period": 0.010322059785300914, + "mean": 0.010322059785300914, + "variance": 0.0028065008097464196, + "sd": 0.0529764174869009, + "sem": 0.0002407024543854945, + "df": 48439, + "critical": 1.96, + "moe": 0.0004717768105955692, + "p75": 0.015486000000237254, + "p99": 0.039346000000023196, + "p995": 0.04509699999971417, + "p999": 0.1321459999999206 + } + ] + }, + { + "fullName": "test/benchmark/connection-operations.bench.mjs > Send Binary Message (1KB)", + "benchmarks": [ + { + "id": "347648886_3_0", + "sampleCount": 108833, + "name": "send binary message (1KB)", + "rank": 1, + "rme": 6.408247345850968, + "totalTime": 500.0128869999694, + "min": 0.0002959999997074192, + "max": 8.217849999999999, + "hz": 217660.39002112093, + "period": 0.004594313186257563, + "mean": 0.004594313186257563, + "variance": 0.0024556597086718116, + "sd": 0.04955461339443394, + "sem": 0.00015021171062164863, + "df": 108832, + "critical": 1.96, + "moe": 0.0002944149528184313, + "p75": 0.004922000000078697, + "p99": 0.027522000000317348, + "p995": 0.03348600000026636, + "p999": 0.11101899999994203 + } + ] + }, + { + "fullName": "test/benchmark/connection-operations.bench.mjs > Send Ping Frame", + "benchmarks": [ + { + "id": "347648886_4_0", + "sampleCount": 1062702, + "name": "send ping frame", + "rank": 1, + "rme": 20.14485654032809, + "totalTime": 508.32634000000144, + "min": 0.00018199999976786785, + "max": 32.92630800000006, + "hz": 2090590.0725112867, + "period": 0.0004783338508819984, + "mean": 0.0004783338508819984, + "variance": 0.002568561363662073, + "sd": 0.05068097634874523, + "sem": 0.00004916309594081911, + "df": 1062701, + "critical": 1.96, + "moe": 0.00009635966804400545, + "p75": 0.00024899999971239595, + "p99": 0.0006320000002233428, + "p995": 0.0008950000001277658, + "p999": 0.011792999999670428 + } + ] + }, + { + "fullName": "test/benchmark/connection-operations.bench.mjs > Send Pong Frame", + "benchmarks": [ + { + "id": "347648886_5_0", + "sampleCount": 962038, + "name": "send pong frame", + "rank": 1, + "rme": 18.913749130520372, + "totalTime": 500.0001680007763, + "min": 0.00019799999972747173, + "max": 31.614644000000226, + "hz": 1924075.353507694, + "period": 0.0005197301645057433, + "mean": 0.0005197301645057433, + "variance": 0.0024198652313353448, + "sd": 0.049192125704581466, + "sem": 0.00005015329564809036, + "df": 962037, + "critical": 1.96, + "moe": 0.0000983004594702571, + "p75": 0.0003349999997226405, + "p99": 0.0009099999997488339, + "p995": 0.0018499999996492988, + "p999": 0.011822000000393018 + } + ] + } + ] }, - "WebSocketFrame Performance 9745ms": { - "serialize small text frame (17 bytes, unmasked)": 4240401.67, - "serialize small text frame (17 bytes, masked)": 3070532.44, - "serialize medium binary frame (1KB)": 4418217.61, - "serialize large binary frame (64KB)": 3918678.38 + { + "filepath": "/home/ubuntu/code/websocket-node/test/benchmark/frame-operations.bench.mjs", + "groups": [ + { + "fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Small Text Frame (Unmasked)", + "benchmarks": [ + { + "id": "2141590085_0_0", + "sampleCount": 2221574, + "name": "serialize small text frame (17 bytes, unmasked)", + "rank": 1, + "rme": 0.9893597558817244, + "totalTime": 500.0001229996669, + "min": 0.0001449999999749707, + "max": 0.5502369999994698, + "hz": 4443146.90698882, + "period": 0.00022506570701658686, + "mean": 0.00022506570701658686, + "variance": 0.00000286731744387617, + "sd": 0.0016933155181111906, + "sem": 0.0000011360762905677454, + "df": 2221573, + "critical": 1.96, + "moe": 0.000002226709529512781, + "p75": 0.00019599999905040022, + "p99": 0.0005499999988387572, + "p995": 0.0006460000004153699, + "p999": 0.006724000000758679 + } + ] + }, + { + "fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Small Text Frame (Masked)", + "benchmarks": [ + { + "id": "2141590085_1_0", + "sampleCount": 1534250, + "name": "serialize small text frame (17 bytes, masked)", + "rank": 1, + "rme": 3.3082056682221554, + "totalTime": 500.0079150010697, + "min": 0.0002190000013797544, + "max": 6.977795999999216, + "hz": 3068451.4264073553, + "period": 0.0003258972885781781, + "mean": 0.0003258972885781781, + "variance": 0.00004642270968057881, + "sd": 0.006813421290407544, + "sem": 0.00000550069008843143, + "df": 1534249, + "critical": 1.96, + "moe": 0.000010781352573325602, + "p75": 0.0002859999985957984, + "p99": 0.0007029999997030245, + "p995": 0.0009759999993548263, + "p999": 0.008757999999943422 + } + ] + }, + { + "fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Medium Binary Frame (1KB)", + "benchmarks": [ + { + "id": "2141590085_2_0", + "sampleCount": 2164023, + "name": "serialize medium binary frame (1KB)", + "rank": 1, + "rme": 1.6649018583076653, + "totalTime": 500.0305290022534, + "min": 0.00015799999891896732, + "max": 2.1783230000000913, + "hz": 4327781.754282142, + "period": 0.00023106525623907575, + "mean": 0.00023106525623907575, + "variance": 0.000008336740867673836, + "sd": 0.0028873414878870557, + "sem": 0.0000019627600739937454, + "df": 2164022, + "critical": 1.96, + "moe": 0.000003847009745027741, + "p75": 0.00020399999993969686, + "p99": 0.0005010000004403992, + "p995": 0.0005930000006628688, + "p999": 0.006003999998938525 + } + ] + }, + { + "fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Large Binary Frame (64KB)", + "benchmarks": [ + { + "id": "2141590085_3_0", + "sampleCount": 2251276, + "name": "serialize large binary frame (64KB)", + "rank": 1, + "rme": 1.1766836796382618, + "totalTime": 500.00007300055404, + "min": 0.00015700000039942097, + "max": 1.1734879999985424, + "hz": 4502551.342622515, + "period": 0.00022209630138665985, + "mean": 0.00022209630138665985, + "variance": 0.000004002383606964811, + "sd": 0.002000595812992922, + "sem": 0.0000013333525160699149, + "df": 2251275, + "critical": 1.96, + "moe": 0.000002613370931497033, + "p75": 0.0001940000001923181, + "p99": 0.0005029999992984813, + "p995": 0.0006020000000717118, + "p999": 0.005696000000170898 + } + ] + } + ] } - } + ] } \ No newline at end of file diff --git a/test/benchmark/connection-operations.bench.mjs b/test/benchmark/connection-operations.bench.mjs index efe48a7f..e0860ec9 100644 --- a/test/benchmark/connection-operations.bench.mjs +++ b/test/benchmark/connection-operations.bench.mjs @@ -2,42 +2,56 @@ import { bench, describe } from 'vitest'; import WebSocketConnection from '../../lib/WebSocketConnection.js'; import { MockSocket } from '../helpers/mocks.mjs'; -describe('WebSocketConnection Performance', () => { - // Pre-allocate messages and buffers outside benchmarks - const smallMessage = 'Hello, WebSocket!'; - const mediumMessage = 'x'.repeat(1024); - const binaryBuffer = Buffer.alloc(1024); - - // Shared connection for send operations (created once, reused across all iterations) - // Note: We initialize this directly rather than using beforeAll() because Vitest's - // benchmark runner doesn't execute hooks before benchmarks in the same way as test() - const sharedSocket = new MockSocket(); - const sharedConnection = new WebSocketConnection(sharedSocket, [], 'echo-protocol', false, {}); - sharedConnection._addSocketEventListeners(); - sharedConnection.state = 'open'; - +// Pre-allocate messages and buffers outside benchmarks +const smallMessage = 'Hello, WebSocket!'; +const mediumMessage = 'x'.repeat(1024); +const binaryBuffer = Buffer.alloc(1024); + +// Shared connection for send operations (created once, reused across all iterations) +// Note: We initialize this directly rather than using beforeAll() because Vitest's +// benchmark runner doesn't execute hooks before benchmarks in the same way as test() +const sharedSocket = new MockSocket(); +const sharedConnection = new WebSocketConnection(sharedSocket, [], 'echo-protocol', false, {}); +sharedConnection._addSocketEventListeners(); +sharedConnection.state = 'open'; + +// Each operation gets its own describe block so Vitest doesn't treat them as +// alternatives for comparison (like comparing different sorting algorithms). +// This allows each benchmark to be measured independently. + +describe('Connection Creation', () => { bench('create connection instance', () => { const socket = new MockSocket(); const connection = new WebSocketConnection(socket, [], 'echo-protocol', false, {}); connection._addSocketEventListeners(); }); +}); +describe('Send Small UTF-8 Message', () => { bench('send small UTF-8 message', () => { sharedConnection.sendUTF(smallMessage); }); +}); +describe('Send Medium UTF-8 Message (1KB)', () => { bench('send medium UTF-8 message (1KB)', () => { sharedConnection.sendUTF(mediumMessage); }); +}); +describe('Send Binary Message (1KB)', () => { bench('send binary message (1KB)', () => { sharedConnection.sendBytes(binaryBuffer); }); +}); +describe('Send Ping Frame', () => { bench('send ping frame', () => { sharedConnection.ping(); }); +}); +describe('Send Pong Frame', () => { bench('send pong frame', () => { sharedConnection.pong(); }); diff --git a/test/benchmark/frame-operations.bench.mjs b/test/benchmark/frame-operations.bench.mjs index f4fc3acc..76f11bbf 100644 --- a/test/benchmark/frame-operations.bench.mjs +++ b/test/benchmark/frame-operations.bench.mjs @@ -1,33 +1,42 @@ import { bench, describe } from 'vitest'; import WebSocketFrame from '../../lib/WebSocketFrame.js'; -describe('WebSocketFrame Performance', () => { - // Pre-allocate payloads outside benchmark loops - const smallPayload = Buffer.from('Hello, WebSocket!'); - const mediumPayload = Buffer.alloc(1024); - mediumPayload.fill('x'); - const largePayload = Buffer.alloc(64 * 1024); - largePayload.fill('y'); +// Pre-allocate payloads outside benchmark loops +const smallPayload = Buffer.from('Hello, WebSocket!'); +const mediumPayload = Buffer.alloc(1024); +mediumPayload.fill('x'); +const largePayload = Buffer.alloc(64 * 1024); +largePayload.fill('y'); - // Pre-allocate mask - const mask = Buffer.from([0x12, 0x34, 0x56, 0x78]); +// Pre-allocate mask +const mask = Buffer.from([0x12, 0x34, 0x56, 0x78]); +// Each operation gets its own describe block so Vitest doesn't treat them as +// alternatives for comparison. This allows each benchmark to be measured independently. + +describe('Serialize Small Text Frame (Unmasked)', () => { bench('serialize small text frame (17 bytes, unmasked)', () => { const frame = new WebSocketFrame(smallPayload, true, 0x01); frame.toBuffer(); }); +}); +describe('Serialize Small Text Frame (Masked)', () => { bench('serialize small text frame (17 bytes, masked)', () => { const frame = new WebSocketFrame(smallPayload, true, 0x01); frame.mask = mask; frame.toBuffer(); }); +}); +describe('Serialize Medium Binary Frame (1KB)', () => { bench('serialize medium binary frame (1KB)', () => { const frame = new WebSocketFrame(mediumPayload, true, 0x02); frame.toBuffer(); }); +}); +describe('Serialize Large Binary Frame (64KB)', () => { bench('serialize large binary frame (64KB)', () => { const frame = new WebSocketFrame(largePayload, true, 0x02); frame.toBuffer(); diff --git a/test/benchmark/track-performance.mjs b/test/benchmark/track-performance.mjs deleted file mode 100755 index 469a6675..00000000 --- a/test/benchmark/track-performance.mjs +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env node - -/** - * Performance Baseline Tracking Script - * - * This script runs benchmarks and tracks performance over time. - * It can save baselines and compare current performance against them. - * - * Usage: - * node test/benchmark/track-performance.mjs save # Save current results as baseline - * node test/benchmark/track-performance.mjs compare # Compare with baseline - * node test/benchmark/track-performance.mjs check # Check for regressions (CI) - */ - -import { exec } from 'child_process'; -import { readFileSync, writeFileSync, existsSync } from 'fs'; -import { promisify } from 'util'; - -const execAsync = promisify(exec); -const BASELINE_FILE = 'test/benchmark/baseline.json'; -const REGRESSION_THRESHOLD = 0.15; // 15% regression threshold - -async function runBenchmarks() { - console.log('Running benchmarks...'); - const { stdout } = await execAsync('pnpm run bench 2>&1'); - return parseBenchmarkOutput(stdout); -} - -function parseBenchmarkOutput(output) { - const results = {}; - const lines = output.split('\n'); - - let currentSuite = null; - for (const line of lines) { - // Detect suite name - if (line.includes('> WebSocket')) { - currentSuite = line.match(/> (.*)/)[1].trim(); - // Don't initialize suite here - wait until first benchmark is found - } - - // Parse benchmark results - const benchMatch = line.match(/^\s*[·•]\s+(.+?)\s+(\d+(?:,\d+)*(?:\.\d+)?)\s/); - if (benchMatch && currentSuite) { - const [, name, hz] = benchMatch; - // Lazily initialize suite only when first benchmark is found - if (!results[currentSuite]) { - results[currentSuite] = {}; - } - results[currentSuite][name.trim()] = parseFloat(hz.replace(/,/g, '')); - } - } - - return results; -} - -async function saveBaseline() { - const results = await runBenchmarks(); - writeFileSync(BASELINE_FILE, JSON.stringify({ - timestamp: new Date().toISOString(), - results - }, null, 2)); - console.log(`\n✅ Baseline saved to ${BASELINE_FILE}`); -} - -async function compareWithBaseline() { - if (!existsSync(BASELINE_FILE)) { - console.error(`❌ No baseline found at ${BASELINE_FILE}`); - console.log('Run: node test/benchmark/track-performance.mjs save'); - process.exit(1); - } - - const baseline = JSON.parse(readFileSync(BASELINE_FILE, 'utf-8')); - const current = await runBenchmarks(); - - console.log(`\n📊 Comparing with baseline from ${baseline.timestamp}\n`); - - let hasRegression = false; - - for (const [suite, tests] of Object.entries(current)) { - if (!baseline.results[suite]) continue; - - console.log(`\n${suite}:`); - - for (const [test, currentHz] of Object.entries(tests)) { - const baselineHz = baseline.results[suite][test]; - if (!baselineHz) continue; - - const change = (currentHz - baselineHz) / baselineHz; - const changePercent = (change * 100).toFixed(2); - - let status = '✓'; - if (change < -REGRESSION_THRESHOLD) { - status = '❌ REGRESSION'; - hasRegression = true; - } else if (change < -0.05) { - status = '⚠️ SLOWER'; - } else if (change > 0.05) { - status = '🚀 FASTER'; - } - - console.log(` ${status} ${test}`); - console.log(` ${baselineHz.toLocaleString()} → ${currentHz.toLocaleString()} ops/sec (${changePercent}%)`); - } - } - - if (hasRegression) { - console.log(`\n❌ Performance regressions detected (>${REGRESSION_THRESHOLD * 100}% slower)`); - process.exit(1); - } else { - console.log('\n✅ No performance regressions detected'); - } -} - -const command = process.argv[2]; - -if (command === 'save') { - await saveBaseline(); -} else if (command === 'compare' || command === 'check') { - await compareWithBaseline(); -} else { - console.log(` -Usage: - node test/benchmark/track-performance.mjs save # Save baseline - node test/benchmark/track-performance.mjs compare # Compare with baseline - node test/benchmark/track-performance.mjs check # Check for regressions (CI) -`); - process.exit(1); -}