transitive-bullshit · divdavem · Jan 16, 2025 · Jan 16, 2025 · Jan 17, 2025 · Jan 17, 2025
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 node_modules/
 dist/
 .DS_Store
+results.js
diff --git a/.prettierrc b/.prettierrc
@@ -1,4 +1,5 @@
 {
+  "printWidth": 140,
   "singleQuote": false,
   "jsxSingleQuote": false,
   "semi": true,

diff --git a/package.json b/package.json
@@ -3,11 +3,12 @@
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
+  "type": "module",
   "scripts": {
     "test": "vitest run",
-    "build": "esbuild src/index.ts --bundle --format=cjs --platform=node --outdir=dist --sourcemap=external",
+    "build": "esbuild src/index.ts --bundle --splitting --format=esm --platform=node --outdir=dist --sourcemap=external",
     "run": "node --expose-gc dist/index.js",
-    "bench": "esbuild src/index.ts --bundle --format=cjs --platform=node | node --expose-gc"
+    "bench": "pnpm run build && pnpm run run"
   },
   "keywords": [],
   "author": "",

diff --git a/results.html b/results.html
@@ -0,0 +1,119 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>js-reactivity-benchmark results</title>
+    <script src="./results.js"></script>
+  </head>
+  <body>
+    <style>
+      canvas {
+        margin-bottom: 30px;
+      }
+    </style>
+    <script
+      src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.js"
+      crossorigin="anonymous"
+      integrity="sha256-KBLLiCX9xXRp6y97sFXpQpJE5ZmSBRHuR36ChJm2Mss="
+    ></script>
+    <h1>js-reactivity-benchmark results</h1>
+    <script type="module">
+      const BENCHMARK_RESULTS = globalThis.BENCHMARK_RESULTS;
+      if (!BENCHMARK_RESULTS) {
+        const msg = document.createElement("div");
+        msg.textContent = 'Run benchmarks with "pnpm bench" then refresh this page to see the results.';
+        document.body.appendChild(msg);
+      } else {
+        const results = BENCHMARK_RESULTS.results;
+        const createChart = (label, data, props) => {
+          const canvas = document.createElement("canvas");
+          document.body.appendChild(canvas);
+          new Chart(canvas.getContext("2d"), {
+            type: "bar",
+            data: {
+              labels: data.map(({ label }) => label),
+              datasets: props.map((prop) => ({
+                label: prop,
+                data: data.map((item) => item[prop]),
+                borderWidth: 1,
+                backgroundColor: data.map(({ color }) => color),
+              })),
+            },
+            options: {
+              indexAxis: "y",
+              plugins: {
+                title: {
+                  display: true,
+                  text: label,
+                },
+                legend: {
+                  display: false,
+                },
+              },
+            },
+          });
+        };
+        const progressiveColors = [
+          [0, 255, 0],
+          [255, 255, 0],
+          [255, 0, 0],
+        ];
+        results.forEach((framework, i) => {
+          const percent = results.length === 1 ? 0 : i / (results.length - 1);
+          const [startColor, endColor] =
+            percent < 0.5 ? [progressiveColors[0], progressiveColors[1]] : [progressiveColors[1], progressiveColors[2]];
+          const relativePercent = percent < 0.5 ? percent * 2 : (percent - 0.5) * 2;
+          const r = Math.round(startColor[0] + (endColor[0] - startColor[0]) * relativePercent);
+          const g = Math.round(startColor[1] + (endColor[1] - startColor[1]) * relativePercent);
+          const b = Math.round(startColor[2] + (endColor[2] - startColor[2]) * relativePercent);
+          framework.color = `rgba(${r}, ${g}, ${b}, 0.9)`;
+        });
+        createChart(
+          "Average time",
+          results.map(({ framework, average, color }) => ({ label: framework, average, color })),
+          ["average"]
+        );
+        Object.keys(results[0].tests).forEach((testName) => {
+          const data = results
+            .map(({ framework, tests, color }) => {
+              const testInfo = tests[testName];
+              if (!testInfo || testInfo.length === 0) {
+                return null;
+              }
+              let min = Infinity;
+              let max = -Infinity;
+              let sum = 0;
+              for (const time of testInfo) {
+                if (time < min) {
+                  min = time;
+                }
+                if (time > max) {
+                  max = time;
+                }
+                sum += time;
+              }
+              const average = sum / testInfo.length;
+              return {
+                label: framework,
+                min,
+                average,
+                max,
+                color,
+              };
+            })
+            .filter((result) => !!result)
+            .sort(({ average: average1 }, { average: average2 }) => average1 - average2);
+          createChart(testName, data, ["min", "average", "max"]);
+        });
+        const addInfo = (text, tag = "div") => {
+          const infos = document.createElement(tag);
+          infos.innerText = text;
+          document.body.appendChild(infos);
+        };
+        addInfo(`Tests start time: ${new Date(BENCHMARK_RESULTS.startTime).toString()}`);
+        addInfo(`Tests end time: ${new Date(BENCHMARK_RESULTS.endTime).toString()}`);
+        addInfo(`${JSON.stringify(BENCHMARK_RESULTS.system, null, 2)}`, "pre");
+      }
+    </script>
+  </body>
+</html>
diff --git a/src/cellxBench.ts b/src/cellxBench.ts
@@ -1,5 +1,4 @@
 // The following is an implementation of the cellx benchmark https://github.com/Riim/cellx/blob/master/perf/perf.html
-import { logPerfResult } from "./util/perfLogging";
 import { Computed, ReactiveFramework } from "./util/reactiveFramework";
 
 const cellx = (framework: ReactiveFramework, layers: number) => {
@@ -117,10 +116,10 @@ export const cellxbench = (framework: ReactiveFramework) => {
       total += elapsed;
     }
 
-    logPerfResult({
+    process.send?.({
       framework: framework.name,
       test: `cellx${layers}`,
-      time: total.toFixed(2),
+      time: total,
     });
   }
 

diff --git a/src/config.ts b/src/config.ts
@@ -1,48 +1,32 @@
 import { TestConfig, FrameworkInfo } from "./util/frameworkTypes";
 
-import { alienFramework } from "./frameworks/alienSignals";
-import { angularFramework } from "./frameworks/angularSignals";
-import { mobxFramework } from "./frameworks/mobx";
-import { tc39SignalsProposalStage0 } from "./frameworks/tc39-proposal-signals-stage-0";
-import { molWireFramework } from "./frameworks/molWire";
-import { obyFramework } from "./frameworks/oby";
-import { preactSignalFramework } from "./frameworks/preactSignals";
-import { reactivelyFramework } from "./frameworks/reactively";
-import { signiaFramework } from "./frameworks/signia";
-import { solidFramework } from "./frameworks/solid";
-import { sFramework } from "./frameworks/s";
-import { usignalFramework } from "./frameworks/uSignal";
-import { vueReactivityFramework } from "./frameworks/vueReactivity";
-import { svelteFramework } from "./frameworks/svelte";
-import { tansuFramework } from "./frameworks/tansu";
-// import { compostateFramework } from "./frameworks/compostate";
-// import { valtioFramework } from "./frameworks/valtio";
+export let executions = 3;
 
-export const frameworkInfo: FrameworkInfo[] = [
-  { framework: alienFramework, testPullCounts: true },
-  { framework: preactSignalFramework, testPullCounts: true },
-  { framework: svelteFramework, testPullCounts: true },
-  { framework: tc39SignalsProposalStage0, testPullCounts: true },
-  { framework: reactivelyFramework, testPullCounts: true },
-  { framework: sFramework },
-  { framework: tansuFramework, testPullCounts: true },
-  { framework: angularFramework, testPullCounts: true },
-  { framework: molWireFramework, testPullCounts: true },
-  { framework: obyFramework, testPullCounts: true },
-  { framework: signiaFramework, testPullCounts: true },
-  { framework: solidFramework },
-  { framework: usignalFramework, testPullCounts: true },
-  { framework: vueReactivityFramework, testPullCounts: true },
+export const frameworkInfo: (() => Promise<FrameworkInfo>)[] = [
+  async () => ({ framework: (await import("./frameworks/alienSignals")).alienFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/preactSignals")).preactSignalFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/svelte")).svelteFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/tc39-proposal-signals-stage-0")).tc39SignalsProposalStage0, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/reactively")).reactivelyFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/s")).sFramework }),
+  async () => ({ framework: (await import("./frameworks/tansu")).tansuFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/angularSignals")).angularFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/molWire")).molWireFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/oby")).obyFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/signia")).signiaFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/solid")).solidFramework }),
+  async () => ({ framework: (await import("./frameworks/uSignal")).usignalFramework, testPullCounts: true }),
+  async () => ({ framework: (await import("./frameworks/vueReactivity")).vueReactivityFramework, testPullCounts: true }),
   // NOTE: MobX currently hangs on some of the `dynamic` tests and `cellx` tests, so disable it if you want to run them. (https://github.com/mobxjs/mobx/issues/3926)
-  { framework: mobxFramework, testPullCounts: false },
+  async () => ({ framework: (await import("./frameworks/mobx")).mobxFramework, testPullCounts: false }),
 
   // --- Disabled frameworks ---
   // NOTE: the compostate adapter is currently broken and unused.
-  // { framework: compostateFramework },
+  // async () => ({ framework: (await import("./frameworks/compostate")).compostateFramework }),
   // NOTE: the kairo adapter is currently broken and unused.
-  // { framework: kairoFramework, testPullCounts: true },
+  // async () => ({ framework: (await import("./frameworks/kairo")).kairoFramework, testPullCounts: true }),
   // NOTE: Valtio currently hangs on some of the `dynamic` tests, so disable it if you want to run them. (https://github.com/pmndrs/valtio/discussions/949)
-  // { framework: valtioFramework },
+  // async () => ({ framework: (await import("./frameworks/valtio")).valtioFramework }),
 ];
 
 export const perfTests: TestConfig[] = [

diff --git a/src/dynamicBench.ts b/src/dynamicBench.ts
@@ -1,5 +1,5 @@
 import { Counter, makeGraph, runGraph } from "./util/dependencyGraph";
-import { logPerfResult, perfRowStrings } from "./util/perfLogging";
+import { makeTitle } from "./util/perfLogging";
 import { verifyBenchResult } from "./util/perfTests";
 import { FrameworkInfo } from "./util/frameworkTypes";
 import { perfTests } from "./config";
@@ -41,7 +41,7 @@ export async function dynamicBench(
       return { sum, count: counter.count };
     });
 
-    logPerfResult(perfRowStrings(framework.name, config, timedResult));
+    process.send?.({ framework: framework.name, test: `${makeTitle(config)} (${config.name || ""})`, time: timedResult.timing.time });
     verifyBenchResult(frameworkTest, config, timedResult);
   }
 }
diff --git a/src/frameworks.test.ts b/src/frameworks.test.ts
@@ -3,7 +3,7 @@ import { expect, test, vi } from "vitest";
 import { FrameworkInfo, TestConfig } from "./util/frameworkTypes";
 import { frameworkInfo } from "./config";
 
-frameworkInfo.forEach((frameworkInfo) => frameworkTests(frameworkInfo));
+(await Promise.all(frameworkInfo.map((frameworkLoader) => frameworkLoader()))).forEach((frameworkInfo) => frameworkTests(frameworkInfo));
 
 function makeConfig(): TestConfig {
   return {

diff --git a/src/index.ts b/src/index.ts
@@ -1,18 +1,21 @@
+import cluster from "cluster";
+import os from "os";
 import { dynamicBench } from "./dynamicBench";
 // import { cellxbench } from "./cellxBench";
 import { sbench } from "./sBench";
-import { frameworkInfo } from "./config";
+import { frameworkInfo, executions } from "./config";
 import { logPerfResult, perfReportHeaders } from "./util/perfLogging";
 import { molBench } from "./molBench";
 import { kairoBench } from "./kairoBench";
+import { FrameworkInfo } from "./util/frameworkTypes";
+import { writeFileSync } from "fs";
 
-async function main() {
-  logPerfResult(perfReportHeaders());
-  (globalThis as any).__DEV__ = true;
+async function testFramework(frameworkTestPromise: () => Promise<FrameworkInfo>) {
+  try {
+    (globalThis as any).__DEV__ = true;
 
-  for (const frameworkTest of frameworkInfo) {
+    const frameworkTest = await frameworkTestPromise();
     const { framework } = frameworkTest;
-
     await kairoBench(framework);
     await molBench(framework);
     sbench(framework);
@@ -24,8 +27,110 @@ async function main() {
 
     await dynamicBench(frameworkTest);
 
-    globalThis.gc?.();
+    process.exit(0);
+  } catch (err: any) {
+    console.error(err);
+    process.exit(1);
   }
 }
 
-main();
+const average = (times: number[]) => times.reduce((a, b) => a + b, 0) / times.length;
+
+async function main() {
+  const system = {
+    engine: process.versions,
+    os: {
+      platform: os.platform(),
+      type: os.type(),
+      release: os.release(),
+      version: os.version(),
+    },
+    hardware: {
+      machine: os.machine(),
+      cpus: os.cpus().map(({ model }) => model),
+    },
+  };
+  const startTime = Date.now();
+  logPerfResult(perfReportHeaders());
+
+  const frameworkSummary = new Map<string, Record<string, number[]>>();
+  cluster.on("message", (_, message) => {
+    logPerfResult({
+      framework: message.framework,
+      test: message.test,
+      time: message.time.toFixed(2),
+    });
+    let framework = frameworkSummary.get(message.framework);
+    if (!framework) {
+      framework = {};
+      frameworkSummary.set(message.framework, framework);
+    }
+    let test = framework[message.test];
+    if (!test) {
+      test = [];
+      framework[message.test] = test;
+    }
+    test.push(message.time);
+  });
+
+  const logSummary = () => {
+    const summary = [...frameworkSummary.entries()]
+      .map(([framework, tests]) => {
+        const averages = Object.values(tests).map(average);
+        return { framework, testsCount: averages.length, average: average(averages), tests };
+      })
+      .sort(({ average: average1 }, { average: average2 }) => average1 - average2);
+    if (summary.length === 0) {
+      return;
+    }
+    console.log("");
+    logPerfResult(perfReportHeaders());
+    for (const { framework, average, testsCount } of summary) {
+      logPerfResult({
+        framework,
+        test: `Average (${testsCount} tests)`,
+        time: average.toFixed(2),
+      });
+    }
+    console.log("");
+    const endTime = Date.now();
+    writeFileSync(
+      "results.js",
+      `globalThis.BENCHMARK_RESULTS = ${JSON.stringify(
+        {
+          startTime,
+          endTime,
+          system,
+          results: summary,
+        },
+        null,
+        "\t"
+      )};`
+    );
+  };
+  process.on("SIGUSR1", logSummary);
+  process.on("SIGINT", () => process.exit(1));
+  process.on("SIGTERM", () => process.exit(1));
+  process.on("exit", logSummary);
+
+  for (let n = 0; n < executions; n++) {
+    for (let i = 0, l = frameworkInfo.length; i < l; i++) {
+      await new Promise<void>((resolve, reject) =>
+        cluster.fork({ FRAMEWORK_ID: i }).addListener("exit", (code, signal) => {
+          if (code === 0) {
+            resolve();
+          } else {
+            reject(new Error(`Framework test failed with code ${code} and signal ${signal}`));
+          }
+        })
+      );
+    }
+    logSummary();
+  }
+}
+
+if (cluster.isPrimary) {
+  main();
+} else {
+  testFramework(frameworkInfo[+process.env.FRAMEWORK_ID!]);
+}