chore: add `pnpm bench:compare` script (#12099)

* WIP compare script

* WIP

* fix

* add script

* update

* fix

* tweak

* fix

* update

* add TODO

* compare

* tmp

* gah

* revert

* fix

* fix

* prevent div/0

* reenable benchmarks

---------

Co-authored-by: Dominic Gannaway <dg@domgan.com>

.gitignore

@@ -22,3 +22,5 @@ coverage
 .DS_Store
 
 tmp
+
+benchmarking/compare/.results

benchmarking/benchmarks.js

@@ -0,0 +1,24 @@
+import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js';
+import { kairo_broad } from './benchmarks/kairo/kairo_broad.js';
+import { kairo_deep } from './benchmarks/kairo/kairo_deep.js';
+import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js';
+import { kairo_mux } from './benchmarks/kairo/kairo_mux.js';
+import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js';
+import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js';
+import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js';
+import { mol_bench } from './benchmarks/mol_bench.js';
+
+// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark)
+// Not all tests are the same, and many parts have been tweaked to capture different data.
+
+export const benchmarks = [
+	kairo_avoidable,
+	kairo_broad,
+	kairo_deep,
+	kairo_diamond,
+	kairo_triangle,
+	kairo_mux,
+	kairo_repeated,
+	kairo_unstable,
+	mol_bench
+];

benchmarking/compare/index.js

@@ -0,0 +1,90 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { execSync, fork } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { benchmarks } from '../benchmarks.js';
+
+// if (execSync('git status --porcelain').toString().trim()) {
+// 	console.error('Working directory is not clean');
+// 	process.exit(1);
+// }
+
+const filename = fileURLToPath(import.meta.url);
+const runner = path.resolve(filename, '../runner.js');
+const outdir = path.resolve(filename, '../.results');
+
+if (fs.existsSync(outdir)) fs.rmSync(outdir, { recursive: true });
+fs.mkdirSync(outdir);
+
+const branches = [];
+
+for (const arg of process.argv.slice(2)) {
+	if (arg.startsWith('--')) continue;
+	if (arg === filename) continue;
+
+	branches.push(arg);
+}
+
+if (branches.length === 0) {
+	branches.push(
+		execSync('git symbolic-ref --short -q HEAD || git rev-parse --short HEAD').toString().trim()
+	);
+}
+
+if (branches.length === 1) {
+	branches.push('main');
+}
+
+process.on('exit', () => {
+	execSync(`git checkout ${branches[0]}`);
+});
+
+for (const branch of branches) {
+	console.group(`Benchmarking ${branch}`);
+
+	execSync(`git checkout ${branch}`);
+
+	await new Promise((fulfil, reject) => {
+		const child = fork(runner);
+
+		child.on('message', (results) => {
+			fs.writeFileSync(`${outdir}/${branch}.json`, JSON.stringify(results, null, '  '));
+			fulfil();
+		});
+
+		child.on('error', reject);
+	});
+
+	console.groupEnd();
+}
+
+const results = branches.map((branch) => {
+	return JSON.parse(fs.readFileSync(`${outdir}/${branch}.json`, 'utf-8'));
+});
+
+for (let i = 0; i < results[0].length; i += 1) {
+	console.group(`${results[0][i].benchmark}`);
+
+	for (const metric of ['time', 'gc_time']) {
+		const times = results.map((result) => +result[i][metric]);
+		let min = Infinity;
+		let min_index = -1;
+
+		for (let b = 0; b < times.length; b += 1) {
+			if (times[b] < min) {
+				min = times[b];
+				min_index = b;
+			}
+		}
+
+		if (min !== 0) {
+			console.group(`${metric}: fastest is ${branches[min_index]}`);
+			times.forEach((time, b) => {
+				console.log(`${branches[b]}: ${time.toFixed(2)}ms (${((time / min) * 100).toFixed(2)}%)`);
+			});
+			console.groupEnd();
+		}
+	}
+
+	console.groupEnd();
+}

benchmarking/compare/runner.js

@@ -0,0 +1,10 @@
+import { benchmarks } from '../benchmarks.js';
+
+const results = [];
+for (const benchmark of benchmarks) {
+	const result = await benchmark();
+	console.error(result.benchmark);
+	results.push(result);
+}
+
+process.send(results);

benchmarking/run.js

@@ -1,58 +1,32 @@
 import * as $ from '../packages/svelte/src/internal/client/index.js';
-import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js';
-import { kairo_broad } from './benchmarks/kairo/kairo_broad.js';
-import { kairo_deep } from './benchmarks/kairo/kairo_deep.js';
-import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js';
-import { kairo_mux } from './benchmarks/kairo/kairo_mux.js';
-import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js';
-import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js';
-import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js';
-import { mol_bench } from './benchmarks/mol_bench.js';
+import { benchmarks } from './benchmarks.js';
 
-// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark)
-// Not all tests are the same, and many parts have been tweaked to capture different data.
+let total_time = 0;
+let total_gc_time = 0;
 
-const benchmarks = [
-	kairo_avoidable,
-	kairo_broad,
-	kairo_deep,
-	kairo_diamond,
-	kairo_triangle,
-	kairo_mux,
-	kairo_repeated,
-	kairo_unstable,
-	mol_bench
-];
-
-async function run_benchmarks() {
-	let total_time = 0;
-	let total_gc_time = 0;
-	// eslint-disable-next-line no-console
-	console.log('-- Benchmarking Started --');
-	$.push({}, true);
-	try {
-		for (const benchmark of benchmarks) {
-			const results = await benchmark();
-			// eslint-disable-next-line no-console
-			console.log(results);
-			total_time += Number(results.time);
-			total_gc_time += Number(results.gc_time);
-		}
-	} catch (e) {
+// eslint-disable-next-line no-console
+console.log('-- Benchmarking Started --');
+$.push({}, true);
+try {
+	for (const benchmark of benchmarks) {
+		const results = await benchmark();
 		// eslint-disable-next-line no-console
-		console.error('-- Benchmarking Failed --');
-		// eslint-disable-next-line no-console
-		console.error(e);
-		process.exit(1);
+		console.log(results);
+		total_time += Number(results.time);
+		total_gc_time += Number(results.gc_time);
 	}
-	$.pop();
+} catch (e) {
 	// eslint-disable-next-line no-console
-	console.log(`-- Benchmarking Complete --`);
+	console.error('-- Benchmarking Failed --');
 	// eslint-disable-next-line no-console
-	console.log({
-		total_time: total_time.toFixed(2),
-		total_gc_time: total_gc_time.toFixed(2)
-	});
+	console.error(e);
+	process.exit(1);
 }
-
-run_benchmarks();
+$.pop();
+// eslint-disable-next-line no-console
+console.log('-- Benchmarking Complete --');
+// eslint-disable-next-line no-console
+console.log({
+	total_time: total_time.toFixed(2),
+	total_gc_time: total_gc_time.toFixed(2)
+});

package.json

@@ -25,6 +25,7 @@
     "changeset:version": "changeset version && pnpm -r generate:version && git add --all",
     "changeset:publish": "changeset publish",
     "bench": "node --allow-natives-syntax ./benchmarking/run.js",
+    "bench:compare": "node --allow-natives-syntax ./benchmarking/compare/index.js",
     "bench:debug": "node --allow-natives-syntax --inspect-brk ./benchmarking/run.js"
   },
   "devDependencies": {