chore: run every benchmark in its own process (#18542)

While benchmarking it became apparent that when running the full `pnpm
bench:compare` suite results could be wildly different compared to only
comparing single benchmarks. The reason (most likely) is that the
heap/GC/JIT state from one benchmark contaminates the others.

Therefore this adjusts the runners such that each benchmark entry is run
in its own forked process, not just each branch. This should give
better, more stable results.

Also a little fix to compare results by name; I made the mistake of
adding a new benchmark to main which wasn't present on other branches
yet, which made the results really confusing.

benchmarking/compare/generate-report.js

@@ -33,11 +33,38 @@ export function generate_report(outdir) {
 
 	write('');
 
-	for (let i = 0; i < results[0].length; i += 1) {
+	// match results by benchmark name — branches may have different benchmark
-		write(`${results[0][i].benchmark}`);
+	// lists (e.g. a benchmark that only exists on one of the branches), so
+	// pairing by array index would misattribute results
+	const by_name = results.map((result) => new Map(result.map((r) => [r.benchmark, r])));
+
+	/** @type {string[]} */
+	const names = [];
+
+	for (const result of results) {
+		for (const { benchmark } of result) {
+			if (!names.includes(benchmark)) {
+				names.push(benchmark);
+			}
+		}
+	}
+
+	for (const name of names) {
+		const entries = by_name.map((map) => map.get(name));
+		const missing = entries
+			.map((entry, b) => (entry === undefined ? branches[b] : null))
+			.filter((branch) => branch !== null);
+
+		write(`${name}`);
+
+		if (missing.length > 0) {
+			write(`  skipped (missing on ${missing.join(', ')})`);
+			write('');
+			continue;
+		}
 
 		for (const metric of ['time', 'gc_time']) {
-			const times = results.map((result) => +result[i][metric]);
+			const times = entries.map((entry) => +entry[metric]);
 			let min = Infinity;
 			let max = -Infinity;
 			let min_index = -1;

benchmarking/compare/runner.js

@@ -1,18 +1,67 @@
+import { fork } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
 import { reactivity_benchmarks } from '../benchmarks/reactivity/index.js';
 import { with_cpu_profile } from '../utils.js';
 
-const results = [];
+const PROFILE_DIR = process.env.BENCH_PROFILE_DIR ?? null;
-const PROFILE_DIR = process.env.BENCH_PROFILE_DIR;
+const single = process.env.BENCH_SINGLE;
 
-for (let i = 0; i < reactivity_benchmarks.length; i += 1) {
+if (single) {
+	// child mode — run a single benchmark and report the result to the parent
+	const benchmark = reactivity_benchmarks.find((b) => b.label === single);
+
+	if (!benchmark) {
+		throw new Error(`Unknown benchmark ${single}`);
+	}
+
+	const result = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn());
+
+	// exit via the callback so the message is guaranteed to be delivered
+	/** @type {NodeJS.Process} */ (process).send(result, () => process.exit(0));
+} else {
+	// parent mode — run every benchmark in its own child process, so that
+	// heap/GC/JIT state from one benchmark cannot contaminate the others
+	const filename = fileURLToPath(import.meta.url);
+	const results = [];
+
+	for (let i = 0; i < reactivity_benchmarks.length; i += 1) {
 		const benchmark = reactivity_benchmarks[i];
 
 		process.stderr.write(`Running ${i + 1}/${reactivity_benchmarks.length} ${benchmark.label} `);
+
+		const result = await new Promise((fulfil, reject) => {
+			const child = fork(filename, [], {
+				env: {
+					...process.env,
+					BENCH_SINGLE: benchmark.label
+				}
+			});
+
+			/** @type {object | null} */
+			let message_received = null;
+
+			child.on('message', (message) => {
+				message_received = /** @type {object} */ (message);
+			});
+
+			child.on('error', reject);
+
+			child.on('exit', (code) => {
+				if (message_received === null) {
+					reject(new Error(`benchmark ${benchmark.label} exited with code ${code}`));
+				} else {
+					fulfil(message_received);
+				}
+			});
+		});
+
 		results.push({
 			benchmark: benchmark.label,
-		...(await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn()))
+			.../** @type {object} */ (result)
 		});
+
 		process.stderr.write('\x1b[2K\r');
-}
+	}
 
-process.send(results);
+	/** @type {NodeJS.Process} */ (process).send(results);
+}

benchmarking/run.js

@@ -1,14 +1,38 @@
+import { fork } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
 import * as $ from '../packages/svelte/src/internal/client/index.js';
 import { reactivity_benchmarks } from './benchmarks/reactivity/index.js';
 import { ssr_benchmarks } from './benchmarks/ssr/index.js';
 import { with_cpu_profile } from './utils.js';
 
-// e.g. `pnpm bench kairo` to only run the kairo benchmarks
-const filters = process.argv.slice(2);
-
 const PROFILE_DIR = './benchmarking/.profiles';
 
-const suites = [
+const single = process.env.BENCH_SINGLE;
+
+if (single) {
+	// child mode — run a single benchmark and report the result to the parent
+	const benchmark = [...reactivity_benchmarks, ...ssr_benchmarks].find((b) => b.label === single);
+
+	if (!benchmark) {
+		throw new Error(`Unknown benchmark ${single}`);
+	}
+
+	$.push({}, true);
+
+	const result = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn());
+
+	$.pop();
+
+	// exit via the callback so the message is guaranteed to be delivered
+	/** @type {NodeJS.Process} */ (process).send(result, () => process.exit(0));
+} else {
+	// parent mode — run every benchmark in its own child process, so that
+	// heap/GC/JIT state from one benchmark cannot contaminate the others
+
+	// e.g. `pnpm bench kairo` to only run the kairo benchmarks
+	const filters = process.argv.slice(2);
+
+	const suites = [
 		{
 			benchmarks: reactivity_benchmarks.filter(
 				(b) => filters.length === 0 || filters.some((f) => b.label.includes(f))
@@ -21,25 +45,59 @@ const suites = [
 			),
 			name: 'server-side rendering benchmarks'
 		}
-].filter((suite) => suite.benchmarks.length > 0);
+	].filter((suite) => suite.benchmarks.length > 0);
 
-if (suites.length === 0) {
+	if (suites.length === 0) {
 		console.log('No benchmarks matched provided filters');
 		process.exit(1);
-}
+	}
 
-const COLUMN_WIDTHS = [25, 9, 9];
+	const filename = fileURLToPath(import.meta.url);
-const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b);
+
+	/**
+	 * @param {string} label
+	 * @returns {Promise<{ time: number, gc_time: number }>}
+	 */
+	const run_benchmark = (label) => {
+		return new Promise((fulfil, reject) => {
+			const child = fork(filename, [], {
+				env: {
+					...process.env,
+					BENCH_SINGLE: label
+				}
+			});
 
-const pad_right = (str, n) => str + ' '.repeat(n - str.length);
+			/** @type {{ time: number, gc_time: number } | null} */
-const pad_left = (str, n) => ' '.repeat(n - str.length) + str;
+			let result = null;
 
-let total_time = 0;
+			child.on('message', (message) => {
-let total_gc_time = 0;
+				result = /** @type {{ time: number, gc_time: number }} */ (message);
+			});
 
-$.push({}, true);
+			child.on('error', reject);
 
-try {
+			child.on('exit', (code) => {
+				if (result === null) {
+					reject(new Error(`benchmark ${label} exited with code ${code}`));
+				} else {
+					fulfil(result);
+				}
+			});
+		});
+	};
+
+	const COLUMN_WIDTHS = [25, 9, 9];
+	const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b);
+
+	/** @type {(str: string, n: number) => string} */
+	const pad_right = (str, n) => str + ' '.repeat(n - str.length);
+	/** @type {(str: string, n: number) => string} */
+	const pad_left = (str, n) => ' '.repeat(n - str.length) + str;
+
+	let total_time = 0;
+	let total_gc_time = 0;
+
+	try {
 		for (const { benchmarks, name } of suites) {
 			let suite_time = 0;
 			let suite_gc_time = 0;
@@ -53,7 +111,7 @@ try {
 			console.log('='.repeat(TOTAL_WIDTH));
 
 			for (const benchmark of benchmarks) {
-			const results = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn());
+				const results = await run_benchmark(benchmark.label);
 				console.log(
 					pad_right(benchmark.label, COLUMN_WIDTHS[0]) +
 						pad_left(results.time.toFixed(2), COLUMN_WIDTHS[1]) +
@@ -74,21 +132,18 @@ try {
 			console.log('='.repeat(TOTAL_WIDTH));
 		}
 
-	if (PROFILE_DIR !== null) {
 		console.log(`\nCPU profiles written to ${PROFILE_DIR}`);
-	}
+	} catch (e) {
-} catch (e) {
 		// eslint-disable-next-line no-console
 		console.error(e);
 		process.exit(1);
-}
+	}
-
-$.pop();
 
-console.log('');
+	console.log('');
 
-console.log(
+	console.log(
 		pad_right('total', COLUMN_WIDTHS[0]) +
 			pad_left(total_time.toFixed(2), COLUMN_WIDTHS[1]) +
 			pad_left(total_gc_time.toFixed(2), COLUMN_WIDTHS[2])
-);
+	);
+}