chore: skippable benchmarks (#17506)

* chore: add benchmark filtering mechanism

* tweak

* fix

* try this

* better logging for bench:compare

* fix

benchmarking/benchmarks/reactivity/sbench.js

@@ -26,31 +26,28 @@ function create_derived(source) {
 /**
  *
  * @param {string} label
- * @param {(n: number, sources: Array<Source<number>>)} fn
+ * @param {(n: number, sources: Array<Source<number>>) => void} fn
  * @param {number} count
  * @param {number} num_sources
  */
 function create_sbench_test(label, count, num_sources, fn) {
-	return async () => {
-		// Do 3 loops to warm up JIT
-		for (let i = 0; i < 3; i++) {
-			fn(count, create_sources(num_sources, []));
-		}
+	return {
+		label,
+		fn: async () => {
+			// Do 3 loops to warm up JIT
+			for (let i = 0; i < 3; i++) {
+				fn(count, create_sources(num_sources, []));
+			}
 
-		const { time, gc_time } = await fastest_test(10, () => {
-			const destroy = $.effect_root(() => {
-				for (let i = 0; i < 10; i++) {
-					fn(count, create_sources(num_sources, []));
-				}
+			return await fastest_test(10, () => {
+				const destroy = $.effect_root(() => {
+					for (let i = 0; i < 10; i++) {
+						fn(count, create_sources(num_sources, []));
+					}
+				});
+				destroy();
 			});
-			destroy();
-		});
-
-		return {
-			benchmark: label,
-			time,
-			gc_time
-		};
+		}
 	};
 }
 

benchmarking/benchmarks/reactivity/util.js

@@ -15,34 +15,9 @@ export function busy() {
  */
 export function create_test(label, setup) {
 	return {
-		unowned: async () => {
-			// Do 10 loops to warm up JIT
-			for (let i = 0; i < 10; i++) {
-				const { run, destroy } = setup();
-				run(0);
-				destroy();
-			}
-
-			const { run, destroy } = setup();
-
-			const { time, gc_time } = await fastest_test(10, () => {
-				for (let i = 0; i < 1000; i++) {
-					run(i);
-				}
-			});
-
-			destroy();
-
-			return {
-				benchmark: `${label}_unowned`,
-				time,
-				gc_time
-			};
-		},
-		owned: async () => {
-			let run, destroy;
-
-			const destroy_owned = $.effect_root(() => {
+		unowned: {
+			label: `${label}_unowned`,
+			fn: async () => {
 				// Do 10 loops to warm up JIT
 				for (let i = 0; i < 10; i++) {
 					const { run, destroy } = setup();
@@ -50,24 +25,47 @@ export function create_test(label, setup) {
 					destroy();
 				}
 
-				({ run, destroy } = setup());
-			});
+				const { run, destroy } = setup();
+
+				const result = await fastest_test(10, () => {
+					for (let i = 0; i < 1000; i++) {
+						run(i);
+					}
+				});
 
-			const { time, gc_time } = await fastest_test(10, () => {
-				for (let i = 0; i < 1000; i++) {
-					run(i);
-				}
-			});
+				destroy();
+
+				return result;
+			}
+		},
+		owned: {
+			label: `${label}_owned`,
+			fn: async () => {
+				let run, destroy;
 
-			// @ts-ignore
-			destroy();
-			destroy_owned();
+				const destroy_owned = $.effect_root(() => {
+					// Do 10 loops to warm up JIT
+					for (let i = 0; i < 10; i++) {
+						const { run, destroy } = setup();
+						run(0);
+						destroy();
+					}
 
-			return {
-				benchmark: `${label}_owned`,
-				time,
-				gc_time
-			};
+					({ run, destroy } = setup());
+				});
+
+				const result = await fastest_test(10, () => {
+					for (let i = 0; i < 1000; i++) {
+						run(i);
+					}
+				});
+
+				// @ts-ignore
+				destroy();
+				destroy_owned();
+
+				return result;
+			}
 		}
 	};
 }

benchmarking/benchmarks/ssr/wrapper/wrapper_bench.js

@@ -18,26 +18,23 @@ async function compile_svelte() {
 	return module.default;
 }
 
-export async function wrapper_bench() {
-	const App = await compile_svelte();
+export const wrapper_bench = {
+	label: 'wrapper_bench',
+	fn: async () => {
+		const App = await compile_svelte();
 
-	// Do 3 loops to warm up JIT
-	for (let i = 0; i < 3; i++) {
-		render(App);
-	}
-
-	const { time, gc_time } = await fastest_test(10, () => {
-		for (let i = 0; i < 100; i++) {
+		// Do 3 loops to warm up JIT
+		for (let i = 0; i < 3; i++) {
 			render(App);
 		}
-	});
 
-	return {
-		benchmark: 'wrapper_bench',
-		time,
-		gc_time
-	};
-}
+		return await fastest_test(10, () => {
+			for (let i = 0; i < 100; i++) {
+				render(App);
+			}
+		});
+	}
+};
 
 /**
  * @param {string} file

benchmarking/compare/runner.js

@@ -1,10 +1,13 @@
 import { reactivity_benchmarks } from '../benchmarks/reactivity/index.js';
 
 const results = [];
-for (const benchmark of reactivity_benchmarks) {
-	const result = await benchmark();
-	console.error(result.benchmark);
-	results.push(result);
+
+for (let i = 0; i < reactivity_benchmarks.length; i += 1) {
+	const benchmark = reactivity_benchmarks[i];
+
+	process.stderr.write(`Running ${i + 1}/${reactivity_benchmarks.length} ${benchmark.label} `);
+	results.push({ benchmark: benchmark.label, ...(await benchmark.fn()) });
+	process.stderr.write('\x1b[2K\r');
 }
 
 process.send(results);

benchmarking/run.js

@@ -2,13 +2,28 @@ import * as $ from '../packages/svelte/src/internal/client/index.js';
 import { reactivity_benchmarks } from './benchmarks/reactivity/index.js';
 import { ssr_benchmarks } from './benchmarks/ssr/index.js';
 
-let total_time = 0;
-let total_gc_time = 0;
+// e.g. `pnpm bench kairo` to only run the kairo benchmarks
+const filters = process.argv.slice(2);
 
 const suites = [
-	{ benchmarks: reactivity_benchmarks, name: 'reactivity benchmarks' },
-	{ benchmarks: ssr_benchmarks, name: 'server-side rendering benchmarks' }
-];
+	{
+		benchmarks: reactivity_benchmarks.filter(
+			(b) => filters.length === 0 || filters.some((f) => b.label.includes(f))
+		),
+		name: 'reactivity benchmarks'
+	},
+	{
+		benchmarks: ssr_benchmarks.filter(
+			(b) => filters.length === 0 || filters.some((f) => b.label.includes(f))
+		),
+		name: 'server-side rendering benchmarks'
+	}
+].filter((suite) => suite.benchmarks.length > 0);
+
+if (suites.length === 0) {
+	console.log('No benchmarks matched provided filters');
+	process.exit(1);
+}
 
 const COLUMN_WIDTHS = [25, 9, 9];
 const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b);
@@ -16,6 +31,9 @@ const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b);
 const pad_right = (str, n) => str + ' '.repeat(n - str.length);
 const pad_left = (str, n) => ' '.repeat(n - str.length) + str;
 
+let total_time = 0;
+let total_gc_time = 0;
+
 $.push({}, true);
 
 try {
@@ -32,9 +50,9 @@ try {
 		console.log('='.repeat(TOTAL_WIDTH));
 
 		for (const benchmark of benchmarks) {
-			const results = await benchmark();
+			const results = await benchmark.fn();
 			console.log(
-				pad_right(results.benchmark, COLUMN_WIDTHS[0]) +
+				pad_right(benchmark.label, COLUMN_WIDTHS[0]) +
 					pad_left(results.time.toFixed(2), COLUMN_WIDTHS[1]) +
 					pad_left(results.gc_time.toFixed(2), COLUMN_WIDTHS[2])
 			);

package.json

@@ -22,8 +22,8 @@
     "changeset:version": "changeset version && pnpm -r generate:version && git add --all",
     "changeset:publish": "changeset publish",
     "bench": "NODE_ENV=production node --allow-natives-syntax ./benchmarking/run.js",
-    "bench:compare": "node --allow-natives-syntax ./benchmarking/compare/index.js",
-    "bench:debug": "node --allow-natives-syntax --inspect-brk ./benchmarking/run.js"
+    "bench:compare": "NODE_ENV=production node --allow-natives-syntax ./benchmarking/compare/index.js",
+    "bench:debug": "NODE_ENV=production node --allow-natives-syntax --inspect-brk ./benchmarking/run.js"
   },
   "devDependencies": {
     "@changesets/cli": "^2.29.8",