compare

benchmarking/compare/index.js

@@ -1,15 +1,16 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import { execSync } from 'node:child_process';
+import { execSync, fork } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
 import { benchmarks } from '../benchmarks.js';
 
-if (execSync('git status --porcelain').toString().trim()) {
-	console.error('Working directory is not clean');
-	process.exit(1);
-}
+// if (execSync('git status --porcelain').toString().trim()) {
+// 	console.error('Working directory is not clean');
+// 	process.exit(1);
+// }
 
 const filename = fileURLToPath(import.meta.url);
+const runner = path.resolve(filename, '../runner.js');
 const outdir = path.resolve(filename, '../.results');
 
 if (fs.existsSync(outdir)) fs.rmSync(outdir, { recursive: true });
@@ -34,16 +35,73 @@ if (branches.length === 1) {
 	branches.push('main');
 }
 
-// TODO run the benchmarks in a child process
+process.on('exit', () => {
+	execSync(`git checkout ${branches[0]}`);
+});
+
 for (const branch of branches) {
-	const results = [];
-	for (const benchmark of benchmarks) {
-		const result = await benchmark();
-		console.log(result.benchmark);
-		results.push(result);
-	}
+	console.group(`Benchmarking ${branch}`);
 
+	execSync(`git checkout ${branch}`);
+
+	await new Promise((fulfil, reject) => {
+		const child = fork(runner);
+
+		child.on('message', (results) => {
 			fs.writeFileSync(`${outdir}/${branch}.json`, JSON.stringify(results, null, '  '));
+			fulfil();
+		});
+
+		child.on('error', reject);
+	});
+
+	console.groupEnd();
 }
 
-// TODO compare the results
+const results = branches.map((branch) => {
+	return JSON.parse(fs.readFileSync(`${outdir}/${branch}.json`, 'utf-8'));
+});
+
+for (let i = 0; i < results[0].length; i += 1) {
+	console.group(`${results[0][i].benchmark}`);
+
+	// time
+	{
+		const times = results.map((result) => result[i].time);
+		let min = Infinity;
+		let min_index = -1;
+
+		for (let b = 0; b < times.length; b += 1) {
+			if (times[b] < min) {
+				min = times[b];
+				min_index = b;
+			}
+		}
+
+		console.log(`fastest is ${branches[min_index]}`);
+		times.forEach((time, b) => {
+			console.log(`${branches[b]}: ${time.toFixed(2)}ms (${((time / min) * 100).toFixed(2)}%)`);
+		});
+	}
+
+	// gc_time
+	{
+		const gc_times = results.map((result) => result[i].gc_time);
+		let min = Infinity;
+		let min_index = -1;
+
+		for (let b = 0; b < gc_times.length; b += 1) {
+			if (gc_times[b] < min) {
+				min = gc_times[b];
+				min_index = b;
+			}
+		}
+
+		console.log(`fastest is ${branches[min_index]}`);
+		gc_times.forEach((time, b) => {
+			console.log(`${branches[b]}: ${time.toFixed(2)}ms (${((time / min) * 100).toFixed(2)}%)`);
+		});
+	}
+
+	console.groupEnd();
+}

benchmarking/compare/runner.js

@@ -0,0 +1,10 @@
+import { benchmarks } from '../benchmarks.js';
+
+const results = [];
+for (const benchmark of benchmarks) {
+	const result = await benchmark();
+	console.error(result.benchmark);
+	results.push(result);
+}
+
+process.send(results);