diff --git a/benchmarking/compare/index.js b/benchmarking/compare/index.js index 5b0259d633..8b174dea04 100644 --- a/benchmarking/compare/index.js +++ b/benchmarking/compare/index.js @@ -1,15 +1,16 @@ import fs from 'node:fs'; import path from 'node:path'; -import { execSync } from 'node:child_process'; +import { execSync, fork } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { benchmarks } from '../benchmarks.js'; -if (execSync('git status --porcelain').toString().trim()) { - console.error('Working directory is not clean'); - process.exit(1); -} +// if (execSync('git status --porcelain').toString().trim()) { +// console.error('Working directory is not clean'); +// process.exit(1); +// } const filename = fileURLToPath(import.meta.url); +const runner = path.resolve(filename, '../runner.js'); const outdir = path.resolve(filename, '../.results'); if (fs.existsSync(outdir)) fs.rmSync(outdir, { recursive: true }); @@ -34,16 +35,73 @@ if (branches.length === 1) { branches.push('main'); } -// TODO run the benchmarks in a child process +process.on('exit', () => { + execSync(`git checkout ${branches[0]}`); +}); + for (const branch of branches) { - const results = []; - for (const benchmark of benchmarks) { - const result = await benchmark(); - console.log(result.benchmark); - results.push(result); - } + console.group(`Benchmarking ${branch}`); + + execSync(`git checkout ${branch}`); + + await new Promise((fulfil, reject) => { + const child = fork(runner); - fs.writeFileSync(`${outdir}/${branch}.json`, JSON.stringify(results, null, ' ')); + child.on('message', (results) => { + fs.writeFileSync(`${outdir}/${branch}.json`, JSON.stringify(results, null, ' ')); + fulfil(); + }); + + child.on('error', reject); + }); + + console.groupEnd(); } -// TODO compare the results +const results = branches.map((branch) => { + return JSON.parse(fs.readFileSync(`${outdir}/${branch}.json`, 'utf-8')); +}); + +for (let i = 0; i < results[0].length; i += 1) { + console.group(`${results[0][i].benchmark}`); + + // time + { + const times = results.map((result) => result[i].time); + let min = Infinity; + let min_index = -1; + + for (let b = 0; b < times.length; b += 1) { + if (times[b] < min) { + min = times[b]; + min_index = b; + } + } + + console.log(`fastest is ${branches[min_index]}`); + times.forEach((time, b) => { + console.log(`${branches[b]}: ${time.toFixed(2)}ms (${((time / min) * 100).toFixed(2)}%)`); + }); + } + + // gc_time + { + const gc_times = results.map((result) => result[i].gc_time); + let min = Infinity; + let min_index = -1; + + for (let b = 0; b < gc_times.length; b += 1) { + if (gc_times[b] < min) { + min = gc_times[b]; + min_index = b; + } + } + + console.log(`fastest is ${branches[min_index]}`); + gc_times.forEach((time, b) => { + console.log(`${branches[b]}: ${time.toFixed(2)}ms (${((time / min) * 100).toFixed(2)}%)`); + }); + } + + console.groupEnd(); +} diff --git a/benchmarking/compare/runner.js b/benchmarking/compare/runner.js new file mode 100644 index 0000000000..6fa58e2bac --- /dev/null +++ b/benchmarking/compare/runner.js @@ -0,0 +1,10 @@ +import { benchmarks } from '../benchmarks.js'; + +const results = []; +for (const benchmark of benchmarks) { + const result = await benchmark(); + console.error(result.benchmark); + results.push(result); +} + +process.send(results);