From 331925356e144491ef03baba2cec7fc1ffac9005 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Thu, 20 Jun 2024 02:05:11 -0700 Subject: [PATCH] chore: add `pnpm bench:compare` script (#12099) * WIP compare script * WIP * fix * add script * update * fix * tweak * fix * update * add TODO * compare * tmp * gah * revert * fix * fix * prevent div/0 * reenable benchmarks --------- Co-authored-by: Dominic Gannaway --- .gitignore | 2 + benchmarking/benchmarks.js | 24 +++++++++ benchmarking/compare/index.js | 90 ++++++++++++++++++++++++++++++++++ benchmarking/compare/runner.js | 10 ++++ benchmarking/run.js | 74 +++++++++------------------- package.json | 1 + 6 files changed, 151 insertions(+), 50 deletions(-) create mode 100644 benchmarking/benchmarks.js create mode 100644 benchmarking/compare/index.js create mode 100644 benchmarking/compare/runner.js diff --git a/.gitignore b/.gitignore index 4ce5c243c7..d503437664 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ coverage .DS_Store tmp + +benchmarking/compare/.results diff --git a/benchmarking/benchmarks.js b/benchmarking/benchmarks.js new file mode 100644 index 0000000000..283112dbe3 --- /dev/null +++ b/benchmarking/benchmarks.js @@ -0,0 +1,24 @@ +import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js'; +import { kairo_broad } from './benchmarks/kairo/kairo_broad.js'; +import { kairo_deep } from './benchmarks/kairo/kairo_deep.js'; +import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js'; +import { kairo_mux } from './benchmarks/kairo/kairo_mux.js'; +import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js'; +import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js'; +import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js'; +import { mol_bench } from './benchmarks/mol_bench.js'; + +// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark) +// Not all tests are the same, and many parts have been tweaked to capture different data. + +export const benchmarks = [ + kairo_avoidable, + kairo_broad, + kairo_deep, + kairo_diamond, + kairo_triangle, + kairo_mux, + kairo_repeated, + kairo_unstable, + mol_bench +]; diff --git a/benchmarking/compare/index.js b/benchmarking/compare/index.js new file mode 100644 index 0000000000..a5fc6d10a9 --- /dev/null +++ b/benchmarking/compare/index.js @@ -0,0 +1,90 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { execSync, fork } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { benchmarks } from '../benchmarks.js'; + +// if (execSync('git status --porcelain').toString().trim()) { +// console.error('Working directory is not clean'); +// process.exit(1); +// } + +const filename = fileURLToPath(import.meta.url); +const runner = path.resolve(filename, '../runner.js'); +const outdir = path.resolve(filename, '../.results'); + +if (fs.existsSync(outdir)) fs.rmSync(outdir, { recursive: true }); +fs.mkdirSync(outdir); + +const branches = []; + +for (const arg of process.argv.slice(2)) { + if (arg.startsWith('--')) continue; + if (arg === filename) continue; + + branches.push(arg); +} + +if (branches.length === 0) { + branches.push( + execSync('git symbolic-ref --short -q HEAD || git rev-parse --short HEAD').toString().trim() + ); +} + +if (branches.length === 1) { + branches.push('main'); +} + +process.on('exit', () => { + execSync(`git checkout ${branches[0]}`); +}); + +for (const branch of branches) { + console.group(`Benchmarking ${branch}`); + + execSync(`git checkout ${branch}`); + + await new Promise((fulfil, reject) => { + const child = fork(runner); + + child.on('message', (results) => { + fs.writeFileSync(`${outdir}/${branch}.json`, JSON.stringify(results, null, ' ')); + fulfil(); + }); + + child.on('error', reject); + }); + + console.groupEnd(); +} + +const results = branches.map((branch) => { + return JSON.parse(fs.readFileSync(`${outdir}/${branch}.json`, 'utf-8')); +}); + +for (let i = 0; i < results[0].length; i += 1) { + console.group(`${results[0][i].benchmark}`); + + for (const metric of ['time', 'gc_time']) { + const times = results.map((result) => +result[i][metric]); + let min = Infinity; + let min_index = -1; + + for (let b = 0; b < times.length; b += 1) { + if (times[b] < min) { + min = times[b]; + min_index = b; + } + } + + if (min !== 0) { + console.group(`${metric}: fastest is ${branches[min_index]}`); + times.forEach((time, b) => { + console.log(`${branches[b]}: ${time.toFixed(2)}ms (${((time / min) * 100).toFixed(2)}%)`); + }); + console.groupEnd(); + } + } + + console.groupEnd(); +} diff --git a/benchmarking/compare/runner.js b/benchmarking/compare/runner.js new file mode 100644 index 0000000000..6fa58e2bac --- /dev/null +++ b/benchmarking/compare/runner.js @@ -0,0 +1,10 @@ +import { benchmarks } from '../benchmarks.js'; + +const results = []; +for (const benchmark of benchmarks) { + const result = await benchmark(); + console.error(result.benchmark); + results.push(result); +} + +process.send(results); diff --git a/benchmarking/run.js b/benchmarking/run.js index ca1f5eb683..35a6ac307f 100644 --- a/benchmarking/run.js +++ b/benchmarking/run.js @@ -1,58 +1,32 @@ import * as $ from '../packages/svelte/src/internal/client/index.js'; -import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js'; -import { kairo_broad } from './benchmarks/kairo/kairo_broad.js'; -import { kairo_deep } from './benchmarks/kairo/kairo_deep.js'; -import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js'; -import { kairo_mux } from './benchmarks/kairo/kairo_mux.js'; -import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js'; -import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js'; -import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js'; -import { mol_bench } from './benchmarks/mol_bench.js'; +import { benchmarks } from './benchmarks.js'; -// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark) -// Not all tests are the same, and many parts have been tweaked to capture different data. +let total_time = 0; +let total_gc_time = 0; -const benchmarks = [ - kairo_avoidable, - kairo_broad, - kairo_deep, - kairo_diamond, - kairo_triangle, - kairo_mux, - kairo_repeated, - kairo_unstable, - mol_bench -]; - -async function run_benchmarks() { - let total_time = 0; - let total_gc_time = 0; - // eslint-disable-next-line no-console - console.log('-- Benchmarking Started --'); - $.push({}, true); - try { - for (const benchmark of benchmarks) { - const results = await benchmark(); - // eslint-disable-next-line no-console - console.log(results); - total_time += Number(results.time); - total_gc_time += Number(results.gc_time); - } - } catch (e) { +// eslint-disable-next-line no-console +console.log('-- Benchmarking Started --'); +$.push({}, true); +try { + for (const benchmark of benchmarks) { + const results = await benchmark(); // eslint-disable-next-line no-console - console.error('-- Benchmarking Failed --'); - // eslint-disable-next-line no-console - console.error(e); - process.exit(1); + console.log(results); + total_time += Number(results.time); + total_gc_time += Number(results.gc_time); } - $.pop(); +} catch (e) { // eslint-disable-next-line no-console - console.log(`-- Benchmarking Complete --`); + console.error('-- Benchmarking Failed --'); // eslint-disable-next-line no-console - console.log({ - total_time: total_time.toFixed(2), - total_gc_time: total_gc_time.toFixed(2) - }); + console.error(e); + process.exit(1); } - -run_benchmarks(); +$.pop(); +// eslint-disable-next-line no-console +console.log('-- Benchmarking Complete --'); +// eslint-disable-next-line no-console +console.log({ + total_time: total_time.toFixed(2), + total_gc_time: total_gc_time.toFixed(2) +}); diff --git a/package.json b/package.json index 27b844bc39..ee8dc4f836 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "changeset:version": "changeset version && pnpm -r generate:version && git add --all", "changeset:publish": "changeset publish", "bench": "node --allow-natives-syntax ./benchmarking/run.js", + "bench:compare": "node --allow-natives-syntax ./benchmarking/compare/index.js", "bench:debug": "node --allow-natives-syntax --inspect-brk ./benchmarking/run.js" }, "devDependencies": {