diff --git a/benchmarking/compare/generate-report.js b/benchmarking/compare/generate-report.js index a61f58909b..a06aaf9fc2 100644 --- a/benchmarking/compare/generate-report.js +++ b/benchmarking/compare/generate-report.js @@ -33,11 +33,38 @@ export function generate_report(outdir) { write(''); - for (let i = 0; i < results[0].length; i += 1) { - write(`${results[0][i].benchmark}`); + // match results by benchmark name — branches may have different benchmark + // lists (e.g. a benchmark that only exists on one of the branches), so + // pairing by array index would misattribute results + const by_name = results.map((result) => new Map(result.map((r) => [r.benchmark, r]))); + + /** @type {string[]} */ + const names = []; + + for (const result of results) { + for (const { benchmark } of result) { + if (!names.includes(benchmark)) { + names.push(benchmark); + } + } + } + + for (const name of names) { + const entries = by_name.map((map) => map.get(name)); + const missing = entries + .map((entry, b) => (entry === undefined ? branches[b] : null)) + .filter((branch) => branch !== null); + + write(`${name}`); + + if (missing.length > 0) { + write(` skipped (missing on ${missing.join(', ')})`); + write(''); + continue; + } for (const metric of ['time', 'gc_time']) { - const times = results.map((result) => +result[i][metric]); + const times = entries.map((entry) => +entry[metric]); let min = Infinity; let max = -Infinity; let min_index = -1; diff --git a/benchmarking/compare/runner.js b/benchmarking/compare/runner.js index 31a8e6b44b..fa746f9869 100644 --- a/benchmarking/compare/runner.js +++ b/benchmarking/compare/runner.js @@ -1,18 +1,67 @@ +import { fork } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; import { reactivity_benchmarks } from '../benchmarks/reactivity/index.js'; import { with_cpu_profile } from '../utils.js'; -const results = []; -const PROFILE_DIR = process.env.BENCH_PROFILE_DIR; +const PROFILE_DIR = process.env.BENCH_PROFILE_DIR ?? null; +const single = process.env.BENCH_SINGLE; -for (let i = 0; i < reactivity_benchmarks.length; i += 1) { - const benchmark = reactivity_benchmarks[i]; +if (single) { + // child mode — run a single benchmark and report the result to the parent + const benchmark = reactivity_benchmarks.find((b) => b.label === single); - process.stderr.write(`Running ${i + 1}/${reactivity_benchmarks.length} ${benchmark.label} `); - results.push({ - benchmark: benchmark.label, - ...(await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn())) - }); - process.stderr.write('\x1b[2K\r'); -} + if (!benchmark) { + throw new Error(`Unknown benchmark ${single}`); + } + + const result = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn()); + + // exit via the callback so the message is guaranteed to be delivered + /** @type {NodeJS.Process} */ (process).send(result, () => process.exit(0)); +} else { + // parent mode — run every benchmark in its own child process, so that + // heap/GC/JIT state from one benchmark cannot contaminate the others + const filename = fileURLToPath(import.meta.url); + const results = []; + + for (let i = 0; i < reactivity_benchmarks.length; i += 1) { + const benchmark = reactivity_benchmarks[i]; + + process.stderr.write(`Running ${i + 1}/${reactivity_benchmarks.length} ${benchmark.label} `); + + const result = await new Promise((fulfil, reject) => { + const child = fork(filename, [], { + env: { + ...process.env, + BENCH_SINGLE: benchmark.label + } + }); + + /** @type {object | null} */ + let message_received = null; -process.send(results); + child.on('message', (message) => { + message_received = /** @type {object} */ (message); + }); + + child.on('error', reject); + + child.on('exit', (code) => { + if (message_received === null) { + reject(new Error(`benchmark ${benchmark.label} exited with code ${code}`)); + } else { + fulfil(message_received); + } + }); + }); + + results.push({ + benchmark: benchmark.label, + .../** @type {object} */ (result) + }); + + process.stderr.write('\x1b[2K\r'); + } + + /** @type {NodeJS.Process} */ (process).send(results); +} diff --git a/benchmarking/run.js b/benchmarking/run.js index 80e40a5ff1..e44e816dd0 100644 --- a/benchmarking/run.js +++ b/benchmarking/run.js @@ -1,94 +1,149 @@ +import { fork } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; import * as $ from '../packages/svelte/src/internal/client/index.js'; import { reactivity_benchmarks } from './benchmarks/reactivity/index.js'; import { ssr_benchmarks } from './benchmarks/ssr/index.js'; import { with_cpu_profile } from './utils.js'; -// e.g. `pnpm bench kairo` to only run the kairo benchmarks -const filters = process.argv.slice(2); - const PROFILE_DIR = './benchmarking/.profiles'; -const suites = [ - { - benchmarks: reactivity_benchmarks.filter( - (b) => filters.length === 0 || filters.some((f) => b.label.includes(f)) - ), - name: 'reactivity benchmarks' - }, - { - benchmarks: ssr_benchmarks.filter( - (b) => filters.length === 0 || filters.some((f) => b.label.includes(f)) - ), - name: 'server-side rendering benchmarks' - } -].filter((suite) => suite.benchmarks.length > 0); - -if (suites.length === 0) { - console.log('No benchmarks matched provided filters'); - process.exit(1); -} - -const COLUMN_WIDTHS = [25, 9, 9]; -const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b); - -const pad_right = (str, n) => str + ' '.repeat(n - str.length); -const pad_left = (str, n) => ' '.repeat(n - str.length) + str; +const single = process.env.BENCH_SINGLE; -let total_time = 0; -let total_gc_time = 0; +if (single) { + // child mode — run a single benchmark and report the result to the parent + const benchmark = [...reactivity_benchmarks, ...ssr_benchmarks].find((b) => b.label === single); -$.push({}, true); + if (!benchmark) { + throw new Error(`Unknown benchmark ${single}`); + } -try { - for (const { benchmarks, name } of suites) { - let suite_time = 0; - let suite_gc_time = 0; + $.push({}, true); + + const result = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn()); + + $.pop(); + + // exit via the callback so the message is guaranteed to be delivered + /** @type {NodeJS.Process} */ (process).send(result, () => process.exit(0)); +} else { + // parent mode — run every benchmark in its own child process, so that + // heap/GC/JIT state from one benchmark cannot contaminate the others + + // e.g. `pnpm bench kairo` to only run the kairo benchmarks + const filters = process.argv.slice(2); + + const suites = [ + { + benchmarks: reactivity_benchmarks.filter( + (b) => filters.length === 0 || filters.some((f) => b.label.includes(f)) + ), + name: 'reactivity benchmarks' + }, + { + benchmarks: ssr_benchmarks.filter( + (b) => filters.length === 0 || filters.some((f) => b.label.includes(f)) + ), + name: 'server-side rendering benchmarks' + } + ].filter((suite) => suite.benchmarks.length > 0); - console.log(`\nRunning ${name}...\n`); - console.log( - pad_right('Benchmark', COLUMN_WIDTHS[0]) + - pad_left('Time', COLUMN_WIDTHS[1]) + - pad_left('GC time', COLUMN_WIDTHS[2]) - ); - console.log('='.repeat(TOTAL_WIDTH)); + if (suites.length === 0) { + console.log('No benchmarks matched provided filters'); + process.exit(1); + } - for (const benchmark of benchmarks) { - const results = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn()); + const filename = fileURLToPath(import.meta.url); + + /** + * @param {string} label + * @returns {Promise<{ time: number, gc_time: number }>} + */ + const run_benchmark = (label) => { + return new Promise((fulfil, reject) => { + const child = fork(filename, [], { + env: { + ...process.env, + BENCH_SINGLE: label + } + }); + + /** @type {{ time: number, gc_time: number } | null} */ + let result = null; + + child.on('message', (message) => { + result = /** @type {{ time: number, gc_time: number }} */ (message); + }); + + child.on('error', reject); + + child.on('exit', (code) => { + if (result === null) { + reject(new Error(`benchmark ${label} exited with code ${code}`)); + } else { + fulfil(result); + } + }); + }); + }; + + const COLUMN_WIDTHS = [25, 9, 9]; + const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b); + + /** @type {(str: string, n: number) => string} */ + const pad_right = (str, n) => str + ' '.repeat(n - str.length); + /** @type {(str: string, n: number) => string} */ + const pad_left = (str, n) => ' '.repeat(n - str.length) + str; + + let total_time = 0; + let total_gc_time = 0; + + try { + for (const { benchmarks, name } of suites) { + let suite_time = 0; + let suite_gc_time = 0; + + console.log(`\nRunning ${name}...\n`); + console.log( + pad_right('Benchmark', COLUMN_WIDTHS[0]) + + pad_left('Time', COLUMN_WIDTHS[1]) + + pad_left('GC time', COLUMN_WIDTHS[2]) + ); + console.log('='.repeat(TOTAL_WIDTH)); + + for (const benchmark of benchmarks) { + const results = await run_benchmark(benchmark.label); + console.log( + pad_right(benchmark.label, COLUMN_WIDTHS[0]) + + pad_left(results.time.toFixed(2), COLUMN_WIDTHS[1]) + + pad_left(results.gc_time.toFixed(2), COLUMN_WIDTHS[2]) + ); + total_time += results.time; + total_gc_time += results.gc_time; + suite_time += results.time; + suite_gc_time += results.gc_time; + } + + console.log('='.repeat(TOTAL_WIDTH)); console.log( - pad_right(benchmark.label, COLUMN_WIDTHS[0]) + - pad_left(results.time.toFixed(2), COLUMN_WIDTHS[1]) + - pad_left(results.gc_time.toFixed(2), COLUMN_WIDTHS[2]) + pad_right('suite', COLUMN_WIDTHS[0]) + + pad_left(suite_time.toFixed(2), COLUMN_WIDTHS[1]) + + pad_left(suite_gc_time.toFixed(2), COLUMN_WIDTHS[2]) ); - total_time += results.time; - total_gc_time += results.gc_time; - suite_time += results.time; - suite_gc_time += results.gc_time; + console.log('='.repeat(TOTAL_WIDTH)); } - console.log('='.repeat(TOTAL_WIDTH)); - console.log( - pad_right('suite', COLUMN_WIDTHS[0]) + - pad_left(suite_time.toFixed(2), COLUMN_WIDTHS[1]) + - pad_left(suite_gc_time.toFixed(2), COLUMN_WIDTHS[2]) - ); - console.log('='.repeat(TOTAL_WIDTH)); - } - - if (PROFILE_DIR !== null) { console.log(`\nCPU profiles written to ${PROFILE_DIR}`); + } catch (e) { + // eslint-disable-next-line no-console + console.error(e); + process.exit(1); } -} catch (e) { - // eslint-disable-next-line no-console - console.error(e); - process.exit(1); -} -$.pop(); + console.log(''); -console.log(''); - -console.log( - pad_right('total', COLUMN_WIDTHS[0]) + - pad_left(total_time.toFixed(2), COLUMN_WIDTHS[1]) + - pad_left(total_gc_time.toFixed(2), COLUMN_WIDTHS[2]) -); + console.log( + pad_right('total', COLUMN_WIDTHS[0]) + + pad_left(total_time.toFixed(2), COLUMN_WIDTHS[1]) + + pad_left(total_gc_time.toFixed(2), COLUMN_WIDTHS[2]) + ); +}