diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78bab024ca..df9f755874 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,7 +107,7 @@ jobs: - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: actions/setup-node@v6 with: - node-version: 18 + node-version: 24 cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm bench diff --git a/.gitignore b/.gitignore index d503437664..d3c1819bd5 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,6 @@ coverage tmp +benchmarking/.profiles benchmarking/compare/.results +benchmarking/compare/.profiles diff --git a/benchmarking/compare/index.js b/benchmarking/compare/index.js index 8f38686a29..dcf5d48dbd 100644 --- a/benchmarking/compare/index.js +++ b/benchmarking/compare/index.js @@ -11,12 +11,17 @@ import { fileURLToPath } from 'node:url'; const filename = fileURLToPath(import.meta.url); const runner = path.resolve(filename, '../runner.js'); const outdir = path.resolve(filename, '../.results'); +const report_file = `${outdir}/report.txt`; if (fs.existsSync(outdir)) fs.rmSync(outdir, { recursive: true }); fs.mkdirSync(outdir); const branches = []; +let PROFILE_DIR = path.resolve(filename, '../.profiles'); +if (fs.existsSync(PROFILE_DIR)) fs.rmSync(PROFILE_DIR, { recursive: true }); +fs.mkdirSync(PROFILE_DIR, { recursive: true }); + for (const arg of process.argv.slice(2)) { if (arg.startsWith('--')) continue; if (arg === filename) continue; @@ -44,7 +49,12 @@ for (const branch of branches) { execSync(`git checkout ${branch}`); await new Promise((fulfil, reject) => { - const child = fork(runner); + const child = fork(runner, [], { + env: { + ...process.env, + BENCH_PROFILE_DIR: `${PROFILE_DIR}/${safe(branch)}` + } + }); child.on('message', (results) => { fs.writeFileSync(`${outdir}/${branch}.json`, JSON.stringify(results, null, ' ')); @@ -57,12 +67,23 @@ for (const branch of branches) { console.groupEnd(); } +if (PROFILE_DIR !== null) { + console.log(`\nCPU profiles written to ${PROFILE_DIR}`); +} + const results = branches.map((branch) => { return JSON.parse(fs.readFileSync(`${outdir}/${branch}.json`, 'utf-8')); }); +fs.writeFileSync(report_file, ''); + +const write = (str) => { + fs.appendFileSync(report_file, str + '\n'); + console.log(str); +}; + for (let i = 0; i < results[0].length; i += 1) { - console.group(`${results[0][i].benchmark}`); + write(`${results[0][i].benchmark}`); for (const metric of ['time', 'gc_time']) { const times = results.map((result) => +result[i][metric]); @@ -84,20 +105,23 @@ for (let i = 0; i < results[0].length; i += 1) { } if (min !== 0) { - console.group(`${metric}: fastest is ${char(min_index)} (${branches[min_index]})`); + write(` ${metric}: fastest is ${char(min_index)} (${branches[min_index]})`); times.forEach((time, b) => { const SIZE = 20; const n = Math.round(SIZE * (time / max)); - console.log(`${char(b)}: ${'◼'.repeat(n)}${' '.repeat(SIZE - n)} ${time.toFixed(2)}ms`); + write(` ${char(b)}: ${'◼'.repeat(n)}${' '.repeat(SIZE - n)} ${time.toFixed(2)}ms`); }); - console.groupEnd(); } } - console.groupEnd(); + write(''); } function char(i) { return String.fromCharCode(97 + i); } + +function safe(name) { + return name.replace(/[^a-z0-9._-]+/gi, '_'); +} diff --git a/benchmarking/compare/runner.js b/benchmarking/compare/runner.js index 11e40ed983..31a8e6b44b 100644 --- a/benchmarking/compare/runner.js +++ b/benchmarking/compare/runner.js @@ -1,12 +1,17 @@ import { reactivity_benchmarks } from '../benchmarks/reactivity/index.js'; +import { with_cpu_profile } from '../utils.js'; const results = []; +const PROFILE_DIR = process.env.BENCH_PROFILE_DIR; for (let i = 0; i < reactivity_benchmarks.length; i += 1) { const benchmark = reactivity_benchmarks[i]; process.stderr.write(`Running ${i + 1}/${reactivity_benchmarks.length} ${benchmark.label} `); - results.push({ benchmark: benchmark.label, ...(await benchmark.fn()) }); + results.push({ + benchmark: benchmark.label, + ...(await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn())) + }); process.stderr.write('\x1b[2K\r'); } diff --git a/benchmarking/run.js b/benchmarking/run.js index 2b09f7c592..80e40a5ff1 100644 --- a/benchmarking/run.js +++ b/benchmarking/run.js @@ -1,10 +1,13 @@ import * as $ from '../packages/svelte/src/internal/client/index.js'; import { reactivity_benchmarks } from './benchmarks/reactivity/index.js'; import { ssr_benchmarks } from './benchmarks/ssr/index.js'; +import { with_cpu_profile } from './utils.js'; // e.g. `pnpm bench kairo` to only run the kairo benchmarks const filters = process.argv.slice(2); +const PROFILE_DIR = './benchmarking/.profiles'; + const suites = [ { benchmarks: reactivity_benchmarks.filter( @@ -50,7 +53,7 @@ try { console.log('='.repeat(TOTAL_WIDTH)); for (const benchmark of benchmarks) { - const results = await benchmark.fn(); + const results = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn()); console.log( pad_right(benchmark.label, COLUMN_WIDTHS[0]) + pad_left(results.time.toFixed(2), COLUMN_WIDTHS[1]) + @@ -70,6 +73,10 @@ try { ); console.log('='.repeat(TOTAL_WIDTH)); } + + if (PROFILE_DIR !== null) { + console.log(`\nCPU profiles written to ${PROFILE_DIR}`); + } } catch (e) { // eslint-disable-next-line no-console console.error(e); diff --git a/benchmarking/utils.js b/benchmarking/utils.js index 5581135e00..b363576963 100644 --- a/benchmarking/utils.js +++ b/benchmarking/utils.js @@ -1,4 +1,7 @@ import { performance, PerformanceObserver } from 'node:perf_hooks'; +import fs from 'node:fs'; +import path from 'node:path'; +import inspector from 'node:inspector/promises'; import v8 from 'v8-natives'; // Credit to https://github.com/milomg/js-reactivity-benchmark for the logic for timing + GC tracking. @@ -41,3 +44,37 @@ export async function fastest_test(times, fn) { return results.reduce((a, b) => (a.time < b.time ? a : b)); } + +function safe(name) { + return name.replace(/[^a-z0-9._-]+/gi, '_'); +} + +/** + * @template T + * @param {string | null} profile_dir + * @param {string} profile_name + * @param {() => T | Promise} fn + * @returns {Promise} + */ +export async function with_cpu_profile(profile_dir, profile_name, fn) { + if (profile_dir === null) { + return await fn(); + } + + fs.mkdirSync(profile_dir, { recursive: true }); + + const session = new inspector.Session(); + session.connect(); + + await session.post('Profiler.enable'); + await session.post('Profiler.start'); + + try { + return await fn(); + } finally { + const { profile } = /** @type {{ profile: object }} */ (await session.post('Profiler.stop')); + const file = path.join(profile_dir, `${safe(profile_name)}.cpuprofile`); + fs.writeFileSync(file, JSON.stringify(profile)); + session.disconnect(); + } +}