msb-public
/
svelte
mirror of https://github.com/sveltejs/svelte
1
0
Code Issues Projects Releases Wiki Activity

chore: run every benchmark in its own process (#18542)

Browse Source 
While benchmarking it became apparent that when running the full `pnpm
bench:compare` suite results could be wildly different compared to only
comparing single benchmarks. The reason (most likely) is that the
heap/GC/JIT state from one benchmark contaminates the others.

Therefore this adjusts the runners such that each benchmark entry is run
in its own forked process, not just each branch. This should give
better, more stable results.

Also a little fix to compare results by name; I made the mistake of
adding a new benchmark to main which wasn't present on other branches
yet, which made the results really confusing.
pull/18543/head
Simon H 1 month ago committed by GitHub
parent b4d1583ae2
commit af6f1d309b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 220 additions and 89 deletions
Show Stats Download Patch File Download Diff File

33
benchmarking/compare/generate-report.js
Unescape View File

@ -33,11 +33,38 @@ export function generate_report(outdir) {
write('');
for (let i = 0; i < results[0].length; i += 1) {
write(`${results[0][i].benchmark}`);
// match results by benchmark name — branches may have different benchmark
// lists (e.g. a benchmark that only exists on one of the branches), so
// pairing by array index would misattribute results
const by_name = results.map((result) => new Map(result.map((r) => [r.benchmark, r])));
/** @type {string[]} */
const names = [];
for (const result of results) {
for (const { benchmark } of result) {
if (!names.includes(benchmark)) {
names.push(benchmark);
}
}
}
for (const name of names) {
const entries = by_name.map((map) => map.get(name));
const missing = entries
.map((entry, b) => (entry === undefined ? branches[b] : null))
.filter((branch) => branch !== null);
write(`${name}`);
if (missing.length > 0) {
write(` skipped (missing on ${missing.join(', ')})`);
write('');
continue;
}
for (const metric of ['time', 'gc_time']) {
const times = results.map((result) => +result[i][metric]);
const times = entries.map((entry) => +entry[metric]);
let min = Infinity;
let max = -Infinity;
let min_index = -1;

73
benchmarking/compare/runner.js
Unescape View File

@ -1,18 +1,67 @@
import { fork } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { reactivity_benchmarks } from '../benchmarks/reactivity/index.js';
import { with_cpu_profile } from '../utils.js';
const results = [];
const PROFILE_DIR = process.env.BENCH_PROFILE_DIR;
const PROFILE_DIR = process.env.BENCH_PROFILE_DIR ?? null;
const single = process.env.BENCH_SINGLE;
for (let i = 0; i < reactivity_benchmarks.length; i += 1) {
const benchmark = reactivity_benchmarks[i];
if (single) {
// child mode — run a single benchmark and report the result to the parent
const benchmark = reactivity_benchmarks.find((b) => b.label === single);
process.stderr.write(`Running ${i + 1}/${reactivity_benchmarks.length} ${benchmark.label} `);
results.push({
benchmark: benchmark.label,
...(await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn()))
});
process.stderr.write('\x1b[2K\r');
}
if (!benchmark) {
throw new Error(`Unknown benchmark ${single}`);
}
const result = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn());
// exit via the callback so the message is guaranteed to be delivered
/** @type {NodeJS.Process} */ (process).send(result, () => process.exit(0));
} else {
// parent mode — run every benchmark in its own child process, so that
// heap/GC/JIT state from one benchmark cannot contaminate the others
const filename = fileURLToPath(import.meta.url);
const results = [];
for (let i = 0; i < reactivity_benchmarks.length; i += 1) {
const benchmark = reactivity_benchmarks[i];
process.stderr.write(`Running ${i + 1}/${reactivity_benchmarks.length} ${benchmark.label} `);
const result = await new Promise((fulfil, reject) => {
const child = fork(filename, [], {
env: {
...process.env,
BENCH_SINGLE: benchmark.label
}
});
/** @type {object | null} */
let message_received = null;
process.send(results);
child.on('message', (message) => {
message_received = /** @type {object} */ (message);
});
child.on('error', reject);
child.on('exit', (code) => {
if (message_received === null) {
reject(new Error(`benchmark ${benchmark.label} exited with code ${code}`));
} else {
fulfil(message_received);
}
});
});
results.push({
benchmark: benchmark.label,
.../** @type {object} */ (result)
});
process.stderr.write('\x1b[2K\r');
}
/** @type {NodeJS.Process} */ (process).send(results);
}

203
benchmarking/run.js
Unescape View File

@ -1,94 +1,149 @@
import { fork } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import * as $ from '../packages/svelte/src/internal/client/index.js';
import { reactivity_benchmarks } from './benchmarks/reactivity/index.js';
import { ssr_benchmarks } from './benchmarks/ssr/index.js';
import { with_cpu_profile } from './utils.js';
// e.g. `pnpm bench kairo` to only run the kairo benchmarks
const filters = process.argv.slice(2);
const PROFILE_DIR = './benchmarking/.profiles';
const suites = [
{
benchmarks: reactivity_benchmarks.filter(
(b) => filters.length === 0 || filters.some((f) => b.label.includes(f))
),
name: 'reactivity benchmarks'
},
{
benchmarks: ssr_benchmarks.filter(
(b) => filters.length === 0 || filters.some((f) => b.label.includes(f))
),
name: 'server-side rendering benchmarks'
}
].filter((suite) => suite.benchmarks.length > 0);
if (suites.length === 0) {
console.log('No benchmarks matched provided filters');
process.exit(1);
}
const COLUMN_WIDTHS = [25, 9, 9];
const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b);
const pad_right = (str, n) => str + ' '.repeat(n - str.length);
const pad_left = (str, n) => ' '.repeat(n - str.length) + str;
const single = process.env.BENCH_SINGLE;
let total_time = 0;
let total_gc_time = 0;
if (single) {
// child mode — run a single benchmark and report the result to the parent
const benchmark = [...reactivity_benchmarks, ...ssr_benchmarks].find((b) => b.label === single);
$.push({}, true);
if (!benchmark) {
throw new Error(`Unknown benchmark ${single}`);
}
try {
for (const { benchmarks, name } of suites) {
let suite_time = 0;
let suite_gc_time = 0;
$.push({}, true);
const result = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn());
$.pop();
// exit via the callback so the message is guaranteed to be delivered
/** @type {NodeJS.Process} */ (process).send(result, () => process.exit(0));
} else {
// parent mode — run every benchmark in its own child process, so that
// heap/GC/JIT state from one benchmark cannot contaminate the others
// e.g. `pnpm bench kairo` to only run the kairo benchmarks
const filters = process.argv.slice(2);
const suites = [
{
benchmarks: reactivity_benchmarks.filter(
(b) => filters.length === 0 || filters.some((f) => b.label.includes(f))
),
name: 'reactivity benchmarks'
},
{
benchmarks: ssr_benchmarks.filter(
(b) => filters.length === 0 || filters.some((f) => b.label.includes(f))
),
name: 'server-side rendering benchmarks'
}
].filter((suite) => suite.benchmarks.length > 0);
console.log(`\nRunning ${name}...\n`);
console.log(
pad_right('Benchmark', COLUMN_WIDTHS[0]) +
pad_left('Time', COLUMN_WIDTHS[1]) +
pad_left('GC time', COLUMN_WIDTHS[2])
);
console.log('='.repeat(TOTAL_WIDTH));
if (suites.length === 0) {
console.log('No benchmarks matched provided filters');
process.exit(1);
}
for (const benchmark of benchmarks) {
const results = await with_cpu_profile(PROFILE_DIR, benchmark.label, () => benchmark.fn());
const filename = fileURLToPath(import.meta.url);
/**
* @param {string} label
* @returns {Promise<{ time: number, gc_time: number }>}
*/
const run_benchmark = (label) => {
return new Promise((fulfil, reject) => {
const child = fork(filename, [], {
env: {
...process.env,
BENCH_SINGLE: label
}
});
/** @type {{ time: number, gc_time: number } | null} */
let result = null;
child.on('message', (message) => {
result = /** @type {{ time: number, gc_time: number }} */ (message);
});
child.on('error', reject);
child.on('exit', (code) => {
if (result === null) {
reject(new Error(`benchmark ${label} exited with code ${code}`));
} else {
fulfil(result);
}
});
});
};
const COLUMN_WIDTHS = [25, 9, 9];
const TOTAL_WIDTH = COLUMN_WIDTHS.reduce((a, b) => a + b);
/** @type {(str: string, n: number) => string} */
const pad_right = (str, n) => str + ' '.repeat(n - str.length);
/** @type {(str: string, n: number) => string} */
const pad_left = (str, n) => ' '.repeat(n - str.length) + str;
let total_time = 0;
let total_gc_time = 0;
try {
for (const { benchmarks, name } of suites) {
let suite_time = 0;
let suite_gc_time = 0;
console.log(`\nRunning ${name}...\n`);
console.log(
pad_right('Benchmark', COLUMN_WIDTHS[0]) +
pad_left('Time', COLUMN_WIDTHS[1]) +
pad_left('GC time', COLUMN_WIDTHS[2])
);
console.log('='.repeat(TOTAL_WIDTH));
for (const benchmark of benchmarks) {
const results = await run_benchmark(benchmark.label);
console.log(
pad_right(benchmark.label, COLUMN_WIDTHS[0]) +
pad_left(results.time.toFixed(2), COLUMN_WIDTHS[1]) +
pad_left(results.gc_time.toFixed(2), COLUMN_WIDTHS[2])
);
total_time += results.time;
total_gc_time += results.gc_time;
suite_time += results.time;
suite_gc_time += results.gc_time;
}
console.log('='.repeat(TOTAL_WIDTH));
console.log(
pad_right(benchmark.label, COLUMN_WIDTHS[0]) +
pad_left(results.time.toFixed(2), COLUMN_WIDTHS[1]) +
pad_left(results.gc_time.toFixed(2), COLUMN_WIDTHS[2])
pad_right('suite', COLUMN_WIDTHS[0]) +
pad_left(suite_time.toFixed(2), COLUMN_WIDTHS[1]) +
pad_left(suite_gc_time.toFixed(2), COLUMN_WIDTHS[2])
);
total_time += results.time;
total_gc_time += results.gc_time;
suite_time += results.time;
suite_gc_time += results.gc_time;
console.log('='.repeat(TOTAL_WIDTH));
}
console.log('='.repeat(TOTAL_WIDTH));
console.log(
pad_right('suite', COLUMN_WIDTHS[0]) +
pad_left(suite_time.toFixed(2), COLUMN_WIDTHS[1]) +
pad_left(suite_gc_time.toFixed(2), COLUMN_WIDTHS[2])
);
console.log('='.repeat(TOTAL_WIDTH));
}
if (PROFILE_DIR !== null) {
console.log(`\nCPU profiles written to ${PROFILE_DIR}`);
} catch (e) {
// eslint-disable-next-line no-console
console.error(e);
process.exit(1);
}
} catch (e) {
// eslint-disable-next-line no-console
console.error(e);
process.exit(1);
}
$.pop();
console.log('');
console.log('');
console.log(
pad_right('total', COLUMN_WIDTHS[0]) +
pad_left(total_time.toFixed(2), COLUMN_WIDTHS[1]) +
pad_left(total_gc_time.toFixed(2), COLUMN_WIDTHS[2])
);
console.log(
pad_right('total', COLUMN_WIDTHS[0]) +
pad_left(total_time.toFixed(2), COLUMN_WIDTHS[1]) +
pad_left(total_gc_time.toFixed(2), COLUMN_WIDTHS[2])
);
}

Write Preview
Loading…
Cancel
Save