diff --git a/.gitignore b/.gitignore index 556cae6344..e4f9f9cf87 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ packages/svelte/scripts/_baseline/ benchmarking/.profiles benchmarking/compare/.results benchmarking/compare/.profiles +benchmarking/compare/results.html diff --git a/benchmarking/compare/generate-report.js b/benchmarking/compare/generate-report.js index a06aaf9fc2..70f7b30fbb 100644 --- a/benchmarking/compare/generate-report.js +++ b/benchmarking/compare/generate-report.js @@ -2,15 +2,27 @@ import fs from 'node:fs'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -export function generate_report(outdir) { +const REPORT_DATA_PLACEHOLDER = '%%REPORT_DATA%%'; +const report_template = fs.readFileSync( + new URL('./results.template.html', import.meta.url), + 'utf-8' +); + +if (!report_template.includes(REPORT_DATA_PLACEHOLDER)) { + throw new Error(`Missing ${REPORT_DATA_PLACEHOLDER} in results.template.html`); +} + +export function generate_report(outdir, branches) { const result_files = fs .readdirSync(outdir) - .filter((file) => file.endsWith('.json')) + .filter((file) => file.endsWith('.json') && (!branches || branches.includes(file.slice(0, -5)))) .sort((a, b) => a.localeCompare(b)); - const branches = result_files.map((file) => file.slice(0, -5)); + // always do this so that ordering lines up (branches argument might be passed in a different order than the result files are sorted + branches = result_files.map((file) => file.slice(0, -5)); + const results = result_files.map((file) => - JSON.parse(fs.readFileSync(`${outdir}/${file}`, 'utf-8')) + JSON.parse(fs.readFileSync(path.join(outdir, file), 'utf-8')) ); if (results.length === 0) { @@ -95,6 +107,32 @@ export function generate_report(outdir) { write(''); } + + const benchmarks = names.map((name) => ({ + name, + values: by_name.map((map) => { + const entry = map.get(name); + + if (entry === undefined) return null; + + return { + time: Number(entry.time), + gc_time: Number(entry.gc_time) + }; + }) + })); + const data = JSON.stringify({ + generated_at: new Date().toISOString(), + branches, + benchmarks + }) + .replaceAll('<', '\\u003c') + .replaceAll('\u2028', '\\u2028') + .replaceAll('\u2029', '\\u2029'); + const html_file = path.resolve(outdir, '../results.html'); + + fs.writeFileSync(html_file, report_template.replace(REPORT_DATA_PLACEHOLDER, data)); + console.log(`\nHTML report written to ${html_file}`); } function char(i) { diff --git a/benchmarking/compare/index.js b/benchmarking/compare/index.js index 9064ee7da9..2e76f46e1b 100644 --- a/benchmarking/compare/index.js +++ b/benchmarking/compare/index.js @@ -85,4 +85,4 @@ if (PROFILE_DIR !== null) { console.log(`\nCPU profiles written to ${PROFILE_DIR}`); } -generate_report(outdir); +generate_report(outdir, requested_branches); diff --git a/benchmarking/compare/results.template.html b/benchmarking/compare/results.template.html new file mode 100644 index 0000000000..02f208ac4f --- /dev/null +++ b/benchmarking/compare/results.template.html @@ -0,0 +1,741 @@ + + + + + + Benchmark comparison + + + +
+

Benchmark comparison

+

+ Runtime results across branches. Green cells are fastest for an entry and red cells expose + the largest regressions. Overall runtime normalizes every benchmark to its fastest result + before averaging, so long-running entries do not outweigh short ones. +

+ +
+ +
+
+
+

Branch standings

+

+ Wins count the fastest branch for each comparable entry. Normalized runtime is the + average slowdown against each entry's fastest result; lower is better. Click a heading + to sort. +

+
+
+
+ + + + + + + + + +
+ +
+
+
+ +
+
+
+

Results by benchmark

+

+ Each cell shows runtime, difference from the winner, and GC time. Missing entries are + excluded from both standings. +

+
+
+
+ + +
+
+
+
+
+ + +
+ + + + +