From 6f7857af5921f03dcad2057c92be4b169ac4df4a Mon Sep 17 00:00:00 2001 From: Simon H <5968653+dummdidumm@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:24:50 +0200 Subject: [PATCH] chore: compiler profiling + parser bench (#18767) Used these to speed up parsing/find compilation bottlenecks. Useful for later analysis therefore committing it. --- benchmarking/analyze-compiler-profile.js | 346 ++++++++++++++++++ benchmarking/benchmarks/compiler/index.js | 3 + .../benchmarks/compiler/parser.bench.js | 58 +++ benchmarking/compiler-profiling.md | 43 +++ benchmarking/profile-compiler.js | 94 +++++ benchmarking/run.js | 12 +- package.json | 1 + 7 files changed, 556 insertions(+), 1 deletion(-) create mode 100644 benchmarking/analyze-compiler-profile.js create mode 100644 benchmarking/benchmarks/compiler/index.js create mode 100644 benchmarking/benchmarks/compiler/parser.bench.js create mode 100644 benchmarking/compiler-profiling.md create mode 100644 benchmarking/profile-compiler.js diff --git a/benchmarking/analyze-compiler-profile.js b/benchmarking/analyze-compiler-profile.js new file mode 100644 index 0000000000..41729b7d93 --- /dev/null +++ b/benchmarking/analyze-compiler-profile.js @@ -0,0 +1,346 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(fileURLToPath(import.meta.url), '../..'); +const compiler_path = 'packages/svelte/src/compiler/'; + +/** + * Merge V8 profiles from a Vitest run into a compiler-only flame graph and hotspot summary. + * Samples retain callees outside the compiler once execution has entered compiler code. + * @param {string} run_dir + */ +export function analyze_compiler_profiles(run_dir) { + const raw_dir = path.join(run_dir, 'raw'); + const profile_files = fs + .readdirSync(raw_dir) + .filter((file) => file.endsWith('.cpuprofile')) + .sort(); + + if (profile_files.length === 0) { + throw new Error(`No CPU profiles found in ${raw_dir}`); + } + + /** @type {Array<{ name: string, file?: string, line?: number, col?: number }>} */ + const frames = []; + /** @type {Map} */ + const frame_indices = new Map(); + /** @type {Map} */ + const stacks = new Map(); + /** @type {Map} */ + const hotspots = new Map(); + /** @type {Map} */ + const files = new Map(); + let total_time = 0; + let compiler_time = 0; + let garbage_collection_time = 0; + let total_samples = 0; + let compiler_samples = 0; + + for (const profile_file of profile_files) { + let profile; + try { + profile = JSON.parse(fs.readFileSync(path.join(raw_dir, profile_file), 'utf8')); + } catch (error) { + throw new Error(`Could not parse CPU profile ${profile_file}`, { cause: error }); + } + const nodes = Array.isArray(profile.nodes) ? profile.nodes : []; + const samples = Array.isArray(profile.samples) ? profile.samples : []; + const time_deltas = Array.isArray(profile.timeDeltas) ? profile.timeDeltas : []; + const nodes_by_id = new Map(nodes.map((node) => [node.id, node])); + const parents = new Map(); + + for (const node of nodes) { + for (const child of node.children || []) { + parents.set(child, node.id); + } + } + + for (let i = 0; i < samples.length; i += 1) { + const weight = typeof time_deltas[i] === 'number' ? time_deltas[i] : 1; + const stack = get_stack(samples[i], nodes_by_id, parents); + total_time += weight; + total_samples += 1; + + if (stack.at(-1)?.function_name === '(garbage collector)') { + garbage_collection_time += weight; + } + + const compiler_index = stack.findIndex((frame) => frame.url.includes(compiler_path)); + if (compiler_index === -1) continue; + + const compiler_stack = stack.slice(compiler_index); + const stack_indices = compiler_stack.map(get_frame_index); + const stack_key = stack_indices.join(','); + const existing_stack = stacks.get(stack_key); + + if (existing_stack) { + existing_stack.weight += weight; + } else { + stacks.set(stack_key, { stack: stack_indices, weight }); + } + + compiler_time += weight; + compiler_samples += 1; + + const leaf = compiler_stack.at(-1); + if (leaf) { + get_hotspot(hotspots, frame_key(leaf), leaf).self += weight; + get_hotspot(files, leaf.url || '(native)', { + ...leaf, + function_name: leaf.url || '(native)' + }).self += weight; + } + + const seen_frames = new Set(); + const seen_files = new Set(); + + for (const frame of compiler_stack) { + const key = frame_key(frame); + if (!seen_frames.has(key)) { + get_hotspot(hotspots, key, frame).inclusive += weight; + seen_frames.add(key); + } + + const file = frame.url || '(native)'; + if (!seen_files.has(file)) { + get_hotspot(files, file, { ...frame, function_name: file }).inclusive += weight; + seen_files.add(file); + } + } + } + } + + if (compiler_time === 0) { + throw new Error('The CPU profiles contain no samples from packages/svelte/src/compiler'); + } + + const hotspot_rows = rank(hotspots, compiler_time); + const file_rows = rank(files, compiler_time); + const compact_stacks = [...stacks.values()]; + const summary = { + profile_files: profile_files.length, + total_samples, + compiler_samples, + total_time_microseconds: total_time, + compiler_time_microseconds: compiler_time, + garbage_collection_time_microseconds: garbage_collection_time, + compiler_share_percent: (compiler_time * 100) / total_time, + hotspots: hotspot_rows, + files: file_rows + }; + const speedscope = { + $schema: 'https://www.speedscope.app/file-format-schema.json', + name: `Svelte compiler: ${path.basename(run_dir)}`, + exporter: 'Svelte compiler test profiler', + activeProfileIndex: 0, + shared: { frames }, + profiles: [ + { + type: 'sampled', + name: 'Compiler-active samples from pnpm test', + unit: 'microseconds', + startValue: 0, + endValue: compiler_time, + samples: compact_stacks.map((entry) => entry.stack), + weights: compact_stacks.map((entry) => entry.weight) + } + ] + }; + + fs.writeFileSync( + path.join(run_dir, 'flamegraph.speedscope.json'), + `${JSON.stringify(speedscope)}\n` + ); + fs.writeFileSync(path.join(run_dir, 'summary.json'), `${JSON.stringify(summary, null, '\t')}\n`); + fs.writeFileSync(path.join(run_dir, 'summary.md'), render_markdown(summary)); + + return summary; + + /** @param {Frame} frame */ + function get_frame_index(frame) { + const key = frame_key(frame); + const existing = frame_indices.get(key); + if (existing !== undefined) return existing; + + const index = frames.length; + const location = frame.url ? ` (${frame.url}:${frame.line})` : ''; + frames.push({ + name: `${frame.function_name}${location}`, + ...(frame.url ? { file: frame.url, line: frame.line, col: frame.column } : {}) + }); + frame_indices.set(key, index); + return index; + } +} + +/** + * @param {number} leaf_id + * @param {Map} nodes_by_id + * @param {Map} parents + * @returns {Frame[]} + */ +function get_stack(leaf_id, nodes_by_id, parents) { + /** @type {Frame[]} */ + const stack = []; + const seen = new Set(); + let id = leaf_id; + + while (typeof id === 'number' && !seen.has(id)) { + seen.add(id); + const node = nodes_by_id.get(id); + if (!node) break; + + const call_frame = node.callFrame || {}; + stack.push({ + function_name: call_frame.functionName || '(anonymous)', + url: normalize_url(call_frame.url || ''), + line: typeof call_frame.lineNumber === 'number' ? call_frame.lineNumber + 1 : 0, + column: typeof call_frame.columnNumber === 'number' ? call_frame.columnNumber + 1 : 0 + }); + id = parents.get(id); + } + + return stack.reverse(); +} + +/** @param {string} url */ +function normalize_url(url) { + if (!url) return ''; + + let pathname = url.replace(/^\/\@fs\//, '/').replace(/[?#].*$/, ''); + if (pathname.startsWith('file://')) { + try { + pathname = fileURLToPath(pathname); + } catch { + return url; + } + } + + if (path.isAbsolute(pathname)) { + const relative = path.relative(root, pathname); + if (!relative.startsWith('..') && !path.isAbsolute(relative)) { + pathname = relative; + } + } + + return pathname.replaceAll(path.sep, '/'); +} + +/** @param {Frame} frame */ +function frame_key(frame) { + return `${frame.function_name}\0${frame.url}\0${frame.line}\0${frame.column}`; +} + +/** + * @param {Map} collection + * @param {string} key + * @param {Frame} frame + */ +function get_hotspot(collection, key, frame) { + let hotspot = collection.get(key); + if (!hotspot) { + hotspot = { ...frame, self: 0, inclusive: 0 }; + collection.set(key, hotspot); + } + return hotspot; +} + +/** + * @param {Map} collection + * @param {number} total + */ +function rank(collection, total) { + return [...collection.values()] + .map((hotspot) => ({ + function: hotspot.function_name, + url: hotspot.url, + line: hotspot.line, + column: hotspot.column, + self_microseconds: hotspot.self, + self_percent: (hotspot.self * 100) / total, + inclusive_microseconds: hotspot.inclusive, + inclusive_percent: (hotspot.inclusive * 100) / total + })) + .sort( + (a, b) => + b.self_microseconds - a.self_microseconds || + b.inclusive_microseconds - a.inclusive_microseconds + ); +} + +/** @param {ReturnType} summary */ +function render_markdown(summary) { + const lines = [ + '# Svelte compiler CPU profile', + '', + 'Open `flamegraph.speedscope.json` in [Speedscope](https://www.speedscope.app/) for the interactive flame graph.', + '', + '## Coverage', + '', + '| Metric | Value |', + '| --- | ---: |', + `| Raw profiles | ${summary.profile_files} |`, + `| All profiled CPU time | ${format_time(summary.total_time_microseconds)} |`, + `| Compiler-active CPU time | ${format_time(summary.compiler_time_microseconds)} |`, + `| Compiler share | ${summary.compiler_share_percent.toFixed(2)}% |`, + `| All-process garbage collection | ${format_time(summary.garbage_collection_time_microseconds)} |`, + '', + 'Compiler-active time includes external callees while a compiler frame is on the stack. Garbage collection is reported for context but cannot be attributed to compiler stacks by V8.', + '', + '## Top Self Hotspots', + '', + '| Rank | Function | Location | Self | Self % | Inclusive | Inclusive % |', + '| ---: | --- | --- | ---: | ---: | ---: | ---: |' + ]; + + for (const [index, row] of summary.hotspots.slice(0, 50).entries()) { + lines.push(render_row(index, row)); + } + + lines.push( + '', + '## Top Files', + '', + '| Rank | File | Self | Self % | Inclusive | Inclusive % |', + '| ---: | --- | ---: | ---: | ---: | ---: |' + ); + + for (const [index, row] of summary.files.slice(0, 50).entries()) { + lines.push( + `| ${index + 1} | ${escape_cell(row.function)} | ${format_time(row.self_microseconds)} | ${row.self_percent.toFixed(2)}% | ${format_time(row.inclusive_microseconds)} | ${row.inclusive_percent.toFixed(2)}% |` + ); + } + + return `${lines.join('\n')}\n`; +} + +/** @param {number} index @param {ReturnType[number]} row */ +function render_row(index, row) { + const location = row.url ? `${row.url}:${row.line}:${row.column}` : '(native)'; + return `| ${index + 1} | ${escape_cell(row.function)} | ${escape_cell(location)} | ${format_time(row.self_microseconds)} | ${row.self_percent.toFixed(2)}% | ${format_time(row.inclusive_microseconds)} | ${row.inclusive_percent.toFixed(2)}% |`; +} + +/** @param {number} microseconds */ +function format_time(microseconds) { + return `${(microseconds / 1000).toFixed(1)} ms`; +} + +/** @param {string} value */ +function escape_cell(value) { + return value.replaceAll('|', '\\|'); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const run_dir = process.argv[2]; + if (!run_dir) { + console.error('Usage: node benchmarking/analyze-compiler-profile.js '); + process.exit(1); + } + analyze_compiler_profiles(path.resolve(run_dir)); +} + +/** + * @typedef {{ function_name: string, url: string, line: number, column: number }} Frame + * @typedef {Frame & { self: number, inclusive: number }} Hotspot + */ diff --git a/benchmarking/benchmarks/compiler/index.js b/benchmarking/benchmarks/compiler/index.js new file mode 100644 index 0000000000..6db5ca139d --- /dev/null +++ b/benchmarking/benchmarks/compiler/index.js @@ -0,0 +1,3 @@ +import { parser_benchmarks } from './parser.bench.js'; + +export const compiler_benchmarks = parser_benchmarks; diff --git a/benchmarking/benchmarks/compiler/parser.bench.js b/benchmarking/benchmarks/compiler/parser.bench.js new file mode 100644 index 0000000000..3ff0fefbe7 --- /dev/null +++ b/benchmarking/benchmarks/compiler/parser.bench.js @@ -0,0 +1,58 @@ +import { parse } from '../../../packages/svelte/src/compiler/phases/1-parse/index.js'; +import { fastest_test } from '../../utils.js'; + +const elements_source = Array.from( + { length: 1000 }, + (_, i) => + `

Item ${i} has a reasonably long entity-free text value

` +).join('\n'); + +const script_style_source = ` + +`; + +const typescript_source = `${Array.from( + { length: 1000 }, + (_, i) => `
Item ${i}
` +).join('\n')} +{value as number} +${' \n'.repeat(1000)}`; + +/** + * @param {string} label + * @param {string} source + * @param {number} iterations + */ +function create_parser_benchmark(label, source, iterations) { + return { + label, + fn: async () => { + for (let i = 0; i < iterations; i++) { + parse(source); + } + + return await fastest_test(10, () => { + for (let i = 0; i < iterations; i++) { + parse(source); + } + }); + } + }; +} + +export const parser_benchmarks = [ + create_parser_benchmark('parser_elements', elements_source, 5), + create_parser_benchmark('parser_script_style', script_style_source, 50), + create_parser_benchmark('parser_typescript', typescript_source, 25) +]; diff --git a/benchmarking/compiler-profiling.md b/benchmarking/compiler-profiling.md new file mode 100644 index 0000000000..cc2196958e --- /dev/null +++ b/benchmarking/compiler-profiling.md @@ -0,0 +1,43 @@ +# Compiler CPU profiling + +Run the full test suite with V8 CPU profiling enabled in every Vitest worker: + +```sh +pnpm profile:compiler +``` + +Arguments are passed to `pnpm test`, so a smaller smoke run can target a suite: + +```sh +pnpm profile:compiler snapshot +``` + +Each run is stored under `benchmarking/.profiles/compiler-tests/`. Set +`SVELTE_PROFILE_NAME` to give a run a stable name; existing runs are never overwritten. +`benchmarking/.profiles/compiler-tests/latest.txt` contains the latest successful run name. + +Each run contains: + +- `raw/*.cpuprofile`: the original V8 profile from every Vitest worker +- `flamegraph.speedscope.json`: compiler-active stacks merged for use in + [Speedscope](https://www.speedscope.app/) +- `summary.md`: ranked self/inclusive hotspots and files +- `summary.json`: the same measurements for further analysis +- `manifest.json`: the command, revision, timestamps, and test result + +The merged profile includes downstream callees while a compiler source frame is on the stack. +This captures parser, walker, and printer dependencies without including unrelated test runtime. +V8 reports garbage collection outside the originating stack, so the summary reports process-wide +GC separately rather than attributing it to the compiler. + +The default sampling interval is 5 ms. It can be changed for shorter, focused runs: + +```sh +SVELTE_CPU_PROF_INTERVAL=1000 pnpm profile:compiler snapshot +``` + +Existing raw profiles can be analyzed again without rerunning tests: + +```sh +node benchmarking/analyze-compiler-profile.js benchmarking/.profiles/compiler-tests/ +``` diff --git a/benchmarking/profile-compiler.js b/benchmarking/profile-compiler.js new file mode 100644 index 0000000000..323c2d27ea --- /dev/null +++ b/benchmarking/profile-compiler.js @@ -0,0 +1,94 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { analyze_compiler_profiles } from './analyze-compiler-profile.js'; + +const root = path.resolve(fileURLToPath(import.meta.url), '../..'); +const profiles_root = path.join(root, 'benchmarking/.profiles/compiler-tests'); +const requested_name = process.env.SVELTE_PROFILE_NAME; +const run_name = safe(requested_name || new Date().toISOString().replaceAll(':', '-')); +const run_dir = path.join(profiles_root, run_name); +const raw_dir = path.join(run_dir, 'raw'); +const test_args = process.argv.slice(2); +const sampling_interval = process.env.SVELTE_CPU_PROF_INTERVAL || '5000'; + +if (fs.existsSync(run_dir)) { + console.error(`Profile run already exists: ${path.relative(root, run_dir)}`); + process.exit(1); +} + +fs.mkdirSync(raw_dir, { recursive: true }); + +const revision = spawnSync('git', ['rev-parse', 'HEAD'], { + cwd: root, + encoding: 'utf8' +}); +const started_at = new Date().toISOString(); +const manifest_file = path.join(run_dir, 'manifest.json'); +const command = [ + 'pnpm', + 'test', + '--execArgv=--cpu-prof', + `--execArgv=--cpu-prof-dir=${raw_dir}`, + `--execArgv=--cpu-prof-interval=${sampling_interval}`, + ...test_args +]; + +write_manifest({ + command, + revision: revision.status === 0 ? revision.stdout.trim() : null, + started_at, + status: 'running' +}); + +console.log(`Compiler profiles will be written to ${path.relative(root, run_dir)}`); + +const result = spawnSync(command[0], command.slice(1), { + cwd: root, + stdio: 'inherit' +}); + +let analysis_error = null; + +try { + analyze_compiler_profiles(run_dir); + fs.writeFileSync(path.join(profiles_root, 'latest.txt'), `${run_name}\n`); +} catch (error) { + analysis_error = error instanceof Error ? error.stack || error.message : String(error); + console.error(analysis_error); +} + +write_manifest({ + command, + revision: revision.status === 0 ? revision.stdout.trim() : null, + started_at, + finished_at: new Date().toISOString(), + status: result.status === 0 && analysis_error === null ? 'completed' : 'failed', + test_exit_code: result.status, + test_signal: result.signal, + analysis_error +}); + +if (result.error) { + console.error(result.error); +} + +if (result.status === 0 && analysis_error === null) { + console.log( + `Compiler flame graph: ${path.relative(root, path.join(run_dir, 'flamegraph.speedscope.json'))}` + ); + console.log(`Compiler hotspot summary: ${path.relative(root, path.join(run_dir, 'summary.md'))}`); +} else { + process.exitCode = result.status || 1; +} + +/** @param {string} value */ +function safe(value) { + return value.replace(/[^a-z0-9._-]+/gi, '_'); +} + +/** @param {Record} manifest */ +function write_manifest(manifest) { + fs.writeFileSync(manifest_file, `${JSON.stringify(manifest, null, '\t')}\n`); +} diff --git a/benchmarking/run.js b/benchmarking/run.js index e44e816dd0..322960e866 100644 --- a/benchmarking/run.js +++ b/benchmarking/run.js @@ -1,6 +1,7 @@ import { fork } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import * as $ from '../packages/svelte/src/internal/client/index.js'; +import { compiler_benchmarks } from './benchmarks/compiler/index.js'; import { reactivity_benchmarks } from './benchmarks/reactivity/index.js'; import { ssr_benchmarks } from './benchmarks/ssr/index.js'; import { with_cpu_profile } from './utils.js'; @@ -11,7 +12,9 @@ const single = process.env.BENCH_SINGLE; if (single) { // child mode — run a single benchmark and report the result to the parent - const benchmark = [...reactivity_benchmarks, ...ssr_benchmarks].find((b) => b.label === single); + const benchmark = [...compiler_benchmarks, ...reactivity_benchmarks, ...ssr_benchmarks].find( + (b) => b.label === single + ); if (!benchmark) { throw new Error(`Unknown benchmark ${single}`); @@ -33,6 +36,13 @@ if (single) { const filters = process.argv.slice(2); const suites = [ + // Commenting out because we rarely need to run it + // { + // benchmarks: compiler_benchmarks.filter( + // (b) => filters.length === 0 || filters.some((f) => b.label.includes(f)) + // ), + // name: 'compiler benchmarks' + // }, { benchmarks: reactivity_benchmarks.filter( (b) => filters.length === 0 || filters.some((f) => b.label.includes(f)) diff --git a/package.json b/package.json index 8004698dd1..41f93d526c 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "lint": "eslint && prettier --check .", "format": "prettier --write .", "test": "vitest run", + "profile:compiler": "node ./benchmarking/profile-compiler.js", "changeset:version": "changeset version && pnpm -r generate:version && git add --all", "changeset:publish": "changeset publish", "bench": "NODE_ENV=production node --allow-natives-syntax ./benchmarking/run.js",