From c7f6805c3711daf7e147c484ffcfe1828cc5968b Mon Sep 17 00:00:00 2001 From: Dominic Gannaway Date: Wed, 19 Jun 2024 20:52:20 +0100 Subject: [PATCH] chore: add more benchmarks (#12094) chore: add more benchmarks --- .github/workflows/ci.yml | 14 ++++ .../benchmarks/kairo/kairo_avoidable.js | 60 ++++++++++++++ benchmarking/benchmarks/kairo/kairo_broad.js | 66 +++++++++++++++ benchmarking/benchmarks/kairo/kairo_deep.js | 66 +++++++++++++++ .../benchmarks/kairo/kairo_diamond.js | 70 ++++++++++++++++ benchmarking/benchmarks/kairo/kairo_mux.js | 63 +++++++++++++++ .../benchmarks/kairo/kairo_repeated.js | 67 ++++++++++++++++ .../benchmarks/kairo/kairo_triangle.js | 80 +++++++++++++++++++ .../benchmarks/kairo/kairo_unstable.js | 66 +++++++++++++++ benchmarking/benchmarks/kairo/util.js | 7 ++ benchmarking/benchmarks/mol_bench.js | 15 ++-- benchmarking/run.js | 43 ++++++++-- benchmarking/utils.js | 13 ++- 13 files changed, 616 insertions(+), 14 deletions(-) create mode 100644 benchmarking/benchmarks/kairo/kairo_avoidable.js create mode 100644 benchmarking/benchmarks/kairo/kairo_broad.js create mode 100644 benchmarking/benchmarks/kairo/kairo_deep.js create mode 100644 benchmarking/benchmarks/kairo/kairo_diamond.js create mode 100644 benchmarking/benchmarks/kairo/kairo_mux.js create mode 100644 benchmarking/benchmarks/kairo/kairo_repeated.js create mode 100644 benchmarking/benchmarks/kairo/kairo_triangle.js create mode 100644 benchmarking/benchmarks/kairo/kairo_unstable.js create mode 100644 benchmarking/benchmarks/kairo/util.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 572de29b85..d3c8b3955c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,3 +60,17 @@ jobs: - name: build and check generated types if: (${{ success() }} || ${{ failure() }}) # ensures this step runs even if previous steps fail run: pnpm build && { [ "`git status --porcelain=v1`" == "" ] || (echo "Generated types have changed — please regenerate types locally and commit the changes after you have reviewed them"; git diff; exit 1); } + Benchmarks: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v3 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm bench + env: + CI: true diff --git a/benchmarking/benchmarks/kairo/kairo_avoidable.js b/benchmarking/benchmarks/kairo/kairo_avoidable.js new file mode 100644 index 0000000000..636e96ccce --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_avoidable.js @@ -0,0 +1,60 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; +import { busy } from './util.js'; + +function setup() { + let head = $.source(0); + let computed1 = $.derived(() => $.get(head)); + let computed2 = $.derived(() => ($.get(computed1), 0)); + let computed3 = $.derived(() => (busy(), $.get(computed2) + 1)); // heavy computation + let computed4 = $.derived(() => $.get(computed3) + 2); + let computed5 = $.derived(() => $.get(computed4) + 3); + + const destroy = $.effect_root(() => { + $.render_effect(() => { + $.get(computed5); + busy(); // heavy side effect + }); + }); + + return { + destroy, + run() { + $.flush_sync(() => { + $.set(head, 1); + }); + assert($.get(computed5) === 6); + for (let i = 0; i < 1000; i++) { + $.flush_sync(() => { + $.set(head, i); + }); + assert($.get(computed5) === 6); + } + } + }; +} + +export async function kairo_avoidable() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_avoidable', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/kairo_broad.js b/benchmarking/benchmarks/kairo/kairo_broad.js new file mode 100644 index 0000000000..682154f0e3 --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_broad.js @@ -0,0 +1,66 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; + +function setup() { + let head = $.source(0); + let last = head; + let counter = 0; + + const destroy = $.effect_root(() => { + for (let i = 0; i < 50; i++) { + let current = $.derived(() => { + return $.get(head) + i; + }); + let current2 = $.derived(() => { + return $.get(current) + 1; + }); + $.render_effect(() => { + $.get(current2); + counter++; + }); + last = current2; + } + }); + + return { + destroy, + run() { + $.flush_sync(() => { + $.set(head, 1); + }); + counter = 0 + for (let i = 0; i < 50; i++) { + $.flush_sync(() => { + $.set(head, i); + }); + assert($.get(last) === i + 50); + } + assert(counter === 50 * 50); + } + }; +} + +export async function kairo_broad() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_broad', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/kairo_deep.js b/benchmarking/benchmarks/kairo/kairo_deep.js new file mode 100644 index 0000000000..af985c3e43 --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_deep.js @@ -0,0 +1,66 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; + +let len = 50; +const iter = 50; + +function setup() { + let head = $.source(0); + let current = head; + for (let i = 0; i < len; i++) { + let c = current; + current = $.derived(() => { + return $.get(c) + 1; + }); + } + let counter = 0; + + const destroy = $.effect_root(() => { + $.render_effect(() => { + $.get(current); + counter++; + }); + }); + + return { + destroy, + run() { + $.flush_sync(() => { + $.set(head, 1); + }); + counter = 0 + for (let i = 0; i < iter; i++) { + $.flush_sync(() => { + $.set(head, i); + }); + assert($.get(current) === len + i); + } + assert(counter === iter); + } + }; +} + +export async function kairo_deep() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_deep', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/kairo_diamond.js b/benchmarking/benchmarks/kairo/kairo_diamond.js new file mode 100644 index 0000000000..879727c99e --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_diamond.js @@ -0,0 +1,70 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; + +let width = 5; + +function setup() { + let head = $.source(0); + let current = []; + for (let i = 0; i < width; i++) { + current.push( + $.derived(() => { + return $.get(head) + 1; + }) + ); + } + let sum = $.derived(() => { + return current.map((x) => $.get(x)).reduce((a, b) => a + b, 0); + }); + let counter = 0; + + const destroy = $.effect_root(() => { + $.render_effect(() => { + $.get(sum); + counter++; + }); + }); + + return { + destroy, + run() { + $.flush_sync(() => { + $.set(head, 1); + }); + assert($.get(sum) === 2 * width); + counter = 0; + for (let i = 0; i < 500; i++) { + $.flush_sync(() => { + $.set(head, i); + }); + assert($.get(sum) === (i + 1) * width); + } + assert(counter === 500); + } + }; +} + +export async function kairo_diamond() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_diamond', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/kairo_mux.js b/benchmarking/benchmarks/kairo/kairo_mux.js new file mode 100644 index 0000000000..d2867f499b --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_mux.js @@ -0,0 +1,63 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; + +function setup() { + let heads = new Array(100).fill(null).map((_) => $.source(0)); + const mux = $.derived(() => { + return Object.fromEntries(heads.map((h) => $.get(h)).entries()); + }); + const splited = heads + .map((_, index) => $.derived(() => $.get(mux)[index])) + .map((x) => $.derived(() => $.get(x) + 1)); + + const destroy = $.effect_root(() => { + splited.forEach((x) => { + $.render_effect(() => { + $.get(x); + }); + }); + }); + + return { + destroy, + run() { + for (let i = 0; i < 10; i++) { + $.flush_sync(() => { + $.set(heads[i], i); + }); + assert($.get(splited[i]) === i + 1); + } + for (let i = 0; i < 10; i++) { + $.flush_sync(() => { + $.set(heads[i], i * 2); + }); + assert($.get(splited[i]) === i * 2 + 1); + } + } + }; +} + +export async function kairo_mux() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_mux', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/kairo_repeated.js b/benchmarking/benchmarks/kairo/kairo_repeated.js new file mode 100644 index 0000000000..fd22c1e563 --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_repeated.js @@ -0,0 +1,67 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; + +let size = 30; + +function setup() { + let head = $.source(0); + let current = $.derived(() => { + let result = 0; + for (let i = 0; i < size; i++) { + result += $.get(head); + } + return result; + }); + + let counter = 0; + + const destroy = $.effect_root(() => { + $.render_effect(() => { + $.get(current); + counter++; + }); + }); + + return { + destroy, + run() { + $.flush_sync(() => { + $.set(head, 1); + }); + assert($.get(current) === size); + counter = 0; + for (let i = 0; i < 100; i++) { + $.flush_sync(() => { + $.set(head, i); + }); + assert($.get(current) === i * size); + } + assert(counter === 100); + } + }; +} + +export async function kairo_repeated() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_repeated', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/kairo_triangle.js b/benchmarking/benchmarks/kairo/kairo_triangle.js new file mode 100644 index 0000000000..4e8afe1b82 --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_triangle.js @@ -0,0 +1,80 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; + +let width = 10; + +function count(number) { + return new Array(number) + .fill(0) + .map((_, i) => i + 1) + .reduce((x, y) => x + y, 0); +} + +function setup() { + let head = $.source(0); + let current = head; + let list = []; + for (let i = 0; i < width; i++) { + let c = current; + list.push(current); + current = $.derived(() => { + return $.get(c) + 1; + }); + } + let sum = $.derived(() => { + return list.map((x) => $.get(x)).reduce((a, b) => a + b, 0); + }); + + let counter = 0; + + const destroy = $.effect_root(() => { + $.render_effect(() => { + $.get(sum); + counter++; + }); + }); + + return { + destroy, + run() { + const constant = count(width); + $.flush_sync(() => { + $.set(head, 1); + }); + assert($.get(sum) === constant); + counter = 0; + for (let i = 0; i < 100; i++) { + $.flush_sync(() => { + $.set(head, i); + }); + assert($.get(sum) === constant - width + i * width); + } + assert(counter === 100); + } + }; +} + +export async function kairo_triangle() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_triangle', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/kairo_unstable.js b/benchmarking/benchmarks/kairo/kairo_unstable.js new file mode 100644 index 0000000000..0185d12868 --- /dev/null +++ b/benchmarking/benchmarks/kairo/kairo_unstable.js @@ -0,0 +1,66 @@ +import { assert, fastest_test } from '../../utils.js'; +import * as $ from '../../../packages/svelte/src/internal/client/index.js'; + +function setup() { + let head = $.source(0); + const double = $.derived(() => $.get(head) * 2); + const inverse = $.derived(() => -$.get(head)); + let current = $.derived(() => { + let result = 0; + for (let i = 0; i < 20; i++) { + result += $.get(head) % 2 ? $.get(double) : $.get(inverse); + } + return result; + }); + + let counter = 0; + + const destroy = $.effect_root(() => { + $.render_effect(() => { + $.get(current); + counter++; + }); + }); + + return { + destroy, + run() { + $.flush_sync(() => { + $.set(head, 1); + }); + assert($.get(current) === 40); + counter = 0; + for (let i = 0; i < 100; i++) { + $.flush_sync(() => { + $.set(head, i); + }); + } + assert(counter === 100); + } + }; +} + +export async function kairo_unstable() { + // Do 10 loops to warm up JIT + for (let i = 0; i < 10; i++) { + const { run, destroy } = setup(); + run(); + destroy(); + } + + const { run, destroy } = setup(); + + const { timing } = await fastest_test(10, () => { + for (let i = 0; i < 100; i++) { + run(); + } + }); + + destroy(); + + return { + benchmark: 'kairo_unstable', + time: timing.time.toFixed(2), + gc_time: timing.gc_time.toFixed(2) + }; +} diff --git a/benchmarking/benchmarks/kairo/util.js b/benchmarking/benchmarks/kairo/util.js new file mode 100644 index 0000000000..5c73e99eec --- /dev/null +++ b/benchmarking/benchmarks/kairo/util.js @@ -0,0 +1,7 @@ + +export function busy() { + let a = 0; + for (let i = 0; i < 1_00; i++) { + a++; + } +} diff --git a/benchmarking/benchmarks/mol_bench.js b/benchmarking/benchmarks/mol_bench.js index f98521c809..7fa49d7df4 100644 --- a/benchmarking/benchmarks/mol_bench.js +++ b/benchmarking/benchmarks/mol_bench.js @@ -1,4 +1,4 @@ -import { fastest_test } from '../utils.js'; +import { assert, fastest_test } from '../utils.js'; import * as $ from '../../packages/svelte/src/internal/client/index.js'; /** @@ -23,10 +23,14 @@ function setup() { const A = $.source(0); const B = $.source(0); const C = $.derived(() => ($.get(A) % 2) + ($.get(B) % 2)); - const D = $.derived(() => numbers.map((i) => ({ x: i + ($.get(A) % 2) - ($.get(B) % 2) }))); - const E = $.derived(() => hard($.get(C) + $.get(A) + $.get(D)[0].x)); - const F = $.derived(() => hard($.get(D)[2].x || $.get(B))); - const G = $.derived(() => $.get(C) + ($.get(C) || $.get(E) % 2) + $.get(D)[4].x + $.get(F)); + const D = $.derived(() => numbers.map((i) => i + ($.get(A) % 2) - ($.get(B) % 2))); + D.equals = function (/** @type {number[]} */ l) { + var r = this.v; + return r !== null && l.length === r.length && l.every((v, i) => v === r[i]); + }; + const E = $.derived(() => hard($.get(C) + $.get(A) + $.get(D)[0])); + const F = $.derived(() => hard($.get(D)[0] && $.get(B))); + const G = $.derived(() => $.get(C) + ($.get(C) || $.get(E) % 2) + $.get(D)[0] + $.get(F)); const destroy = $.effect_root(() => { $.render_effect(() => { @@ -55,6 +59,7 @@ function setup() { $.set(A, 2 + i * 2); $.set(B, 2); }); + assert(res[0] === 3198 && res[1] === 1601 && res[2] === 3195 && res[3] === 1598); } }; } diff --git a/benchmarking/run.js b/benchmarking/run.js index 060b982a85..8114641be6 100644 --- a/benchmarking/run.js +++ b/benchmarking/run.js @@ -1,19 +1,48 @@ import * as $ from '../packages/svelte/src/internal/client/index.js'; +import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js'; +import { kairo_broad } from './benchmarks/kairo/kairo_broad.js'; +import { kairo_deep } from './benchmarks/kairo/kairo_deep.js'; +import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js'; +import { kairo_mux } from './benchmarks/kairo/kairo_mux.js'; +import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js'; +import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js'; +import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js'; import { mol_bench } from './benchmarks/mol_bench.js'; -const benchmarks = [mol_bench]; +// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark) +// Not all tests are the same, and many parts have been tweaked to capture different data. -async function run_benchmarks() { - const results = []; +const benchmarks = [ + kairo_avoidable, + kairo_broad, + kairo_deep, + kairo_diamond, + kairo_triangle, + kairo_mux, + kairo_repeated, + kairo_unstable, + mol_bench +]; +async function run_benchmarks() { + // eslint-disable-next-line no-console + console.log('-- Benchmarking Started --'); $.push({}, true); - for (const benchmark of benchmarks) { - results.push(await benchmark()); + try { + for (const benchmark of benchmarks) { + // eslint-disable-next-line no-console + console.log(await benchmark()); + } + } catch (e) { + // eslint-disable-next-line no-console + console.error('-- Benchmarking Failed --'); + // eslint-disable-next-line no-console + console.error(e); + process.exit(1); } $.pop(); - // eslint-disable-next-line no-console - console.log(results); + console.log('-- Benchmarking Complete --'); } run_benchmarks(); diff --git a/benchmarking/utils.js b/benchmarking/utils.js index d9594c1f03..db2fa753ee 100644 --- a/benchmarking/utils.js +++ b/benchmarking/utils.js @@ -20,8 +20,8 @@ class GarbageTrack { } /** - * @param {number} track_id - */ + * @param {number} track_id + */ async gcDuration(track_id) { await promise_delay(10); @@ -87,3 +87,12 @@ export async function fastest_test(times, fn) { return fastest; } + +/** + * @param {boolean} a + */ +export function assert(a) { + if (!a) { + throw new Error('Assertion failed'); + } +}