From c7f6805c3711daf7e147c484ffcfe1828cc5968b Mon Sep 17 00:00:00 2001
From: Dominic Gannaway <trueadm@users.noreply.github.com>
Date: Wed, 19 Jun 2024 20:52:20 +0100
Subject: [PATCH] chore: add more benchmarks (#12094)

chore: add more benchmarks
---
 .github/workflows/ci.yml                      | 14 ++++
 .../benchmarks/kairo/kairo_avoidable.js       | 60 ++++++++++++++
 benchmarking/benchmarks/kairo/kairo_broad.js  | 66 +++++++++++++++
 benchmarking/benchmarks/kairo/kairo_deep.js   | 66 +++++++++++++++
 .../benchmarks/kairo/kairo_diamond.js         | 70 ++++++++++++++++
 benchmarking/benchmarks/kairo/kairo_mux.js    | 63 +++++++++++++++
 .../benchmarks/kairo/kairo_repeated.js        | 67 ++++++++++++++++
 .../benchmarks/kairo/kairo_triangle.js        | 80 +++++++++++++++++++
 .../benchmarks/kairo/kairo_unstable.js        | 66 +++++++++++++++
 benchmarking/benchmarks/kairo/util.js         |  7 ++
 benchmarking/benchmarks/mol_bench.js          | 15 ++--
 benchmarking/run.js                           | 43 ++++++++--
 benchmarking/utils.js                         | 13 ++-
 13 files changed, 616 insertions(+), 14 deletions(-)
 create mode 100644 benchmarking/benchmarks/kairo/kairo_avoidable.js
 create mode 100644 benchmarking/benchmarks/kairo/kairo_broad.js
 create mode 100644 benchmarking/benchmarks/kairo/kairo_deep.js
 create mode 100644 benchmarking/benchmarks/kairo/kairo_diamond.js
 create mode 100644 benchmarking/benchmarks/kairo/kairo_mux.js
 create mode 100644 benchmarking/benchmarks/kairo/kairo_repeated.js
 create mode 100644 benchmarking/benchmarks/kairo/kairo_triangle.js
 create mode 100644 benchmarking/benchmarks/kairo/kairo_unstable.js
 create mode 100644 benchmarking/benchmarks/kairo/util.js

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 572de29b85..d3c8b3955c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -60,3 +60,17 @@ jobs:
       - name: build and check generated types
         if: (${{ success() }} || ${{ failure() }}) # ensures this step runs even if previous steps fail
         run: pnpm build && { [ "`git status --porcelain=v1`" == "" ] || (echo "Generated types have changed — please regenerate types locally and commit the changes after you have reviewed them"; git diff; exit 1); }
+  Benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v3
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm bench
+        env:
+          CI: true
diff --git a/benchmarking/benchmarks/kairo/kairo_avoidable.js b/benchmarking/benchmarks/kairo/kairo_avoidable.js
new file mode 100644
index 0000000000..636e96ccce
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_avoidable.js
@@ -0,0 +1,60 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+import { busy } from './util.js';
+
+function setup() {
+	let head = $.source(0);
+	let computed1 = $.derived(() => $.get(head));
+	let computed2 = $.derived(() => ($.get(computed1), 0));
+	let computed3 = $.derived(() => (busy(), $.get(computed2) + 1)); // heavy computation
+	let computed4 = $.derived(() => $.get(computed3) + 2);
+	let computed5 = $.derived(() => $.get(computed4) + 3);
+
+	const destroy = $.effect_root(() => {
+		$.render_effect(() => {
+			$.get(computed5);
+			busy(); // heavy side effect
+		});
+	});
+
+	return {
+		destroy,
+		run() {
+			$.flush_sync(() => {
+				$.set(head, 1);
+			});
+			assert($.get(computed5) === 6);
+			for (let i = 0; i < 1000; i++) {
+				$.flush_sync(() => {
+					$.set(head, i);
+				});
+				assert($.get(computed5) === 6);
+			}
+		}
+	};
+}
+
+export async function kairo_avoidable() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_avoidable',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/kairo_broad.js b/benchmarking/benchmarks/kairo/kairo_broad.js
new file mode 100644
index 0000000000..682154f0e3
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_broad.js
@@ -0,0 +1,66 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+
+function setup() {
+	let head = $.source(0);
+	let last = head;
+	let counter = 0;
+
+	const destroy = $.effect_root(() => {
+		for (let i = 0; i < 50; i++) {
+			let current = $.derived(() => {
+				return $.get(head) + i;
+			});
+			let current2 = $.derived(() => {
+				return $.get(current) + 1;
+			});
+			$.render_effect(() => {
+				$.get(current2);
+				counter++;
+			});
+			last = current2;
+		}
+	});
+
+	return {
+		destroy,
+		run() {
+			$.flush_sync(() => {
+				$.set(head, 1);
+			});
+			counter = 0
+			for (let i = 0; i < 50; i++) {
+				$.flush_sync(() => {
+					$.set(head, i);
+				});
+				assert($.get(last) === i + 50);
+			}
+			assert(counter === 50 * 50);
+		}
+	};
+}
+
+export async function kairo_broad() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_broad',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/kairo_deep.js b/benchmarking/benchmarks/kairo/kairo_deep.js
new file mode 100644
index 0000000000..af985c3e43
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_deep.js
@@ -0,0 +1,66 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+
+let len = 50;
+const iter = 50;
+
+function setup() {
+	let head = $.source(0);
+	let current = head;
+	for (let i = 0; i < len; i++) {
+    let c = current;
+    current = $.derived(() => {
+      return $.get(c) + 1;
+    });
+  }
+  let counter = 0;
+
+	const destroy = $.effect_root(() => {
+		$.render_effect(() => {
+			$.get(current);
+			counter++;
+		});
+	});
+
+	return {
+		destroy,
+		run() {
+			$.flush_sync(() => {
+				$.set(head, 1);
+			});
+			counter = 0
+			for (let i = 0; i < iter; i++) {
+				$.flush_sync(() => {
+					$.set(head, i);
+				});
+				assert($.get(current) === len + i);
+			}
+			assert(counter === iter);
+		}
+	};
+}
+
+export async function kairo_deep() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_deep',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/kairo_diamond.js b/benchmarking/benchmarks/kairo/kairo_diamond.js
new file mode 100644
index 0000000000..879727c99e
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_diamond.js
@@ -0,0 +1,70 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+
+let width = 5;
+
+function setup() {
+	let head = $.source(0);
+  let current = [];
+  for (let i = 0; i < width; i++) {
+    current.push(
+      $.derived(() => {
+        return $.get(head) + 1;
+      })
+    );
+  }
+  let sum = $.derived(() => {
+    return current.map((x) => $.get(x)).reduce((a, b) => a + b, 0);
+  });
+	let counter = 0;
+
+	const destroy = $.effect_root(() => {
+		$.render_effect(() => {
+			$.get(sum);
+			counter++;
+		});
+	});
+
+	return {
+		destroy,
+		run() {
+			$.flush_sync(() => {
+				$.set(head, 1);
+			});
+			assert($.get(sum) === 2 * width);
+			counter = 0;
+			for (let i = 0; i < 500; i++) {
+				$.flush_sync(() => {
+					$.set(head, i);
+				});
+				assert($.get(sum) === (i + 1) * width);
+			}
+			assert(counter === 500);
+		}
+	};
+}
+
+export async function kairo_diamond() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_diamond',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/kairo_mux.js b/benchmarking/benchmarks/kairo/kairo_mux.js
new file mode 100644
index 0000000000..d2867f499b
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_mux.js
@@ -0,0 +1,63 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+
+function setup() {
+	let heads = new Array(100).fill(null).map((_) => $.source(0));
+  const mux = $.derived(() => {
+    return Object.fromEntries(heads.map((h) => $.get(h)).entries());
+  });
+  const splited = heads
+    .map((_, index) => $.derived(() => $.get(mux)[index]))
+    .map((x) => $.derived(() => $.get(x) + 1));
+
+	const destroy = $.effect_root(() => {
+		splited.forEach((x) => {
+			$.render_effect(() => {
+				$.get(x);
+			});
+		});
+	});
+
+	return {
+		destroy,
+		run() {
+			for (let i = 0; i < 10; i++) {
+				$.flush_sync(() => {
+					$.set(heads[i], i);
+				});
+				assert($.get(splited[i]) === i + 1);
+			}
+			for (let i = 0; i < 10; i++) {
+				$.flush_sync(() => {
+					$.set(heads[i], i * 2);
+				});
+				assert($.get(splited[i]) === i * 2 + 1);
+			}
+		}
+	};
+}
+
+export async function kairo_mux() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_mux',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/kairo_repeated.js b/benchmarking/benchmarks/kairo/kairo_repeated.js
new file mode 100644
index 0000000000..fd22c1e563
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_repeated.js
@@ -0,0 +1,67 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+
+let size = 30;
+
+function setup() {
+  let head = $.source(0);
+  let current = $.derived(() => {
+    let result = 0;
+    for (let i = 0; i < size; i++) {
+      result += $.get(head);
+    }
+    return result;
+  });
+
+	let counter = 0;
+
+	const destroy = $.effect_root(() => {
+		$.render_effect(() => {
+			$.get(current);
+			counter++;
+		});
+	});
+
+	return {
+		destroy,
+		run() {
+			$.flush_sync(() => {
+				$.set(head, 1);
+			});
+			assert($.get(current) === size);
+			counter = 0;
+			for (let i = 0; i < 100; i++) {
+				$.flush_sync(() => {
+					$.set(head, i);
+				});
+				assert($.get(current) === i * size);
+			}
+			assert(counter === 100);
+		}
+	};
+}
+
+export async function kairo_repeated() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_repeated',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/kairo_triangle.js b/benchmarking/benchmarks/kairo/kairo_triangle.js
new file mode 100644
index 0000000000..4e8afe1b82
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_triangle.js
@@ -0,0 +1,80 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+
+let width = 10;
+
+function count(number) {
+  return new Array(number)
+    .fill(0)
+    .map((_, i) => i + 1)
+    .reduce((x, y) => x + y, 0);
+}
+
+function setup() {
+	let head = $.source(0);
+  let current = head;
+  let list = [];
+  for (let i = 0; i < width; i++) {
+    let c = current;
+    list.push(current);
+    current = $.derived(() => {
+      return $.get(c) + 1;
+    });
+  }
+  let sum = $.derived(() => {
+    return list.map((x) => $.get(x)).reduce((a, b) => a + b, 0);
+  });
+
+	let counter = 0;
+
+	const destroy = $.effect_root(() => {
+		$.render_effect(() => {
+			$.get(sum);
+			counter++;
+		});
+	});
+
+	return {
+		destroy,
+		run() {
+			const constant = count(width);
+			$.flush_sync(() => {
+				$.set(head, 1);
+			});
+			assert($.get(sum) === constant);
+			counter = 0;
+			for (let i = 0; i < 100; i++) {
+				$.flush_sync(() => {
+					$.set(head, i);
+				});
+				assert($.get(sum) === constant - width + i * width);
+			}
+			assert(counter === 100);
+		}
+	};
+}
+
+export async function kairo_triangle() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_triangle',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/kairo_unstable.js b/benchmarking/benchmarks/kairo/kairo_unstable.js
new file mode 100644
index 0000000000..0185d12868
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/kairo_unstable.js
@@ -0,0 +1,66 @@
+import { assert, fastest_test } from '../../utils.js';
+import * as $ from '../../../packages/svelte/src/internal/client/index.js';
+
+function setup() {
+  let head = $.source(0);
+  const double = $.derived(() => $.get(head) * 2);
+  const inverse = $.derived(() => -$.get(head));
+  let current = $.derived(() => {
+    let result = 0;
+    for (let i = 0; i < 20; i++) {
+      result += $.get(head) % 2 ? $.get(double) : $.get(inverse);
+    }
+    return result;
+  });
+
+	let counter = 0;
+
+	const destroy = $.effect_root(() => {
+		$.render_effect(() => {
+			$.get(current);
+			counter++;
+		});
+	});
+
+	return {
+		destroy,
+		run() {
+			$.flush_sync(() => {
+				$.set(head, 1);
+			});
+			assert($.get(current) === 40);
+			counter = 0;
+			for (let i = 0; i < 100; i++) {
+				$.flush_sync(() => {
+					$.set(head, i);
+				});
+			}
+			assert(counter === 100);
+		}
+	};
+}
+
+export async function kairo_unstable() {
+	// Do 10 loops to warm up JIT
+	for (let i = 0; i < 10; i++) {
+		const { run, destroy } = setup();
+		run();
+		destroy();
+	}
+
+	const { run, destroy } = setup();
+
+	const { timing } = await fastest_test(10, () => {
+		for (let i = 0; i < 100; i++) {
+			run();
+		}
+	});
+
+	destroy();
+
+	return {
+		benchmark: 'kairo_unstable',
+		time: timing.time.toFixed(2),
+		gc_time: timing.gc_time.toFixed(2)
+	};
+}
diff --git a/benchmarking/benchmarks/kairo/util.js b/benchmarking/benchmarks/kairo/util.js
new file mode 100644
index 0000000000..5c73e99eec
--- /dev/null
+++ b/benchmarking/benchmarks/kairo/util.js
@@ -0,0 +1,7 @@
+
+export function busy() {
+  let a = 0;
+  for (let i = 0; i < 1_00; i++) {
+    a++;
+  }
+}
diff --git a/benchmarking/benchmarks/mol_bench.js b/benchmarking/benchmarks/mol_bench.js
index f98521c809..7fa49d7df4 100644
--- a/benchmarking/benchmarks/mol_bench.js
+++ b/benchmarking/benchmarks/mol_bench.js
@@ -1,4 +1,4 @@
-import { fastest_test } from '../utils.js';
+import { assert, fastest_test } from '../utils.js';
 import * as $ from '../../packages/svelte/src/internal/client/index.js';
 
 /**
@@ -23,10 +23,14 @@ function setup() {
 	const A = $.source(0);
 	const B = $.source(0);
 	const C = $.derived(() => ($.get(A) % 2) + ($.get(B) % 2));
-	const D = $.derived(() => numbers.map((i) => ({ x: i + ($.get(A) % 2) - ($.get(B) % 2) })));
-	const E = $.derived(() => hard($.get(C) + $.get(A) + $.get(D)[0].x));
-	const F = $.derived(() => hard($.get(D)[2].x || $.get(B)));
-	const G = $.derived(() => $.get(C) + ($.get(C) || $.get(E) % 2) + $.get(D)[4].x + $.get(F));
+	const D = $.derived(() => numbers.map((i) => i + ($.get(A) % 2) - ($.get(B) % 2)));
+	D.equals = function (/** @type {number[]} */ l) {
+		var r = this.v;
+		return r !== null && l.length === r.length && l.every((v, i) => v === r[i]);
+	};
+	const E = $.derived(() => hard($.get(C) + $.get(A) + $.get(D)[0]));
+	const F = $.derived(() => hard($.get(D)[0] && $.get(B)));
+	const G = $.derived(() => $.get(C) + ($.get(C) || $.get(E) % 2) + $.get(D)[0] + $.get(F));
 
 	const destroy = $.effect_root(() => {
 		$.render_effect(() => {
@@ -55,6 +59,7 @@ function setup() {
 				$.set(A, 2 + i * 2);
 				$.set(B, 2);
 			});
+			assert(res[0] === 3198 && res[1] === 1601 && res[2] === 3195 && res[3] === 1598);
 		}
 	};
 }
diff --git a/benchmarking/run.js b/benchmarking/run.js
index 060b982a85..8114641be6 100644
--- a/benchmarking/run.js
+++ b/benchmarking/run.js
@@ -1,19 +1,48 @@
 import * as $ from '../packages/svelte/src/internal/client/index.js';
+import { kairo_avoidable } from './benchmarks/kairo/kairo_avoidable.js';
+import { kairo_broad } from './benchmarks/kairo/kairo_broad.js';
+import { kairo_deep } from './benchmarks/kairo/kairo_deep.js';
+import { kairo_diamond } from './benchmarks/kairo/kairo_diamond.js';
+import { kairo_mux } from './benchmarks/kairo/kairo_mux.js';
+import { kairo_repeated } from './benchmarks/kairo/kairo_repeated.js';
+import { kairo_triangle } from './benchmarks/kairo/kairo_triangle.js';
+import { kairo_unstable } from './benchmarks/kairo/kairo_unstable.js';
 import { mol_bench } from './benchmarks/mol_bench.js';
 
-const benchmarks = [mol_bench];
+// This benchmark has been adapted from the js-reactivity-benchmark (https://github.com/milomg/js-reactivity-benchmark)
+// Not all tests are the same, and many parts have been tweaked to capture different data.
 
-async function run_benchmarks() {
-	const results = [];
+const benchmarks = [
+	kairo_avoidable,
+	kairo_broad,
+	kairo_deep,
+	kairo_diamond,
+	kairo_triangle,
+	kairo_mux,
+	kairo_repeated,
+	kairo_unstable,
+	mol_bench
+];
 
+async function run_benchmarks() {
+	// eslint-disable-next-line no-console
+	console.log('-- Benchmarking Started --');
 	$.push({}, true);
-	for (const benchmark of benchmarks) {
-		results.push(await benchmark());
+	try {
+		for (const benchmark of benchmarks) {
+			// eslint-disable-next-line no-console
+			console.log(await benchmark());
+		}
+	} catch (e) {
+		// eslint-disable-next-line no-console
+		console.error('-- Benchmarking Failed --');
+		// eslint-disable-next-line no-console
+		console.error(e);
+		process.exit(1);
 	}
 	$.pop();
-
 	// eslint-disable-next-line no-console
-	console.log(results);
+	console.log('-- Benchmarking Complete --');
 }
 
 run_benchmarks();
diff --git a/benchmarking/utils.js b/benchmarking/utils.js
index d9594c1f03..db2fa753ee 100644
--- a/benchmarking/utils.js
+++ b/benchmarking/utils.js
@@ -20,8 +20,8 @@ class GarbageTrack {
 	}
 
 	/**
-     * @param {number} track_id
-     */
+	 * @param {number} track_id
+	 */
 	async gcDuration(track_id) {
 		await promise_delay(10);
 
@@ -87,3 +87,12 @@ export async function fastest_test(times, fn) {
 
 	return fastest;
 }
+
+/**
+ * @param {boolean} a
+ */
+export function assert(a) {
+	if (!a) {
+		throw new Error('Assertion failed');
+	}
+}