From 38839bc6678beb1a326b55eaaf55ad92e5914d9c Mon Sep 17 00:00:00 2001 From: NGPixel Date: Mon, 21 Sep 2026 01:50:41 -0400 Subject: [PATCH] docs: add wkblock spec document --- CLAUDE.md | 8 +- backend/core/maintenance.ts | 7 + backend/helpers/wkblock.ts | 30 ++-- backend/index.ts | 4 + backend/locales/en.json | 50 +++++- backend/models/blocks.ts | 66 +++++++ blocks/package.mjs | 41 ++--- dev/specs/wkblock.md | 333 ++++++++++++++++++++++++++++++++++++ 8 files changed, 492 insertions(+), 47 deletions(-) create mode 100644 dev/specs/wkblock.md diff --git a/CLAUDE.md b/CLAUDE.md index 415598af8..7c2c2fbe4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -179,9 +179,11 @@ and where its definition is read from. So a block being written is developed in `npm run build` and the compiled tree — and packaged once it works. There is no second authoring API. `blocks/package.mjs` is the packager and `backend/helpers/wkblock.ts` reads what it writes. **The -format is stated in full in both files and has to be kept in step by hand**: `blocks/` and `backend/` -are separately installed workspaces and the backend does not type-check JavaScript, so there is no -module the two halves could share. +format itself is specified in `dev/specs/wkblock.md`, and that document is the contract**: `blocks/` +and `backend/` are separately installed workspaces and the backend does not type-check JavaScript, so +there is no module the two halves could share and nothing that checks one against the other. The two +files are independent implementations of the spec and carry only the rationale for how each is +written — so a change to the format is a change to the spec first, then both files, in one commit. - **One block per package, and the directory name is the identity.** `blocks/block-xyz/` declares `block: 'xyz'`, is packaged as `block-xyz.wkblock`, serves as `block-xyz.js` and renders as diff --git a/backend/core/maintenance.ts b/backend/core/maintenance.ts index 31b0d7b32..6b760f626 100644 --- a/backend/core/maintenance.ts +++ b/backend/core/maintenance.ts @@ -82,8 +82,15 @@ export default { // -> Which blocks a site imported is held per instance too, and is what `/_blocks` answers from. // The files themselves need no propagating: each instance unpacks a block it has not got, or // has at the wrong checksum, the first time a browser asks it for one. + // + // Going the other way does need doing here. The event says the set of custom blocks changed + // but not how, and the instance that deleted one dropped its own files as it went -- so this + // is where every other instance finds out that files it holds are for a block that no longer + // exists. Reconciled rather than told, so an instance that was down for the delete gets there + // too, on the sweep at boot. WIKI.events.inbound.on('reloadBlocks', async () => { await WIKI.models.blocks.refreshCustomIndex() + await WIKI.models.blocks.sweepCache() }) } } diff --git a/backend/helpers/wkblock.ts b/backend/helpers/wkblock.ts index c3906bcbc..40212e76a 100644 --- a/backend/helpers/wkblock.ts +++ b/backend/helpers/wkblock.ts @@ -6,22 +6,30 @@ import type { BlockDefinition, BlockProp } from '../models/blocks.ts' /** * Reading a `.wkblock` — the single file a block is distributed as. * - * `blocks/package.mjs` is the other half of this, and the two have to agree. There is no module to - * share between them: `blocks/` and `backend/` are separately installed workspaces and the backend - * does not type-check JavaScript, so the format is written twice and stated in full in both places. + * ## The format is specified in `dev/specs/wkblock.md` * - * magic 8 bytes "WKBLOCK\0" - * version uint32be format version, 1 - * headerLen uint32be byte length of the header that follows - * header gzip'd JSON — see `PackageHeader` - * payload each file's gzip'd bytes, concatenated in the header's order + * Read it before changing anything here. This file and `blocks/package.mjs` are two independent + * implementations of that document — `blocks/` and `backend/` are separately installed workspaces + * with no module between them, and the backend does not type-check JavaScript, so there is nothing + * the two halves could share and nothing that checks one against the other. The spec is what holds + * them together, so a change to the format is a change to the spec first. + * + * ## Why this file is written the way it is * * Everything here treats the package as something a person uploaded, because that is what it is: * `manage:sites` is the trust boundary for the CODE in it — which runs in every reader's browser on * that site, and is no more and no less than what the raw head and body fields under Theme already - * allow — but the container itself is parsed before anybody has vouched for anything. So every - * length is bounded before it is acted on, every digest is checked, and every path has to fall inside - * the block's own namespace. + * allow — but the container itself is parsed before anybody has vouched for anything. + * + * So every length the header CLAIMS is bounded before a byte of the payload is touched, every file's + * digest is checked against what was decompressed, and every path has to fall inside the block's own + * namespace. That last one is not tidiness: the serving route decides which block answers a request + * from the first path segment alone, so a package reaching outside its namespace would answer for a + * block it is not. + * + * Every refusal is a `CustomError` naming what is wrong, because every one of them is something the + * administrator who uploaded the file can act on — a truncated download, the wrong file, a package + * built by a newer wiki. */ const MAGIC = Buffer.from('WKBLOCK\0', 'latin1') diff --git a/backend/index.ts b/backend/index.ts index 702f90994..6673a0708 100644 --- a/backend/index.ts +++ b/backend/index.ts @@ -203,6 +203,10 @@ async function postBoot() { await WIKI.models.blocks.syncAllSites() // -> And which blocks were imported rather than installed, which is what `/_blocks` answers from await WIKI.models.blocks.refreshCustomIndex() + // -> Then drop whatever is cached for a block or a site that has gone since this instance last ran. + // The cache is a directory in a container while the blocks are rows elsewhere, so an instance + // coming back to a volume it left behind is exactly where stale files accumulate. + await WIKI.models.blocks.sweepCache() // -> Same: every site gets a row per installed storage module await WIKI.models.storage.refreshFromDisk() diff --git a/backend/locales/en.json b/backend/locales/en.json index da7888446..05b14f464 100644 --- a/backend/locales/en.json +++ b/backend/locales/en.json @@ -1444,8 +1444,22 @@ "admin.utilities.generateSampleSuccess": "No sample pages were written. | Wrote 1 sample page. | Wrote {count} sample pages.", "admin.utilities.graphEndpointSubtitle": "Change the GraphQL endpoint for Wiki.js", "admin.utilities.graphEndpointTitle": "GraphQL Endpoint", - "admin.utilities.import": "Import", - "admin.utilities.importHint": "Import content from a tarball backup or a 2.X backup.", + "admin.utilities.groupAuthentication": "Authentication", + "admin.utilities.groupMaintenance": "Maintenance", + "admin.utilities.groupMigration": "Migration / Backup", + "admin.utilities.groupTesting": "Testing", + "admin.utilities.importBackup": "Import from Backup", + "admin.utilities.importBackupHint": "Restore content from a tarball previously created by Export.", + "admin.utilities.importConfluence": "Import from Confluence", + "admin.utilities.importConfluenceHint": "Migrate spaces and pages from a Confluence export.", + "admin.utilities.importDokuwiki": "Import from DokuWiki", + "admin.utilities.importDokuwikiHint": "Migrate pages and media from a DokuWiki data directory.", + "admin.utilities.importNotion": "Import from Notion", + "admin.utilities.importNotionHint": "Migrate pages from a Notion workspace export.", + "admin.utilities.importWikijs2": "Import from Wiki.js 2.x", + "admin.utilities.importWikijs2Hint": "Migrate content from a Wiki.js 2.x backup.", + "admin.utilities.importXwiki": "Import from XWiki", + "admin.utilities.importXwikiHint": "Migrate pages and attachments from an XWiki XAR export.", "admin.utilities.importv1Subtitle": "Migrate data from a previous 1.x installation", "admin.utilities.importv1Title": "Import from Wiki.js 1.x", "admin.utilities.invalidApiCertificates": "Invalidate API Keys Certificates", @@ -1497,6 +1511,31 @@ "admin.utilities.telemetryTitle": "Telemetry", "admin.utilities.title": "Utilities", "admin.utilities.tools": "Tools", + "admin.utilities.wikijs2Import.archive": "Archive", + "admin.utilities.wikijs2Import.archiveHint": "A .wkbackup file produced by a Wiki.js 2.x instance.", + "admin.utilities.wikijs2Import.assets": "Assets", + "admin.utilities.wikijs2Import.comments": "Comments", + "admin.utilities.wikijs2Import.groups": "Groups", + "admin.utilities.wikijs2Import.history": "History", + "admin.utilities.wikijs2Import.navigation": "Navigation", + "admin.utilities.wikijs2Import.noArchive": "No archive selected", + "admin.utilities.wikijs2Import.options": "Options", + "admin.utilities.wikijs2Import.overwrite": "Overwrite on conflict", + "admin.utilities.wikijs2Import.overwriteHint": "What to do when the destination already has something at the same path, name or ID.", + "admin.utilities.wikijs2Import.overwriteWarning": "This is a destructive action. Existing content cannot be recovered!", + "admin.utilities.wikijs2Import.pages": "Pages", + "admin.utilities.wikijs2Import.progress": "Progress Log", + "admin.utilities.wikijs2Import.progressEmpty": "The import has not been started. Progress will be reported here.", + "admin.utilities.wikijs2Import.selectArchive": "Select archive...", + "admin.utilities.wikijs2Import.settings": "Settings", + "admin.utilities.wikijs2Import.source": "Destination", + "admin.utilities.wikijs2Import.start": "Start Import", + "admin.utilities.wikijs2Import.subtitle": "Migrate content from a Wiki.js 2.x backup", + "admin.utilities.wikijs2Import.targetSite": "Site to import into", + "admin.utilities.wikijs2Import.targetSiteHint": "Everything in the archive is written to this site.", + "admin.utilities.wikijs2Import.users": "Users", + "admin.utilities.wikijs2Import.whatToImport": "What to import", + "admin.utilities.wikijs2Import.whatToImportHint": "Anything unticked is left in the archive.", "admin.utitilies.purgeHistoryMonth": "1 Month | {count} Months", "admin.utitilies.purgeHistoryToday": "Today", "admin.utitilies.purgeHistoryYear": "1 Year | {count} Years", @@ -1735,6 +1774,7 @@ "common.clipboard.uuid": "Copy UUID to clipboard.", "common.clipboard.uuidFailure": "Failed to copy UUID to clipboard.", "common.clipboard.uuidSuccess": "Copied UUID to clipboard successfully.", + "common.comingSoon": "Coming soon", "common.comments.beFirst": "Be the first to comment.", "common.comments.charsLeft": "{count} left", "common.comments.closed": "Comments are turned off for this page.", @@ -2676,12 +2716,12 @@ "mail.resetPwd.body": "Somebody asked to reset the password for your account on {siteName}.", "mail.resetPwd.expiry": "This link is valid for 24 hours and can only be used once. If you did not ask for this, nothing has changed and you can ignore this message.", "mail.resetPwd.footer": "You are receiving this because a password reset was requested for this address on {siteName}.", - "mail.resetPwd.subject": "Reset your password \u2014 {siteName}", + "mail.resetPwd.subject": "Reset your password — {siteName}", "mail.resetPwd.title": "Reset your password", "mail.test.action": "Go to the wiki", "mail.test.body": "If you are reading it, {siteName} can send mail through the SMTP server it is configured with.", "mail.test.footer": "You are receiving this because somebody sent a test email from the Wiki.js admin area.", - "mail.test.subject": "Test email \u2014 {siteName}", + "mail.test.subject": "Test email — {siteName}", "mail.test.title": "This is a test email", "mail.welcome.action": "Go to the wiki", "mail.welcome.body": "Your account on {siteName} is ready. You can sign in at any time.", @@ -2690,7 +2730,7 @@ "mail.welcome.verify.action": "Confirm my email address", "mail.welcome.verify.body": "An account was created for this address on {siteName}. Confirm that it is yours to finish signing up.", "mail.welcome.verify.expiry": "This link is valid for 24 hours. If you did not create this account, you can ignore this message.", - "mail.welcome.verify.subject": "Confirm your email address \u2014 {siteName}", + "mail.welcome.verify.subject": "Confirm your email address — {siteName}", "mail.welcome.verify.title": "Confirm your email address", "navEdit.clearItems": "Clear All Items", "navEdit.editMenuItems": "Edit Menu Items", diff --git a/backend/models/blocks.ts b/backend/models/blocks.ts index 0cc1fe462..1a7afe21d 100644 --- a/backend/models/blocks.ts +++ b/backend/models/blocks.ts @@ -6,6 +6,20 @@ import { blocks as blocksTable, sites as sitesTable } from '../db/schema.ts' import { CustomError } from '../helpers/common.ts' import { readBlockPackage } from '../helpers/wkblock.ts' +/** A site's directory in the block cache, which is named for its id and nothing else. */ +const CACHE_SITE_DIR = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i + +/** + * Everything `materialize` can leave in a site's cache directory for one block, and the key it + * belongs to: the unpacked directory, the checksum marker beside it, and a staging directory that a + * process killed mid-unpack never cleaned up. + * + * A block key cannot contain a dot, so the three are told apart by what follows the key without any + * ambiguity. Anything else in the directory matches nothing and is left alone — a sweep deleting + * what it does not recognise is a worse failure than the leak it is fixing. + */ +const CACHE_ENTRY = /^block-([a-z0-9][a-z0-9-]*)(?:\.checksum|\.[0-9a-f]{12})?$/ + /** One authorable attribute of a block, as its `static definition` describes it. */ export interface BlockProp { name: string @@ -710,6 +724,58 @@ class Blocks { await rm(blockDir, { recursive: true, force: true }) } + /** + * Delete cached files for every block this instance has no business serving, and report how many + * entries went. + * + * `discardCached` covers the instance that handled the delete and only that one. The `reloadBlocks` + * event carries no payload, so the others learn that a block is gone but not which files to drop — + * and an instance that was down when it happened hears nothing at all. So this reconciles the cache + * against the index rather than being told what changed, which is the same bargain `materialize` + * already makes: every instance works out for itself whether the files it holds are the right ones. + * + * Whole site directories go too. `customIndex` holds only the sites that have a custom block, so a + * directory for any other site is stale by definition — which is what eventually clears up after a + * site that was deleted. + * + * Removing a block's files while a request is mid-`materialize` for that same block is possible and + * harmless: it only happens for a block already gone from the index, so the request is going to 404 + * whichever of the two lands last, and anything left behind goes on the next sweep. + */ + async sweepCache(): Promise { + // -> No cache directory at all is the normal state of an instance that has never served a custom + // block, not an error + const siteDirs = await readdir(this.cachePath, { withFileTypes: true }).catch(() => []) + let removed = 0 + for (const siteDir of siteDirs) { + if (!siteDir.isDirectory() || !CACHE_SITE_DIR.test(siteDir.name)) { + continue + } + const sitePath = path.join(this.cachePath, siteDir.name) + const forSite = this.customIndex.get(siteDir.name) + if (!forSite || forSite.size < 1) { + await rm(sitePath, { recursive: true, force: true }) + removed++ + continue + } + for (const entry of await readdir(sitePath)) { + const key = CACHE_ENTRY.exec(entry)?.[1] + if (!key || forSite.has(key)) { + continue + } + this.materialized.delete(`${siteDir.name}:${key}`) + await rm(path.join(sitePath, entry), { recursive: true, force: true }) + removed++ + } + } + if (removed > 0) { + WIKI.logger.info( + `Swept ${removed} stale ${removed === 1 ? 'entry' : 'entries'} from the block cache.` + ) + } + return removed + } + /** * Throw away every unpacked block, for `flushCaches`. * diff --git a/blocks/package.mjs b/blocks/package.mjs index f16c9206f..0b2ca420c 100644 --- a/blocks/package.mjs +++ b/blocks/package.mjs @@ -8,39 +8,24 @@ * package, together with the `static definition` read off the component. The result lands in * `packages/block-xyz.wkblock` and is uploaded from the instance's Administration → Content Blocks. * - * ## The container + * ## The format is specified in `dev/specs/wkblock.md` * - * `backend/helpers/wkblock.ts` is the other half of this and the two have to agree. There is no - * module to share between them: `blocks/` and `backend/` are separately installed workspaces and the - * backend does not type-check JavaScript, so the format is written twice and stated in full in both - * places. + * Read it before changing anything here. This file and `backend/helpers/wkblock.ts` are two + * independent implementations of that document — `blocks/` and `backend/` are separately installed + * workspaces with no module between them, and the backend does not type-check JavaScript, so there is + * nothing the two halves could share and nothing that checks one against the other. The spec is what + * holds them together, so a change to the format is a change to the spec first. * - * magic 8 bytes "WKBLOCK\0" - * version uint32be format version, 1 - * headerLen uint32be byte length of the header that follows - * header gzip'd JSON — see below - * payload each file's gzip'd bytes, concatenated in the header's order + * Two decisions worth knowing while reading this file, both explained at length there: * - * The header: + * **Per-file gzip rather than one stream over the lot.** A block's assets are often already-compressed + * images and fonts sitting beside a bundle that compresses four to one, and a file at a time means the + * reader can check a digest as it goes rather than after holding the whole package twice. * - * { - * "block": "xyz", // the key, i.e. the element's suffix - * "definition": { ... }, // the component's `static definition`, verbatim - * "packagedAt": "2026-09-19T...Z", - * "packagedWith": "3.0.0", // the wiki the packager came from, for diagnostics only - * "files": [ - * { "path": "block-xyz.js", "size": 12345, "compressedSize": 4321, "sha256": "..." } - * ] - * } - * - * Per-file gzip rather than one stream over the lot: a block's assets are often already-compressed - * images and fonts sitting beside a bundle that compresses four to one, and a file at a time means - * the reader can check a digest as it goes rather than after holding the whole package twice. - * - * Every path is relative to where the block is served from, and the reader refuses anything outside - * the block's own namespace — `block-.js`, `block-.worker.js` and `block-/**`. That + * **Everything is namespaced under the block's own name.** Every path is relative to where the block + * is served from, and must be `block-.js`, `block-.worker.js` or `block-/**`. That * namespace is the whole of what keeps an imported block from overwriting a built-in one, so it is - * checked here as well, where the author can still do something about it. + * checked here as well as in the reader — here, where the author can still do something about it. */ import crypto from 'node:crypto' diff --git a/dev/specs/wkblock.md b/dev/specs/wkblock.md new file mode 100644 index 000000000..e7f82e7da --- /dev/null +++ b/dev/specs/wkblock.md @@ -0,0 +1,333 @@ +# `.wkblock` — block package format + +**Status:** implemented, format version 1. +**Implemented by:** [`blocks/package.mjs`](../../blocks/package.mjs) (writer) and +[`backend/helpers/wkblock.ts`](../../backend/helpers/wkblock.ts) (reader). + +A `.wkblock` is the single file one content block is distributed as. An administrator uploads it +under **Administration → Content Blocks → Install Block…** and the block is available to authors +immediately — nothing on the instance is rebuilt and nothing is restarted. + +> **This document is the format.** The writer and the reader are two independent implementations of +> what is written here — `blocks/` and `backend/` are separately installed workspaces with no module +> between them, and the backend does not type-check JavaScript, so there is nothing either half could +> import and nothing that could check one against the other. Stating it in both files meant two +> statements drifting apart by hand; stating it here means both implement one document. +> +> If an implementation disagrees with this document, **the implementation is wrong** — with the single +> exception that a stated limit found to be unsafe should be tightened in code first and written down +> immediately after. See [Changing the format](#9-changing-the-format). + +--- + +## 1. What the format is for + +A block author clones this repository, writes a directory under `blocks/`, runs the packager, and +uploads the result to a wiki they have never otherwise touched. The design follows from that: + +- **A packaged block is the same thing as a built-in one arriving by a different road.** Same + `component.js`, same `static definition`, same rollup build. What differs is where its files end up + and where its definition is read from. There is deliberately no second authoring API. +- **The package is the only copy.** It is stored verbatim on the block's row and unpacked into a cache + on demand. Nothing is ever written into `blocks/` on the server, which is a build output and, in a + container, part of the image. +- **The container is parsed as something a stranger uploaded.** The trust boundary is about the *code* + — see [§7](#7-trust-boundary) — but the file itself is untrusted input before anyone has vouched for + anything. + +--- + +## 2. Container + +All integers are big-endian. The file is a fixed 16-byte preamble, a gzipped JSON header, then each +file's gzipped bytes concatenated in the header's order. + +``` +offset size field +0 8 magic "WKBLOCK\0" (0x57 4B 42 4C 4F 43 4B 00) +8 4 version uint32be, currently 1 +12 4 headerLen uint32be, byte length of the gzipped header +16 headerLen header gzip(JSON, level 9) — see §3 +16+hl … payload gzip(file bytes, level 9) × N, in header order +``` + +There is no index of payload offsets. Each file's position is the running sum of the +`compressedSize` values before it, which is why the reader walks the entries in order and why +`compressedSize` is in the header at all. + +**The file length must equal `16 + headerLen + Σ compressedSize` exactly.** The reader checks this +before decompressing anything; a mismatch means a truncated download or a doctored file, and both are +refused with the same message. + +### Why per-file gzip rather than one stream over the lot + +A block's assets are frequently already-compressed images and fonts sitting beside a bundle that +compresses four to one. Compressing each file separately means the incompressible ones cost almost +nothing, and — more importantly — the reader can verify a digest as it goes rather than having to +hold the whole package twice. + +### Why not ZIP + +The sibling format in this repo, [`.wkbackup`](./wkbackup.md), *is* a ZIP, and the reasoning there +is worth contrasting. A `.wkbackup` is gigabytes, read in a browser, and needs random access to one +entry out of thousands. A `.wkblock` is a couple of megabytes read whole into server memory in one +pass. ZIP's central directory buys nothing at that size, and a hand-written 300-line reader with +explicit bounds on every claimed length is easier to audit than a ZIP parser's edge cases (data +descriptors, ZIP64, filename encodings). Different problems, different answers. + +--- + +## 3. Header + +Gzipped UTF-8 JSON: + +```json +{ + "block": "xyz", + "definition": { "block": "xyz", "name": "…", "description": "…", "icon": "…", "props": [] }, + "packagedAt": "2026-09-19T12:00:00.000Z", + "packagedWith": "3.0.0", + "files": [ + { "path": "block-xyz.js", "size": 12345, "compressedSize": 4321, "sha256": "…64 hex chars…" } + ] +} +``` + +| Field | Meaning | +| --- | --- | +| `block` | The key. Must match `/^[a-z0-9][a-z0-9-]{0,62}$/`. | +| `definition` | The component's `static definition`, verbatim — see [§5](#5-the-definition). | +| `packagedAt` | ISO 8601. Diagnostics only; the reader defaults it to `''` if absent or not a string. | +| `packagedWith` | Version of the wiki the packager came from. Diagnostics only, same tolerance. | +| `files` | Payload entries, in payload order. `sha256` is of the **uncompressed** bytes. | + +`packagedAt` and `packagedWith` are the only optional fields, and nothing branches on either. + +--- + +## 4. Namespacing: the rule everything else rests on + +**One block per package, and the directory name is the identity.** `blocks/block-xyz/` declares +`block: 'xyz'`, packages as `block-xyz.wkblock`, serves as `block-xyz.js`, and renders as +``. The packager refuses a mismatch, because every one of those names is derived from the +same key. + +**Every path in a package must fall inside that block's own namespace:** + +``` +block-.js ← required; the file the wiki loads the block from +block-.worker.js ← optional +block-/** ← assets and shared chunks +``` + +This is what lets an imported block and a built-in one be served from the same `/_blocks/` without +either standing on the other, and it is enforced **on both sides** — by the packager, where the author +can still do something about it, and by the reader, where it is a security check. + +Note this differs from a full `npm run build`: there, shared chunks sit at the output root. +`buildConfig({ only })` names them into `block-/` instead, precisely so a package can be +namespaced. + +A path is rejected unless it is a plain relative path. The reader refuses any of: empty, longer than +255 characters, containing `\`, leading `/`, any `.` or `..` segment, trailing `/`, `//`, or any +character below U+0020. The backslash rule matters because the path is joined onto a cache directory +and a Windows instance would read `\` as a separator where this check would not. + +### Child blocks are refused outright + +A block with `isChild` is part of whatever holds it — it has no row, no enable toggle and nothing to +switch on — so a package of one would install nothing. Both halves refuse it, with a message saying to +package the parent instead. + +--- + +## 5. The definition + +The definition is **read key by key, not spread**. What comes back is stored on the block's row, +handed to the editor to build a form from, and turned into the sanitiser's allow list for the block's +tag — an unknown key would travel all of that way meaning nothing. + +| Key | Rule | +| --- | --- | +| `block` | Must equal the header's `block`. | +| `name` | String, ≤ 255. Required. | +| `description` | String, ≤ 255. Defaults to `''`. | +| `icon` | String, ≤ 255. Defaults to `''`. | +| `props` | Array, ≤ 64 entries. | +| `template` | Optional string, ≤ 8192. | +| `asciidocTemplate` | Optional string, ≤ 8192. | +| `contentEditor` | Optional string, ≤ 64. | + +Each prop: + +| Key | Rule | +| --- | --- | +| `name` | `/^[A-Za-z][A-Za-z0-9-]{0,63}$/`. **This becomes an attribute on the block's tag**, so a name that is not a valid attribute name would either be dropped silently or widen the sanitiser's allow list in a way nobody wrote down. | +| `type` | One of `string`, `number`, `boolean`, `select`, `icon`. | +| `label`, `hint` | Optional strings, truncated to 1024 rather than refused. | +| `required` | Kept only when exactly `true`. | +| `default` | Kept only when a string, number or boolean. | +| `options` | Optional array, first 128 kept. A string, or coerced to `{ label, value }` with both stringified. | + +--- + +## 6. Limits + +All checked **before** anything is decompressed, so a package cannot talk the reader into +decompressing more than it is prepared to hold. `gunzipSync` is additionally called with +`maxOutputLength` set per entry, so a lying `size` cannot become a zip bomb. + +| Limit | Value | Note | +| --- | --- | --- | +| `MAX_PACKAGE_SIZE` | 32 MiB | Also the body limit of the import route. | +| `MAX_HEADER_SIZE` | 4 MiB | Applies to the gzipped header *and* its output. | +| `MAX_UNPACKED_SIZE` | 128 MiB | Sum of every entry's `size`. | +| `MAX_FILE_COUNT` | 4096 | At least 1 required. | + +`MAX_PACKAGE_SIZE` is deliberately **not** the site's asset upload limit. That one is about what +readers may attach to pages and is usually turned down; a block carrying a PDF engine and its +character maps is legitimately a couple of dozen megabytes. + +### Reader checks, in order + +1. Length ≥ 16 and magic matches → else "not a Wiki.js block package". +2. `version === 1` → else a message naming both versions, since the likely cause is a newer wiki. +3. `headerLen` ≥ 1, ≤ `MAX_HEADER_SIZE`, and `16 + headerLen ≤ file length`. +4. Header gunzips and parses as JSON. +5. `block` matches the key pattern; definition validates ([§5](#5-the-definition)). +6. `files` is an array, 1 … `MAX_FILE_COUNT`. +7. **Every entry**: path is servable and namespaced; `size` and `compressedSize` are non-negative + integers; `sha256` is 64 lowercase hex. +8. `Σ size ≤ MAX_UNPACKED_SIZE`. +9. `16 + headerLen + Σ compressedSize === file length`. +10. Then, per entry in order: no duplicate path, gunzip with `maxOutputLength`, and the decompressed + bytes must match both the stated `size` and the stated `sha256`. +11. `block-.js` is present. + +Every failure is a `CustomError('blockPackageInvalid', …)` naming what is wrong, because every one of +them is something the administrator who uploaded the file can act on — a truncated download, the wrong +file, a package built by a newer wiki. + +--- + +## 7. Trust boundary + +**`manage:sites` is what it takes to import one** — the same permission the screen already needs, and +deliberately not something stricter. + +A block is code that runs in every reader's browser on that site. That is exactly what the raw head +and body fields under **Administration → Theme** already are, and those take `manage:theme`. Block +import is not a new kind of power; it is a tidier way to exercise one the admin area already grants. + +What that permission covers is the **code**. It says nothing about the **container**, which is parsed +before anybody has vouched for anything — hence the bounds in [§6](#6-limits), the digest on every +file, and the namespace check on every path. + +--- + +## 8. Lifecycle on the server + +### Import + +`POST /sites/:siteId/blocks/import`, `manage:sites`, body is the file itself as +`application/octet-stream` — not a multipart form. + +- **Re-importing the same key is an upgrade, not a second block.** The row is updated, so what the + site had switched on and configured on it survives. The reply says `isNew: false`. +- **A key a built-in block already uses is refused with 409** (`blockPackageConflict`), since both + would be served from the same address. Two conditions raise it: the key belongs to a block compiled + into this wiki, or the site already has a non-custom row under it. +- The block is registered **enabled** and is available to authors immediately. + +### Storage + +On the `blocks` row: + +| Column | Contents | +| --- | --- | +| `packageData` | The `.wkblock` verbatim. Null for a built-in. **This is the only copy.** | +| `definition` | The package's copy of the definition. Empty for a built-in, whose definition is read from the compiled manifest so that an updated block describes itself the moment it is deployed. | +| `checksum` | SHA-256 of the package, and the name of its directory in the disk cache. Empty for a built-in. | + +### Serving + +`backend/controllers/blocks.ts` answers `/_blocks/` for both kinds. It replaced the +`@fastify/static` registration for that prefix, because a static plugin claims the whole prefix and +leaves nothing to ask the question in front of it; the plugin is still registered with `serve: false` +for `reply.sendFile`. + +- **The first path segment names the block, and that is the whole decision** — which is why the + namespace rule in [§4](#4-namespacing-the-rule-everything-else-rests-on) is enforced so hard. +- **It depends on which site was asked.** Two sites on one instance may each have imported a different + block under the same key. The frontend has no site in hand when it loads a block — it reads a tag out + of a page and asks for it — so the hostname resolves it through the same `WIKI.sitesMappings` lookup + the request hooks use. +- **`/cache/blocks//block-/` is a cache, not storage.** `servingPathFor` + unpacks the stored package into it on first request. `block-.checksum` beside it says which + version is there and **is written last**, so an unpack that died halfway is redone rather than half + served; the directory is built under a temporary name and moved into place for the same reason. +- Consequently an upgrade reaches every instance of an HA set on its own — including one that was not + running when the upload happened — and a fresh container needs nothing restored. +- **Files for a block that has gone are swept, not left.** Deleting a block discards its unpacked + files on the instance that handled the request. Every other instance reconciles instead of being + told: the `reloadBlocks` event says the set of custom blocks changed but not how, so on receiving + it each instance re-reads the index and then deletes anything cached for a block the index no + longer has. The same sweep runs at boot, which is what catches an instance that was down when the + delete happened and comes back to a volume it left behind. +- **A whole site's directory goes the same way.** The index holds only the sites that have a custom + block, so a directory for any other site is stale by definition — which is what clears up after a + site that was deleted. +- The sweep acts only on names it recognises: `block-`, `block-.checksum`, and a + `block-.<12 hex>` staging directory left by a process killed mid-unpack. Anything else in the + cache is left alone, since a sweep deleting what it does not recognise would be a worse failure + than the leak it is fixing. +- **Custom files are revalidated (`no-cache` + ETag); built-ins are held for an hour.** The file names + are the same across versions of a custom block, and the point of uploading a fixed one is that the + fix is live. + +### Rendering + +A custom block's definition is read from its row wherever a built-in's is read from the manifest — its +props in `getSiteBlocks`, and its tag and attributes in the sanitiser's allow list via +`getEnabledForRender`, which fetches both in the one query `postProcess` was already making. That +query is deliberately not cached: a definition it misses is a block stripped out of somebody's page. + +Everything else is identical to a built-in, the enable toggle included — a custom block that is +switched off is stripped from a page being saved exactly as a built-in one is. + +--- + +## 9. Changing the format + +This document is the contract. Both implementations follow it, and neither is a place to decide +something new about the format. + +1. **Change this document first.** A change that is not written here has not been agreed with the + other half. +2. Change `blocks/package.mjs` and `backend/helpers/wkblock.ts` to match. They are separate + workspaces and can be edited in either order, but both belong in the same commit as the change + here. +3. **Bump `FORMAT_VERSION` in both files** if the change is not backward compatible. The reader + refuses any version but its own and says so in a message naming both numbers, which is the whole + of the compatibility story: there is no migration path for a package, because rebuilding one is a + single command. + +Nothing enforces step 2 automatically — there is no test runner in any of the three workspaces, so +there is nowhere such a check could currently live. If the two implementations do drift, the cheapest +guard would be a committed fixture `.wkblock` and something that reads it: that catches the reader +drifting from a file the writer actually produced, which is the failure that matters. + +--- + +## 10. Known gaps + +- **No signature or publisher identity.** Anyone who can reach the import screen can install any + block, and nothing about the file says where it came from. That is consistent with the trust + boundary in [§7](#7-trust-boundary) — `manage:sites` already implies running arbitrary code in + readers' browsers — but it means there is no way to distribute a block with any assurance attached. + A signature would be a meaningful addition *only* alongside a notion of trusted publishers; on its + own it proves nothing useful. +- **No dependency or compatibility declaration.** A package does not say which wiki versions it works + with. `packagedWith` is recorded but never acted on. +- **One block per file.** A suite of related blocks is *n* uploads. This is a consequence of the key + being the identity and is not obviously worth changing.