import { and, asc, eq, gt, sql } from 'drizzle-orm' import { alias } from 'drizzle-orm/pg-core' import { navigation as navigationTable, pages as pagesTable, tree as treeTable } from '../db/schema.ts' import { decodeTreePath, generatePathHash } from '../helpers/common.ts' import { resolveLink } from '../helpers/pageLinks.ts' import type { ResolvedLink } from '../helpers/pageLinks.ts' import { PENDING_RENDER_HTML } from './import.ts' import { isBodylessEditor, isKnownEditor, normalizeBodylessContent } from './pages.ts' /** What a problem means for the page: `error` is broken for readers, `warning` is out of step. */ export type ProblemSeverity = 'error' | 'warning' /** * Every check the scan runs, in the order the admin screen lists them. * * The key is also the translation key of the check's label (`admin.utilities.pageProblems.checks.*`) * and of the line a problem it finds is logged as (`admin.utilities.pageProblems.problems.*`), so a * check added here needs both strings. */ export const PAGE_PROBLEM_CHECKS = [ { key: 'contentEmpty', group: 'content', severity: 'error' }, { key: 'renderEmpty', group: 'content', severity: 'error' }, { key: 'renderPending', group: 'content', severity: 'warning' }, { key: 'contentInvalidJson', group: 'content', severity: 'error' }, { key: 'contentInvalid', group: 'content', severity: 'error' }, { key: 'editorUnknown', group: 'editor', severity: 'error' }, { key: 'editorDisabled', group: 'editor', severity: 'warning' }, { key: 'redirectTargetMissing', group: 'redirects', severity: 'error' }, { key: 'redirectSelf', group: 'redirects', severity: 'error' }, { key: 'redirectLoop', group: 'redirects', severity: 'error' }, { key: 'redirectChainTooLong', group: 'redirects', severity: 'error' }, { key: 'redirectChain', group: 'redirects', severity: 'warning' }, { key: 'treeEntryMissing', group: 'tree', severity: 'error' }, { key: 'treeEntryMismatch', group: 'tree', severity: 'error' }, { key: 'treeEntryOrphaned', group: 'tree', severity: 'error' }, { key: 'treeEntryStale', group: 'tree', severity: 'warning' }, { key: 'hashMismatch', group: 'address', severity: 'error' }, { key: 'aliasDuplicate', group: 'address', severity: 'error' }, { key: 'localeInactive', group: 'locales', severity: 'warning' }, { key: 'localeGroupOrphaned', group: 'locales', severity: 'warning' }, { key: 'scheduledWithoutDates', group: 'publishing', severity: 'warning' }, { key: 'publishDatesInverted', group: 'publishing', severity: 'warning' }, { key: 'searchIndexMissing', group: 'search', severity: 'warning' }, { key: 'navigationMenuMissing', group: 'navigation', severity: 'warning' } ] as const satisfies readonly { key: string; group: string; severity: ProblemSeverity }[] export type PageProblemCheck = (typeof PAGE_PROBLEM_CHECKS)[number]['key'] const SEVERITY_OF = new Map( PAGE_PROBLEM_CHECKS.map((check) => [check.key, check.severity]) ) /** One thing found wrong with one page, or with a tree entry that has no page. */ export interface PageProblem { check: PageProblemCheck severity: ProblemSeverity /** Null for an orphaned tree entry, which is the one problem that has no page. */ pageId: string | null siteId: string locale: string path: string title: string /** Where to open the page, or null where there is no page to open. */ url: string | null /** Filled into the problem's line, by the names its string uses. */ params: Record } export interface PageProblemBatch { problems: PageProblem[] /** Rows looked at by this call, pages and tree entries alike. */ scanned: number /** What to send back for the next batch, or null once there is nothing left. */ cursor: string | null } /** * Pages read per call. Small enough that one call answers in well under a second on a large wiki, * which is what keeps the progress bar moving and a Stop button meaningful. */ const PAGE_BATCH_SIZE = 250 /** Tree entries per call in the second phase, which reads only ids and so can take more. */ const TREE_BATCH_SIZE = 1000 /** * How many redirections a reader's browser follows in a row before calling it a loop — `MAX_HOPS` * in `frontend/src/components/PageRedirect.vue`, which stops on the one after it. The two have to * agree, or this reports as fine a chain that readers are stopped at. */ const MAX_REDIRECT_HOPS = 5 /** * The two editors behind the experimental flag (`BODYLESS_EDITORS` in `frontend/src/stores/site.js`). * The server has no content type for them yet, so they are not "unknown", and nothing about what they * store is settled enough to check. */ const EXPERIMENTAL_EDITORS = new Set(['channel', 'api']) const REDIRECT_EDITOR = 'redirect' /** The two navigation modes under which a page draws a menu of its own rather than an ancestor's. */ const OVERRIDE_MODES = new Set(['override', 'overrideExact']) /** * Where the scan is. `p:` is the pages phase and `t:` the orphaned tree entries after it, each with * the last id the previous call read; an id alone is enough to carry on because both walk in id * order. */ function parseCursor(cursor?: string): { phase: 'pages' | 'tree'; after: string | null } { if (cursor?.startsWith('t:')) { return { phase: 'tree', after: cursor.slice(2) || null } } if (cursor?.startsWith('p:')) { return { phase: 'pages', after: cursor.slice(2) || null } } return { phase: 'pages', after: null } } /** A page a redirection lands on, with what is needed to follow it further. */ interface RedirectHop { id: string siteId: string locale: string path: string editor: string content: string | null } /** * Page problems model * * A read-only scan of every page on every site for data that is broken or out of step: the things * nothing at save time can see, because they are the result of an import, a failed write halfway * through a multi-statement change, or a setting changed after the page was written. * * **It never changes anything.** Each check reports; fixing is a separate decision per check, and some * of them have more than one right answer. * * **Driven by the caller, a batch at a time**, the way the 2.x import is: the admin screen asks for * the next batch until the cursor comes back null, so progress is the screen's own count and stopping * is simply not asking again. There is no job and nothing to clean up after a scan abandoned halfway. * Pages are walked in id order, so one created mid-scan is either reached or not — never read twice. */ class PageProblems { /** * How big a full scan is: `pages` to say so, and `total` rows for the progress bar — every page, * then every page entry in the tree, since the second phase reads those too. */ async size(): Promise<{ pages: number; total: number }> { const [pages] = await WIKI.db.select({ n: sql`count(*)::int` }).from(pagesTable) const [entries] = await WIKI.db .select({ n: sql`count(*)::int` }) .from(treeTable) .where(eq(treeTable.type, 'page')) const pageCount = Number(pages?.n ?? 0) return { pages: pageCount, total: pageCount + Number(entries?.n ?? 0) } } /** The next batch of the scan, starting after `cursor` — or from the beginning without one. */ async scan(cursor?: string): Promise { const { phase, after } = parseCursor(cursor) return phase === 'pages' ? this.#scanPages(after) : this.#scanTree(after) } /** * Every check that is a question about a page, for one batch of pages. * * One query answers all of them but the redirections: the tree entry comes in by join, and the * locale group, the alias and the menu are subqueries per row. The content and the render are * never read whole — only whether they are empty — except a bodyless page's content, which is the * small settings document the JSON checks parse. */ async #scanPages(after: string | null): Promise { const groupMember = alias(pagesTable, 'groupMember') const aliasTwin = alias(pagesTable, 'aliasTwin') const rows = await WIKI.db .select({ id: pagesTable.id, siteId: pagesTable.siteId, locale: pagesTable.locale, path: pagesTable.path, hash: pagesTable.hash, alias: pagesTable.alias, title: pagesTable.title, description: pagesTable.description, editor: pagesTable.editor, contentType: pagesTable.contentType, publishState: pagesTable.publishState, publishStartDate: pagesTable.publishStartDate, publishEndDate: pagesTable.publishEndDate, isBrowsable: pagesTable.isBrowsable, isSearchable: pagesTable.isSearchableComputed, tags: pagesTable.tags, authorId: pagesTable.authorId, creatorId: pagesTable.creatorId, ownerId: pagesTable.ownerId, localeGroupId: pagesTable.localeGroupId, contentEmpty: sql`coalesce(btrim(${pagesTable.content}), '') = ''`, renderEmpty: sql`coalesce(btrim(${pagesTable.render}), '') = ''`, renderPending: sql`coalesce(${pagesTable.render} = ${PENDING_RENDER_HTML}, false)`, searchTextMissing: sql`${pagesTable.searchContent} is null`, searchVectorMissing: sql`${pagesTable.ts} is null`, bodylessContent: sql< string | null >`case when ${pagesTable.editor} in ('redirect', 'blog') then ${pagesTable.content} end`, treeId: treeTable.id, treeType: treeTable.type, treeSiteId: treeTable.siteId, treeLocale: treeTable.locale, treeFolderPath: treeTable.folderPath, treeFileName: treeTable.fileName, treeTitle: treeTable.title, treeTags: treeTable.tags, treeMeta: treeTable.meta, navigationMode: treeTable.navigationMode, navigationId: treeTable.navigationId, navigationExists: sql`exists (${WIKI.db .select({ one: sql`1` }) .from(navigationTable) .where(eq(navigationTable.id, treeTable.navigationId))})`, groupSize: sql`(${WIKI.db .select({ n: sql`count(*)::int` }) .from(groupMember) .where( and( eq(groupMember.siteId, pagesTable.siteId), eq(groupMember.localeGroupId, pagesTable.localeGroupId) ) )})`, aliasTaken: sql`exists (${WIKI.db .select({ one: sql`1` }) .from(aliasTwin) .where( and( eq(aliasTwin.siteId, pagesTable.siteId), eq(aliasTwin.alias, pagesTable.alias), sql`${aliasTwin.id} <> ${pagesTable.id}` ) )})` }) .from(pagesTable) .leftJoin(treeTable, eq(treeTable.id, pagesTable.id)) .where(after ? gt(pagesTable.id, after) : undefined) .orderBy(asc(pagesTable.id)) .limit(PAGE_BATCH_SIZE) const problems: PageProblem[] = [] for (const row of rows) { const report = (check: PageProblemCheck, params: Record = {}) => { problems.push({ check, severity: SEVERITY_OF.get(check)!, pageId: row.id, siteId: row.siteId, locale: row.locale, path: row.path, title: row.title, url: this.#pageUrl(row.siteId, row.locale, row.path, row.editor), params }) } const site = WIKI.sites[row.siteId] const isExperimental = EXPERIMENTAL_EDITORS.has(row.editor) const isKnown = isKnownEditor(row.editor) const isBodyless = isBodylessEditor(row.editor) // -> Content if (row.contentEmpty) { report('contentEmpty') } if (isKnown && !isBodyless && row.renderEmpty) { report('renderEmpty', { editor: row.editor }) } if (row.renderPending) { report('renderPending') } if (isBodyless && !row.contentEmpty) { this.#checkBodylessContent(row.editor, row.bodylessContent, report) } // -> Editor if (!isKnown && !isExperimental) { report('editorUnknown', { editor: row.editor }) } else if (isKnown && site?.config?.editors?.[row.editor]?.isActive === false) { report('editorDisabled', { editor: row.editor }) } // -> Redirections if (row.editor === REDIRECT_EDITOR && !row.contentEmpty) { await this.#checkRedirect(row, report) } // -> Tree if (!row.treeId) { report('treeEntryMissing') } else { const folder = row.treeFolderPath ? decodeTreePath(row.treeFolderPath) : '' const treePath = folder ? `${folder}/${row.treeFileName}` : (row.treeFileName ?? '') if ( row.treeType !== 'page' || row.treeSiteId !== row.siteId || row.treeLocale !== row.locale || treePath.toLowerCase() !== row.path.toLowerCase() ) { report('treeEntryMismatch', { treePath: `${row.treeLocale}/${treePath}` }) } else { const stale = this.#staleTreeFields(row) if (stale.length > 0) { report('treeEntryStale', { fields: stale.join(', ') }) } } // -> Navigation, which lives on the tree entry const menuMissing = row.navigationId ? !row.navigationExists : OVERRIDE_MODES.has(row.navigationMode ?? '') if (menuMissing) { report('navigationMenuMissing', { mode: row.navigationMode ?? 'inherit' }) } } // -> Address if (row.hash !== generatePathHash(row.path)) { report('hashMismatch') } if (row.alias && row.aliasTaken) { report('aliasDuplicate', { alias: row.alias }) } // -> Locales const activeLocales: string[] = site?.config?.locales?.active ?? [] if (!activeLocales.includes(row.locale)) { report('localeInactive', { locale: row.locale }) } if (row.localeGroupId && Number(row.groupSize) < 2) { report('localeGroupOrphaned') } // -> Publishing if (row.publishState === 'scheduled' && !row.publishStartDate && !row.publishEndDate) { report('scheduledWithoutDates') } if ( row.publishStartDate && row.publishEndDate && row.publishEndDate.getTime() <= row.publishStartDate.getTime() ) { report('publishDatesInverted') } // -> Search. A render with no extracted text is a body the index cannot see; a page with no // vector at all is missing from results even by title if ( row.isSearchable && (row.searchVectorMissing || (!isBodyless && !row.renderEmpty && row.searchTextMissing)) ) { report('searchIndexMissing') } } const last = rows.at(-1) return { problems, scanned: rows.length, // -> A short batch is the last one; the tree phase starts from its own beginning cursor: rows.length < PAGE_BATCH_SIZE ? 't:' : `p:${last!.id}` } } /** * Tree entries of type page with no page behind them. * * The one check that cannot be asked of a page, since there is none, so it is a phase of its own * after the pages. Such an entry is listed in the file browser and the sidebar and leads to a 404. */ async #scanTree(after: string | null): Promise { const rows = await WIKI.db .select({ id: treeTable.id, siteId: treeTable.siteId, locale: treeTable.locale, folderPath: treeTable.folderPath, fileName: treeTable.fileName, title: treeTable.title, pageId: pagesTable.id }) .from(treeTable) .leftJoin(pagesTable, eq(pagesTable.id, treeTable.id)) .where(and(eq(treeTable.type, 'page'), after ? gt(treeTable.id, after) : undefined)) .orderBy(asc(treeTable.id)) .limit(TREE_BATCH_SIZE) const problems: PageProblem[] = rows .filter((row) => !row.pageId) .map((row) => { const folder = row.folderPath ? decodeTreePath(row.folderPath) : '' return { check: 'treeEntryOrphaned', severity: SEVERITY_OF.get('treeEntryOrphaned')!, pageId: null, siteId: row.siteId, locale: row.locale, path: folder ? `${folder}/${row.fileName}` : row.fileName, title: row.title, url: null, params: {} } }) const last = rows.at(-1) return { problems, scanned: rows.length, cursor: rows.length < TREE_BATCH_SIZE ? null : `t:${last!.id}` } } /** * A redirection's or a blog's settings document, read the way saving it would. * * Two checks, because they are two different breakages: a document that is not a JSON object at * all (what the editor opens is blank, and a blog quietly falls back to every default), and one that * parses but that the save path would refuse — a redirection with no target, a blog asking for * no posts per page. The second is the save path's own normalizer, so the two cannot disagree about * what is valid, and the reason given is the one a save would have been refused with. */ #checkBodylessContent( editor: string, content: string | null, report: (check: PageProblemCheck, params?: Record) => void ): void { let parsed: unknown try { parsed = JSON.parse(content ?? '') } catch { parsed = undefined } if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { report('contentInvalidJson', { editor }) return } try { normalizeBodylessContent(editor, content ?? '') } catch (err: any) { report('contentInvalid', { reason: err.message }) } } /** * Follow a redirection the way a reader's browser would, and report where that goes wrong. * * Resolved with `resolveLink`, which is what the link table uses and what puts the site's locale * prefix and page extensions into play, then looked up page by page — a chain is followed until it * reaches something that is not a redirection, comes back on itself, or runs past what a browser * will follow. Only this page's own problem is reported: a chain that breaks further along is * reported against the page where it breaks, when the scan gets to that one. * * A URL target, or a path to something that is not a page (`/_admin`, a file), is not followed. */ async #checkRedirect( page: { id: string siteId: string locale: string path: string bodylessContent: string | null }, report: (check: PageProblemCheck, params?: Record) => void ): Promise { const visited = new Set([page.id]) let current: { siteId: string; locale: string; path: string; content: string | null } = { ...page, content: page.bodylessContent } let redirections = 1 let firstTarget = '' while (true) { const target = this.#redirectTargetOf(current.content) if (!target) { return } firstTarget ||= target const link = resolveLink(target, current) if (!link || (link.kind !== 'page' && link.kind !== 'alias' && link.kind !== 'pageId')) { return } const hop = await this.#findPage(link) if (!hop) { if (redirections === 1) { report('redirectTargetMissing', { target }) } return } if (hop.id === page.id) { report(redirections === 1 ? 'redirectSelf' : 'redirectLoop', { target: firstTarget }) return } if (visited.has(hop.id)) { // -> This page leads into a loop that it is not part of: the pages in the loop report // themselves, and a reader arriving here goes round it just the same report('redirectLoop', { target: firstTarget }) return } if (hop.editor !== REDIRECT_EDITOR) { if (redirections > 1) { report('redirectChain', { target: firstTarget, hops: redirections }) } return } visited.add(hop.id) redirections++ if (redirections > MAX_REDIRECT_HOPS) { report('redirectChainTooLong', { target: firstTarget, max: MAX_REDIRECT_HOPS }) return } current = hop } } /** A redirection's target when it points at a page of this wiki, or null. */ #redirectTargetOf(content: string | null): string | null { try { const parsed = JSON.parse(content ?? '') if (parsed?.kind === 'url' || typeof parsed?.target !== 'string') { return null } const target = parsed.target.trim() return target.length > 0 ? target : null } catch { return null } } /** The page a resolved link addresses, by the same three keys `pageLinks.outboundFor` joins on. */ async #findPage(link: ResolvedLink): Promise { const where = link.kind === 'page' ? and( eq(pagesTable.siteId, link.targetSiteId), eq(pagesTable.locale, link.targetLocale ?? ''), eq(pagesTable.path, link.targetPath ?? '') ) : link.kind === 'alias' ? and( eq(pagesTable.siteId, link.targetSiteId), eq(pagesTable.alias, link.targetRef ?? '') ) : sql`${pagesTable.id}::text = ${link.targetRef ?? ''}` const [row] = await WIKI.db .select({ id: pagesTable.id, siteId: pagesTable.siteId, locale: pagesTable.locale, path: pagesTable.path, editor: pagesTable.editor, content: sql< string | null >`case when ${pagesTable.editor} = 'redirect' then ${pagesTable.content} end` }) .from(pagesTable) .where(where) .limit(1) return row ?? null } /** * Which of the copies a tree entry keeps of its page no longer match it. * * Compared through `pages.treeMeta`, the function that writes them, and through JSON, since the * stored copy has been through a `jsonb` column and a date in it is the ISO string of the one on the * page. */ #staleTreeFields(row: any): string[] { const stale: string[] = [] if (row.treeTitle !== row.title) { stale.push('title') } if (JSON.stringify(row.treeTags ?? []) !== JSON.stringify(row.tags ?? [])) { stale.push('tags') } const expected = JSON.parse(JSON.stringify(WIKI.models.pages.treeMeta(row))) const stored = (row.treeMeta ?? {}) as Record for (const [key, value] of Object.entries(expected)) { if (JSON.stringify(stored[key] ?? null) !== JSON.stringify(value ?? null)) { stale.push(key) } } return stale } /** * Where to open a page from the admin area, which may be on another site's hostname. * * A redirection carries `?redirect=no`, since the whole reason to open one from here is to look at * it rather than be sent on. */ #pageUrl(siteId: string, locale: string, path: string, editor: string): string { const hostname = WIKI.sites[siteId]?.hostname const origin = hostname && hostname !== '*' ? `//${hostname}` : '' const query = editor === REDIRECT_EDITOR ? '?redirect=no' : '' return `${origin}${WIKI.models.pages.urlFor(siteId, locale, path)}${query}` } } export const pageProblems = new PageProblems()