feat: rerender all pages utility + avoid blanking render on imports / render failure

scarlett
NGPixel 9 hours ago
parent 6dc3c3b10f
commit 9712cbfefc
No known key found for this signature in database

@ -1732,6 +1732,174 @@ async function routes(app: FastifyInstance) {
async () => ({ ok: true, checks: PAGE_PROBLEM_CHECKS })
)
/**
* RERENDER ALL PAGES
*
* Every page rendered again from its source, by the admin's own browser rather than by Puppeteer —
* so it works on every instance, and it is the way to put right a wiki whose renders went blank or
* stale with no headless browser to fix them. The browser drives it: this starts a run and lists the
* pages, `sources` hands out their content a batch at a time, and each render comes back through the
* PUT below. Stopping is not asking again, as for the page problem scan.
*
* A POST, though it changes nothing, because it is where a run starts and so where the audit log
* records one. The renders themselves are not recorded one by one: a run is thousands of them, and
* a row each would bury everything else in the log under one button press.
*/
app.post(
'/page-renders',
{
config: {
permissions: ['manage:system']
},
schema: {
summary: 'Start re-rendering every page from its source',
description:
'Lists every page on every site whose editor the frontend can render — markdown, visual and AsciiDoc — for the admin area to render one by one in the browser, which is what lets this work without the Puppeteer extension. Fetch the sources with `POST /system/page-renders/sources` and store each render with `PUT /system/page-renders/{pageId}`. Pages of other editors have no source to render and are only counted.',
tags: ['System'],
response: {
200: {
description: 'The pages to render',
type: 'object',
properties: {
ok: { type: 'boolean' },
skipped: {
type: 'number',
description: 'Pages left out because their editor cannot be rendered.'
},
pages: {
type: 'array',
items: {
type: 'object',
properties: {
id: { type: 'string', format: 'uuid' },
siteId: { type: 'string', format: 'uuid' },
locale: { type: 'string' },
path: { type: 'string' },
title: { type: 'string' },
editor: { type: 'string' }
}
}
}
}
}
}
}
},
async (req) => {
const { pages, skipped } = await WIKI.models.rendering.listForRerender()
await audit(req, 'admin', 'rerenderPages', { pages: pages.length })
return { ok: true, pages, skipped }
}
)
app.post<{ Body: { ids: string[] } }>(
'/page-renders/sources',
{
config: {
permissions: ['manage:system']
},
schema: {
summary: 'Fetch the sources of a batch of pages to re-render',
description:
'Each page comes with the site’s config for its editor, which is what to render it with, and a hash of its content to send back with the render. A page that has gone, or whose editor cannot be rendered, is left out of the reply.',
tags: ['System'],
body: {
type: 'object',
required: ['ids'],
properties: {
ids: {
type: 'array',
items: { type: 'string', format: 'uuid' },
maxItems: 50
}
}
},
response: {
200: {
type: 'object',
properties: {
ok: { type: 'boolean' },
pages: {
type: 'array',
items: {
type: 'object',
properties: {
id: { type: 'string', format: 'uuid' },
siteId: { type: 'string', format: 'uuid' },
path: { type: 'string' },
editor: { type: 'string' },
content: { type: 'string' },
contentHash: { type: 'string' },
config: { type: 'object', additionalProperties: true }
}
}
}
}
}
}
}
},
async (req) => ({
ok: true,
pages: await WIKI.models.rendering.sourcesForRerender(req.body.ids)
})
)
app.put<{ Params: { pageId: string }; Body: { render: string; contentHash: string } }>(
'/page-renders/:pageId',
{
config: {
permissions: ['manage:system']
},
schema: {
summary: 'Store a page render produced by the browser',
description:
'Sanitized like any render, against what the page’s current render shows it was allowed to carry — not against the caller’s permissions, so a re-render never brings back a script or a style that was stripped from the page when it was saved. Answers 409 when the page’s content has changed since its source was fetched, since the render would describe the content before that change.',
tags: ['System'],
params: {
type: 'object',
required: ['pageId'],
properties: {
pageId: { type: 'string', format: 'uuid' }
}
},
body: {
type: 'object',
required: ['render', 'contentHash'],
properties: {
render: { type: 'string' },
contentHash: { type: 'string' }
}
},
response: {
200: {
description: 'Render stored',
type: 'object',
properties: {
ok: { type: 'boolean' }
}
}
}
}
},
async (req, reply) => {
const outcome = await WIKI.models.rendering.storeRerender(
req.params.pageId,
req.body.render,
req.body.contentHash
)
switch (outcome) {
case 'missing':
return reply.notFound('This page no longer exists.')
case 'changed':
return reply.conflict('The page was changed while it was being rendered.')
case 'unsupported':
return reply.badRequest('Pages of this editor cannot be rendered.')
}
return { ok: true }
}
)
/**
* REBUILD PAGE LINKS
*

@ -192,6 +192,7 @@
"admin.audit.actions.registerPasskey": "Registered a passkey",
"admin.audit.actions.rejectPageEdit": "Declined an edit suggestion",
"admin.audit.actions.renderPage": "Queued a page for rendering",
"admin.audit.actions.rerenderPages": "Re-rendered every page",
"admin.audit.actions.requestPasswordReset": "Requested a password reset",
"admin.audit.actions.resetPassword": "Reset a password from an emailed link",
"admin.audit.actions.resetUserPassword": "Set a user's password",
@ -1558,6 +1559,35 @@
"admin.utilities.pageProblems.problems.publishDatesInverted": "Its publishing window ends before it starts.",
"admin.utilities.pageProblems.problems.searchIndexMissing": "It is searchable, but missing from the search index. Rebuilding the search index adds it.",
"admin.utilities.pageProblems.problems.navigationMenuMissing": "Its sidebar points at a menu that does not exist, so it shows no menu (navigation mode: {mode}).",
"admin.utilities.pageRenders.count": "{processed} / {total}",
"admin.utilities.pageRenders.logBatch": "Batch of pages {from}–{to} of {total}",
"admin.utilities.pageRenders.logFailed": "failed: {message}",
"admin.utilities.pageRenders.logFetched": "fetched {count} in {ms} ms ({size})",
"admin.utilities.pageRenders.logFetching": "Fetching their sources",
"admin.utilities.pageRenders.logFinished": "Rendering complete in {seconds} s: {rendered} rendered, {failed} failed, {skipped} skipped.",
"admin.utilities.pageRenders.logGone": "Skipped: it was deleted before it could be rendered.",
"admin.utilities.pageRenders.logListed": "{pages} page(s) to render, on every site ({ms} ms)",
"admin.utilities.pageRenders.logListing": "Listing the pages",
"admin.utilities.pageRenders.logRendered": "done in {ms} ms ({size})",
"admin.utilities.pageRenders.logRendering": "Rendering ({editor})",
"admin.utilities.pageRenders.logSaved": "done in {ms} ms",
"admin.utilities.pageRenders.logSaving": "Saving to the server",
"admin.utilities.pageRenders.logSkippedChanged": "skipped: it was changed while it was being rendered, and keeps the render that change stored.",
"admin.utilities.pageRenders.logSkippedEditors": "{count} page(s) use an editor with no source to render, and were left out.",
"admin.utilities.pageRenders.logSkippedGone": "skipped: it was deleted while it was being rendered.",
"admin.utilities.pageRenders.logStarted": "Rerendering every page on every site.",
"admin.utilities.pageRenders.logStopped": "Rendering stopped after {processed} of {total} page(s), in {seconds} s.",
"admin.utilities.pageRenders.pagesEmpty": "Every page to render is listed here once rendering starts.",
"admin.utilities.pageRenders.pagesTitle": "Pages",
"admin.utilities.pageRenders.progress": "Progress",
"admin.utilities.pageRenders.progressEmpty": "Start rendering to render every page on every site again from its source. It runs in this browser, so keep this screen open until it finishes. A page keeps the scripts and styles its current render has, and gains none.",
"admin.utilities.pageRenders.start": "Start Rendering",
"admin.utilities.pageRenders.status.done": "Rendered",
"admin.utilities.pageRenders.status.error": "Failed",
"admin.utilities.pageRenders.status.pending": "Waiting",
"admin.utilities.pageRenders.status.skipped": "Skipped",
"admin.utilities.pageRenders.stop": "Stop Rendering",
"admin.utilities.pageRenders.subtitle": "Render every page on every site again from its source",
"admin.utilities.purgeEmptyFolders": "Delete Empty Folders",
"admin.utilities.purgeEmptyFoldersConfirm": "Every folder holding no page and no asset will be deleted, on every site.",
"admin.utilities.purgeEmptyFoldersConfirmWarn": "A folder containing only empty folders goes too, and so does the branch above it once its last folder is gone. Nothing but folders is deleted: a folder holding a page, an asset or a draft is left exactly as it is.",
@ -1595,6 +1625,8 @@
"admin.utilities.rebuildPageRatingsFailed": "The page ratings could not be rebuilt.",
"admin.utilities.rebuildPageRatingsHint": "Recalculate the rating totals shown on every page from the ratings actually recorded, in case they have fallen out of step.",
"admin.utilities.rebuildPageRatingsSuccess": "Every page's ratings were already correct. | Corrected the ratings of 1 page. | Corrected the ratings of {count} pages.",
"admin.utilities.rerenderPages": "Rerender All Pages",
"admin.utilities.rerenderPagesHint": "Render every page on every site again from its source, in this browser. For pages that show a blank or out-of-date render, or after the markdown settings changed.",
"admin.utilities.scanPageProblems": "Scan for Page Problems",
"admin.utilities.scanPageProblemsHint": "Check every page on every site for empty or unreadable content, broken redirections, file tree entries out of step and more. Nothing is changed.",
"admin.utilities.subtitle": "Maintenance and miscellaneous tools",

@ -124,6 +124,7 @@ export const AUDIT_ACTIONS = {
'rebuildSearchIndex',
'rebuildPageLinks',
'rebuildPageRatings',
'rerenderPages',
'installExtension',
'startImport',
'finishImport',

@ -1826,11 +1826,11 @@ class Import {
/**
* Say that a page is waiting to be rendered, in the column a reader is served from.
*
* `adoptStoredPage` stores an empty render and queues a real one, which is right for a folder
* import of a handful of files and wrong for twelve thousand pages at once: the queue is one
* headless browser doing one page at a time, so for most of a day most of the wiki would be blank
* with nothing saying why. Written only over an empty render, so the first real one replaces it and
* nothing here can overwrite a page that already has HTML.
* `adoptStoredPage` creates a page with an empty render and queues a real one, which is right for
* a folder import of a handful of files and wrong for twelve thousand pages at once: the queue is
* one headless browser doing one page at a time, so for most of a day most of the wiki would be
* blank with nothing saying why. Written only over an empty render, so the first real one replaces
* it and nothing here can overwrite a page that already has HTML.
*/
async #markPendingRender(siteId: string, pageId: string, editor: string): Promise<void> {
if (BODYLESS_EDITORS.has(editor)) {

@ -3201,13 +3201,15 @@ class Pages {
? await this.updatePage(
siteId,
existing[0].id,
// -> No `render`: the page keeps the HTML it has until the queued render replaces it. Stale
// for a moment is better than blank — and blank for good if that render never lands,
// which is every pull on an instance without the Puppeteer extension.
{
title,
description,
content,
tags,
publishState: isPublished === false ? 'draft' : 'published',
render: ''
publishState: isPublished === false ? 'draft' : 'published'
} as Partial<PageInput>,
actor
)
@ -3256,9 +3258,9 @@ class Pages {
await WIKI.models.storage.mirrorPage(restored.ref, restored.content)
}
// -> An imported page has no HTML until something renders it, which takes a headless browser this
// instance may not have. Best effort: the page is in the wiki either way, and a re-render can
// be asked for from the admin area once one is available.
// -> A new page has no HTML until something renders it, and an overwritten one still has the HTML
// of its previous content. Rendering takes a headless browser this instance may not have. Best
// effort: the page is in the wiki either way, and a re-render can be asked for once one is.
try {
await this.queueRerender(siteId, page.id, actor)
} catch (err: any) {

@ -1,8 +1,13 @@
import { createHash } from 'node:crypto'
import * as cheerio from 'cheerio'
import sanitizeHtml from 'sanitize-html'
import { eq, inArray, sql } from 'drizzle-orm'
import { asc, eq, inArray, sql } from 'drizzle-orm'
import { flipFromString, rotateFromString } from '@iconify/utils'
import { jobs as jobsTable, pageRenderQueue as renderQueueTable } from '../db/schema.ts'
import {
jobs as jobsTable,
pageRenderQueue as renderQueueTable,
pages as pagesTable
} from '../db/schema.ts'
import { CustomError } from '../helpers/common.ts'
import { hrefsFrom } from '../helpers/pageLinks.ts'
import type { IconifyIcon } from '@iconify/types'
@ -129,6 +134,32 @@ export interface RenderPermissions {
styles: boolean
}
/** A page the admin area's Rerender All Pages works through, as it lists them. */
export interface RerenderablePage {
id: string
siteId: string
locale: string
path: string
title: string
editor: string
}
/** What a browser needs to render one page the way its editor would have. */
export interface RerenderSource {
id: string
siteId: string
path: string
editor: string
content: string
/** Sent back with the render, so that one made from content that has since changed is refused. */
contentHash: string
/** The site's config for that editor — see `editorConfigFor`. */
config: Record<string, any>
}
/** Why a render posted back by Rerender All Pages was or was not stored. */
export type RerenderOutcome = 'stored' | 'missing' | 'changed' | 'unsupported'
/**
* Tags and attributes a page may use whoever wrote it.
*
@ -1056,7 +1087,8 @@ class Rendering {
* Claiming is a delete, so an instance can never pick up a page another one is already rendering,
* and a render that fails is a render that was asked for and did not happen — logged, with the page
* keeping the HTML it had. Re-queueing it here would be a loop, since whatever made it fail is still
* true.
* true. The log is all that reports it, since the job itself completes either way, so a failure is
* an error and the batch ends with a count of what did not render.
*
* A failure also drops the browser rather than trusting it: the likeliest one is a render that ran
* out of time, which leaves a page wedged in whatever loop it was in, and the pages behind it in the
@ -1080,6 +1112,8 @@ class Rendering {
}
let renderer: PageRenderer | null = null
let rendered = 0
let failed = 0
try {
while (!signal?.aborted) {
/*
@ -1104,6 +1138,7 @@ class Rendering {
return
}
let label = entry.pageId
try {
const page = await WIKI.models.pages.getPage({
siteId: entry.siteId,
@ -1115,32 +1150,181 @@ class Rendering {
// for a page that went between the claim and here.
continue
}
label = `${page.id} (${page.path})`
if (!RENDERABLE_EDITORS.has(page.editor)) {
WIKI.logger.warn(
`Cannot render page ${page.id}: server-side rendering is not implemented for the ${page.editor} editor.`
WIKI.logger.error(
`Cannot render page ${label}: server-side rendering is not implemented for the ${page.editor} editor.`
)
failed++
continue
}
const configKey = EDITOR_CONFIG_ALIAS[page.editor] ?? page.editor
const html = await renderer.render(
page.content ?? '',
WIKI.sites[entry.siteId]?.config?.editors?.[configKey]?.config ?? {},
this.editorConfigFor(entry.siteId, page.editor),
{ pagePath: page.path, editor: page.editor }
)
await WIKI.models.pages.storeRender(entry.siteId, page.id, html, {
scripts: entry.allowScripts,
styles: entry.allowStyles
})
WIKI.logger.debug(`Rendered page ${page.id} (${page.path}) from its source.`)
WIKI.logger.debug(`Rendered page ${label} from its source.`)
rendered++
} catch (err: any) {
WIKI.logger.warn(`Failed to render page ${entry.pageId}: ${err.message}`)
WIKI.logger.error(`Failed to render page ${label}: ${err.message}`)
failed++
await this.discardRenderer(renderer)
renderer = null
}
}
} finally {
await this.discardRenderer(renderer)
if (failed > 0) {
WIKI.logger.error(
`Render queue: ${failed} of ${rendered + failed} queued pages failed to render and were left as they were. Re-render them once the cause is fixed.`
)
} else if (rendered > 0) {
WIKI.logger.info(`Render queue: rendered ${rendered} pages.`)
}
}
}
/**
* The site's config for an editor, which is what a render of one of its pages is made with.
*
* Through `EDITOR_CONFIG_ALIAS`, so a visual page is rendered with the markdown settings its editor
* previews with.
*/
editorConfigFor(siteId: string, editor: string): Record<string, any> {
const configKey = EDITOR_CONFIG_ALIAS[editor] ?? editor
return WIKI.sites[siteId]?.config?.editors?.[configKey]?.config ?? {}
}
/**
* Every page on every site that a browser can render, for the admin area's Rerender All Pages.
*
* That utility is the other way to re-render, and the one that needs no Puppeteer: the admin's own
* browser runs the pipeline the editor runs, a page at a time, and posts each render back through
* `storeRerender`. So this is only the list to work through — no content, which for a large wiki
* would be the whole of it in one reply — and `sourcesForRerender` hands the sources out a batch at
* a time.
*
* @returns The pages, and how many were left out for an editor whose pages have no source to render
*/
async listForRerender(): Promise<{ pages: RerenderablePage[]; skipped: number }> {
const rows = await WIKI.db
.select({
id: pagesTable.id,
siteId: pagesTable.siteId,
locale: pagesTable.locale,
path: pagesTable.path,
title: pagesTable.title,
editor: pagesTable.editor
})
.from(pagesTable)
.orderBy(asc(pagesTable.siteId), asc(pagesTable.locale), asc(pagesTable.path))
const pages = rows.filter((row) => RENDERABLE_EDITORS.has(row.editor))
return { pages, skipped: rows.length - pages.length }
}
/**
* The sources of a batch of pages, with what rendering each one needs. A page that has gone, or
* whose editor cannot be rendered, is simply not in the answer.
*/
async sourcesForRerender(ids: string[]): Promise<RerenderSource[]> {
if (ids.length < 1) {
return []
}
const rows = await WIKI.db
.select({
id: pagesTable.id,
siteId: pagesTable.siteId,
path: pagesTable.path,
editor: pagesTable.editor,
content: pagesTable.content
})
.from(pagesTable)
.where(inArray(pagesTable.id, ids))
return rows
.filter((row) => RENDERABLE_EDITORS.has(row.editor))
.map((row) => ({
id: row.id,
siteId: row.siteId,
path: row.path,
editor: row.editor,
content: row.content ?? '',
contentHash: hashContent(row.content),
config: this.editorConfigFor(row.siteId, row.editor)
}))
}
/**
* Store a render a browser produced for Rerender All Pages.
*
* Refused as `changed` when the page's content is no longer what the browser was handed: somebody
* saved it in between, and that save stored a render of its own that this one would replace with
* HTML describing the content before it.
*
* What the render may carry is the page's own say, read off the render it has now — see
* `inheritedPermissions`. Not the permissions of whoever pressed the button, which on this route is
* always `manage:system`: that would bring back, on every page at once, each `<script>` the
* sanitizer stripped from an author who was not allowed one, and everything an import brought in.
*/
async storeRerender(id: string, html: string, contentHash: string): Promise<RerenderOutcome> {
const rows = await WIKI.db
.select({
siteId: pagesTable.siteId,
editor: pagesTable.editor,
content: pagesTable.content,
render: pagesTable.render
})
.from(pagesTable)
.where(eq(pagesTable.id, id))
.limit(1)
const page = rows[0]
if (!page) {
return 'missing'
}
if (!RENDERABLE_EDITORS.has(page.editor)) {
return 'unsupported'
}
if (hashContent(page.content) !== contentHash) {
return 'changed'
}
await WIKI.models.pages.storeRender(
page.siteId,
id,
html,
this.inheritedPermissions(page.render)
)
return 'stored'
}
/**
* What a stored render shows its page was allowed to carry, which nothing else records.
*
* A save sanitizes against what its author may embed, so whatever the sanitizer only lets through
* with a permission being IN the render means that permission was held: a `<script>`, an `<iframe>`
* or an inline handler for `scripts`, a `<style>` for `styles`. Absent, it was either refused or never
* written, and refusing it again is right either way. A blank render — an import, or one whose
* render failed — therefore grants nothing, which is what an import is rendered with.
*
* A `<style>` inside an `<svg>` is not counted: `inlineIcons` runs after the sanitizer, and an
* animated icon carries one of its own.
*/
inheritedPermissions(render: string | null): RenderPermissions {
if (!render) {
return { scripts: false, styles: false }
}
const $ = cheerio.load(render, null, false)
const scripts =
$('script, iframe').length > 0 ||
$('*')
.toArray()
.some((el) => 'attribs' in el && Object.keys(el.attribs).some((name) => /^on/i.test(name)))
const styles = $('style')
.toArray()
.some((el) => $(el).closest('svg').length < 1)
return { scripts, styles }
}
/**
@ -1266,6 +1450,13 @@ class Rendering {
}
}
/** How a re-render recognises the content it was made from, without carrying the content back. */
function hashContent(content: string | null): string {
return createHash('sha256')
.update(content ?? '')
.digest('hex')
}
/**
* Why the browser Puppeteer launched may not be one it can drive, or null when nothing says so.
*

@ -0,0 +1,569 @@
<template>
<w-layout view="hHh lpR fFf" container>
<!--
The bar and the progress strip under it are two rows of the ONE header element, as in
`ImportWikijs2Overlay` -- see the note there for why the strip cannot be a sibling.
-->
<w-header class="card-header render-header">
<div class="flex items-center px-4 py-2">
<w-icon name="img:/_assets/icons/ultraviolet-html.svg" left size="md" />
<div>
<span>{{ t('admin.utilities.rerenderPages') }}</span>
<div class="text-caption">{{ t('admin.utilities.pageRenders.subtitle') }}</div>
</div>
<w-space />
<w-btn
class="mr-2"
flat
rounded
color="white"
:aria-label="t(`common.actions.viewDocs`)"
icon="la:question-circle"
:href="siteStore.docsBase + `/admin/utilities`"
target="_blank"
type="a" />
<w-btn-group push>
<w-btn
push
color="white"
text-color="grey-7"
:label="t(`common.actions.close`)"
:aria-label="t(`common.actions.close`)"
icon="la:times"
@click="close" />
</w-btn-group>
</div>
<w-linear-progress
v-if="showProgress"
size="10px"
:striped="isRunning"
:color="state.phase === 'failed' ? 'negative' : 'positive'"
:value="state.progress"
:aria-label="t('admin.utilities.pageRenders.progress')" />
</w-header>
<w-page-container>
<w-page class="p-4">
<div class="grid grid-cols-12 gap-4 items-start">
<!-- ----------------------- -->
<!-- Pages -->
<!-- ----------------------- -->
<div class="col-span-12 lg:col-span-5 flex flex-col gap-4">
<!--
Spelt out in the slot rather than through `icon`/`label`, for the reason given on the
import's own button: `WBtn`'s `loading` would hide the label, and the label is what
says which of the two this button currently does.
-->
<w-btn
unelevated
size="lg"
class="w-full"
:color="isRunning ? 'negative' : 'positive'"
text-color="white"
:disabled="state.phase === 'starting'"
@click="toggleRun">
<w-icon :name="isRunning ? 'la:stop-circle' : 'la:play-circle'" class="shrink-0" />
<span>{{
isRunning
? t('admin.utilities.pageRenders.stop')
: t('admin.utilities.pageRenders.start')
}}</span>
</w-btn>
<w-card class="pb-2">
<w-card-header>
{{ t('admin.utilities.pageRenders.pagesTitle') }}
<template v-if="state.pages.length > 0" #action>
<span class="text-sm font-normal text-grey-6">{{
t('admin.utilities.pageRenders.count', {
processed: state.processed,
total: state.pages.length
})
}}</span>
</template>
</w-card-header>
<div v-if="state.pages.length < 1" class="px-4 py-2 text-sm text-grey-6">
{{ t('admin.utilities.pageRenders.pagesEmpty') }}
</div>
<!--
A table rather than a list, because each row is read ACROSS: how its render went,
then which page it is. Scrolls on its own so that the button above stays in reach on
a wiki of thousands of pages.
-->
<div v-else class="pages-scroll">
<table class="pages-table w-full border-collapse text-left">
<tbody>
<tr v-for="page of state.pages" :key="page.id" class="pages-table__row">
<td class="w-16 pl-4 pr-0 py-1.5 align-middle">
<w-badge color="grey-7" class="max-w-full truncate" :label="page.locale" />
</td>
<td class="px-2 py-1.5 min-w-0">
<div class="text-sm truncate">{{ page.title }}</div>
<div class="text-xs text-grey-6 truncate">{{ pageSubtitle(page) }}</div>
</td>
<td class="w-10 pl-1 pr-4 py-1.5 align-middle text-right">
<w-spinner v-if="page.status === 'running'" size="16px" color="primary" />
<w-icon
v-else
:name="statusIcons[page.status].icon"
size="16px"
:class="statusIcons[page.status].class"
:aria-label="t(`admin.utilities.pageRenders.status.${page.status}`)" />
</td>
</tr>
</tbody>
</table>
</div>
</w-card>
</div>
<!-- ----------------------- -->
<!-- Progress log -->
<!-- ----------------------- -->
<div class="col-span-12 lg:col-span-7">
<w-card>
<w-card-header>{{ t('admin.utilities.pageRenders.progress') }}</w-card-header>
<div class="p-4 pt-0">
<progress-log
:entries="state.log"
:empty-text="t('admin.utilities.pageRenders.progressEmpty')" />
</div>
</w-card>
</div>
</div>
</w-page>
</w-page-container>
</w-layout>
</template>
<script setup>
import { computed, onBeforeUnmount, reactive } from 'vue'
import { useI18n } from 'vue-i18n'
import ProgressLog from '@/components/ProgressLog.vue'
import { apiErrorMessage } from '@/helpers/apiError'
import { renderSource } from '@/renderers/source'
import { useAdminStore } from '@/stores/admin'
import { useSiteStore } from '@/stores/site'
/**
* Render every page on every site again from its source, in this browser.
*
* The other way to re-render is the server's queue, which drives the same pipeline in a headless
* browser and so needs the Puppeteer extension. This needs nothing: `renderSource` is that same
* pipeline, and the admin's browser is a browser. So it is the way to put right renders that went
* blank or stale on an instance with no extension, and to bring every page up to date after the
* markdown settings change.
*
* Driven from here a batch at a time, the way the page problem scan is: `POST /system/page-renders`
* lists the pages, `sources` hands out a batch of their content, and each render goes back on its own
* `PUT`. Stopping is not asking again -- there is no job on the server to cancel, and a page is either
* stored or left exactly as it was. What a render may carry (scripts, styles) is the server's to
* decide, from the page's current render, and nothing sent from here can widen it.
*/
/** Sources fetched per request. Small enough to keep a request quick, large enough to matter. */
const BATCH_SIZE = 20
/** Lines the log keeps before it starts dropping the oldest routine ones -- see `trimLog`. */
const MAX_LOG_LINES = 5000
// STORES
const adminStore = useAdminStore()
const siteStore = useSiteStore()
// I18N
const { t, locale } = useI18n()
// DATA
const state = reactive({
/**
* `{ id, siteId, locale, path, title, editor, status }`, in the server's order. `status` is
* `pending`, `running`, `done`, `error` or `skipped` -- the last for a page that was deleted or
* changed while the run reached it, which keeps whatever render that change gave it.
*/
pages: [],
/** Pages finished one way or another, which is what the count and the strip measure. */
processed: 0,
/** `{ ts, level, message, location?, url? }` — see `ProgressLog`. */
log: [],
/** `idle`, `starting`, `running`, then `done`, `stopped` or `failed`. */
phase: 'idle',
/** 0..1 */
progress: 0
})
/**
* Asked between pages. Not state: nothing draws it, and it has to be readable from inside the loop
* the moment it is set.
*/
let stopRequested = false
/** Literal names, so that the icon bundle picks them up -- see `scripts/generate-icons.mjs`. */
const statusIcons = {
pending: { icon: 'la:clock', class: 'text-grey-5' },
done: { icon: 'la:check-circle', class: 'text-positive' },
error: { icon: 'la:times-circle', class: 'text-negative' },
skipped: { icon: 'la:exclamation-triangle', class: 'text-orange-8' }
}
// COMPUTED
const isRunning = computed(() => state.phase === 'running')
/** As for the page problem scan: the strip is for watching a run, and for how far a failed one got. */
const showProgress = computed(() => state.phase === 'running' || state.phase === 'failed')
/** Which site a page is on is only worth saying when there is more than one to choose from. */
const siteTitles = computed(() =>
adminStore.sites.length > 1
? Object.fromEntries(adminStore.sites.map((site) => [site.id, site.title]))
: null
)
/** Sizes in the log, in the reader's own number format. */
const sizeFormat = computed(
() =>
new Intl.NumberFormat(locale.value, {
style: 'unit',
unit: 'kilobyte',
maximumFractionDigits: 1
})
)
// METHODS
function close() {
adminStore.$patch({ overlay: '' })
}
/**
* Push a line, and hand back the reactive copy of it so that a step can finish its own line in place
* rather than writing a second one.
*/
function addLog(level, message, extra = {}) {
state.log.push({
ts: Temporal.Now.plainTimeISO().toString({ smallestUnit: 'second' }),
level,
message,
...extra
})
trimLog()
return state.log.at(-1)
}
/**
* Keep the log to a size the panel can draw. A run writes three lines a page, so a large wiki would
* otherwise put tens of thousands of rows in the DOM. What goes is the oldest routine output; the
* warnings and errors, which are what anybody scrolls back for, are kept however many there are.
*/
function trimLog() {
while (state.log.length > MAX_LOG_LINES) {
const idx = state.log.findIndex((entry) => entry.level === 'info' || entry.level === 'success')
if (idx < 0) {
return
}
state.log.splice(idx, 1)
}
}
/**
* One sub-task of the run, as a line that says what is being done and is then finished in place with
* how it went: `↳ Rendering (markdown)… done in 12 ms, 4.1 kB`.
*/
function startStep(message) {
const label = ` ↳ ${message}…`
const entry = addLog('info', label)
const started = performance.now()
return {
elapsed: () => Math.round(performance.now() - started),
finish(detail, level = 'info') {
entry.level = level
entry.message = `${label} ${detail}`
}
}
}
/** The line a page's log starts with: where it is, then what it is called. */
function pageLocation(page) {
const site = siteTitles.value?.[page.siteId]
return `${site ? `${site} · ` : ''}${page.locale}/${page.path}`
}
/** The second line of a row in the table, whose locale is already in the badge beside it. */
function pageSubtitle(page) {
const site = siteTitles.value?.[page.siteId]
return `${site ? `${site} · ` : ''}/${page.path}`
}
function formatSize(text) {
return sizeFormat.value.format(new Blob([text]).size / 1000)
}
function finishPage(page, status) {
page.status = status
state.processed++
state.progress = state.pages.length > 0 ? state.processed / state.pages.length : 1
}
/** Render one page here and store it there. Never throws: a page that fails is the page's problem. */
async function renderPage(page, source) {
addLog('info', page.title, { location: pageLocation(page) })
if (!source) {
addLog('warn', ` ↳ ${t('admin.utilities.pageRenders.logGone')}`)
finishPage(page, 'skipped')
return
}
page.status = 'running'
const rendering = startStep(
t('admin.utilities.pageRenders.logRendering', { editor: source.editor })
)
let html
try {
html = await renderSource(source.content, source.config, {
pagePath: source.path,
editor: source.editor
})
rendering.finish(
t('admin.utilities.pageRenders.logRendered', {
ms: rendering.elapsed(),
size: formatSize(html)
})
)
} catch (err) {
rendering.finish(t('admin.utilities.pageRenders.logFailed', { message: err.message }), 'error')
finishPage(page, 'error')
return
}
const saving = startStep(t('admin.utilities.pageRenders.logSaving'))
try {
await API_CLIENT.put(`system/page-renders/${page.id}`, {
json: { render: html, contentHash: source.contentHash }
})
saving.finish(t('admin.utilities.pageRenders.logSaved', { ms: saving.elapsed() }))
finishPage(page, 'done')
} catch (err) {
const status = err.response?.status
if (status === 404) {
saving.finish(t('admin.utilities.pageRenders.logSkippedGone'), 'warn')
finishPage(page, 'skipped')
} else if (status === 409) {
saving.finish(t('admin.utilities.pageRenders.logSkippedChanged'), 'warn')
finishPage(page, 'skipped')
} else {
saving.finish(
t('admin.utilities.pageRenders.logFailed', { message: apiErrorMessage(err) }),
'error'
)
finishPage(page, 'error')
}
}
}
async function startRun() {
stopRequested = false
state.phase = 'starting'
state.pages = []
state.processed = 0
state.progress = 0
state.log = []
const runStarted = performance.now()
try {
addLog('info', t('admin.utilities.pageRenders.logStarted'))
const listing = startStep(t('admin.utilities.pageRenders.logListing'))
const resp = await API_CLIENT.post('system/page-renders').json()
state.pages = (resp?.pages ?? []).map((page) => ({ ...page, status: 'pending' }))
listing.finish(
t('admin.utilities.pageRenders.logListed', {
pages: state.pages.length,
ms: listing.elapsed()
})
)
if (resp?.skipped > 0) {
addLog('info', t('admin.utilities.pageRenders.logSkippedEditors', { count: resp.skipped }))
}
state.phase = 'running'
for (let idx = 0; idx < state.pages.length && !stopRequested; idx += BATCH_SIZE) {
const batch = state.pages.slice(idx, idx + BATCH_SIZE)
addLog(
'info',
t('admin.utilities.pageRenders.logBatch', {
from: idx + 1,
to: idx + batch.length,
total: state.pages.length
})
)
const fetching = startStep(t('admin.utilities.pageRenders.logFetching'))
let sources
try {
sources = await API_CLIENT.post('system/page-renders/sources', {
json: { ids: batch.map((page) => page.id) }
}).json()
} catch (err) {
fetching.finish(
t('admin.utilities.pageRenders.logFailed', { message: apiErrorMessage(err) }),
'error'
)
throw err
}
const byId = new Map((sources?.pages ?? []).map((source) => [source.id, source]))
fetching.finish(
t('admin.utilities.pageRenders.logFetched', {
count: byId.size,
ms: fetching.elapsed(),
size: formatSize(JSON.stringify(sources?.pages ?? []))
})
)
for (const page of batch) {
if (stopRequested) {
break
}
await renderPage(page, byId.get(page.id))
}
}
const counts = { done: 0, error: 0, skipped: 0 }
for (const page of state.pages) {
if (page.status in counts) {
counts[page.status]++
}
}
const seconds = Math.round((performance.now() - runStarted) / 100) / 10
if (state.processed < state.pages.length) {
state.phase = 'stopped'
addLog(
'warn',
t('admin.utilities.pageRenders.logStopped', {
processed: state.processed,
total: state.pages.length,
seconds
})
)
} else {
state.phase = 'done'
addLog(
counts.error > 0 ? 'warn' : 'success',
t('admin.utilities.pageRenders.logFinished', {
rendered: counts.done,
failed: counts.error,
skipped: counts.skipped,
seconds
})
)
}
} catch (err) {
// -> Only the list and the batches get here: a page of its own never throws out of `renderPage`
addLog('error', apiErrorMessage(err))
state.phase = 'failed'
}
}
/** Takes effect once the page in hand is finished; nothing on the server needs telling. */
function stopRun() {
stopRequested = true
}
function toggleRun() {
if (isRunning.value) {
stopRun()
} else {
startRun()
}
}
// -> Closing the overlay mid-run ends it, rather than leaving a loop rendering for nobody
onBeforeUnmount(stopRun)
</script>
<style scoped lang="scss">
/*
The header's closing line, redrawn under the progress strip. The same treatment as
`.import-header` in `ImportWikijs2Overlay`, which says why: `.card-header`'s own border and shadow
read as one thick rule beneath a progress bar, and a 1px border lands on a fractional device row
under display scaling.
*/
.render-header {
flex-direction: column;
align-items: stretch;
position: relative;
}
body.body--light .render-header,
body.body--dark .render-header {
border-bottom: 0;
box-shadow: none;
}
.render-header::after {
content: '';
position: absolute;
left: 0;
right: 0;
bottom: 0;
height: 1px;
background-color: $dark-6;
transform: scaleY(calc(1 / var(--w-dpr, 1)));
transform-origin: bottom;
}
body.body--dark .render-header::after {
background-color: #000;
}
/* -> Tall enough to be worth scrolling, short enough that the card never runs off the overlay */
.pages-scroll {
max-height: calc(100vh - 290px);
overflow-y: auto;
}
/*
The pages table, ruled and striped as the page problem scan's checklist is: hairlines drawn by a
pseudo-element on each cell, scaled to one device pixel the way `WTable` draws its own. `fixed`
layout, so that a long path truncates instead of widening the column.
*/
.pages-table {
--pages-rule: rgb(0 0 0 / 0.12);
--pages-stripe: rgb(0 0 0 / 0.02);
--pages-hover: rgb(0 0 0 / 0.05);
table-layout: fixed;
td {
position: relative;
}
tbody tr > *::before {
content: '';
position: absolute;
top: 0;
left: 0;
right: 0;
height: 1px;
background-color: var(--pages-rule);
transform: scaleY(calc(1 / var(--w-dpr, 1)));
transform-origin: top left;
pointer-events: none;
}
}
.pages-table__row:nth-child(odd) {
background-color: var(--pages-stripe);
}
.pages-table__row:hover {
background-color: var(--pages-hover);
}
body.body--dark .pages-table {
--pages-rule: rgb(255 255 255 / 0.15);
--pages-stripe: rgb(255 255 255 / 0.025);
--pages-hover: rgb(255 255 255 / 0.07);
}
</style>

@ -514,6 +514,10 @@ const overlays = {
loader: () => import('../components/PageProblemsOverlay.vue'),
loadingComponent: LoadingGeneric
}),
RerenderPagesOverlay: defineAsyncComponent({
loader: () => import('../components/RerenderPagesOverlay.vue'),
loadingComponent: LoadingGeneric
}),
UserEditOverlay: defineAsyncComponent({
loader: () => import('../components/UserEditOverlay.vue'),
loadingComponent: LoadingGeneric

@ -197,6 +197,22 @@
:label="t(`common.actions.proceed`)" />
</w-item-section>
</w-item>
<w-item>
<blueprint-icon icon="html" :hue-rotate="45" />
<w-item-section>
<w-item-label>{{ t(`admin.utilities.rerenderPages`) }}</w-item-label>
<w-item-label caption>{{ t(`admin.utilities.rerenderPagesHint`) }}</w-item-label>
</w-item-section>
<w-item-section side>
<w-btn
class="acrylic-btn"
flat
icon="la:arrow-circle-right"
color="primary"
@click="openRerenderPages"
:label="t(`common.actions.proceed`)" />
</w-item-section>
</w-item>
<w-item>
<blueprint-icon icon="rescan-document" :hue-rotate="45" />
<w-item-section>
@ -490,6 +506,14 @@ function openPageProblems() {
adminStore.$patch({ overlay: 'PageProblemsOverlay' })
}
/**
* Rerendering every page as well: it renders in this browser, a page at a time, and its screen is the
* list of pages and how each one went.
*/
function openRerenderPages() {
adminStore.$patch({ overlay: 'RerenderPagesOverlay' })
}
/**
* Close every websocket the wiki holds — the editors of anyone collaborating on a page, and any open
* admin terminal. Confirmed first because it interrupts people who are working: their clients

@ -9,33 +9,10 @@
* Built to a fixed filename (`_assets/renderer.js`, see `vite.config.js`) because the backend has to
* reference it from a static page and cannot resolve a hashed one.
*/
import { MarkdownRenderer } from './markdown'
import { renderSource } from './source'
/**
* A page's source, rendered the way the editor that wrote it would have rendered it.
*
* Which pipeline is the caller's to say. A source is not self-describing — the server holds the
* editor in a column and passes it in, and guessing from the text would be guessing.
*
* Asciidoctor is reached through a dynamic import, so the chunk it lives in is fetched the first time
* an AsciiDoc page is rendered and never on an instance that has none. It is roughly as large as
* everything else in this bundle put together, and most wikis will never ask for it.
*
* @param {string} content The page source
* @param {object} config The site's config for that editor, so the result matches what an author
* would have produced in it
* @param {object} context What the source cannot say about itself: `pagePath`, which a relative image
* in it resolves against, and `editor`, which picks the pipeline
* @returns {Promise<string>} Rendered HTML, before the server's own post-processing
*/
window.__wikiRender = async function (content, config = {}, context = {}) {
const { editor = 'markdown', ...rest } = context
if (editor === 'asciidoc') {
const { AsciidocRenderer } = await import('./asciidoc')
return new AsciidocRenderer(config).render(content ?? '', rest)
}
return new MarkdownRenderer(config).render(content ?? '', rest)
}
/** See `renderSource`, which the admin area's Rerender All Pages calls too. */
window.__wikiRender = renderSource
// -> Polled by the caller: a module script is deferred, so the page can be "loaded" before this ran
window.__wikiRenderReady = true

@ -0,0 +1,32 @@
import { MarkdownRenderer } from './markdown'
/**
* A page's source, rendered the way the editor that wrote it would have rendered it.
*
* Shared by the two things that render a page away from its editor: the headless bundle Puppeteer
* drives (`headless.js`) and the admin area's Rerender All Pages, which runs in the admin's own
* browser so that an instance without Puppeteer can re-render too. One function, so that the two can
* never produce different HTML for the same page.
*
* Which pipeline is the caller's to say. A source is not self-describing — the server holds the
* editor in a column and passes it in, and guessing from the text would be guessing.
*
* Asciidoctor is reached through a dynamic import, so the chunk it lives in is fetched the first time
* an AsciiDoc page is rendered and never on an instance that has none. It is roughly as large as
* everything else in this bundle put together, and most wikis will never ask for it.
*
* @param {string} content The page source
* @param {object} config The site's config for that editor, so the result matches what an author
* would have produced in it
* @param {object} context What the source cannot say about itself: `pagePath`, which a relative image
* in it resolves against, and `editor`, which picks the pipeline
* @returns {Promise<string>} Rendered HTML, before the server's own post-processing
*/
export async function renderSource(content, config = {}, context = {}) {
const { editor = 'markdown', ...rest } = context
if (editor === 'asciidoc') {
const { AsciidocRenderer } = await import('./asciidoc')
return new AsciidocRenderer(config).render(content ?? '', rest)
}
return new MarkdownRenderer(config).render(content ?? '', rest)
}
Loading…
Cancel
Save