feat: serve sitemap.xml and robots.txt + headers

CLAUDE.md

@@ -40,7 +40,7 @@ The backend is **TypeScript 7**; `frontend/` and `blocks/` are JavaScript. See
   confirmation link or a password reset lands in a web inbox at `http://localhost:8025` rather than a
   real mailbox. Point the wiki at it under **Admin → Mail** — host `localhost`, port 1025, TLS off,
   no credentials.
-- `localazy.json` — translation sync config; locale strings live in `backend/locales/`.
+- Locale strings live in `backend/locales/`.
 
 ### `backend/`
 
@@ -65,6 +65,10 @@ path in silence.
   background) under `/_site`; `icons.ts` serves icons under `/_icons`, implementing the part of the
   Iconify API protocol the frontend speaks (`/_icons/<prefix>.json?icons=a,b` and
   `/_icons/<prefix>/<name>.svg`). Public and cached hard — see [Icons](#icons).
+  `rootFiles.ts` is the exception that registers at the root rather than under a prefix: `robots.txt`
+  and `sitemap.xml`, the two names in `RESERVED_ROOT_FILES` a crawler asks for by convention, both
+  driven by the site's **General → SEO** settings. The sitemap lists what the GUESTS group may read
+  and nothing else.
 - `core/` — long-lived singletons: `config.ts` (yml + db-backed settings), `db.ts` (pg pool, Drizzle
   instance, migrations, LISTEN/NOTIFY pubsub), `logger.ts`, `scheduler.ts` (poolifier thread pool +
   postgres-backed job queue).
@@ -86,7 +90,7 @@ path in silence.
   file-tree half of the storage modules that address content by path (see [Storage targets](#storage-targets)).
 - `types/` — ambient declarations: `global.d.ts` (the `WIKI` global) and `fastify.d.ts` (session +
   route-permission augmentations).
-- `locales/` — `en.json` source strings (Localazy-managed) + `metadata.js` language table (the one
+- `locales/` — `en.json` source strings (CrowdIn-managed) + `metadata.js` language table (the one
   remaining JavaScript file; typed by its sibling `metadata.d.ts`).
 
 ### `frontend/`
@@ -245,8 +249,7 @@ tracked and will turn up in the next commit.
 ## TypeScript (backend)
 
 The backend is entirely **TypeScript 7** (the native Go compiler — `tsc` is a platform binary, not a
-JS bundle). The only remaining `.js` is `locales/metadata.js`, which is Localazy-generated output and
+JS bundle).
-is typed by a sibling `locales/metadata.d.ts`.
 
 **There is no build step.** Node 26 runs `.ts` files directly by stripping types at load time, so
 `node backend` and nodemon keep working unchanged as files are converted. `tsc` is used purely as a

backend/controllers/rootFiles.ts

@@ -0,0 +1,263 @@
+import { chunk } from 'es-toolkit/array'
+import type { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'
+import type { SitemapPage } from '../models/pages.ts'
+
+/**
+ * The sitemap protocol's own ceiling: 50,000 URLs, and 50MB uncompressed, per file. Past it the
+ * document served is a sitemap index naming numbered parts of this size instead.
+ */
+const MAX_URLS_PER_FILE = 50000
+
+/**
+ * How long a crawler — or anything caching in front of this — may hold one of these files.
+ *
+ * `public` whoever asked: neither document varies by requester. The sitemap is `listForSitemap`'s
+ * answer, which is the guests group's view and nobody else's, and robots.txt is a site setting — so a
+ * shared cache has nothing to leak from one reader to another. Ten minutes because the alternative to
+ * a slightly stale sitemap is re-reading every page of a wiki for a file a crawler fetches a few
+ * times a day.
+ */
+const ROOT_FILE_CACHE = 'public, max-age=600'
+
+/** Everything written into the document goes through here, including anything that came off a header. */
+function xmlEscape(value: string): string {
+  return value
+    .replaceAll('&', '&')
+    .replaceAll('<', '&lt;')
+    .replaceAll('>', '&gt;')
+    .replaceAll('"', '&quot;')
+    .replaceAll("'", '&apos;')
+}
+
+/**
+ * The origin a `<loc>` — or robots.txt's `Sitemap:` line — is written against.
+ *
+ * The requester's own, and deliberately not the site's configured hostname: a sitemap may only list
+ * URLs on the host it was itself fetched from — a crawler discards the rest as a cross-submission —
+ * so the host in the request IS the answer, whether the site is bound to it or is the catch-all `*`.
+ * It comes off a header and is therefore whatever the client said, which is why it is escaped before
+ * it reaches the XML. It cannot carry a line break into robots.txt: a header value holding one is
+ * rejected by the HTTP parser long before this.
+ */
+function originOf(req: FastifyRequest): string {
+  return `${req.protocol}://${req.host}`
+}
+
+/**
+ * Where a page is, absolute and ready to be written into an element.
+ *
+ * `urlFor` is what decides the path, the same as for every link the wiki makes of its own pages, so
+ * the locale prefix is bracketed exactly where that site's settings put one. A page path is held to
+ * `[a-zA-Z0-9-_/]` when it is saved, but a locale's short code is an administrator's to alias, so the
+ * result is percent-encoded before it is escaped.
+ */
+function locOf(origin: string, siteId: string, page: SitemapPage): string {
+  return xmlEscape(
+    `${origin}${encodeURI(WIKI.models.pages.urlFor(siteId, page.locale, page.path))}`
+  )
+}
+
+/** W3C datetime, which is what `<lastmod>` takes. Seconds: a crawler has no use for the nanoseconds. */
+function lastmodOf(date: Date): string {
+  return date.toTemporalInstant().toString({ smallestUnit: 'second' })
+}
+
+/**
+ * The translations of each page, keyed by locale group.
+ *
+ * Built over the WHOLE list rather than the part being rendered, so that a page whose other languages
+ * fell into a different numbered file still names all of them. A page with no counterparts has a null
+ * group and is absent from here — nulls are distinct, so they would otherwise all be one group.
+ */
+function localeGroupsOf(pages: SitemapPage[]): Map<string, SitemapPage[]> {
+  const groups = new Map<string, SitemapPage[]>()
+  for (const page of pages) {
+    if (!page.localeGroupId) {
+      continue
+    }
+    const group = groups.get(page.localeGroupId)
+    if (group) {
+      group.push(page)
+    } else {
+      groups.set(page.localeGroupId, [page])
+    }
+  }
+  return groups
+}
+
+/**
+ * One file of page URLs.
+ *
+ * `changefreq` and `priority` are deliberately absent: no search engine has read either for years,
+ * and the wiki has nothing honest to put in them — every page would claim the same numbers.
+ *
+ * What is here instead is `xhtml:link`, one per language a page exists in, which is how a crawler is
+ * told that two paths are the same page rather than duplicates of each other. Emitted only where a
+ * page actually has counterparts, and including the page itself, which is what the annotation calls
+ * for. Only counterparts that are in this document at all: a translation the guests group may not
+ * read is not an alternate a crawler should be sent to.
+ */
+function renderUrlset(
+  origin: string,
+  siteId: string,
+  pages: SitemapPage[],
+  all: SitemapPage[]
+): string {
+  const groups = localeGroupsOf(all)
+  const lines = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">'
+  ]
+  for (const page of pages) {
+    lines.push('  <url>')
+    lines.push(`    <loc>${locOf(origin, siteId, page)}</loc>`)
+    lines.push(`    <lastmod>${lastmodOf(page.updatedAt)}</lastmod>`)
+    const alternates = page.localeGroupId ? (groups.get(page.localeGroupId) ?? []) : []
+    if (alternates.length > 1) {
+      for (const alternate of alternates) {
+        lines.push(
+          `    <xhtml:link rel="alternate" hreflang="${xmlEscape(alternate.locale)}" href="${locOf(origin, siteId, alternate)}"/>`
+        )
+      }
+    }
+    lines.push('  </url>')
+  }
+  lines.push('</urlset>')
+  return `${lines.join('\n')}\n`
+}
+
+/**
+ * The index served in place of the list once it no longer fits in one file.
+ *
+ * The parts are `?p=N` on this same path rather than files of their own, because a sitemap may only
+ * list URLs at or below its own directory — a part under `/_sitemap/` could name nothing outside it —
+ * and because the reserved root files a crawler may ask for are a fixed set that no numbered name
+ * could join. A query string is a URL like any other to a crawler.
+ */
+function renderIndex(origin: string, parts: SitemapPage[][]): string {
+  const lines = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+  ]
+  parts.forEach((pages, index) => {
+    const newest = pages.reduce<Date | null>(
+      (latest, page) => (!latest || page.updatedAt > latest ? page.updatedAt : latest),
+      null
+    )
+    lines.push('  <sitemap>')
+    lines.push(`    <loc>${xmlEscape(`${origin}/sitemap.xml?p=${index + 1}`)}</loc>`)
+    if (newest) {
+      lines.push(`    <lastmod>${lastmodOf(newest)}</lastmod>`)
+    }
+    lines.push('  </sitemap>')
+  })
+  lines.push('</sitemapindex>')
+  return `${lines.join('\n')}\n`
+}
+
+function sendXml(reply: FastifyReply, body: string) {
+  return reply
+    .header('Cache-Control', ROOT_FILE_CACHE)
+    .type('application/xml; charset=utf-8')
+    .send(body)
+}
+
+/**
+ * Root file routes: `robots.txt` and `sitemap.xml`.
+ *
+ * The two files a crawler asks for at the root by convention rather than because the wiki has a page
+ * there — which is why they are registered at the root rather than under one of the server's own
+ * prefixes, and why `RESERVED_ROOT_FILES` holds both names: without that, the SEO hook would treat
+ * `/sitemap.xml` as a page path and send a crawler off to the site's locale prefix, and `/robots.txt`
+ * would be redirected to the page `robots`, `txt` being a page extension on a default site.
+ * (`favicon.ico` is the third such name and is served by `@fastify/favicon`.)
+ *
+ * Both are public, both answer for whichever site the request's host resolves to, and both are driven
+ * by that site's own settings in the admin area's **General → SEO** card.
+ */
+async function routes(app: FastifyInstance) {
+  /**
+   * robots.txt
+   *
+   * Two of the three settings on that card are about crawlers, and robots.txt can carry exactly one
+   * of them:
+   *
+   * - **Allow Indexing** off becomes `Disallow: /`, which is as close as this file gets. Worth being
+   *   clear that it is not the same instruction: `Disallow` says do not CRAWL, `noindex` says do not
+   *   INDEX, and a page that is never crawled can still be listed from its inbound links alone. The
+   *   thorough form of the setting is the `X-Robots-Tag` on every HTML response — `robotsTagFor` in
+   *   `index.ts` — and this line is the coarse one that keeps a crawler off the wiki to begin with.
+   * - **Allow Follow** has no expression here at all: `nofollow` is a directive about a document a
+   *   crawler is holding, and robots.txt has no concept of one. That toggle is honoured by the same
+   *   header, and this route does not pretend to carry it.
+   * - **Allow Sitemap** adds the `Sitemap:` line, which is how a crawler that was given nothing but a
+   *   hostname finds the sitemap at all. Omitted when indexing is off, since pointing a crawler at an
+   *   index of pages it has just been told not to crawl says nothing coherent.
+   *
+   * Nothing per-page is ever written here — not the pages the guests group may not read, and not the
+   * ones marked out of search results. robots.txt is world-readable, so a `Disallow` naming a path is
+   * a published list of what a wiki considers worth hiding. What keeps those out of a crawler's way is
+   * that they are absent from the sitemap and refused when asked for.
+   *
+   * Served whatever the settings say — there is no toggle for having a robots.txt, because a site
+   * always has an answer to the question it asks. Only a host that resolves to no site at all 404s.
+   */
+  app.get('/robots.txt', async (req, reply) => {
+    const site = await WIKI.models.sites.getSiteByHostname({ hostname: req.hostname })
+    if (!site) {
+      return reply.notFound()
+    }
+
+    const lines = ['User-agent: *']
+    if (site.config?.robots?.index) {
+      lines.push('Allow: /')
+      if (site.config.sitemap) {
+        lines.push('', `Sitemap: ${originOf(req)}/sitemap.xml`)
+      }
+    } else {
+      lines.push('Disallow: /')
+    }
+
+    return reply
+      .header('Cache-Control', ROOT_FILE_CACHE)
+      .type('text/plain; charset=utf-8')
+      .send(`${lines.join('\n')}\n`)
+  })
+
+  /**
+   * sitemap.xml
+   *
+   * Off for a site whose **Allow Sitemap** is unticked, on for a new one.
+   */
+  app.get<{ Querystring: { p?: string } }>('/sitemap.xml', async (req, reply) => {
+    const site = await WIKI.models.sites.getSiteByHostname({ hostname: req.hostname })
+    if (!site?.config?.sitemap) {
+      // -> A site with the setting off has no sitemap rather than an empty one, and 404 is what tells
+      //    a crawler to stop asking for it
+      return reply.notFound()
+    }
+
+    const pages = await WIKI.models.pages.listForSitemap(site.id)
+    const origin = originOf(req)
+    // -> A site with nothing a guest may read still answers, with an empty list: it is the truthful
+    //    answer, and a 404 would read as the feature being broken rather than as the wiki being shut
+    const parts = pages.length > 0 ? chunk(pages, MAX_URLS_PER_FILE) : [[]]
+
+    if (req.query.p === undefined) {
+      return sendXml(
+        reply,
+        parts.length > 1
+          ? renderIndex(origin, parts)
+          : renderUrlset(origin, site.id, parts[0]!, pages)
+      )
+    }
+
+    const part = Number.parseInt(req.query.p, 10)
+    if (!Number.isInteger(part) || part < 1 || part > parts.length) {
+      return reply.notFound()
+    }
+    return sendXml(reply, renderUrlset(origin, site.id, parts[part - 1]!, pages))
+  })
+}
+
+export default routes

backend/index.ts

@@ -97,6 +97,33 @@ function isPageUrl(urlPath: string): boolean {
   return !firstSegment.startsWith('_') && !RESERVED_ROOT_FILES.has(firstSegment.toLowerCase())
 }
 
+/**
+ * What a site tells a crawler about a document it has already fetched, as an `X-Robots-Tag` value.
+ *
+ * The other half of the **General → SEO** settings, and the half that carries what robots.txt cannot:
+ * that file can only say whether to CRAWL a path (see `controllers/rootFiles.ts`), while `noindex`
+ * and `nofollow` are instructions about a document in hand. A search engine reads this header exactly
+ * as it reads a `<meta name="robots">` tag, and — unlike a tag the frontend would set once it booted —
+ * it is there for a crawler that does not run the page's JavaScript.
+ *
+ * `noindex` is also the setting's only thorough form. `Disallow: /` keeps a crawler off the page, but
+ * a page nobody fetched can still be listed from its inbound links alone; this is what says not to
+ * list it.
+ *
+ * Null when both settings are on, which is every crawler's default anyway: no header says the same
+ * thing as `index, follow`, and a wiki that wants to be found should not have to repeat it on every
+ * response.
+ */
+function robotsTagFor(siteId: string | undefined): string | null {
+  // -> A host matching no site at all is still handed the app shell, and is told not to index it:
+  //    there is no site here whose settings could say otherwise
+  const robots = siteId ? WIKI.sites[siteId]?.config?.robots : undefined
+  if (robots?.index && robots.follow) {
+    return null
+  }
+  return `${robots?.index ? 'index' : 'noindex'}, ${robots?.follow ? 'follow' : 'nofollow'}`
+}
+
 /**
  * The segments a site's locale-prefixed URLs may start with, mapped to the locale each names.
  *
@@ -764,6 +791,11 @@ async function initHTTPServer() {
   app.register(import('./controllers/thumb.ts'), { prefix: '/_thumb' })
   app.register(import('./controllers/user.ts'), { prefix: '/_user' })
 
+  // -> At the root and with no prefix of their own: `robots.txt` and `sitemap.xml` are names a crawler
+  //    asks for by convention, the same way `favicon.ico` is, and `RESERVED_ROOT_FILES` is what keeps
+  //    the SEO hook above from mistaking either for a page path
+  app.register(import('./controllers/rootFiles.ts'))
+
   // ----------------------------------------
   // App Shell
   // ----------------------------------------
@@ -796,6 +828,17 @@ async function initHTTPServer() {
     }
     try {
       const shell = await readFile(appShellPath, 'utf8')
+      /*
+        Every HTML document this wiki serves leaves through here — a page and an app route alike — so
+        this is the one place the site's indexing settings can be attached to all of them.
+
+        Straight off the site caches for the same reason the SEO hook above reads them that way: both
+        lookups are what `getSiteByHostname` would do, minus its optional reload.
+      */
+      const robotsTag = robotsTagFor(WIKI.sitesMappings[req.hostname] || WIKI.sitesMappings['*'])
+      if (robotsTag) {
+        reply.header('X-Robots-Tag', robotsTag)
+      }
       return reply.header('Cache-Control', 'no-store').type('text/html; charset=utf-8').send(shell)
     } catch (err: any) {
       // -> Nothing to serve means the frontend was never built, which is a setup step rather than a

backend/locales/en.json

@@ -506,9 +506,9 @@
   "admin.general.reasonForChangeRequired": "Required",
   "admin.general.saveSuccess": "Site configuration saved successfully.",
   "admin.general.searchAllowFollow": "Allow Search Engines to Follow Links",
-  "admin.general.searchAllowFollowHint": "This sets the meta-robots property to follow or nofollow.",
+  "admin.general.searchAllowFollowHint": "When off, pages are served with a nofollow header, asking search engines not to follow the links they contain.",
   "admin.general.searchAllowIndexing": "Allow Indexing by Search Engines",
-  "admin.general.searchAllowIndexingHint": "This sets the meta-robots property to index or noindex.",
+  "admin.general.searchAllowIndexingHint": "When off, pages are served with a noindex header and robots.txt disallows crawling, keeping the site out of search results.",
   "admin.general.senderEmailHint": "Email address of the sender.",
   "admin.general.senderNameHint": "Name of the sender.",
   "admin.general.siteBranding": "Site Branding",

backend/models/groups.ts

@@ -158,6 +158,11 @@ class Groups {
     for (const row of rows) {
       rulesCache[row.id] = (row.rules ?? []) as GroupRule[]
     }
+    // -> The sitemap is a list of what the GUESTS group may read, held for minutes at a time. Every
+    //    other consumer of these rules asks per request and is correct the moment this returns; that
+    //    one would go on publishing paths a rule had just taken away, which is a permission waiting
+    //    for a timer rather than a document being a little out of date
+    WIKI.models.pages.invalidateSitemaps()
     WIKI.logger.info(`Loaded page rules for ${rows.length} groups [ OK ]`)
   }
 

backend/models/pages.ts

@@ -1,4 +1,4 @@
-import { and, desc, eq, inArray, ne, notInArray, sql } from 'drizzle-orm'
+import { and, desc, eq, inArray, isNull, ne, notInArray, sql } from 'drizzle-orm'
 import { pages as pagesTable, tree as treeTable, users as usersTable } from '../db/schema.ts'
 import {
   CustomError,
@@ -67,6 +67,30 @@ export function pageEditorForExtension(ext: string): string | null {
  */
 const REDIRECT_EDITOR = 'redirect'
 
+/**
+ * How long a site's sitemap list is held before it is read again, in seconds.
+ *
+ * The same figure as the `Cache-Control` the file goes out with, for the same reason: a sitemap may
+ * be a few minutes out of date without anybody being misled. Note the two compound — see
+ * `listForSitemap`.
+ */
+const SITEMAP_TTL = 600
+
+/** Namespaced so that `invalidateSitemaps` can find every site's entry without knowing the sites. */
+const SITEMAP_CACHE_PREFIX = 'sitemap:'
+
+function sitemapCacheKey(siteId: string): string {
+  return `${SITEMAP_CACHE_PREFIX}${siteId}`
+}
+
+/**
+ * Sitemap lists being built right now, one at most per site.
+ *
+ * Deliberately NOT in `WIKI.cache`: a promise is this process's, and the point of it is to make
+ * concurrent callers in this process wait on one query rather than start their own.
+ */
+const sitemapBuilds = new Map<string, Promise<SitemapPage[]>>()
+
 /** A page path is what ends up in a URL, so it is held to what reads and routes cleanly. */
 const rePagePath = /^[a-zA-Z0-9-_/]*$/
 const reAlias = /^[a-zA-Z0-9-_]*$/
@@ -233,6 +257,15 @@ export interface RecentPage {
   authorName: string | null
 }
 
+/** A page as a sitemap entry sees it: where it is, when it last changed, and what it is a translation of. */
+export interface SitemapPage {
+  locale: string
+  path: string
+  updatedAt: Date
+  /** The set of pages this one is a translation of, or null when it has none — see the column. */
+  localeGroupId: string | null
+}
+
 export interface PageActor {
   id: string
   permissions: string[]
@@ -800,6 +833,118 @@ class Pages {
     }))
   }
 
+  /**
+   * Every page of a site that belongs in its sitemap, cached.
+   *
+   * The list is what a sitemap costs: one read of every published page of a site, and then the guests
+   * group's rules resolved against each of them. That is far too much to spend per request on a
+   * document whose whole audience is a handful of crawlers, so it is built on demand and held for
+   * `SITEMAP_TTL` seconds.
+   *
+   * **The LIST is what is cached, not the file.** Rendering reads three things that a cached file
+   * would freeze: the request's own origin, which differs between the hosts a catch-all site answers
+   * on; `urlFor`, which brackets a path by locale exactly as that site's settings say; and whether the
+   * sitemap is enabled at all. Keeping those per request means a change to any of them shows up at
+   * once, that paging re-slices one list instead of caching every part separately, and that a site
+   * reached on several hostnames needs one entry rather than one per host.
+   *
+   * It is not the smaller of the two, mind: measured at about **275 bytes per page** — roughly 2.7 MB
+   * for a 10,000-page site — where the rendered XML for the same pages is nearer 1.5 MB, since a long
+   * one-byte string costs less than an object with four fields and a `Date`. The reasons above are
+   * what pay for the difference.
+   *
+   * What does wait for the TTL is a page appearing, changing or going. That is the deliberate trade —
+   * the file is already served `max-age=600`, so a crawler may be holding one that old regardless, and
+   * a sitemap is a hint about where to look rather than a statement of record. The two compound: a
+   * crawler can be reading a list up to twice the TTL old. Admin → System's **Flush Cache** empties
+   * this with everything else, and does it across an HA set, for when that is not good enough.
+   *
+   * @see `models/groups.ts` → `reloadCache`, which drops this when a group's rules change — the one
+   *      input where being out of date is a permissions question rather than a freshness one.
+   */
+  async listForSitemap(siteId: string): Promise<SitemapPage[]> {
+    const cached = WIKI.cache.get<SitemapPage[]>(sitemapCacheKey(siteId))
+    if (cached) {
+      return cached
+    }
+    /*
+      A burst against a cold cache is one query and not one per request. Without this the endpoint is
+      public, unauthenticated and trivially made to stack full table scans — which is most of what the
+      cache is here to prevent, and exactly when it is not yet populated.
+    */
+    const inFlight = sitemapBuilds.get(siteId)
+    if (inFlight) {
+      return inFlight
+    }
+    const build = this.buildSitemapList(siteId).finally(() => sitemapBuilds.delete(siteId))
+    sitemapBuilds.set(siteId, build)
+    return build
+  }
+
+  /** Drop every site's cached sitemap list, for a change that could have altered any of them. */
+  invalidateSitemaps(): void {
+    WIKI.cache.del(WIKI.cache.keys().filter((key) => key.startsWith(SITEMAP_CACHE_PREFIX)))
+  }
+
+  /**
+   * The read behind `listForSitemap`, which is the only thing that should call it.
+   *
+   * A sitemap is a list handed to search engines, so the question is not what exists but what a
+   * crawler may both reach and index. Four things decide it, three of them in SQL:
+   *
+   * - **Published**, on the same reading as everywhere else — a `scheduled` page is not published yet
+   *   whatever its dates say, and a draft never was.
+   * - **Not a redirection**, which has no body to index and sends its reader elsewhere anyway.
+   * - **Not password protected**, since what a crawler would reach there is the lock screen.
+   * - **`isSearchable`**, the page property whose whole purpose is keeping a page out of search
+   *   results. A sitemap is the most direct way there is of putting one in them.
+   *
+   * The fourth is the guests group's page rules, which no query can express — they match on path,
+   * locale and tags, and are resolved a page at a time — so the candidates are filtered here. The
+   * guests group and nothing else: this document is the same for whoever asks, and what the public
+   * may read is exactly what that group's rules say. An administrator fetching it therefore gets the
+   * file a crawler would, which is also what makes it safe to cache publicly.
+   */
+  private async buildSitemapList(siteId: string): Promise<SitemapPage[]> {
+    const rows = await WIKI.db
+      .select({
+        locale: pagesTable.locale,
+        path: pagesTable.path,
+        tags: pagesTable.tags,
+        updatedAt: pagesTable.updatedAt,
+        localeGroupId: pagesTable.localeGroupId
+      })
+      .from(pagesTable)
+      .where(
+        and(
+          eq(pagesTable.siteId, siteId),
+          eq(pagesTable.publishState, 'published'),
+          eq(pagesTable.isSearchable, true),
+          ne(pagesTable.editor, REDIRECT_EDITOR),
+          isNull(pagesTable.password)
+        )
+      )
+      // -> A stable order, so that a sitemap split into numbered parts means the same thing on the
+      //    next fetch as it did on the one that handed the crawler those numbers
+      .orderBy(pagesTable.locale, pagesTable.path)
+
+    // -> No group-wide permissions rather than the guests group's own: `checkAccess` reads that list
+    //    only for `manage:system`, and a wiki that had somehow handed the public an administrator's
+    //    permission should not also publish an index of every page it has as a consequence
+    const guests = { groupIds: [WIKI.data.systemIds.guestsGroupId], permissions: [] }
+    const pages = rows
+      .filter((row) => WIKI.models.groups.checkAccess(guests, 'read:pages', row))
+      .map(({ locale, path, updatedAt, localeGroupId }) => ({
+        locale,
+        path,
+        updatedAt,
+        localeGroupId
+      }))
+
+    WIKI.cache.set(sitemapCacheKey(siteId), pages, SITEMAP_TTL)
+    return pages
+  }
+
   /**
    * Where a page lives, as a path.
    *

backend/models/sites.ts

@@ -83,6 +83,14 @@ class Sites {
     for (const site of sites) {
       WIKI.sitesMappings[site.hostname] = site.id
     }
+    /*
+      Sitemap lists are held per site for minutes at a time, and `WIKI.cache` has no expiry sweeper —
+      an entry is only dropped when its own key is next read. So a site that was deleted, or whose
+      sitemap was just switched off, would hold its last list for the life of the process: nothing
+      will ever ask for that key again. This is the one place every create, update and delete passes
+      through, which makes it the place to let them go.
+    */
+    WIKI.models.pages.invalidateSitemaps()
     WIKI.logger.info(`Loaded ${sites.length} site configurations [ OK ]`)
   }
 

frontend/index.html

@@ -8,15 +8,6 @@
     <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width">
     <title>Wiki.js</title>
     <!--preload-links-->
-    <!--
-      The app's only webfont, and self-hosted: Roboto 300-900 as woff2, covering latin, latin-ext,
-      greek, cyrillic and vietnamese. Icon data is inlined at build time by scripts/generate-icons.mjs,
-      so there is no icon webfont to load.
-
-      Declared here rather than imported from a package, which is what `@quasar/extras/roboto-font`
-      used to do from main.js. Bundled CSS is linked after this, so those rules won -- leaving the app
-      on a latin-only woff copy of the same family while these files went unfetched.
-    -->
     <link href="/_assets/fonts/roboto/roboto.css" rel="stylesheet" />
     <style type="text/css">
       @keyframes initspinner {