import { chunk } from 'es-toolkit/array' import type { FastifyInstance, FastifyReply } from 'fastify' import type { SitemapPage } from '../models/pages.ts' import { originOf } from '../helpers/common.ts' /** * The sitemap protocol's own ceiling: 50,000 URLs, and 50MB uncompressed, per file. Past it the * document served is a sitemap index naming numbered parts of this size instead. */ const MAX_URLS_PER_FILE = 50000 /** * How long a crawler — or anything caching in front of this — may hold one of these files. * * `public` whoever asked: neither document varies by requester. The sitemap is `listForSitemap`'s * answer, which is the guests group's view and nobody else's, and robots.txt is a site setting — so a * shared cache has nothing to leak from one reader to another. Ten minutes because the alternative to * a slightly stale sitemap is re-reading every page of a wiki for a file a crawler fetches a few * times a day. */ const ROOT_FILE_CACHE = 'public, max-age=600' /** Everything written into the document goes through here, including anything that came off a header. */ function xmlEscape(value: string): string { return value .replaceAll('&', '&') .replaceAll('<', '<') .replaceAll('>', '>') .replaceAll('"', '"') .replaceAll("'", ''') } /* `originOf` (helpers/common.ts) is the origin every `` — and robots.txt's `Sitemap:` line — is written against: a sitemap may only list URLs on the host it was itself fetched from, since a crawler discards the rest as a cross-submission, so the host in the request IS the answer whether the site is bound to it or is the catch-all `*`. It comes off a header and is therefore whatever the client said, which is why it is escaped before it reaches the XML. It cannot carry a line break into robots.txt: a header value holding one is rejected by the HTTP parser long before this. */ /** * Where a page is, absolute and ready to be written into an element. * * `urlFor` is what decides the path, the same as for every link the wiki makes of its own pages, so * the locale prefix is bracketed exactly where that site's settings put one. A page path is held to * `[a-zA-Z0-9-_/]` when it is saved, but a locale's short code is an administrator's to alias, so the * result is percent-encoded before it is escaped. */ function locOf(origin: string, siteId: string, page: SitemapPage): string { return xmlEscape( `${origin}${encodeURI(WIKI.models.pages.urlFor(siteId, page.locale, page.path))}` ) } /** W3C datetime, which is what `` takes. Seconds: a crawler has no use for the nanoseconds. */ function lastmodOf(date: Date): string { return date.toTemporalInstant().toString({ smallestUnit: 'second' }) } /** * The translations of each page, keyed by locale group. * * Built over the WHOLE list rather than the part being rendered, so that a page whose other languages * fell into a different numbered file still names all of them. A page with no counterparts has a null * group and is absent from here — nulls are distinct, so they would otherwise all be one group. */ function localeGroupsOf(pages: SitemapPage[]): Map { const groups = new Map() for (const page of pages) { if (!page.localeGroupId) { continue } const group = groups.get(page.localeGroupId) if (group) { group.push(page) } else { groups.set(page.localeGroupId, [page]) } } return groups } /** * One file of page URLs. * * `changefreq` and `priority` are deliberately absent: no search engine has read either for years, * and the wiki has nothing honest to put in them — every page would claim the same numbers. * * What is here instead is `xhtml:link`, one per language a page exists in, which is how a crawler is * told that two paths are the same page rather than duplicates of each other. Emitted only where a * page actually has counterparts, and including the page itself, which is what the annotation calls * for. Only counterparts that are in this document at all: a translation the guests group may not * read is not an alternate a crawler should be sent to. */ function renderUrlset( origin: string, siteId: string, pages: SitemapPage[], all: SitemapPage[] ): string { const groups = localeGroupsOf(all) const lines = [ '', '' ] for (const page of pages) { lines.push(' ') lines.push(` ${locOf(origin, siteId, page)}`) lines.push(` ${lastmodOf(page.updatedAt)}`) const alternates = page.localeGroupId ? (groups.get(page.localeGroupId) ?? []) : [] if (alternates.length > 1) { for (const alternate of alternates) { lines.push( ` ` ) } } lines.push(' ') } lines.push('') return `${lines.join('\n')}\n` } /** * The index served in place of the list once it no longer fits in one file. * * The parts are `?p=N` on this same path rather than files of their own, because a sitemap may only * list URLs at or below its own directory — a part under `/_sitemap/` could name nothing outside it — * and because the reserved root files a crawler may ask for are a fixed set that no numbered name * could join. A query string is a URL like any other to a crawler. */ function renderIndex(origin: string, parts: SitemapPage[][]): string { const lines = [ '', '' ] parts.forEach((pages, index) => { const newest = pages.reduce( (latest, page) => (!latest || page.updatedAt > latest ? page.updatedAt : latest), null ) lines.push(' ') lines.push(` ${xmlEscape(`${origin}/sitemap.xml?p=${index + 1}`)}`) if (newest) { lines.push(` ${lastmodOf(newest)}`) } lines.push(' ') }) lines.push('') return `${lines.join('\n')}\n` } function sendXml(reply: FastifyReply, body: string) { return reply .header('Cache-Control', ROOT_FILE_CACHE) .type('application/xml; charset=utf-8') .send(body) } /** * Root file routes: `robots.txt` and `sitemap.xml`. * * The two files a crawler asks for at the root by convention rather than because the wiki has a page * there — which is why they are registered at the root rather than under one of the server's own * prefixes, and why `RESERVED_ROOT_FILES` holds both names: without that, the SEO hook would treat * `/sitemap.xml` as a page path and send a crawler off to the site's locale prefix, and `/robots.txt` * would be redirected to the page `robots`, `txt` being a page extension on a default site. * (`favicon.ico` is the third such name and is served by `@fastify/favicon`.) * * Both are public, both answer for whichever site the request's host resolves to, and both are driven * by that site's own settings in the admin area's **General → SEO** card. */ async function routes(app: FastifyInstance) { /** * robots.txt * * Two of the three settings on that card are about crawlers, and robots.txt can carry exactly one * of them: * * - **Allow Indexing** off becomes `Disallow: /`, which is as close as this file gets. Worth being * clear that it is not the same instruction: `Disallow` says do not CRAWL, `noindex` says do not * INDEX, and a page that is never crawled can still be listed from its inbound links alone. The * thorough form of the setting is the `X-Robots-Tag` on every HTML response — `robotsTagFor` in * `index.ts` — and this line is the coarse one that keeps a crawler off the wiki to begin with. * - **Allow Follow** has no expression here at all: `nofollow` is a directive about a document a * crawler is holding, and robots.txt has no concept of one. That toggle is honoured by the same * header, and this route does not pretend to carry it. * - **Allow Sitemap** adds the `Sitemap:` line, which is how a crawler that was given nothing but a * hostname finds the sitemap at all. Omitted when indexing is off, since pointing a crawler at an * index of pages it has just been told not to crawl says nothing coherent. * * Nothing per-page is ever written here — not the pages the guests group may not read, and not the * ones marked out of search results. robots.txt is world-readable, so a `Disallow` naming a path is * a published list of what a wiki considers worth hiding. What keeps those out of a crawler's way is * that they are absent from the sitemap and refused when asked for. * * Served whatever the settings say — there is no toggle for having a robots.txt, because a site * always has an answer to the question it asks. Only a host that resolves to no site at all 404s. */ app.get('/robots.txt', async (req, reply) => { const site = await WIKI.models.sites.getSiteByHostname({ hostname: req.hostname }) if (!site) { return reply.notFound() } const lines = ['User-agent: *'] if (site.config?.robots?.index) { lines.push('Allow: /') if (site.config.sitemap) { lines.push('', `Sitemap: ${originOf(req)}/sitemap.xml`) } } else { lines.push('Disallow: /') } return reply .header('Cache-Control', ROOT_FILE_CACHE) .type('text/plain; charset=utf-8') .send(`${lines.join('\n')}\n`) }) /** * sitemap.xml * * Off for a site whose **Allow Sitemap** is unticked, on for a new one. */ app.get<{ Querystring: { p?: string } }>('/sitemap.xml', async (req, reply) => { const site = await WIKI.models.sites.getSiteByHostname({ hostname: req.hostname }) if (!site?.config?.sitemap) { // -> A site with the setting off has no sitemap rather than an empty one, and 404 is what tells // a crawler to stop asking for it return reply.notFound() } const pages = await WIKI.models.pages.listForSitemap(site.id) const origin = originOf(req) // -> A site with nothing a guest may read still answers, with an empty list: it is the truthful // answer, and a 404 would read as the feature being broken rather than as the wiki being shut const parts = pages.length > 0 ? chunk(pages, MAX_URLS_PER_FILE) : [[]] if (req.query.p === undefined) { return sendXml( reply, parts.length > 1 ? renderIndex(origin, parts) : renderUrlset(origin, site.id, parts[0]!, pages) ) } const part = Number.parseInt(req.query.p, 10) if (!Number.isInteger(part) || part < 1 || part > parts.length) { return reply.notFound() } return sendXml(reply, renderUrlset(origin, site.id, parts[part - 1]!, pages)) }) } export default routes