From 23dbb27212fba805b408e808ce71b3b7c16155d8 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Sat, 8 Aug 2026 23:15:24 -0400 Subject: [PATCH] refactor: improve scheduler connection usage + move webhooks to worker jobs --- backend/api/assets.ts | 9 +- backend/api/tree.ts | 14 +- backend/base.yml | 13 + backend/controllers/files.ts | 15 +- backend/core/scheduler.ts | 373 +++++++++++++++------ backend/models/assets.ts | 320 +++++++++++++++++- backend/models/hooks.ts | 29 +- backend/models/pages.ts | 48 ++- backend/models/tree.ts | 95 ++++-- backend/tasks/workers/dispatch-webhook.ts | 25 +- backend/worker.ts | 9 + frontend/src/components/FileManager.vue | 137 +++++++- frontend/src/components/PageActionsCol.vue | 2 +- frontend/src/helpers/assets.js | 19 ++ frontend/src/layouts/ProfileLayout.vue | 20 +- frontend/src/renderers/markdown.js | 4 +- frontend/src/stores/page.js | 10 +- 17 files changed, 962 insertions(+), 180 deletions(-) diff --git a/backend/api/assets.ts b/backend/api/assets.ts index 7c1874582..f9974a557 100644 --- a/backend/api/assets.ts +++ b/backend/api/assets.ts @@ -226,7 +226,10 @@ async function routes(app: FastifyInstance) { if (!asset || !mayOnAsset(req, 'read:assets', asset)) { return reply.notFound('This asset does not exist.') } - const content = await WIKI.models.assets.getContent(req.params.assetId) + // -> Through the same local disk cache `/_files/` serves from, since this is the download + // button in the file manager rather than an administrative route: anyone who may read a + // file may press it + const content = await WIKI.models.assets.readContent(asset) if (!content) { return reply.notFound('This asset has no content.') } @@ -240,7 +243,9 @@ async function routes(app: FastifyInstance) { // -> The bytes came from a user, so the browser must take the type at its word rather than // looking for something more interesting in them reply.header('X-Content-Type-Options', 'nosniff') - return reply.type(content.mimeType).send(content.data) + // -> Set by hand because the body may be a stream, which Fastify would otherwise send chunked + reply.header('Content-Length', content.size) + return reply.type(asset.mimeType).send(content.body) } ) diff --git a/backend/api/tree.ts b/backend/api/tree.ts index b1d54b858..4ebdc7fe4 100644 --- a/backend/api/tree.ts +++ b/backend/api/tree.ts @@ -1,6 +1,7 @@ import type { FastifyInstance, FastifyRequest } from 'fastify' import { TREE_ORDER_BY, type TreeItemType, type TreeOrderBy } from '../models/tree.ts' import { decodeTreePath } from '../helpers/common.ts' +import { actorFrom } from './pages.ts' interface TreeQuery { parentId?: string @@ -624,7 +625,7 @@ async function routes(app: FastifyInstance) { schema: { summary: 'Delete a folder', description: - 'Everything under the folder goes with it, assets included. Pages are not implemented yet, so their tree entries are removed but nothing else is.', + 'Everything under the folder goes with it, pages and assets included. Each deleted page is recorded in its history first, so the branch can be recovered from there.', tags: ['Tree'], params: folderIdParam, response: { @@ -635,6 +636,12 @@ async function routes(app: FastifyInstance) { } }, async (req, reply) => { + // -> As deleting a single page does: every page going with the folder is recorded against + // whoever deleted it, so there has to be somebody to record + const actor = actorFrom(req) + if (!actor) { + return reply.unauthorized('Deleting a folder requires a logged in user.') + } const existing = await WIKI.models.tree.getFolderById(req.params.folderId) if (!existing || existing.siteId !== req.params.siteId) { return reply.notFound('This folder does not exist.') @@ -643,7 +650,10 @@ async function routes(app: FastifyInstance) { return reply.forbidden('You are not allowed to delete this folder.') } const removed = await WIKI.models.tree.deleteFolder(req.params.folderId) - await WIKI.models.assets.deleteOrphaned(removed.assets) + // -> The tree entries are gone; these are the rows behind them, which is where a page and an + // asset actually live + await WIKI.models.pages.deleteOrphaned(req.params.siteId, removed.pages, actor) + await WIKI.models.assets.deleteOrphaned(req.params.siteId, removed.assets) return reply.code(204).send() } ) diff --git a/backend/base.yml b/backend/base.yml index 376e09e19..5bc5e7fd3 100644 --- a/backend/base.yml +++ b/backend/base.yml @@ -31,6 +31,11 @@ defaults: # Iconify API the wiki fetches icons from the first time they are used. Point this at a # self-hosted Iconify API to keep icon lookups inside your network. apiUrl: 'https://api.iconify.design' + files: + # How much of /cache/files the served copies of uploaded files may take up, in bytes. + # Trimmed back oldest-first once it is exceeded. Set to 0 to serve every request from the + # database instead. + cacheMaxSize: 536870912 scheduler: workers: 3 pollingCheck: 5 @@ -38,6 +43,14 @@ defaults: maxRetries: 2 retryBackoff: 60 historyExpiration: 90000 + # How long, in seconds, a job running in a worker thread may take before the scheduler stops + # waiting for it and counts it as failed. A safety net rather than a schedule: a worker thread + # that dies mid-task never answers at all, and without a ceiling the scheduler waits forever. + taskTimeout: 300 + # How long, in seconds, a job may sit marked as running before it is assumed that whatever was + # running it is gone, and it is requeued. Deliberately generous: it has to outlast the longest + # job the instance legitimately runs, since the alternative is starting a second copy of one. + staleJobTimeout: 3600 # DB defaults api: isEnabled: false diff --git a/backend/controllers/files.ts b/backend/controllers/files.ts index ff0a4faa1..809e417d4 100644 --- a/backend/controllers/files.ts +++ b/backend/controllers/files.ts @@ -24,6 +24,10 @@ const FILE_CACHE = 'private, max-age=600, must-revalidate' * Public in the sense that `_site` and `_thumb` are — no session is required — but not unguarded: * assets are addressed by the same rules as the pages they sit among, so every request is judged * against `read:assets` for the path it asked for. + * + * Every image on every page comes through here, so neither half of the lookup normally reaches the + * database: the path resolves out of memory and the bytes stream off the local disk cache. See the + * assets model for what that caches and when it lets go of it. */ async function routes(app: FastifyInstance) { app.get<{ Params: { '*': string } }>('/*', async (req, reply) => { @@ -32,7 +36,7 @@ async function routes(app: FastifyInstance) { return reply.notFound('Site not found') } - const asset = await WIKI.models.assets.getAssetByPath(site.id, req.params['*'] ?? '') + const asset = await WIKI.models.assets.resolveAssetPath(site.id, req.params['*'] ?? '') // -> Not readable is answered as not there, so the URL cannot be used to probe for files if ( !asset || @@ -58,8 +62,10 @@ async function routes(app: FastifyInstance) { return reply.code(304).send() } - const content = await WIKI.models.assets.getContent(asset.id) + const content = await WIKI.models.assets.readContent(asset) if (!content) { + // -> The path resolved to a row that is no longer there, so the resolution was a stale one + WIKI.models.assets.forgetPath(site.id, asset.folderPath, asset.fileName) return reply.notFound('File not found') } @@ -69,7 +75,10 @@ async function routes(app: FastifyInstance) { `attachment; filename="${encodeURIComponent(asset.fileName)}"` ) } - return reply.type(content.mimeType).send(content.data) + // -> Set by hand because the body may be a stream, which Fastify would otherwise send chunked — + // and a download with no length is a download with no progress bar + reply.header('Content-Length', content.size) + return reply.type(asset.mimeType).send(content.body) }) } diff --git a/backend/core/scheduler.ts b/backend/core/scheduler.ts index 74f13dc60..cc7c81b08 100644 --- a/backend/core/scheduler.ts +++ b/backend/core/scheduler.ts @@ -14,12 +14,27 @@ import { jobSchedule as jobScheduleTable, jobHistory as jobHistoryTable } from '../db/schema.ts' -import { eq, inArray, sql } from 'drizzle-orm' +import { and, eq, inArray, lt, sql } from 'drizzle-orm' import type { PoolClient } from 'pg' /** An in-process task, loaded from `tasks/simple/`. */ export type SimpleTask = (payload?: any) => Promise | void +/** Fallback for `scheduler.taskTimeout`, in seconds, when nothing is configured. */ +const DEFAULT_TASK_TIMEOUT = 300 + +/** Fallback for `scheduler.staleJobTimeout`, in seconds, when nothing is configured. */ +const DEFAULT_STALE_JOB_TIMEOUT = 3600 + +/** + * How much longer than the task timeout the scheduler waits before giving up on its own. + * + * The abort is the polite route — the pool aborts a task that is merely slow, and rejects with a + * `TimeoutError` naming what happened. This grace period lets that answer arrive first, and only + * covers the case where nothing is going to answer at all. + */ +const TASK_TIMEOUT_GRACE = 5000 + /** * Sends the scheduler's cross-instance notifications, one at a time. * @@ -106,10 +121,10 @@ export default { const decoded = JSON.parse(msg.payload!) switch (decoded?.event) { case 'newJob': { + // -> No counting here: `processJob` accounts for the jobs it actually claims, and + // counting this call as a worker as well would hide one slot for its duration if (this.activeWorkers < this.maxWorkers) { - this.activeWorkers++ await this.processJob() - this.activeWorkers-- } break } @@ -134,11 +149,19 @@ export default { // -> Start scheduled jobs check this.scheduledRef = setInterval(async () => { this.addScheduled() + this.reapStaleJobs() }, WIKI.config.scheduler.scheduledCheck * 1000) // -> Add scheduled jobs on init await this.addScheduled() + /* + Anything left claimed but unfinished, before this instance starts claiming more. Most often + that is what this very instance abandoned when it last went down — but it runs on the interval + as well, since an instance that never comes back cannot clean up after itself. + */ + await this.reapStaleJobs() + // -> Start job polling this.pollingRef = setInterval(async () => { this.processJob() @@ -197,17 +220,76 @@ export default { WIKI.logger.warn(`Failed to add job to scheduler: ${err.message}`) } }, - async processJob(): Promise { - const jobIds: string[] = [] + /** + * Run a job in a worker thread, and stop waiting for it if it does not come back. + * + * A task promise that never settles is not a hypothetical: a worker thread that dies mid-task — + * `process.exit`, an OOM kill, a native crash — takes the answer with it. Poolifier reports the + * exit through its `exitHandler` but has nothing to attach it to, so the promise this awaits stays + * pending forever, and with it everything the caller is holding: the job stays claimed, its history + * row stays `active`, and the transaction around this never commits. + * + * Two ceilings, because they cover different failures. The abort signal is for a task that is still + * running and merely slow — the pool aborts it and rejects, so the worker stops doing the work as + * well. The timer is for the case where there is no longer anybody to abort, and is what makes the + * wait finite no matter what happened to the thread. + * + * Either way the job ends up in the same place a thrown task does: recorded as failed, and retried + * with the usual backoff. + */ + async executeOnWorker(job: { task: string; payload?: any }): Promise { + const timeoutMs = (WIKI.config.scheduler.taskTimeout ?? DEFAULT_TASK_TIMEOUT) * 1000 + let timer: NodeJS.Timeout | undefined try { - const availableWorkers = this.maxWorkers - this.activeWorkers - if (availableWorkers < 1) { - WIKI.logger.debug('All workers are busy. Cannot process more jobs at the moment.') - return - } + await Promise.race([ + this.workerPool!.execute( + { ...job, INSTANCE_ID: `${WIKI.INSTANCE_ID}:WKR` }, + undefined, + AbortSignal.timeout(timeoutMs) + ), + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject( + new Error( + `The worker running this task did not answer within ${timeoutMs / 1000}s. It may have crashed.` + ) + ) + }, timeoutMs + TASK_TIMEOUT_GRACE) + }) + ]) + } finally { + clearTimeout(timer) + } + }, - await WIKI.db.transaction(async (trx: any) => { - const jobs = await trx + /** + * Take a batch of due jobs and run them. + * + * Two steps, deliberately not one transaction. Claiming a job has to be atomic — the `DELETE` with + * `SKIP LOCKED` is what stops two instances running the same job, and the history row saying it + * started belongs with it — but running it does not: a task takes as long as whatever it is waiting + * on, and a transaction held open across that pins a pooled connection, holds the locks the claim + * took, and stops postgres vacuuming anything newer than its snapshot for the duration. + * + * So the transaction covers the claim and nothing else, and the work happens after it commits, all + * of the batch at once rather than one job at a time — the worker pool is there to be used, and the + * batch was sized to it. + * + * The cost of committing the claim first is that a process which dies mid-job no longer has its + * claim rolled back: the job is gone from the queue and its history row is left saying `active`. + * That is what `reapStaleJobs` is for. + */ + async processJob(): Promise { + const availableWorkers = this.maxWorkers - this.activeWorkers + if (availableWorkers < 1) { + WIKI.logger.debug('All workers are busy. Cannot process more jobs at the moment.') + return + } + + let jobs: any[] = [] + try { + jobs = await WIKI.db.transaction(async (trx: any) => { + const claimed = await trx .delete(jobsTable) .where( inArray( @@ -216,112 +298,187 @@ export default { ) ) .returning() - if (jobs && jobs.length > 0) { - for (const job of jobs) { - WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`) - // -> Add to Job History - await WIKI.db - .insert(jobHistoryTable) - .values({ - id: job.id, - task: job.task, - state: 'active', - useWorker: job.useWorker, - wasScheduled: job.isScheduled, - payload: job.payload, - attempt: job.retries + 1, - maxRetries: job.maxRetries, - executedBy: WIKI.INSTANCE_ID, - createdAt: job.createdAt - }) - .onConflictDoUpdate({ - target: jobHistoryTable.id, - set: { executedBy: WIKI.INSTANCE_ID, startedAt: sql`now()` } - }) - jobIds.push(job.id) - - // -> Start working on it - try { - if (job.useWorker) { - await this.workerPool!.execute({ - ...job, - INSTANCE_ID: `${WIKI.INSTANCE_ID}:WKR` - }) - } else { - await this.tasks![job.task](job.payload) - } - // -> Update job history (success) - await WIKI.db - .update(jobHistoryTable) - .set({ - state: 'completed', - completedAt: sql`now()` - }) - .where(eq(jobHistoryTable.id, job.id)) - WIKI.logger.info(`Completed job ${job.id}: ${job.task}`) - notifier.send( - 'scheduler', - JSON.stringify({ - source: WIKI.INSTANCE_ID, - event: 'jobCompleted', - state: 'success', - id: job.id - }) - ) - } catch (err: any) { - WIKI.logger.warn(`Failed to complete job ${job.id}: ${job.task} [ FAILED ]`) - WIKI.logger.warn(err) - // -> Update job history (fail) - await WIKI.db - .update(jobHistoryTable) - .set({ - attempt: job.retries + 1, - state: 'failed', - lastErrorMessage: err.message - }) - .where(eq(jobHistoryTable.id, job.id)) - notifier.send( - 'scheduler', - JSON.stringify({ - source: WIKI.INSTANCE_ID, - event: 'jobCompleted', - state: 'failed', - id: job.id, - errorMessage: err.message - }) - ) - // -> Reschedule for retry - if (job.retries < job.maxRetries) { - const backoffDelay = 2 ** job.retries * WIKI.config.scheduler.retryBackoff - await trx.insert(jobsTable).values({ - ...job, - retries: job.retries + 1, - waitUntil: new Date( - Temporal.Now.instant().add({ seconds: backoffDelay }).epochMilliseconds - ), - updatedAt: new Date() - }) - WIKI.logger.warn(`Rescheduling new attempt for job ${job.id}: ${job.task}...`) - } - } - } + for (const job of claimed) { + // -> In the same transaction as the claim: a claim that rolls back must not leave a history + // row behind saying the job started + await trx + .insert(jobHistoryTable) + .values({ + id: job.id, + task: job.task, + state: 'active', + useWorker: job.useWorker, + wasScheduled: job.isScheduled, + payload: job.payload, + attempt: job.retries + 1, + maxRetries: job.maxRetries, + executedBy: WIKI.INSTANCE_ID, + createdAt: job.createdAt + }) + .onConflictDoUpdate({ + target: jobHistoryTable.id, + set: { state: 'active', executedBy: WIKI.INSTANCE_ID, startedAt: sql`now()` } + }) } + return claimed }) } catch (err: any) { + // -> Nothing was claimed: the transaction rolled back, so the jobs are still queued + WIKI.logger.warn(err) + return + } + + if (jobs.length < 1) { + return + } + + this.activeWorkers += jobs.length + try { + // -> `allSettled`, though `runJob` handles its own failures: one job that manages to throw + // anyway must not abandon the bookkeeping of the others + await Promise.allSettled(jobs.map((job) => this.runJob(job))) + } finally { + this.activeWorkers -= jobs.length + } + }, + /** + * Run one already-claimed job and record how it went. + * + * Runs outside any transaction, so every write here is on its own — which is also why a failure + * cannot undo the ones before it. A job that fails is recorded as failed and requeued with the + * scheduler's backoff, and its siblings in the batch are unaffected either way. + */ + async runJob(job: any): Promise { + WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`) + try { + if (job.useWorker) { + await this.executeOnWorker(job) + } else { + await this.tasks![job.task](job.payload) + } + await WIKI.db + .update(jobHistoryTable) + .set({ + state: 'completed', + completedAt: sql`now()` + }) + .where(eq(jobHistoryTable.id, job.id)) + WIKI.logger.info(`Completed job ${job.id}: ${job.task}`) + notifier.send( + 'scheduler', + JSON.stringify({ + source: WIKI.INSTANCE_ID, + event: 'jobCompleted', + state: 'success', + id: job.id + }) + ) + } catch (err: any) { + WIKI.logger.warn(`Failed to complete job ${job.id}: ${job.task} [ FAILED ]`) WIKI.logger.warn(err) - if (jobIds && jobIds.length > 0) { - // -> The filter must name the table being updated: `jobs.id` here produced - // `UPDATE "jobHistory" ... WHERE "jobs"."id" IN (...)`, which postgres rejects with - // "missing FROM-clause entry", and the statement was not awaited so the rejection was - // lost. Interrupted jobs were therefore never recorded as such. + try { await WIKI.db .update(jobHistoryTable) .set({ - state: 'interrupted', + attempt: job.retries + 1, + state: 'failed', lastErrorMessage: err.message }) - .where(inArray(jobHistoryTable.id, jobIds)) + .where(eq(jobHistoryTable.id, job.id)) + notifier.send( + 'scheduler', + JSON.stringify({ + source: WIKI.INSTANCE_ID, + event: 'jobCompleted', + state: 'failed', + id: job.id, + errorMessage: err.message + }) + ) + // -> Reschedule for retry + if (job.retries < job.maxRetries) { + const backoffDelay = 2 ** job.retries * WIKI.config.scheduler.retryBackoff + await WIKI.db.insert(jobsTable).values({ + ...job, + retries: job.retries + 1, + waitUntil: new Date( + Temporal.Now.instant().add({ seconds: backoffDelay }).epochMilliseconds + ), + updatedAt: new Date() + }) + WIKI.logger.warn(`Rescheduling new attempt for job ${job.id}: ${job.task}...`) + } + } catch (recordErr: any) { + // -> The task's failure is already logged; this is the database refusing to hear about it, + // which leaves the job looking active until `reapStaleJobs` picks it up + WIKI.logger.warn(`Could not record the failure of job ${job.id}: ${recordErr.message}`) + } + } + }, + /** + * Requeue jobs that were claimed and never finished. + * + * A job is claimed out of `jobs` and marked `active` in the history before it runs, so an instance + * that dies mid-job — or a worker that takes its answer with it — leaves a row saying a job started + * that nothing is going to finish. Nothing else notices those: they are no longer in the queue. + * + * Age is the only usable signal. `INSTANCE_ID` is a fresh nanoid on every boot, so an instance + * cannot pick out the rows of its own previous life, and another instance's `active` row may well + * be a job that is running perfectly happily. `staleJobTimeout` is therefore a "nobody could still + * be working on this" threshold rather than a deadline — generous on purpose, because the cost of + * setting it too low is running a job that was already running. + * + * The `UPDATE` is the claim: two instances sweeping at once both filter on `state = 'active'`, so + * whichever commits second matches nothing and returns nothing. + * + * @returns How many jobs were requeued + */ + async reapStaleJobs(): Promise { + try { + const staleAfter = WIKI.config.scheduler.staleJobTimeout ?? DEFAULT_STALE_JOB_TIMEOUT + const cutoff = new Date( + Temporal.Now.instant().subtract({ seconds: staleAfter }).epochMilliseconds + ) + const stranded = await WIKI.db + .update(jobHistoryTable) + .set({ + state: 'interrupted', + lastErrorMessage: `No instance reported on this job within ${staleAfter}s. Whatever was running it is gone.` + }) + .where(and(eq(jobHistoryTable.state, 'active'), lt(jobHistoryTable.startedAt, cutoff))) + .returning() + + let requeued = 0 + for (const job of stranded) { + // -> Its remaining attempts are what they were: being interrupted is a failed attempt, and a + // job that had already used them up is not owed another one + if (job.attempt > job.maxRetries) { + WIKI.logger.warn( + `Job ${job.id}: ${job.task} was interrupted and has no attempts left [ SKIPPED ]` + ) + continue + } + await WIKI.db.insert(jobsTable).values({ + id: job.id, + task: job.task, + useWorker: job.useWorker, + payload: job.payload, + retries: job.attempt, + maxRetries: job.maxRetries, + isScheduled: job.wasScheduled, + createdBy: WIKI.INSTANCE_ID + }) + requeued++ + } + if (stranded.length > 0) { + WIKI.logger.warn( + `Found ${stranded.length} interrupted job(s), ${requeued} of them requeued [ OK ]` + ) } + return requeued + } catch (err: any) { + WIKI.logger.warn(`Failed to requeue interrupted jobs: ${err.message}`) + return 0 } }, async addScheduled(): Promise { diff --git a/backend/models/assets.ts b/backend/models/assets.ts index 1b7a5ad7c..be1bae9fa 100644 --- a/backend/models/assets.ts +++ b/backend/models/assets.ts @@ -1,13 +1,38 @@ +import fs from 'node:fs/promises' import path from 'node:path' import mime from 'mime' import { and, desc, eq, inArray, sql } from 'drizzle-orm' import { assets as assetsTable, tree as treeTable } from '../db/schema.ts' import { CustomError, decodeTreePath, encodeTreePath } from '../helpers/common.ts' import { makeImageThumbnail } from '../helpers/images.ts' +import type { Readable } from 'node:stream' +import type { DeletedEntry } from './tree.ts' /** How large the file manager renders a preview. Generated once, at upload time. */ const THUMBNAIL_SIZE = { width: 320, height: 200 } +/** + * How long a path resolution is trusted before it is looked up again. + * + * The backstop rather than the mechanism: the mutations that move an asset drop the entries they + * affect, but only on the instance that ran them, and a second instance has no way to hear about it — + * so every entry expires on its own as well. Short enough that a rename made elsewhere shows up + * quickly, long enough that a busy page's images resolve once rather than once per request. + */ +const PATH_CACHE_TTL_MS = 60_000 + +/** How many path resolutions to hold per instance. Each is one row of metadata, so this is small. */ +const PATH_CACHE_MAX = 5000 + +/** Ceiling for the disk cache when nothing is configured. */ +const DEFAULT_CACHE_MAX_SIZE = 512 * 1024 * 1024 + +/** Sweep once this much of the ceiling has been written since the last one. */ +const SWEEP_TRIGGER_RATIO = 0.25 + +/** How far under the ceiling a sweep trims, so that the next write does not trigger another. */ +const SWEEP_TARGET_RATIO = 0.8 + /** * Extensions a browser may render inline. Everything else is sent as a download. * @@ -81,6 +106,16 @@ export function sanitizeFileName(input: string): string { return cleaned.slice(0, 255) } +/** + * The form a file path is cached under. + * + * Matches what the lookup does with it — empty segments dropped, lowercased — so that the spellings + * of a path that reach the same asset share one cache entry instead of each getting their own. + */ +function normalizePath(filePath: string): string { + return filePath.split('/').filter(Boolean).join('/').toLowerCase() +} + /** * The extension, lowercase and without its dot. Empty when the name has none. */ @@ -109,9 +144,27 @@ function kindOf(mimeType: string, fileExt: string): AssetKind { * in the site live in the matching `tree` row, which shares its ID. Both are written together — an * asset with no tree row would be unreachable, and a tree row with no asset would be a broken link. * - * Storage targets are not implemented yet, so the database is the only copy. + * Storage targets are not implemented yet, so the database is the only copy — but not the one that + * answers a request for a file. Serving goes through two caches, because `/_files/` is hit by every + * image on every page view and neither half of that lookup needs the database twice: + * + * 1. **memory**, holding path → metadata for `PATH_CACHE_TTL_MS`, which is what decides the ETag and + * answers the conditional requests a browser sends once its own copy goes stale + * 2. **disk**, under `/cache/files`, holding the bytes, streamed straight to the response + * + * Only the database is permanent; both caches are derived and can be deleted at any point, which is + * also what makes a cold instance correct rather than empty-handed. */ class Assets { + /** Path resolutions, keyed `siteId:path`. Insertion-ordered, so the oldest entry is evictable. */ + pathCache = new Map() + + /** Bytes written to the disk cache since the last sweep, for `SWEEP_TRIGGER_RATIO`. */ + writtenSinceSweep = 0 + + /** Whether a sweep is running, so that a burst of writes queues no more than one. */ + sweeping = false + /** * Store an uploaded file. * @@ -342,6 +395,242 @@ class Assets { return results[0]?.preview ?? null } + // == SERVING CACHE ================== + + /** + * An asset addressed by path, answered from memory where it can be. + * + * What `/_files/` resolves every request through: the metadata decides whether the caller may read + * the file and what its ETag is, both of which are needed before any bytes are worth fetching. + */ + async resolveAssetPath(siteId: string, filePath: string): Promise { + const key = `${siteId}:${normalizePath(filePath)}` + const cached = this.pathCache.get(key) + if (cached && Date.now() - cached.cachedAt < PATH_CACHE_TTL_MS) { + return cached.asset + } + + const asset = await this.getAssetByPath(siteId, filePath) + if (!asset) { + // -> A path with nothing at it is not remembered as empty: a file uploaded there has no way to + // find the entry and clear it, and it would answer 404 for as long as the entry lived + this.pathCache.delete(key) + return null + } + if (this.pathCache.size >= PATH_CACHE_MAX) { + const oldest = this.pathCache.keys().next().value + if (oldest) { + this.pathCache.delete(oldest) + } + } + this.pathCache.set(key, { asset, cachedAt: Date.now() }) + return asset + } + + /** + * Forget what sits at a path, for a change that moved one asset + */ + forgetPath(siteId: string, folderPath: string, fileName: string): void { + this.pathCache.delete( + `${siteId}:${normalizePath(folderPath ? `${folderPath}/${fileName}` : fileName)}` + ) + } + + /** + * Forget every path resolution, for a change that moved assets in bulk — a folder renamed or + * deleted, where the paths that changed are no longer enumerable from what is left in the tree. + */ + forgetAllPaths(): void { + this.pathCache.clear() + } + + /** + * An asset's bytes, ready to be sent — from the disk cache, or from the database and into it. + * + * @returns A stream when the cache holds the file, the buffer when it had to be read, and null when + * there is no such asset, i.e. when a cached path resolution has outlived the row behind it + */ + async readContent(asset: { + id: string + updatedAt: Date + }): Promise<{ body: Readable | Buffer; size: number } | null> { + const cached = await this.readContentCache(asset) + if (cached) { + return cached + } + + const content = await this.getContent(asset.id) + if (!content) { + return null + } + await this.writeContentCache(asset, content.data) + return { body: content.data, size: content.data.length } + } + + /** + * Where an asset's bytes sit in the disk cache. + * + * Named for the ID and the modification time together, which is what makes an entry immutable: + * anything that changes a file changes the name it would be cached under, so a stale entry is never + * read, only left behind for the sweep. Sharded by the first byte of the ID, to keep a wiki's worth + * of files out of a single directory. + */ + contentCachePath(asset: { id: string; updatedAt: Date }): string { + return path.join( + this.cachePath, + asset.id.slice(0, 2), + `${asset.id}-${asset.updatedAt.getTime()}.bin` + ) + } + + /** + * Open an asset's cached bytes. + * + * The file is opened before it is streamed rather than as it is streamed, so that a sweep removing + * it midway through a response cannot truncate what is being sent: the handle keeps the bytes + * readable until the stream closes it, whatever happens to the directory entry. + * + * @returns Null when this instance has not cached the file, which is the normal state of a fresh + * container and the state of every entry after a change to the file + */ + async readContentCache(asset: { + id: string + updatedAt: Date + }): Promise<{ body: Readable; size: number } | null> { + let handle + try { + handle = await fs.open(this.contentCachePath(asset), 'r') + } catch { + return null + } + try { + const { size } = await handle.stat() + return { body: handle.createReadStream({ autoClose: true }), size } + } catch { + await handle.close().catch(() => {}) + return null + } + } + + /** + * Write an asset's bytes to the disk cache, best effort. + * + * A full or read-only disk must not stop a file from being served, hence the swallowed error — the + * database answers every request the cache cannot. The file is written under a temporary name and + * renamed, so a concurrent reader sees either nothing or the whole thing. + */ + async writeContentCache(asset: { id: string; updatedAt: Date }, data: Buffer): Promise { + // -> A file larger than the whole cache would be evicted by the sweep it triggers + if (this.cacheMaxSize < 1 || data.length > this.cacheMaxSize) { + return + } + const filePath = this.contentCachePath(asset) + const tempPath = `${filePath}.${process.pid}.tmp` + try { + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile(tempPath, data) + await fs.rename(tempPath, filePath) + } catch (err: any) { + WIKI.logger.warn(`Could not write ${filePath} to the file cache [ SKIPPED ]`) + WIKI.logger.warn(err.message) + await fs.rm(tempPath, { force: true }).catch(() => {}) + return + } + + this.writtenSinceSweep += data.length + if (this.writtenSinceSweep >= this.cacheMaxSize * SWEEP_TRIGGER_RATIO) { + // -> Nothing waits on this: the request that filled the cache is not the one that should pay + // for measuring it + void this.sweepCache() + } + } + + /** + * Drop whatever the disk cache holds for these assets. + * + * Every entry an asset has, not just its current one — a file renamed twice leaves two behind, and + * the point of this is to reclaim the space rather than to correct an answer, which the naming + * already does. + */ + async dropCachedContent(ids: string[]): Promise { + for (const id of ids) { + const shard = path.join(this.cachePath, id.slice(0, 2)) + try { + const entries = await fs.readdir(shard) + await Promise.all( + entries + .filter((name) => name.startsWith(`${id}-`)) + .map((name) => fs.rm(path.join(shard, name), { force: true })) + ) + } catch { + // -> Nothing cached for it on this instance, which is not worth reporting + } + } + } + + /** + * Trim the disk cache back under its ceiling, oldest entry first. + * + * Oldest by when it was written rather than when it was last read: keeping a true LRU would mean + * touching a file on every hit, which puts a write back on the path this cache exists to keep + * writes off. An entry evicted while still in demand is refilled by the next request for it. + */ + async sweepCache(): Promise { + if (this.sweeping) { + return + } + this.sweeping = true + this.writtenSinceSweep = 0 + try { + const files: { path: string; size: number; writtenAt: number }[] = [] + let total = 0 + const entries = await fs.readdir(this.cachePath, { recursive: true, withFileTypes: true }) + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith('.bin')) { + continue + } + const filePath = path.join(entry.parentPath, entry.name) + const stat = await fs.stat(filePath).catch(() => null) + if (!stat) { + continue + } + files.push({ path: filePath, size: stat.size, writtenAt: stat.mtimeMs }) + total += stat.size + } + if (total <= this.cacheMaxSize) { + return + } + + files.sort((a, b) => a.writtenAt - b.writtenAt) + const target = this.cacheMaxSize * SWEEP_TARGET_RATIO + let removed = 0 + for (const file of files) { + if (total <= target) { + break + } + await fs.rm(file.path, { force: true }) + total -= file.size + removed++ + } + WIKI.logger.debug(`Trimmed ${removed} file(s) from the file cache [ OK ]`) + } catch (err: any) { + WIKI.logger.warn('Could not sweep the file cache [ SKIPPED ]') + WIKI.logger.warn(err.message) + } finally { + this.sweeping = false + } + } + + /** Where the disk cache lives. Derived data — deleting it costs a refill and nothing else. */ + get cachePath(): string { + return path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'cache/files') + } + + /** How large the disk cache may grow, in bytes. Zero turns it off. */ + get cacheMaxSize(): number { + return WIKI.config.files?.cacheMaxSize ?? DEFAULT_CACHE_MAX_SIZE + } + /** * Rename an asset, in both of the rows that describe it. * @@ -379,6 +668,12 @@ class Assets { .set({ meta: { fileSize: asset.fileSize, fileExt, mimeType: resolvedMime } }) .where(eq(treeTable.id, id)) + // -> Both ends of the move: the name it left, and the name it took, which something else may have + // been resolved at before it was freed up + this.forgetPath(siteId, asset.folderPath, asset.fileName) + this.forgetPath(siteId, asset.folderPath, safeName) + await this.dropCachedContent([id]) + WIKI.models.hooks.emit('asset:rename', { id, fileName: safeName, @@ -403,6 +698,9 @@ class Assets { await WIKI.db.delete(assetsTable).where(eq(assetsTable.id, id)) await WIKI.models.tree.deleteEntry(id) + this.forgetPath(siteId, asset.folderPath, asset.fileName) + await this.dropCachedContent([id]) + WIKI.models.hooks.emit('asset:delete', { id, fileName: asset.fileName, @@ -416,11 +714,27 @@ class Assets { /** * Delete the assets left behind by a folder deletion, which removed their tree entries already. */ - async deleteOrphaned(ids: string[]): Promise { - if (ids.length < 1) { + async deleteOrphaned(siteId: string, entries: DeletedEntry[]): Promise { + if (entries.length < 1) { return } + const ids = entries.map((entry) => entry.id) await WIKI.db.delete(assetsTable).where(inArray(assetsTable.id, ids)) + + // -> Which paths they sat at is no longer knowable from the tree: those rows went with the folder + this.forgetAllPaths() + await this.dropCachedContent(ids) + + // -> One per file, as deleting them one at a time would have sent: a subscriber mirroring the + // wiki has to hear about each file, not about the folder it happened to sit in + for (const entry of entries) { + await WIKI.models.hooks.emit('asset:delete', { + id: entry.id, + fileName: entry.fileName, + folderPath: entry.folderPath, + siteId + }) + } } } diff --git a/backend/models/hooks.ts b/backend/models/hooks.ts index e04359c7b..aff53bcbe 100644 --- a/backend/models/hooks.ts +++ b/backend/models/hooks.ts @@ -6,8 +6,8 @@ import { desc, eq, sql } from 'drizzle-orm' /** * The events a webhook can subscribe to, as offered by the admin area. * - * Not all of them have emit points today — pages and comments are not implemented yet, so subscribing - * to those stores a subscription that nothing triggers. + * Not all of them have emit points today — comments are not implemented yet, so subscribing to those + * stores a subscription that nothing triggers. */ export const HOOK_EVENTS = [ 'page:create', @@ -31,10 +31,14 @@ export type HookEvent = (typeof HOOK_EVENTS)[number] /** * The events something in the server actually emits today. * - * Kept as an explicit list rather than inferred from the prefix, since the page and comment events - * have no emit point yet. Add an event here when you add its `emit()` call. + * Kept as an explicit list rather than inferred from the prefix, since the comment events have no + * emit point yet. Add an event here when you add its `emit()` call. */ export const EMITTED_EVENTS: HookEvent[] = [ + 'page:create', + 'page:edit', + 'page:rename', + 'page:delete', 'asset:upload', 'asset:rename', 'asset:delete', @@ -245,7 +249,10 @@ class Hooks { } const added = await WIKI.scheduler.addJob({ task: 'dispatchWebhook', - payload: { hookId: hook.id, event, data: payload } + // -> The instance travels with the job because the delivery does not happen here: it runs + // in a worker thread, whose `INSTANCE_ID` names the thread rather than the wiki, and + // what a subscriber wants to know is which instance the event came from + payload: { hookId: hook.id, event, data: payload, instance: WIKI.INSTANCE_ID } }) if (added?.id) { queued++ @@ -261,16 +268,22 @@ class Hooks { /** * Deliver one event to one webhook, recording the outcome on the webhook. * - * Called by the `dispatchWebhook` task. Throws on failure so that the scheduler retries it. + * Called by the `dispatchWebhook` task, which runs in a worker thread — so everything it needs + * comes from the job or the database, and `instance` in particular is the one that queued the + * delivery rather than whatever thread is making it. + * + * Throws on failure so that the scheduler retries it. */ async deliver({ hookId, event, - data + data, + instance }: { hookId: string event: string data: Record + instance: string }): Promise { const hook = await this.getHookById(hookId) if (!hook) { @@ -282,7 +295,7 @@ class Hooks { const body = JSON.stringify({ event, sentAt: Temporal.Now.instant().toString({ smallestUnit: 'millisecond' }), - instance: WIKI.INSTANCE_ID, + instance, data }) diff --git a/backend/models/pages.ts b/backend/models/pages.ts index 64a2af28c..a48b36ec7 100644 --- a/backend/models/pages.ts +++ b/backend/models/pages.ts @@ -1,7 +1,8 @@ -import { and, eq, ne, sql } from 'drizzle-orm' +import { and, eq, inArray, ne, sql } from 'drizzle-orm' import { pages as pagesTable, tree as treeTable, users as usersTable } from '../db/schema.ts' import { CustomError, generatePathHash, timingSafeCompare } from '../helpers/common.ts' import type { RenderPermissions, TocNode } from './rendering.ts' +import type { DeletedEntry } from './tree.ts' /** What each editor produces, which is what the content column holds. */ const EDITOR_CONTENT_TYPES: Record = { @@ -742,6 +743,51 @@ class Pages { return true } + /** + * Delete the pages left behind by a folder deletion, which removed their tree entries already. + * + * Not optional tidying: a page is served from its own row, found by the hash of its path, and the + * tree is only consulted for where it sits in the site. A page whose tree entry went with the + * folder is therefore still live at its URL while being invisible to everything that lists the + * wiki -- including the file manager somebody would have to use to delete it. + * + * Each one is recorded as deleted first, exactly as deleting a single page does. `pageHistory` + * carries no foreign key back to `pages` precisely so that it outlives the row, which is what makes + * a folder deleted by mistake recoverable. + */ + async deleteOrphaned(siteId: string, entries: DeletedEntry[], actor: PageActor): Promise { + if (entries.length < 1) { + return + } + for (const entry of entries) { + await WIKI.models.pageHistory.record({ + siteId, + pageId: entry.id, + action: 'deleted', + authorId: actor.id + }) + } + await WIKI.db.delete(pagesTable).where( + inArray( + pagesTable.id, + entries.map((entry) => entry.id) + ) + ) + + // -> One per page, as deleting them one at a time would have sent: a subscriber mirroring the + // wiki has to hear about each page, not about the folder it happened to sit in + for (const entry of entries) { + await WIKI.models.hooks.emit('page:delete', { + id: entry.id, + path: entry.folderPath ? `${entry.folderPath}/${entry.fileName}` : entry.fileName, + locale: entry.locale, + siteId, + authorId: actor.id + }) + } + WIKI.logger.debug(`Deleted ${entries.length} page(s) that went with a deleted folder.`) + } + /** * Ask for a page to be rendered again from its source, without going through an editor. * diff --git a/backend/models/tree.ts b/backend/models/tree.ts index c52a255be..baf8db1c9 100644 --- a/backend/models/tree.ts +++ b/backend/models/tree.ts @@ -1,7 +1,13 @@ import { and, asc, desc, eq, exists, inArray, ne, or, sql, type SQL } from 'drizzle-orm' import { alias, type PgColumn } from 'drizzle-orm/pg-core' import { pages as pagesTable, tree as treeTable } from '../db/schema.ts' -import { CustomError, decodeTreePath, encodeTreePath, generateHash } from '../helpers/common.ts' +import { + CustomError, + decodeTreePath, + encodeTreePath, + generateHash, + generatePathHash +} from '../helpers/common.ts' /** What a tree entry can be. Mirrors the `treeType` enum in the schema. */ export type TreeItemType = 'folder' | 'page' | 'asset' @@ -83,6 +89,20 @@ export interface ListedPage { icon: string } +/** + * An entry that went with a deleted folder, and where it used to sit. + * + * What the caller needs to finish the job: the row behind it to delete, and enough to say what was + * deleted once nothing in the database records that any more. + */ +export interface DeletedEntry { + id: string + /** Slash-separated, without the file name. Empty at the site root. */ + folderPath: string + fileName: string + locale: string +} + /** A raw `tree` row, as the model passes it around internally. */ export interface TreeRow { id: string @@ -695,12 +715,11 @@ class Tree { eq(treeTable.locale, effectiveLocale), eq(treeTable.type, 'folder'), or( - ...expected.map( - (ancestor) => - and( - eq(treeTable.folderPath, ancestor.folderPath), - eq(treeTable.fileName, ancestor.fileName) - )! + ...expected.map((ancestor) => + and( + eq(treeTable.folderPath, ancestor.folderPath), + eq(treeTable.fileName, ancestor.fileName) + )! ) ) ) @@ -844,29 +863,44 @@ class Tree { .where(eq(treeTable.id, folder.id)) .returning() - await this.refreshHashes(folder.siteId, newPath) + await this.refreshDescendantPaths(folder.siteId, newPath) + + // -> Every asset under it is served from a different path now, and nothing about the assets + // themselves changed for the file cache to notice + WIKI.models.assets.forgetAllPaths() WIKI.logger.debug(`Renamed folder ${folder.id} successfully.`) return updated[0] as TreeRow } /** - * Recompute the path hash of everything at or below a folder. + * Rewrite where everything at or below a folder now sits. + * + * Two rows carry a path and both have to be redone. The tree's own `hash` is how an entry is found + * by its path, so leaving it would make every page and asset under the folder unreachable by URL. + * A page then keeps a second copy of its path on `pages` -- the `path` itself and the `hash` a + * reader's request is actually resolved through -- so leaving that would move the page in the tree + * while still serving it from where it used to be, and nothing at all from where it now is. * - * The hash is how an entry is found by its path, so moving a branch without redoing them would - * leave every page and asset under it unreachable by URL. It is a SHA-1 of the full path, which - * postgres has no function for, so each row is rewritten from here. + * An asset has no path of its own: its tree row is the only thing that places it, and moving that + * row is the whole job. + * + * The two hashes are not the same function and neither exists in postgres, so each row is rewritten + * from here. What is deliberately not touched is `updatedAt`: the folder moved, the pages under it + * did not change, and marking a few hundred of them as freshly edited would say otherwise. */ - private async refreshHashes(siteId: string, path: string): Promise { + private async refreshDescendantPaths(siteId: string, path: string): Promise { const rows = await WIKI.db .select({ id: treeTable.id, + type: treeTable.type, folderPath: treeTable.folderPath, fileName: treeTable.fileName }) .from(treeTable) .where(and(eq(treeTable.siteId, siteId), sql`${treeTable.folderPath} <@ ${path}::ltree`)) + let pageCount = 0 for (const row of rows) { const folderPath = decodeTreePath(row.folderPath ?? '') const fullPath = folderPath ? `${folderPath}/${row.fileName}` : row.fileName @@ -874,18 +908,29 @@ class Tree { .update(treeTable) .set({ hash: generateHash(fullPath) }) .where(eq(treeTable.id, row.id)) + if (row.type === 'page') { + await WIKI.db + .update(pagesTable) + .set({ path: fullPath, hash: generatePathHash(fullPath) }) + .where(eq(pagesTable.id, row.id)) + pageCount++ + } } if (rows.length > 0) { - WIKI.logger.debug(`Refreshed the path hash of ${rows.length} moved entrie(s).`) + WIKI.logger.debug( + `Refreshed the path of ${rows.length} moved entrie(s), ${pageCount} of them page(s).` + ) } } /** * Delete a folder and everything under it. * - * @returns The IDs of the deleted pages and assets, for the caller to clean up after + * @returns The deleted pages and assets, for the caller to clean up after. Where each one sat comes + * back with it: the tree row is the only record of that, and it is gone by then — but what + * was deleted is exactly what a webhook subscriber is owed. */ - async deleteFolder(folderId: string): Promise<{ pages: string[]; assets: string[] }> { + async deleteFolder(folderId: string): Promise<{ pages: DeletedEntry[]; assets: DeletedEntry[] }> { const folder = await this.getFolderById(folderId) if (!folder) { throw new CustomError('treeInvalidFolder', 'This folder does not exist.', 404) @@ -900,7 +945,13 @@ class Tree { .where( and(eq(treeTable.siteId, folder.siteId), sql`${treeTable.folderPath} <@ ${path}::ltree`) ) - .returning({ id: treeTable.id, type: treeTable.type }) + .returning({ + id: treeTable.id, + type: treeTable.type, + folderPath: treeTable.folderPath, + fileName: treeTable.fileName, + locale: treeTable.locale + }) await WIKI.db.delete(treeTable).where(eq(treeTable.id, folder.id)) @@ -911,9 +962,15 @@ class Tree { WIKI.logger.debug(`Deleted folder ${folder.id} and ${deleted.length} descendant(s).`) + const asEntry = (row: (typeof deleted)[number]): DeletedEntry => ({ + id: row.id, + folderPath: decodeTreePath(row.folderPath ?? '') ?? '', + fileName: row.fileName, + locale: row.locale + }) return { - pages: deleted.filter((n) => n.type === 'page').map((n) => n.id), - assets: deleted.filter((n) => n.type === 'asset').map((n) => n.id) + pages: deleted.filter((n) => n.type === 'page').map(asEntry), + assets: deleted.filter((n) => n.type === 'asset').map(asEntry) } } diff --git a/backend/tasks/workers/dispatch-webhook.ts b/backend/tasks/workers/dispatch-webhook.ts index eefd2ef16..d853df7cd 100644 --- a/backend/tasks/workers/dispatch-webhook.ts +++ b/backend/tasks/workers/dispatch-webhook.ts @@ -1,13 +1,28 @@ +import { hooks } from '../../models/hooks.ts' + /** * Deliver one event to one webhook. * * Queued by `models/hooks.ts` → `emit()`, one job per subscribed webhook, so that a slow endpoint * delays nothing else and a failing one is retried with the scheduler's backoff. + * + * Runs in a worker thread rather than in-process, because what it does is wait on somebody else's + * server: an endpoint is allowed to take up to `DELIVERY_TIMEOUT` to answer, and a wiki with a few + * webhooks on a busy event queues those deliveries in bursts. On the main thread that is time the + * event loop spends on other people's HTTP instead of on serving pages. + * + * A worker task is handed the whole job rather than its payload — see `worker.ts` — and starts with + * nothing but config and a logger, so the database connection is opened on demand and the one model + * this needs is imported here rather than taken off `WIKI.models`, which a worker does not carry. */ -export async function task(payload: { - hookId: string - event: string - data: Record +export async function task(job: { + payload: { + hookId: string + event: string + data: Record + instance: string + } }): Promise { - await WIKI.models.hooks.deliver(payload) + await WIKI.ensureDb!() + await hooks.deliver(job.payload) } diff --git a/backend/worker.ts b/backend/worker.ts index 9f3457947..b63a2f474 100644 --- a/backend/worker.ts +++ b/backend/worker.ts @@ -21,6 +21,15 @@ const WIKI = { } WIKI.db = await dbManager.init(true) + /* + Only the settings model, which is what `loadFromDb` reads through — not the whole registry. + A worker thread pays the import cost of everything it pulls in, and importing all of them + brings cheerio, sanitize-html, bcrypt and the rest into a thread that wanted one `select`. + A task that needs another model imports that model itself. + */ + WIKI.models = { + settings: (await import('./models/settings.ts')).settings + } as WikiGlobal['models'] try { await WIKI.configSvc.loadFromDb() diff --git a/frontend/src/components/FileManager.vue b/frontend/src/components/FileManager.vue index 35cf7b2df..ec1235e57 100644 --- a/frontend/src/components/FileManager.vue +++ b/frontend/src/components/FileManager.vue @@ -355,24 +355,37 @@ {{ t(`common.actions.download`) }} - + Duplicate... - + + - + - Rename... - - - - - - Move to... + Rename / Move Page... + @@ -429,6 +442,7 @@ import Fuse from 'fuse.js/basic' import NewMenu from './PageNewMenu.vue' import Tree from './TreeNav.vue' import { apiErrorMessage } from '@/helpers/apiError' +import { assetUrl } from '@/helpers/assets' import fileTypes from '@/helpers/fileTypes' import FolderCreateDialog from '@/components/FolderCreateDialog.vue' import FolderDeleteDialog from '@/components/FolderDeleteDialog.vue' @@ -936,6 +950,87 @@ function rerenderPage(item) { }) } +/** + * Copy a page, through the same dialog the page view's action rail opens. + * + * The copy is not written here: what comes back is where it should go, and the store opens the editor + * on an unsaved page holding the source's content -- so the author lands in the same place they would + * have from the page itself, and nothing exists until they save it. + */ +function duplicatePage(item) { + dialog({ + component: defineAsyncComponent(() => import('@/components/TreeBrowserDialog.vue')), + componentProps: { + mode: 'duplicatePage', + itemId: item.id, + itemTitle: item.title, + folderPath: item.folderPath, + itemFileName: item.fileName + } + }).onOk(async (opts) => { + try { + await pageStore.pageDuplicate({ + sourcePageId: item.id, + path: opts.path, + title: opts.title + }) + // -> The editor is now underneath this overlay, as it is after opening a page to edit + close() + } catch (err) { + notify({ + type: 'negative', + message: 'Failed to duplicate page.', + caption: apiErrorMessage(err, 'An unexpected error occured.') + }) + } + }) +} + +/** + * Rename a page, move it, or both. + * + * One action rather than two, through the same dialog the page view's action rail opens: what it + * hands back is a title and the full path the page should sit at, and only the path decides which of + * the two endpoints that is -- a page whose title changed in place was never moved. + */ +function renameMovePage(item) { + const currentPath = item.folderPath ? `${item.folderPath}/${item.fileName}` : item.fileName + dialog({ + component: defineAsyncComponent(() => import('@/components/TreeBrowserDialog.vue')), + componentProps: { + mode: 'renamePage', + itemId: item.id, + itemTitle: item.title, + folderPath: item.folderPath, + itemFileName: item.fileName + } + }).onOk(async (opts) => { + try { + if (opts.path === currentPath) { + await pageStore.pageRename({ id: item.id, title: opts.title }) + notify({ + type: 'positive', + message: 'Page renamed successfully.' + }) + } else { + await pageStore.pageMove({ id: item.id, path: opts.path, title: opts.title }) + notify({ + type: 'positive', + message: 'Page moved successfully.' + }) + } + // -> Reload current view + await loadTree({ parentId: state.currentFolderId }) + } catch (err) { + notify({ + type: 'negative', + message: 'Failed to rename or move page.', + caption: apiErrorMessage(err, 'An unexpected error occured.') + }) + } + }) +} + function delPage(pageId, pageName) { dialog({ component: defineAsyncComponent(() => import('@/components/PageDeleteDialog.vue')), @@ -1106,8 +1201,11 @@ async function copyItemURL(item) { break } case 'asset': { - const assetPath = item.folderPath ? `${item.folderPath}/${item.fileName}` : item.fileName - await navigator.clipboard.writeText(`${window.location.origin}/${assetPath}`) + // -> Under `/_files/`, which is where a file is served from: the page tree it is listed + // alongside in here is not a place a browser can fetch it from + await navigator.clipboard.writeText( + `${window.location.origin}${assetUrl(item.folderPath, item.fileName)}` + ) break } default: { @@ -1161,7 +1259,7 @@ function renameItem(item) { break } case 'page': { - // TODO: Rename page + renameMovePage(item) break } case 'asset': { @@ -1171,6 +1269,19 @@ function renameItem(item) { } } +/** + * Duplicating a folder or an asset has no endpoint behind it yet, so those two keep the inert entry + * they already had rather than being offered something that would fail. + */ +function duplicateItem(item) { + switch (item.type) { + case 'page': { + duplicatePage(item) + break + } + } +} + function delItem(item) { switch (item.type) { case 'asset': { diff --git a/frontend/src/components/PageActionsCol.vue b/frontend/src/components/PageActionsCol.vue index b3fcf1b20..8838dec06 100644 --- a/frontend/src/components/PageActionsCol.vue +++ b/frontend/src/components/PageActionsCol.vue @@ -292,7 +292,7 @@ function duplicatePage() { } }).onOk((newPageOpts) => { pageStore.pageDuplicate({ - sourecePageId: pageStore.id, + sourcePageId: pageStore.id, path: newPageOpts.path, title: newPageOpts.title }) diff --git a/frontend/src/helpers/assets.js b/frontend/src/helpers/assets.js index 547bcab64..1f3074c27 100644 --- a/frontend/src/helpers/assets.js +++ b/frontend/src/helpers/assets.js @@ -20,3 +20,22 @@ export function assetPath(folderPath, fileName) { return folderPath ? `/${folderPath}/${fileName}` : `/${fileName}` } + +/** Where uploaded files are served from — `backend/controllers/files.ts`. */ +export const FILES_PREFIX = '/_files/' + +/** + * Where an uploaded file actually loads from. + * + * The other half of the pair: `assetPath` is what a page's source stores and this is what it resolves + * to, so anything handing a file straight to a browser -- a link to copy, an `` built outside the + * renderer -- uses this one. Writing it into a page instead would nail the content to the shape this + * server happens to serve files under. + * + * @param {string} folderPath Folder the asset sits in, slash-separated, empty at the site root. + * @param {string} fileName The asset's stored file name. + * @returns {string} A root-relative URL, e.g. `/_files/media/photo.png`. + */ +export function assetUrl(folderPath, fileName) { + return `${FILES_PREFIX}${folderPath ? `${folderPath}/${fileName}` : fileName}` +} diff --git a/frontend/src/layouts/ProfileLayout.vue b/frontend/src/layouts/ProfileLayout.vue index e9aefeb6b..1131d5846 100644 --- a/frontend/src/layouts/ProfileLayout.vue +++ b/frontend/src/layouts/ProfileLayout.vue @@ -22,15 +22,17 @@ - - - - - - - {{ t('profile.viewPublicProfile') }} - - + diff --git a/frontend/src/renderers/markdown.js b/frontend/src/renderers/markdown.js index 4ecdb2c5b..7e8f48431 100644 --- a/frontend/src/renderers/markdown.js +++ b/frontend/src/renderers/markdown.js @@ -23,6 +23,7 @@ import { escape } from 'es-toolkit/string' // -> Relative, like this file's other in-repo imports: it is also the entry point of the headless // renderer bundle, which is built on its own import { isServerPath } from '../helpers/serverPaths' +import { FILES_PREFIX } from '../helpers/assets' const quoteStyles = { chinese: '””‘’', @@ -65,9 +66,6 @@ function isExternalHref(href) { } } -/** Where uploaded files are served from — `backend/controllers/files.ts`. */ -const FILES_PREFIX = '/_files/' - /** * Where an image in a page should actually load from. * diff --git a/frontend/src/stores/page.js b/frontend/src/stores/page.js index dffd22e4b..a13f968fd 100644 --- a/frontend/src/stores/page.js +++ b/frontend/src/stores/page.js @@ -402,11 +402,11 @@ export const usePageStore = defineStore('page', { /** * PAGE - DUPLICATE */ - async pageDuplicate({ sourecePageId, title, path }) { + async pageDuplicate({ sourcePageId, title, path }) { const siteStore = useSiteStore() try { const pageData = await API_CLIENT.get( - `sites/${siteStore.id}/pages/${sourecePageId ?? this.id}`, + `sites/${siteStore.id}/pages/${sourcePageId ?? this.id}`, { searchParams: { withContent: true } } ).json() if (!pageData?.id) { @@ -565,7 +565,11 @@ export const usePageStore = defineStore('page', { } }).json() ) - this.router.replace(`/${path}`) + // -> Following the page only makes sense when it is the one being viewed. Moved from the file + // manager, it is some other page, and the reader is still on theirs. + if (id === this.id) { + this.router.replace(`/${path}`) + } }, /** * PAGE - Rename