refactor: wire admin scheduler + metrics views

backend/api/index.ts

@@ -8,6 +8,7 @@ async function routes(app: FastifyInstance) {
   await import('./schemas/block.ts').then((m) => m.registerSchemas(app))
   await import('./schemas/group.ts').then((m) => m.registerSchemas(app))
   await import('./schemas/mail.ts').then((m) => m.registerSchemas(app))
+  await import('./schemas/scheduler.ts').then((m) => m.registerSchemas(app))
   await import('./schemas/site.ts').then((m) => m.registerSchemas(app))
   await import('./schemas/user.ts').then((m) => m.registerSchemas(app))
 
@@ -18,6 +19,7 @@ async function routes(app: FastifyInstance) {
   app.register(import('./locales.ts'), { prefix: '/locales' })
   app.register(import('./mail.ts'), { prefix: '/mail' })
   app.register(import('./pages.ts'))
+  app.register(import('./scheduler.ts'), { prefix: '/scheduler' })
   app.register(import('./sites.ts'), { prefix: '/sites' })
   app.register(import('./system.ts'), { prefix: '/system' })
   app.register(import('./users.ts'), { prefix: '/users' })

backend/api/scheduler.ts

@@ -0,0 +1,296 @@
+import type { FastifyInstance } from 'fastify'
+import { JOB_STATES, type JobState } from '../models/jobs.ts'
+
+/**
+ * Scheduler API Routes
+ */
+async function routes(app: FastifyInstance) {
+  /**
+   * LIST SCHEDULED TASKS
+   */
+  app.get(
+    '/schedule',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'List the cron schedule',
+        description:
+          'The tasks the scheduler runs automatically. These are definitions, not executions — the jobs they produce show up under upcoming and then in the history.',
+        tags: ['Scheduler'],
+        response: {
+          200: {
+            description: 'List of scheduled tasks',
+            type: 'array',
+            items: { $ref: 'SchedulerTask#' }
+          }
+        }
+      }
+    },
+    async () => {
+      return WIKI.models.jobs.getSchedule()
+    }
+  )
+
+  /**
+   * RUN A SCHEDULED TASK NOW
+   */
+  app.post<{ Params: { scheduleId: string } }>(
+    '/schedule/:scheduleId/run',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'Run a scheduled task now',
+        description:
+          'Queues the task immediately, without waiting for its cron expression and without disturbing the planned iterations. The run is recorded in the history like any other job.',
+        tags: ['Scheduler'],
+        params: {
+          type: 'object',
+          properties: {
+            scheduleId: {
+              type: 'string',
+              format: 'uuid'
+            }
+          },
+          required: ['scheduleId']
+        },
+        response: {
+          200: {
+            description: 'Task queued successfully',
+            type: 'object',
+            properties: {
+              ok: {
+                type: 'boolean'
+              },
+              message: {
+                type: 'string'
+              },
+              id: {
+                type: 'string',
+                format: 'uuid',
+                description: 'The ID of the queued job.'
+              }
+            }
+          }
+        }
+      }
+    },
+    async (req, reply) => {
+      const entry = await WIKI.models.jobs.getScheduleEntry(req.params.scheduleId)
+      if (!entry) {
+        return reply.notFound('Scheduled task does not exist.')
+      }
+
+      const id = await WIKI.models.jobs.runScheduledTask(entry)
+      if (!id) {
+        return reply.internalServerError('The scheduler could not queue the job.')
+      }
+
+      return {
+        ok: true,
+        message: 'Task queued successfully.',
+        id
+      }
+    }
+  )
+
+  /**
+   * LIST UPCOMING JOBS
+   */
+  app.get(
+    '/upcoming',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'List the pending job queue',
+        description:
+          'Jobs waiting to be picked up, soonest first. A job with no `waitUntil` is eligible immediately.',
+        tags: ['Scheduler'],
+        response: {
+          200: {
+            description: 'List of upcoming jobs',
+            type: 'array',
+            items: { $ref: 'SchedulerUpcomingJob#' }
+          }
+        }
+      }
+    },
+    async () => {
+      return WIKI.models.jobs.getUpcoming()
+    }
+  )
+
+  /**
+   * CANCEL UPCOMING JOB
+   */
+  app.delete<{ Params: { jobId: string } }>(
+    '/upcoming/:jobId',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'Cancel a pending job',
+        description:
+          'Removes the job from the queue. A job that an instance has already picked up cannot be cancelled and answers 404, as it is no longer pending.',
+        tags: ['Scheduler'],
+        params: {
+          type: 'object',
+          properties: {
+            jobId: {
+              type: 'string',
+              format: 'uuid'
+            }
+          },
+          required: ['jobId']
+        },
+        response: {
+          204: {
+            description: 'Job cancelled successfully'
+          }
+        }
+      }
+    },
+    async (req, reply) => {
+      const cancelled = await WIKI.models.jobs.cancelUpcoming(req.params.jobId)
+      if (!cancelled) {
+        return reply.notFound('No pending job with this ID.')
+      }
+      return reply.code(204).send()
+    }
+  )
+
+  /**
+   * LIST JOB HISTORY
+   */
+  app.get<{ Querystring: { states?: JobState[]; limit?: number } }>(
+    '/jobs',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'List job execution history',
+        description:
+          'Past and running jobs, most recently started first. Older entries are purged by the `cleanJobHistory` task.',
+        tags: ['Scheduler'],
+        querystring: {
+          type: 'object',
+          properties: {
+            states: {
+              type: 'array',
+              description: 'Keep only jobs in these states. All states when omitted.',
+              items: {
+                type: 'string',
+                enum: JOB_STATES
+              }
+            },
+            limit: { type: 'integer', minimum: 1, maximum: 500, default: 100 }
+          }
+        },
+        response: {
+          200: {
+            description: 'List of jobs',
+            type: 'object',
+            properties: {
+              total: {
+                type: 'integer',
+                description:
+                  'How many jobs match the requested states, which can exceed the number returned.'
+              },
+              limit: {
+                type: 'integer'
+              },
+              jobs: {
+                type: 'array',
+                items: { $ref: 'SchedulerJob#' }
+              }
+            }
+          }
+        }
+      }
+    },
+    async (req) => {
+      const limit = req.query.limit ?? 100
+      const { total, jobs } = await WIKI.models.jobs.getHistory({
+        states: req.query.states ?? [],
+        limit
+      })
+      return { total, limit, jobs }
+    }
+  )
+
+  /**
+   * RETRY JOB
+   */
+  app.post<{ Params: { jobId: string } }>(
+    '/jobs/:jobId/retry',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'Run a past job again',
+        description:
+          'Queues a new job with the same task and payload. The original history entry is left as it is, and the new run is recorded separately with a full retry budget.',
+        tags: ['Scheduler'],
+        params: {
+          type: 'object',
+          properties: {
+            jobId: {
+              type: 'string',
+              format: 'uuid'
+            }
+          },
+          required: ['jobId']
+        },
+        response: {
+          200: {
+            description: 'Job queued successfully',
+            type: 'object',
+            properties: {
+              ok: {
+                type: 'boolean'
+              },
+              message: {
+                type: 'string'
+              },
+              id: {
+                type: 'string',
+                format: 'uuid',
+                description: 'The ID of the newly queued job, not the one it was created from.'
+              }
+            }
+          }
+        }
+      }
+    },
+    async (req, reply) => {
+      const entry = await WIKI.models.jobs.getHistoryEntry(req.params.jobId)
+      if (!entry) {
+        return reply.notFound('Job does not exist.')
+      }
+      if (entry.state === 'active') {
+        return reply.conflict('This job is still running.')
+      }
+
+      const id = await WIKI.models.jobs.retryJob(entry)
+      if (!id) {
+        return reply.internalServerError('The scheduler could not queue the job.')
+      }
+
+      return {
+        ok: true,
+        message: 'Job queued successfully.',
+        id
+      }
+    }
+  )
+}
+
+export default routes

backend/api/schemas/scheduler.ts

@@ -0,0 +1,155 @@
+import type { FastifyInstance } from 'fastify'
+import { JOB_STATES } from '../../models/jobs.ts'
+
+export async function registerSchemas(app: FastifyInstance): Promise<void> {
+  /**
+   * SCHEDULER TASK - A cron entry, i.e. a task that runs automatically
+   */
+  app.addSchema({
+    $id: 'SchedulerTask',
+    type: 'object',
+    properties: {
+      id: {
+        type: 'string',
+        format: 'uuid'
+      },
+      task: {
+        type: 'string',
+        description: 'Task name, matching a file under `tasks/simple/` or `tasks/workers/`.'
+      },
+      cron: {
+        type: 'string',
+        description: 'Cron expression, evaluated in UTC.'
+      },
+      type: {
+        type: 'string',
+        description: 'Where the entry comes from, e.g. `system`.'
+      },
+      createdAt: {
+        type: 'string',
+        format: 'date-time',
+        description: 'RFC 3339 Date Time'
+      },
+      updatedAt: {
+        type: 'string',
+        format: 'date-time',
+        description: 'RFC 3339 Date Time'
+      }
+    }
+  })
+
+  /**
+   * SCHEDULER UPCOMING JOB - A job waiting in the queue
+   */
+  app.addSchema({
+    $id: 'SchedulerUpcomingJob',
+    type: 'object',
+    properties: {
+      id: {
+        type: 'string',
+        format: 'uuid'
+      },
+      task: {
+        type: 'string'
+      },
+      useWorker: {
+        type: 'boolean',
+        description: 'True when the task runs in a worker thread rather than in-process.'
+      },
+      retries: {
+        type: 'integer',
+        description: 'How many attempts have already failed. Zero on a first attempt.'
+      },
+      maxRetries: {
+        type: 'integer'
+      },
+      waitUntil: {
+        // -> Jobs meant to run as soon as a worker is free have no date at all
+        type: 'string',
+        nullable: true,
+        format: 'date-time',
+        description: 'RFC 3339 Date Time, or null to run at the next opportunity'
+      },
+      isScheduled: {
+        type: 'boolean',
+        description: 'True when the job was created from a cron entry rather than on demand.'
+      },
+      createdBy: {
+        type: 'string',
+        nullable: true,
+        description: 'ID of the instance that queued the job.'
+      },
+      createdAt: {
+        type: 'string',
+        format: 'date-time',
+        description: 'RFC 3339 Date Time'
+      },
+      updatedAt: {
+        type: 'string',
+        format: 'date-time',
+        description: 'RFC 3339 Date Time'
+      }
+    }
+  })
+
+  /**
+   * SCHEDULER JOB - One execution, as recorded in the job history
+   */
+  app.addSchema({
+    $id: 'SchedulerJob',
+    type: 'object',
+    properties: {
+      id: {
+        type: 'string',
+        format: 'uuid'
+      },
+      task: {
+        type: 'string'
+      },
+      state: {
+        type: 'string',
+        enum: JOB_STATES,
+        description:
+          '`active` while running, `interrupted` when the run was cut short rather than failing on its own.'
+      },
+      useWorker: {
+        type: 'boolean'
+      },
+      wasScheduled: {
+        type: 'boolean'
+      },
+      attempt: {
+        type: 'integer',
+        description: 'Which attempt this execution was, starting at 1.'
+      },
+      maxRetries: {
+        type: 'integer'
+      },
+      lastErrorMessage: {
+        type: 'string',
+        nullable: true
+      },
+      executedBy: {
+        type: 'string',
+        nullable: true,
+        description: 'ID of the instance that ran the job.'
+      },
+      createdAt: {
+        type: 'string',
+        format: 'date-time',
+        description: 'RFC 3339 Date Time — when the job was queued'
+      },
+      startedAt: {
+        type: 'string',
+        format: 'date-time',
+        description: 'RFC 3339 Date Time'
+      },
+      completedAt: {
+        type: 'string',
+        nullable: true,
+        format: 'date-time',
+        description: 'RFC 3339 Date Time, or null while the job has not finished'
+      }
+    }
+  })
+}

backend/api/system.ts

@@ -56,8 +56,14 @@ async function routes(app: FastifyInstance) {
               isMailConfigured: {
                 type: 'boolean'
               },
+              isMetricsEnabled: {
+                type: 'boolean',
+                description: 'Whether the Prometheus metrics endpoint is turned on.'
+              },
               isSchedulerHealthy: {
-                type: 'boolean'
+                type: 'boolean',
+                description:
+                  'False when no instance has refreshed the scheduler cron lock recently, i.e. scheduled jobs are no longer being queued.'
               },
               latestVersion: {
                 type: 'string'
@@ -112,7 +118,8 @@ async function routes(app: FastifyInstance) {
         hostname: os.hostname(),
         httpPort: 0,
         isMailConfigured: WIKI.config?.mail?.host?.length > 2,
-        isSchedulerHealthy: true, // TODO:
+        isMetricsEnabled: WIKI.config.metrics.isEnabled === true,
+        isSchedulerHealthy: await WIKI.models.jobs.isHealthy(),
         latestVersion: WIKI.config.update.version,
         latestVersionReleaseDate: WIKI.config.update.versionDate,
         loginsPastDay: await WIKI.db.$count(
@@ -165,6 +172,99 @@ async function routes(app: FastifyInstance) {
     }
   )
 
+  /**
+   * GET METRICS ENDPOINT STATE
+   */
+  app.get(
+    '/metrics',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'Get the metrics endpoint state',
+        description:
+          'Whether the Prometheus metrics endpoint is turned on. The endpoint itself is not implemented yet — see the description of the PUT counterpart.',
+        tags: ['System'],
+        response: {
+          200: {
+            description: 'Metrics endpoint state',
+            type: 'object',
+            properties: {
+              isEnabled: {
+                type: 'boolean'
+              }
+            }
+          }
+        }
+      }
+    },
+    async () => {
+      return { isEnabled: WIKI.config.metrics.isEnabled === true }
+    }
+  )
+
+  /**
+   * SET METRICS ENDPOINT STATE
+   */
+  app.put<{ Body: { isEnabled: boolean } }>(
+    '/metrics',
+    {
+      config: {
+        permissions: ['manage:system']
+      },
+      schema: {
+        summary: 'Turn the metrics endpoint on or off',
+        description:
+          'Stores the state and nothing more, for now: the `/metrics` endpoint it governs is not implemented, and its documented `read:metrics` bearer authentication depends on API keys, which are not implemented either.',
+        tags: ['System'],
+        body: {
+          type: 'object',
+          required: ['isEnabled'],
+          properties: {
+            isEnabled: {
+              type: 'boolean'
+            }
+          }
+        },
+        response: {
+          200: {
+            description: 'Metrics endpoint state updated successfully',
+            type: 'object',
+            properties: {
+              ok: {
+                type: 'boolean'
+              },
+              message: {
+                type: 'string'
+              },
+              isEnabled: {
+                type: 'boolean'
+              }
+            }
+          }
+        }
+      }
+    },
+    async (req, reply) => {
+      const previousConfig = WIKI.config.metrics
+      WIKI.config.metrics = { ...previousConfig, isEnabled: req.body.isEnabled }
+
+      if (!(await WIKI.configSvc.saveToDb(['metrics']))) {
+        WIKI.config.metrics = previousConfig
+        return reply.internalServerError('Failed to save the metrics endpoint state.')
+      }
+
+      return {
+        ok: true,
+        message: req.body.isEnabled
+          ? 'Metrics endpoint enabled successfully.'
+          : 'Metrics endpoint disabled successfully.',
+        isEnabled: req.body.isEnabled
+      }
+    }
+  )
+
   /**
    * LIST SYSTEM INSTANCES
    */

backend/core/scheduler.ts

@@ -301,13 +301,17 @@ export default {
     } catch (err: any) {
       WIKI.logger.warn(err)
       if (jobIds && jobIds.length > 0) {
-        WIKI.db
+        // -> The filter must name the table being updated: `jobs.id` here produced
+        //    `UPDATE "jobHistory" ... WHERE "jobs"."id" IN (...)`, which postgres rejects with
+        //    "missing FROM-clause entry", and the statement was not awaited so the rejection was
+        //    lost. Interrupted jobs were therefore never recorded as such.
+        await WIKI.db
           .update(jobHistoryTable)
           .set({
             state: 'interrupted',
             lastErrorMessage: err.message
           })
-          .where(inArray(jobsTable.id, jobIds))
+          .where(inArray(jobHistoryTable.id, jobIds))
       }
     }
   },

backend/locales/en.json

@@ -497,11 +497,14 @@
   "admin.metrics.enabled": "Endpoint Enabled",
   "admin.metrics.endpoint": "The metrics endpoint can be scraped at {endpoint}",
   "admin.metrics.endpointWarning": "Note that this override any page at this path.",
+  "admin.metrics.loadFailed": "Failed to load the metrics endpoint state.",
+  "admin.metrics.notImplemented": "The endpoint itself is not available yet: this setting is saved, but nothing serves {endpoint} so far.",
   "admin.metrics.refreshSuccess": "Metrics endpoint state has been refreshed.",
   "admin.metrics.subtitle": "Manage the Prometheus metrics endpoint",
   "admin.metrics.title": "Metrics",
   "admin.metrics.toggleStateDisabledSuccess": "Metrics endpoint disabled successfully.",
   "admin.metrics.toggleStateEnabledSuccess": "Metrics endpoint enabled successfully.",
+  "admin.metrics.toggleStateFailed": "Failed to switch the metrics endpoint state.",
   "admin.nav.modules": "Modules",
   "admin.nav.site": "Site",
   "admin.nav.system": "System",
@@ -550,6 +553,7 @@
   "admin.scheduler.activeNone": "There are no active jobs at the moment.",
   "admin.scheduler.attempt": "Attempt",
   "admin.scheduler.cancelJob": "Cancel Job",
+  "admin.scheduler.cancelJobFailed": "Failed to cancel the job.",
   "admin.scheduler.cancelJobSuccess": "Job cancelled successfully.",
   "admin.scheduler.completed": "Completed",
   "admin.scheduler.completedIn": "Completed in {duration}",
@@ -560,11 +564,17 @@
   "admin.scheduler.error": "Error",
   "admin.scheduler.failed": "Failed",
   "admin.scheduler.failedNone": "There are no recently failed job to display.",
+  "admin.scheduler.historyCapped": "Showing the {shown} most recent of {total} jobs.",
   "admin.scheduler.interrupted": "Interrupted",
+  "admin.scheduler.loadFailed": "Failed to load jobs.",
   "admin.scheduler.pending": "Pending",
   "admin.scheduler.result": "Result",
   "admin.scheduler.retryJob": "Retry Job",
+  "admin.scheduler.retryJobFailed": "Failed to retry the job.",
   "admin.scheduler.retryJobSuccess": "Job has been rescheduled and will execute shortly.",
+  "admin.scheduler.runNow": "Run Now",
+  "admin.scheduler.runNowFailed": "Failed to queue the task.",
+  "admin.scheduler.runNowSuccess": "{task} has been queued and will run shortly.",
   "admin.scheduler.schedule": "Schedule",
   "admin.scheduler.scheduled": "Scheduled",
   "admin.scheduler.scheduledNone": "There are no scheduled jobs at the moment.",

backend/models/jobs.ts

@@ -1,12 +1,27 @@
 import {
+  jobs as jobsTable,
   jobSchedule as jobScheduleTable,
   jobLock as jobLockTable,
   jobHistory as jobHistoryTable
 } from '../db/schema.ts'
-import { and, eq, lte, not } from 'drizzle-orm'
+import { and, count, desc, eq, inArray, lte, not, sql } from 'drizzle-orm'
+
+/** The states a job can be in once it has been picked up for execution. */
+export const JOB_STATES = ['active', 'completed', 'failed', 'interrupted'] as const
+export type JobState = (typeof JOB_STATES)[number]
+
+/** One page of job history, with the total matching the requested states. */
+export interface JobHistoryPage {
+  total: number
+  jobs: (typeof jobHistoryTable.$inferSelect)[]
+}
 
 /**
  * Jobs model
+ *
+ * Three tables back the scheduler, and the admin area shows all three: `jobSchedule` holds the cron
+ * definitions, `jobs` is the pending queue, and `jobHistory` records every execution. A job moves
+ * from `jobs` to `jobHistory` when a worker picks it up — see `core/scheduler.ts`.
  */
 class Jobs {
   /**
@@ -50,6 +65,146 @@ class Jobs {
     })
   }
 
+  /**
+   * Whether the scheduler is keeping up with its cron duties.
+   *
+   * Exactly one instance holds the `cron` lock at a time and refreshes it as it queues the next
+   * batch of scheduled jobs, so a stale timestamp means no instance is running that check any more.
+   * The lock is only re-acquired once it is 5 minutes old, and the check itself runs on an interval,
+   * so the threshold has to be a comfortable multiple of that to avoid crying wolf.
+   */
+  async isHealthy(): Promise<boolean> {
+    const results = await WIKI.db
+      .select({ lastCheckedAt: jobLockTable.lastCheckedAt })
+      .from(jobLockTable)
+      .where(eq(jobLockTable.key, 'cron'))
+      .limit(1)
+    const lastCheckedAt = results[0]?.lastCheckedAt
+    if (!lastCheckedAt) {
+      return false
+    }
+    return (
+      Temporal.Instant.compare(
+        lastCheckedAt.toTemporalInstant(),
+        Temporal.Now.instant().subtract({ minutes: 15 })
+      ) > 0
+    )
+  }
+
+  /**
+   * The cron schedule: which tasks run automatically and how often
+   */
+  async getSchedule() {
+    return WIKI.db.select().from(jobScheduleTable).orderBy(jobScheduleTable.task)
+  }
+
+  /**
+   * A single cron entry, or null if there is no such entry
+   */
+  async getScheduleEntry(id: string) {
+    const results = await WIKI.db
+      .select()
+      .from(jobScheduleTable)
+      .where(eq(jobScheduleTable.id, id))
+      .limit(1)
+    return results[0] ?? null
+  }
+
+  /**
+   * Queue a cron entry's task to run at the next opportunity.
+   *
+   * The job is deliberately *not* flagged as scheduled: it is an on-demand run, so it must not be
+   * mistaken for one of the planned iterations that `scheduler.addScheduled()` reconciles.
+   *
+   * @returns The new job's ID, or null if the scheduler refused it
+   */
+  async runScheduledTask(entry: typeof jobScheduleTable.$inferSelect): Promise<string | null> {
+    const added = await WIKI.scheduler.addJob({
+      task: entry.task,
+      payload: entry.payload ?? {}
+    })
+    return added?.id ?? null
+  }
+
+  /**
+   * The pending queue, soonest first. Jobs with no `waitUntil` are eligible right away, so they
+   * come before any dated ones.
+   */
+  async getUpcoming() {
+    return WIKI.db
+      .select()
+      .from(jobsTable)
+      .orderBy(sql`${jobsTable.waitUntil} ASC NULLS FIRST`, jobsTable.createdAt)
+  }
+
+  /**
+   * Job execution history, most recently started first.
+   *
+   * @param states Keep only these states; all of them when empty
+   * @param limit Caps the rows returned — `total` still counts every match, so a caller can tell
+   *              that it is looking at a truncated view
+   */
+  async getHistory({
+    states = [],
+    limit = 100
+  }: { states?: JobState[]; limit?: number } = {}): Promise<JobHistoryPage> {
+    const where = states.length > 0 ? inArray(jobHistoryTable.state, states) : undefined
+    const totals = await WIKI.db.select({ total: count() }).from(jobHistoryTable).where(where)
+    const jobs = await WIKI.db
+      .select()
+      .from(jobHistoryTable)
+      .where(where)
+      .orderBy(desc(jobHistoryTable.startedAt))
+      .limit(limit)
+
+    return {
+      total: totals[0]?.total ?? 0,
+      jobs
+    }
+  }
+
+  /**
+   * A single history entry, or null if no such job ever ran
+   */
+  async getHistoryEntry(id: string) {
+    const results = await WIKI.db
+      .select()
+      .from(jobHistoryTable)
+      .where(eq(jobHistoryTable.id, id))
+      .limit(1)
+    return results[0] ?? null
+  }
+
+  /**
+   * Drop a job from the pending queue.
+   *
+   * Only queued jobs can be cancelled: once an instance has picked one up it is gone from `jobs`
+   * and already running.
+   *
+   * @returns Whether a queued job was removed
+   */
+  async cancelUpcoming(id: string): Promise<boolean> {
+    const result = await WIKI.db.delete(jobsTable).where(eq(jobsTable.id, id))
+    return (result.rowCount ?? 0) > 0
+  }
+
+  /**
+   * Queue a fresh run of a past job.
+   *
+   * The original history entry is left alone and the new run gets its own entry with a full retry
+   * budget — history is a log of executions, not a mutable job record.
+   *
+   * @returns The new job's ID, or null if the scheduler refused it
+   */
+  async retryJob(entry: typeof jobHistoryTable.$inferSelect): Promise<string | null> {
+    const added = await WIKI.scheduler.addJob({
+      task: entry.task,
+      payload: entry.payload ?? {},
+      maxRetries: entry.maxRetries
+    })
+    return added?.id ?? null
+  }
+
   /**
    * Purge old job history
    */

frontend/src/pages/AdminMetrics.vue

@@ -58,6 +58,11 @@ q-page.admin-api
               template(#endpoint)
                 strong.font-robotomono /metrics
             .text-caption {{ t('admin.metrics.endpointWarning') }}
+            //- The state is stored, but no route serves it yet — say so rather than let the card
+            //- above read as a promise
+            i18n-t.text-caption.text-orange(tag='div', keypath='admin.metrics.notImplemented', scope='global')
+              template(#endpoint)
+                strong.font-robotomono /metrics
       q-card.rounded-borders.q-mt-md(
         flat
         :class='$q.dark.isActive ? `bg-dark-5 text-white` : `bg-grey-3 text-dark`'
@@ -78,10 +83,9 @@ q-page.admin-api
 
 <script setup>
 
-import { cloneDeep } from 'lodash-es'
 import { useI18n } from 'vue-i18n'
 import { useMeta, useQuasar } from 'quasar'
-import { computed, onMounted, reactive, watch } from 'vue'
+import { onMounted, reactive } from 'vue'
 
 import { useAdminStore } from '@/stores/admin'
 import { useSiteStore } from '@/stores/site'
@@ -118,16 +122,18 @@ const state = reactive({
 async function load () {
   state.loading++
   $q.loading.show()
-  const resp = await APOLLO_CLIENT.query({
+  try {
-    query: `
+    const resp = await API_CLIENT.get('system/metrics').json()
-      query getMetricsState {
+    state.enabled = resp?.isEnabled === true
-        metricsState
+    // -> Keeps the status light in the admin sidebar in step without another round trip
-      }
-    `,
-    fetchPolicy: 'network-only'
-  })
-  state.enabled = resp?.data?.metricsState === true
     adminStore.info.isMetricsEnabled = state.enabled
+  } catch (err) {
+    $q.notify({
+      type: 'negative',
+      message: t('admin.metrics.loadFailed'),
+      caption: err.message
+    })
+  }
   $q.loading.hide()
   state.loading--
 }
@@ -142,36 +148,25 @@ async function refresh () {
 
 async function globalSwitch () {
   state.isToggleLoading = true
+  const wanted = !state.enabled
   try {
-    const resp = await APOLLO_CLIENT.mutate({
+    const resp = await API_CLIENT.put('system/metrics', {
-      mutation: `
+      json: { isEnabled: wanted }
-        mutation ($enabled: Boolean!) {
+    }).json()
-          setMetricsState (enabled: $enabled) {
+    if (!resp?.ok) {
-            operation {
+      throw new Error(resp?.message || 'An unexpected error occurred.')
-              succeeded
-              message
-            }
-          }
     }
-      `,
-      variables: {
-        enabled: !state.enabled
-      }
-    })
-    if (resp?.data?.setMetricsState?.operation?.succeeded) {
     $q.notify({
       type: 'positive',
-        message: state.enabled ? t('admin.metrics.toggleStateDisabledSuccess') : t('admin.metrics.toggleStateEnabledSuccess')
+      message: wanted ? t('admin.metrics.toggleStateEnabledSuccess') : t('admin.metrics.toggleStateDisabledSuccess')
     })
     await load()
-    } else {
-      throw new Error(resp?.data?.setMetricsState?.operation?.message || 'An unexpected error occurred.')
-    }
   } catch (err) {
+    const apiMessage = await err.response?.json().then(b => b?.message).catch(() => null)
     $q.notify({
       type: 'negative',
-      message: 'Failed to switch metrics endpoint state.',
+      message: t('admin.metrics.toggleStateFailed'),
-      caption: err.message
+      caption: apiMessage || err.message
     })
   }
   state.isToggleLoading = false

frontend/src/pages/AdminScheduler.vue

@@ -60,7 +60,7 @@ q-page.admin-terminal
         q-table(
           :rows='state.scheduledJobs'
           :columns='scheduledJobsHeaders'
-          row-key='name'
+          row-key='id'
           flat
           hide-bottom
           :rows-per-page-options='[0]'
@@ -103,6 +103,16 @@ q-page.admin-terminal
             q-td(:props='props')
               span {{props.value}}
               div: small.text-grey {{humanizeDate(props.row.updatedAt)}}
+          template(v-slot:body-cell-run='props')
+            q-td(:props='props')
+              q-btn.acrylic-btn.q-px-sm(
+                flat
+                icon='las la-play'
+                color='positive'
+                :aria-label='t(`admin.scheduler.runNow`)'
+                @click='runNow(props.row)'
+                )
+                q-tooltip(anchor='center left', self='center right') {{ t('admin.scheduler.runNow') }}
     template(v-else-if='state.displayMode === `upcoming`')
       q-card.rounded-borders(
         v-if='state.upcomingJobs.length < 1'
@@ -117,7 +127,7 @@ q-page.admin-terminal
         q-table(
           :rows='state.upcomingJobs'
           :columns='upcomingJobsHeaders'
-          row-key='name'
+          row-key='id'
           flat
           hide-bottom
           :rows-per-page-options='[0]'
@@ -175,7 +185,7 @@ q-page.admin-terminal
         q-table(
           :rows='state.jobs'
           :columns='jobsHeaders'
-          row-key='name'
+          row-key='id'
           flat
           hide-bottom
           :rows-per-page-options='[0]'
@@ -253,15 +263,20 @@ q-page.admin-terminal
                     strong {{props.row.executedBy}}
           template(v-slot:body-cell-actions='props')
             q-td(:props='props')
+              //- Only withheld while the scheduler still owes the job an automatic attempt
+              //- (`attempt` counts from 1, `maxRetries` is how many *extra* attempts it gets)
               q-btn.acrylic-btn.q-px-sm(
                 v-if='props.row.state !== `active`'
                 flat
                 icon='las la-undo-alt'
                 color='orange'
+                :aria-label='t(`admin.scheduler.retryJob`)'
                 @click='retryJob(props.row.id)'
-                :disable='props.row.state === `interrupted` || props.row.state === `failed` && props.row.attempt < props.row.maxRetries'
+                :disable='props.row.state === `failed` && props.row.attempt <= props.row.maxRetries'
                 )
                 q-tooltip(anchor='center left', self='center right') {{ t('admin.scheduler.retryJob') }}
+      .text-caption.text-grey(v-if='state.jobsTotal > state.jobs.length')
+        | {{ t('admin.scheduler.historyCapped', { shown: state.jobs.length, total: state.jobsTotal }) }}
 
 </template>
 
@@ -270,8 +285,6 @@ import { onMounted, reactive, watch } from 'vue'
 import { useMeta, useQuasar } from 'quasar'
 import { useI18n } from 'vue-i18n'
 
-import { DateTime, Duration, Interval } from 'luxon'
-
 import { useSiteStore } from '@/stores/site'
 
 // QUASAR
@@ -299,9 +312,21 @@ const state = reactive({
   scheduledJobs: [],
   upcomingJobs: [],
   jobs: [],
+  jobsTotal: 0,
   loading: 0
 })
 
+/** How many history entries a tab shows. The API caps this at 500. */
+const HISTORY_LIMIT = 100
+
+/** The history states behind each display mode. */
+const MODE_STATES = {
+  active: ['active'],
+  completed: ['completed'],
+  // -> An interrupted job never reported a result of its own, so it belongs with the failures
+  failed: ['failed', 'interrupted']
+}
+
 const scheduledJobsHeaders = [
   {
     align: 'center',
@@ -337,7 +362,7 @@ const scheduledJobsHeaders = [
     field: 'createdAt',
     name: 'created',
     sortable: true,
-    format: v => DateTime.fromISO(v).toRelative()
+    format: relativeDate
   },
   {
     label: t('admin.scheduler.updatedAt'),
@@ -345,7 +370,14 @@ const scheduledJobsHeaders = [
     field: 'updatedAt',
     name: 'updated',
     sortable: true,
-    format: v => DateTime.fromISO(v).toRelative()
+    format: relativeDate
+  },
+  {
+    align: 'center',
+    field: 'id',
+    name: 'run',
+    sortable: false,
+    style: 'width: 15px;'
   }
 ]
 
@@ -370,7 +402,7 @@ const upcomingJobsHeaders = [
     field: 'waitUntil',
     name: 'waituntil',
     sortable: true,
-    format: v => DateTime.fromISO(v).toRelative()
+    format: relativeDate
   },
   {
     label: t('admin.scheduler.attempt'),
@@ -392,7 +424,7 @@ const upcomingJobsHeaders = [
     field: 'createdAt',
     name: 'date',
     sortable: true,
-    format: v => DateTime.fromISO(v).toRelative()
+    format: relativeDate
   },
   {
     align: 'center',
@@ -445,7 +477,7 @@ const jobsHeaders = [
     field: 'startedAt',
     name: 'date',
     sortable: true,
-    format: v => DateTime.fromISO(v).toRelative()
+    format: relativeDate
   },
   {
     align: 'center',
@@ -458,142 +490,138 @@ const jobsHeaders = [
 
 // WATCHERS
 
-watch(() => state.displayMode, (newValue) => {
+watch(() => state.displayMode, () => {
   load()
 })
 
 // METHODS
 
+/** Largest-first. `week` is deliberately absent, so output reads e.g. "21 days ago". */
+const RELATIVE_UNITS = [
+  ['year', 31536000],
+  ['month', 2592000],
+  ['day', 86400],
+  ['hour', 3600],
+  ['minute', 60],
+  ['second', 1]
+]
+const relativeTimeFormat = new Intl.RelativeTimeFormat(undefined, { numeric: 'auto' })
+
+/** Reads both ways: past for history, future for a job still waiting its turn. */
+function relativeDate (val) {
+  if (!val) { return '---' }
+  const seconds = Temporal.Instant.from(val).until(Temporal.Now.instant()).total('seconds')
+  for (const [unit, secondsPerUnit] of RELATIVE_UNITS) {
+    if (Math.abs(seconds) >= secondsPerUnit || unit === 'second') {
+      return relativeTimeFormat.format(-Math.round(seconds / secondsPerUnit), unit)
+    }
+  }
+}
+
 function humanizeDate (val) {
-  return DateTime.fromISO(val).toFormat('fff')
+  if (!val) { return '---' }
+  return Temporal.Instant.from(val).toLocaleString(undefined, {
+    year: 'numeric',
+    month: 'long',
+    day: 'numeric',
+    hour: 'numeric',
+    minute: '2-digit',
+    second: '2-digit',
+    timeZoneName: 'short'
+  })
 }
 
+/** Narrow, largest-first and skipping empty units — "1h 4m 32s", or "820ms" for a quick job. */
+const DURATION_UNITS = ['hour', 'minute', 'second', 'millisecond']
+const durationListFormat = new Intl.ListFormat(undefined, { style: 'narrow', type: 'unit' })
+
 function humanizeDuration (start, end) {
-  const dur = Interval.fromDateTimes(DateTime.fromISO(start), DateTime.fromISO(end))
+  if (!start || !end) { return '---' }
-    .toDuration(['hours', 'minutes', 'seconds', 'milliseconds'])
+  const dur = Temporal.Instant.from(start).until(Temporal.Instant.from(end)).round({
-  return Duration.fromObject({
+    largestUnit: 'hour',
-    ...dur.hours > 0 && { hours: dur.hours },
+    smallestUnit: 'millisecond'
-    ...dur.minutes > 0 && { minutes: dur.minutes },
+  })
-    ...dur.seconds > 0 && { seconds: dur.seconds },
+  const parts = DURATION_UNITS
-    ...dur.milliseconds > 0 && { milliseconds: dur.milliseconds }
+    .filter(unit => dur[`${unit}s`] > 0)
-  }).toHuman({ unitDisplay: 'narrow', listStyle: 'short' })
+    .map(unit => new Intl.NumberFormat(undefined, {
+      style: 'unit',
+      unit,
+      unitDisplay: 'narrow'
+    }).format(dur[`${unit}s`]))
+  // -> A job that took under a millisecond still has to render as something
+  return parts.length > 0 ? durationListFormat.format(parts) : '0ms'
 }
 
 async function load () {
   state.loading++
   try {
     if (state.displayMode === 'scheduled') {
-      const resp = await APOLLO_CLIENT.query({
+      state.scheduledJobs = await API_CLIENT.get('scheduler/schedule').json() ?? []
-        query: `
-          query getSystemJobsScheduled {
-            systemJobsScheduled {
-              id
-              task
-              cron
-              type
-              createdAt
-              updatedAt
-            }
-          }
-        `,
-        fetchPolicy: 'network-only'
-      })
-      state.scheduledJobs = resp?.data?.systemJobsScheduled
     } else if (state.displayMode === 'upcoming') {
-      const resp = await APOLLO_CLIENT.query({
+      state.upcomingJobs = await API_CLIENT.get('scheduler/upcoming').json() ?? []
-        query: `
-          query getSystemJobsUpcoming {
-            systemJobsUpcoming {
-              id
-              task
-              useWorker
-              retries
-              maxRetries
-              waitUntil
-              isScheduled
-              createdBy
-              createdAt
-              updatedAt
-            }
-          }
-        `,
-        fetchPolicy: 'network-only'
-      })
-      state.upcomingJobs = resp?.data?.systemJobsUpcoming
     } else {
-      const states = state.displayMode === 'failed' ? ['FAILED', 'INTERRUPTED'] : [state.displayMode.toUpperCase()]
+      // -> Repeated `states` params rather than a comma-joined value: that is what the route's
-      const resp = await APOLLO_CLIENT.query({
+      //    array schema validates against
-        query: `
+      const searchParams = new URLSearchParams(
-          query getSystemJobs (
+        MODE_STATES[state.displayMode].map(s => ['states', s])
-            $states: [SystemJobState]
+      )
-          ) {
+      searchParams.set('limit', HISTORY_LIMIT)
-            systemJobs (
+      const resp = await API_CLIENT.get('scheduler/jobs', { searchParams }).json()
-              states: $states
+      state.jobs = resp?.jobs ?? []
-              ) {
+      state.jobsTotal = resp?.total ?? 0
-                id
-                task
-                state
-                useWorker
-                wasScheduled
-                attempt
-                maxRetries
-                lastErrorMessage
-                executedBy
-                createdAt
-                startedAt
-                completedAt
-            }
-          }
-        `,
-        variables: {
-          states
-        },
-        fetchPolicy: 'network-only'
-      })
-      state.jobs = resp?.data?.systemJobs?.map(j => ({ ...j, state: j.state.toLowerCase() }))
     }
   } catch (err) {
     $q.notify({
       type: 'negative',
-      message: 'Failed to load scheduled jobs.',
+      message: t('admin.scheduler.loadFailed'),
       caption: err.message
     })
   }
   state.loading--
 }
 
-async function cancelJob (jobId) {
+async function runNow (entry) {
   state.loading++
   try {
-    const resp = await APOLLO_CLIENT.mutate({
+    const resp = await API_CLIENT.post(`scheduler/schedule/${entry.id}/run`).json()
-      mutation: `
+    if (!resp?.ok) {
-        mutation cancelJob ($id: UUID!) {
+      throw new Error(resp?.message || 'An unexpected error occured.')
-          cancelJob(id: $id) {
-            operation {
-              succeeded
-              message
     }
+    // -> Nothing on this tab changes: the job it queued shows up under upcoming, then in the history
+    $q.notify({
+      type: 'positive',
+      message: t('admin.scheduler.runNowSuccess', { task: entry.task })
+    })
+  } catch (err) {
+    const apiMessage = await err.response?.json().then(b => b?.message).catch(() => null)
+    $q.notify({
+      type: 'negative',
+      message: t('admin.scheduler.runNowFailed'),
+      caption: apiMessage || err.message
+    })
   }
+  state.loading--
 }
-      `,
+
-      variables: {
+async function cancelJob (jobId) {
-        id: jobId
+  state.loading++
+  try {
+    const resp = await API_CLIENT.delete(`scheduler/upcoming/${jobId}`)
+    if (!resp?.ok) {
+      throw new Error((await resp.json())?.message || 'An unexpected error occured.')
     }
-    })
-    if (resp?.data?.cancelJob?.operation?.succeeded) {
-      load()
     $q.notify({
       type: 'positive',
       message: t('admin.scheduler.cancelJobSuccess')
     })
-    } else {
+    await load()
-      throw new Error(resp?.data?.cancelJob?.operation?.message || 'An unexpected error occured.')
-    }
   } catch (err) {
+    // -> ky throws above 400 — a job picked up between the render and the click answers 404
+    const apiMessage = await err.response?.json().then(b => b?.message).catch(() => null)
     $q.notify({
       type: 'negative',
-      message: 'Failed to cancel job.',
+      message: t('admin.scheduler.cancelJobFailed'),
-      caption: err.message
+      caption: apiMessage || err.message
     })
   }
   state.loading--
@@ -602,35 +630,21 @@ async function cancelJob (jobId) {
 async function retryJob (jobId) {
   state.loading++
   try {
-    const resp = await APOLLO_CLIENT.mutate({
+    const resp = await API_CLIENT.post(`scheduler/jobs/${jobId}/retry`).json()
-      mutation: `
+    if (!resp?.ok) {
-        mutation retryJob ($id: UUID!) {
+      throw new Error(resp?.message || 'An unexpected error occured.')
-          retryJob(id: $id) {
-            operation {
-              succeeded
-              message
     }
-          }
-        }
-      `,
-      variables: {
-        id: jobId
-      }
-    })
-    if (resp?.data?.retryJob?.operation?.succeeded) {
-      this.load()
     $q.notify({
       type: 'positive',
       message: t('admin.scheduler.retryJobSuccess')
     })
-    } else {
+    await load()
-      throw new Error(resp?.data?.retryJob?.operation?.message || 'An unexpected error occured.')
-    }
   } catch (err) {
+    const apiMessage = await err.response?.json().then(b => b?.message).catch(() => null)
     $q.notify({
       type: 'negative',
-      message: 'Failed to retry the job.',
+      message: t('admin.scheduler.retryJobFailed'),
-      caption: err.message
+      caption: apiMessage || err.message
     })
   }
   state.loading--

frontend/src/stores/admin.js

@@ -16,6 +16,7 @@ export const useAdminStore = defineStore('admin', {
       loginsPastDay: 0,
       isApiEnabled: false,
       isMailConfigured: false,
+      isMetricsEnabled: false,
       isSchedulerHealthy: false
     },
     overlay: null,
@@ -50,7 +51,7 @@ export const useAdminStore = defineStore('admin', {
       this.info.currentVersion = clone(resp?.currentVersion ?? 'n/a')
       this.info.latestVersion = clone(resp?.latestVersion ?? 'n/a')
       this.info.isApiEnabled = clone(resp?.apiState ?? false)
-      this.info.isMetricsEnabled = clone(resp?.metricsState ?? false)
+      this.info.isMetricsEnabled = clone(resp?.isMetricsEnabled ?? false)
       this.info.isMailConfigured = clone(resp?.isMailConfigured ?? false)
       this.info.isSchedulerHealthy = clone(resp?.isSchedulerHealthy ?? false)
     },