mirror of https://github.com/requarks/wiki
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
248 lines
7.6 KiB
248 lines
7.6 KiB
import {
|
|
jobs as jobsTable,
|
|
jobSchedule as jobScheduleTable,
|
|
jobLock as jobLockTable,
|
|
jobHistory as jobHistoryTable
|
|
} from '../db/schema.ts'
|
|
import { and, count, desc, eq, inArray, lte, not, sql } from 'drizzle-orm'
|
|
|
|
/** The states a job can be in once it has been picked up for execution. */
|
|
export const JOB_STATES = ['active', 'completed', 'failed', 'interrupted'] as const
|
|
export type JobState = (typeof JOB_STATES)[number]
|
|
|
|
/** One page of job history, with the total matching the requested states. */
|
|
export interface JobHistoryPage {
|
|
total: number
|
|
jobs: (typeof jobHistoryTable.$inferSelect)[]
|
|
}
|
|
|
|
/**
|
|
* Jobs model
|
|
*
|
|
* Three tables back the scheduler, and the admin area shows all three: `jobSchedule` holds the cron
|
|
* definitions, `jobs` is the pending queue, and `jobHistory` records every execution. A job moves
|
|
* from `jobs` to `jobHistory` when a worker picks it up — see `core/scheduler.ts`.
|
|
*/
|
|
class Jobs {
|
|
/**
|
|
* Initialize jobs table
|
|
*/
|
|
async init(): Promise<void> {
|
|
WIKI.logger.info('Inserting scheduled jobs...')
|
|
|
|
await WIKI.db.insert(jobScheduleTable).values([
|
|
{
|
|
task: 'checkVersion',
|
|
cron: '0 0 * * *',
|
|
type: 'system'
|
|
},
|
|
{
|
|
task: 'cleanJobHistory',
|
|
cron: '5 0 * * *',
|
|
type: 'system'
|
|
},
|
|
// {
|
|
// task: 'refreshAutocomplete',
|
|
// cron: '0 */6 * * *',
|
|
// type: 'system'
|
|
// },
|
|
{
|
|
task: 'purgeRateLimits',
|
|
cron: '10 * * * *',
|
|
type: 'system'
|
|
},
|
|
{
|
|
task: 'updateLocales',
|
|
cron: '0 0 * * *',
|
|
type: 'system'
|
|
}
|
|
])
|
|
|
|
await WIKI.db.insert(jobLockTable).values({
|
|
key: 'cron',
|
|
lastCheckedBy: 'init',
|
|
// NOTE: an ISO string, not a Date, is passed deliberately — pg sends it verbatim and
|
|
// postgres parses it as UTC, whereas a JS Date would be serialized in the process's local
|
|
// timezone. Kept as-is; the cast only silences the column's `Date` type.
|
|
lastCheckedAt: Temporal.Now.instant()
|
|
.subtract({ hours: 1 })
|
|
.toString({ smallestUnit: 'millisecond' }) as any
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Whether the scheduler is keeping up with its cron duties.
|
|
*
|
|
* Exactly one instance holds the `cron` lock at a time and refreshes it as it queues the next
|
|
* batch of scheduled jobs, so a stale timestamp means no instance is running that check any more.
|
|
* The lock is only re-acquired once it is 5 minutes old, and the check itself runs on an interval,
|
|
* so the threshold has to be a comfortable multiple of that to avoid crying wolf.
|
|
*/
|
|
async isHealthy(): Promise<boolean> {
|
|
const results = await WIKI.db
|
|
.select({ lastCheckedAt: jobLockTable.lastCheckedAt })
|
|
.from(jobLockTable)
|
|
.where(eq(jobLockTable.key, 'cron'))
|
|
.limit(1)
|
|
const lastCheckedAt = results[0]?.lastCheckedAt
|
|
if (!lastCheckedAt) {
|
|
return false
|
|
}
|
|
return (
|
|
Temporal.Instant.compare(
|
|
lastCheckedAt.toTemporalInstant(),
|
|
Temporal.Now.instant().subtract({ minutes: 15 })
|
|
) > 0
|
|
)
|
|
}
|
|
|
|
/**
|
|
* How many jobs are running right now, across every instance.
|
|
*
|
|
* A job occupies exactly one worker slot from the moment it is claimed — `core/scheduler.ts`
|
|
* moves it into the history as `active` and bumps `activeWorkers` in the same step — so this is
|
|
* the cluster-wide equivalent of that per-instance counter.
|
|
*
|
|
* An instance that dies mid-job leaves its row saying `active` until `reapStaleJobs` picks it up,
|
|
* which counts here in the meantime, exactly as it still shows under the scheduler's active tab.
|
|
*/
|
|
async countActive(): Promise<number> {
|
|
return WIKI.db.$count(jobHistoryTable, eq(jobHistoryTable.state, 'active'))
|
|
}
|
|
|
|
/**
|
|
* The cron schedule: which tasks run automatically and how often
|
|
*/
|
|
async getSchedule() {
|
|
return WIKI.db.select().from(jobScheduleTable).orderBy(jobScheduleTable.task)
|
|
}
|
|
|
|
/**
|
|
* A single cron entry, or null if there is no such entry
|
|
*/
|
|
async getScheduleEntry(id: string) {
|
|
const results = await WIKI.db
|
|
.select()
|
|
.from(jobScheduleTable)
|
|
.where(eq(jobScheduleTable.id, id))
|
|
.limit(1)
|
|
return results[0] ?? null
|
|
}
|
|
|
|
/**
|
|
* Queue a cron entry's task to run at the next opportunity.
|
|
*
|
|
* The job is deliberately *not* flagged as scheduled: it is an on-demand run, so it must not be
|
|
* mistaken for one of the planned iterations that `scheduler.addScheduled()` reconciles.
|
|
*
|
|
* @returns The new job's ID, or null if the scheduler refused it
|
|
*/
|
|
async runScheduledTask(entry: typeof jobScheduleTable.$inferSelect): Promise<string | null> {
|
|
const added = await WIKI.scheduler.addJob({
|
|
task: entry.task,
|
|
payload: entry.payload ?? {}
|
|
})
|
|
return added?.id ?? null
|
|
}
|
|
|
|
/**
|
|
* The pending queue, soonest first. Jobs with no `waitUntil` are eligible right away, so they
|
|
* come before any dated ones.
|
|
*/
|
|
async getUpcoming() {
|
|
return WIKI.db
|
|
.select()
|
|
.from(jobsTable)
|
|
.orderBy(sql`${jobsTable.waitUntil} ASC NULLS FIRST`, jobsTable.createdAt)
|
|
}
|
|
|
|
/**
|
|
* Job execution history, most recently started first.
|
|
*
|
|
* @param states Keep only these states; all of them when empty
|
|
* @param limit Caps the rows returned — `total` still counts every match, so a caller can tell
|
|
* that it is looking at a truncated view
|
|
*/
|
|
async getHistory({
|
|
states = [],
|
|
limit = 100
|
|
}: { states?: JobState[]; limit?: number } = {}): Promise<JobHistoryPage> {
|
|
const where = states.length > 0 ? inArray(jobHistoryTable.state, states) : undefined
|
|
const totals = await WIKI.db.select({ total: count() }).from(jobHistoryTable).where(where)
|
|
const jobs = await WIKI.db
|
|
.select()
|
|
.from(jobHistoryTable)
|
|
.where(where)
|
|
.orderBy(desc(jobHistoryTable.startedAt))
|
|
.limit(limit)
|
|
|
|
return {
|
|
total: totals[0]?.total ?? 0,
|
|
jobs
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A single history entry, or null if no such job ever ran
|
|
*/
|
|
async getHistoryEntry(id: string) {
|
|
const results = await WIKI.db
|
|
.select()
|
|
.from(jobHistoryTable)
|
|
.where(eq(jobHistoryTable.id, id))
|
|
.limit(1)
|
|
return results[0] ?? null
|
|
}
|
|
|
|
/**
|
|
* Drop a job from the pending queue.
|
|
*
|
|
* Only queued jobs can be cancelled: once an instance has picked one up it is gone from `jobs`
|
|
* and already running.
|
|
*
|
|
* @returns Whether a queued job was removed
|
|
*/
|
|
async cancelUpcoming(id: string): Promise<boolean> {
|
|
const result = await WIKI.db.delete(jobsTable).where(eq(jobsTable.id, id))
|
|
return (result.rowCount ?? 0) > 0
|
|
}
|
|
|
|
/**
|
|
* Queue a fresh run of a past job.
|
|
*
|
|
* The original history entry is left alone and the new run gets its own entry with a full retry
|
|
* budget — history is a log of executions, not a mutable job record.
|
|
*
|
|
* @returns The new job's ID, or null if the scheduler refused it
|
|
*/
|
|
async retryJob(entry: typeof jobHistoryTable.$inferSelect): Promise<string | null> {
|
|
const added = await WIKI.scheduler.addJob({
|
|
task: entry.task,
|
|
payload: entry.payload ?? {},
|
|
maxRetries: entry.maxRetries
|
|
})
|
|
return added?.id ?? null
|
|
}
|
|
|
|
/**
|
|
* Purge old job history
|
|
*/
|
|
async cleanHistory(): Promise<void> {
|
|
await WIKI.db.delete(jobHistoryTable).where(
|
|
and(
|
|
not(eq(jobHistoryTable.state, 'active')),
|
|
lte(
|
|
jobHistoryTable.startedAt,
|
|
new Date(
|
|
Temporal.Now.instant().subtract({
|
|
seconds: WIKI.config.scheduler.historyExpiration
|
|
}).epochMilliseconds
|
|
)
|
|
)
|
|
)
|
|
)
|
|
}
|
|
}
|
|
|
|
export const jobs = new Jobs()
|