feat: scheduler - regular cron check + add future jobs

pull/5733/head
Nicolas Giard 2 years ago
parent 011be49ab4
commit 39b273b224
No known key found for this signature in database
GPG Key ID: 85061B8F9D55B7C8

@ -38,20 +38,6 @@ html(lang=siteConfig.lang)
script. script.
siteConfig.devMode = true siteConfig.devMode = true
//- Icon Set
if config.theming.iconset === 'fa'
link(
type='text/css'
rel='stylesheet'
href='https://use.fontawesome.com/releases/v5.10.0/css/all.css'
)
else if config.theming.iconset === 'fa4'
link(
type='text/css'
rel='stylesheet'
href='https://cdn.jsdelivr.net/npm/font-awesome@4.7.0/css/font-awesome.min.css'
)
//- CSS //- CSS
<% for (var index in htmlWebpackPlugin.files.css) { %> <% for (var index in htmlWebpackPlugin.files.css) { %>
<% if (htmlWebpackPlugin.files.cssIntegrity) { %> <% if (htmlWebpackPlugin.files.cssIntegrity) { %>

@ -31,6 +31,8 @@ defaults:
bodyParserLimit: 5mb bodyParserLimit: 5mb
scheduler: scheduler:
workers: 3 workers: 3
pollingCheck: 5
scheduledCheck: 300
# DB defaults # DB defaults
api: api:
isEnabled: false isEnabled: false

@ -158,15 +158,15 @@ router.get(['/_edit', '/_edit/*'], async (req, res, next) => {
return res.redirect(`/_edit/home`) return res.redirect(`/_edit/home`)
} }
if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) { // if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) {
return res.redirect(`/_edit/${pageArgs.locale}/${pageArgs.path}`) // return res.redirect(`/_edit/${pageArgs.locale}/${pageArgs.path}`)
} // }
req.i18n.changeLanguage(pageArgs.locale) // req.i18n.changeLanguage(pageArgs.locale)
// -> Set Editor Lang // -> Set Editor Lang
_.set(res, 'locals.siteConfig.lang', pageArgs.locale) _.set(res, 'locals.siteConfig.lang', pageArgs.locale)
_.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl') // _.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl')
// -> Check for reserved path // -> Check for reserved path
if (pageHelper.isReservedPath(pageArgs.path)) { if (pageHelper.isReservedPath(pageArgs.path)) {
@ -187,9 +187,9 @@ router.get(['/_edit', '/_edit/*'], async (req, res, next) => {
const effectivePermissions = WIKI.auth.getEffectivePermissions(req, pageArgs) const effectivePermissions = WIKI.auth.getEffectivePermissions(req, pageArgs)
const injectCode = { const injectCode = {
css: WIKI.config.theming.injectCSS, css: '', // WIKI.config.theming.injectCSS,
head: WIKI.config.theming.injectHead, head: '', // WIKI.config.theming.injectHead,
body: WIKI.config.theming.injectBody body: '' // WIKI.config.theming.injectBody
} }
if (page) { if (page) {
@ -462,11 +462,11 @@ router.get('/*', async (req, res, next) => {
const isPage = (stripExt || pageArgs.path.indexOf('.') === -1) const isPage = (stripExt || pageArgs.path.indexOf('.') === -1)
if (isPage) { if (isPage) {
if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) { // if (WIKI.config.lang.namespacing && !pageArgs.explicitLocale) {
return res.redirect(`/${pageArgs.locale}/${pageArgs.path}`) // return res.redirect(`/${pageArgs.locale}/${pageArgs.path}`)
} // }
req.i18n.changeLanguage(pageArgs.locale) // req.i18n.changeLanguage(pageArgs.locale)
try { try {
// -> Get Page from cache // -> Get Page from cache
@ -494,7 +494,7 @@ router.get('/*', async (req, res, next) => {
} }
_.set(res, 'locals.siteConfig.lang', pageArgs.locale) _.set(res, 'locals.siteConfig.lang', pageArgs.locale)
_.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl') // _.set(res, 'locals.siteConfig.rtl', req.i18n.dir() === 'rtl')
if (page) { if (page) {
_.set(res.locals, 'pageMeta.title', page.title) _.set(res.locals, 'pageMeta.title', page.title)

@ -453,9 +453,9 @@ module.exports = {
getEffectivePermissions (req, page) { getEffectivePermissions (req, page) {
return { return {
comments: { comments: {
read: WIKI.config.features.featurePageComments ? WIKI.auth.checkAccess(req.user, ['read:comments'], page) : false, read: WIKI.auth.checkAccess(req.user, ['read:comments'], page),
write: WIKI.config.features.featurePageComments ? WIKI.auth.checkAccess(req.user, ['write:comments'], page) : false, write: WIKI.auth.checkAccess(req.user, ['write:comments'], page),
manage: WIKI.config.features.featurePageComments ? WIKI.auth.checkAccess(req.user, ['manage:comments'], page) : false manage: WIKI.auth.checkAccess(req.user, ['manage:comments'], page)
}, },
history: { history: {
read: WIKI.auth.checkAccess(req.user, ['read:history'], page) read: WIKI.auth.checkAccess(req.user, ['read:history'], page)

@ -1,18 +1,21 @@
const { DynamicThreadPool } = require('poolifier') const { DynamicThreadPool } = require('poolifier')
const os = require('node:os') const os = require('node:os')
const { setTimeout } = require('node:timers/promises')
const autoload = require('auto-load') const autoload = require('auto-load')
const path = require('node:path') const path = require('node:path')
const cronparser = require('cron-parser')
const { DateTime } = require('luxon')
module.exports = { module.exports = {
pool: null, workerPool: null,
maxWorkers: 1, maxWorkers: 1,
activeWorkers: 0, activeWorkers: 0,
pollingRef: null,
scheduledRef: null,
tasks: null, tasks: null,
async init () { async init () {
this.maxWorkers = WIKI.config.scheduler.workers === 'auto' ? os.cpus().length : WIKI.config.scheduler.workers this.maxWorkers = WIKI.config.scheduler.workers === 'auto' ? os.cpus().length : WIKI.config.scheduler.workers
WIKI.logger.info(`Initializing Worker Pool (Limit: ${this.maxWorkers})...`) WIKI.logger.info(`Initializing Worker Pool (Limit: ${this.maxWorkers})...`)
this.pool = new DynamicThreadPool(1, this.maxWorkers, './server/worker.js', { this.workerPool = new DynamicThreadPool(1, this.maxWorkers, './server/worker.js', {
errorHandler: (err) => WIKI.logger.warn(err), errorHandler: (err) => WIKI.logger.warn(err),
exitHandler: () => WIKI.logger.debug('A worker has gone offline.'), exitHandler: () => WIKI.logger.debug('A worker has gone offline.'),
onlineHandler: () => WIKI.logger.debug('New worker is online.') onlineHandler: () => WIKI.logger.debug('New worker is online.')
@ -22,39 +25,68 @@ module.exports = {
}, },
async start () { async start () {
WIKI.logger.info('Starting Scheduler...') WIKI.logger.info('Starting Scheduler...')
WIKI.db.listener.addChannel('scheduler', payload => {
// -> Add PostgreSQL Sub Channel
WIKI.db.listener.addChannel('scheduler', async payload => {
switch (payload.event) { switch (payload.event) {
case 'newJob': { case 'newJob': {
if (this.activeWorkers < this.maxWorkers) { if (this.activeWorkers < this.maxWorkers) {
this.activeWorkers++ this.activeWorkers++
this.processJob() await this.processJob()
this.activeWorkers--
} }
break break
} }
} }
}) })
// await WIKI.db.knex('jobs').insert({
// task: 'test', // -> Start scheduled jobs check
// payload: { foo: 'bar' } this.scheduledRef = setInterval(async () => {
// }) this.addScheduled()
// WIKI.db.listener.publish('scheduler', { }, WIKI.config.scheduler.scheduledCheck * 1000)
// source: WIKI.INSTANCE_ID,
// event: 'newJob' // -> Add scheduled jobs on init
// }) await this.addScheduled()
// -> Start job polling
this.pollingRef = setInterval(async () => {
this.processJob()
}, WIKI.config.scheduler.pollingCheck * 1000)
WIKI.logger.info('Scheduler: [ STARTED ]') WIKI.logger.info('Scheduler: [ STARTED ]')
}, },
async addJob ({ task, payload, waitUntil, isScheduled = false, notify = true }) {
try {
await WIKI.db.knex('jobs').insert({
task,
useWorker: !(typeof this.tasks[task] === 'function'),
payload,
isScheduled,
waitUntil,
createdBy: WIKI.INSTANCE_ID
})
if (notify) {
WIKI.db.listener.publish('scheduler', {
source: WIKI.INSTANCE_ID,
event: 'newJob'
})
}
} catch (err) {
WIKI.logger.warn(`Failed to add job to scheduler: ${err.message}`)
}
},
async processJob () { async processJob () {
try { try {
await WIKI.db.knex.transaction(async trx => { await WIKI.db.knex.transaction(async trx => {
const jobs = await trx('jobs') const jobs = await trx('jobs')
.where('id', WIKI.db.knex.raw('(SELECT id FROM jobs ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1)')) .where('id', WIKI.db.knex.raw('(SELECT id FROM jobs WHERE ("waitUntil" IS NULL OR "waitUntil" <= NOW()) ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1)'))
.returning('*') .returning('*')
.del() .del()
if (jobs && jobs.length === 1) { if (jobs && jobs.length === 1) {
const job = jobs[0] const job = jobs[0]
WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`) WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`)
if (job.useWorker) { if (job.useWorker) {
await this.pool.execute({ await this.workerPool.execute({
id: job.id, id: job.id,
name: job.task, name: job.task,
data: job.payload data: job.payload
@ -68,9 +100,74 @@ module.exports = {
WIKI.logger.warn(err) WIKI.logger.warn(err)
} }
}, },
async addScheduled () {
try {
await WIKI.db.knex.transaction(async trx => {
// -> Acquire lock
const jobLock = await trx('jobLock')
.where(
'key',
WIKI.db.knex('jobLock')
.select('key')
.where('key', 'cron')
.andWhere('lastCheckedAt', '<=', DateTime.utc().minus({ minutes: 5 }).toISO())
.forUpdate()
.skipLocked()
.limit(1)
).update({
lastCheckedBy: WIKI.INSTANCE_ID,
lastCheckedAt: DateTime.utc().toISO()
})
if (jobLock > 0) {
WIKI.logger.info(`Scheduling future planned jobs...`)
const scheduledJobs = await WIKI.db.knex('jobSchedule')
if (scheduledJobs?.length > 0) {
// -> Get existing scheduled jobs
const existingJobs = await WIKI.db.knex('jobs').where('isScheduled', true)
for (const job of scheduledJobs) {
// -> Get next planned iterations
const plannedIterations = cronparser.parseExpression(job.cron, {
startDate: DateTime.utc().toJSDate(),
endDate: DateTime.utc().plus({ days: 1, minutes: 5 }).toJSDate(),
iterator: true,
tz: 'UTC'
})
// -> Add a maximum of 10 future iterations for a single task
let addedFutureJobs = 0
while (true) {
try {
const next = plannedIterations.next()
// -> Ensure this iteration isn't already scheduled
if (!existingJobs.some(j => j.task === job.task && j.waitUntil.getTime() === next.value.getTime())) {
this.addJob({
task: job.task,
useWorker: !(typeof this.tasks[job.task] === 'function'),
payload: job.payload,
isScheduled: true,
waitUntil: next.value.toISOString(),
notify: false
})
addedFutureJobs++
}
// -> No more iterations for this period or max iterations count reached
if (next.done || addedFutureJobs >= 10) { break }
} catch (err) {
break
}
}
}
}
}
})
} catch (err) {
WIKI.logger.warn(err)
}
},
async stop () { async stop () {
WIKI.logger.info('Stopping Scheduler...') WIKI.logger.info('Stopping Scheduler...')
await this.pool.destroy() clearInterval(this.scheduledRef)
clearInterval(this.pollingRef)
await this.workerPool.destroy()
WIKI.logger.info('Scheduler: [ STOPPED ]') WIKI.logger.info('Scheduler: [ STOPPED ]')
} }
} }

@ -1,6 +1,7 @@
const { v4: uuid } = require('uuid') const { v4: uuid } = require('uuid')
const bcrypt = require('bcryptjs-then') const bcrypt = require('bcryptjs-then')
const crypto = require('crypto') const crypto = require('crypto')
const { DateTime } = require('luxon')
const pem2jwk = require('pem-jwk').pem2jwk const pem2jwk = require('pem-jwk').pem2jwk
exports.up = async knex => { exports.up = async knex => {
@ -120,16 +121,6 @@ exports.up = async knex => {
table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now()) table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now()) table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
}) })
// JOB SCHEDULE ------------------------
.createTable('jobSchedule', table => {
table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
table.string('task').notNullable()
table.string('cron').notNullable()
table.string('type').notNullable().defaultTo('system')
table.jsonb('payload')
table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
})
// JOB HISTORY ------------------------- // JOB HISTORY -------------------------
.createTable('jobHistory', table => { .createTable('jobHistory', table => {
table.uuid('id').notNullable().primary() table.uuid('id').notNullable().primary()
@ -141,6 +132,22 @@ exports.up = async knex => {
table.timestamp('startedAt').notNullable() table.timestamp('startedAt').notNullable()
table.timestamp('completedAt').notNullable().defaultTo(knex.fn.now()) table.timestamp('completedAt').notNullable().defaultTo(knex.fn.now())
}) })
// JOB SCHEDULE ------------------------
.createTable('jobSchedule', table => {
table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
table.string('task').notNullable()
table.string('cron').notNullable()
table.string('type').notNullable().defaultTo('system')
table.jsonb('payload')
table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
})
// JOB SCHEDULE ------------------------
.createTable('jobLock', table => {
table.string('key').notNullable().primary()
table.string('lastCheckedBy')
table.timestamp('lastCheckedAt').notNullable().defaultTo(knex.fn.now())
})
// JOBS -------------------------------- // JOBS --------------------------------
.createTable('jobs', table => { .createTable('jobs', table => {
table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()')) table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
@ -148,6 +155,8 @@ exports.up = async knex => {
table.boolean('useWorker').notNullable().defaultTo(false) table.boolean('useWorker').notNullable().defaultTo(false)
table.jsonb('payload') table.jsonb('payload')
table.timestamp('waitUntil') table.timestamp('waitUntil')
table.boolean('isScheduled').notNullable().defaultTo(false)
table.string('createdBy')
table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now()) table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now()) table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
}) })
@ -680,6 +689,12 @@ exports.up = async knex => {
} }
]) ])
await knex('jobLock').insert({
key: 'cron',
lastCheckedBy: 'init',
lastCheckedAt: DateTime.utc().minus({ hours: 1 }).toISO()
})
WIKI.logger.info('Completed 3.0.0 database migration.') WIKI.logger.info('Completed 3.0.0 database migration.')
} }

@ -20,7 +20,8 @@ module.exports = {
*/ */
parsePath (rawPath, opts = {}) { parsePath (rawPath, opts = {}) {
let pathObj = { let pathObj = {
locale: WIKI.config.lang.code, // TODO: use site base lang
locale: 'en', // WIKI.config.lang.code,
path: 'home', path: 'home',
private: false, private: false,
privateNS: '', privateNS: '',

@ -38,20 +38,6 @@ html(lang=siteConfig.lang)
script. script.
siteConfig.devMode = true siteConfig.devMode = true
//- Icon Set
if config.theming.iconset === 'fa'
link(
type='text/css'
rel='stylesheet'
href='https://use.fontawesome.com/releases/v5.10.0/css/all.css'
)
else if config.theming.iconset === 'fa4'
link(
type='text/css'
rel='stylesheet'
href='https://cdn.jsdelivr.net/npm/font-awesome@4.7.0/css/font-awesome.min.css'
)
//- CSS //- CSS
@ -68,14 +54,14 @@ html(lang=siteConfig.lang)
script( script(
type='text/javascript' type='text/javascript'
src='/_assets-legacy/js/runtime.js?1662846772' src='/_assets-legacy/js/runtime.js?1664769154'
) )
script( script(
type='text/javascript' type='text/javascript'
src='/_assets-legacy/js/app.js?1662846772' src='/_assets-legacy/js/app.js?1664769154'
) )

@ -134,7 +134,6 @@ module.exports = async () => {
// View accessible data // View accessible data
// ---------------------------------------- // ----------------------------------------
app.locals.siteConfig = {}
app.locals.analyticsCode = {} app.locals.analyticsCode = {}
app.locals.basedir = WIKI.ROOTPATH app.locals.basedir = WIKI.ROOTPATH
app.locals.config = WIKI.config app.locals.config = WIKI.config
@ -173,6 +172,9 @@ module.exports = async () => {
rtl: false, // TODO: handle RTL rtl: false, // TODO: handle RTL
company: currentSite.config.company, company: currentSite.config.company,
contentLicense: currentSite.config.contentLicense contentLicense: currentSite.config.contentLicense
}
res.locals.theming = {
} }
res.locals.langs = await WIKI.db.locales.getNavLocales({ cache: true }) res.locals.langs = await WIKI.db.locales.getNavLocales({ cache: true })
res.locals.analyticsCode = await WIKI.db.analytics.getCode({ cache: true }) res.locals.analyticsCode = await WIKI.db.analytics.getCode({ cache: true })

Loading…
Cancel
Save