feat: admin scheduler + worker

pull/6078/head
Nicolas Giard 2 years ago
parent 24ddab73fc
commit 7aa4a0e9ee
No known key found for this signature in database
GPG Key ID: 85061B8F9D55B7C8

@ -35,6 +35,7 @@ defaults:
scheduledCheck: 300
maxRetries: 5
retryBackoff: 60
historyExpiration: 90000
# DB defaults
api:
isEnabled: false

@ -9,7 +9,7 @@ module.exports = {
/**
* Load root config from disk
*/
init() {
init(silent = false) {
let confPaths = {
config: path.join(WIKI.ROOTPATH, 'config.yml'),
data: path.join(WIKI.SERVERPATH, 'app/data.yml'),
@ -24,7 +24,9 @@ module.exports = {
confPaths.config = path.resolve(WIKI.ROOTPATH, process.env.CONFIG_FILE)
}
process.stdout.write(chalk.blue(`Loading configuration from ${confPaths.config}... `))
if (!silent) {
process.stdout.write(chalk.blue(`Loading configuration from ${confPaths.config}... `))
}
let appconfig = {}
let appdata = {}
@ -37,7 +39,9 @@ module.exports = {
)
appdata = yaml.load(fs.readFileSync(confPaths.data, 'utf8'))
appdata.regex = require(confPaths.dataRegex)
console.info(chalk.green.bold(`OK`))
if (!silent) {
console.info(chalk.green.bold(`OK`))
}
} catch (err) {
console.error(chalk.red.bold(`FAILED`))
console.error(err.message)
@ -66,7 +70,9 @@ module.exports = {
// Load DB Password from Docker Secret File
if (process.env.DB_PASS_FILE) {
console.info(chalk.blue(`DB_PASS_FILE is defined. Will use secret from file.`))
if (!silent) {
console.info(chalk.blue(`DB_PASS_FILE is defined. Will use secret from file.`))
}
try {
appconfig.db.pass = fs.readFileSync(process.env.DB_PASS_FILE, 'utf8').trim()
} catch (err) {

@ -4,6 +4,7 @@ const path = require('path')
const Knex = require('knex')
const fs = require('fs')
const Objection = require('objection')
const PGPubSub = require('pg-pubsub')
const migrationSource = require('../db/migrator-source')
const migrateFromLegacy = require('../db/legacy')
@ -87,7 +88,7 @@ module.exports = {
...WIKI.config.pool,
async afterCreate(conn, done) {
// -> Set Connection App Name
await conn.query(`set application_name = 'Wiki.js'`)
await conn.query(`set application_name = 'Wiki.js - ${WIKI.INSTANCE_ID}:MAIN'`)
done()
}
},
@ -159,9 +160,18 @@ module.exports = {
* Subscribe to database LISTEN / NOTIFY for multi-instances events
*/
async subscribeToNotifications () {
const PGPubSub = require('pg-pubsub')
this.listener = new PGPubSub(this.knex.client.connectionSettings, {
let connSettings = this.knex.client.connectionSettings
if (typeof connSettings === 'string') {
const encodedName = encodeURIComponent(`Wiki.js - ${WIKI.INSTANCE_ID}:PSUB`)
if (connSettings.indexOf('?') > 0) {
connSettings = `${connSettings}&ApplicationName=${encodedName}`
} else {
connSettings = `${connSettings}?ApplicationName=${encodedName}`
}
} else {
connSettings.application_name = `Wiki.js - ${WIKI.INSTANCE_ID}:PSUB`
}
this.listener = new PGPubSub(connSettings, {
log (ev) {
WIKI.logger.debug(ev)
}

@ -13,7 +13,8 @@ module.exports = {
scheduledRef: null,
tasks: null,
async init () {
this.maxWorkers = WIKI.config.scheduler.workers === 'auto' ? os.cpus().length : WIKI.config.scheduler.workers
this.maxWorkers = WIKI.config.scheduler.workers === 'auto' ? (os.cpus().length - 1) : WIKI.config.scheduler.workers
if (this.maxWorkers < 1) { this.maxWorkers = 1 }
WIKI.logger.info(`Initializing Worker Pool (Limit: ${this.maxWorkers})...`)
this.workerPool = new DynamicThreadPool(1, this.maxWorkers, './server/worker.js', {
errorHandler: (err) => WIKI.logger.warn(err),
@ -77,80 +78,87 @@ module.exports = {
}
},
async processJob () {
let jobId = null
let jobIds = []
try {
const availableWorkers = this.maxWorkers - this.activeWorkers
if (availableWorkers < 1) {
WIKI.logger.debug('All workers are busy. Cannot process more jobs at the moment.')
return
}
await WIKI.db.knex.transaction(async trx => {
const jobs = await trx('jobs')
.where('id', WIKI.db.knex.raw('(SELECT id FROM jobs WHERE ("waitUntil" IS NULL OR "waitUntil" <= NOW()) ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1)'))
.whereIn('id', WIKI.db.knex.raw(`(SELECT id FROM jobs WHERE ("waitUntil" IS NULL OR "waitUntil" <= NOW()) ORDER BY id FOR UPDATE SKIP LOCKED LIMIT ${availableWorkers})`))
.returning('*')
.del()
if (jobs && jobs.length === 1) {
const job = jobs[0]
WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`)
jobId = job.id
// -> Add to Job History
await WIKI.db.knex('jobHistory').insert({
id: job.id,
task: job.task,
state: 'active',
useWorker: job.useWorker,
wasScheduled: job.isScheduled,
payload: job.payload,
attempt: job.retries + 1,
maxRetries: job.maxRetries,
createdAt: job.createdAt
}).onConflict('id').merge({
startedAt: new Date()
})
// -> Start working on it
try {
if (job.useWorker) {
await this.workerPool.execute({
id: job.id,
name: job.task,
data: job.payload
})
} else {
await this.tasks[job.task](job.payload)
}
// -> Update job history (success)
await WIKI.db.knex('jobHistory').where({
id: job.id
}).update({
state: 'completed',
completedAt: new Date()
})
WIKI.logger.info(`Completed job ${job.id}: ${job.task} [ SUCCESS ]`)
} catch (err) {
WIKI.logger.warn(`Failed to complete job ${job.id}: ${job.task} [ FAILED ]`)
WIKI.logger.warn(err)
// -> Update job history (fail)
await WIKI.db.knex('jobHistory').where({
id: job.id
}).update({
state: 'failed',
lastErrorMessage: err.message
if (jobs && jobs.length > 0) {
for (const job of jobs) {
WIKI.logger.info(`Processing new job ${job.id}: ${job.task}...`)
// -> Add to Job History
await WIKI.db.knex('jobHistory').insert({
id: job.id,
task: job.task,
state: 'active',
useWorker: job.useWorker,
wasScheduled: job.isScheduled,
payload: job.payload,
attempt: job.retries + 1,
maxRetries: job.maxRetries,
executedBy: WIKI.INSTANCE_ID,
createdAt: job.createdAt
}).onConflict('id').merge({
executedBy: WIKI.INSTANCE_ID,
startedAt: new Date()
})
// -> Reschedule for retry
if (job.retries < job.maxRetries) {
const backoffDelay = (2 ** job.retries) * WIKI.config.scheduler.retryBackoff
await trx('jobs').insert({
...job,
retries: job.retries + 1,
waitUntil: DateTime.utc().plus({ seconds: backoffDelay }).toJSDate(),
updatedAt: new Date()
jobIds.push(job.id)
// -> Start working on it
try {
if (job.useWorker) {
await this.workerPool.execute({
...job,
INSTANCE_ID: `${WIKI.INSTANCE_ID}:WKR`
})
} else {
await this.tasks[job.task](job.payload)
}
// -> Update job history (success)
await WIKI.db.knex('jobHistory').where({
id: job.id
}).update({
state: 'completed',
completedAt: new Date()
})
WIKI.logger.info(`Completed job ${job.id}: ${job.task}`)
} catch (err) {
WIKI.logger.warn(`Failed to complete job ${job.id}: ${job.task} [ FAILED ]`)
WIKI.logger.warn(err)
// -> Update job history (fail)
await WIKI.db.knex('jobHistory').where({
id: job.id
}).update({
state: 'failed',
lastErrorMessage: err.message
})
WIKI.logger.warn(`Rescheduling new attempt for job ${job.id}: ${job.task}...`)
// -> Reschedule for retry
if (job.retries < job.maxRetries) {
const backoffDelay = (2 ** job.retries) * WIKI.config.scheduler.retryBackoff
await trx('jobs').insert({
...job,
retries: job.retries + 1,
waitUntil: DateTime.utc().plus({ seconds: backoffDelay }).toJSDate(),
updatedAt: new Date()
})
WIKI.logger.warn(`Rescheduling new attempt for job ${job.id}: ${job.task}...`)
}
}
}
}
})
} catch (err) {
WIKI.logger.warn(err)
if (jobId) {
WIKI.db.knex('jobHistory').where({
id: jobId
}).update({
if (jobIds && jobIds.length > 0) {
WIKI.db.knex('jobHistory').whereIn('id', jobIds).update({
state: 'interrupted',
lastErrorMessage: err.message
})
@ -181,6 +189,7 @@ module.exports = {
if (scheduledJobs?.length > 0) {
// -> Get existing scheduled jobs
const existingJobs = await WIKI.db.knex('jobs').where('isScheduled', true)
let totalAdded = 0
for (const job of scheduledJobs) {
// -> Get next planned iterations
const plannedIterations = cronparser.parseExpression(job.cron, {
@ -205,6 +214,7 @@ module.exports = {
notify: false
})
addedFutureJobs++
totalAdded++
}
// -> No more iterations for this period or max iterations count reached
if (next.done || addedFutureJobs >= 10) { break }
@ -213,6 +223,11 @@ module.exports = {
}
}
}
if (totalAdded > 0) {
WIKI.logger.info(`Scheduled ${totalAdded} new future planned jobs: [ OK ]`)
} else {
WIKI.logger.info(`No new future planned jobs to schedule: [ OK ]`)
}
}
}
})

@ -132,6 +132,7 @@ exports.up = async knex => {
table.integer('attempt').notNullable().defaultTo(1)
table.integer('maxRetries').notNullable().defaultTo(0)
table.text('lastErrorMessage')
table.string('executedBy')
table.timestamp('createdAt').notNullable()
table.timestamp('startedAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('completedAt')
@ -684,12 +685,17 @@ exports.up = async knex => {
await knex('jobSchedule').insert([
{
task: 'updateLocales',
task: 'checkVersion',
cron: '0 0 * * *',
type: 'system'
},
{
task: 'checkVersion',
task: 'cleanJobHistory',
cron: '5 0 * * *',
type: 'system'
},
{
task: 'updateLocales',
cron: '0 0 * * *',
type: 'system'
}

@ -27,25 +27,13 @@ module.exports = {
return WIKI.config.security
},
async systemJobs (obj, args) {
switch (args.state) {
case 'ACTIVE': {
// const result = await WIKI.scheduler.boss.fetch('*', 25, { includeMeta: true })
return []
}
case 'COMPLETED': {
return []
}
case 'FAILED': {
return []
}
case 'INTERRUPTED': {
return []
}
default: {
WIKI.logger.warn('Invalid Job State requested.')
return []
}
}
const results = args.states?.length > 0 ?
await WIKI.db.knex('jobHistory').whereIn('state', args.states.map(s => s.toLowerCase())).orderBy('startedAt') :
await WIKI.db.knex('jobHistory').orderBy('startedAt')
return results.map(r => ({
...r,
state: r.state.toUpperCase()
}))
},
async systemJobsScheduled (obj, args) {
return WIKI.db.knex('jobSchedule').orderBy('task')

@ -8,7 +8,7 @@ extend type Query {
systemInfo: SystemInfo
systemSecurity: SystemSecurity
systemJobs(
state: SystemJobState
states: [SystemJobState]
): [SystemJob]
systemJobsScheduled: [SystemJobScheduled]
systemJobsUpcoming: [SystemJobUpcoming]
@ -159,6 +159,7 @@ type SystemJob {
attempt: Int
maxRetries: Int
lastErrorMessage: String
executedBy: String
createdAt: Date
startedAt: Date
completedAt: Date

@ -0,0 +1,17 @@
const { DateTime } = require('luxon')
module.exports = async (payload) => {
WIKI.logger.info('Cleaning scheduler job history...')
try {
await WIKI.db.knex('jobHistory')
.whereNot('state', 'active')
.andWhere('startedAt', '<=', DateTime.utc().minus({ seconds: WIKI.config.scheduler.historyExpiration }).toISO())
.del()
WIKI.logger.info('Cleaned scheduler job history: [ COMPLETED ]')
} catch (err) {
WIKI.logger.error('Cleaning scheduler job history: [ FAILED ]')
WIKI.logger.error(err.message)
}
}

@ -2,8 +2,8 @@ const path = require('node:path')
const fs = require('fs-extra')
const { DateTime } = require('luxon')
module.exports = async (payload, helpers) => {
helpers.logger.info('Purging orphaned upload files...')
module.exports = async ({ payload }) => {
WIKI.logger.info('Purging orphaned upload files...')
try {
const uplTempPath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'uploads')
@ -18,9 +18,9 @@ module.exports = async (payload, helpers) => {
}
}
helpers.logger.info('Purging orphaned upload files: [ COMPLETED ]')
WIKI.logger.info('Purging orphaned upload files: [ COMPLETED ]')
} catch (err) {
helpers.logger.error('Purging orphaned upload files: [ FAILED ]')
helpers.logger.error(err.message)
WIKI.logger.error('Purging orphaned upload files: [ FAILED ]')
WIKI.logger.error(err.message)
}
}

@ -1,6 +1,31 @@
const { ThreadWorker } = require('poolifier')
const { kebabCase } = require('lodash')
const path = require('node:path')
// ----------------------------------------
// Init Minimal Core
// ----------------------------------------
let WIKI = {
IS_DEBUG: process.env.NODE_ENV === 'development',
ROOTPATH: process.cwd(),
INSTANCE_ID: 'worker',
SERVERPATH: path.join(process.cwd(), 'server'),
Error: require('./helpers/error'),
configSvc: require('./core/config')
}
global.WIKI = WIKI
WIKI.configSvc.init(true)
WIKI.logger = require('./core/logger').init()
// ----------------------------------------
// Execute Task
// ----------------------------------------
module.exports = new ThreadWorker(async (job) => {
// TODO: Call external task file
return { ok: true }
WIKI.INSTANCE_ID = job.INSTANCE_ID
const task = require(`./tasks/workers/${kebabCase(job.task)}.js`)
await task(job)
return true
}, { async: true })

@ -1523,7 +1523,16 @@
"admin.scheduler.updatedAt": "Last Updated",
"common.field.task": "Task",
"admin.scheduler.upcomingNone": "There are no upcoming job for the moment.",
"admin.scheduler.failedNone": "There are no recently failed job to display.",
"admin.scheduler.waitUntil": "Start",
"admin.scheduler.attempt": "Attempt",
"admin.scheduler.useWorker": "Execution Mode"
"admin.scheduler.useWorker": "Execution Mode",
"admin.scheduler.schedule": "Schedule",
"admin.scheduler.state": "State",
"admin.scheduler.startedAt": "Started",
"admin.scheduler.result": "Result",
"admin.scheduler.completedIn": "Completed in {duration}",
"admin.scheduler.pending": "Pending",
"admin.scheduler.error": "Error",
"admin.scheduler.interrupted": "Interrupted"
}

@ -17,8 +17,9 @@ q-page.admin-terminal
:text-color='$q.dark.isActive ? `white` : `black`'
:color='$q.dark.isActive ? `dark-1` : `white`'
:options=`[
{ label: t('admin.scheduler.scheduled'), value: 'scheduled' },
{ label: t('admin.scheduler.schedule'), value: 'scheduled' },
{ label: t('admin.scheduler.upcoming'), value: 'upcoming' },
{ label: t('admin.scheduler.active'), value: 'active' },
{ label: t('admin.scheduler.completed'), value: 'completed' },
{ label: t('admin.scheduler.failed'), value: 'failed' },
]`
@ -67,14 +68,37 @@ q-page.admin-terminal
color='indigo'
size='xs'
)
//- q-icon(name='las la-stopwatch', color='primary', size='sm')
template(v-slot:body-cell-task='props')
q-td(:props='props')
strong {{props.value}}
div: small.text-grey {{props.row.id}}
template(v-slot:body-cell-cron='props')
q-td(:props='props')
span {{ props.value }}
q-chip(
square
size='md'
color='blue'
text-color='white'
)
span.font-robotomono {{ props.value }}
template(v-slot:body-cell-type='props')
q-td(:props='props')
q-chip(
square
size='md'
dense
color='deep-orange'
text-color='white'
)
small.text-uppercase {{ props.value }}
template(v-slot:body-cell-created='props')
q-td(:props='props')
span {{props.value}}
div: small.text-grey {{humanizeDate(props.row.createdAt)}}
template(v-slot:body-cell-updated='props')
q-td(:props='props')
span {{props.value}}
div: small.text-grey {{humanizeDate(props.row.updatedAt)}}
template(v-else-if='state.displayMode === `upcoming`')
q-card.rounded-borders(
v-if='state.upcomingJobs.length < 1'
@ -97,7 +121,7 @@ q-page.admin-terminal
)
template(v-slot:body-cell-id='props')
q-td(:props='props')
q-icon(name='las la-chess-knight', color='primary', size='sm')
q-icon(name='las la-clock', color='primary', size='sm')
template(v-slot:body-cell-task='props')
q-td(:props='props')
strong {{props.value}}
@ -133,8 +157,87 @@ q-page.admin-terminal
q-card-section.items-center(horizontal)
q-card-section.col-auto.q-pr-none
q-icon(name='las la-info-circle', size='sm')
q-card-section.text-caption {{ t('admin.scheduler.completedNone') }}
q-card.shadow-1(v-else) ---
q-card-section.text-caption {{ t('admin.scheduler.' + state.displayMode + 'None') }}
q-card.shadow-1(v-else)
q-table(
:rows='state.jobs'
:columns='jobsHeaders'
row-key='name'
flat
hide-bottom
:rows-per-page-options='[0]'
:loading='state.loading > 0'
)
template(v-slot:body-cell-id='props')
q-td(:props='props')
q-avatar(
v-if='props.row.state === `completed`'
icon='las la-check'
color='positive'
text-color='white'
size='sm'
rounded
)
q-avatar(
v-else-if='props.row.state === `failed`'
icon='las la-times'
color='negative'
text-color='white'
size='sm'
rounded
)
q-avatar(
v-else-if='props.row.state === `interrupted`'
icon='las la-square-full'
color='orange'
text-color='white'
size='sm'
rounded
)
q-circular-progress(
v-else-if='props.row.state === `active`'
indeterminate
size='sm'
:thickness='0.4'
color='blue'
track-color='blue-1'
center-color='blue-2'
)
template(v-slot:body-cell-task='props')
q-td(:props='props')
strong {{props.value}}
div: small.text-grey {{props.row.id}}
template(v-slot:body-cell-state='props')
q-td(:props='props')
template(v-if='props.value === `completed`')
i18n-t(keypath='admin.scheduler.completedIn', tag='span')
template(#duration)
strong {{humanizeDuration(props.row.startedAt, props.row.completedAt)}}
div: small.text-grey {{ humanizeDate(props.row.completedAt) }}
template(v-else-if='props.value === `active`')
em.text-grey {{ t('admin.scheduler.pending') }}
template(v-else)
strong.text-negative {{ props.value === 'failed' ? t('admin.scheduler.error') : t('admin.scheduler.interrupted') }}
div: small {{ props.row.lastErrorMessage }}
template(v-slot:body-cell-attempt='props')
q-td(:props='props')
span #[strong {{props.value}}] #[span.text-grey / {{props.row.maxRetries}}]
template(v-slot:body-cell-useworker='props')
q-td(:props='props')
template(v-if='props.value')
q-icon(name='las la-microchip', color='brown', size='sm')
small.q-ml-xs.text-brown Worker
template(v-else)
q-icon(name='las la-leaf', color='teal', size='sm')
small.q-ml-xs.text-teal In-Process
template(v-slot:body-cell-date='props')
q-td(:props='props')
span {{props.value}}
div: small.text-grey {{humanizeDate(props.row.startedAt)}}
div
i18n-t.text-grey(keypath='admin.scheduler.createdBy', tag='small')
template(#instance)
strong {{props.row.executedBy}}
</template>
@ -143,7 +246,7 @@ import { onMounted, reactive, watch } from 'vue'
import { useMeta, useQuasar } from 'quasar'
import { useI18n } from 'vue-i18n'
import gql from 'graphql-tag'
import { DateTime } from 'luxon'
import { DateTime, Duration, Interval } from 'luxon'
import { useSiteStore } from 'src/stores/site'
@ -168,7 +271,7 @@ useMeta({
// DATA
const state = reactive({
displayMode: 'upcoming',
displayMode: 'completed',
scheduledJobs: [],
upcomingJobs: [],
jobs: [],
@ -260,7 +363,7 @@ const upcomingJobsHeaders = [
sortable: true
},
{
label: t('admin.scheduler.createdAt'),
label: t('admin.scheduler.scheduled'),
align: 'left',
field: 'createdAt',
name: 'date',
@ -269,6 +372,52 @@ const upcomingJobsHeaders = [
}
]
const jobsHeaders = [
{
align: 'center',
field: 'id',
name: 'id',
sortable: false,
style: 'width: 15px; padding-right: 0;'
},
{
label: t('common.field.task'),
align: 'left',
field: 'task',
name: 'task',
sortable: true
},
{
label: t('admin.scheduler.result'),
align: 'left',
field: 'state',
name: 'state',
sortable: true
},
{
label: t('admin.scheduler.attempt'),
align: 'left',
field: 'attempt',
name: 'attempt',
sortable: true
},
{
label: t('admin.scheduler.useWorker'),
align: 'left',
field: 'useWorker',
name: 'useworker',
sortable: true
},
{
label: t('admin.scheduler.startedAt'),
align: 'left',
field: 'startedAt',
name: 'date',
sortable: true,
format: v => DateTime.fromISO(v).toRelative()
}
]
// WATCHERS
watch(() => state.displayMode, (newValue) => {
@ -281,6 +430,17 @@ function humanizeDate (val) {
return DateTime.fromISO(val).toFormat('fff')
}
function humanizeDuration (start, end) {
const dur = Interval.fromDateTimes(DateTime.fromISO(start), DateTime.fromISO(end))
.toDuration(['hours', 'minutes', 'seconds', 'milliseconds'])
return Duration.fromObject({
...dur.hours > 0 && { hours: dur.hours },
...dur.minutes > 0 && { minutes: dur.minutes },
...dur.seconds > 0 && { seconds: dur.seconds },
...dur.milliseconds > 0 && { milliseconds: dur.milliseconds }
}).toHuman({ unitDisplay: 'narrow', listStyle: 'short' })
}
async function load () {
state.loading++
try {
@ -323,27 +483,36 @@ async function load () {
})
state.upcomingJobs = resp?.data?.systemJobsUpcoming
} else {
const states = state.displayMode === 'failed' ? ['FAILED', 'INTERRUPTED'] : [state.displayMode.toUpperCase()]
const resp = await APOLLO_CLIENT.query({
query: gql`
query getSystemJobs (
$state: SystemJobState!
$states: [SystemJobState]
) {
systemJobs (
state: $state
states: $states
) {
id
name
priority
state
id
task
state
useWorker
wasScheduled
attempt
maxRetries
lastErrorMessage
executedBy
createdAt
startedAt
completedAt
}
}
`,
variables: {
state: state.displayMode.toUpperCase()
states
},
fetchPolicy: 'network-only'
})
state.jobs = resp?.data?.systemJobs
state.jobs = resp?.data?.systemJobs?.map(j => ({ ...j, state: j.state.toLowerCase() }))
}
} catch (err) {
$q.notify({
@ -361,15 +530,3 @@ onMounted(() => {
load()
})
</script>
<style lang='scss'>
.admin-terminal {
&-term {
width: 100%;
background-color: #000;
border-radius: 5px;
overflow: hidden;
padding: 10px;
}
}
</style>

Loading…
Cancel
Save