feat: scheduler + admin page

pull/5698/head
Nicolas Giard 2 years ago
parent e1ebaf5b31
commit 7be5943c26
No known key found for this signature in database
GPG Key ID: 85061B8F9D55B7C8

@ -64,6 +64,7 @@
"connect-session-knex": "3.0.0",
"cookie-parser": "1.4.6",
"cors": "2.8.5",
"cron-parser": "4.6.0",
"cuint": "0.2.2",
"custom-error-instance": "2.1.2",
"dependency-graph": "0.9.0",

@ -80,17 +80,17 @@ defaults:
jobs:
purgeUploads:
onInit: true
schedule: PT15M
schedule: '*/15 * * * *'
offlineSkip: false
repeat: true
syncGraphLocales:
onInit: true
schedule: P1D
schedule: '0 0 * * *'
offlineSkip: true
repeat: true
syncGraphUpdates:
onInit: true
schedule: P1D
schedule: '0 0 * * *'
offlineSkip: true
repeat: true
rebuildTree:

@ -26,13 +26,13 @@ router.get('/.well-known/acme-challenge/:token', (req, res, next) => {
/**
* Redirect to HTTPS if HTTP Redirection is enabled
*/
router.all('/*', (req, res, next) => {
if (WIKI.config.server.sslRedir && !req.secure && WIKI.servers.servers.https) {
let query = (!_.isEmpty(req.query)) ? `?${qs.stringify(req.query)}` : ``
return res.redirect(`https://${req.hostname}${req.originalUrl}${query}`)
} else {
next()
}
})
// router.all('/*', (req, res, next) => {
// if (WIKI.config.server.sslRedir && !req.secure && WIKI.servers.servers.https) {
// let query = (!_.isEmpty(req.query)) ? `?${qs.stringify(req.query)}` : ``
// return res.redirect(`https://${req.hostname}${req.originalUrl}${query}`)
// } else {
// next()
// }
// })
module.exports = router

@ -76,7 +76,8 @@ module.exports = {
await WIKI.models.commentProviders.initProvider()
await WIKI.models.sites.reloadCache()
await WIKI.models.storage.initTargets()
WIKI.scheduler.start()
await WIKI.scheduler.start()
await WIKI.scheduler.registerScheduledJobs()
await WIKI.models.subscribeToNotifications()
},

@ -6,10 +6,11 @@ const os = require('node:os')
module.exports = {
pool: null,
scheduler: null,
boss: null,
maxWorkers: 1,
async init () {
WIKI.logger.info('Initializing Scheduler...')
this.scheduler = new PgBoss({
this.boss = new PgBoss({
db: {
close: () => Promise.resolve('ok'),
executeSql: async (text, values) => {
@ -27,12 +28,14 @@ module.exports = {
// ...WIKI.models.knex.client.connectionSettings,
application_name: 'Wiki.js Scheduler',
schema: WIKI.config.db.schemas.scheduler,
uuid: 'v4'
uuid: 'v4',
archiveCompletedAfterSeconds: 120,
deleteAfterHours: 24
})
const maxWorkers = WIKI.config.workers === 'auto' ? os.cpus().length : WIKI.config.workers
WIKI.logger.info(`Initializing Worker Pool (Max ${maxWorkers})...`)
this.pool = new DynamicThreadPool(1, maxWorkers, './server/worker.js', {
this.maxWorkers = WIKI.config.workers === 'auto' ? os.cpus().length : WIKI.config.workers
WIKI.logger.info(`Initializing Worker Pool (Limit: ${this.maxWorkers})...`)
this.pool = new DynamicThreadPool(1, this.maxWorkers, './server/worker.js', {
errorHandler: (err) => WIKI.logger.warn(err),
exitHandler: () => WIKI.logger.debug('A worker has gone offline.'),
onlineHandler: () => WIKI.logger.debug('New worker is online.')
@ -41,20 +44,40 @@ module.exports = {
},
async start () {
WIKI.logger.info('Starting Scheduler...')
await this.scheduler.start()
this.scheduler.work('*', async job => {
return this.pool.execute({
id: job.id,
name: job.name,
data: job.data
})
await this.boss.start()
this.boss.work('wk-*', {
teamSize: this.maxWorkers,
teamConcurrency: this.maxWorkers
}, async job => {
WIKI.logger.debug(`Starting job ${job.name}:${job.id}...`)
try {
const result = await this.pool.execute({
id: job.id,
name: job.name,
data: job.data
})
WIKI.logger.debug(`Completed job ${job.name}:${job.id}.`)
job.done(null, result)
} catch (err) {
WIKI.logger.warn(`Failed job ${job.name}:${job.id}): ${err.message}`)
job.done(err)
}
this.boss.complete(job.id)
})
WIKI.logger.info('Scheduler: [ STARTED ]')
},
async stop () {
WIKI.logger.info('Stopping Scheduler...')
await this.scheduler.stop()
await this.boss.stop({ timeout: 5000 })
await this.pool.destroy()
WIKI.logger.info('Scheduler: [ STOPPED ]')
},
async registerScheduledJobs () {
for (const [key, job] of Object.entries(WIKI.data.jobs)) {
if (job.schedule) {
WIKI.logger.debug(`Scheduling regular job ${key}...`)
await this.boss.schedule(`wk-${key}`, job.schedule)
}
}
}
}

@ -7,6 +7,7 @@ const path = require('path')
const fs = require('fs-extra')
const { DateTime } = require('luxon')
const graphHelper = require('../../helpers/graph')
const cronParser = require('cron-parser')
/* global WIKI */
@ -27,6 +28,35 @@ module.exports = {
},
systemSecurity () {
return WIKI.config.security
},
async systemJobs (obj, args) {
switch (args.type) {
case 'ACTIVE': {
// const result = await WIKI.scheduler.boss.fetch('*', 25, { includeMeta: true })
return []
}
case 'COMPLETED': {
const result = await WIKI.scheduler.boss.fetchCompleted('*', 25, { includeMeta: true })
console.info(result)
return result ?? []
}
default: {
WIKI.logger.warn('Invalid Job Type requested.')
return []
}
}
},
async systemScheduledJobs (obj, args) {
const jobs = await WIKI.scheduler.boss.getSchedules()
return jobs.map(job => ({
id: job.name,
name: job.name,
cron: job.cron,
timezone: job.timezone,
nextExecution: cronParser.parseExpression(job.cron, { tz: job.timezone }).next(),
createdAt: job.created_on,
updatedAt: job.updated_on
}))
}
},
Mutation: {

@ -7,6 +7,10 @@ extend type Query {
systemFlags: [SystemFlag]
systemInfo: SystemInfo
systemSecurity: SystemSecurity
systemJobs(
type: SystemJobType!
): [SystemJob]
systemScheduledJobs: [SystemScheduledJob]
}
extend type Mutation {
@ -144,3 +148,25 @@ enum SystemSecurityCorsMode {
HOSTNAMES
REGEX
}
type SystemJob {
id: UUID
name: String
priority: Int
state: String
}
type SystemScheduledJob {
id: String
name: String
cron: String
timezone: String
nextExecution: Date
createdAt: Date
updatedAt: Date
}
enum SystemJobType {
ACTIVE
COMPLETED
}

@ -39,25 +39,26 @@ module.exports = class Locale extends Model {
}
static async getNavLocales({ cache = false } = {}) {
if (!WIKI.config.lang.namespacing) {
return []
}
return []
// if (!WIKI.config.lang.namespacing) {
// return []
// }
if (cache) {
const navLocalesCached = await WIKI.cache.get('nav:locales')
if (navLocalesCached) {
return navLocalesCached
}
}
const navLocales = await WIKI.models.locales.query().select('code', 'nativeName AS name').whereIn('code', WIKI.config.lang.namespaces).orderBy('code')
if (navLocales) {
if (cache) {
await WIKI.cache.set('nav:locales', navLocales, 300)
}
return navLocales
} else {
WIKI.logger.warn('Site Locales for navigation are missing or corrupted.')
return []
}
// if (cache) {
// const navLocalesCached = await WIKI.cache.get('nav:locales')
// if (navLocalesCached) {
// return navLocalesCached
// }
// }
// const navLocales = await WIKI.models.locales.query().select('code', 'nativeName AS name').whereIn('code', WIKI.config.lang.namespaces).orderBy('code')
// if (navLocales) {
// if (cache) {
// await WIKI.cache.set('nav:locales', navLocales, 300)
// }
// return navLocales
// } else {
// WIKI.logger.warn('Site Locales for navigation are missing or corrupted.')
// return []
// }
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 48 48" width="96px" height="96px"><defs><linearGradient id="p0leOTPLvuNkjL_fSa~qVa" x1="24" x2="24" y1="9.109" y2="13.568" data-name="Безымянный градиент 6" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0077d2"/><stop offset="1" stop-color="#0b59a2"/></linearGradient><linearGradient id="p0leOTPLvuNkjL_fSa~qVb" x1="4.5" x2="4.5" y1="26.717" y2="41.786" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/><linearGradient id="p0leOTPLvuNkjL_fSa~qVc" x1="43.5" x2="43.5" y1="26.717" y2="41.786" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/><linearGradient id="p0leOTPLvuNkjL_fSa~qVd" x1="16" x2="16" y1="25.054" y2="43.495" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/><linearGradient id="p0leOTPLvuNkjL_fSa~qVe" x1="32" x2="32" y1="25.054" y2="43.495" xlink:href="#p0leOTPLvuNkjL_fSa~qVa"/></defs><rect width="2" height="6" x="23" y="8" fill="url(#p0leOTPLvuNkjL_fSa~qVa)"/><path fill="url(#p0leOTPLvuNkjL_fSa~qVb)" d="M6,27H8a0,0,0,0,1,0,0V37a0,0,0,0,1,0,0H6a5,5,0,0,1-5-5v0A5,5,0,0,1,6,27Z"/><path fill="url(#p0leOTPLvuNkjL_fSa~qVc)" d="M40,27h2a5,5,0,0,1,5,5v0a5,5,0,0,1-5,5H40a0,0,0,0,1,0,0V27A0,0,0,0,1,40,27Z"/><path fill="#199be2" d="M24,13h0A18,18,0,0,1,42,31v8a2,2,0,0,1-2,2H8a2,2,0,0,1-2-2V31A18,18,0,0,1,24,13Z"/><circle cx="16" cy="31" r="6" fill="url(#p0leOTPLvuNkjL_fSa~qVd)"/><circle cx="32" cy="31" r="6" fill="url(#p0leOTPLvuNkjL_fSa~qVe)"/><circle cx="32" cy="31" r="4" fill="#50e6ff"/><circle cx="32" cy="31" r="2" fill="url(#p0leOTPLvuNkjL_fSa~qVe)"/><circle cx="16" cy="31" r="4" fill="#50e6ff"/><circle cx="16" cy="31" r="2" fill="url(#p0leOTPLvuNkjL_fSa~qVd)"/><circle cx="24" cy="8" r="2" fill="#199be2"/></svg>

After

Width:  |  Height:  |  Size: 1.7 KiB

@ -1505,5 +1505,17 @@
"profile.saveFailed": "Failed to save profile changes.",
"admin.users.pronouns": "Pronouns",
"admin.users.pronounsHint": "The pronouns used to address this user.",
"admin.users.appearance": "Site Appearance"
"admin.users.appearance": "Site Appearance",
"admin.scheduler.title": "Scheduler",
"admin.scheduler.subtitle": "View scheduled and completed jobs",
"admin.scheduler.active": "Active",
"admin.scheduler.completed": "Completed",
"admin.scheduler.scheduled": "Scheduled",
"admin.scheduler.activeNone": "There are no active jobs at the moment.",
"admin.scheduler.completedNone": "There are no recently completed job to display.",
"admin.scheduler.scheduledNone": "There are no scheduled jobs at the moment.",
"admin.scheduler.cron": "Cron",
"admin.scheduler.nextExecutionIn": "Next run {date}",
"admin.scheduler.nextExecution": "Next Run",
"admin.scheduler.timezone": "Timezone"
}

@ -152,6 +152,10 @@ q-layout.admin(view='hHh Lpr lff')
q-item-section {{ t('admin.mail.title') }}
q-item-section(side)
status-light(:color='adminStore.info.isMailConfigured ? `positive` : `warning`')
q-item(to='/_admin/scheduler', v-ripple, active-class='bg-primary text-white')
q-item-section(avatar)
q-icon(name='img:/_assets/icons/fluent-bot.svg')
q-item-section {{ t('admin.scheduler.title') }}
q-item(to='/_admin/security', v-ripple, active-class='bg-primary text-white')
q-item-section(avatar)
q-icon(name='img:/_assets/icons/fluent-protect.svg')

@ -0,0 +1,250 @@
<template lang='pug'>
q-page.admin-terminal
.row.q-pa-md.items-center
.col-auto
img.admin-icon.animated.fadeInLeft(src='/_assets/icons/fluent-bot.svg')
.col.q-pl-md
.text-h5.text-primary.animated.fadeInLeft {{ t('admin.scheduler.title') }}
.text-subtitle1.text-grey.animated.fadeInLeft.wait-p2s {{ t('admin.scheduler.subtitle') }}
.col-auto.flex
q-btn-toggle.q-mr-md(
v-model='state.displayMode'
push
no-caps
:disable='state.loading > 0'
:toggle-color='$q.dark.isActive ? `white` : `black`'
:toggle-text-color='$q.dark.isActive ? `black` : `white`'
:text-color='$q.dark.isActive ? `white` : `black`'
:color='$q.dark.isActive ? `dark-1` : `white`'
:options=`[
{ label: t('admin.scheduler.scheduled'), value: 'scheduled' },
{ label: t('admin.scheduler.completed'), value: 'completed' }
]`
)
q-separator.q-mr-md(vertical)
q-btn.q-mr-sm.acrylic-btn(
icon='las la-question-circle'
flat
color='grey'
:href='siteStore.docsBase + `/admin/scheduler`'
target='_blank'
type='a'
)
q-btn.q-mr-sm.acrylic-btn(
icon='las la-redo-alt'
flat
color='secondary'
:loading='state.loading > 0'
@click='load'
)
q-separator(inset)
.q-pa-md.q-gutter-md
template(v-if='state.displayMode === `scheduled`')
q-card.rounded-borders(
v-if='state.scheduledJobs.length < 1'
flat
:class='$q.dark.isActive ? `bg-dark-5 text-white` : `bg-grey-3 text-dark`'
)
q-card-section.items-center(horizontal)
q-card-section.col-auto.q-pr-none
q-icon(name='las la-info-circle', size='sm')
q-card-section.text-caption {{ t('admin.scheduler.scheduledNone') }}
q-card.shadow-1(v-else)
q-table(
:rows='state.scheduledJobs'
:columns='scheduledJobsHeaders'
row-key='name'
flat
hide-bottom
:rows-per-page-options='[0]'
:loading='state.loading > 0'
)
template(v-slot:body-cell-id='props')
q-td(:props='props')
q-spinner-clock.q-mr-sm(
color='indigo'
size='xs'
)
//- q-icon(name='las la-stopwatch', color='primary', size='sm')
template(v-slot:body-cell-name='props')
q-td(:props='props')
strong {{props.value}}
template(v-slot:body-cell-cron='props')
q-td(:props='props')
span {{ props.value }}
template(v-slot:body-cell-date='props')
q-td(:props='props')
i18n-t.text-caption(keypath='admin.scheduler.nextExecutionIn', tag='div')
template(#date)
strong {{ humanizeDate(props.value) }}
small {{props.value}}
template(v-else)
q-card.rounded-borders(
v-if='state.jobs.length < 1'
flat
:class='$q.dark.isActive ? `bg-dark-5 text-white` : `bg-grey-3 text-dark`'
)
q-card-section.items-center(horizontal)
q-card-section.col-auto.q-pr-none
q-icon(name='las la-info-circle', size='sm')
q-card-section.text-caption {{ t('admin.scheduler.completedNone') }}
q-card.shadow-1(v-else) ---
</template>
<script setup>
import { onMounted, reactive, watch } from 'vue'
import { useMeta, useQuasar } from 'quasar'
import { useI18n } from 'vue-i18n'
import gql from 'graphql-tag'
import { DateTime } from 'luxon'
import { useSiteStore } from 'src/stores/site'
// QUASAR
const $q = useQuasar()
// STORES
const siteStore = useSiteStore()
// I18N
const { t } = useI18n()
// META
useMeta({
title: t('admin.scheduler.title')
})
// DATA
const state = reactive({
displayMode: 'scheduled',
scheduledJobs: [],
jobs: [],
loading: 0
})
const scheduledJobsHeaders = [
{
align: 'center',
field: 'id',
name: 'id',
sortable: false,
style: 'width: 15px; padding-right: 0;'
},
{
label: t('common.field.name'),
align: 'left',
field: 'name',
name: 'name',
sortable: true
},
{
label: t('admin.scheduler.cron'),
align: 'left',
field: 'cron',
name: 'cron',
sortable: false
},
{
label: t('admin.scheduler.timezone'),
align: 'left',
field: 'timezone',
name: 'timezone',
sortable: false
},
{
label: t('admin.scheduler.nextExecution'),
align: 'left',
field: 'nextExecution',
name: 'date',
sortable: false
}
]
// WATCHERS
watch(() => state.displayMode, (newValue) => {
load()
})
// METHODS
async function load () {
state.loading++
try {
if (state.displayMode === 'scheduled') {
const resp = await APOLLO_CLIENT.query({
query: gql`
query getSystemScheduledJobs {
systemScheduledJobs {
id
name
cron
timezone
nextExecution
}
}
`,
fetchPolicy: 'network-only'
})
state.scheduledJobs = resp?.data?.systemScheduledJobs
} else {
const resp = await APOLLO_CLIENT.query({
query: gql`
query getSystemJobs (
$type: SystemJobType!
) {
systemJobs (
type: $type
) {
id
name
priority
state
}
}
`,
variables: {
type: state.displayMode.toUpperCase()
},
fetchPolicy: 'network-only'
})
state.jobs = resp?.data?.systemJobs
}
} catch (err) {
$q.notify({
type: 'negative',
message: 'Failed to load scheduled jobs.',
caption: err.message
})
}
state.loading--
}
function humanizeDate (val) {
return DateTime.fromISO(val).toRelative()
}
// MOUNTED
onMounted(() => {
load()
})
</script>
<style lang='scss'>
.admin-terminal {
&-term {
width: 100%;
background-color: #000;
border-radius: 5px;
overflow: hidden;
padding: 10px;
}
}
</style>

@ -47,6 +47,7 @@ const routes = [
{ path: 'api', component: () => import('pages/AdminApi.vue') },
{ path: 'extensions', component: () => import('pages/AdminExtensions.vue') },
{ path: 'mail', component: () => import('pages/AdminMail.vue') },
{ path: 'scheduler', component: () => import('pages/AdminScheduler.vue') },
{ path: 'security', component: () => import('pages/AdminSecurity.vue') },
{ path: 'system', component: () => import('pages/AdminSystem.vue') },
{ path: 'terminal', component: () => import('pages/AdminTerminal.vue') },

@ -6393,7 +6393,7 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4:
safe-buffer "^5.0.1"
sha.js "^2.4.8"
cron-parser@^4.0.0:
cron-parser@4.6.0, cron-parser@^4.0.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.6.0.tgz#404c3fdbff10ae80eef6b709555d577ef2fd2e0d"
integrity sha512-guZNLMGUgg6z4+eGhmHGw7ft+v6OQeuHzd1gcLxCo9Yg/qoxmG3nindp2/uwGCLizEisf2H0ptqeVXeoCpP6FA==

Loading…
Cancel
Save