From 6ce29bdb772ce819d5ab23bff031071d9b84cbeb Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 19 Sep 2022 06:54:03 +0000 Subject: [PATCH] feat: add scheduler --- package.json | 1 + server/core/kernel.js | 2 +- server/core/letsencrypt.js | 3 +- server/core/scheduler.js | 142 +++++-------------------------------- server/core/worker.js | 24 ------- ux/src/pages/Welcome.vue | 1 + yarn.lock | 83 +++++++++++++++++++++- 7 files changed, 103 insertions(+), 153 deletions(-) delete mode 100644 server/core/worker.js diff --git a/package.json b/package.json index d3990164..b90426e1 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "passport-twitch-strategy": "2.2.0", "pem-jwk": "2.0.0", "pg": "8.8.0", + "pg-boss": "8.0.0", "pg-hstore": "2.3.4", "pg-pubsub": "0.8.0", "pg-query-stream": "4.2.4", diff --git a/server/core/kernel.js b/server/core/kernel.js index cf6903dd..7ab5ba1c 100644 --- a/server/core/kernel.js +++ b/server/core/kernel.js @@ -75,7 +75,7 @@ module.exports = { await WIKI.models.commentProviders.initProvider() await WIKI.models.sites.reloadCache() await WIKI.models.storage.initTargets() - // WIKI.scheduler.start() + WIKI.scheduler.start() await WIKI.models.subscribeToNotifications() }, diff --git a/server/core/letsencrypt.js b/server/core/letsencrypt.js index bbad33da..9cfabada 100644 --- a/server/core/letsencrypt.js +++ b/server/core/letsencrypt.js @@ -4,8 +4,7 @@ const _ = require('lodash') const moment = require('moment') const CSR = require('@root/csr') const PEM = require('@root/pem') -// eslint-disable-next-line node/no-deprecated-api -const punycode = require('punycode') +const punycode = require('punycode/') /* global WIKI */ diff --git a/server/core/scheduler.js b/server/core/scheduler.js index 9eec70a7..bb2adbd6 100644 --- a/server/core/scheduler.js +++ b/server/core/scheduler.js @@ -1,134 +1,28 @@ -const moment = require('moment') -const childProcess = require('child_process') -const _ = require('lodash') -const configHelper = require('../helpers/config') +const PgBoss = require('pg-boss') /* global WIKI */ -class Job { - constructor({ - name, - immediate = false, - schedule = 'P1D', - repeat = false, - worker = false - }, queue) { - this.queue = queue - this.finished = Promise.resolve() - this.name = name - this.immediate = immediate - this.schedule = moment.duration(schedule) - this.repeat = repeat - this.worker = worker - } - - /** - * Start Job - * - * @param {Object} data Job Data - */ - start(data) { - this.queue.jobs.push(this) - if (this.immediate) { - this.invoke(data) - } else { - this.enqueue(data) - } - } - - /** - * Queue the next job run according to the wait duration - * - * @param {Object} data Job Data - */ - enqueue(data) { - this.timeout = setTimeout(this.invoke.bind(this), this.schedule.asMilliseconds(), data) - } - - /** - * Run the actual job - * - * @param {Object} data Job Data - */ - async invoke(data) { - try { - if (this.worker) { - const proc = childProcess.fork(`server/core/worker.js`, [ - `--job=${this.name}`, - `--data=${data}` - ], { - cwd: WIKI.ROOTPATH, - stdio: ['inherit', 'inherit', 'pipe', 'ipc'] - }) - const stderr = []; - proc.stderr.on('data', chunk => stderr.push(chunk)) - this.finished = new Promise((resolve, reject) => { - proc.on('exit', (code, signal) => { - const data = Buffer.concat(stderr).toString() - if (code === 0) { - resolve(data) - } else { - const err = new Error(`Error when running job ${this.name}: ${data}`) - err.exitSignal = signal - err.exitCode = code - err.stderr = data - reject(err) - } - proc.kill() - }) - }) - } else { - this.finished = require(`../jobs/${this.name}`)(data) - } - await this.finished - } catch (err) { - WIKI.logger.warn(err) - } - if (this.repeat && this.queue.jobs.includes(this)) { - this.enqueue(data) - } else { - this.stop().catch(() => {}) - } - } - - /** - * Stop any future job invocation from occuring - */ - async stop() { - clearTimeout(this.timeout) - this.queue.jobs = this.queue.jobs.filter(x => x !== this) - return this.finished - } -} - module.exports = { + scheduler: null, jobs: [], - init() { - return this - }, - start() { - _.forOwn(WIKI.data.jobs, (queueParams, queueName) => { - if (WIKI.config.offline && queueParams.offlineSkip) { - WIKI.logger.warn(`Skipping job ${queueName} because offline mode is enabled. [SKIPPED]`) - return - } - - const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams.schedule : 'P1D' - this.registerJob({ - name: _.kebabCase(queueName), - immediate: _.get(queueParams, 'onInit', false), - schedule: schedule, - repeat: _.get(queueParams, 'repeat', false), - worker: _.get(queueParams, 'worker', false) - }) + init () { + WIKI.logger.info('Initializing Scheduler...') + this.scheduler = new PgBoss({ + ...WIKI.models.knex.client.connectionSettings, + application_name: 'Wiki.js Scheduler', + schema: WIKI.config.db.schemas.scheduler, + uuid: 'v4' }) + return this }, - registerJob(opts, data) { - const job = new Job(opts, this) - job.start(data) - return job + async start () { + WIKI.logger.info('Starting Scheduler...') + await this.scheduler.start() + WIKI.logger.info('Scheduler: [ STARTED ]') }, - async stop() { - return Promise.all(this.jobs.map(job => job.stop())) + async stop () { + WIKI.logger.info('Stopping Scheduler...') + await this.scheduler.stop() + WIKI.logger.info('Scheduler: [ STOPPED ]') } } diff --git a/server/core/worker.js b/server/core/worker.js deleted file mode 100644 index d648a678..00000000 --- a/server/core/worker.js +++ /dev/null @@ -1,24 +0,0 @@ -const path = require('path') - -let WIKI = { - IS_DEBUG: process.env.NODE_ENV === 'development', - ROOTPATH: process.cwd(), - SERVERPATH: path.join(process.cwd(), 'server'), - Error: require('../helpers/error'), - configSvc: require('./config') -} -global.WIKI = WIKI - -WIKI.configSvc.init() -WIKI.logger = require('./logger').init('JOB') -const args = require('yargs').argv - -;(async () => { - try { - await require(`../jobs/${args.job}`)(args.data) - process.exit(0) - } catch (e) { - await new Promise(resolve => process.stderr.write(e.message, resolve)) - process.exit(1) - } -})() diff --git a/ux/src/pages/Welcome.vue b/ux/src/pages/Welcome.vue index 2ed4db3c..4ec9497f 100644 --- a/ux/src/pages/Welcome.vue +++ b/ux/src/pages/Welcome.vue @@ -77,6 +77,7 @@ useMeta({ > img { height: 200px; + user-select: none; } } diff --git a/yarn.lock b/yarn.lock index 9cd0b5ae..b6a55a09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3812,6 +3812,14 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" +aggregate-error@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-4.0.1.tgz#25091fe1573b9e0be892aeda15c7c66a545f758e" + integrity sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w== + dependencies: + clean-stack "^4.0.0" + indent-string "^5.0.0" + ajv-errors@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d" @@ -5720,6 +5728,13 @@ clean-stack@^2.0.0: resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== +clean-stack@^4.0.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-4.2.0.tgz#c464e4cde4ac789f4e0735c5d75beb49d7b30b31" + integrity sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg== + dependencies: + escape-string-regexp "5.0.0" + clean-webpack-plugin@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/clean-webpack-plugin/-/clean-webpack-plugin-3.0.0.tgz#a99d8ec34c1c628a4541567aa7b457446460c62b" @@ -6386,6 +6401,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4: safe-buffer "^5.0.1" sha.js "^2.4.8" +cron-parser@^4.0.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.6.0.tgz#404c3fdbff10ae80eef6b709555d577ef2fd2e0d" + integrity sha512-guZNLMGUgg6z4+eGhmHGw7ft+v6OQeuHzd1gcLxCo9Yg/qoxmG3nindp2/uwGCLizEisf2H0ptqeVXeoCpP6FA== + dependencies: + luxon "^3.0.1" + cross-fetch@3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" @@ -7486,6 +7508,11 @@ delaunator@4: resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-4.0.1.tgz#3d779687f57919a7a418f8ab947d3bddb6846957" integrity sha512-WNPWi1IRKZfCt/qIDMfERkDp93+iZEmOxN2yy4Jg+Xhv8SLk2UTqqbe1sfiipn0and9QrE914/ihdx82Y/Giag== +delay@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/delay/-/delay-5.0.0.tgz#137045ef1b96e5071060dd5be60bf9334436bd1d" + integrity sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw== + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -8166,6 +8193,11 @@ escape-html@^1.0.3, escape-html@~1.0.3: resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg= +escape-string-regexp@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8" + integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== + escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" @@ -10225,6 +10257,11 @@ indent-string@^4.0.0: resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== +indent-string@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-5.0.0.tgz#4fd2980fccaf8622d14c64d694f4cf33c81951a5" + integrity sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg== + indexes-of@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/indexes-of/-/indexes-of-1.0.1.tgz#f30f716c8e2bd346c7b67d3df3915566a7c05607" @@ -11884,6 +11921,11 @@ lodash.clonedeep@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ== +lodash.debounce@^4.0.8: + version "4.0.8" + resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" + integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== + lodash.includes@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" @@ -12073,6 +12115,11 @@ luxon@2.3.1: resolved "https://registry.yarnpkg.com/luxon/-/luxon-2.3.1.tgz#f276b1b53fd9a740a60e666a541a7f6dbed4155a" integrity sha512-I8vnjOmhXsMSlNMZlMkSOvgrxKJl0uOsEzdGgGNZuZPaS9KlefpE9KV95QFftlJSC+1UyCC9/I69R02cz/zcCA== +luxon@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.3.tgz#573e65531efd3d92265feb640f02ba7a192e2388" + integrity sha512-+EfHWnF+UT7GgTnq5zXg3ldnTKL2zdv7QJgsU5bjjpbH17E3qi/puMhQyJVYuCq+FRkogvB5WB6iVvUr+E4a7w== + make-dir@^2.0.0, make-dir@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5" @@ -13467,6 +13514,13 @@ p-map@^4.0.0: dependencies: aggregate-error "^3.0.0" +p-map@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-5.5.0.tgz#054ca8ca778dfa4cf3f8db6638ccb5b937266715" + integrity sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg== + dependencies: + aggregate-error "^4.0.0" + p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -13957,6 +14011,19 @@ persistgraphql@^0.3.11: "@types/graphql" "^0.9.0" "@types/isomorphic-fetch" "0.0.34" +pg-boss@8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/pg-boss/-/pg-boss-8.0.0.tgz#a32cd2c6f09c894b9e3ace027ecdecc06929edf6" + integrity sha512-WTchkRcTS9/AFuXhNzCQ9KlHgvi9VI3YpPk2EqGDhf2Od+/5Ug1e8b+NB+A81swe8LusAoQ6ka6n4pBkpkgrkw== + dependencies: + cron-parser "^4.0.0" + delay "^5.0.0" + lodash.debounce "^4.0.8" + p-map "^5.3.0" + pg "^8.5.1" + serialize-error "^11.0.0" + uuid "^8.3.2" + pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" @@ -14027,7 +14094,7 @@ pg-types@^2.1.0: postgres-date "~1.0.4" postgres-interval "^1.1.0" -pg@8.8.0, pg@^8.7.3: +pg@8.8.0, pg@^8.5.1, pg@^8.7.3: version "8.8.0" resolved "https://registry.yarnpkg.com/pg/-/pg-8.8.0.tgz#a77f41f9d9ede7009abfca54667c775a240da686" integrity sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw== @@ -16732,6 +16799,13 @@ send@0.18.0: range-parser "~1.2.1" statuses "2.0.1" +serialize-error@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-11.0.0.tgz#0129f2b07b19b09bc7a5f2d850ffe9cd2d561582" + integrity sha512-YKrURWDqcT3VGX/s/pCwaWtpfJEEaEw5Y4gAnQDku92b/HjVj4r4UhA5QrMVMFotymK2wIWs5xthny5SMFu7Vw== + dependencies: + type-fest "^2.12.2" + serialize-javascript@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-2.1.2.tgz#ecec53b0e0317bdc95ef76ab7074b7384785fa61" @@ -18098,6 +18172,11 @@ type-fest@^0.8.1: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== +type-fest@^2.12.2: + version "2.19.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.19.0.tgz#88068015bb33036a598b952e55e9311a60fd3a9b" + integrity sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA== + type-is@^1.6.4, type-is@~1.6.17, type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" @@ -18446,7 +18525,7 @@ uuid@8.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== -uuid@8.3.2: +uuid@8.3.2, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==