From e1ebaf5b31260be634123e3b4888a5c1fe31b89f Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 24 Sep 2022 06:08:34 +0000 Subject: [PATCH] feat: scheduler + worker pool --- config.sample.yml | 8 +++ package.json | 2 +- server/app/data.yml | 66 ++++++-------------- server/core/kernel.js | 8 ++- server/core/localization.js | 118 ------------------------------------ server/core/scheduler.js | 38 +++++++++++- server/web.js | 7 --- server/worker.js | 5 ++ yarn.lock | 27 ++------- 9 files changed, 81 insertions(+), 198 deletions(-) delete mode 100644 server/core/localization.js create mode 100644 server/worker.js diff --git a/config.sample.yml b/config.sample.yml index f4ba837e..5819d46d 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -122,3 +122,11 @@ dataPath: ./data # file uploads. bodyParserLimit: 5mb + +# --------------------------------------------------------------------- +# Workers Limit +# --------------------------------------------------------------------- +# Maximum number of workers that can run background cpu-intensive jobs. +# Leave to 'auto' to use CPU cores count as maximum. + +workers: auto diff --git a/package.json b/package.json index b90426e1..2219b184 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,6 @@ "he": "1.2.0", "highlight.js": "10.3.1", "i18next": "19.8.3", - "i18next-express-middleware": "2.0.0", "i18next-node-fs-backend": "2.1.3", "image-size": "0.9.2", "js-base64": "3.7.2", @@ -153,6 +152,7 @@ "pg-pubsub": "0.8.0", "pg-query-stream": "4.2.4", "pg-tsquery": "8.4.0", + "poolifier": "2.2.0", "pug": "3.0.2", "punycode": "2.1.1", "puppeteer-core": "17.1.3", diff --git a/server/app/data.yml b/server/app/data.yml index 101e4e48..38ab3caf 100644 --- a/server/app/data.yml +++ b/server/app/data.yml @@ -27,34 +27,16 @@ defaults: logLevel: info logFormat: default offline: false + dataPath: ./data bodyParserLimit: 5mb + workers: auto # DB defaults api: isEnabled: false - graphEndpoint: 'https://graph.requarks.io' - lang: - code: en - autoUpdate: true - namespaces: [] - namespacing: false - rtl: false - telemetry: - clientId: '' - isEnabled: false - title: Wiki.js - company: '' - contentLicense: '' - logoUrl: https://static.requarks.io/logo/wikijs-butterfly.svg mail: host: '' secure: true verifySSL: true - nav: - mode: 'MIXED' - theming: - theme: 'default' - iconset: 'md' - darkMode: false auth: autoLogin: false enforce2FA: false @@ -63,34 +45,30 @@ defaults: audience: 'urn:wiki.js' tokenExpiration: '30m' tokenRenewal: '14d' - features: - featurePageRatings: true - featurePageComments: true - featurePersonalWikis: true security: - securityOpenRedirect: true - securityIframe: true - securityReferrerPolicy: true - securityTrustProxy: true - securitySRI: true - securityHSTS: false - securityHSTSDuration: 300 - securityCSP: false - securityCSPDirectives: '' - server: - sslRedir: false - uploads: - maxFileSize: 5242880 - maxFiles: 10 - scanSVG: true - forceDownload: true + corsMode: 'OFF' + corsConfig: '' + enforceCsp: false + trustProxy: false + enforceHsts: false + disallowFloc: true + hstsDuration: 0 + cspDirectives: '' + uploadScanSVG: true + disallowIframe: true + uploadMaxFiles: 20 + authJwtAudience: 'urn:wiki.js' + authJwtExpiration: '30m' + uploadMaxFileSize: 10485760 + forceAssetDownload: true + disallowOpenRedirect: true + authJwtRenewablePeriod: '14d' + enforceSameOriginReferrerPolicy: true flags: ldapdebug: false sqllog: false # System defaults channel: NEXT - setup: false - dataPath: ./data cors: credentials: true maxAge: 600 @@ -99,10 +77,6 @@ defaults: search: maxHits: 100 maintainerEmail: security@requarks.io -localeNamespaces: - - admin - - auth - - common jobs: purgeUploads: onInit: true diff --git a/server/core/kernel.js b/server/core/kernel.js index 7ab5ba1c..34e89ef5 100644 --- a/server/core/kernel.js +++ b/server/core/kernel.js @@ -1,6 +1,8 @@ const _ = require('lodash') const EventEmitter = require('eventemitter2').EventEmitter2 +let isShuttingDown = false + /* global WIKI */ module.exports = { @@ -31,9 +33,8 @@ module.exports = { */ async preBootWeb() { try { - WIKI.sideloader = await require('./sideloader').init() WIKI.cache = require('./cache').init() - WIKI.scheduler = require('./scheduler').init() + WIKI.scheduler = await require('./scheduler').init() WIKI.servers = require('./servers') WIKI.events = { inbound: new EventEmitter(), @@ -83,6 +84,8 @@ module.exports = { * Graceful shutdown */ async shutdown (devMode = false) { + if (isShuttingDown) { return } + isShuttingDown = true if (WIKI.servers) { await WIKI.servers.stopServers() } @@ -99,6 +102,7 @@ module.exports = { await WIKI.asar.unload() } if (!devMode) { + WIKI.logger.info('Terminating process...') process.exit(0) } } diff --git a/server/core/localization.js b/server/core/localization.js deleted file mode 100644 index 0fe2367d..00000000 --- a/server/core/localization.js +++ /dev/null @@ -1,118 +0,0 @@ -const _ = require('lodash') -const dotize = require('dotize') -const i18nMW = require('i18next-express-middleware') -const i18next = require('i18next') -const Promise = require('bluebird') -const fs = require('fs-extra') -const path = require('path') -const yaml = require('js-yaml') - -/* global WIKI */ - -module.exports = { - engine: null, - namespaces: [], - init() { - this.namespaces = WIKI.data.localeNamespaces - this.engine = i18next - this.engine.init({ - load: 'languageOnly', - ns: this.namespaces, - defaultNS: 'common', - saveMissing: false, - lng: WIKI.config.lang.code, - fallbackLng: 'en' - }) - - // Load current language + namespaces - this.refreshNamespaces(true) - - return this - }, - /** - * Attach i18n middleware for Express - * - * @param {Object} app Express Instance - */ - attachMiddleware (app) { - app.use(i18nMW.handle(this.engine)) - }, - /** - * Get all entries for a specific locale and namespace - * - * @param {String} locale Locale code - * @param {String} namespace Namespace - */ - async getByNamespace(locale, namespace) { - if (this.engine.hasResourceBundle(locale, namespace)) { - let data = this.engine.getResourceBundle(locale, namespace) - return _.map(dotize.convert(data), (value, key) => { - return { - key, - value - } - }) - } else { - throw new Error('Invalid locale or namespace') - } - }, - /** - * Load entries from the DB for a single locale - * - * @param {String} locale Locale code - * @param {*} opts Additional options - */ - async loadLocale(locale, opts = { silent: false }) { - const res = await WIKI.models.locales.query().findOne('code', locale) - if (res) { - if (_.isPlainObject(res.strings)) { - _.forOwn(res.strings, (data, ns) => { - this.namespaces.push(ns) - this.engine.addResourceBundle(locale, ns, data, true, true) - }) - } - } else if (!opts.silent) { - throw new Error('No such locale in local store.') - } - - // -> Load dev locale files if present - if (WIKI.IS_DEBUG) { - try { - const devEntriesRaw = await fs.readFile(path.join(WIKI.SERVERPATH, `locales/${locale}.yml`), 'utf8') - if (devEntriesRaw) { - const devEntries = yaml.safeLoad(devEntriesRaw) - _.forOwn(devEntries, (data, ns) => { - this.namespaces.push(ns) - this.engine.addResourceBundle(locale, ns, data, true, true) - }) - WIKI.logger.info(`Loaded dev locales from ${locale}.yml`) - } - } catch (err) { - // ignore - } - } - }, - /** - * Reload all namespaces for all active locales from the DB - * - * @param {Boolean} silent No error on fail - */ - async refreshNamespaces (silent = false) { - await this.loadLocale(WIKI.config.lang.code, { silent }) - if (WIKI.config.lang.namespacing) { - for (let ns of WIKI.config.lang.namespaces) { - await this.loadLocale(ns, { silent }) - } - } - }, - /** - * Set the active locale - * - * @param {String} locale Locale code - */ - async setCurrentLocale(locale) { - await Promise.fromCallback(cb => { - return this.engine.changeLanguage(locale, cb) - }) - } -} diff --git a/server/core/scheduler.js b/server/core/scheduler.js index bb2adbd6..3bea2c9b 100644 --- a/server/core/scheduler.js +++ b/server/core/scheduler.js @@ -1,28 +1,60 @@ const PgBoss = require('pg-boss') +const { DynamicThreadPool } = require('poolifier') +const os = require('node:os') /* global WIKI */ module.exports = { + pool: null, scheduler: null, - jobs: [], - init () { + async init () { WIKI.logger.info('Initializing Scheduler...') this.scheduler = new PgBoss({ - ...WIKI.models.knex.client.connectionSettings, + db: { + close: () => Promise.resolve('ok'), + executeSql: async (text, values) => { + try { + const resource = await WIKI.models.knex.client.pool.acquire().promise + const res = await resource.query(text, values) + WIKI.models.knex.client.pool.release(resource) + return res + } catch (err) { + WIKI.logger.error('Failed to acquire DB connection during scheduler query execution.') + WIKI.logger.error(err) + } + } + }, + // ...WIKI.models.knex.client.connectionSettings, application_name: 'Wiki.js Scheduler', schema: WIKI.config.db.schemas.scheduler, uuid: 'v4' }) + + const maxWorkers = WIKI.config.workers === 'auto' ? os.cpus().length : WIKI.config.workers + WIKI.logger.info(`Initializing Worker Pool (Max ${maxWorkers})...`) + this.pool = new DynamicThreadPool(1, maxWorkers, './server/worker.js', { + errorHandler: (err) => WIKI.logger.warn(err), + exitHandler: () => WIKI.logger.debug('A worker has gone offline.'), + onlineHandler: () => WIKI.logger.debug('New worker is online.') + }) return this }, async start () { WIKI.logger.info('Starting Scheduler...') await this.scheduler.start() + this.scheduler.work('*', async job => { + return this.pool.execute({ + id: job.id, + name: job.name, + data: job.data + }) + }) WIKI.logger.info('Scheduler: [ STARTED ]') }, async stop () { WIKI.logger.info('Stopping Scheduler...') await this.scheduler.stop() + await this.pool.destroy() WIKI.logger.info('Scheduler: [ STOPPED ]') } } diff --git a/server/web.js b/server/web.js index 6b2196a1..889266d3 100644 --- a/server/web.js +++ b/server/web.js @@ -18,7 +18,6 @@ module.exports = async () => { // ---------------------------------------- WIKI.auth = require('./core/auth').init() - WIKI.lang = require('./core/localization').init() WIKI.mail = require('./core/mail').init() WIKI.system = require('./core/system').init() @@ -133,12 +132,6 @@ module.exports = async () => { app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' })) - // ---------------------------------------- - // Localization - // ---------------------------------------- - - WIKI.lang.attachMiddleware(app) - // ---------------------------------------- // View accessible data // ---------------------------------------- diff --git a/server/worker.js b/server/worker.js new file mode 100644 index 00000000..18d4ed54 --- /dev/null +++ b/server/worker.js @@ -0,0 +1,5 @@ +const { ThreadWorker } = require('poolifier') + +module.exports = new ThreadWorker(async (job) => { + return { ok: true } +}, { async: true }) diff --git a/yarn.lock b/yarn.lock index b6a55a09..82b8ec6b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6264,14 +6264,6 @@ cookiejar@^2.1.3: resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.3.tgz#fc7a6216e408e74414b90230050842dacda75acc" integrity sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ== -cookies@0.7.1: - version "0.7.1" - resolved "https://registry.yarnpkg.com/cookies/-/cookies-0.7.1.tgz#7c8a615f5481c61ab9f16c833731bcb8f663b99b" - integrity sha1-fIphX1SBxhq58WyDNzG8uPZjuZs= - dependencies: - depd "~1.1.1" - keygrip "~1.0.2" - copy-concurrently@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/copy-concurrently/-/copy-concurrently-1.0.5.tgz#92297398cae34937fcafd6ec8139c18051f0b5e0" @@ -7533,7 +7525,7 @@ depd@2.0.0, depd@~2.0.0: resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== -depd@~1.1.1, depd@~1.1.2: +depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= @@ -10084,13 +10076,6 @@ i18next-chained-backend@2.0.1: dependencies: "@babel/runtime" "^7.4.5" -i18next-express-middleware@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/i18next-express-middleware/-/i18next-express-middleware-2.0.0.tgz#e6ab3be8d2db3c715dc084880d100d235b6fd62e" - integrity sha512-TGlSkYsQHikggv4mIp5B+CiXsZzwbpHaZgmOkRNGStLOdKHABH5cHr136g2PC1+p2VPMf3y3UoQZ1TfPfVOrgg== - dependencies: - cookies "0.7.1" - i18next-localstorage-backend@3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/i18next-localstorage-backend/-/i18next-localstorage-backend-3.1.3.tgz#5eaad25a515bdadebeb13e1486acfa6fa1686cbe" @@ -11595,11 +11580,6 @@ katex@0.12.0: dependencies: commander "^2.19.0" -keygrip@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/keygrip/-/keygrip-1.0.3.tgz#399d709f0aed2bab0a059e0cdd3a5023a053e1dc" - integrity sha512-/PpesirAIfaklxUzp4Yb7xBper9MwP6hNRA6BGGUFCgbJ+BM5CKBtsoxinNXkLHAr+GXS1/lSlF2rP7cv5Fl+g== - keyv@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.1.0.tgz#ecc228486f69991e49e9476485a5be1e8fc5c4d9" @@ -14223,6 +14203,11 @@ pony-cause@^2.0.0: resolved "https://registry.yarnpkg.com/pony-cause/-/pony-cause-2.1.4.tgz#18ef4799b5207ad0a7bacf5ea4602e6b06c75c8c" integrity sha512-6jNyaeEi1I4rGD338qmNmx2yLg8N/JZJZU8JCrqDtfxCEYZttfuN6AnKhBGfMyTydW4t2iBioxDzKeZJC2mJVw== +poolifier@2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/poolifier/-/poolifier-2.2.0.tgz#7a8bf0bd6e7a0b3e0633cd54a6ffeb6ed89ec41b" + integrity sha512-bzthEM2MRwt4mDVC7sWYBRu7pTpb+aDj+Bw2EhoxtYTSL6wbWTu94gW7wuTwjCPClNRDiCPxV5bHQz5X/M/PnA== + posix-character-classes@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"