msb-public
/
wiki
mirror of https://github.com/requarks/wiki
1
0
Code Issues Projects Releases Wiki Activity

feat: queue handling

Browse Source
pull/621/head
NGPixel 7 years ago
parent 2020e457cf
commit 749766e9bd
13 changed files with 236 additions and 144 deletions
Show Stats Download Patch File Download Diff File

11
config.sample.yml
Unescape View File

@ -42,6 +42,15 @@ redis:
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Background Workers # Background Workers
# --------------------------------------------------------------------- # ---------------------------------------------------------------------
# Leave 0 for auto based on CPU cores # Leave 0 for auto based on CPU cores
workers: 0 workers: 0
# ---------------------------------------------------------------------
# High Availability
# ---------------------------------------------------------------------
# Read the docs BEFORE changing these settings!
ha:
nodeuid: primary
readonly: false

6
server/app/data.yml
Unescape View File

@ -21,6 +21,12 @@ defaults:
db: 0 db: 0
password: null password: null
workers: 0 workers: 0
ha:
nodeuid: primary
readonly: false
queues:
- gitSync
- uplClearTemp
authProviders: authProviders:
- local - local
- microsoft - microsoft

21
server/index.js
Unescape View File

@ -48,18 +48,27 @@ if (numWorkers > numCPUs) {
} }
if (cluster.isMaster) { if (cluster.isMaster) {
wiki.logger.info('--------------------------')
wiki.logger.info('Wiki.js is initializing...') wiki.logger.info('Wiki.js is initializing...')
wiki.logger.info('--------------------------')
require('./master') require('./master').then(() => {
// -> Create background workers
for (let i = 0; i < numWorkers; i++) {
cluster.fork()
}
for (let i = 0; i < numWorkers; i++) { // -> Queue post-init tasks
cluster.fork()
} wiki.queue.uplClearTemp.add({}, {
repeat: { cron: '*/15 * * * *' }
})
})
cluster.on('exit', (worker, code, signal) => { cluster.on('exit', (worker, code, signal) => {
wiki.logger.info(`Worker #${worker.id} died.`) wiki.logger.info(`Background Worker #${worker.id} was terminated.`)
}) })
} else { } else {
wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`) wiki.logger.info(`Background Worker #${cluster.worker.id} is initializing...`)
require('./worker') require('./worker')
} }

11
server/master.js
Unescape View File

@ -4,20 +4,23 @@
const Promise = require('bluebird') const Promise = require('bluebird')
wiki.redis = require('./modules/redis').init()
wiki.queue = require('./modules/queue').init()
module.exports = Promise.join( module.exports = Promise.join(
wiki.db.onReady, wiki.db.onReady,
wiki.configSvc.loadFromDb() wiki.configSvc.loadFromDb(),
wiki.queue.clean()
).then(() => { ).then(() => {
// ---------------------------------------- // ----------------------------------------
// Load global modules // Load global modules
// ---------------------------------------- // ----------------------------------------
wiki.disk = require('./modules/disk').init() wiki.disk = require('./modules/disk').init()
wiki.entries = require('./modules/entries').init() wiki.docs = require('./modules/documents').init()
wiki.git = require('./modules/git').init(false) wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next') wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown') wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init() wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init() wiki.upl = require('./modules/uploads').init()
@ -75,7 +78,7 @@ module.exports = Promise.join(
// Passport Authentication // Passport Authentication
// ---------------------------------------- // ----------------------------------------
require('./modules/auth')(passport) require('./modules/auth').init(passport)
wiki.rights = require('./modules/rights') wiki.rights = require('./modules/rights')
// wiki.rights.init() // wiki.rights.init()

144
server/modules/auth.js
Unescape View File

@ -4,87 +4,89 @@
const _ = require('lodash') const _ = require('lodash')
module.exports = (passport) => { module.exports = {
init(passport) {
// Serialization user methods // Serialization user methods
passport.serializeUser(function (user, done) { passport.serializeUser(function (user, done) {
done(null, user._id) done(null, user._id)
}) })
passport.deserializeUser(function (id, done) { passport.deserializeUser(function (id, done) {
wiki.db.User.findById(id).then((user) => { wiki.db.User.findById(id).then((user) => {
if (user) { if (user) {
done(null, user) done(null, user)
} else { } else {
done(new Error(wiki.lang.t('auth:errors:usernotfound')), null) done(new Error(wiki.lang.t('auth:errors:usernotfound')), null)
} }
return true return true
}).catch((err) => { }).catch((err) => {
done(err, null) done(err, null)
})
}) })
})
// Load authentication strategies // Load authentication strategies
wiki.config.authStrategies = { wiki.config.authStrategies = {
list: _.pickBy(wiki.config.auth, strategy => strategy.enabled), list: _.pickBy(wiki.config.auth, strategy => strategy.enabled),
socialEnabled: (_.chain(wiki.config.auth).omit('local').filter(['enabled', true]).value().length > 0) socialEnabled: (_.chain(wiki.config.auth).omit('local').filter(['enabled', true]).value().length > 0)
} }
_.forOwn(wiki.config.authStrategies.list, (strategyConfig, strategyName) => { _.forOwn(wiki.config.authStrategies.list, (strategyConfig, strategyName) => {
strategyConfig.callbackURL = `${wiki.config.site.host}/login/${strategyName}/callback` strategyConfig.callbackURL = `${wiki.config.site.host}/login/${strategyName}/callback`
require(`../authentication/${strategyName}`)(passport, strategyConfig) require(`../authentication/${strategyName}`)(passport, strategyConfig)
wiki.logger.info(`Authentication Provider ${_.upperFirst(strategyName)}: OK`) wiki.logger.info(`Authentication Provider ${_.upperFirst(strategyName)}: OK`)
}) })
// Create Guest account for first-time // Create Guest account for first-time
return wiki.db.User.findOne({ return wiki.db.User.findOne({
where: { where: {
provider: 'local',
email: 'guest@example.com'
}
}).then((c) => {
if (c < 1) {
return wiki.db.User.create({
provider: 'local', provider: 'local',
email: 'guest@example.com', email: 'guest@example.com'
name: 'Guest', }
password: '', }).then((c) => {
role: 'guest' if (c < 1) {
}).then(() => { return wiki.db.User.create({
wiki.logger.info('[AUTH] Guest account created successfully!') provider: 'local',
return true email: 'guest@example.com',
}).catch((err) => { name: 'Guest',
wiki.logger.error('[AUTH] An error occured while creating guest account:') password: '',
wiki.logger.error(err) role: 'guest'
return err }).then(() => {
}) wiki.logger.info('[AUTH] Guest account created successfully!')
} return true
}) }).catch((err) => {
wiki.logger.error('[AUTH] An error occured while creating guest account:')
wiki.logger.error(err)
return err
})
}
})
// .then(() => { // .then(() => {
// if (process.env.WIKI_JS_HEROKU) { // if (process.env.WIKI_JS_HEROKU) {
// return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => { // return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => {
// if (c < 1) { // if (c < 1) {
// // Create root admin account (HEROKU ONLY) // // Create root admin account (HEROKU ONLY)
// return wiki.db.User.create({ // return wiki.db.User.create({
// provider: 'local', // provider: 'local',
// email: process.env.WIKI_ADMIN_EMAIL, // email: process.env.WIKI_ADMIN_EMAIL,
// name: 'Administrator', // name: 'Administrator',
// password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default) // password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default)
// role: 'admin' // role: 'admin'
// }).then(() => { // }).then(() => {
// wiki.logger.info('[AUTH] Root admin account created successfully!') // wiki.logger.info('[AUTH] Root admin account created successfully!')
// return true // return true
// }).catch((err) => { // }).catch((err) => {
// wiki.logger.error('[AUTH] An error occured while creating root admin account:') // wiki.logger.error('[AUTH] An error occured while creating root admin account:')
// wiki.logger.error(err) // wiki.logger.error(err)
// return err // return err
// }) // })
// } else { return true } // } else { return true }
// }) // })
// } else { return true } // } else { return true }
// }) // })
}
} }

3
server/modules/config.js
Unescape View File

@ -13,9 +13,6 @@ module.exports = {
/** /**
* Load root config from disk * Load root config from disk
*
* @param {any} confPaths
* @returns
*/ */
init() { init() {
let confPaths = { let confPaths = {

32
server/modules/db.js
Unescape View File

@ -63,12 +63,32 @@ module.exports = {
require(path.join(dbModelsPath, '_relations.js'))(self) require(path.join(dbModelsPath, '_relations.js'))(self)
// Sync DB // Set init tasks
self.onReady = (wiki.IS_MASTER) ? self.inst.sync({ let initTasks = {
force: false, // -> Sync DB Schemas
logging: false syncSchemas() {
}) : Promise.resolve() return self.inst.sync({
force: false,
logging: false
})
},
// -> Set Connection App Name
setAppName() {
return self.inst.query(`set application_name = 'Wiki.js'`, { raw: true })
}
}
let initTasksQueue = (wiki.IS_MASTER) ? [
initTasks.syncSchemas,
initTasks.setAppName
] : [
initTasks.setAppName
]
// Perform init tasks
self.onReady = Promise.each(initTasksQueue, t => t())
return self return self
} }

2
server/modules/entries.js → server/modules/documents.js
Unescape View File

@ -10,7 +10,7 @@ const _ = require('lodash')
const entryHelper = require('../helpers/entry') const entryHelper = require('../helpers/entry')
/** /**
* Entries Model * Documents Model
*/ */
module.exports = { module.exports = {

37
server/modules/queue.js
Unescape View File

@ -0,0 +1,37 @@
'use strict'
/* global wiki */
const Bull = require('bull')
const Promise = require('bluebird')
module.exports = {
init() {
wiki.data.queues.forEach(queueName => {
this[queueName] = new Bull(queueName, {
prefix: `q-${wiki.config.ha.nodeuid}`,
redis: wiki.config.redis
})
})
return this
},
clean() {
return Promise.each(wiki.data.queues, queueName => {
return new Promise((resolve, reject) => {
let keyStream = wiki.redis.scanStream({
match: `q-${wiki.config.ha.nodeuid}:${queueName}:*`
})
keyStream.on('data', resultKeys => {
if (resultKeys.length > 0) {
wiki.redis.del(resultKeys)
}
})
keyStream.on('end', resolve)
})
}).then(() => {
wiki.logger.info('Purging old queue jobs: OK')
}).catch(err => {
wiki.logger.error(err)
})
}
}

6
server/modules/redis.js
Unescape View File

@ -19,7 +19,11 @@ module.exports = {
*/ */
init() { init() {
if (isPlainObject(wiki.config.redis)) { if (isPlainObject(wiki.config.redis)) {
return new Redis(wiki.config.redis) let red = new Redis(wiki.config.redis)
red.on('ready', () => {
wiki.logger.info('Redis connection: OK')
})
return red
} else { } else {
wiki.logger.error('Invalid Redis configuration!') wiki.logger.error('Invalid Redis configuration!')
process.exit(1) process.exit(1)

3
server/queues/git-sync.js
Unescape View File

@ -61,5 +61,8 @@ module.exports = (job, done) => {
}) })
return jobCbStreamDocs return jobCbStreamDocs
}).then(() => {
wiki.logger.info('Git remote repository sync: DONE')
return true
}) })
} }

3
server/queues/upl-clear-temp.js
Unescape View File

@ -22,5 +22,8 @@ module.exports = (job, done) => {
} }
}) })
}) })
}).then(() => {
wiki.logger.info('Purging temporary upload files: DONE')
return true
}) })
} }

101
server/worker.js
Unescape View File

@ -2,69 +2,68 @@
/* global wiki */ /* global wiki */
const path = require('path') const Promise = require('bluebird')
wiki.REPOPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
wiki.DATAPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
wiki.UPLTEMPPATH = path.join(wiki.DATAPATH, 'temp-upload')
// ---------------------------------------- module.exports = Promise.join(
// Load global modules wiki.db.onReady,
// ---------------------------------------- wiki.configSvc.loadFromDb(['features', 'git', 'logging', 'site', 'uploads'])
).then(() => {
const path = require('path')
// wiki.upl = require('./modules/uploads-agent').init() wiki.REPOPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
// wiki.git = require('./modules/git').init() wiki.DATAPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
// wiki.entries = require('./modules/entries').init() wiki.UPLTEMPPATH = path.join(wiki.DATAPATH, 'temp-upload')
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
// ---------------------------------------- // ----------------------------------------
// Load local modules // Load global modules
// ---------------------------------------- // ----------------------------------------
const Promise = require('bluebird') // wiki.upl = require('./modules/uploads-agent').init()
const i18nBackend = require('i18next-node-fs-backend') // wiki.git = require('./modules/git').init()
// wiki.entries = require('./modules/entries').init()
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
// ---------------------------------------- // ----------------------------------------
// Localization Engine // Localization Engine
// ---------------------------------------- // ----------------------------------------
wiki.lang.use(i18nBackend).init({ const i18nBackend = require('i18next-node-fs-backend')
load: 'languageOnly', wiki.lang.use(i18nBackend).init({
ns: ['common', 'admin', 'auth', 'errors', 'git'], load: 'languageOnly',
defaultNS: 'common', ns: ['common', 'admin', 'auth', 'errors', 'git'],
saveMissing: false, defaultNS: 'common',
preload: [wiki.config.lang], saveMissing: false,
lng: wiki.config.lang, preload: [wiki.config.lang],
fallbackLng: 'en', lng: wiki.config.lang,
backend: { fallbackLng: 'en',
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') backend: {
} loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}) }
})
// ---------------------------------------- // ----------------------------------------
// Start Queues // Start Queues
// ---------------------------------------- // ----------------------------------------
const Bull = require('bull') const Bull = require('bull')
const autoload = require('auto-load') const autoload = require('auto-load')
let queues = autoload(path.join(wiki.SERVERPATH, 'queues')) let queues = autoload(path.join(wiki.SERVERPATH, 'queues'))
Promise.join(
wiki.db.onReady
// wiki.upl.initialScan()
).then(() => {
for (let queueName in queues) { for (let queueName in queues) {
new Bull(queueName, { redis: wiki.config.redis }).process(queues[queueName]) new Bull(queueName, {
prefix: `q-${wiki.config.ha.nodeuid}`,
redis: wiki.config.redis
}).process(queues[queueName])
} }
})
// ---------------------------------------- // ----------------------------------------
// Shutdown gracefully // Shutdown gracefully
// ---------------------------------------- // ----------------------------------------
process.on('disconnect', () => { process.on('disconnect', () => {
wiki.logger.warn('Lost connection to Master. Exiting...') wiki.logger.warn('Lost connection to Master. Exiting...')
process.exit() process.exit()
})
}) })

Write Preview
Loading…
Cancel
Save