From 9c112ab53555839424bcbf2dae48b2ae2314b60e Mon Sep 17 00:00:00 2001 From: NGPixel Date: Sat, 29 Jul 2017 00:11:22 -0400 Subject: [PATCH] feat: cluster implementation --- config.sample.yml | 153 ++-------------- package.json | 1 + server/index.js | 254 ++------------------------ server/master.js | 232 +++++++++++++++++++++++ server/models/_relations.js | 7 +- server/models/comment.js | 18 ++ server/models/document.js | 5 + server/models/tag.js | 24 +++ server/modules/db.js | 3 +- server/modules/logger.js | 9 +- server/modules/search-index/index.js | 81 -------- server/modules/search-index/siUtil.js | 36 ---- server/modules/search.js | 6 +- server/{agent.js => worker.js} | 86 ++++----- yarn.lock | 63 ++++++- 15 files changed, 414 insertions(+), 564 deletions(-) create mode 100644 server/master.js create mode 100644 server/models/comment.js create mode 100644 server/models/tag.js delete mode 100644 server/modules/search-index/index.js delete mode 100644 server/modules/search-index/siUtil.js rename server/{agent.js => worker.js} (69%) diff --git a/config.sample.yml b/config.sample.yml index 7f10129f..d3359c9c 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -4,23 +4,9 @@ # Full explanation + examples in the documentation: # https://docs.requarks.io/wiki/install -# --------------------------------------------------------------------- -# Title of this site -# --------------------------------------------------------------------- - -title: Wiki - -# --------------------------------------------------------------------- -# Full public path to the site, without the trailing slash -# --------------------------------------------------------------------- -# INCLUDE CLIENT PORT IF NOT 80/443! - -host: http://localhost - # --------------------------------------------------------------------- # Port the main server should listen to (80 by default) # --------------------------------------------------------------------- -# To use process.env.PORT, comment the line below: port: 80 @@ -33,136 +19,23 @@ paths: data: ./data # --------------------------------------------------------------------- -# Upload Limits -# --------------------------------------------------------------------- -# In megabytes (MB) - -uploads: - maxImageFileSize: 3 - maxOtherFileSize: 100 - -# --------------------------------------------------------------------- -# Site Language -# --------------------------------------------------------------------- -# Possible values: en, es, fr, ko, ru or zh - -lang: en - -# --------------------------------------------------------------------- -# Site Authentication -# --------------------------------------------------------------------- - -public: false - -auth: - defaultReadAccess: false - local: - enabled: true - google: - enabled: true - clientId: GOOGLE_CLIENT_ID - clientSecret: GOOGLE_CLIENT_SECRET - microsoft: - enabled: true - clientId: MS_APP_ID - clientSecret: MS_APP_SECRET - facebook: - enabled: false - clientId: FACEBOOK_APP_ID - clientSecret: FACEBOOK_APP_SECRET - github: - enabled: false - clientId: GITHUB_CLIENT_ID - clientSecret: GITHUB_CLIENT_SECRET - slack: - enabled: false - clientId: SLACK_CLIENT_ID - clientSecret: SLACK_CLIENT_SECRET - ldap: - enabled: false - url: ldap://serverhost:389 - bindDn: cn='root' - bindCredentials: BIND_PASSWORD - searchBase: o=users,o=example.com - searchFilter: (uid={{username}}) - tlsEnabled: false - tlsCertPath: C:\example\root_ca_cert.crt - azure: - enabled: false - clientID: APP_ID - clientSecret: APP_SECRET_KEY - resource: '00000002-0000-0000-c000-000000000000' - tenant: 'YOUR_TENANT.onmicrosoft.com' - -# --------------------------------------------------------------------- -# Secret key to use when encrypting sessions +# Database # --------------------------------------------------------------------- -# Use a long and unique random string (256-bit keys are perfect!) -sessionSecret: 1234567890abcdefghijklmnopqrstuvxyz +db: + host: localhost + port: 5432 + user: wikijs + pass: wikijsrocks + db: wiki # --------------------------------------------------------------------- -# Database Connection String +# Redis # --------------------------------------------------------------------- -# You can also use an ENV variable by using $ENV_VAR_NAME as the value -db: mongodb://localhost:27017/wiki - -# --------------------------------------------------------------------- -# Git Connection Info -# --------------------------------------------------------------------- - -git: - url: https://github.com/Organization/Repo - branch: master - auth: - - # Type: basic or ssh - type: ssh - - # Only for Basic authentication: - username: marty - password: MartyMcFly88 - - # Only for SSH authentication: - privateKey: /etc/wiki/keys/git.pem - - sslVerify: true - - # Default email to use as commit author - serverEmail: marty@example.com - - # Whether to use user email as author in commits - showUserEmail: true - -# --------------------------------------------------------------------- -# Features -# --------------------------------------------------------------------- -# You can enable / disable specific features below - -features: - linebreaks: true - mathjax: true - -# --------------------------------------------------------------------- -# External Logging -# --------------------------------------------------------------------- - -externalLogging: - bugsnag: false - loggly: false - papertrail: false - rollbar: false - sentry: false - -# --------------------------------------------------------------------- -# Color Theme -# --------------------------------------------------------------------- +redis: + host: localhost + port: 6379 + db: 0 + password: null -theme: - primary: indigo - alt: blue-grey - footer: blue-grey - code: - dark: true - colorize: true diff --git a/package.json b/package.json index 862d8fff..d20fd14e 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "bcryptjs-then": "~1.0.1", "bluebird": "~3.5.0", "body-parser": "~1.17.2", + "bull": "/home/nick3.0.0-rc.4", "bunyan": "~1.8.10", "cheerio": "~1.0.0-rc.2", "child-process-promise": "~2.2.1", diff --git a/server/index.js b/server/index.js index d53f3fba..aeb2f8a5 100644 --- a/server/index.js +++ b/server/index.js @@ -2,7 +2,6 @@ // =========================================== // Wiki.js -// 1.0.1 // Licensed under AGPLv3 // =========================================== @@ -28,251 +27,28 @@ wiki.data = appconf.data // Load Winston // ---------------------------------------- -wiki.logger = require('./modules/logger')(wiki.IS_DEBUG, 'SERVER') -wiki.logger.info('Wiki.js is initializing...') +wiki.logger = require('./modules/logger')() // ---------------------------------------- -// Load global modules +// Start Cluster // ---------------------------------------- -wiki.disk = require('./modules/disk').init() -wiki.db = require('./modules/db').init() -wiki.entries = require('./modules/entries').init() -wiki.git = require('./modules/git').init(false) -wiki.lang = require('i18next') -wiki.mark = require('./modules/markdown') -wiki.redis = require('./modules/redis').init() -wiki.search = require('./modules/search').init() -wiki.upl = require('./modules/uploads').init() +const cluster = require('cluster') +const numCPUs = require('os').cpus().length -// ---------------------------------------- -// Load modules -// ---------------------------------------- - -const autoload = require('auto-load') -const bodyParser = require('body-parser') -const compression = require('compression') -const cookieParser = require('cookie-parser') -const express = require('express') -const favicon = require('serve-favicon') -const flash = require('connect-flash') -const fork = require('child_process').fork -const http = require('http') -const i18nBackend = require('i18next-node-fs-backend') -const passport = require('passport') -const passportSocketIo = require('passport.socketio') -const session = require('express-session') -const SessionRedisStore = require('connect-redis')(session) -const graceful = require('node-graceful') -const socketio = require('socket.io') -const graphqlApollo = require('apollo-server-express') -const graphqlSchema = require('./modules/graphql') - -var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares')) -var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers')) - -// ---------------------------------------- -// Define Express App -// ---------------------------------------- - -const app = express() -wiki.app = app -app.use(compression()) - -// ---------------------------------------- -// Security -// ---------------------------------------- - -app.use(mw.security) - -// ---------------------------------------- -// Public Assets -// ---------------------------------------- - -app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico'))) -app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), { - index: false, - maxAge: '7d' -})) - -// ---------------------------------------- -// Passport Authentication -// ---------------------------------------- - -require('./modules/auth')(passport) -wiki.rights = require('./modules/rights') -wiki.rights.init() - -let sessionStore = new SessionRedisStore({ - client: wiki.redis -}) - -app.use(cookieParser()) -app.use(session({ - name: 'wikijs.sid', - store: sessionStore, - secret: wiki.config.sessionSecret, - resave: false, - saveUninitialized: false -})) -app.use(flash()) -app.use(passport.initialize()) -app.use(passport.session()) - -// ---------------------------------------- -// SEO -// ---------------------------------------- - -app.use(mw.seo) - -// ---------------------------------------- -// Localization Engine -// ---------------------------------------- - -wiki.lang.use(i18nBackend).init({ - load: 'languageOnly', - ns: ['common', 'admin', 'auth', 'errors', 'git'], - defaultNS: 'common', - saveMissing: false, - preload: [wiki.config.lang], - lng: wiki.config.lang, - fallbackLng: 'en', - backend: { - loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') - } -}) - -// ---------------------------------------- -// View Engine Setup -// ---------------------------------------- - -app.set('views', path.join(wiki.SERVERPATH, 'views')) -app.set('view engine', 'pug') - -app.use(bodyParser.json({ limit: '1mb' })) -app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' })) - -// ---------------------------------------- -// View accessible data -// ---------------------------------------- - -app.locals._ = require('lodash') -app.locals.t = wiki.lang.t.bind(wiki.lang) -app.locals.moment = require('moment') -app.locals.moment.locale(wiki.config.lang) -app.locals.appconfig = wiki.config -app.use(mw.flash) - -// ---------------------------------------- -// Controllers -// ---------------------------------------- - -app.use('/', ctrl.auth) - -app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema })) -app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' })) -app.use('/uploads', mw.auth, ctrl.uploads) -app.use('/admin', mw.auth, ctrl.admin) -app.use('/', mw.auth, ctrl.pages) - -// ---------------------------------------- -// Error handling -// ---------------------------------------- - -app.use(function (req, res, next) { - var err = new Error('Not Found') - err.status = 404 - next(err) -}) - -app.use(function (err, req, res, next) { - res.status(err.status || 500) - res.render('error', { - message: err.message, - error: wiki.IS_DEBUG ? err : {} - }) -}) - -// ---------------------------------------- -// Start HTTP server -// ---------------------------------------- - -wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...') - -app.set('port', wiki.config.port) -var server = http.createServer(app) -var io = socketio(server) - -server.listen(wiki.config.port) -server.on('error', (error) => { - if (error.syscall !== 'listen') { - throw error - } - - // handle specific listen errors with friendly messages - switch (error.code) { - case 'EACCES': - wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!') - return process.exit(1) - case 'EADDRINUSE': - wiki.logger.error('Port ' + wiki.config.port + ' is already in use!') - return process.exit(1) - default: - throw error - } -}) +if (cluster.isMaster) { + wiki.logger.info('Wiki.js is initializing...') -server.on('listening', () => { - wiki.logger.info('HTTP/WS server started successfully! [RUNNING]') -}) - -// ---------------------------------------- -// WebSocket -// ---------------------------------------- + require('./master') -io.use(passportSocketIo.authorize({ - key: 'wikijs.sid', - store: sessionStore, - secret: wiki.config.sessionSecret, - cookieParser, - success: (data, accept) => { - accept() - }, - fail: (data, message, error, accept) => { - accept() + for (let i = 0; i < numCPUs; i++) { + cluster.fork() } -})) - -io.on('connection', ctrl.ws) - -// ---------------------------------------- -// Start child processes -// ---------------------------------------- -let bgAgent = fork(path.join(wiki.SERVERPATH, 'agent.js')) - -bgAgent.on('message', m => { - if (!m.action) { - return - } - - switch (m.action) { - case 'searchAdd': - wiki.search.add(m.content) - break - } -}) - -// ---------------------------------------- -// Graceful shutdown -// ---------------------------------------- - -graceful.on('exit', () => { - wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...') - bgAgent.kill() - wiki.logger.info('- SHUTTING DOWN - Performing git sync...') - return global.git.resync().then(() => { - wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.') - process.exit() + cluster.on('exit', (worker, code, signal) => { + wiki.logger.info(`Worker #${worker.id} died.`) }) -}) +} else { + wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`) + // require('./worker') +} diff --git a/server/master.js b/server/master.js new file mode 100644 index 00000000..75a609c8 --- /dev/null +++ b/server/master.js @@ -0,0 +1,232 @@ +'use strict' + +/* global wiki */ + +const path = require('path') + +// ---------------------------------------- +// Load global modules +// ---------------------------------------- + +wiki.disk = require('./modules/disk').init() +wiki.db = require('./modules/db').init() +wiki.entries = require('./modules/entries').init() +wiki.git = require('./modules/git').init(false) +wiki.lang = require('i18next') +wiki.mark = require('./modules/markdown') +wiki.redis = require('./modules/redis').init() +wiki.search = require('./modules/search').init() +wiki.upl = require('./modules/uploads').init() + +// ---------------------------------------- +// Load modules +// ---------------------------------------- + +const autoload = require('auto-load') +const bodyParser = require('body-parser') +const compression = require('compression') +const cookieParser = require('cookie-parser') +const express = require('express') +const favicon = require('serve-favicon') +const flash = require('connect-flash') +const http = require('http') +const i18nBackend = require('i18next-node-fs-backend') +const passport = require('passport') +const passportSocketIo = require('passport.socketio') +const session = require('express-session') +const SessionRedisStore = require('connect-redis')(session) +const graceful = require('node-graceful') +const socketio = require('socket.io') +const graphqlApollo = require('apollo-server-express') +const graphqlSchema = require('./modules/graphql') + +var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares')) +var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers')) + +// ---------------------------------------- +// Define Express App +// ---------------------------------------- + +const app = express() +wiki.app = app +app.use(compression()) + +// ---------------------------------------- +// Security +// ---------------------------------------- + +app.use(mw.security) + +// ---------------------------------------- +// Public Assets +// ---------------------------------------- + +app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico'))) +app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), { + index: false, + maxAge: '7d' +})) + +// ---------------------------------------- +// Passport Authentication +// ---------------------------------------- + +require('./modules/auth')(passport) +wiki.rights = require('./modules/rights') +wiki.rights.init() + +let sessionStore = new SessionRedisStore({ + client: wiki.redis +}) + +app.use(cookieParser()) +app.use(session({ + name: 'wikijs.sid', + store: sessionStore, + secret: wiki.config.sessionSecret, + resave: false, + saveUninitialized: false +})) +app.use(flash()) +app.use(passport.initialize()) +app.use(passport.session()) + +// ---------------------------------------- +// SEO +// ---------------------------------------- + +app.use(mw.seo) + +// ---------------------------------------- +// Localization Engine +// ---------------------------------------- + +wiki.lang.use(i18nBackend).init({ + load: 'languageOnly', + ns: ['common', 'admin', 'auth', 'errors', 'git'], + defaultNS: 'common', + saveMissing: false, + preload: [wiki.config.lang], + lng: wiki.config.lang, + fallbackLng: 'en', + backend: { + loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') + } +}) + +// ---------------------------------------- +// View Engine Setup +// ---------------------------------------- + +app.set('views', path.join(wiki.SERVERPATH, 'views')) +app.set('view engine', 'pug') + +app.use(bodyParser.json({ limit: '1mb' })) +app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' })) + +// ---------------------------------------- +// View accessible data +// ---------------------------------------- + +app.locals._ = require('lodash') +app.locals.t = wiki.lang.t.bind(wiki.lang) +app.locals.moment = require('moment') +app.locals.moment.locale(wiki.config.lang) +app.locals.appconfig = wiki.config +app.use(mw.flash) + +// ---------------------------------------- +// Controllers +// ---------------------------------------- + +app.use('/', ctrl.auth) + +app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema })) +app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' })) +app.use('/uploads', mw.auth, ctrl.uploads) +app.use('/admin', mw.auth, ctrl.admin) +app.use('/', mw.auth, ctrl.pages) + +// ---------------------------------------- +// Error handling +// ---------------------------------------- + +app.use(function (req, res, next) { + var err = new Error('Not Found') + err.status = 404 + next(err) +}) + +app.use(function (err, req, res, next) { + res.status(err.status || 500) + res.render('error', { + message: err.message, + error: wiki.IS_DEBUG ? err : {} + }) +}) + +// ---------------------------------------- +// Start HTTP server +// ---------------------------------------- + +wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...') + +app.set('port', wiki.config.port) +var server = http.createServer(app) +var io = socketio(server) + +server.listen(wiki.config.port) +server.on('error', (error) => { + if (error.syscall !== 'listen') { + throw error + } + + // handle specific listen errors with friendly messages + switch (error.code) { + case 'EACCES': + wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!') + return process.exit(1) + case 'EADDRINUSE': + wiki.logger.error('Port ' + wiki.config.port + ' is already in use!') + return process.exit(1) + default: + throw error + } +}) + +server.on('listening', () => { + wiki.logger.info('HTTP/WS server started successfully! [RUNNING]') +}) + +// ---------------------------------------- +// WebSocket +// ---------------------------------------- + +io.use(passportSocketIo.authorize({ + key: 'wikijs.sid', + store: sessionStore, + secret: wiki.config.sessionSecret, + cookieParser, + success: (data, accept) => { + accept() + }, + fail: (data, message, error, accept) => { + accept() + } +})) + +io.on('connection', ctrl.ws) + +// ---------------------------------------- +// Graceful shutdown +// ---------------------------------------- + +graceful.on('exit', () => { + // wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...') + // bgAgent.kill() + wiki.logger.info('- SHUTTING DOWN - Performing git sync...') + return global.git.resync().then(() => { + wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.') + process.exit() + }) +}) diff --git a/server/models/_relations.js b/server/models/_relations.js index 1f2f373f..7a851396 100644 --- a/server/models/_relations.js +++ b/server/models/_relations.js @@ -4,7 +4,10 @@ * Associate DB Model relations */ module.exports = db => { - db.User.belongsToMany(db.Group, { through: 'UserGroups' }) - db.Group.hasMany(db.Right, { as: 'GroupRights' }) + db.User.belongsToMany(db.Group, { through: 'userGroups' }) + db.Group.hasMany(db.Right, { as: 'groupRights' }) + db.Document.hasMany(db.Tag, { as: 'documentTags' }) db.File.belongsTo(db.Folder) + db.Comment.belongsTo(db.Document) + db.Comment.belongsTo(db.User, { as: 'author' }) } diff --git a/server/models/comment.js b/server/models/comment.js new file mode 100644 index 00000000..ecdb5806 --- /dev/null +++ b/server/models/comment.js @@ -0,0 +1,18 @@ +'use strict' + +/** + * Comment schema + */ +module.exports = (sequelize, DataTypes) => { + let commentSchema = sequelize.define('comment', { + content: { + type: DataTypes.STRING, + allowNull: false + } + }, { + timestamps: true, + version: true + }) + + return commentSchema +} diff --git a/server/models/document.js b/server/models/document.js index e742d492..76c9e0c7 100644 --- a/server/models/document.js +++ b/server/models/document.js @@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false + }, + searchContent: { + type: DataTypes.TEXT, + allowNull: true, + defaultValue: '' } }, { timestamps: true, diff --git a/server/models/tag.js b/server/models/tag.js new file mode 100644 index 00000000..ccd9363a --- /dev/null +++ b/server/models/tag.js @@ -0,0 +1,24 @@ +'use strict' + +/** + * Tags schema + */ +module.exports = (sequelize, DataTypes) => { + let tagSchema = sequelize.define('tag', { + key: { + type: DataTypes.STRING, + allowNull: false + } + }, { + timestamps: true, + version: true, + indexes: [ + { + unique: true, + fields: ['key'] + } + ] + }) + + return tagSchema +} diff --git a/server/modules/db.js b/server/modules/db.js index a849031d..89ca5ce1 100644 --- a/server/modules/db.js +++ b/server/modules/db.js @@ -64,7 +64,8 @@ module.exports = { // Sync DB self.onReady = self.inst.sync({ - force: false + force: false, + logging: false }) return self diff --git a/server/modules/logger.js b/server/modules/logger.js index d45a3ec4..08ecc175 100644 --- a/server/modules/logger.js +++ b/server/modules/logger.js @@ -2,12 +2,10 @@ /* global wiki */ -module.exports = (processName) => { - let winston = require('winston') +const cluster = require('cluster') - if (typeof processName === 'undefined') { - processName = 'SERVER' - } +module.exports = () => { + let winston = require('winston') // Console @@ -25,6 +23,7 @@ module.exports = (processName) => { }) logger.filters.push((level, msg) => { + let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}` return '[' + processName + '] ' + msg }) diff --git a/server/modules/search-index/index.js b/server/modules/search-index/index.js deleted file mode 100644 index 807219d9..00000000 --- a/server/modules/search-index/index.js +++ /dev/null @@ -1,81 +0,0 @@ -const bunyan = require('bunyan') -const level = require('levelup') -const down = require('memdown') -const SearchIndexAdder = require('search-index-adder') -const SearchIndexSearcher = require('search-index-searcher') - -module.exports = function (givenOptions, moduleReady) { - const optionsLoaded = function (err, SearchIndex) { - const siUtil = require('./siUtil.js')(SearchIndex.options) - if (err) return moduleReady(err) - SearchIndex.close = siUtil.close - SearchIndex.countDocs = siUtil.countDocs - getAdder(SearchIndex, adderLoaded) - } - - const adderLoaded = function (err, SearchIndex) { - if (err) return moduleReady(err) - getSearcher(SearchIndex, searcherLoaded) - } - - const searcherLoaded = function (err, SearchIndex) { - if (err) return moduleReady(err) - return moduleReady(err, SearchIndex) - } - - getOptions(givenOptions, optionsLoaded) -} - -const getAdder = function (SearchIndex, done) { - SearchIndexAdder(SearchIndex.options, function (err, searchIndexAdder) { - SearchIndex.add = searchIndexAdder.add - SearchIndex.callbackyAdd = searchIndexAdder.concurrentAdd // deprecated - SearchIndex.concurrentAdd = searchIndexAdder.concurrentAdd - SearchIndex.createWriteStream = searchIndexAdder.createWriteStream - SearchIndex.dbWriteStream = searchIndexAdder.dbWriteStream - SearchIndex.defaultPipeline = searchIndexAdder.defaultPipeline - SearchIndex.del = searchIndexAdder.deleter - SearchIndex.deleteStream = searchIndexAdder.deleteStream - SearchIndex.flush = searchIndexAdder.flush - done(err, SearchIndex) - }) -} - -const getSearcher = function (SearchIndex, done) { - SearchIndexSearcher(SearchIndex.options, function (err, searchIndexSearcher) { - SearchIndex.availableFields = searchIndexSearcher.availableFields - SearchIndex.buckets = searchIndexSearcher.bucketStream - SearchIndex.categorize = searchIndexSearcher.categoryStream - SearchIndex.dbReadStream = searchIndexSearcher.dbReadStream - SearchIndex.get = searchIndexSearcher.get - SearchIndex.match = searchIndexSearcher.match - SearchIndex.scan = searchIndexSearcher.scan - SearchIndex.search = searchIndexSearcher.search - SearchIndex.totalHits = searchIndexSearcher.totalHits - done(err, SearchIndex) - }) -} - -const getOptions = function (options, done) { - var SearchIndex = {} - SearchIndex.options = Object.assign({}, { - indexPath: 'si', - keySeparator: '￮', - logLevel: 'error' - }, options) - options.log = bunyan.createLogger({ - name: 'search-index', - level: options.logLevel - }) - if (!options.indexes) { - level(SearchIndex.options.indexPath || 'si', { - valueEncoding: 'json', - db: down - }, function (err, db) { - SearchIndex.options.indexes = db - return done(err, SearchIndex) - }) - } else { - return done(null, SearchIndex) - } -} diff --git a/server/modules/search-index/siUtil.js b/server/modules/search-index/siUtil.js deleted file mode 100644 index bad264a7..00000000 --- a/server/modules/search-index/siUtil.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict' - -module.exports = function (siOptions) { - var siUtil = {} - - siUtil.countDocs = function (callback) { - var count = 0 - const gte = 'DOCUMENT' + siOptions.keySeparator - const lte = 'DOCUMENT' + siOptions.keySeparator + siOptions.keySeparator - siOptions.indexes.createReadStream({gte: gte, lte: lte}) - .on('data', function (data) { - count++ - }) - .on('error', function (err) { - return callback(err, null) - }) - .on('end', function () { - return callback(null, count) - }) - } - - siUtil.close = function (callback) { - siOptions.indexes.close(function (err) { - while (!siOptions.indexes.isClosed()) { - // log not always working here- investigate - if (siOptions.log) siOptions.log.info('closing...') - } - if (siOptions.indexes.isClosed()) { - if (siOptions.log) siOptions.log.info('closed...') - callback(err) - } - }) - } - - return siUtil -} diff --git a/server/modules/search.js b/server/modules/search.js index d2978aa2..7801a739 100644 --- a/server/modules/search.js +++ b/server/modules/search.js @@ -4,7 +4,7 @@ const Promise = require('bluebird') const _ = require('lodash') -const searchIndex = require('./search-index') +// const searchIndex = require('./search-index') const stopWord = require('stopword') const streamToPromise = require('stream-to-promise') const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g') @@ -22,7 +22,7 @@ module.exports = { init () { let self = this self._isReady = new Promise((resolve, reject) => { - searchIndex({ + /*searchIndex({ deletable: true, fieldedSearch: true, indexPath: 'wiki', @@ -39,7 +39,7 @@ module.exports = { resolve(true) }) } - }) + }) */ }) return self diff --git a/server/agent.js b/server/worker.js similarity index 69% rename from server/agent.js rename to server/worker.js index 327496e8..7ad76277 100644 --- a/server/agent.js +++ b/server/worker.js @@ -1,39 +1,19 @@ -// =========================================== -// Wiki.js - Background Agent -// 1.0.1 -// Licensed under AGPLv3 -// =========================================== +'use strict' -const path = require('path') -const ROOTPATH = process.cwd() -const SERVERPATH = path.join(ROOTPATH, 'server') - -global.ROOTPATH = ROOTPATH -global.SERVERPATH = SERVERPATH -const IS_DEBUG = process.env.NODE_ENV === 'development' - -let appconf = require('./modules/config')() -global.appconfig = appconf.config -global.appdata = appconf.data - -// ---------------------------------------- -// Load Winston -// ---------------------------------------- +/* global wiki */ -global.winston = require('./modules/logger')(IS_DEBUG, 'AGENT') +const path = require('path') // ---------------------------------------- // Load global modules // ---------------------------------------- -global.winston.info('Background Agent is initializing...') - -global.db = require('./modules/db').init() -global.upl = require('./modules/uploads-agent').init() -global.git = require('./modules/git').init() -global.entries = require('./modules/entries').init() -global.lang = require('i18next') -global.mark = require('./modules/markdown') +wiki.db = require('./modules/db').init() +wiki.upl = require('./modules/uploads-agent').init() +wiki.git = require('./modules/git').init() +wiki.entries = require('./modules/entries').init() +wiki.lang = require('i18next') +wiki.mark = require('./modules/markdown') // ---------------------------------------- // Load modules @@ -52,20 +32,18 @@ const entryHelper = require('./helpers/entry') // Localization Engine // ---------------------------------------- -global.lang - .use(i18nBackend) - .init({ - load: 'languageOnly', - ns: ['common', 'admin', 'auth', 'errors', 'git'], - defaultNS: 'common', - saveMissing: false, - preload: [appconfig.lang], - lng: appconfig.lang, - fallbackLng: 'en', - backend: { - loadPath: path.join(SERVERPATH, 'locales/{{lng}}/{{ns}}.json') - } - }) +wiki.lang.use(i18nBackend).init({ + load: 'languageOnly', + ns: ['common', 'admin', 'auth', 'errors', 'git'], + defaultNS: 'common', + saveMissing: false, + preload: [wiki.config.lang], + lng: wiki.config.lang, + fallbackLng: 'en', + backend: { + loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') + } +}) // ---------------------------------------- // Start Cron @@ -75,8 +53,8 @@ let job let jobIsBusy = false let jobUplWatchStarted = false -global.db.onReady.then(() => { - return global.db.Entry.remove({}) +wiki.db.onReady.then(() => { + return wiki.db.Entry.remove({}) }).then(() => { job = new Cron({ cronTime: '0 */5 * * * *', @@ -84,17 +62,17 @@ global.db.onReady.then(() => { // Make sure we don't start two concurrent jobs if (jobIsBusy) { - global.winston.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)') + wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)') return } - global.winston.info('Running all jobs...') + wiki.logger.info('Running all jobs...') jobIsBusy = true // Prepare async job collector let jobs = [] - let repoPath = path.resolve(ROOTPATH, appconfig.paths.repo) - let dataPath = path.resolve(ROOTPATH, appconfig.paths.data) + let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo) + let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data) let uploadsTempPath = path.join(dataPath, 'temp-upload') // ---------------------------------------- @@ -105,7 +83,7 @@ global.db.onReady.then(() => { // -> Sync with Git remote //* **************************************** - jobs.push(global.git.resync().then(() => { + jobs.push(wiki.git.resync().then(() => { // -> Stream all documents let cacheJobs = [] @@ -185,18 +163,18 @@ global.db.onReady.then(() => { // ---------------------------------------- Promise.all(jobs).then(() => { - global.winston.info('All jobs completed successfully! Going to sleep for now.') + wiki.logger.info('All jobs completed successfully! Going to sleep for now.') if (!jobUplWatchStarted) { jobUplWatchStarted = true - global.upl.initialScan().then(() => { + wiki.upl.initialScan().then(() => { job.start() }) } return true }).catch((err) => { - global.winston.error('One or more jobs have failed: ', err) + wiki.logger.error('One or more jobs have failed: ', err) }).finally(() => { jobIsBusy = false }) @@ -212,7 +190,7 @@ global.db.onReady.then(() => { // ---------------------------------------- process.on('disconnect', () => { - global.winston.warn('Lost connection to main server. Exiting...') + wiki.logger.warn('Lost connection to main server. Exiting...') job.stop() process.exit() }) diff --git a/yarn.lock b/yarn.lock index 0bac2239..e52fcd68 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1036,7 +1036,7 @@ block-stream@*: dependencies: inherits "~2.0.0" -bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@~3.5.0: +bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@~3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c" @@ -1124,6 +1124,18 @@ builtin-modules@^1.0.0, builtin-modules@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" +bull@/home/nick3.0.0-rc.4: + version "3.0.0-rc.4" + resolved "https://registry.yarnpkg.com/bull/-/bull-3.0.0-rc.4.tgz#dea18e870787037183849fc0198982ed756589b7" + dependencies: + bluebird "^3.5.0" + cron-parser "^2.4.1" + debuglog "^1.0.0" + ioredis "^3.1.1" + lodash "^4.17.4" + semver "^5.3.0" + uuid "^3.1.0" + bunyan@^1.8.1, bunyan@^1.8.10, bunyan@^1.8.3, bunyan@~1.8.10: version "1.8.10" resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.10.tgz#201fedd26c7080b632f416072f53a90b9a52981c" @@ -1590,6 +1602,13 @@ crc@3.4.4, crc@^3.4.0: version "3.4.4" resolved "https://registry.yarnpkg.com/crc/-/crc-3.4.4.tgz#9da1e980e3bd44fc5c93bf5ab3da3378d85e466b" +cron-parser@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.4.1.tgz#022befce1af293e4d3144ff04c2cbd2edb491271" + dependencies: + is-nan "^1.2.1" + moment-timezone "^0.5.0" + cron@1.2.1, cron@~1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/cron/-/cron-1.2.1.tgz#3a86c09b41b8f261ac863a7cc85ea4735857eab2" @@ -1702,6 +1721,10 @@ debug@~2.2.0: dependencies: ms "0.7.1" +debuglog@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" + decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1732,6 +1755,13 @@ deferred-leveldown@~1.2.1: dependencies: abstract-leveldown "~2.4.0" +define-properties@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94" + dependencies: + foreach "^2.0.5" + object-keys "^1.0.8" + del@^2.0.2: version "2.2.2" resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8" @@ -2573,6 +2603,10 @@ for-own@^0.1.4: dependencies: for-in "^1.0.1" +foreach@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -3207,6 +3241,19 @@ invert-kv@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" +ioredis@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.2.tgz#2579e3eba6dc490f68f14c7b51346281332b467b" + dependencies: + bluebird "^3.3.4" + cluster-key-slot "^1.0.6" + debug "^2.2.0" + denque "^1.1.0" + flexbuffer "0.0.6" + lodash "^4.8.2" + redis-commands "^1.2.0" + redis-parser "^2.4.0" + ioredis@~3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.1.tgz#cc2f1d3232b8c95cc153046bce168f2baa1186e8" @@ -3320,6 +3367,12 @@ is-glob@^2.0.0, is-glob@^2.0.1: dependencies: is-extglob "^1.0.0" +is-nan@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.2.1.tgz#9faf65b6fb6db24b7f5c0628475ea71f988401e2" + dependencies: + define-properties "^1.1.1" + is-npm@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4" @@ -4574,7 +4627,7 @@ mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1: dependencies: minimist "0.0.8" -moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13: +moment-timezone@^0.5.0, moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13: version "0.5.13" resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.13.tgz#99ce5c7d827262eb0f1f702044177f60745d7b90" dependencies: @@ -4877,6 +4930,10 @@ object-component@0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291" +object-keys@^1.0.8: + version "1.0.11" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d" + object.omit@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa" @@ -7045,7 +7102,7 @@ uuid@^2.0.1, uuid@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a" -uuid@^3.0.0, uuid@^3.0.1: +uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"