From f32429325cc39edb8243052a266e750fde346f6c Mon Sep 17 00:00:00 2001 From: NGPixel Date: Sat, 29 Jul 2017 17:33:08 -0400 Subject: [PATCH] feat: modular auth + queue tasks --- config.sample.yml | 8 +- package.json | 1 - server/app/data.yml | 70 +---- server/authentication/azure.js | 33 +++ server/authentication/facebook.js | 28 ++ server/authentication/github.js | 28 ++ server/authentication/google.js | 27 ++ server/authentication/ldap.js | 41 +++ server/authentication/local.js | 34 +++ server/authentication/microsoft.js | 27 ++ server/authentication/slack.js | 27 ++ server/index.js | 29 +- server/master.js | 428 +++++++++++++++-------------- server/modules/auth.js | 269 +++--------------- server/modules/config.js | 121 +++++--- server/modules/db.js | 8 +- server/modules/disk.js | 8 +- server/modules/git.js | 2 +- server/modules/logger.js | 124 +++++---- server/modules/markdown.js | 6 +- server/modules/uploads-agent.js | 24 +- server/queues/git-sync.js | 65 +++++ server/queues/upl-clear-temp.js | 26 ++ server/worker.js | 170 ++---------- yarn.lock | 2 +- 25 files changed, 829 insertions(+), 777 deletions(-) create mode 100644 server/authentication/azure.js create mode 100644 server/authentication/facebook.js create mode 100644 server/authentication/github.js create mode 100644 server/authentication/google.js create mode 100644 server/authentication/ldap.js create mode 100644 server/authentication/local.js create mode 100644 server/authentication/microsoft.js create mode 100644 server/authentication/slack.js create mode 100644 server/queues/git-sync.js create mode 100644 server/queues/upl-clear-temp.js diff --git a/config.sample.yml b/config.sample.yml index d3359c9c..a42a67ba 100644 --- a/config.sample.yml +++ b/config.sample.yml @@ -5,7 +5,7 @@ # https://docs.requarks.io/wiki/install # --------------------------------------------------------------------- -# Port the main server should listen to (80 by default) +# Port the main server should listen to # --------------------------------------------------------------------- port: 80 @@ -39,3 +39,9 @@ redis: db: 0 password: null +# --------------------------------------------------------------------- +# Background Workers +# --------------------------------------------------------------------- + +# Leave 0 for auto based on CPU cores +workers: 0 diff --git a/package.json b/package.json index d20fd14e..06f4fbec 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,6 @@ "connect-flash": "~0.1.1", "connect-redis": "~3.3.0", "cookie-parser": "~1.4.3", - "cron": "~1.2.1", "diff2html": "~2.3.0", "execa": "~0.7.0", "express": "~4.15.3", diff --git a/server/app/data.yml b/server/app/data.yml index cd8e839d..8ab67828 100644 --- a/server/app/data.yml +++ b/server/app/data.yml @@ -5,65 +5,22 @@ name: Wiki.js defaults: config: - title: Wiki - host: http://localhost port: 80 paths: repo: ./repo data: ./data - uploads: - maxImageFileSize: 3, - maxOtherFileSize: 100 - lang: en - public: false - auth: - defaultReadAccess: false - local: - enabled: true - microsoft: - enabled: false - google: - enabled: false - facebook: - enabled: false - github: - enabled: false - slack: - enabled: false - ldap: - enabled: false - azure: - enabled: false - db: mongodb://localhost/wiki - sessionSecret: null - admin: null - git: - url: null - branch: master - auth: - type: basic - username: null - password: null - privateKey: null - sslVerify: true - serverEmail: wiki@example.com - showUserEmail: true - features: - linebreaks: true - mathjax: true - externalLogging: - bugsnap: false - loggly: false - papertrail: false - rollbar: false - sentry: false - theme: - primary: indigo - alt: blue-grey - footer: blue-grey - code: - dark: true - colorize: true + db: + host: localhost + port: 5432 + user: wikijs + pass: wikijsrocks + db: wiki + redis: + host: localhost + port: 6379 + db: 0 + password: null + workers: 0 authProviders: - local - microsoft @@ -112,6 +69,9 @@ langs: - id: ko name: Korean - 한국어 + - + id: pt + name: Portuguese - Português - id: ru name: Russian - Русский diff --git a/server/authentication/azure.js b/server/authentication/azure.js new file mode 100644 index 00000000..1759aa9f --- /dev/null +++ b/server/authentication/azure.js @@ -0,0 +1,33 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// Azure AD Account +// ------------------------------------ + +const AzureAdOAuth2Strategy = require('passport-azure-ad-oauth2').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.azure && wiki.config.auth.azure.enabled) { + const jwt = require('jsonwebtoken') + passport.use('azure_ad_oauth2', + new AzureAdOAuth2Strategy({ + clientID: wiki.config.auth.azure.clientId, + clientSecret: wiki.config.auth.azure.clientSecret, + callbackURL: wiki.config.host + '/login/azure/callback', + resource: wiki.config.auth.azure.resource, + tenant: wiki.config.auth.azure.tenant + }, (accessToken, refreshToken, params, profile, cb) => { + let waadProfile = jwt.decode(params.id_token) + waadProfile.id = waadProfile.oid + waadProfile.provider = 'azure' + wiki.db.User.processProfile(waadProfile).then((user) => { + return cb(null, user) || true + }).catch((err) => { + return cb(err, null) || true + }) + } + )) + } +} diff --git a/server/authentication/facebook.js b/server/authentication/facebook.js new file mode 100644 index 00000000..db1c4ef9 --- /dev/null +++ b/server/authentication/facebook.js @@ -0,0 +1,28 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// Facebook Account +// ------------------------------------ + +const FacebookStrategy = require('passport-facebook').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.facebook && wiki.config.auth.facebook.enabled) { + passport.use('facebook', + new FacebookStrategy({ + clientID: wiki.config.auth.facebook.clientId, + clientSecret: wiki.config.auth.facebook.clientSecret, + callbackURL: wiki.config.host + '/login/facebook/callback', + profileFields: ['id', 'displayName', 'email'] + }, function (accessToken, refreshToken, profile, cb) { + wiki.db.User.processProfile(profile).then((user) => { + return cb(null, user) || true + }).catch((err) => { + return cb(err, null) || true + }) + } + )) + } +} diff --git a/server/authentication/github.js b/server/authentication/github.js new file mode 100644 index 00000000..c26df1b3 --- /dev/null +++ b/server/authentication/github.js @@ -0,0 +1,28 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// GitHub Account +// ------------------------------------ + +const GitHubStrategy = require('passport-github2').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.github && wiki.config.auth.github.enabled) { + passport.use('github', + new GitHubStrategy({ + clientID: wiki.config.auth.github.clientId, + clientSecret: wiki.config.auth.github.clientSecret, + callbackURL: wiki.config.host + '/login/github/callback', + scope: ['user:email'] + }, (accessToken, refreshToken, profile, cb) => { + wiki.db.User.processProfile(profile).then((user) => { + return cb(null, user) || true + }).catch((err) => { + return cb(err, null) || true + }) + } + )) + } +} diff --git a/server/authentication/google.js b/server/authentication/google.js new file mode 100644 index 00000000..12d51f2e --- /dev/null +++ b/server/authentication/google.js @@ -0,0 +1,27 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// Google ID Account +// ------------------------------------ + +const GoogleStrategy = require('passport-google-oauth20').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.google && wiki.config.auth.google.enabled) { + passport.use('google', + new GoogleStrategy({ + clientID: wiki.config.auth.google.clientId, + clientSecret: wiki.config.auth.google.clientSecret, + callbackURL: wiki.config.host + '/login/google/callback' + }, (accessToken, refreshToken, profile, cb) => { + wiki.db.User.processProfile(profile).then((user) => { + return cb(null, user) || true + }).catch((err) => { + return cb(err, null) || true + }) + } + )) + } +} diff --git a/server/authentication/ldap.js b/server/authentication/ldap.js new file mode 100644 index 00000000..cc3ebf7c --- /dev/null +++ b/server/authentication/ldap.js @@ -0,0 +1,41 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// LDAP Account +// ------------------------------------ + +const LdapStrategy = require('passport-ldapauth').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.ldap && wiki.config.auth.ldap.enabled) { + passport.use('ldapauth', + new LdapStrategy({ + server: { + url: wiki.config.auth.ldap.url, + bindDn: wiki.config.auth.ldap.bindDn, + bindCredentials: wiki.config.auth.ldap.bindCredentials, + searchBase: wiki.config.auth.ldap.searchBase, + searchFilter: wiki.config.auth.ldap.searchFilter, + searchAttributes: ['displayName', 'name', 'cn', 'mail'], + tlsOptions: (wiki.config.auth.ldap.tlsEnabled) ? { + ca: [ + fs.readFileSync(wiki.config.auth.ldap.tlsCertPath) + ] + } : {} + }, + usernameField: 'email', + passReqToCallback: false + }, (profile, cb) => { + profile.provider = 'ldap' + profile.id = profile.dn + wiki.db.User.processProfile(profile).then((user) => { + return cb(null, user) || true + }).catch((err) => { + return cb(err, null) || true + }) + } + )) + } +} diff --git a/server/authentication/local.js b/server/authentication/local.js new file mode 100644 index 00000000..3af57126 --- /dev/null +++ b/server/authentication/local.js @@ -0,0 +1,34 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// Local Account +// ------------------------------------ + +const LocalStrategy = require('passport-local').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.local && wiki.config.auth.local.enabled) { + passport.use('local', + new LocalStrategy({ + usernameField: 'email', + passwordField: 'password' + }, (uEmail, uPassword, done) => { + wiki.db.User.findOne({ email: uEmail, provider: 'local' }).then((user) => { + if (user) { + return user.validatePassword(uPassword).then(() => { + return done(null, user) || true + }).catch((err) => { + return done(err, null) + }) + } else { + return done(new Error('INVALID_LOGIN'), null) + } + }).catch((err) => { + done(err, null) + }) + } + )) + } +} diff --git a/server/authentication/microsoft.js b/server/authentication/microsoft.js new file mode 100644 index 00000000..1d39eb47 --- /dev/null +++ b/server/authentication/microsoft.js @@ -0,0 +1,27 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// Microsoft Account +// ------------------------------------ + +const WindowsLiveStrategy = require('passport-windowslive').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.microsoft && wiki.config.auth.microsoft.enabled) { + passport.use('windowslive', + new WindowsLiveStrategy({ + clientID: wiki.config.auth.microsoft.clientId, + clientSecret: wiki.config.auth.microsoft.clientSecret, + callbackURL: wiki.config.host + '/login/ms/callback' + }, function (accessToken, refreshToken, profile, cb) { + wiki.db.User.processProfile(profile).then((user) => { + return cb(null, user) || true + }).catch((err) => { + return cb(err, null) || true + }) + } + )) + } +} diff --git a/server/authentication/slack.js b/server/authentication/slack.js new file mode 100644 index 00000000..dcc8a7f2 --- /dev/null +++ b/server/authentication/slack.js @@ -0,0 +1,27 @@ +'use strict' + +/* global wiki */ + +// ------------------------------------ +// Slack Account +// ------------------------------------ + +const SlackStrategy = require('passport-slack').Strategy + +module.exports = (passport) => { + if (wiki.config.auth.slack && wiki.config.auth.slack.enabled) { + passport.use('slack', + new SlackStrategy({ + clientID: wiki.config.auth.slack.clientId, + clientSecret: wiki.config.auth.slack.clientSecret, + callbackURL: wiki.config.host + '/login/slack/callback' + }, (accessToken, refreshToken, profile, cb) => { + wiki.db.User.processProfile(profile).then((user) => { + return cb(null, user) || true + }).catch((err) => { + return cb(err, null) || true + }) + } + )) + } +} diff --git a/server/index.js b/server/index.js index aeb2f8a5..24f65480 100644 --- a/server/index.js +++ b/server/index.js @@ -6,10 +6,14 @@ // =========================================== const path = require('path') +const cluster = require('cluster') + let wiki = { IS_DEBUG: process.env.NODE_ENV === 'development', + IS_MASTER: cluster.isMaster, ROOTPATH: process.cwd(), - SERVERPATH: path.join(process.cwd(), 'server') + SERVERPATH: path.join(process.cwd(), 'server'), + configSvc: require('./modules/config') } global.wiki = wiki @@ -19,29 +23,36 @@ process.env.VIPS_WARNING = false // require('@glimpse/glimpse').init() // } -let appconf = require('./modules/config')() -wiki.config = appconf.config -wiki.data = appconf.data +wiki.configSvc.init() + +// ---------------------------------------- +// Init Logger +// ---------------------------------------- + +wiki.logger = require('./modules/logger').init() // ---------------------------------------- -// Load Winston +// Init DB // ---------------------------------------- -wiki.logger = require('./modules/logger')() +wiki.db = require('./modules/db').init() // ---------------------------------------- // Start Cluster // ---------------------------------------- -const cluster = require('cluster') const numCPUs = require('os').cpus().length +let numWorkers = (wiki.config.workers > 0) ? wiki.config.workers : numCPUs +if (numWorkers > numCPUs) { + numWorkers = numCPUs +} if (cluster.isMaster) { wiki.logger.info('Wiki.js is initializing...') require('./master') - for (let i = 0; i < numCPUs; i++) { + for (let i = 0; i < numWorkers; i++) { cluster.fork() } @@ -50,5 +61,5 @@ if (cluster.isMaster) { }) } else { wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`) - // require('./worker') + require('./worker') } diff --git a/server/master.js b/server/master.js index 75a609c8..12248ba4 100644 --- a/server/master.js +++ b/server/master.js @@ -2,231 +2,241 @@ /* global wiki */ -const path = require('path') - -// ---------------------------------------- -// Load global modules -// ---------------------------------------- - -wiki.disk = require('./modules/disk').init() -wiki.db = require('./modules/db').init() -wiki.entries = require('./modules/entries').init() -wiki.git = require('./modules/git').init(false) -wiki.lang = require('i18next') -wiki.mark = require('./modules/markdown') -wiki.redis = require('./modules/redis').init() -wiki.search = require('./modules/search').init() -wiki.upl = require('./modules/uploads').init() - -// ---------------------------------------- -// Load modules -// ---------------------------------------- - -const autoload = require('auto-load') -const bodyParser = require('body-parser') -const compression = require('compression') -const cookieParser = require('cookie-parser') -const express = require('express') -const favicon = require('serve-favicon') -const flash = require('connect-flash') -const http = require('http') -const i18nBackend = require('i18next-node-fs-backend') -const passport = require('passport') -const passportSocketIo = require('passport.socketio') -const session = require('express-session') -const SessionRedisStore = require('connect-redis')(session) -const graceful = require('node-graceful') -const socketio = require('socket.io') -const graphqlApollo = require('apollo-server-express') -const graphqlSchema = require('./modules/graphql') - -var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares')) -var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers')) - -// ---------------------------------------- -// Define Express App -// ---------------------------------------- - -const app = express() -wiki.app = app -app.use(compression()) - -// ---------------------------------------- -// Security -// ---------------------------------------- - -app.use(mw.security) - -// ---------------------------------------- -// Public Assets -// ---------------------------------------- - -app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico'))) -app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), { - index: false, - maxAge: '7d' -})) - -// ---------------------------------------- -// Passport Authentication -// ---------------------------------------- - -require('./modules/auth')(passport) -wiki.rights = require('./modules/rights') -wiki.rights.init() - -let sessionStore = new SessionRedisStore({ - client: wiki.redis -}) +const Promise = require('bluebird') + +module.exports = Promise.join( + wiki.db.onReady, + wiki.configSvc.loadFromDb() +).then(() => { + // ---------------------------------------- + // Load global modules + // ---------------------------------------- + + wiki.disk = require('./modules/disk').init() + wiki.entries = require('./modules/entries').init() + wiki.git = require('./modules/git').init(false) + wiki.lang = require('i18next') + wiki.mark = require('./modules/markdown') + wiki.redis = require('./modules/redis').init() + wiki.search = require('./modules/search').init() + wiki.upl = require('./modules/uploads').init() + + // ---------------------------------------- + // Load modules + // ---------------------------------------- + + const autoload = require('auto-load') + const bodyParser = require('body-parser') + const compression = require('compression') + const cookieParser = require('cookie-parser') + const express = require('express') + const favicon = require('serve-favicon') + const flash = require('connect-flash') + const http = require('http') + const i18nBackend = require('i18next-node-fs-backend') + const path = require('path') + const passport = require('passport') + const passportSocketIo = require('passport.socketio') + const session = require('express-session') + const SessionRedisStore = require('connect-redis')(session) + const graceful = require('node-graceful') + const socketio = require('socket.io') + const graphqlApollo = require('apollo-server-express') + const graphqlSchema = require('./modules/graphql') + + var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares')) + var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers')) + + // ---------------------------------------- + // Define Express App + // ---------------------------------------- + + const app = express() + wiki.app = app + app.use(compression()) + + // ---------------------------------------- + // Security + // ---------------------------------------- + + app.use(mw.security) + + // ---------------------------------------- + // Public Assets + // ---------------------------------------- + + app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico'))) + app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), { + index: false, + maxAge: '7d' + })) + + // ---------------------------------------- + // Passport Authentication + // ---------------------------------------- + + require('./modules/auth')(passport) + wiki.rights = require('./modules/rights') + // wiki.rights.init() + + let sessionStore = new SessionRedisStore({ + client: wiki.redis + }) -app.use(cookieParser()) -app.use(session({ - name: 'wikijs.sid', - store: sessionStore, - secret: wiki.config.sessionSecret, - resave: false, - saveUninitialized: false -})) -app.use(flash()) -app.use(passport.initialize()) -app.use(passport.session()) - -// ---------------------------------------- -// SEO -// ---------------------------------------- - -app.use(mw.seo) - -// ---------------------------------------- -// Localization Engine -// ---------------------------------------- - -wiki.lang.use(i18nBackend).init({ - load: 'languageOnly', - ns: ['common', 'admin', 'auth', 'errors', 'git'], - defaultNS: 'common', - saveMissing: false, - preload: [wiki.config.lang], - lng: wiki.config.lang, - fallbackLng: 'en', - backend: { - loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') - } -}) + app.use(cookieParser()) + app.use(session({ + name: 'wikijs.sid', + store: sessionStore, + secret: wiki.config.site.sessionSecret, + resave: false, + saveUninitialized: false + })) + app.use(flash()) + app.use(passport.initialize()) + app.use(passport.session()) + + // ---------------------------------------- + // SEO + // ---------------------------------------- + + app.use(mw.seo) + + // ---------------------------------------- + // Localization Engine + // ---------------------------------------- + + wiki.lang.use(i18nBackend).init({ + load: 'languageOnly', + ns: ['common', 'admin', 'auth', 'errors', 'git'], + defaultNS: 'common', + saveMissing: false, + preload: [wiki.config.site.lang], + lng: wiki.config.site.lang, + fallbackLng: 'en', + backend: { + loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json') + } + }) -// ---------------------------------------- -// View Engine Setup -// ---------------------------------------- + // ---------------------------------------- + // View Engine Setup + // ---------------------------------------- -app.set('views', path.join(wiki.SERVERPATH, 'views')) -app.set('view engine', 'pug') + app.set('views', path.join(wiki.SERVERPATH, 'views')) + app.set('view engine', 'pug') -app.use(bodyParser.json({ limit: '1mb' })) -app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' })) + app.use(bodyParser.json({ limit: '1mb' })) + app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' })) -// ---------------------------------------- -// View accessible data -// ---------------------------------------- + // ---------------------------------------- + // View accessible data + // ---------------------------------------- -app.locals._ = require('lodash') -app.locals.t = wiki.lang.t.bind(wiki.lang) -app.locals.moment = require('moment') -app.locals.moment.locale(wiki.config.lang) -app.locals.appconfig = wiki.config -app.use(mw.flash) + app.locals._ = require('lodash') + app.locals.t = wiki.lang.t.bind(wiki.config.site.lang) + app.locals.moment = require('moment') + app.locals.moment.locale(wiki.config.site.lang) + app.locals.appconfig = wiki.config + app.use(mw.flash) -// ---------------------------------------- -// Controllers -// ---------------------------------------- + // ---------------------------------------- + // Controllers + // ---------------------------------------- -app.use('/', ctrl.auth) + app.use('/', ctrl.auth) -app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema })) -app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' })) -app.use('/uploads', mw.auth, ctrl.uploads) -app.use('/admin', mw.auth, ctrl.admin) -app.use('/', mw.auth, ctrl.pages) + app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema })) + app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' })) + app.use('/uploads', mw.auth, ctrl.uploads) + app.use('/admin', mw.auth, ctrl.admin) + app.use('/', mw.auth, ctrl.pages) -// ---------------------------------------- -// Error handling -// ---------------------------------------- + // ---------------------------------------- + // Error handling + // ---------------------------------------- -app.use(function (req, res, next) { - var err = new Error('Not Found') - err.status = 404 - next(err) -}) + app.use(function (req, res, next) { + var err = new Error('Not Found') + err.status = 404 + next(err) + }) -app.use(function (err, req, res, next) { - res.status(err.status || 500) - res.render('error', { - message: err.message, - error: wiki.IS_DEBUG ? err : {} + app.use(function (err, req, res, next) { + res.status(err.status || 500) + res.render('error', { + message: err.message, + error: wiki.IS_DEBUG ? err : {} + }) }) -}) -// ---------------------------------------- -// Start HTTP server -// ---------------------------------------- - -wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...') - -app.set('port', wiki.config.port) -var server = http.createServer(app) -var io = socketio(server) - -server.listen(wiki.config.port) -server.on('error', (error) => { - if (error.syscall !== 'listen') { - throw error - } - - // handle specific listen errors with friendly messages - switch (error.code) { - case 'EACCES': - wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!') - return process.exit(1) - case 'EADDRINUSE': - wiki.logger.error('Port ' + wiki.config.port + ' is already in use!') - return process.exit(1) - default: + // ---------------------------------------- + // Start HTTP server + // ---------------------------------------- + + wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...') + + app.set('port', wiki.config.port) + var server = http.createServer(app) + var io = socketio(server) + + server.listen(wiki.config.port) + server.on('error', (error) => { + if (error.syscall !== 'listen') { throw error - } -}) + } + + // handle specific listen errors with friendly messages + switch (error.code) { + case 'EACCES': + wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!') + return process.exit(1) + case 'EADDRINUSE': + wiki.logger.error('Port ' + wiki.config.port + ' is already in use!') + return process.exit(1) + default: + throw error + } + }) -server.on('listening', () => { - wiki.logger.info('HTTP/WS server started successfully! [RUNNING]') -}) + server.on('listening', () => { + wiki.logger.info('HTTP/WS server started successfully! [RUNNING]') + }) -// ---------------------------------------- -// WebSocket -// ---------------------------------------- - -io.use(passportSocketIo.authorize({ - key: 'wikijs.sid', - store: sessionStore, - secret: wiki.config.sessionSecret, - cookieParser, - success: (data, accept) => { - accept() - }, - fail: (data, message, error, accept) => { - accept() - } -})) - -io.on('connection', ctrl.ws) - -// ---------------------------------------- -// Graceful shutdown -// ---------------------------------------- - -graceful.on('exit', () => { - // wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...') - // bgAgent.kill() - wiki.logger.info('- SHUTTING DOWN - Performing git sync...') - return global.git.resync().then(() => { - wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.') - process.exit() + // ---------------------------------------- + // WebSocket + // ---------------------------------------- + + io.use(passportSocketIo.authorize({ + key: 'wikijs.sid', + store: sessionStore, + secret: wiki.config.site.sessionSecret, + cookieParser, + success: (data, accept) => { + accept() + }, + fail: (data, message, error, accept) => { + accept() + } + })) + + io.on('connection', ctrl.ws) + + // ---------------------------------------- + // Graceful shutdown + // ---------------------------------------- + + graceful.on('exit', () => { + // wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...') + // bgAgent.kill() + wiki.logger.info('- SHUTTING DOWN - Performing git sync...') + return global.git.resync().then(() => { + wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.') + process.exit() + }) }) + + return true +}).catch(err => { + wiki.logger.error(err) + process.exit(1) }) diff --git a/server/modules/auth.js b/server/modules/auth.js index 60bc49d8..3e96e730 100644 --- a/server/modules/auth.js +++ b/server/modules/auth.js @@ -24,232 +24,51 @@ module.exports = function (passport) { }) }) - // Local Account - - if (wiki.config.auth.local && wiki.config.auth.local.enabled) { - const LocalStrategy = require('passport-local').Strategy - passport.use('local', - new LocalStrategy({ - usernameField: 'email', - passwordField: 'password' - }, (uEmail, uPassword, done) => { - wiki.db.User.findOne({ email: uEmail, provider: 'local' }).then((user) => { - if (user) { - return user.validatePassword(uPassword).then(() => { - return done(null, user) || true - }).catch((err) => { - return done(err, null) - }) - } else { - return done(new Error('INVALID_LOGIN'), null) - } - }).catch((err) => { - done(err, null) - }) - } - )) - } - - // Google ID - - if (wiki.config.auth.google && wiki.config.auth.google.enabled) { - const GoogleStrategy = require('passport-google-oauth20').Strategy - passport.use('google', - new GoogleStrategy({ - clientID: wiki.config.auth.google.clientId, - clientSecret: wiki.config.auth.google.clientSecret, - callbackURL: wiki.config.host + '/login/google/callback' - }, (accessToken, refreshToken, profile, cb) => { - wiki.db.User.processProfile(profile).then((user) => { - return cb(null, user) || true - }).catch((err) => { - return cb(err, null) || true - }) - } - )) - } - - // Microsoft Accounts - - if (wiki.config.auth.microsoft && wiki.config.auth.microsoft.enabled) { - const WindowsLiveStrategy = require('passport-windowslive').Strategy - passport.use('windowslive', - new WindowsLiveStrategy({ - clientID: wiki.config.auth.microsoft.clientId, - clientSecret: wiki.config.auth.microsoft.clientSecret, - callbackURL: wiki.config.host + '/login/ms/callback' - }, function (accessToken, refreshToken, profile, cb) { - wiki.db.User.processProfile(profile).then((user) => { - return cb(null, user) || true - }).catch((err) => { - return cb(err, null) || true - }) - } - )) - } - - // Facebook - - if (wiki.config.auth.facebook && wiki.config.auth.facebook.enabled) { - const FacebookStrategy = require('passport-facebook').Strategy - passport.use('facebook', - new FacebookStrategy({ - clientID: wiki.config.auth.facebook.clientId, - clientSecret: wiki.config.auth.facebook.clientSecret, - callbackURL: wiki.config.host + '/login/facebook/callback', - profileFields: ['id', 'displayName', 'email'] - }, function (accessToken, refreshToken, profile, cb) { - wiki.db.User.processProfile(profile).then((user) => { - return cb(null, user) || true - }).catch((err) => { - return cb(err, null) || true - }) - } - )) - } - - // GitHub - - if (wiki.config.auth.github && wiki.config.auth.github.enabled) { - const GitHubStrategy = require('passport-github2').Strategy - passport.use('github', - new GitHubStrategy({ - clientID: wiki.config.auth.github.clientId, - clientSecret: wiki.config.auth.github.clientSecret, - callbackURL: wiki.config.host + '/login/github/callback', - scope: ['user:email'] - }, (accessToken, refreshToken, profile, cb) => { - wiki.db.User.processProfile(profile).then((user) => { - return cb(null, user) || true - }).catch((err) => { - return cb(err, null) || true - }) - } - )) - } - - // Slack - - if (wiki.config.auth.slack && wiki.config.auth.slack.enabled) { - const SlackStrategy = require('passport-slack').Strategy - passport.use('slack', - new SlackStrategy({ - clientID: wiki.config.auth.slack.clientId, - clientSecret: wiki.config.auth.slack.clientSecret, - callbackURL: wiki.config.host + '/login/slack/callback' - }, (accessToken, refreshToken, profile, cb) => { - wiki.db.User.processProfile(profile).then((user) => { - return cb(null, user) || true - }).catch((err) => { - return cb(err, null) || true - }) - } - )) - } - - // LDAP - - if (wiki.config.auth.ldap && wiki.config.auth.ldap.enabled) { - const LdapStrategy = require('passport-ldapauth').Strategy - passport.use('ldapauth', - new LdapStrategy({ - server: { - url: wiki.config.auth.ldap.url, - bindDn: wiki.config.auth.ldap.bindDn, - bindCredentials: wiki.config.auth.ldap.bindCredentials, - searchBase: wiki.config.auth.ldap.searchBase, - searchFilter: wiki.config.auth.ldap.searchFilter, - searchAttributes: ['displayName', 'name', 'cn', 'mail'], - tlsOptions: (wiki.config.auth.ldap.tlsEnabled) ? { - ca: [ - fs.readFileSync(wiki.config.auth.ldap.tlsCertPath) - ] - } : {} - }, - usernameField: 'email', - passReqToCallback: false - }, (profile, cb) => { - profile.provider = 'ldap' - profile.id = profile.dn - wiki.db.User.processProfile(profile).then((user) => { - return cb(null, user) || true - }).catch((err) => { - return cb(err, null) || true - }) - } - )) - } - - // AZURE AD - - if (wiki.config.auth.azure && wiki.config.auth.azure.enabled) { - const AzureAdOAuth2Strategy = require('passport-azure-ad-oauth2').Strategy - const jwt = require('jsonwebtoken') - passport.use('azure_ad_oauth2', - new AzureAdOAuth2Strategy({ - clientID: wiki.config.auth.azure.clientId, - clientSecret: wiki.config.auth.azure.clientSecret, - callbackURL: wiki.config.host + '/login/azure/callback', - resource: wiki.config.auth.azure.resource, - tenant: wiki.config.auth.azure.tenant - }, (accessToken, refreshToken, params, profile, cb) => { - let waadProfile = jwt.decode(params.id_token) - waadProfile.id = waadProfile.oid - waadProfile.provider = 'azure' - wiki.db.User.processProfile(waadProfile).then((user) => { - return cb(null, user) || true - }).catch((err) => { - return cb(err, null) || true - }) - } - )) - } - // Create users for first-time - wiki.db.onReady.then(() => { - return wiki.db.User.findOne({ provider: 'local', email: 'guest' }).then((c) => { - if (c < 1) { - // Create guest account - - return wiki.db.User.create({ - provider: 'local', - email: 'guest@example.com', - name: 'Guest', - password: '', - role: 'guest' - }).then(() => { - wiki.logger.info('[AUTH] Guest account created successfully!') - return true - }).catch((err) => { - wiki.logger.error('[AUTH] An error occured while creating guest account:') - wiki.logger.error(err) - return err - }) - } - }).then(() => { - if (process.env.WIKI_JS_HEROKU) { - return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => { - if (c < 1) { - // Create root admin account (HEROKU ONLY) - - return wiki.db.User.create({ - provider: 'local', - email: process.env.WIKI_ADMIN_EMAIL, - name: 'Administrator', - password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default) - role: 'admin' - }).then(() => { - wiki.logger.info('[AUTH] Root admin account created successfully!') - return true - }).catch((err) => { - wiki.logger.error('[AUTH] An error occured while creating root admin account:') - wiki.logger.error(err) - return err - }) - } else { return true } - }) - } else { return true } - }) + return wiki.db.User.findOne({ provider: 'local', email: 'guest@example.com' }).then((c) => { + if (c < 1) { + // Create guest account + + return wiki.db.User.create({ + provider: 'local', + email: 'guest@example.com', + name: 'Guest', + password: '', + role: 'guest' + }).then(() => { + wiki.logger.info('[AUTH] Guest account created successfully!') + return true + }).catch((err) => { + wiki.logger.error('[AUTH] An error occured while creating guest account:') + wiki.logger.error(err) + return err + }) + } }) + + // .then(() => { + // if (process.env.WIKI_JS_HEROKU) { + // return wiki.db.User.findOne({ provider: 'local', email: process.env.WIKI_ADMIN_EMAIL }).then((c) => { + // if (c < 1) { + // // Create root admin account (HEROKU ONLY) + + // return wiki.db.User.create({ + // provider: 'local', + // email: process.env.WIKI_ADMIN_EMAIL, + // name: 'Administrator', + // password: '$2a$04$MAHRw785Xe/Jd5kcKzr3D.VRZDeomFZu2lius4gGpZZ9cJw7B7Mna', // admin123 (default) + // role: 'admin' + // }).then(() => { + // wiki.logger.info('[AUTH] Root admin account created successfully!') + // return true + // }).catch((err) => { + // wiki.logger.error('[AUTH] An error occured while creating root admin account:') + // wiki.logger.error(err) + // return err + // }) + // } else { return true } + // }) + // } else { return true } + // }) } diff --git a/server/modules/config.js b/server/modules/config.js index 4c120338..bb07c02f 100644 --- a/server/modules/config.js +++ b/server/modules/config.js @@ -8,62 +8,95 @@ const _ = require('lodash') const path = require('path') const cfgHelper = require('../helpers/config') -/** - * Load Application Configuration - * - * @param {Object} confPaths Path to the configuration files - * @return {Object} Application Configuration - */ -module.exports = (confPaths) => { - confPaths = _.defaults(confPaths, { - config: path.join(wiki.ROOTPATH, 'config.yml'), - data: path.join(wiki.SERVERPATH, 'app/data.yml'), - dataRegex: path.join(wiki.SERVERPATH, 'app/regex.js') - }) - - let appconfig = {} - let appdata = {} - - try { - appconfig = yaml.safeLoad( - cfgHelper.parseConfigValue( - fs.readFileSync(confPaths.config, 'utf8') +module.exports = { + SUBSETS: ['auth', 'features', 'git', 'logging', 'site', 'theme', 'uploads'], + + /** + * Load root config from disk + * + * @param {any} confPaths + * @returns + */ + init() { + let confPaths = { + config: path.join(wiki.ROOTPATH, 'config.yml'), + data: path.join(wiki.SERVERPATH, 'app/data.yml'), + dataRegex: path.join(wiki.SERVERPATH, 'app/regex.js') + } + + let appconfig = {} + let appdata = {} + + try { + appconfig = yaml.safeLoad( + cfgHelper.parseConfigValue( + fs.readFileSync(confPaths.config, 'utf8') + ) ) - ) - appdata = yaml.safeLoad(fs.readFileSync(confPaths.data, 'utf8')) - appdata.regex = require(confPaths.dataRegex) - } catch (ex) { - console.error(ex) - process.exit(1) - } + appdata = yaml.safeLoad(fs.readFileSync(confPaths.data, 'utf8')) + appdata.regex = require(confPaths.dataRegex) + } catch (ex) { + console.error(ex) + process.exit(1) + } - // Merge with defaults + // Merge with defaults - appconfig = _.defaultsDeep(appconfig, appdata.defaults.config) + appconfig = _.defaultsDeep(appconfig, appdata.defaults.config) - // Check port + // Check port - if (appconfig.port < 1) { - appconfig.port = process.env.PORT || 80 - } + if (appconfig.port < 1) { + appconfig.port = process.env.PORT || 80 + } // Convert booleans appconfig.public = (appconfig.public === true || _.toLower(appconfig.public) === 'true') // List authentication strategies + wiki.config = appconfig + wiki.data = appdata - appconfig.authStrategies = { - list: _.filter(appconfig.auth, ['enabled', true]), - socialEnabled: (_.chain(appconfig.auth).omit(['local', 'ldap']).filter(['enabled', true]).value().length > 0) - } - if (appconfig.authStrategies.list.length < 1) { - console.error(new Error('You must enable at least 1 authentication strategy!')) - process.exit(1) - } + // List authentication strategies + + // appconfig.authStrategies = { + // list: _.filter(appconfig.auth, ['enabled', true]), + // socialEnabled: (_.chain(appconfig.auth).omit('local').filter(['enabled', true]).value().length > 0) + // } + // if (appconfig.authStrategies.list.length < 1) { + // console.error(new Error('You must enable at least 1 authentication strategy!')) + // process.exit(1) + // } + }, + + /** + * Load config from DB + * + * @param {Array} subsets Array of subsets to load + * @returns Promise + */ + loadFromDb(subsets) { + if (!_.isArray(subsets) || subsets.length === 0) { + subsets = this.SUBSETS + } - return { - config: appconfig, - data: appdata + return wiki.db.Setting.findAll({ + attributes: ['key', 'config'], + where: { + key: { + $in: subsets + } + } + }).then(results => { + if (_.isArray(results) && results.length > 0) { + results.forEach(result => { + wiki.config[result.key] = result.config + }) + return true + } else { + return Promise.reject(new Error('Invalid DB Configuration result set')) + } + }) } } diff --git a/server/modules/db.js b/server/modules/db.js index 89ca5ce1..36bbfadc 100644 --- a/server/modules/db.js +++ b/server/modules/db.js @@ -5,6 +5,7 @@ const fs = require('fs') const path = require('path') const _ = require('lodash') +const Promise = require('bluebird') /** * PostgreSQL DB module @@ -33,7 +34,8 @@ module.exports = { max: 10, min: 0, idle: 10000 - } + }, + logging: false }) // Attempt to connect and authenticate to DB @@ -63,10 +65,10 @@ module.exports = { // Sync DB - self.onReady = self.inst.sync({ + self.onReady = (wiki.IS_MASTER) ? self.inst.sync({ force: false, logging: false - }) + }) : Promise.resolve() return self } diff --git a/server/modules/disk.js b/server/modules/disk.js index 9e919229..88c54279 100644 --- a/server/modules/disk.js +++ b/server/modules/disk.js @@ -27,7 +27,7 @@ module.exports = { this._uploadsThumbsPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data, 'thumbs') this.createBaseDirectories() - this.initMulter() + // this.initMulter() return this }, @@ -37,8 +37,10 @@ module.exports = { */ initMulter () { let maxFileSizes = { - img: wiki.config.uploads.maxImageFileSize * 1024 * 1024, - file: wiki.config.uploads.maxOtherFileSize * 1024 * 1024 + // img: wiki.config.uploads.maxImageFileSize * 1024 * 1024, + // file: wiki.config.uploads.maxOtherFileSize * 1024 * 1024 + img: 3 * 1024 * 1024, + file: 10 * 1024 * 1024 } // -> IMAGES diff --git a/server/modules/git.js b/server/modules/git.js index b0155ce9..4c58a48f 100644 --- a/server/modules/git.js +++ b/server/modules/git.js @@ -51,7 +51,7 @@ module.exports = { // -> Initialize repository - self.onReady = self._initRepo() + self.onReady = (wiki.IS_MASTER) ? self._initRepo() : Promise.resolve() // Define signature diff --git a/server/modules/logger.js b/server/modules/logger.js index 08ecc175..8b134d05 100644 --- a/server/modules/logger.js +++ b/server/modules/logger.js @@ -4,75 +4,77 @@ const cluster = require('cluster') -module.exports = () => { - let winston = require('winston') +module.exports = { + init() { + let winston = require('winston') - // Console + // Console - let logger = new (winston.Logger)({ - level: (wiki.IS_DEBUG) ? 'debug' : 'info', - transports: [ - new (winston.transports.Console)({ - level: (wiki.IS_DEBUG) ? 'debug' : 'info', - prettyPrint: true, - colorize: true, - silent: false, - timestamp: true - }) - ] - }) + let logger = new (winston.Logger)({ + level: (wiki.IS_DEBUG) ? 'debug' : 'info', + transports: [ + new (winston.transports.Console)({ + level: (wiki.IS_DEBUG) ? 'debug' : 'info', + prettyPrint: true, + colorize: true, + silent: false, + timestamp: true + }) + ] + }) - logger.filters.push((level, msg) => { - let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}` - return '[' + processName + '] ' + msg - }) + logger.filters.push((level, msg) => { + let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}` + return '[' + processName + '] ' + msg + }) - // External services + // External services - if (wiki.config.externalLogging.bugsnag) { - const bugsnagTransport = require('./winston-transports/bugsnag') - logger.add(bugsnagTransport, { - level: 'warn', - key: wiki.config.externalLogging.bugsnag - }) - } + // if (wiki.config.externalLogging.bugsnag) { + // const bugsnagTransport = require('./winston-transports/bugsnag') + // logger.add(bugsnagTransport, { + // level: 'warn', + // key: wiki.config.externalLogging.bugsnag + // }) + // } - if (wiki.config.externalLogging.loggly) { - require('winston-loggly-bulk') - logger.add(winston.transports.Loggly, { - token: wiki.config.externalLogging.loggly.token, - subdomain: wiki.config.externalLogging.loggly.subdomain, - tags: ['wiki-js'], - level: 'warn', - json: true - }) - } + // if (wiki.config.externalLogging.loggly) { + // require('winston-loggly-bulk') + // logger.add(winston.transports.Loggly, { + // token: wiki.config.externalLogging.loggly.token, + // subdomain: wiki.config.externalLogging.loggly.subdomain, + // tags: ['wiki-js'], + // level: 'warn', + // json: true + // }) + // } - if (wiki.config.externalLogging.papertrail) { - require('winston-papertrail').Papertrail // eslint-disable-line no-unused-expressions - logger.add(winston.transports.Papertrail, { - host: wiki.config.externalLogging.papertrail.host, - port: wiki.config.externalLogging.papertrail.port, - level: 'warn', - program: 'wiki.js' - }) - } + // if (wiki.config.externalLogging.papertrail) { + // require('winston-papertrail').Papertrail // eslint-disable-line no-unused-expressions + // logger.add(winston.transports.Papertrail, { + // host: wiki.config.externalLogging.papertrail.host, + // port: wiki.config.externalLogging.papertrail.port, + // level: 'warn', + // program: 'wiki.js' + // }) + // } - if (wiki.config.externalLogging.rollbar) { - const rollbarTransport = require('./winston-transports/rollbar') - logger.add(rollbarTransport, { - level: 'warn', - key: wiki.config.externalLogging.rollbar - }) - } + // if (wiki.config.externalLogging.rollbar) { + // const rollbarTransport = require('./winston-transports/rollbar') + // logger.add(rollbarTransport, { + // level: 'warn', + // key: wiki.config.externalLogging.rollbar + // }) + // } - if (wiki.config.externalLogging.sentry) { - const sentryTransport = require('./winston-transports/sentry') - logger.add(sentryTransport, { - level: 'warn', - key: wiki.config.externalLogging.sentry - }) - } + // if (wiki.config.externalLogging.sentry) { + // const sentryTransport = require('./winston-transports/sentry') + // logger.add(sentryTransport, { + // level: 'warn', + // key: wiki.config.externalLogging.sentry + // }) + // } - return logger + return logger + } } diff --git a/server/modules/markdown.js b/server/modules/markdown.js index c88266d7..71e0ea54 100644 --- a/server/modules/markdown.js +++ b/server/modules/markdown.js @@ -23,7 +23,8 @@ const mdRemove = require('remove-markdown') var mkdown = md({ html: true, - breaks: wiki.config.features.linebreaks, + // breaks: wiki.config.features.linebreaks, + breaks: true, linkify: true, typography: true, highlight(str, lang) { @@ -57,7 +58,8 @@ var mkdown = md({ }) .use(mdAttrs) -if (wiki.config.features.mathjax) { +// if (wiki.config.features.mathjax) { +if (true) { mkdown.use(mdMathjax) } diff --git a/server/modules/uploads-agent.js b/server/modules/uploads-agent.js index a8ae79ec..6b861733 100644 --- a/server/modules/uploads-agent.js +++ b/server/modules/uploads-agent.js @@ -1,6 +1,6 @@ 'use strict' -/* global db, git, lang, upl */ +/* global wiki */ const path = require('path') const Promise = require('bluebird') @@ -32,8 +32,8 @@ module.exports = { init () { let self = this - self._uploadsPath = path.resolve(ROOTPATH, appconfig.paths.repo, 'uploads') - self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.paths.data, 'thumbs') + self._uploadsPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo, 'uploads') + self._uploadsThumbsPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data, 'thumbs') return self }, @@ -59,16 +59,16 @@ module.exports = { self._watcher.on('add', (p) => { let pInfo = self.parseUploadsRelPath(p) return self.processFile(pInfo.folder, pInfo.filename).then((mData) => { - return db.UplFile.findByIdAndUpdate(mData._id, mData, { upsert: true }) + return wiki.db.UplFile.findByIdAndUpdate(mData._id, mData, { upsert: true }) }).then(() => { - return git.commitUploads(lang.t('git:uploaded', { path: p })) + return wiki.git.commitUploads(wiki.lang.t('git:uploaded', { path: p })) }) }) // -> Remove upload file self._watcher.on('unlink', (p) => { - return git.commitUploads(lang.t('git:deleted', { path: p })) + return wiki.git.commitUploads(wiki.lang.t('git:deleted', { path: p })) }) }, @@ -91,8 +91,8 @@ module.exports = { // Add folders to DB - return db.UplFolder.remove({}).then(() => { - return db.UplFolder.insertMany(_.map(folderNames, (f) => { + return wiki.db.UplFolder.remove({}).then(() => { + return wiki.db.UplFolder.insertMany(_.map(folderNames, (f) => { return { _id: 'f:' + f, name: f @@ -107,7 +107,7 @@ module.exports = { let fldPath = path.join(self._uploadsPath, fldName) return fs.readdirAsync(fldPath).then((fList) => { return Promise.map(fList, (f) => { - return upl.processFile(fldName, f).then((mData) => { + return wiki.upl.processFile(fldName, f).then((mData) => { if (mData) { allFiles.push(mData) } @@ -118,9 +118,9 @@ module.exports = { }, {concurrency: 1}).finally(() => { // Add files to DB - return db.UplFile.remove({}).then(() => { + return wiki.db.UplFile.remove({}).then(() => { if (_.isArray(allFiles) && allFiles.length > 0) { - return db.UplFile.insertMany(allFiles) + return wiki.db.UplFile.insertMany(allFiles) } else { return true } @@ -131,7 +131,7 @@ module.exports = { }).then(() => { // Watch for new changes - return upl.watch() + return wiki.upl.watch() }) }, diff --git a/server/queues/git-sync.js b/server/queues/git-sync.js new file mode 100644 index 00000000..9f9b1f57 --- /dev/null +++ b/server/queues/git-sync.js @@ -0,0 +1,65 @@ +'use strict' + +/* global wiki */ + +const Promise = require('bluebird') +const fs = Promise.promisifyAll(require('fs-extra')) +const klaw = require('klaw') +const moment = require('moment') +const path = require('path') +const entryHelper = require('../helpers/entry') + +module.exports = (job, done) => { + return wiki.git.resync().then(() => { + // -> Stream all documents + + let cacheJobs = [] + let jobCbStreamDocsResolve = null + let jobCbStreamDocs = new Promise((resolve, reject) => { + jobCbStreamDocsResolve = resolve + }) + + klaw(wiki.REPOPATH).on('data', function (item) { + if (path.extname(item.path) === '.md' && path.basename(item.path) !== 'README.md') { + let entryPath = entryHelper.parsePath(entryHelper.getEntryPathFromFullPath(item.path)) + let cachePath = entryHelper.getCachePath(entryPath) + + // -> Purge outdated cache + + cacheJobs.push( + fs.statAsync(cachePath).then((st) => { + return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active' + }).catch((err) => { + return (err.code !== 'EEXIST') ? err : 'new' + }).then((fileStatus) => { + // -> Delete expired cache file + + if (fileStatus === 'expired') { + return fs.unlinkAsync(cachePath).return(fileStatus) + } + + return fileStatus + }).then((fileStatus) => { + // -> Update cache and search index + + if (fileStatus !== 'active') { + return global.entries.updateCache(entryPath).then(entry => { + process.send({ + action: 'searchAdd', + content: entry + }) + return true + }) + } + + return true + }) + ) + } + }).on('end', () => { + jobCbStreamDocsResolve(Promise.all(cacheJobs)) + }) + + return jobCbStreamDocs + }) +} diff --git a/server/queues/upl-clear-temp.js b/server/queues/upl-clear-temp.js new file mode 100644 index 00000000..2162aae4 --- /dev/null +++ b/server/queues/upl-clear-temp.js @@ -0,0 +1,26 @@ +'use strict' + +/* global wiki */ + +const Promise = require('bluebird') +const fs = Promise.promisifyAll(require('fs-extra')) +const moment = require('moment') +const path = require('path') + +module.exports = (job, done) => { + return fs.readdirAsync(wiki.UPLTEMPPATH).then((ls) => { + let fifteenAgo = moment().subtract(15, 'minutes') + + return Promise.map(ls, (f) => { + return fs.statAsync(path.join(wiki.UPLTEMPPATH, f)).then((s) => { return { filename: f, stat: s } }) + }).filter((s) => { return s.stat.isFile() }).then((arrFiles) => { + return Promise.map(arrFiles, (f) => { + if (moment(f.stat.ctime).isBefore(fifteenAgo, 'minute')) { + return fs.unlinkAsync(path.join(wiki.UPLTEMPPATH, f.filename)) + } else { + return true + } + }) + }) + }) +} diff --git a/server/worker.js b/server/worker.js index 7ad76277..f64deca5 100644 --- a/server/worker.js +++ b/server/worker.js @@ -4,30 +4,27 @@ const path = require('path') +wiki.REPOPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo) +wiki.DATAPATH = path.resolve(wiki.ROOTPATH, wiki.config.paths.data) +wiki.UPLTEMPPATH = path.join(wiki.DATAPATH, 'temp-upload') + // ---------------------------------------- // Load global modules // ---------------------------------------- -wiki.db = require('./modules/db').init() -wiki.upl = require('./modules/uploads-agent').init() -wiki.git = require('./modules/git').init() -wiki.entries = require('./modules/entries').init() +// wiki.upl = require('./modules/uploads-agent').init() +// wiki.git = require('./modules/git').init() +// wiki.entries = require('./modules/entries').init() wiki.lang = require('i18next') wiki.mark = require('./modules/markdown') // ---------------------------------------- -// Load modules +// Load local modules // ---------------------------------------- -const moment = require('moment') const Promise = require('bluebird') -const fs = Promise.promisifyAll(require('fs-extra')) -const klaw = require('klaw') -const Cron = require('cron').CronJob const i18nBackend = require('i18next-node-fs-backend') -const entryHelper = require('./helpers/entry') - // ---------------------------------------- // Localization Engine // ---------------------------------------- @@ -46,143 +43,21 @@ wiki.lang.use(i18nBackend).init({ }) // ---------------------------------------- -// Start Cron +// Start Queues // ---------------------------------------- -let job -let jobIsBusy = false -let jobUplWatchStarted = false - -wiki.db.onReady.then(() => { - return wiki.db.Entry.remove({}) -}).then(() => { - job = new Cron({ - cronTime: '0 */5 * * * *', - onTick: () => { - // Make sure we don't start two concurrent jobs - - if (jobIsBusy) { - wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)') - return - } - wiki.logger.info('Running all jobs...') - jobIsBusy = true - - // Prepare async job collector - - let jobs = [] - let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo) - let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data) - let uploadsTempPath = path.join(dataPath, 'temp-upload') - - // ---------------------------------------- - // REGULAR JOBS - // ---------------------------------------- - - //* **************************************** - // -> Sync with Git remote - //* **************************************** - - jobs.push(wiki.git.resync().then(() => { - // -> Stream all documents - - let cacheJobs = [] - let jobCbStreamDocsResolve = null - let jobCbStreamDocs = new Promise((resolve, reject) => { - jobCbStreamDocsResolve = resolve - }) - - klaw(repoPath).on('data', function (item) { - if (path.extname(item.path) === '.md' && path.basename(item.path) !== 'README.md') { - let entryPath = entryHelper.parsePath(entryHelper.getEntryPathFromFullPath(item.path)) - let cachePath = entryHelper.getCachePath(entryPath) - - // -> Purge outdated cache - - cacheJobs.push( - fs.statAsync(cachePath).then((st) => { - return moment(st.mtime).isBefore(item.stats.mtime) ? 'expired' : 'active' - }).catch((err) => { - return (err.code !== 'EEXIST') ? err : 'new' - }).then((fileStatus) => { - // -> Delete expired cache file +const Bull = require('bull') +const autoload = require('auto-load') - if (fileStatus === 'expired') { - return fs.unlinkAsync(cachePath).return(fileStatus) - } +let queues = autoload(path.join(wiki.SERVERPATH, 'queues')) - return fileStatus - }).then((fileStatus) => { - // -> Update cache and search index - - if (fileStatus !== 'active') { - return global.entries.updateCache(entryPath).then(entry => { - process.send({ - action: 'searchAdd', - content: entry - }) - return true - }) - } - - return true - }) - ) - } - }).on('end', () => { - jobCbStreamDocsResolve(Promise.all(cacheJobs)) - }) - - return jobCbStreamDocs - })) - - //* **************************************** - // -> Clear failed temporary upload files - //* **************************************** - - jobs.push( - fs.readdirAsync(uploadsTempPath).then((ls) => { - let fifteenAgo = moment().subtract(15, 'minutes') - - return Promise.map(ls, (f) => { - return fs.statAsync(path.join(uploadsTempPath, f)).then((s) => { return { filename: f, stat: s } }) - }).filter((s) => { return s.stat.isFile() }).then((arrFiles) => { - return Promise.map(arrFiles, (f) => { - if (moment(f.stat.ctime).isBefore(fifteenAgo, 'minute')) { - return fs.unlinkAsync(path.join(uploadsTempPath, f.filename)) - } else { - return true - } - }) - }) - }) - ) - - // ---------------------------------------- - // Run - // ---------------------------------------- - - Promise.all(jobs).then(() => { - wiki.logger.info('All jobs completed successfully! Going to sleep for now.') - - if (!jobUplWatchStarted) { - jobUplWatchStarted = true - wiki.upl.initialScan().then(() => { - job.start() - }) - } - - return true - }).catch((err) => { - wiki.logger.error('One or more jobs have failed: ', err) - }).finally(() => { - jobIsBusy = false - }) - }, - start: false, - timeZone: 'UTC', - runOnInit: true - }) +Promise.join( + wiki.db.onReady + // wiki.upl.initialScan() +).then(() => { + for (let queueName in queues) { + new Bull(queueName, { redis: wiki.config.redis }).process(queues[queueName]) + } }) // ---------------------------------------- @@ -190,11 +65,6 @@ wiki.db.onReady.then(() => { // ---------------------------------------- process.on('disconnect', () => { - wiki.logger.warn('Lost connection to main server. Exiting...') - job.stop() + wiki.logger.warn('Lost connection to Master. Exiting...') process.exit() }) - -process.on('exit', () => { - job.stop() -}) diff --git a/yarn.lock b/yarn.lock index e52fcd68..534390bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1609,7 +1609,7 @@ cron-parser@^2.4.1: is-nan "^1.2.1" moment-timezone "^0.5.0" -cron@1.2.1, cron@~1.2.1: +cron@1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/cron/-/cron-1.2.1.tgz#3a86c09b41b8f261ac863a7cc85ea4735857eab2" dependencies: