feat: cluster implementation

pull/621/head
NGPixel 7 years ago
parent 60750eeed8
commit 9c112ab535

@ -4,23 +4,9 @@
# Full explanation + examples in the documentation:
# https://docs.requarks.io/wiki/install
# ---------------------------------------------------------------------
# Title of this site
# ---------------------------------------------------------------------
title: Wiki
# ---------------------------------------------------------------------
# Full public path to the site, without the trailing slash
# ---------------------------------------------------------------------
# INCLUDE CLIENT PORT IF NOT 80/443!
host: http://localhost
# ---------------------------------------------------------------------
# Port the main server should listen to (80 by default)
# ---------------------------------------------------------------------
# To use process.env.PORT, comment the line below:
port: 80
@ -33,136 +19,23 @@ paths:
data: ./data
# ---------------------------------------------------------------------
# Upload Limits
# ---------------------------------------------------------------------
# In megabytes (MB)
uploads:
maxImageFileSize: 3
maxOtherFileSize: 100
# ---------------------------------------------------------------------
# Site Language
# ---------------------------------------------------------------------
# Possible values: en, es, fr, ko, ru or zh
lang: en
# ---------------------------------------------------------------------
# Site Authentication
# ---------------------------------------------------------------------
public: false
auth:
defaultReadAccess: false
local:
enabled: true
google:
enabled: true
clientId: GOOGLE_CLIENT_ID
clientSecret: GOOGLE_CLIENT_SECRET
microsoft:
enabled: true
clientId: MS_APP_ID
clientSecret: MS_APP_SECRET
facebook:
enabled: false
clientId: FACEBOOK_APP_ID
clientSecret: FACEBOOK_APP_SECRET
github:
enabled: false
clientId: GITHUB_CLIENT_ID
clientSecret: GITHUB_CLIENT_SECRET
slack:
enabled: false
clientId: SLACK_CLIENT_ID
clientSecret: SLACK_CLIENT_SECRET
ldap:
enabled: false
url: ldap://serverhost:389
bindDn: cn='root'
bindCredentials: BIND_PASSWORD
searchBase: o=users,o=example.com
searchFilter: (uid={{username}})
tlsEnabled: false
tlsCertPath: C:\example\root_ca_cert.crt
azure:
enabled: false
clientID: APP_ID
clientSecret: APP_SECRET_KEY
resource: '00000002-0000-0000-c000-000000000000'
tenant: 'YOUR_TENANT.onmicrosoft.com'
# ---------------------------------------------------------------------
# Secret key to use when encrypting sessions
# Database
# ---------------------------------------------------------------------
# Use a long and unique random string (256-bit keys are perfect!)
sessionSecret: 1234567890abcdefghijklmnopqrstuvxyz
db:
host: localhost
port: 5432
user: wikijs
pass: wikijsrocks
db: wiki
# ---------------------------------------------------------------------
# Database Connection String
# Redis
# ---------------------------------------------------------------------
# You can also use an ENV variable by using $ENV_VAR_NAME as the value
db: mongodb://localhost:27017/wiki
# ---------------------------------------------------------------------
# Git Connection Info
# ---------------------------------------------------------------------
git:
url: https://github.com/Organization/Repo
branch: master
auth:
# Type: basic or ssh
type: ssh
# Only for Basic authentication:
username: marty
password: MartyMcFly88
# Only for SSH authentication:
privateKey: /etc/wiki/keys/git.pem
sslVerify: true
# Default email to use as commit author
serverEmail: marty@example.com
# Whether to use user email as author in commits
showUserEmail: true
# ---------------------------------------------------------------------
# Features
# ---------------------------------------------------------------------
# You can enable / disable specific features below
features:
linebreaks: true
mathjax: true
# ---------------------------------------------------------------------
# External Logging
# ---------------------------------------------------------------------
externalLogging:
bugsnag: false
loggly: false
papertrail: false
rollbar: false
sentry: false
# ---------------------------------------------------------------------
# Color Theme
# ---------------------------------------------------------------------
redis:
host: localhost
port: 6379
db: 0
password: null
theme:
primary: indigo
alt: blue-grey
footer: blue-grey
code:
dark: true
colorize: true

@ -43,6 +43,7 @@
"bcryptjs-then": "~1.0.1",
"bluebird": "~3.5.0",
"body-parser": "~1.17.2",
"bull": "/home/nick3.0.0-rc.4",
"bunyan": "~1.8.10",
"cheerio": "~1.0.0-rc.2",
"child-process-promise": "~2.2.1",

@ -2,7 +2,6 @@
// ===========================================
// Wiki.js
// 1.0.1
// Licensed under AGPLv3
// ===========================================
@ -28,251 +27,28 @@ wiki.data = appconf.data
// Load Winston
// ----------------------------------------
wiki.logger = require('./modules/logger')(wiki.IS_DEBUG, 'SERVER')
wiki.logger.info('Wiki.js is initializing...')
// ----------------------------------------
// Load global modules
// ----------------------------------------
wiki.disk = require('./modules/disk').init()
wiki.db = require('./modules/db').init()
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
// ----------------------------------------
// Load modules
// ----------------------------------------
const autoload = require('auto-load')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const fork = require('child_process').fork
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
// ----------------------------------------
// View Engine Setup
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
wiki.logger = require('./modules/logger')()
// ----------------------------------------
// Controllers
// Start Cluster
// ----------------------------------------
app.use('/', ctrl.auth)
const cluster = require('cluster')
const numCPUs = require('os').cpus().length
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
})
})
// ----------------------------------------
// Start HTTP server
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
if (cluster.isMaster) {
wiki.logger.info('Wiki.js is initializing...')
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
require('./master')
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
for (let i = 0; i < numCPUs; i++) {
cluster.fork()
}
})
server.on('listening', () => {
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
cluster.on('exit', (worker, code, signal) => {
wiki.logger.info(`Worker #${worker.id} died.`)
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({
key: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
} else {
wiki.logger.info(`Background Worker #${cluster.worker.id} is starting...`)
// require('./worker')
}
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Start child processes
// ----------------------------------------
let bgAgent = fork(path.join(wiki.SERVERPATH, 'agent.js'))
bgAgent.on('message', m => {
if (!m.action) {
return
}
switch (m.action) {
case 'searchAdd':
wiki.search.add(m.content)
break
}
})
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
})
})

@ -0,0 +1,232 @@
'use strict'
/* global wiki */
const path = require('path')
// ----------------------------------------
// Load global modules
// ----------------------------------------
wiki.disk = require('./modules/disk').init()
wiki.db = require('./modules/db').init()
wiki.entries = require('./modules/entries').init()
wiki.git = require('./modules/git').init(false)
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
wiki.redis = require('./modules/redis').init()
wiki.search = require('./modules/search').init()
wiki.upl = require('./modules/uploads').init()
// ----------------------------------------
// Load modules
// ----------------------------------------
const autoload = require('auto-load')
const bodyParser = require('body-parser')
const compression = require('compression')
const cookieParser = require('cookie-parser')
const express = require('express')
const favicon = require('serve-favicon')
const flash = require('connect-flash')
const http = require('http')
const i18nBackend = require('i18next-node-fs-backend')
const passport = require('passport')
const passportSocketIo = require('passport.socketio')
const session = require('express-session')
const SessionRedisStore = require('connect-redis')(session)
const graceful = require('node-graceful')
const socketio = require('socket.io')
const graphqlApollo = require('apollo-server-express')
const graphqlSchema = require('./modules/graphql')
var mw = autoload(path.join(wiki.SERVERPATH, '/middlewares'))
var ctrl = autoload(path.join(wiki.SERVERPATH, '/controllers'))
// ----------------------------------------
// Define Express App
// ----------------------------------------
const app = express()
wiki.app = app
app.use(compression())
// ----------------------------------------
// Security
// ----------------------------------------
app.use(mw.security)
// ----------------------------------------
// Public Assets
// ----------------------------------------
app.use(favicon(path.join(wiki.ROOTPATH, 'assets', 'favicon.ico')))
app.use(express.static(path.join(wiki.ROOTPATH, 'assets'), {
index: false,
maxAge: '7d'
}))
// ----------------------------------------
// Passport Authentication
// ----------------------------------------
require('./modules/auth')(passport)
wiki.rights = require('./modules/rights')
wiki.rights.init()
let sessionStore = new SessionRedisStore({
client: wiki.redis
})
app.use(cookieParser())
app.use(session({
name: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
resave: false,
saveUninitialized: false
}))
app.use(flash())
app.use(passport.initialize())
app.use(passport.session())
// ----------------------------------------
// SEO
// ----------------------------------------
app.use(mw.seo)
// ----------------------------------------
// Localization Engine
// ----------------------------------------
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
// ----------------------------------------
// View Engine Setup
// ----------------------------------------
app.set('views', path.join(wiki.SERVERPATH, 'views'))
app.set('view engine', 'pug')
app.use(bodyParser.json({ limit: '1mb' }))
app.use(bodyParser.urlencoded({ extended: false, limit: '1mb' }))
// ----------------------------------------
// View accessible data
// ----------------------------------------
app.locals._ = require('lodash')
app.locals.t = wiki.lang.t.bind(wiki.lang)
app.locals.moment = require('moment')
app.locals.moment.locale(wiki.config.lang)
app.locals.appconfig = wiki.config
app.use(mw.flash)
// ----------------------------------------
// Controllers
// ----------------------------------------
app.use('/', ctrl.auth)
app.use('/graphql', graphqlApollo.graphqlExpress({ schema: graphqlSchema }))
app.use('/graphiql', graphqlApollo.graphiqlExpress({ endpointURL: '/graphql' }))
app.use('/uploads', mw.auth, ctrl.uploads)
app.use('/admin', mw.auth, ctrl.admin)
app.use('/', mw.auth, ctrl.pages)
// ----------------------------------------
// Error handling
// ----------------------------------------
app.use(function (req, res, next) {
var err = new Error('Not Found')
err.status = 404
next(err)
})
app.use(function (err, req, res, next) {
res.status(err.status || 500)
res.render('error', {
message: err.message,
error: wiki.IS_DEBUG ? err : {}
})
})
// ----------------------------------------
// Start HTTP server
// ----------------------------------------
wiki.logger.info('Starting HTTP/WS server on port ' + wiki.config.port + '...')
app.set('port', wiki.config.port)
var server = http.createServer(app)
var io = socketio(server)
server.listen(wiki.config.port)
server.on('error', (error) => {
if (error.syscall !== 'listen') {
throw error
}
// handle specific listen errors with friendly messages
switch (error.code) {
case 'EACCES':
wiki.logger.error('Listening on port ' + wiki.config.port + ' requires elevated privileges!')
return process.exit(1)
case 'EADDRINUSE':
wiki.logger.error('Port ' + wiki.config.port + ' is already in use!')
return process.exit(1)
default:
throw error
}
})
server.on('listening', () => {
wiki.logger.info('HTTP/WS server started successfully! [RUNNING]')
})
// ----------------------------------------
// WebSocket
// ----------------------------------------
io.use(passportSocketIo.authorize({
key: 'wikijs.sid',
store: sessionStore,
secret: wiki.config.sessionSecret,
cookieParser,
success: (data, accept) => {
accept()
},
fail: (data, message, error, accept) => {
accept()
}
}))
io.on('connection', ctrl.ws)
// ----------------------------------------
// Graceful shutdown
// ----------------------------------------
graceful.on('exit', () => {
// wiki.logger.info('- SHUTTING DOWN - Terminating Background Agent...')
// bgAgent.kill()
wiki.logger.info('- SHUTTING DOWN - Performing git sync...')
return global.git.resync().then(() => {
wiki.logger.info('- SHUTTING DOWN - Git sync successful. Now safe to exit.')
process.exit()
})
})

@ -4,7 +4,10 @@
* Associate DB Model relations
*/
module.exports = db => {
db.User.belongsToMany(db.Group, { through: 'UserGroups' })
db.Group.hasMany(db.Right, { as: 'GroupRights' })
db.User.belongsToMany(db.Group, { through: 'userGroups' })
db.Group.hasMany(db.Right, { as: 'groupRights' })
db.Document.hasMany(db.Tag, { as: 'documentTags' })
db.File.belongsTo(db.Folder)
db.Comment.belongsTo(db.Document)
db.Comment.belongsTo(db.User, { as: 'author' })
}

@ -0,0 +1,18 @@
'use strict'
/**
* Comment schema
*/
module.exports = (sequelize, DataTypes) => {
let commentSchema = sequelize.define('comment', {
content: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true
})
return commentSchema
}

@ -40,6 +40,11 @@ module.exports = (sequelize, DataTypes) => {
type: DataTypes.BOOLEAN,
allowNull: false,
defaultValue: false
},
searchContent: {
type: DataTypes.TEXT,
allowNull: true,
defaultValue: ''
}
}, {
timestamps: true,

@ -0,0 +1,24 @@
'use strict'
/**
* Tags schema
*/
module.exports = (sequelize, DataTypes) => {
let tagSchema = sequelize.define('tag', {
key: {
type: DataTypes.STRING,
allowNull: false
}
}, {
timestamps: true,
version: true,
indexes: [
{
unique: true,
fields: ['key']
}
]
})
return tagSchema
}

@ -64,7 +64,8 @@ module.exports = {
// Sync DB
self.onReady = self.inst.sync({
force: false
force: false,
logging: false
})
return self

@ -2,12 +2,10 @@
/* global wiki */
module.exports = (processName) => {
let winston = require('winston')
const cluster = require('cluster')
if (typeof processName === 'undefined') {
processName = 'SERVER'
}
module.exports = () => {
let winston = require('winston')
// Console
@ -25,6 +23,7 @@ module.exports = (processName) => {
})
logger.filters.push((level, msg) => {
let processName = (cluster.isMaster) ? 'MASTER' : `WORKER-${cluster.worker.id}`
return '[' + processName + '] ' + msg
})

@ -1,81 +0,0 @@
const bunyan = require('bunyan')
const level = require('levelup')
const down = require('memdown')
const SearchIndexAdder = require('search-index-adder')
const SearchIndexSearcher = require('search-index-searcher')
module.exports = function (givenOptions, moduleReady) {
const optionsLoaded = function (err, SearchIndex) {
const siUtil = require('./siUtil.js')(SearchIndex.options)
if (err) return moduleReady(err)
SearchIndex.close = siUtil.close
SearchIndex.countDocs = siUtil.countDocs
getAdder(SearchIndex, adderLoaded)
}
const adderLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
getSearcher(SearchIndex, searcherLoaded)
}
const searcherLoaded = function (err, SearchIndex) {
if (err) return moduleReady(err)
return moduleReady(err, SearchIndex)
}
getOptions(givenOptions, optionsLoaded)
}
const getAdder = function (SearchIndex, done) {
SearchIndexAdder(SearchIndex.options, function (err, searchIndexAdder) {
SearchIndex.add = searchIndexAdder.add
SearchIndex.callbackyAdd = searchIndexAdder.concurrentAdd // deprecated
SearchIndex.concurrentAdd = searchIndexAdder.concurrentAdd
SearchIndex.createWriteStream = searchIndexAdder.createWriteStream
SearchIndex.dbWriteStream = searchIndexAdder.dbWriteStream
SearchIndex.defaultPipeline = searchIndexAdder.defaultPipeline
SearchIndex.del = searchIndexAdder.deleter
SearchIndex.deleteStream = searchIndexAdder.deleteStream
SearchIndex.flush = searchIndexAdder.flush
done(err, SearchIndex)
})
}
const getSearcher = function (SearchIndex, done) {
SearchIndexSearcher(SearchIndex.options, function (err, searchIndexSearcher) {
SearchIndex.availableFields = searchIndexSearcher.availableFields
SearchIndex.buckets = searchIndexSearcher.bucketStream
SearchIndex.categorize = searchIndexSearcher.categoryStream
SearchIndex.dbReadStream = searchIndexSearcher.dbReadStream
SearchIndex.get = searchIndexSearcher.get
SearchIndex.match = searchIndexSearcher.match
SearchIndex.scan = searchIndexSearcher.scan
SearchIndex.search = searchIndexSearcher.search
SearchIndex.totalHits = searchIndexSearcher.totalHits
done(err, SearchIndex)
})
}
const getOptions = function (options, done) {
var SearchIndex = {}
SearchIndex.options = Object.assign({}, {
indexPath: 'si',
keySeparator: '○',
logLevel: 'error'
}, options)
options.log = bunyan.createLogger({
name: 'search-index',
level: options.logLevel
})
if (!options.indexes) {
level(SearchIndex.options.indexPath || 'si', {
valueEncoding: 'json',
db: down
}, function (err, db) {
SearchIndex.options.indexes = db
return done(err, SearchIndex)
})
} else {
return done(null, SearchIndex)
}
}

@ -1,36 +0,0 @@
'use strict'
module.exports = function (siOptions) {
var siUtil = {}
siUtil.countDocs = function (callback) {
var count = 0
const gte = 'DOCUMENT' + siOptions.keySeparator
const lte = 'DOCUMENT' + siOptions.keySeparator + siOptions.keySeparator
siOptions.indexes.createReadStream({gte: gte, lte: lte})
.on('data', function (data) {
count++
})
.on('error', function (err) {
return callback(err, null)
})
.on('end', function () {
return callback(null, count)
})
}
siUtil.close = function (callback) {
siOptions.indexes.close(function (err) {
while (!siOptions.indexes.isClosed()) {
// log not always working here- investigate
if (siOptions.log) siOptions.log.info('closing...')
}
if (siOptions.indexes.isClosed()) {
if (siOptions.log) siOptions.log.info('closed...')
callback(err)
}
})
}
return siUtil
}

@ -4,7 +4,7 @@
const Promise = require('bluebird')
const _ = require('lodash')
const searchIndex = require('./search-index')
// const searchIndex = require('./search-index')
const stopWord = require('stopword')
const streamToPromise = require('stream-to-promise')
const searchAllowedChars = new RegExp('[^a-z0-9' + wiki.data.regex.cjk + wiki.data.regex.arabic + ' ]', 'g')
@ -22,7 +22,7 @@ module.exports = {
init () {
let self = this
self._isReady = new Promise((resolve, reject) => {
searchIndex({
/*searchIndex({
deletable: true,
fieldedSearch: true,
indexPath: 'wiki',
@ -39,7 +39,7 @@ module.exports = {
resolve(true)
})
}
})
}) */
})
return self

@ -1,39 +1,19 @@
// ===========================================
// Wiki.js - Background Agent
// 1.0.1
// Licensed under AGPLv3
// ===========================================
'use strict'
const path = require('path')
const ROOTPATH = process.cwd()
const SERVERPATH = path.join(ROOTPATH, 'server')
global.ROOTPATH = ROOTPATH
global.SERVERPATH = SERVERPATH
const IS_DEBUG = process.env.NODE_ENV === 'development'
let appconf = require('./modules/config')()
global.appconfig = appconf.config
global.appdata = appconf.data
// ----------------------------------------
// Load Winston
// ----------------------------------------
/* global wiki */
global.winston = require('./modules/logger')(IS_DEBUG, 'AGENT')
const path = require('path')
// ----------------------------------------
// Load global modules
// ----------------------------------------
global.winston.info('Background Agent is initializing...')
global.db = require('./modules/db').init()
global.upl = require('./modules/uploads-agent').init()
global.git = require('./modules/git').init()
global.entries = require('./modules/entries').init()
global.lang = require('i18next')
global.mark = require('./modules/markdown')
wiki.db = require('./modules/db').init()
wiki.upl = require('./modules/uploads-agent').init()
wiki.git = require('./modules/git').init()
wiki.entries = require('./modules/entries').init()
wiki.lang = require('i18next')
wiki.mark = require('./modules/markdown')
// ----------------------------------------
// Load modules
@ -52,18 +32,16 @@ const entryHelper = require('./helpers/entry')
// Localization Engine
// ----------------------------------------
global.lang
.use(i18nBackend)
.init({
wiki.lang.use(i18nBackend).init({
load: 'languageOnly',
ns: ['common', 'admin', 'auth', 'errors', 'git'],
defaultNS: 'common',
saveMissing: false,
preload: [appconfig.lang],
lng: appconfig.lang,
preload: [wiki.config.lang],
lng: wiki.config.lang,
fallbackLng: 'en',
backend: {
loadPath: path.join(SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
loadPath: path.join(wiki.SERVERPATH, 'locales/{{lng}}/{{ns}}.json')
}
})
@ -75,8 +53,8 @@ let job
let jobIsBusy = false
let jobUplWatchStarted = false
global.db.onReady.then(() => {
return global.db.Entry.remove({})
wiki.db.onReady.then(() => {
return wiki.db.Entry.remove({})
}).then(() => {
job = new Cron({
cronTime: '0 */5 * * * *',
@ -84,17 +62,17 @@ global.db.onReady.then(() => {
// Make sure we don't start two concurrent jobs
if (jobIsBusy) {
global.winston.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
wiki.logger.warn('Previous job has not completed gracefully or is still running! Skipping for now. (This is not normal, you should investigate)')
return
}
global.winston.info('Running all jobs...')
wiki.logger.info('Running all jobs...')
jobIsBusy = true
// Prepare async job collector
let jobs = []
let repoPath = path.resolve(ROOTPATH, appconfig.paths.repo)
let dataPath = path.resolve(ROOTPATH, appconfig.paths.data)
let repoPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.repo)
let dataPath = path.resolve(wiki.ROOTPATH, wiki.config.paths.data)
let uploadsTempPath = path.join(dataPath, 'temp-upload')
// ----------------------------------------
@ -105,7 +83,7 @@ global.db.onReady.then(() => {
// -> Sync with Git remote
//* ****************************************
jobs.push(global.git.resync().then(() => {
jobs.push(wiki.git.resync().then(() => {
// -> Stream all documents
let cacheJobs = []
@ -185,18 +163,18 @@ global.db.onReady.then(() => {
// ----------------------------------------
Promise.all(jobs).then(() => {
global.winston.info('All jobs completed successfully! Going to sleep for now.')
wiki.logger.info('All jobs completed successfully! Going to sleep for now.')
if (!jobUplWatchStarted) {
jobUplWatchStarted = true
global.upl.initialScan().then(() => {
wiki.upl.initialScan().then(() => {
job.start()
})
}
return true
}).catch((err) => {
global.winston.error('One or more jobs have failed: ', err)
wiki.logger.error('One or more jobs have failed: ', err)
}).finally(() => {
jobIsBusy = false
})
@ -212,7 +190,7 @@ global.db.onReady.then(() => {
// ----------------------------------------
process.on('disconnect', () => {
global.winston.warn('Lost connection to main server. Exiting...')
wiki.logger.warn('Lost connection to main server. Exiting...')
job.stop()
process.exit()
})

@ -1036,7 +1036,7 @@ block-stream@*:
dependencies:
inherits "~2.0.0"
bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@~3.5.0:
bluebird@^3.1.1, bluebird@^3.3.4, bluebird@^3.4.1, bluebird@^3.4.6, bluebird@^3.5.0, bluebird@~3.5.0:
version "3.5.0"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c"
@ -1124,6 +1124,18 @@ builtin-modules@^1.0.0, builtin-modules@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f"
bull@/home/nick3.0.0-rc.4:
version "3.0.0-rc.4"
resolved "https://registry.yarnpkg.com/bull/-/bull-3.0.0-rc.4.tgz#dea18e870787037183849fc0198982ed756589b7"
dependencies:
bluebird "^3.5.0"
cron-parser "^2.4.1"
debuglog "^1.0.0"
ioredis "^3.1.1"
lodash "^4.17.4"
semver "^5.3.0"
uuid "^3.1.0"
bunyan@^1.8.1, bunyan@^1.8.10, bunyan@^1.8.3, bunyan@~1.8.10:
version "1.8.10"
resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.8.10.tgz#201fedd26c7080b632f416072f53a90b9a52981c"
@ -1590,6 +1602,13 @@ crc@3.4.4, crc@^3.4.0:
version "3.4.4"
resolved "https://registry.yarnpkg.com/crc/-/crc-3.4.4.tgz#9da1e980e3bd44fc5c93bf5ab3da3378d85e466b"
cron-parser@^2.4.1:
version "2.4.1"
resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.4.1.tgz#022befce1af293e4d3144ff04c2cbd2edb491271"
dependencies:
is-nan "^1.2.1"
moment-timezone "^0.5.0"
cron@1.2.1, cron@~1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/cron/-/cron-1.2.1.tgz#3a86c09b41b8f261ac863a7cc85ea4735857eab2"
@ -1702,6 +1721,10 @@ debug@~2.2.0:
dependencies:
ms "0.7.1"
debuglog@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@ -1732,6 +1755,13 @@ deferred-leveldown@~1.2.1:
dependencies:
abstract-leveldown "~2.4.0"
define-properties@^1.1.1:
version "1.1.2"
resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94"
dependencies:
foreach "^2.0.5"
object-keys "^1.0.8"
del@^2.0.2:
version "2.2.2"
resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8"
@ -2573,6 +2603,10 @@ for-own@^0.1.4:
dependencies:
for-in "^1.0.1"
foreach@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
forever-agent@~0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@ -3207,6 +3241,19 @@ invert-kv@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6"
ioredis@^3.1.1:
version "3.1.2"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.2.tgz#2579e3eba6dc490f68f14c7b51346281332b467b"
dependencies:
bluebird "^3.3.4"
cluster-key-slot "^1.0.6"
debug "^2.2.0"
denque "^1.1.0"
flexbuffer "0.0.6"
lodash "^4.8.2"
redis-commands "^1.2.0"
redis-parser "^2.4.0"
ioredis@~3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-3.1.1.tgz#cc2f1d3232b8c95cc153046bce168f2baa1186e8"
@ -3320,6 +3367,12 @@ is-glob@^2.0.0, is-glob@^2.0.1:
dependencies:
is-extglob "^1.0.0"
is-nan@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.2.1.tgz#9faf65b6fb6db24b7f5c0628475ea71f988401e2"
dependencies:
define-properties "^1.1.1"
is-npm@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-1.0.0.tgz#f2fb63a65e4905b406c86072765a1a4dc793b9f4"
@ -4574,7 +4627,7 @@ mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
dependencies:
minimist "0.0.8"
moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13:
moment-timezone@^0.5.0, moment-timezone@^0.5.4, moment-timezone@^0.5.x, moment-timezone@~0.5.13:
version "0.5.13"
resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.13.tgz#99ce5c7d827262eb0f1f702044177f60745d7b90"
dependencies:
@ -4877,6 +4930,10 @@ object-component@0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291"
object-keys@^1.0.8:
version "1.0.11"
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
object.omit@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa"
@ -7045,7 +7102,7 @@ uuid@^2.0.1, uuid@^2.0.2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
uuid@^3.0.0, uuid@^3.0.1:
uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"

Loading…
Cancel
Save