msb-public
/
wiki
mirror of https://github.com/requarks/wiki
1
0
Code Issues Projects Releases Wiki Activity

fix: elasticsearch module upgrade + deps update

Browse Source
pull/901/head
Nick 5 years ago
parent d0b73c5a9b
commit b34aa5bab4
6 changed files with 785 additions and 567 deletions
Show Stats Download Patch File Download Diff File

1
dev/index.js
Unescape View File

@ -41,6 +41,7 @@ const init = {
'./server', './server',
'!./server/views/master.pug' '!./server/views/master.pug'
], { ], {
cwd: process.cwd(),
ignoreInitial: true, ignoreInitial: true,
atomic: 400 atomic: 400
}) })

105
package.json
Unescape View File

@ -35,13 +35,13 @@
}, },
"dependencies": { "dependencies": {
"@aoberoi/passport-slack": "1.0.5", "@aoberoi/passport-slack": "1.0.5",
"@bugsnag/js": "6.3.0", "@bugsnag/js": "6.3.1",
"algoliasearch": "3.33.0", "algoliasearch": "3.33.0",
"apollo-fetch": "0.7.0", "apollo-fetch": "0.7.0",
"apollo-server": "2.6.1", "apollo-server": "2.6.4",
"apollo-server-express": "2.6.1", "apollo-server-express": "2.6.4",
"auto-load": "3.0.4", "auto-load": "3.0.4",
"aws-sdk": "2.466.0", "aws-sdk": "2.480.0",
"axios": "0.19.0", "axios": "0.19.0",
"azure-search-client": "3.1.5", "azure-search-client": "3.1.5",
"bcryptjs-then": "1.0.1", "bcryptjs-then": "1.0.1",
@ -49,7 +49,7 @@
"body-parser": "1.19.0", "body-parser": "1.19.0",
"chalk": "2.4.2", "chalk": "2.4.2",
"cheerio": "1.0.0-rc.3", "cheerio": "1.0.0-rc.3",
"chokidar": "3.0.0", "chokidar": "3.0.1",
"clean-css": "4.2.1", "clean-css": "4.2.1",
"compression": "1.7.4", "compression": "1.7.4",
"connect-session-knex": "1.4.0", "connect-session-knex": "1.4.0",
@ -58,14 +58,15 @@
"custom-error-instance": "2.1.1", "custom-error-instance": "2.1.1",
"dependency-graph": "0.8.0", "dependency-graph": "0.8.0",
"diff": "4.0.1", "diff": "4.0.1",
"diff2html": "2.9.0", "diff2html": "2.10.0",
"dotize": "0.3.0", "dotize": "0.3.0",
"elasticsearch": "16.1.0", "elasticsearch6": "npm:@elastic/elasticsearch@6",
"elasticsearch7": "npm:@elastic/elasticsearch@7",
"emoji-regex": "8.0.0", "emoji-regex": "8.0.0",
"express": "4.17.1", "express": "4.17.1",
"express-brute": "1.0.1", "express-brute": "1.0.1",
"express-session": "1.16.1", "express-session": "1.16.2",
"file-type": "11.1.0", "file-type": "12.0.0",
"filesize": "4.1.2", "filesize": "4.1.2",
"fs-extra": "8.0.1", "fs-extra": "8.0.1",
"getos": "3.1.1", "getos": "3.1.1",
@ -73,9 +74,9 @@
"graphql-list-fields": "2.0.2", "graphql-list-fields": "2.0.2",
"graphql-rate-limit-directive": "1.1.0", "graphql-rate-limit-directive": "1.1.0",
"graphql-subscriptions": "1.1.0", "graphql-subscriptions": "1.1.0",
"graphql-tools": "4.0.4", "graphql-tools": "4.0.5",
"highlight.js": "9.15.8", "highlight.js": "9.15.8",
"i18next": "15.1.3", "i18next": "17.0.4",
"i18next-express-middleware": "1.8.0", "i18next-express-middleware": "1.8.0",
"i18next-localstorage-cache": "1.1.1", "i18next-localstorage-cache": "1.1.1",
"i18next-node-fs-backend": "2.1.3", "i18next-node-fs-backend": "2.1.3",
@ -85,11 +86,11 @@
"js-yaml": "3.13.1", "js-yaml": "3.13.1",
"jsonwebtoken": "8.5.1", "jsonwebtoken": "8.5.1",
"klaw": "3.0.0", "klaw": "3.0.0",
"knex": "0.17.1", "knex": "0.17.6",
"lodash": "4.17.11", "lodash": "4.17.11",
"markdown-it": "8.4.2", "markdown-it": "8.4.2",
"markdown-it-abbr": "1.0.4", "markdown-it-abbr": "1.0.4",
"markdown-it-anchor": "5.2.3", "markdown-it-anchor": "5.2.4",
"markdown-it-attrs": "2.4.1", "markdown-it-attrs": "2.4.1",
"markdown-it-emoji": "1.4.0", "markdown-it-emoji": "1.4.0",
"markdown-it-expand-tabs": "1.0.13", "markdown-it-expand-tabs": "1.0.13",
@ -105,7 +106,7 @@
"mime-types": "2.1.24", "mime-types": "2.1.24",
"moment": "2.24.0", "moment": "2.24.0",
"moment-timezone": "0.5.25", "moment-timezone": "0.5.25",
"mongodb": "3.2.6", "mongodb": "3.2.7",
"mssql": "5.1.0", "mssql": "5.1.0",
"multer": "1.4.1", "multer": "1.4.1",
"mysql2": "1.6.5", "mysql2": "1.6.5",
@ -134,10 +135,10 @@
"passport-twitch": "1.0.3", "passport-twitch": "1.0.3",
"pem-jwk": "2.0.0", "pem-jwk": "2.0.0",
"pg": "7.11.0", "pg": "7.11.0",
"pg-hstore": "2.3.2", "pg-hstore": "2.3.3",
"pg-query-stream": "2.0.0", "pg-query-stream": "2.0.0",
"pg-tsquery": "8.0.4", "pg-tsquery": "8.0.4",
"pug": "2.0.3", "pug": "2.0.4",
"qr-image": "3.2.0", "qr-image": "3.2.0",
"raven": "2.6.4", "raven": "2.6.4",
"remove-markdown": "0.3.0", "remove-markdown": "0.3.0",
@ -148,16 +149,16 @@
"scim-query-filter-parser": "1.1.0", "scim-query-filter-parser": "1.1.0",
"semver": "6.1.1", "semver": "6.1.1",
"serve-favicon": "2.5.0", "serve-favicon": "2.5.0",
"simple-git": "1.113.0", "simple-git": "1.116.0",
"solr-node": "1.2.1", "solr-node": "1.2.1",
"sqlite3": "4.0.8", "sqlite3": "4.0.9",
"striptags": "3.1.1", "striptags": "3.1.1",
"subscriptions-transport-ws": "0.9.16", "subscriptions-transport-ws": "0.9.16",
"tar-fs": "2.0.0", "tar-fs": "2.0.0",
"twemoji": "12.0.4", "twemoji": "12.0.4",
"uslug": "1.0.4", "uslug": "1.0.4",
"uuid": "3.3.2", "uuid": "3.3.2",
"validate.js": "0.12.0", "validate.js": "0.13.1",
"validator": "11.0.0", "validator": "11.0.0",
"validator-as-promised": "1.0.2", "validator-as-promised": "1.0.2",
"winston": "3.2.1", "winston": "3.2.1",
@ -177,36 +178,36 @@
"@babel/plugin-syntax-import-meta": "^7.2.0", "@babel/plugin-syntax-import-meta": "^7.2.0",
"@babel/polyfill": "^7.4.4", "@babel/polyfill": "^7.4.4",
"@babel/preset-env": "^7.4.5", "@babel/preset-env": "^7.4.5",
"@panter/vue-i18next": "0.15.0", "@panter/vue-i18next": "0.15.1",
"animate-sass": "0.8.2", "animate-sass": "0.8.2",
"animated-number-vue": "0.1.5", "animated-number-vue": "0.1.5",
"apollo-cache-inmemory": "1.6.0", "apollo-cache-inmemory": "1.6.2",
"apollo-client": "2.6.0", "apollo-client": "2.6.3",
"apollo-link": "1.2.11", "apollo-link": "1.2.12",
"apollo-link-batch-http": "1.2.11", "apollo-link-batch-http": "1.2.12",
"apollo-link-error": "1.1.10", "apollo-link-error": "1.1.11",
"apollo-link-http": "1.5.14", "apollo-link-http": "1.5.15",
"apollo-link-persisted-queries": "0.2.2", "apollo-link-persisted-queries": "0.2.2",
"apollo-link-ws": "1.0.17", "apollo-link-ws": "1.0.18",
"apollo-utilities": "1.3.0", "apollo-utilities": "1.3.2",
"autoprefixer": "9.5.1", "autoprefixer": "9.6.0",
"babel-eslint": "10.0.1", "babel-eslint": "10.0.2",
"babel-jest": "24.8.0", "babel-jest": "24.8.0",
"babel-loader": "^8.0.6", "babel-loader": "^8.0.6",
"babel-plugin-graphql-tag": "2.4.0", "babel-plugin-graphql-tag": "2.4.0",
"babel-plugin-lodash": "3.3.4", "babel-plugin-lodash": "3.3.4",
"babel-plugin-transform-imports": "1.5.1", "babel-plugin-transform-imports": "1.5.1",
"brace": "0.11.1", "brace": "0.11.1",
"cache-loader": "3.0.1", "cache-loader": "4.0.0",
"chart.js": "2.8.0", "chart.js": "2.8.0",
"clean-webpack-plugin": "3.0.0", "clean-webpack-plugin": "3.0.0",
"copy-webpack-plugin": "5.0.3", "copy-webpack-plugin": "5.0.3",
"core-js": "3.1.3", "core-js": "3.1.4",
"css-loader": "2.1.1", "css-loader": "3.0.0",
"cssnano": "4.1.10", "cssnano": "4.1.10",
"duplicate-package-checker-webpack-plugin": "3.0.0", "duplicate-package-checker-webpack-plugin": "3.0.0",
"epic-spinners": "1.1.0", "epic-spinners": "1.1.0",
"eslint": "5.16.0", "eslint": "6.0.0",
"eslint-config-requarks": "1.0.7", "eslint-config-requarks": "1.0.7",
"eslint-config-standard": "12.0.0", "eslint-config-standard": "12.0.0",
"eslint-plugin-import": "2.17.3", "eslint-plugin-import": "2.17.3",
@ -214,23 +215,23 @@
"eslint-plugin-promise": "4.1.1", "eslint-plugin-promise": "4.1.1",
"eslint-plugin-standard": "4.0.0", "eslint-plugin-standard": "4.0.0",
"eslint-plugin-vue": "5.2.2", "eslint-plugin-vue": "5.2.2",
"file-loader": "3.0.1", "file-loader": "4.0.0",
"filepond": "4.4.8", "filepond": "4.4.9",
"filepond-plugin-file-validate-type": "1.2.4", "filepond-plugin-file-validate-type": "1.2.4",
"filesize.js": "1.0.2", "filesize.js": "1.0.2",
"grapesjs": "0.14.61", "grapesjs": "0.14.62",
"graphiql": "0.13.0", "graphiql": "0.13.2",
"graphql-persisted-document-loader": "1.0.1", "graphql-persisted-document-loader": "1.0.1",
"graphql-tag": "^2.10.1", "graphql-tag": "^2.10.1",
"graphql-voyager": "1.0.0-rc.27", "graphql-voyager": "1.0.0-rc.27",
"hammerjs": "2.0.8", "hammerjs": "2.0.8",
"html-webpack-plugin": "3.2.0", "html-webpack-plugin": "3.2.0",
"html-webpack-pug-plugin": "0.3.0", "html-webpack-pug-plugin": "0.3.0",
"i18next-xhr-backend": "2.0.1", "i18next-xhr-backend": "3.0.0",
"ignore-loader": "0.1.2", "ignore-loader": "0.1.2",
"js-cookie": "2.2.0", "js-cookie": "2.2.0",
"mini-css-extract-plugin": "0.7.0", "mini-css-extract-plugin": "0.7.0",
"moment-duration-format": "2.2.2", "moment-duration-format": "2.3.2",
"node-sass": "4.12.0", "node-sass": "4.12.0",
"offline-plugin": "5.0.7", "offline-plugin": "5.0.7",
"optimize-css-assets-webpack-plugin": "5.0.1", "optimize-css-assets-webpack-plugin": "5.0.1",
@ -244,7 +245,7 @@
"pug-lint": "2.5.0", "pug-lint": "2.5.0",
"pug-loader": "2.4.0", "pug-loader": "2.4.0",
"pug-plain-loader": "1.0.0", "pug-plain-loader": "1.0.0",
"raw-loader": "2.0.0", "raw-loader": "3.0.0",
"react": "16.8.6", "react": "16.8.6",
"react-dom": "16.8.6", "react-dom": "16.8.6",
"resolve-url-loader": "3.1.0", "resolve-url-loader": "3.1.0",
@ -257,14 +258,14 @@
"stylus-loader": "3.0.2", "stylus-loader": "3.0.2",
"terser": "4.0.0", "terser": "4.0.0",
"twemoji-awesome": "1.0.6", "twemoji-awesome": "1.0.6",
"url-loader": "1.1.2", "url-loader": "2.0.0",
"vee-validate": "2.2.9", "vee-validate": "2.2.11",
"velocity-animate": "1.5.2", "velocity-animate": "1.5.2",
"viz.js": "2.1.2", "viz.js": "2.1.2",
"vue": "2.6.10", "vue": "2.6.10",
"vue-apollo": "3.0.0-beta.30", "vue-apollo": "3.0.0-rc.1",
"vue-chartjs": "3.4.2", "vue-chartjs": "3.4.2",
"vue-clipboards": "1.2.4", "vue-clipboards": "1.3.0",
"vue-codemirror": "4.0.6", "vue-codemirror": "4.0.6",
"vue-filepond": "5.1.1", "vue-filepond": "5.1.1",
"vue-hot-reload-api": "2.3.3", "vue-hot-reload-api": "2.3.3",
@ -277,22 +278,22 @@
"vue-template-compiler": "2.6.10", "vue-template-compiler": "2.6.10",
"vue-tour": "1.1.0", "vue-tour": "1.1.0",
"vue2-animate": "2.1.0", "vue2-animate": "2.1.0",
"vuedraggable": "2.21.0", "vuedraggable": "2.22.0",
"vuescroll": "4.13.0-beta", "vuescroll": "4.13.0",
"vuetify": "1.5.14", "vuetify": "1.5.16",
"vuex": "3.1.1", "vuex": "3.1.1",
"vuex-pathify": "1.2.2", "vuex-pathify": "1.2.4",
"vuex-persistedstate": "2.5.4", "vuex-persistedstate": "2.5.4",
"webpack": "4.32.2", "webpack": "4.35.0",
"webpack-bundle-analyzer": "3.3.2", "webpack-bundle-analyzer": "3.3.2",
"webpack-cli": "3.3.2", "webpack-cli": "3.3.4",
"webpack-dev-middleware": "3.7.0", "webpack-dev-middleware": "3.7.0",
"webpack-hot-middleware": "2.25.0", "webpack-hot-middleware": "2.25.0",
"webpack-merge": "4.2.1", "webpack-merge": "4.2.1",
"webpack-subresource-integrity": "1.3.2", "webpack-subresource-integrity": "1.3.2",
"whatwg-fetch": "3.0.0", "whatwg-fetch": "3.0.0",
"write-file-webpack-plugin": "4.5.0", "write-file-webpack-plugin": "4.5.0",
"xterm": "3.14.1", "xterm": "3.14.2",
"zxcvbn": "4.4.2" "zxcvbn": "4.4.2"
}, },
"browserslist": [ "browserslist": [

16
server/graph/resolvers/search.js
Unescape View File

@ -19,14 +19,16 @@ module.exports = {
...searchEngineInfo, ...searchEngineInfo,
...searchEngine, ...searchEngine,
config: _.sortBy(_.transform(searchEngine.config, (res, value, key) => { config: _.sortBy(_.transform(searchEngine.config, (res, value, key) => {
const configData = _.get(searchEngineInfo.props, key, {}) const configData = _.get(searchEngineInfo.props, key, false)
res.push({ if (configData) {
key, res.push({
value: JSON.stringify({ key,
...configData, value: JSON.stringify({
value ...configData,
value
})
}) })
}) }
}, []), 'key') }, []), 'key')
} }
}) })

28
server/modules/search/elasticsearch/definition.yml
Unescape View File

@ -8,46 +8,34 @@ isAvailable: true
props: props:
apiVersion: apiVersion:
type: String type: String
title: API Version title: Elasticsearch Version
hint: Should match the version of the Elasticsearch nodes you are connecting to hint: Should match the version of the Elasticsearch nodes you are connecting to
order: 1 order: 1
enum: enum:
- '6.6' - '7.x'
- '6.5' - '6.x'
- '6.4' default: '6.x'
- '6.3'
default: '6.6'
hosts: hosts:
type: String type: String
title: Host(s) title: Host(s)
hint: Comma-separated list of Elasticsearch hosts to connect to. (including the port) hint: Comma-separated list of Elasticsearch hosts to connect to, including the port, username and password if necessary. (e.g. http://localhost:9200, https://user:pass@es1.example.com:9200)
order: 2 order: 2
user:
type: String
title: Username
hint: (Optional) Username to use if using the security feature from X-Pack
order: 3
pass:
type: String
title: Password
hint: (Optional) Password to use if using the security feature from X-Pack
order: 4
indexName: indexName:
type: String type: String
title: Index Name title: Index Name
hint: The index name to use during creation hint: The index name to use during creation
default: wiki default: wiki
order: 5 order: 3
sniffOnStart: sniffOnStart:
type: Boolean type: Boolean
title: Sniff on start title: Sniff on start
hint: 'Should Wiki.js attempt to detect the rest of the cluster on first connect? (Default: off)' hint: 'Should Wiki.js attempt to detect the rest of the cluster on first connect? (Default: off)'
default: false default: false
order: 6 order: 4
sniffInterval: sniffInterval:
type: Number type: Number
title: Sniff Interval title: Sniff Interval
hint: '0 = disabled, Interval in seconds to check for updated list of nodes in cluster. (Default: 0)' hint: '0 = disabled, Interval in seconds to check for updated list of nodes in cluster. (Default: 0)'
default: 0 default: 0
order: 7 order: 5

80
server/modules/search/elasticsearch/engine.js
Unescape View File

@ -1,5 +1,4 @@
const _ = require('lodash') const _ = require('lodash')
const elasticsearch = require('elasticsearch')
const stream = require('stream') const stream = require('stream')
const Promise = require('bluebird') const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline) const pipeline = Promise.promisify(stream.pipeline)
@ -18,13 +17,28 @@ module.exports = {
*/ */
async init() { async init() {
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Initializing...`) WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Initializing...`)
this.client = new elasticsearch.Client({ switch (this.config.apiVersion) {
apiVersion: this.config.apiVersion, case '7.x':
hosts: this.config.hosts.split(',').map(_.trim), const { Client: Client7 } = require('elasticsearch7')
httpAuth: (this.config.user.length > 0) ? `${this.config.user}:${this.config.pass}` : null, this.client = new Client7({
sniffOnStart: this.config.sniffOnStart, nodes: this.config.hosts.split(',').map(_.trim),
sniffInterval: (this.config.sniffInterval > 0) ? this.config.sniffInterval : false sniffOnStart: this.config.sniffOnStart,
}) sniffInterval: (this.config.sniffInterval > 0) ? this.config.sniffInterval : false,
name: 'wiki-js'
})
break
case '6.x':
const { Client: Client6 } = require('elasticsearch6')
this.client = new Client6({
nodes: this.config.hosts.split(',').map(_.trim),
sniffOnStart: this.config.sniffOnStart,
sniffInterval: (this.config.sniffInterval > 0) ? this.config.sniffInterval : false,
name: 'wiki-js'
})
break
default:
throw new Error('Unsupported version of elasticsearch! Update your settings in the Administration Area.')
}
// -> Create Search Index // -> Create Search Index
await this.createIndex() await this.createIndex()
@ -35,26 +49,35 @@ module.exports = {
* Create Index * Create Index
*/ */
async createIndex() { async createIndex() {
const indexExists = await this.client.indices.exists({ index: this.config.indexName }) try {
if (!indexExists) { const indexExists = await this.client.indices.exists({ index: this.config.indexName })
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Creating index...`) if (!indexExists.body) {
await this.client.indices.create({ WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Creating index...`)
index: this.config.indexName, try {
body: { const idxBody = {
mappings: { properties: {
_doc: { suggest: { type: 'completion' },
properties: { title: { type: 'text', boost: 4.0 },
suggest: { type: 'completion' }, description: { type: 'text', boost: 3.0 },
title: { type: 'text', boost: 4.0 }, content: { type: 'text', boost: 1.0 },
description: { type: 'text', boost: 3.0 }, locale: { type: 'keyword' },
content: { type: 'text', boost: 1.0 }, path: { type: 'text' }
locale: { type: 'keyword' },
path: { type: 'text' }
}
} }
} }
await this.client.indices.create({
index: this.config.indexName,
body: {
mappings: (this.config.apiVersion === '6.x') ? {
_doc: idxBody
} : idxBody
}
})
} catch (err) {
WIKI.logger.error(`(SEARCH/ELASTICSEARCH) Create Index Error: `, _.get(err, 'meta.body.error', err))
} }
}) }
} catch (err) {
WIKI.logger.error(`(SEARCH/ELASTICSEARCH) Index Check Error: `, _.get(err, 'meta.body.error', err))
} }
}, },
/** /**
@ -90,7 +113,7 @@ module.exports = {
} }
}) })
return { return {
results: _.get(results, 'hits.hits', []).map(r => ({ results: _.get(results, 'body.hits.hits', []).map(r => ({
id: r._id, id: r._id,
locale: r._source.locale, locale: r._source.locale,
path: r._source.path, path: r._source.path,
@ -98,11 +121,10 @@ module.exports = {
description: r._source.description description: r._source.description
})), })),
suggestions: _.reject(_.get(results, 'suggest.suggestions', []).map(s => _.get(s, 'options[0].text', false)), s => !s), suggestions: _.reject(_.get(results, 'suggest.suggestions', []).map(s => _.get(s, 'options[0].text', false)), s => !s),
totalHits: results.hits.total totalHits: _.get(results, 'body.hits.total.value', _.get(results, 'body.hits.total', 0))
} }
} catch (err) { } catch (err) {
WIKI.logger.warn('Search Engine Error:') WIKI.logger.warn('Search Engine Error: ', _.get(err, 'meta.body.error', err))
WIKI.logger.warn(err)
} }
}, },
/** /**

1122
yarn.lock
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save