mirror of https://github.com/requarks/wiki
Merge 0982ecba91 into 42a72183d0
byteplow
2 weeks ago
committed by
GitHub
commit
5fd9e3f595
@ -0,0 +1,57 @@
|
||||
key: opensearch
|
||||
title: OpenSearch
|
||||
description: OpenSearch is a community-driven, Apache 2.0-licensed open source search and analytics suite that makes it easy to ingest, search, visualize, and analyze data.
|
||||
author: Metaways Infosystems
|
||||
logo: https://opensearch.org/wp-content/uploads/2024/11/favicon.webp
|
||||
website: https://opensearch.org/
|
||||
isAvailable: true
|
||||
props:
|
||||
apiVersion:
|
||||
type: String
|
||||
title: OpenSearch Version
|
||||
hint: Should match the version of the OpenSearch nodes you are connecting to
|
||||
order: 1
|
||||
enum:
|
||||
- '2.x'
|
||||
- '2.x'
|
||||
- '1.x'
|
||||
default: '2.x'
|
||||
hosts:
|
||||
type: String
|
||||
title: Host(s)
|
||||
hint: Comma-separated list of OpenSearch hosts to connect to, including the port, username and password if necessary. (e.g. http://localhost:9200, https://user:pass@es1.example.com:9200)
|
||||
order: 2
|
||||
verifyTLSCertificate:
|
||||
title: Verify TLS Certificate
|
||||
type: Boolean
|
||||
default: true
|
||||
order: 3
|
||||
tlsCertPath:
|
||||
title: TLS Certificate Path
|
||||
type: String
|
||||
hint: Absolute path to the TLS certificate on the server.
|
||||
order: 4
|
||||
indexName:
|
||||
type: String
|
||||
title: Index Name
|
||||
hint: The index name to use during creation
|
||||
default: wiki
|
||||
order: 5
|
||||
analyzer:
|
||||
type: String
|
||||
title: Analyzer
|
||||
hint: 'The token analyzer in OpenSearch'
|
||||
default: simple
|
||||
order: 6
|
||||
sniffOnStart:
|
||||
type: Boolean
|
||||
title: Sniff on start
|
||||
hint: 'Should Wiki.js attempt to detect the rest of the cluster on first connect? (Default: off)'
|
||||
default: false
|
||||
order: 7
|
||||
sniffInterval:
|
||||
type: Number
|
||||
title: Sniff Interval
|
||||
hint: '0 = disabled, Interval in seconds to check for updated list of nodes in cluster. (Default: 0)'
|
||||
default: 0
|
||||
order: 8
|
||||
@ -0,0 +1,345 @@
|
||||
const _ = require('lodash')
|
||||
const fs = require('fs')
|
||||
const { pipeline } = require('node:stream/promises')
|
||||
const { Transform } = require('node:stream')
|
||||
const ElasticSearch = require('../elasticsearch/engine')
|
||||
|
||||
/* global WIKI */
|
||||
|
||||
module.exports = {
|
||||
async activate() {
|
||||
// not used
|
||||
},
|
||||
async deactivate() {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* INIT
|
||||
*/
|
||||
async init() {
|
||||
WIKI.logger.info(`(SEARCH/OPENSEARCH) Initializing...`)
|
||||
|
||||
|
||||
switch (this.config.apiVersion) {
|
||||
case '3.x':
|
||||
case '2.x':
|
||||
case '1.x':
|
||||
var { Client } = require('@opensearch-project/opensearch');
|
||||
this.client = new Client({
|
||||
nodes: this.config.hosts.split(',').map(_.trim),
|
||||
sniffOnStart: this.config.sniffOnStart,
|
||||
sniffInterval: (this.config.sniffInterval > 0) ? this.config.sniffInterval : false,
|
||||
ssl: getTlsOptions(this.config),
|
||||
name: 'wiki-js'
|
||||
})
|
||||
|
||||
break
|
||||
default:
|
||||
throw new Error('Unsupported version of OpenSearch! Update your settings in the Administration Area.')
|
||||
}
|
||||
|
||||
// -> Create Search Index
|
||||
await this.createIndex()
|
||||
|
||||
WIKI.logger.info(`(SEARCH/OPENSEARCH) Initialization completed.`)
|
||||
},
|
||||
/**
|
||||
* Create Index
|
||||
*/
|
||||
async createIndex() {
|
||||
try {
|
||||
const indexExists = await this.client.indices.exists({ index: this.config.indexName })
|
||||
|
||||
if (!indexExists.body) {
|
||||
WIKI.logger.info(`(SEARCH/OPENSEARCH) Creating index...`)
|
||||
try {
|
||||
await this.client.indices.create({
|
||||
index: this.config.indexName,
|
||||
body: {
|
||||
mappings: {
|
||||
properties: {
|
||||
suggest: { type: 'completion' },
|
||||
title: { type: 'text', boost: 10.0 },
|
||||
description: { type: 'text', boost: 3.0 },
|
||||
content: { type: 'text', boost: 1.0 },
|
||||
locale: { type: 'keyword' },
|
||||
path: { type: 'text' },
|
||||
tags: { type: 'text', boost: 8.0 }
|
||||
}
|
||||
},
|
||||
settings: {
|
||||
analysis: {
|
||||
analyzer: {
|
||||
default: {
|
||||
type: this.config.analyzer
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
WIKI.logger.error(`(SEARCH/OPENSEARCH) Create Index Error: `, _.get(err, 'meta.body.error', err))
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
WIKI.logger.error(`(SEARCH/OPENSEARCH) Index Check Error: `, _.get(err, 'meta.body.error', err))
|
||||
}
|
||||
},
|
||||
/**
|
||||
* QUERY
|
||||
*
|
||||
* @param {String} q Query
|
||||
* @param {Object} opts Additional options
|
||||
*/
|
||||
async query(q, opts) {
|
||||
try {
|
||||
const results = await this.client.search({
|
||||
index: this.config.indexName,
|
||||
body: {
|
||||
query: {
|
||||
simple_query_string: {
|
||||
query: `*${q}*`,
|
||||
fields: ['title^20', 'description^3', 'tags^8', 'content^1'],
|
||||
default_operator: 'and',
|
||||
analyze_wildcard: true
|
||||
}
|
||||
},
|
||||
from: 0,
|
||||
size: 50,
|
||||
_source: ['title', 'description', 'path', 'locale'],
|
||||
suggest: {
|
||||
suggestions: {
|
||||
text: q,
|
||||
completion: {
|
||||
field: 'suggest',
|
||||
size: 5,
|
||||
skip_duplicates: true,
|
||||
fuzzy: true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
return {
|
||||
results: _.get(results, 'body.hits.hits', []).map(r => ({
|
||||
id: r._id,
|
||||
locale: r._source.locale,
|
||||
path: r._source.path,
|
||||
title: r._source.title,
|
||||
description: r._source.description
|
||||
})),
|
||||
suggestions: _.reject(_.get(results, 'suggest.suggestions', []).map(s => _.get(s, 'options[0].text', false)), s => !s),
|
||||
totalHits: _.get(results, 'body.hits.total.value', _.get(results, 'body.hits.total', 0))
|
||||
}
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('Search Engine Error: ', _.get(err, 'meta.body.error', err))
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Build tags field
|
||||
* @param id
|
||||
* @returns {Promise<*|*[]>}
|
||||
*/
|
||||
async buildTags(id) {
|
||||
return ElasticSearch.buildTags(id)
|
||||
},
|
||||
/**
|
||||
* Build suggest field
|
||||
*/
|
||||
buildSuggest(page) {
|
||||
return ElasticSearch.buildSuggest(page)
|
||||
},
|
||||
/**
|
||||
* CREATE
|
||||
*
|
||||
* @param {Object} page Page to create
|
||||
*/
|
||||
async created(page) {
|
||||
await this.client.index({
|
||||
index: this.config.indexName,
|
||||
id: page.hash,
|
||||
body: {
|
||||
suggest: this.buildSuggest(page),
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent,
|
||||
tags: await this.buildTags(page.id)
|
||||
},
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* UPDATE
|
||||
*
|
||||
* @param {Object} page Page to update
|
||||
*/
|
||||
async updated(page) {
|
||||
await this.client.index({
|
||||
index: this.config.indexName,
|
||||
id: page.hash,
|
||||
body: {
|
||||
suggest: this.buildSuggest(page),
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent,
|
||||
tags: await this.buildTags(page.id)
|
||||
},
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* DELETE
|
||||
*
|
||||
* @param {Object} page Page to delete
|
||||
*/
|
||||
async deleted(page) {
|
||||
await this.client.delete({
|
||||
index: this.config.indexName,
|
||||
id: page.hash,
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* RENAME
|
||||
*
|
||||
* @param {Object} page Page to rename
|
||||
*/
|
||||
async renamed(page) {
|
||||
await this.client.delete({
|
||||
index: this.config.indexName,
|
||||
id: page.hash,
|
||||
refresh: true
|
||||
})
|
||||
await this.client.index({
|
||||
index: this.config.indexName,
|
||||
id: page.destinationHash,
|
||||
body: {
|
||||
suggest: this.buildSuggest(page),
|
||||
locale: page.destinationLocaleCode,
|
||||
path: page.destinationPath,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent,
|
||||
tags: await this.buildTags(page.id)
|
||||
},
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* REBUILD INDEX
|
||||
*/
|
||||
async rebuild() {
|
||||
WIKI.logger.info(`(SEARCH/OPENSEARCH) Rebuilding Index...`)
|
||||
await this.client.indices.delete({ index: this.config.indexName })
|
||||
await this.createIndex()
|
||||
|
||||
const MAX_INDEXING_BYTES = 10 * Math.pow(2, 20) - Buffer.from('[').byteLength - Buffer.from(']').byteLength // 10 MB
|
||||
const MAX_INDEXING_COUNT = 1000
|
||||
const COMMA_BYTES = Buffer.from(',').byteLength
|
||||
|
||||
let chunks = []
|
||||
let bytes = 0
|
||||
|
||||
const processDocument = async (cb, doc) => {
|
||||
try {
|
||||
if (doc) {
|
||||
const docBytes = Buffer.from(JSON.stringify(doc)).byteLength
|
||||
|
||||
doc['tags'] = await this.buildTags(doc.realId)
|
||||
// -> Current batch exceeds size limit, flush
|
||||
if (docBytes + COMMA_BYTES + bytes >= MAX_INDEXING_BYTES) {
|
||||
await flushBuffer()
|
||||
}
|
||||
|
||||
if (chunks.length > 0) {
|
||||
bytes += COMMA_BYTES
|
||||
}
|
||||
bytes += docBytes
|
||||
chunks.push(doc)
|
||||
|
||||
// -> Current batch exceeds count limit, flush
|
||||
if (chunks.length >= MAX_INDEXING_COUNT) {
|
||||
await flushBuffer()
|
||||
}
|
||||
} else {
|
||||
// -> End of stream, flush
|
||||
await flushBuffer()
|
||||
}
|
||||
cb()
|
||||
} catch (err) {
|
||||
cb(err)
|
||||
}
|
||||
}
|
||||
|
||||
const flushBuffer = async () => {
|
||||
WIKI.logger.info(`(SEARCH/OPENSEARCH) Sending batch of ${chunks.length}...`)
|
||||
try {
|
||||
await this.client.bulk({
|
||||
index: this.config.indexName,
|
||||
body: _.reduce(chunks, (result, doc) => {
|
||||
result.push({
|
||||
index: {
|
||||
_index: this.config.indexName,
|
||||
_id: doc.id,
|
||||
}
|
||||
})
|
||||
doc.safeContent = WIKI.models.pages.cleanHTML(doc.render)
|
||||
result.push({
|
||||
suggest: this.buildSuggest(doc),
|
||||
tags: doc.tags,
|
||||
locale: doc.locale,
|
||||
path: doc.path,
|
||||
title: doc.title,
|
||||
description: doc.description,
|
||||
content: doc.safeContent
|
||||
})
|
||||
return result
|
||||
}, []),
|
||||
refresh: true
|
||||
})
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('(SEARCH/OPENSEARCH) Failed to send batch to OpenSearch: ', err)
|
||||
}
|
||||
chunks.length = 0
|
||||
bytes = 0
|
||||
}
|
||||
|
||||
// Added real id in order to fetch page tags from the query
|
||||
await pipeline(
|
||||
WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'render', { realId: 'id' }).select().from('pages').where({
|
||||
isPublished: true,
|
||||
isPrivate: false
|
||||
}).stream(),
|
||||
new Transform({
|
||||
objectMode: true,
|
||||
transform: async (chunk, enc, cb) => processDocument(cb, chunk),
|
||||
flush: async (cb) => processDocument(cb)
|
||||
})
|
||||
)
|
||||
WIKI.logger.info(`(SEARCH/OPENSEARCH) Index rebuilt successfully.`)
|
||||
}
|
||||
}
|
||||
|
||||
function getTlsOptions(conf) {
|
||||
if (!conf.tlsCertPath) {
|
||||
return {
|
||||
rejectUnauthorized: conf.verifyTLSCertificate
|
||||
}
|
||||
}
|
||||
|
||||
const caList = []
|
||||
if (conf.verifyTLSCertificate) {
|
||||
caList.push(fs.readFileSync(conf.tlsCertPath))
|
||||
}
|
||||
|
||||
return {
|
||||
rejectUnauthorized: conf.verifyTLSCertificate,
|
||||
ca: caList
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue