mirror of https://github.com/requarks/wiki
chore: DB ref refactor + add scheduler db model + remove search modules (#5699)
Nicolas Giard
2 years ago
committed by
GitHub
parent
5c6ae08b84
commit
10cc2ef4b9
@ -1,24 +0,0 @@
|
||||
key: algolia
|
||||
title: Algolia
|
||||
description: Algolia is a powerful search-as-a-service solution, made easy to use with API clients, UI libraries, and pre-built integrations.
|
||||
author: requarks.io
|
||||
logo: https://static.requarks.io/logo/algolia.svg
|
||||
website: https://www.algolia.com/
|
||||
isAvailable: true
|
||||
props:
|
||||
appId:
|
||||
type: String
|
||||
title: App ID
|
||||
hint: Your Algolia Application ID, found under API Keys
|
||||
order: 1
|
||||
apiKey:
|
||||
type: String
|
||||
title: Admin API Key
|
||||
hint: Your Algolia Admin API Key, found under API Keys.
|
||||
order: 2
|
||||
indexName:
|
||||
type: String
|
||||
title: Index Name
|
||||
hint: The name of the index you created under Indices.
|
||||
default: wiki
|
||||
order: 3
|
||||
@ -1,203 +0,0 @@
|
||||
const _ = require('lodash')
|
||||
const algoliasearch = require('algoliasearch')
|
||||
const stream = require('stream')
|
||||
const Promise = require('bluebird')
|
||||
const pipeline = Promise.promisify(stream.pipeline)
|
||||
|
||||
/* global WIKI */
|
||||
|
||||
module.exports = {
|
||||
async activate() {
|
||||
// not used
|
||||
},
|
||||
async deactivate() {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* INIT
|
||||
*/
|
||||
async init() {
|
||||
WIKI.logger.info(`(SEARCH/ALGOLIA) Initializing...`)
|
||||
this.client = algoliasearch(this.config.appId, this.config.apiKey)
|
||||
this.index = this.client.initIndex(this.config.indexName)
|
||||
|
||||
// -> Create Search Index
|
||||
WIKI.logger.info(`(SEARCH/ALGOLIA) Setting index configuration...`)
|
||||
await this.index.setSettings({
|
||||
searchableAttributes: [
|
||||
'title',
|
||||
'description',
|
||||
'content'
|
||||
],
|
||||
attributesToRetrieve: [
|
||||
'locale',
|
||||
'path',
|
||||
'title',
|
||||
'description'
|
||||
],
|
||||
advancedSyntax: true
|
||||
})
|
||||
WIKI.logger.info(`(SEARCH/ALGOLIA) Initialization completed.`)
|
||||
},
|
||||
/**
|
||||
* QUERY
|
||||
*
|
||||
* @param {String} q Query
|
||||
* @param {Object} opts Additional options
|
||||
*/
|
||||
async query(q, opts) {
|
||||
try {
|
||||
const results = await this.index.search(q, {
|
||||
hitsPerPage: 50
|
||||
})
|
||||
return {
|
||||
results: _.map(results.hits, r => ({
|
||||
id: r.objectID,
|
||||
locale: r.locale,
|
||||
path: r.path,
|
||||
title: r.title,
|
||||
description: r.description
|
||||
})),
|
||||
suggestions: [],
|
||||
totalHits: results.nbHits
|
||||
}
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('Search Engine Error:')
|
||||
WIKI.logger.warn(err)
|
||||
}
|
||||
},
|
||||
/**
|
||||
* CREATE
|
||||
*
|
||||
* @param {Object} page Page to create
|
||||
*/
|
||||
async created(page) {
|
||||
await this.index.saveObject({
|
||||
objectID: page.hash,
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
})
|
||||
},
|
||||
/**
|
||||
* UPDATE
|
||||
*
|
||||
* @param {Object} page Page to update
|
||||
*/
|
||||
async updated(page) {
|
||||
await this.index.partialUpdateObject({
|
||||
objectID: page.hash,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
})
|
||||
},
|
||||
/**
|
||||
* DELETE
|
||||
*
|
||||
* @param {Object} page Page to delete
|
||||
*/
|
||||
async deleted(page) {
|
||||
await this.index.deleteObject(page.hash)
|
||||
},
|
||||
/**
|
||||
* RENAME
|
||||
*
|
||||
* @param {Object} page Page to rename
|
||||
*/
|
||||
async renamed(page) {
|
||||
await this.index.deleteObject(page.hash)
|
||||
await this.index.saveObject({
|
||||
objectID: page.destinationHash,
|
||||
locale: page.destinationLocaleCode,
|
||||
path: page.destinationPath,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
})
|
||||
},
|
||||
/**
|
||||
* REBUILD INDEX
|
||||
*/
|
||||
async rebuild() {
|
||||
WIKI.logger.info(`(SEARCH/ALGOLIA) Rebuilding Index...`)
|
||||
await this.index.clearObjects()
|
||||
|
||||
const MAX_DOCUMENT_BYTES = 10 * Math.pow(2, 10) // 10 KB
|
||||
const MAX_INDEXING_BYTES = 10 * Math.pow(2, 20) - Buffer.from('[').byteLength - Buffer.from(']').byteLength // 10 MB
|
||||
const MAX_INDEXING_COUNT = 1000
|
||||
const COMMA_BYTES = Buffer.from(',').byteLength
|
||||
|
||||
let chunks = []
|
||||
let bytes = 0
|
||||
|
||||
const processDocument = async (cb, doc) => {
|
||||
try {
|
||||
if (doc) {
|
||||
const docBytes = Buffer.from(JSON.stringify(doc)).byteLength
|
||||
// -> Document too large
|
||||
if (docBytes >= MAX_DOCUMENT_BYTES) {
|
||||
throw new Error('Document exceeds maximum size allowed by Algolia.')
|
||||
}
|
||||
|
||||
// -> Current batch exceeds size hard limit, flush
|
||||
if (docBytes + COMMA_BYTES + bytes >= MAX_INDEXING_BYTES) {
|
||||
await flushBuffer()
|
||||
}
|
||||
|
||||
if (chunks.length > 0) {
|
||||
bytes += COMMA_BYTES
|
||||
}
|
||||
bytes += docBytes
|
||||
chunks.push(doc)
|
||||
|
||||
// -> Current batch exceeds count soft limit, flush
|
||||
if (chunks.length >= MAX_INDEXING_COUNT) {
|
||||
await flushBuffer()
|
||||
}
|
||||
} else {
|
||||
// -> End of stream, flush
|
||||
await flushBuffer()
|
||||
}
|
||||
cb()
|
||||
} catch (err) {
|
||||
cb(err)
|
||||
}
|
||||
}
|
||||
|
||||
const flushBuffer = async () => {
|
||||
WIKI.logger.info(`(SEARCH/ALGOLIA) Sending batch of ${chunks.length}...`)
|
||||
try {
|
||||
await this.index.saveObjects(
|
||||
_.map(chunks, doc => ({
|
||||
objectID: doc.id,
|
||||
locale: doc.locale,
|
||||
path: doc.path,
|
||||
title: doc.title,
|
||||
description: doc.description,
|
||||
content: WIKI.models.pages.cleanHTML(doc.render)
|
||||
}))
|
||||
)
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('(SEARCH/ALGOLIA) Failed to send batch to Algolia: ', err)
|
||||
}
|
||||
chunks.length = 0
|
||||
bytes = 0
|
||||
}
|
||||
|
||||
await pipeline(
|
||||
WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'render').select().from('pages').where({
|
||||
isPublished: true,
|
||||
isPrivate: false
|
||||
}).stream(),
|
||||
new stream.Transform({
|
||||
objectMode: true,
|
||||
transform: async (chunk, enc, cb) => processDocument(cb, chunk),
|
||||
flush: async (cb) => processDocument(cb)
|
||||
})
|
||||
)
|
||||
WIKI.logger.info(`(SEARCH/ALGOLIA) Index rebuilt successfully.`)
|
||||
}
|
||||
}
|
||||
@ -1,88 +0,0 @@
|
||||
key: aws
|
||||
title: AWS CloudSearch
|
||||
description: Amazon CloudSearch is a managed service in the AWS Cloud that makes it simple and cost-effective to set up, manage, and scale a search solution for your website or application.
|
||||
author: requarks.io
|
||||
logo: https://static.requarks.io/logo/aws-cloudsearch.svg
|
||||
website: https://aws.amazon.com/cloudsearch/
|
||||
isAvailable: true
|
||||
props:
|
||||
domain:
|
||||
type: String
|
||||
title: Search Domain
|
||||
hint: The name of your CloudSearch service.
|
||||
order: 1
|
||||
endpoint:
|
||||
type: String
|
||||
title: Document Endpoint
|
||||
hint: The Document Endpoint specified in the domain AWS console dashboard.
|
||||
order: 2
|
||||
region:
|
||||
type: String
|
||||
title: Region
|
||||
hint: The AWS datacenter region where the instance was created.
|
||||
default: us-east-1
|
||||
enum:
|
||||
- ap-northeast-1
|
||||
- ap-northeast-2
|
||||
- ap-southeast-1
|
||||
- ap-southeast-2
|
||||
- eu-central-1
|
||||
- eu-west-1
|
||||
- sa-east-1
|
||||
- us-east-1
|
||||
- us-west-1
|
||||
- us-west-2
|
||||
order: 3
|
||||
accessKeyId:
|
||||
type: String
|
||||
title: Access Key ID
|
||||
hint: The Access Key ID with CloudSearchFullAccess role access to the CloudSearch instance.
|
||||
order: 4
|
||||
secretAccessKey :
|
||||
type: String
|
||||
title: Secret Access Key
|
||||
hint: The Secret Access Key for the Access Key ID provided above.
|
||||
order: 5
|
||||
AnalysisSchemeLang:
|
||||
type: String
|
||||
title: Analysis Scheme Language
|
||||
hint: The language used to analyse content.
|
||||
default: en
|
||||
enum:
|
||||
- 'ar'
|
||||
- 'bg'
|
||||
- 'ca'
|
||||
- 'cs'
|
||||
- 'da'
|
||||
- 'de'
|
||||
- 'el'
|
||||
- 'en'
|
||||
- 'es'
|
||||
- 'eu'
|
||||
- 'fa'
|
||||
- 'fi'
|
||||
- 'fr'
|
||||
- 'ga'
|
||||
- 'gl'
|
||||
- 'he'
|
||||
- 'hi'
|
||||
- 'hu'
|
||||
- 'hy'
|
||||
- 'id'
|
||||
- 'it'
|
||||
- 'ja'
|
||||
- 'ko'
|
||||
- 'lv'
|
||||
- 'mul'
|
||||
- 'nl'
|
||||
- 'no'
|
||||
- 'pt'
|
||||
- 'ro'
|
||||
- 'ru'
|
||||
- 'sv'
|
||||
- 'th'
|
||||
- 'tr'
|
||||
- 'zh-Hans'
|
||||
- 'zh-Hant'
|
||||
order: 6
|
||||
|
||||
@ -1,370 +0,0 @@
|
||||
const _ = require('lodash')
|
||||
const AWS = require('aws-sdk')
|
||||
const stream = require('stream')
|
||||
const Promise = require('bluebird')
|
||||
const pipeline = Promise.promisify(stream.pipeline)
|
||||
|
||||
/* global WIKI */
|
||||
|
||||
module.exports = {
|
||||
async activate() {
|
||||
// not used
|
||||
},
|
||||
async deactivate() {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* INIT
|
||||
*/
|
||||
async init() {
|
||||
WIKI.logger.info(`(SEARCH/AWS) Initializing...`)
|
||||
this.client = new AWS.CloudSearch({
|
||||
apiVersion: '2013-01-01',
|
||||
accessKeyId: this.config.accessKeyId,
|
||||
secretAccessKey: this.config.secretAccessKey,
|
||||
region: this.config.region
|
||||
})
|
||||
this.clientDomain = new AWS.CloudSearchDomain({
|
||||
apiVersion: '2013-01-01',
|
||||
endpoint: this.config.endpoint,
|
||||
accessKeyId: this.config.accessKeyId,
|
||||
secretAccessKey: this.config.secretAccessKey,
|
||||
region: this.config.region
|
||||
})
|
||||
|
||||
let rebuildIndex = false
|
||||
|
||||
// -> Define Analysis Schemes
|
||||
const schemes = await this.client.describeAnalysisSchemes({
|
||||
DomainName: this.config.domain,
|
||||
AnalysisSchemeNames: ['default_anlscheme']
|
||||
}).promise()
|
||||
if (_.get(schemes, 'AnalysisSchemes', []).length < 1) {
|
||||
WIKI.logger.info(`(SEARCH/AWS) Defining Analysis Scheme...`)
|
||||
await this.client.defineAnalysisScheme({
|
||||
DomainName: this.config.domain,
|
||||
AnalysisScheme: {
|
||||
AnalysisSchemeLanguage: this.config.AnalysisSchemeLang,
|
||||
AnalysisSchemeName: 'default_anlscheme'
|
||||
}
|
||||
}).promise()
|
||||
rebuildIndex = true
|
||||
}
|
||||
|
||||
// -> Define Index Fields
|
||||
const fields = await this.client.describeIndexFields({
|
||||
DomainName: this.config.domain
|
||||
}).promise()
|
||||
if (_.get(fields, 'IndexFields', []).length < 1) {
|
||||
WIKI.logger.info(`(SEARCH/AWS) Defining Index Fields...`)
|
||||
await this.client.defineIndexField({
|
||||
DomainName: this.config.domain,
|
||||
IndexField: {
|
||||
IndexFieldName: 'id',
|
||||
IndexFieldType: 'literal'
|
||||
}
|
||||
}).promise()
|
||||
await this.client.defineIndexField({
|
||||
DomainName: this.config.domain,
|
||||
IndexField: {
|
||||
IndexFieldName: 'path',
|
||||
IndexFieldType: 'literal'
|
||||
}
|
||||
}).promise()
|
||||
await this.client.defineIndexField({
|
||||
DomainName: this.config.domain,
|
||||
IndexField: {
|
||||
IndexFieldName: 'locale',
|
||||
IndexFieldType: 'literal'
|
||||
}
|
||||
}).promise()
|
||||
await this.client.defineIndexField({
|
||||
DomainName: this.config.domain,
|
||||
IndexField: {
|
||||
IndexFieldName: 'title',
|
||||
IndexFieldType: 'text',
|
||||
TextOptions: {
|
||||
ReturnEnabled: true,
|
||||
AnalysisScheme: 'default_anlscheme'
|
||||
}
|
||||
}
|
||||
}).promise()
|
||||
await this.client.defineIndexField({
|
||||
DomainName: this.config.domain,
|
||||
IndexField: {
|
||||
IndexFieldName: 'description',
|
||||
IndexFieldType: 'text',
|
||||
TextOptions: {
|
||||
ReturnEnabled: true,
|
||||
AnalysisScheme: 'default_anlscheme'
|
||||
}
|
||||
}
|
||||
}).promise()
|
||||
await this.client.defineIndexField({
|
||||
DomainName: this.config.domain,
|
||||
IndexField: {
|
||||
IndexFieldName: 'content',
|
||||
IndexFieldType: 'text',
|
||||
TextOptions: {
|
||||
ReturnEnabled: false,
|
||||
AnalysisScheme: 'default_anlscheme'
|
||||
}
|
||||
}
|
||||
}).promise()
|
||||
rebuildIndex = true
|
||||
}
|
||||
|
||||
// -> Define suggester
|
||||
const suggesters = await this.client.describeSuggesters({
|
||||
DomainName: this.config.domain,
|
||||
SuggesterNames: ['default_suggester']
|
||||
}).promise()
|
||||
if (_.get(suggesters, 'Suggesters', []).length < 1) {
|
||||
WIKI.logger.info(`(SEARCH/AWS) Defining Suggester...`)
|
||||
await this.client.defineSuggester({
|
||||
DomainName: this.config.domain,
|
||||
Suggester: {
|
||||
SuggesterName: 'default_suggester',
|
||||
DocumentSuggesterOptions: {
|
||||
SourceField: 'title',
|
||||
FuzzyMatching: 'high'
|
||||
}
|
||||
}
|
||||
}).promise()
|
||||
rebuildIndex = true
|
||||
}
|
||||
|
||||
// -> Rebuild Index
|
||||
if (rebuildIndex) {
|
||||
WIKI.logger.info(`(SEARCH/AWS) Requesting Index Rebuild...`)
|
||||
await this.client.indexDocuments({
|
||||
DomainName: this.config.domain
|
||||
}).promise()
|
||||
}
|
||||
|
||||
WIKI.logger.info(`(SEARCH/AWS) Initialization completed.`)
|
||||
},
|
||||
/**
|
||||
* QUERY
|
||||
*
|
||||
* @param {String} q Query
|
||||
* @param {Object} opts Additional options
|
||||
*/
|
||||
async query(q, opts) {
|
||||
try {
|
||||
let suggestions = []
|
||||
const results = await this.clientDomain.search({
|
||||
query: q,
|
||||
partial: true,
|
||||
size: 50
|
||||
}).promise()
|
||||
if (results.hits.found < 5) {
|
||||
const suggestResults = await this.clientDomain.suggest({
|
||||
query: q,
|
||||
suggester: 'default_suggester',
|
||||
size: 5
|
||||
}).promise()
|
||||
suggestions = suggestResults.suggest.suggestions.map(s => s.suggestion)
|
||||
}
|
||||
return {
|
||||
results: _.map(results.hits.hit, r => ({
|
||||
id: r.id,
|
||||
path: _.head(r.fields.path),
|
||||
locale: _.head(r.fields.locale),
|
||||
title: _.head(r.fields.title) || '',
|
||||
description: _.head(r.fields.description) || ''
|
||||
})),
|
||||
suggestions: suggestions,
|
||||
totalHits: results.hits.found
|
||||
}
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('Search Engine Error:')
|
||||
WIKI.logger.warn(err)
|
||||
}
|
||||
},
|
||||
/**
|
||||
* CREATE
|
||||
*
|
||||
* @param {Object} page Page to create
|
||||
*/
|
||||
async created(page) {
|
||||
await this.clientDomain.uploadDocuments({
|
||||
contentType: 'application/json',
|
||||
documents: JSON.stringify([
|
||||
{
|
||||
type: 'add',
|
||||
id: page.hash,
|
||||
fields: {
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
}
|
||||
}
|
||||
])
|
||||
}).promise()
|
||||
},
|
||||
/**
|
||||
* UPDATE
|
||||
*
|
||||
* @param {Object} page Page to update
|
||||
*/
|
||||
async updated(page) {
|
||||
await this.clientDomain.uploadDocuments({
|
||||
contentType: 'application/json',
|
||||
documents: JSON.stringify([
|
||||
{
|
||||
type: 'add',
|
||||
id: page.hash,
|
||||
fields: {
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
}
|
||||
}
|
||||
])
|
||||
}).promise()
|
||||
},
|
||||
/**
|
||||
* DELETE
|
||||
*
|
||||
* @param {Object} page Page to delete
|
||||
*/
|
||||
async deleted(page) {
|
||||
await this.clientDomain.uploadDocuments({
|
||||
contentType: 'application/json',
|
||||
documents: JSON.stringify([
|
||||
{
|
||||
type: 'delete',
|
||||
id: page.hash
|
||||
}
|
||||
])
|
||||
}).promise()
|
||||
},
|
||||
/**
|
||||
* RENAME
|
||||
*
|
||||
* @param {Object} page Page to rename
|
||||
*/
|
||||
async renamed(page) {
|
||||
await this.clientDomain.uploadDocuments({
|
||||
contentType: 'application/json',
|
||||
documents: JSON.stringify([
|
||||
{
|
||||
type: 'delete',
|
||||
id: page.hash
|
||||
}
|
||||
])
|
||||
}).promise()
|
||||
await this.clientDomain.uploadDocuments({
|
||||
contentType: 'application/json',
|
||||
documents: JSON.stringify([
|
||||
{
|
||||
type: 'add',
|
||||
id: page.destinationHash,
|
||||
fields: {
|
||||
locale: page.destinationLocaleCode,
|
||||
path: page.destinationPath,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
}
|
||||
}
|
||||
])
|
||||
}).promise()
|
||||
},
|
||||
/**
|
||||
* REBUILD INDEX
|
||||
*/
|
||||
async rebuild() {
|
||||
WIKI.logger.info(`(SEARCH/AWS) Rebuilding Index...`)
|
||||
|
||||
const MAX_DOCUMENT_BYTES = Math.pow(2, 20)
|
||||
const MAX_INDEXING_BYTES = 5 * Math.pow(2, 20) - Buffer.from('[').byteLength - Buffer.from(']').byteLength
|
||||
const MAX_INDEXING_COUNT = 1000
|
||||
const COMMA_BYTES = Buffer.from(',').byteLength
|
||||
|
||||
let chunks = []
|
||||
let bytes = 0
|
||||
|
||||
const processDocument = async (cb, doc) => {
|
||||
try {
|
||||
if (doc) {
|
||||
const docBytes = Buffer.from(JSON.stringify(doc)).byteLength
|
||||
// -> Document too large
|
||||
if (docBytes >= MAX_DOCUMENT_BYTES) {
|
||||
throw new Error('Document exceeds maximum size allowed by AWS CloudSearch.')
|
||||
}
|
||||
|
||||
// -> Current batch exceeds size hard limit, flush
|
||||
if (docBytes + COMMA_BYTES + bytes >= MAX_INDEXING_BYTES) {
|
||||
await flushBuffer()
|
||||
}
|
||||
|
||||
if (chunks.length > 0) {
|
||||
bytes += COMMA_BYTES
|
||||
}
|
||||
bytes += docBytes
|
||||
chunks.push(doc)
|
||||
|
||||
// -> Current batch exceeds count soft limit, flush
|
||||
if (chunks.length >= MAX_INDEXING_COUNT) {
|
||||
await flushBuffer()
|
||||
}
|
||||
} else {
|
||||
// -> End of stream, flush
|
||||
await flushBuffer()
|
||||
}
|
||||
cb()
|
||||
} catch (err) {
|
||||
cb(err)
|
||||
}
|
||||
}
|
||||
|
||||
const flushBuffer = async () => {
|
||||
WIKI.logger.info(`(SEARCH/AWS) Sending batch of ${chunks.length}...`)
|
||||
try {
|
||||
await this.clientDomain.uploadDocuments({
|
||||
contentType: 'application/json',
|
||||
documents: JSON.stringify(_.map(chunks, doc => ({
|
||||
type: 'add',
|
||||
id: doc.id,
|
||||
fields: {
|
||||
locale: doc.locale,
|
||||
path: doc.path,
|
||||
title: doc.title,
|
||||
description: doc.description,
|
||||
content: WIKI.models.pages.cleanHTML(doc.render)
|
||||
}
|
||||
})))
|
||||
}).promise()
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('(SEARCH/AWS) Failed to send batch to AWS CloudSearch: ', err)
|
||||
}
|
||||
chunks.length = 0
|
||||
bytes = 0
|
||||
}
|
||||
|
||||
await pipeline(
|
||||
WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'render').select().from('pages').where({
|
||||
isPublished: true,
|
||||
isPrivate: false
|
||||
}).stream(),
|
||||
new stream.Transform({
|
||||
objectMode: true,
|
||||
transform: async (chunk, enc, cb) => processDocument(cb, chunk),
|
||||
flush: async (cb) => processDocument(cb)
|
||||
})
|
||||
)
|
||||
|
||||
WIKI.logger.info(`(SEARCH/AWS) Requesting Index Rebuild...`)
|
||||
await this.client.indexDocuments({
|
||||
DomainName: this.config.domain
|
||||
}).promise()
|
||||
|
||||
WIKI.logger.info(`(SEARCH/AWS) Index rebuilt successfully.`)
|
||||
}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
key: azure
|
||||
title: Azure Search
|
||||
description: AI-Powered cloud search service for web and mobile app development.
|
||||
author: requarks.io
|
||||
logo: https://static.requarks.io/logo/azure.svg
|
||||
website: https://azure.microsoft.com/services/search/
|
||||
isAvailable: true
|
||||
props:
|
||||
serviceName:
|
||||
type: String
|
||||
title: Service Name
|
||||
hint: The name of the Azure Search Service. Found under Properties.
|
||||
order: 1
|
||||
adminKey:
|
||||
type: String
|
||||
title: Admin API Key
|
||||
hint: Either the primary or secondary admin key. Found under Keys.
|
||||
order: 2
|
||||
indexName:
|
||||
type: String
|
||||
title: Index Name
|
||||
hint: 'Name to use when creating the index. (default: wiki)'
|
||||
default: wiki
|
||||
order: 3
|
||||
@ -1,235 +0,0 @@
|
||||
const _ = require('lodash')
|
||||
const { SearchService, QueryType } = require('azure-search-client')
|
||||
const request = require('request-promise')
|
||||
const stream = require('stream')
|
||||
const Promise = require('bluebird')
|
||||
const pipeline = Promise.promisify(stream.pipeline)
|
||||
|
||||
/* global WIKI */
|
||||
|
||||
module.exports = {
|
||||
async activate() {
|
||||
// not used
|
||||
},
|
||||
async deactivate() {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* INIT
|
||||
*/
|
||||
async init() {
|
||||
WIKI.logger.info(`(SEARCH/AZURE) Initializing...`)
|
||||
this.client = new SearchService(this.config.serviceName, this.config.adminKey)
|
||||
|
||||
// -> Create Search Index
|
||||
const indexes = await this.client.indexes.list()
|
||||
if (!_.find(_.get(indexes, 'result.value', []), ['name', this.config.indexName])) {
|
||||
WIKI.logger.info(`(SEARCH/AZURE) Creating index...`)
|
||||
await this.client.indexes.create({
|
||||
name: this.config.indexName,
|
||||
fields: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'Edm.String',
|
||||
key: true,
|
||||
searchable: false
|
||||
},
|
||||
{
|
||||
name: 'locale',
|
||||
type: 'Edm.String',
|
||||
searchable: false
|
||||
},
|
||||
{
|
||||
name: 'path',
|
||||
type: 'Edm.String',
|
||||
searchable: false
|
||||
},
|
||||
{
|
||||
name: 'title',
|
||||
type: 'Edm.String',
|
||||
searchable: true
|
||||
},
|
||||
{
|
||||
name: 'description',
|
||||
type: 'Edm.String',
|
||||
searchable: true
|
||||
},
|
||||
{
|
||||
name: 'content',
|
||||
type: 'Edm.String',
|
||||
searchable: true
|
||||
}
|
||||
],
|
||||
scoringProfiles: [
|
||||
{
|
||||
name: 'fieldWeights',
|
||||
text: {
|
||||
weights: {
|
||||
title: 4,
|
||||
description: 3,
|
||||
content: 1
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
suggesters: [
|
||||
{
|
||||
name: 'suggestions',
|
||||
searchMode: 'analyzingInfixMatching',
|
||||
sourceFields: ['title', 'description', 'content']
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
WIKI.logger.info(`(SEARCH/AZURE) Initialization completed.`)
|
||||
},
|
||||
/**
|
||||
* QUERY
|
||||
*
|
||||
* @param {String} q Query
|
||||
* @param {Object} opts Additional options
|
||||
*/
|
||||
async query(q, opts) {
|
||||
try {
|
||||
let suggestions = []
|
||||
const results = await this.client.indexes.use(this.config.indexName).search({
|
||||
count: true,
|
||||
scoringProfile: 'fieldWeights',
|
||||
search: q,
|
||||
select: 'id, locale, path, title, description',
|
||||
queryType: QueryType.simple,
|
||||
top: 50
|
||||
})
|
||||
if (results.result.value.length < 5) {
|
||||
// Using plain request, not yet available in library...
|
||||
try {
|
||||
const suggestResults = await request({
|
||||
uri: `https://${this.config.serviceName}.search.windows.net/indexes/${this.config.indexName}/docs/autocomplete`,
|
||||
method: 'post',
|
||||
qs: {
|
||||
'api-version': '2017-11-11-Preview'
|
||||
},
|
||||
headers: {
|
||||
'api-key': this.config.adminKey,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
json: true,
|
||||
body: {
|
||||
autocompleteMode: 'oneTermWithContext',
|
||||
search: q,
|
||||
suggesterName: 'suggestions'
|
||||
}
|
||||
})
|
||||
suggestions = suggestResults.value.map(s => s.queryPlusText)
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('Search Engine suggestion failure: ', err)
|
||||
}
|
||||
}
|
||||
return {
|
||||
results: results.result.value,
|
||||
suggestions,
|
||||
totalHits: results.result['@odata.count']
|
||||
}
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('Search Engine Error:')
|
||||
WIKI.logger.warn(err)
|
||||
}
|
||||
},
|
||||
/**
|
||||
* CREATE
|
||||
*
|
||||
* @param {Object} page Page to create
|
||||
*/
|
||||
async created(page) {
|
||||
await this.client.indexes.use(this.config.indexName).index([
|
||||
{
|
||||
id: page.hash,
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
}
|
||||
])
|
||||
},
|
||||
/**
|
||||
* UPDATE
|
||||
*
|
||||
* @param {Object} page Page to update
|
||||
*/
|
||||
async updated(page) {
|
||||
await this.client.indexes.use(this.config.indexName).index([
|
||||
{
|
||||
id: page.hash,
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
}
|
||||
])
|
||||
},
|
||||
/**
|
||||
* DELETE
|
||||
*
|
||||
* @param {Object} page Page to delete
|
||||
*/
|
||||
async deleted(page) {
|
||||
await this.client.indexes.use(this.config.indexName).index([
|
||||
{
|
||||
'@search.action': 'delete',
|
||||
id: page.hash
|
||||
}
|
||||
])
|
||||
},
|
||||
/**
|
||||
* RENAME
|
||||
*
|
||||
* @param {Object} page Page to rename
|
||||
*/
|
||||
async renamed(page) {
|
||||
await this.client.indexes.use(this.config.indexName).index([
|
||||
{
|
||||
'@search.action': 'delete',
|
||||
id: page.hash
|
||||
}
|
||||
])
|
||||
await this.client.indexes.use(this.config.indexName).index([
|
||||
{
|
||||
id: page.destinationHash,
|
||||
locale: page.destinationLocaleCode,
|
||||
path: page.destinationPath,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent
|
||||
}
|
||||
])
|
||||
},
|
||||
/**
|
||||
* REBUILD INDEX
|
||||
*/
|
||||
async rebuild() {
|
||||
WIKI.logger.info(`(SEARCH/AZURE) Rebuilding Index...`)
|
||||
await pipeline(
|
||||
WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'render').select().from('pages').where({
|
||||
isPublished: true,
|
||||
isPrivate: false
|
||||
}).stream(),
|
||||
new stream.Transform({
|
||||
objectMode: true,
|
||||
transform: (chunk, enc, cb) => {
|
||||
cb(null, {
|
||||
id: chunk.id,
|
||||
path: chunk.path,
|
||||
locale: chunk.locale,
|
||||
title: chunk.title,
|
||||
description: chunk.description,
|
||||
content: WIKI.models.pages.cleanHTML(chunk.render)
|
||||
})
|
||||
}
|
||||
}),
|
||||
this.client.indexes.use(this.config.indexName).createIndexingStream()
|
||||
)
|
||||
WIKI.logger.info(`(SEARCH/AZURE) Index rebuilt successfully.`)
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
key: db
|
||||
title: Database - Basic
|
||||
description: Default basic database-based search engine.
|
||||
author: requarks.io
|
||||
logo: https://static.requarks.io/logo/database.svg
|
||||
website: https://www.requarks.io/
|
||||
isAvailable: true
|
||||
props: {}
|
||||
@ -1,94 +0,0 @@
|
||||
/* global WIKI */
|
||||
|
||||
module.exports = {
|
||||
activate() {
|
||||
// not used
|
||||
},
|
||||
deactivate() {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* INIT
|
||||
*/
|
||||
init() {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* QUERY
|
||||
*
|
||||
* @param {String} q Query
|
||||
* @param {Object} opts Additional options
|
||||
*/
|
||||
async query(q, opts) {
|
||||
const results = await WIKI.models.pages.query()
|
||||
.column('pages.id', 'title', 'description', 'path', 'localeCode as locale')
|
||||
.withGraphJoined('tags') // Adding page tags since they can be used to check resource access permissions
|
||||
.modifyGraph('tags', builder => {
|
||||
builder.select('tag')
|
||||
})
|
||||
.where(builder => {
|
||||
builder.where('isPublished', true)
|
||||
if (opts.locale) {
|
||||
builder.andWhere('localeCode', opts.locale)
|
||||
}
|
||||
if (opts.path) {
|
||||
builder.andWhere('path', 'like', `${opts.path}%`)
|
||||
}
|
||||
builder.andWhere(builderSub => {
|
||||
if (WIKI.config.db.type === 'postgres') {
|
||||
builderSub.where('title', 'ILIKE', `%${q}%`)
|
||||
builderSub.orWhere('description', 'ILIKE', `%${q}%`)
|
||||
builderSub.orWhere('path', 'ILIKE', `%${q.toLowerCase()}%`)
|
||||
} else {
|
||||
builderSub.where('title', 'LIKE', `%${q}%`)
|
||||
builderSub.orWhere('description', 'LIKE', `%${q}%`)
|
||||
builderSub.orWhere('path', 'LIKE', `%${q.toLowerCase()}%`)
|
||||
}
|
||||
})
|
||||
})
|
||||
.limit(WIKI.config.search.maxHits)
|
||||
return {
|
||||
results,
|
||||
suggestions: [],
|
||||
totalHits: results.length
|
||||
}
|
||||
},
|
||||
/**
|
||||
* CREATE
|
||||
*
|
||||
* @param {Object} page Page to create
|
||||
*/
|
||||
async created(page) {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* UPDATE
|
||||
*
|
||||
* @param {Object} page Page to update
|
||||
*/
|
||||
async updated(page) {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* DELETE
|
||||
*
|
||||
* @param {Object} page Page to delete
|
||||
*/
|
||||
async deleted(page) {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* RENAME
|
||||
*
|
||||
* @param {Object} page Page to rename
|
||||
*/
|
||||
async renamed(page) {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* REBUILD INDEX
|
||||
*/
|
||||
async rebuild() {
|
||||
// not used
|
||||
}
|
||||
}
|
||||
@ -1,47 +0,0 @@
|
||||
key: elasticsearch
|
||||
title: Elasticsearch
|
||||
description: Elasticsearch is a distributed, RESTful search and analytics engine capable of solving a growing number of use cases.
|
||||
author: requarks.io
|
||||
logo: https://static.requarks.io/logo/elasticsearch.svg
|
||||
website: https://www.elastic.co/products/elasticsearch
|
||||
isAvailable: true
|
||||
props:
|
||||
apiVersion:
|
||||
type: String
|
||||
title: Elasticsearch Version
|
||||
hint: Should match the version of the Elasticsearch nodes you are connecting to
|
||||
order: 1
|
||||
enum:
|
||||
- '7.x'
|
||||
- '6.x'
|
||||
default: '6.x'
|
||||
hosts:
|
||||
type: String
|
||||
title: Host(s)
|
||||
hint: Comma-separated list of Elasticsearch hosts to connect to, including the port, username and password if necessary. (e.g. http://localhost:9200, https://user:pass@es1.example.com:9200)
|
||||
order: 2
|
||||
indexName:
|
||||
type: String
|
||||
title: Index Name
|
||||
hint: The index name to use during creation
|
||||
default: wiki
|
||||
order: 3
|
||||
analyzer:
|
||||
type: String
|
||||
title: Analyzer
|
||||
hint: 'The token analyzer in elasticsearch'
|
||||
default: simple
|
||||
order: 4
|
||||
sniffOnStart:
|
||||
type: Boolean
|
||||
title: Sniff on start
|
||||
hint: 'Should Wiki.js attempt to detect the rest of the cluster on first connect? (Default: off)'
|
||||
default: false
|
||||
order: 5
|
||||
sniffInterval:
|
||||
type: Number
|
||||
title: Sniff Interval
|
||||
hint: '0 = disabled, Interval in seconds to check for updated list of nodes in cluster. (Default: 0)'
|
||||
default: 0
|
||||
order: 6
|
||||
|
||||
@ -1,353 +0,0 @@
|
||||
const _ = require('lodash')
|
||||
const stream = require('stream')
|
||||
const Promise = require('bluebird')
|
||||
const pipeline = Promise.promisify(stream.pipeline)
|
||||
|
||||
/* global WIKI */
|
||||
|
||||
module.exports = {
|
||||
async activate() {
|
||||
// not used
|
||||
},
|
||||
async deactivate() {
|
||||
// not used
|
||||
},
|
||||
/**
|
||||
* INIT
|
||||
*/
|
||||
async init() {
|
||||
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Initializing...`)
|
||||
switch (this.config.apiVersion) {
|
||||
case '7.x':
|
||||
const { Client: Client7 } = require('elasticsearch7')
|
||||
this.client = new Client7({
|
||||
nodes: this.config.hosts.split(',').map(_.trim),
|
||||
sniffOnStart: this.config.sniffOnStart,
|
||||
sniffInterval: (this.config.sniffInterval > 0) ? this.config.sniffInterval : false,
|
||||
name: 'wiki-js'
|
||||
})
|
||||
break
|
||||
case '6.x':
|
||||
const { Client: Client6 } = require('elasticsearch6')
|
||||
this.client = new Client6({
|
||||
nodes: this.config.hosts.split(',').map(_.trim),
|
||||
sniffOnStart: this.config.sniffOnStart,
|
||||
sniffInterval: (this.config.sniffInterval > 0) ? this.config.sniffInterval : false,
|
||||
name: 'wiki-js'
|
||||
})
|
||||
break
|
||||
default:
|
||||
throw new Error('Unsupported version of elasticsearch! Update your settings in the Administration Area.')
|
||||
}
|
||||
|
||||
// -> Create Search Index
|
||||
await this.createIndex()
|
||||
|
||||
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Initialization completed.`)
|
||||
},
|
||||
/**
|
||||
* Create Index
|
||||
*/
|
||||
async createIndex() {
|
||||
try {
|
||||
const indexExists = await this.client.indices.exists({ index: this.config.indexName })
|
||||
if (!indexExists.body) {
|
||||
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Creating index...`)
|
||||
try {
|
||||
const idxBody = {
|
||||
properties: {
|
||||
suggest: { type: 'completion' },
|
||||
title: { type: 'text', boost: 10.0 },
|
||||
description: { type: 'text', boost: 3.0 },
|
||||
content: { type: 'text', boost: 1.0 },
|
||||
locale: { type: 'keyword' },
|
||||
path: { type: 'text' },
|
||||
tags: { type: 'text', boost: 8.0 }
|
||||
}
|
||||
}
|
||||
await this.client.indices.create({
|
||||
index: this.config.indexName,
|
||||
body: {
|
||||
mappings: (this.config.apiVersion === '6.x') ? {
|
||||
_doc: idxBody
|
||||
} : idxBody,
|
||||
settings: {
|
||||
analysis: {
|
||||
analyzer: {
|
||||
default: {
|
||||
type: this.config.analyzer
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
WIKI.logger.error(`(SEARCH/ELASTICSEARCH) Create Index Error: `, _.get(err, 'meta.body.error', err))
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
WIKI.logger.error(`(SEARCH/ELASTICSEARCH) Index Check Error: `, _.get(err, 'meta.body.error', err))
|
||||
}
|
||||
},
|
||||
/**
|
||||
* QUERY
|
||||
*
|
||||
* @param {String} q Query
|
||||
* @param {Object} opts Additional options
|
||||
*/
|
||||
async query(q, opts) {
|
||||
try {
|
||||
const results = await this.client.search({
|
||||
index: this.config.indexName,
|
||||
body: {
|
||||
query: {
|
||||
simple_query_string: {
|
||||
query: `*${q}*`,
|
||||
fields: ['title^20', 'description^3', 'tags^8', 'content^1'],
|
||||
default_operator: 'and',
|
||||
analyze_wildcard: true
|
||||
}
|
||||
},
|
||||
from: 0,
|
||||
size: 50,
|
||||
_source: ['title', 'description', 'path', 'locale'],
|
||||
suggest: {
|
||||
suggestions: {
|
||||
text: q,
|
||||
completion: {
|
||||
field: 'suggest',
|
||||
size: 5,
|
||||
skip_duplicates: true,
|
||||
fuzzy: true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
return {
|
||||
results: _.get(results, 'body.hits.hits', []).map(r => ({
|
||||
id: r._id,
|
||||
locale: r._source.locale,
|
||||
path: r._source.path,
|
||||
title: r._source.title,
|
||||
description: r._source.description
|
||||
})),
|
||||
suggestions: _.reject(_.get(results, 'suggest.suggestions', []).map(s => _.get(s, 'options[0].text', false)), s => !s),
|
||||
totalHits: _.get(results, 'body.hits.total.value', _.get(results, 'body.hits.total', 0))
|
||||
}
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('Search Engine Error: ', _.get(err, 'meta.body.error', err))
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Build tags field
|
||||
* @param id
|
||||
* @returns {Promise<*|*[]>}
|
||||
*/
|
||||
async buildTags(id) {
|
||||
const tags = await WIKI.models.pages.query().findById(id).select('*').withGraphJoined('tags')
|
||||
return (tags.tags && tags.tags.length > 0) ? tags.tags.map(function (tag) {
|
||||
return tag.title
|
||||
}) : []
|
||||
},
|
||||
/**
|
||||
* Build suggest field
|
||||
*/
|
||||
buildSuggest(page) {
|
||||
return _.reject(_.uniq(_.concat(
|
||||
page.title.split(' ').map(s => ({
|
||||
input: s,
|
||||
weight: 10
|
||||
})),
|
||||
page.description.split(' ').map(s => ({
|
||||
input: s,
|
||||
weight: 3
|
||||
})),
|
||||
page.safeContent.split(' ').map(s => ({
|
||||
input: s,
|
||||
weight: 1
|
||||
}))
|
||||
)), ['input', ''])
|
||||
},
|
||||
/**
|
||||
* CREATE
|
||||
*
|
||||
* @param {Object} page Page to create
|
||||
*/
|
||||
async created(page) {
|
||||
await this.client.index({
|
||||
index: this.config.indexName,
|
||||
type: '_doc',
|
||||
id: page.hash,
|
||||
body: {
|
||||
suggest: this.buildSuggest(page),
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent,
|
||||
tags: await this.buildTags(page.id)
|
||||
},
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* UPDATE
|
||||
*
|
||||
* @param {Object} page Page to update
|
||||
*/
|
||||
async updated(page) {
|
||||
await this.client.index({
|
||||
index: this.config.indexName,
|
||||
type: '_doc',
|
||||
id: page.hash,
|
||||
body: {
|
||||
suggest: this.buildSuggest(page),
|
||||
locale: page.localeCode,
|
||||
path: page.path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent,
|
||||
tags: await this.buildTags(page.id)
|
||||
},
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* DELETE
|
||||
*
|
||||
* @param {Object} page Page to delete
|
||||
*/
|
||||
async deleted(page) {
|
||||
await this.client.delete({
|
||||
index: this.config.indexName,
|
||||
type: '_doc',
|
||||
id: page.hash,
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* RENAME
|
||||
*
|
||||
* @param {Object} page Page to rename
|
||||
*/
|
||||
async renamed(page) {
|
||||
await this.client.delete({
|
||||
index: this.config.indexName,
|
||||
type: '_doc',
|
||||
id: page.hash,
|
||||
refresh: true
|
||||
})
|
||||
await this.client.index({
|
||||
index: this.config.indexName,
|
||||
type: '_doc',
|
||||
id: page.destinationHash,
|
||||
body: {
|
||||
suggest: this.buildSuggest(page),
|
||||
locale: page.destinationLocaleCode,
|
||||
path: page.destinationPath,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
content: page.safeContent,
|
||||
tags: await this.buildTags(page.id)
|
||||
},
|
||||
refresh: true
|
||||
})
|
||||
},
|
||||
/**
|
||||
* REBUILD INDEX
|
||||
*/
|
||||
async rebuild() {
|
||||
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Rebuilding Index...`)
|
||||
await this.client.indices.delete({ index: this.config.indexName })
|
||||
await this.createIndex()
|
||||
|
||||
const MAX_INDEXING_BYTES = 10 * Math.pow(2, 20) - Buffer.from('[').byteLength - Buffer.from(']').byteLength // 10 MB
|
||||
const MAX_INDEXING_COUNT = 1000
|
||||
const COMMA_BYTES = Buffer.from(',').byteLength
|
||||
|
||||
let chunks = []
|
||||
let bytes = 0
|
||||
|
||||
const processDocument = async (cb, doc) => {
|
||||
try {
|
||||
if (doc) {
|
||||
const docBytes = Buffer.from(JSON.stringify(doc)).byteLength
|
||||
|
||||
doc['tags'] = await this.buildTags(doc.realId)
|
||||
// -> Current batch exceeds size limit, flush
|
||||
if (docBytes + COMMA_BYTES + bytes >= MAX_INDEXING_BYTES) {
|
||||
await flushBuffer()
|
||||
}
|
||||
|
||||
if (chunks.length > 0) {
|
||||
bytes += COMMA_BYTES
|
||||
}
|
||||
bytes += docBytes
|
||||
chunks.push(doc)
|
||||
|
||||
// -> Current batch exceeds count limit, flush
|
||||
if (chunks.length >= MAX_INDEXING_COUNT) {
|
||||
await flushBuffer()
|
||||
}
|
||||
} else {
|
||||
// -> End of stream, flush
|
||||
await flushBuffer()
|
||||
}
|
||||
cb()
|
||||
} catch (err) {
|
||||
cb(err)
|
||||
}
|
||||
}
|
||||
|
||||
const flushBuffer = async () => {
|
||||
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Sending batch of ${chunks.length}...`)
|
||||
try {
|
||||
await this.client.bulk({
|
||||
index: this.config.indexName,
|
||||
body: _.reduce(chunks, (result, doc) => {
|
||||
result.push({
|
||||
index: {
|
||||
_index: this.config.indexName,
|
||||
_type: '_doc',
|
||||
_id: doc.id
|
||||
}
|
||||
})
|
||||
doc.safeContent = WIKI.models.pages.cleanHTML(doc.render)
|
||||
result.push({
|
||||
suggest: this.buildSuggest(doc),
|
||||
tags: doc.tags,
|
||||
locale: doc.locale,
|
||||
path: doc.path,
|
||||
title: doc.title,
|
||||
description: doc.description,
|
||||
content: doc.safeContent
|
||||
})
|
||||
return result
|
||||
}, []),
|
||||
refresh: true
|
||||
})
|
||||
} catch (err) {
|
||||
WIKI.logger.warn('(SEARCH/ELASTICSEARCH) Failed to send batch to elasticsearch: ', err)
|
||||
}
|
||||
chunks.length = 0
|
||||
bytes = 0
|
||||
}
|
||||
|
||||
// Added real id in order to fetch page tags from the query
|
||||
await pipeline(
|
||||
WIKI.models.knex.column({ id: 'hash' }, 'path', { locale: 'localeCode' }, 'title', 'description', 'render', { realId: 'id' }).select().from('pages').where({
|
||||
isPublished: true,
|
||||
isPrivate: false
|
||||
}).stream(),
|
||||
new stream.Transform({
|
||||
objectMode: true,
|
||||
transform: async (chunk, enc, cb) => processDocument(cb, chunk),
|
||||
flush: async (cb) => processDocument(cb)
|
||||
})
|
||||
)
|
||||
WIKI.logger.info(`(SEARCH/ELASTICSEARCH) Index rebuilt successfully.`)
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
key: manticore
|
||||
title: Manticore Search
|
||||
description: High performance full-text search engine with SQL and JSON support.
|
||||
author: requarks.io
|
||||
logo: https://static.requarks.io/logo/manticore.svg
|
||||
website: https://manticoresearch.com/
|
||||
isAvailable: false
|
||||
props: {}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue