diff --git a/server/app/data.yml b/server/app/data.yml index 17cd99f1..aa6043c2 100644 --- a/server/app/data.yml +++ b/server/app/data.yml @@ -85,9 +85,36 @@ defaults: maxAge: 600 methods: 'GET,POST' origin: true - search: - maxHits: 100 maintainerEmail: security@requarks.io +tsDictMappings: + ar: arabic + hy: armenian + eu: basque + ca: catalan + da: danish + nl: dutch + en: english + fi: finnish + fr: french + de: german + el: greek + hi: hindi + hu: hungarian + id: indonesian + ga: irish + it: italian + lt: lithuanian + ne: nepali + no: norwegian + pt: portuguese + ro: romanian + ru: russian + sr: serbian + es: spanish + sv: swedish + ta: tamil + tr: turkish + yi: yiddish editors: asciidoc: contentType: html diff --git a/server/db/migrations/3.0.0.mjs b/server/db/migrations/3.0.0.mjs index 5e677269..f309a3f7 100644 --- a/server/db/migrations/3.0.0.mjs +++ b/server/db/migrations/3.0.0.mjs @@ -228,7 +228,9 @@ export async function up (knex) { table.jsonb('relations').notNullable().defaultTo('[]') table.text('content') table.text('render') + table.text('searchContent') table.specificType('ts', 'tsvector').index('ts_idx', { indexType: 'GIN' }) + table.specificType('tags', 'int[]').index('tags_idx', { indexType: 'GIN' }) table.jsonb('toc') table.string('editor').notNullable() table.string('contentType').notNullable() @@ -277,7 +279,6 @@ export async function up (knex) { .createTable('tags', table => { table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()')) table.string('tag').notNullable() - table.jsonb('display').notNullable().defaultTo('{}') table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now()) table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now()) }) @@ -334,12 +335,6 @@ export async function up (knex) { // ===================================== // RELATION TABLES // ===================================== - // PAGE TAGS --------------------------- - .createTable('pageTags', table => { - table.increments('id').primary() - table.uuid('pageId').references('id').inTable('pages').onDelete('CASCADE') - table.uuid('tagId').references('id').inTable('tags').onDelete('CASCADE') - }) // USER GROUPS ------------------------- .createTable('userGroups', table => { table.increments('id').primary() @@ -493,7 +488,7 @@ export async function up (knex) { key: 'search', value: { termHighlighting: true, - dictOverrides: [] + dictOverrides: {} } }, { diff --git a/server/graph/resolvers/page.mjs b/server/graph/resolvers/page.mjs index 31171edb..c9f1134e 100644 --- a/server/graph/resolvers/page.mjs +++ b/server/graph/resolvers/page.mjs @@ -43,24 +43,61 @@ export default { * SEARCH PAGES */ async searchPages (obj, args, context) { - if (WIKI.data.searchEngine) { - const resp = await WIKI.data.searchEngine.query(args.query, args) - return { - ...resp, - results: _.filter(resp.results, r => { - return WIKI.auth.checkAccess(context.req.user, ['read:pages'], { - path: r.path, - locale: r.locale, - tags: r.tags // Tags are needed since access permissions can be limited by page tags too - }) + if (!args.siteId) { + throw new Error('Missing Site ID') + } + if (!args.query?.trim()) { + throw new Error('Missing Query') + } + if (args.offset && args.offset < 0) { + throw new Error('Invalid offset value.') + } + if (args.limit && (args.limit < 1 || args.limit > 100)) { + throw new Error('Limit must be between 1 and 100.') + } + try { + const dictName = 'english' + const results = await WIKI.db.knex + .select( + 'id', + 'path', + 'localeCode AS locale', + 'title', + 'description', + 'icon', + 'updatedAt', + WIKI.db.knex.raw('ts_rank_cd(ts, query) AS relevancy'), + WIKI.db.knex.raw(`ts_headline(?, "searchContent", query, 'MaxWords=5, MinWords=3, MaxFragments=5') AS highlight`, [dictName]), + WIKI.db.knex.raw('count(*) OVER() AS total') + ) + .fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query]) + .where('siteId', args.siteId) + .where(builder => { + if (args.path) { + builder.where('path', 'ILIKE', `${path}%`) + } + if (args.locale?.length > 0) { + builder.whereIn('localeCode', args.locale) + } + if (args.editor) { + builder.where('editor', args.editor) + } + if (args.publishState) { + builder.where('publishState', args.publishState) + } }) - } - } else { + .whereRaw('query @@ ts') + .orderBy(args.orderBy || 'relevancy', args.orderByDirection || 'desc') + .offset(args.offset || 0) + .limit(args.limit || 25) + return { - results: [], - suggestions: [], - totalHits: 0 + results, + totalHits: results?.length > 0 ? results[0].total : 0 } + } catch (err) { + WIKI.logger.warn(`Search Query Error: ${err.message}`) + throw err } }, /** @@ -645,9 +682,9 @@ export default { password (obj) { return obj.password ? '********' : '' }, - async tags (obj) { - return WIKI.db.pages.relatedQuery('tags').for(obj.id) - }, + // async tags (obj) { + // return WIKI.db.pages.relatedQuery('tags').for(obj.id) + // }, tocDepth (obj) { return { min: obj.extra?.tocDepth?.min ?? 1, diff --git a/server/graph/resolvers/system.mjs b/server/graph/resolvers/system.mjs index 14ecf884..d193d673 100644 --- a/server/graph/resolvers/system.mjs +++ b/server/graph/resolvers/system.mjs @@ -78,7 +78,10 @@ export default { ]) }, systemSearch () { - return WIKI.config.search + return { + ...WIKI.config.search, + dictOverrides: JSON.stringify(WIKI.config.search.dictOverrides, null, 2) + } } }, Mutation: { @@ -183,7 +186,11 @@ export default { } }, async updateSystemSearch (obj, args, context) { - WIKI.config.search = _.defaultsDeep(_.omit(args, ['__typename']), WIKI.config.search) + WIKI.config.search = { + ...WIKI.config.search, + termHighlighting: args.termHighlighting ?? WIKI.config.search.termHighlighting, + dictOverrides: args.dictOverrides ? JSON.parse(args.dictOverrides) : WIKI.config.search.dictOverrides + } // TODO: broadcast config update await WIKI.configSvc.saveToDb(['search']) return { diff --git a/server/graph/schemas/page.graphql b/server/graph/schemas/page.graphql index ed083b86..7a733a4d 100644 --- a/server/graph/schemas/page.graphql +++ b/server/graph/schemas/page.graphql @@ -15,16 +15,53 @@ extend type Query { ): PageVersion searchPages( + """ + Site ID to search in (required) + """ siteId: UUID! + """ + Search Query (required) + """ query: String! + """ + The locale to perform the query as. Affects how the query is parsed by the search engine. + """ + queryLocale: String + """ + Only match pages that starts with the provided path. + """ path: String + """ + Only match pages having one of the provided locales. + """ locale: [String] + """ + Only match pages having one of the provided tags. + """ tags: [String] + """ + Only match pages using the provided editor. + """ editor: String + """ + Only match pages is the provided state. + """ publishState: PagePublishState + """ + Result ordering. Defaults to relevancy. + """ orderBy: PageSearchSort + """ + Result ordering direction. Defaults to descending. + """ orderByDirection: OrderByDirection + """ + Result offset. Defaults to 0. + """ offset: Int + """ + Results amount to return. Defaults to 25. Maximum 100. + """ limit: Int ): PageSearchResponse @@ -264,19 +301,20 @@ type PageHistoryResult { type PageSearchResponse { results: [PageSearchResult] - suggestions: [String] totalHits: Int } type PageSearchResult { - id: UUID - title: String description: String + highlight: String icon: String + id: UUID + locale: String path: String + relevancy: Float tags: [String] + title: String updatedAt: Date - locale: String } type PageListItem { @@ -392,7 +430,7 @@ input PageTocDepthInput { enum PageSearchSort { relevancy title - updated + updatedAt } enum PageOrderBy { diff --git a/server/helpers/common.mjs b/server/helpers/common.mjs index 275747da..f8d4de83 100644 --- a/server/helpers/common.mjs +++ b/server/helpers/common.mjs @@ -120,3 +120,12 @@ export function parseModuleProps (props) { return result }, {}) } + +export function getDictNameFromLocale (locale) { + const localeCode = locale.length > 2 ? locale.substring(0, 2) : locale + if (localeCode in WIKI.config.search.dictOverrides) { + return WIKI.config.search.dictOverrides[localeCode] + } else { + return WIKI.data.tsDictMappings[localeCode] ?? 'simple' + } +} diff --git a/server/models/pages.mjs b/server/models/pages.mjs index da5746ce..1aff0f17 100644 --- a/server/models/pages.mjs +++ b/server/models/pages.mjs @@ -1,6 +1,7 @@ import { Model } from 'objection' import { find, get, has, initial, isEmpty, isString, last, pick } from 'lodash-es' import { Type as JSBinType } from 'js-binary' +import { getDictNameFromLocale } from '../helpers/common.mjs' import { generateHash, getFileExtension, injectPageMetadata } from '../helpers/page.mjs' import path from 'node:path' import fse from 'fs-extra' @@ -27,9 +28,6 @@ const frontmatterRegex = { markdown: /^(-{3}(?:\n|\r)([\w\W]+?)(?:\n|\r)-{3})?(?:\n|\r)*([\w\W]*)*/ } -const punctuationRegex = /[!,:;/\\_+\-=()&#@<>$~%^*[\]{}"'|]+|(\.\s)|(\s\.)/ig -// const htmlEntitiesRegex = /(&#[0-9]{3};)|(&#x[a-zA-Z0-9]{2};)/ig - /** * Pages model */ @@ -66,18 +64,18 @@ export class Page extends Model { static get relationMappings() { return { - tags: { - relation: Model.ManyToManyRelation, - modelClass: Tag, - join: { - from: 'pages.id', - through: { - from: 'pageTags.pageId', - to: 'pageTags.tagId' - }, - to: 'tags.id' - } - }, + // tags: { + // relation: Model.ManyToManyRelation, + // modelClass: Tag, + // join: { + // from: 'pages.id', + // through: { + // from: 'pageTags.pageId', + // to: 'pageTags.tagId' + // }, + // to: 'tags.id' + // } + // }, links: { relation: Model.HasManyRelation, modelClass: PageLink, @@ -319,6 +317,12 @@ export class Page extends Model { scriptJsUnload = opts.scriptJsUnload || '' } + // -> Get Tags + let tags = [] + if (opts.tags && opts.tags.length > 0) { + tags = await WIKI.db.tags.fetchIds({ tags: opts.tags, siteId: opts.siteId }) + } + // -> Create page const page = await WIKI.db.pages.query().insert({ alias: opts.alias, @@ -348,6 +352,7 @@ export class Page extends Model { publishStartDate: opts.publishStartDate?.toISO(), relations: opts.relations ?? [], siteId: opts.siteId, + tags, title: opts.title, toc: '[]', scripts: JSON.stringify({ @@ -357,11 +362,6 @@ export class Page extends Model { }) }).returning('*') - // -> Save Tags - if (opts.tags && opts.tags.length > 0) { - await WIKI.db.tags.associateTags({ tags: opts.tags, page }) - } - // -> Render page to HTML await WIKI.db.pages.renderPage(page) @@ -387,31 +387,23 @@ export class Page extends Model { siteId: page.siteId }) - return page - // TODO: Handle remaining flow - - // -> Rebuild page tree - await WIKI.db.pages.rebuildTree() + // -> Update search vector + WIKI.db.pages.updatePageSearchVector(page.id) - // -> Add to Search Index - const pageContents = await WIKI.db.pages.query().findById(page.id).select('render') - page.safeContent = WIKI.db.pages.cleanHTML(pageContents.render) - await WIKI.data.searchEngine.created(page) - - // -> Add to Storage - if (!opts.skipStorage) { - await WIKI.db.storage.pageEvent({ - event: 'created', - page - }) - } + // // -> Add to Storage + // if (!opts.skipStorage) { + // await WIKI.db.storage.pageEvent({ + // event: 'created', + // page + // }) + // } - // -> Reconnect Links - await WIKI.db.pages.reconnectLinks({ - locale: page.localeCode, - path: page.path, - mode: 'create' - }) + // // -> Reconnect Links + // await WIKI.db.pages.reconnectLinks({ + // locale: page.localeCode, + // path: page.path, + // mode: 'create' + // }) // -> Get latest updatedAt page.updatedAt = await WIKI.db.pages.query().findById(page.id).select('updatedAt').then(r => r.updatedAt) @@ -445,6 +437,7 @@ export class Page extends Model { action: 'updated', affectedFields: [] } + let shouldUpdateSearch = false // -> Create version snapshot await WIKI.db.pageHistory.addVersion(ogPage) @@ -453,6 +446,7 @@ export class Page extends Model { if ('title' in opts.patch) { patch.title = opts.patch.title.trim() historyData.affectedFields.push('title') + shouldUpdateSearch = true if (patch.title.length < 1) { throw new Error('ERR_PAGE_TITLE_MISSING') @@ -462,6 +456,7 @@ export class Page extends Model { if ('description' in opts.patch) { patch.description = opts.patch.description.trim() historyData.affectedFields.push('description') + shouldUpdateSearch = true } if ('icon' in opts.patch) { @@ -488,9 +483,10 @@ export class Page extends Model { } } - if ('content' in opts.patch) { + if ('content' in opts.patch && opts.patch.content) { patch.content = opts.patch.content historyData.affectedFields.push('content') + shouldUpdateSearch = true } // -> Publish State @@ -674,10 +670,10 @@ export class Page extends Model { updatedAt: page.updatedAt }) - // // -> Update Search Index - // const pageContents = await WIKI.db.pages.query().findById(page.id).select('render') - // page.safeContent = WIKI.db.pages.cleanHTML(pageContents.render) - // await WIKI.data.searchEngine.updated(page) + // -> Update search vector + if (shouldUpdateSearch) { + WIKI.db.pages.updatePageSearchVector(page.id) + } // -> Update on Storage // if (!opts.skipStorage) { @@ -711,6 +707,24 @@ export class Page extends Model { return page } + /** + * Update a page text search vector value + * + * @param {String} id Page UUID + */ + static async updatePageSearchVector (id) { + const page = await WIKI.db.pages.query().findById(id).select('localeCode', 'render') + const safeContent = WIKI.db.pages.cleanHTML(page.render) + const dictName = getDictNameFromLocale(page.localeCode) + return WIKI.db.knex('pages').where('id', id).update({ + searchContent: safeContent, + ts: WIKI.db.knex.raw(` + setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') || + setweight(to_tsvector('${dictName}', coalesce(description,'')), 'B') || + setweight(to_tsvector('${dictName}', coalesce(?,'')), 'C')`, [safeContent]) + }) + } + /** * Convert an Existing Page * @@ -1214,10 +1228,10 @@ export class Page extends Model { ]) .joinRelated('author') .joinRelated('creator') - .withGraphJoined('tags') - .modifyGraph('tags', builder => { - builder.select('tag') - }) + // .withGraphJoined('tags') + // .modifyGraph('tags', builder => { + // builder.select('tag') + // }) .where(queryModeID ? { 'pages.id': opts } : { @@ -1346,14 +1360,11 @@ export class Page extends Model { * @returns {string} Cleaned Content Text */ static cleanHTML(rawHTML = '') { - let data = striptags(rawHTML || '', [], ' ') + const data = striptags(rawHTML || '', [], ' ') .replace(emojiRegex(), '') - // .replace(htmlEntitiesRegex, '') return he.decode(data) - .replace(punctuationRegex, ' ') .replace(/(\r\n|\n|\r)/gm, ' ') .replace(/\s\s+/g, ' ') - .split(' ').filter(w => w.length > 1).join(' ').toLowerCase() } /** diff --git a/ux/src/pages/AdminSearch.vue b/ux/src/pages/AdminSearch.vue index 850e8b91..af18edd3 100644 --- a/ux/src/pages/AdminSearch.vue +++ b/ux/src/pages/AdminSearch.vue @@ -56,14 +56,13 @@ q-page.admin-flags blueprint-icon.self-start(icon='search') q-item-section q-item-label {{t(`admin.search.dictOverrides`)}} - q-input.q-mt-sm( - type='textarea' - v-model='state.config.dictOverrides' - outlined - :aria-label='t(`admin.search.dictOverrides`)' - :hint='t(`admin.search.dictOverridesHint`)' - input-style='min-height: 200px;' + q-no-ssr(:placeholder='t(`common.loading`)') + util-code-editor.admin-theme-cm.q-my-sm( + v-model='state.config.dictOverrides' + language='json' + :min-height='250' ) + q-item-label(caption) JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. { "en": "english" } .col-12.col-lg-5.gt-md .q-pa-md.text-center @@ -80,6 +79,8 @@ import { useI18n } from 'vue-i18n' import { useSiteStore } from 'src/stores/site' import { useFlagsStore } from 'src/stores/flags' +import UtilCodeEditor from 'src/components/UtilCodeEditor.vue' + // QUASAR const $q = useQuasar() diff --git a/ux/src/pages/Search.vue b/ux/src/pages/Search.vue index 8a3bd969..7981ee61 100644 --- a/ux/src/pages/Search.vue +++ b/ux/src/pages/Search.vue @@ -94,15 +94,19 @@ q-layout(view='hHh Lpr lff') .text-header.flex span {{t('search.results')}} q-space - span.text-caption #[strong {{ state.items }}] results - q-list(separator, padding) - q-item(v-for='item of state.items', clickable) + span.text-caption #[strong {{ state.total }}] results + q-list(separator) + q-item( + v-for='item of state.results' + clickable + :to='`/` + item.path' + ) q-item-section(avatar) - q-avatar(color='primary' text-color='white' rounded icon='las la-file-alt') + q-avatar(color='primary' text-color='white' rounded :icon='item.icon') q-item-section - q-item-label Page ABC def {{ item }} - q-item-label(caption) Lorem ipsum beep boop foo bar - q-item-label(caption) ...Abc def #[span.text-highlight home] efg hig klm... + q-item-label {{ item.title }} + q-item-label(caption) {{ item.description }} + q-item-label.text-highlight(caption, v-html='item.highlight') q-item-section(side) .flex q-chip( @@ -114,8 +118,8 @@ q-layout(view='hHh Lpr lff') size='sm' ) tag {{ tag }} .flex - .text-caption.q-mr-sm.text-grey /beep/boop/hello - .text-caption 2023-01-25 + .text-caption.q-mr-sm.text-grey /{{ item.path }} + .text-caption {{ humanizeDate(item.updatedAt) }} q-inner-loading(:showing='state.loading > 0') main-overlay-dialog @@ -127,6 +131,9 @@ import { useI18n } from 'vue-i18n' import { useMeta, useQuasar } from 'quasar' import { computed, onMounted, reactive, watch } from 'vue' import { useRouter, useRoute } from 'vue-router' +import gql from 'graphql-tag' +import { cloneDeep } from 'lodash-es' +import { DateTime } from 'luxon' import { useFlagsStore } from 'src/stores/flags' import { useSiteStore } from 'src/stores/site' @@ -170,7 +177,8 @@ const state = reactive({ filterLocale: ['en'], filterEditor: '', filterPublishState: '', - items: 25 + results: [], + total: 0 }) const editors = computed(() => { @@ -196,6 +204,7 @@ const publishStates = computed(() => { watch(() => route.query, async (newQueryObj) => { if (newQueryObj.q) { siteStore.search = newQueryObj.q + performSearch() } }, { immediate: true }) @@ -207,10 +216,85 @@ function pageStyle (offset, height) { } } +function humanizeDate (val) { + return DateTime.fromISO(val).toFormat(userStore.preferredDateFormat) +} + +async function performSearch () { + siteStore.searchIsLoading = true + try { + const resp = await APOLLO_CLIENT.query({ + query: gql` + query searchPages ( + $siteId: UUID! + $query: String! + $path: String + $locale: [String] + $tags: [String] + $editor: String + $publishState: PagePublishState + $orderBy: PageSearchSort + $orderByDirection: OrderByDirection + $offset: Int + $limit: Int + ) { + searchPages( + siteId: $siteId + query: $query + path: $path + locale: $locale + tags: $tags + editor: $editor + publishState: $publishState + orderBy: $orderBy + orderByDirection: $orderByDirection + offset: $offset + limit: $limit + ) { + results { + id + path + locale + title + description + icon + updatedAt + relevancy + highlight + } + totalHits + } + } + `, + variables: { + siteId: siteStore.id, + query: siteStore.search + }, + fetchPolicy: 'network-only' + }) + if (!resp?.data?.searchPages) { + throw new Error('Unexpected error') + } + state.results = cloneDeep(resp.data.searchPages.results) + state.total = resp.data.searchPages.totalHits + } catch (err) { + $q.notify({ + type: 'negative', + message: 'Failed to perform search query.', + caption: err.message + }) + } + siteStore.searchIsLoading = false +} + // MOUNTED onMounted(() => { - siteStore.searchIsLoading = false + if (siteStore.search) { + // performSearch() + } else { + siteStore.searchIsLoading = false + } }) @@ -298,9 +382,13 @@ onMounted(() => { } .text-highlight { - background-color: rgba($yellow-7, .5); - padding: 0 3px; - border-radius: 3px; + font-style: italic; + + > b { + background-color: rgba($yellow-7, .5); + padding: 0 3px; + border-radius: 3px; + } } .q-page {