diff --git a/server/db/migrations/3.0.0.mjs b/server/db/migrations/3.0.0.mjs index f309a3f7..9a1d43cd 100644 --- a/server/db/migrations/3.0.0.mjs +++ b/server/db/migrations/3.0.0.mjs @@ -12,6 +12,7 @@ export async function up (knex) { // ===================================== await knex.raw('CREATE EXTENSION IF NOT EXISTS ltree;') await knex.raw('CREATE EXTENSION IF NOT EXISTS pgcrypto;') + await knex.raw('CREATE EXTENSION IF NOT EXISTS pg_trgm;') await knex.schema // ===================================== @@ -235,6 +236,8 @@ export async function up (knex) { table.string('editor').notNullable() table.string('contentType').notNullable() table.boolean('isBrowsable').notNullable().defaultTo(true) + table.boolean('isSearchable').notNullable().defaultTo(true) + table.specificType('isSearchableComputed', `boolean GENERATED ALWAYS AS ("publishState" != 'draft' AND "isSearchable") STORED`).index() table.string('password') table.integer('ratingScore').notNullable().defaultTo(0) table.integer('ratingCount').notNullable().defaultTo(0) @@ -393,6 +396,13 @@ export async function up (knex) { .table('userKeys', table => { table.uuid('userId').notNullable().references('id').inTable('users') }) + // ===================================== + // TS WORD SUGGESTION TABLE + // ===================================== + .createTable('autocomplete', table => { + table.text('word') + }) + .raw(`CREATE INDEX "autocomplete_idx" ON "autocomplete" USING GIN (word gin_trgm_ops)`) // ===================================== // DEFAULT DATA @@ -518,8 +528,8 @@ export async function up (knex) { key: 'update', value: { lastCheckedAt: null, - version: null, - versionDate: null + version: WIKI.version, + versionDate: WIKI.releaseDate } }, { @@ -771,6 +781,11 @@ export async function up (knex) { cron: '5 0 * * *', type: 'system' }, + { + task: 'refreshAutocomplete', + cron: '0 */3 * * *', + type: 'system' + }, { task: 'updateLocales', cron: '0 0 * * *', diff --git a/server/graph/resolvers/page.mjs b/server/graph/resolvers/page.mjs index 0c161d96..174c1758 100644 --- a/server/graph/resolvers/page.mjs +++ b/server/graph/resolvers/page.mjs @@ -77,6 +77,7 @@ export default { .select(searchCols) .fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query]) .where('siteId', args.siteId) + .where('isSearchableComputed', true) .where(builder => { if (args.path) { builder.where('path', 'ILIKE', `${args.path}%`) diff --git a/server/graph/resolvers/search.mjs b/server/graph/resolvers/search.mjs deleted file mode 100644 index 22e9a4d7..00000000 --- a/server/graph/resolvers/search.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { generateError, generateSuccess } from '../../helpers/graph.mjs' - -export default { - Mutation: { - async rebuildSearchIndex (obj, args, context) { - try { - await WIKI.data.searchEngine.rebuild() - return { - responseResult: generateSuccess('Index rebuilt successfully') - } - } catch (err) { - return generateError(err) - } - } - } -} diff --git a/server/graph/resolvers/system.mjs b/server/graph/resolvers/system.mjs index d193d673..49401850 100644 --- a/server/graph/resolvers/system.mjs +++ b/server/graph/resolvers/system.mjs @@ -141,6 +141,19 @@ export default { return generateError(err) } }, + async rebuildSearchIndex (obj, args, context) { + try { + await WIKI.scheduler.addJob({ + task: 'rebuildSearchIndex', + maxRetries: 0 + }) + return { + operation: generateSuccess('Search index rebuild has been scheduled and will start shortly.') + } + } catch (err) { + return generateError(err) + } + }, async retryJob (obj, args, context) { WIKI.logger.info(`Admin requested rescheduling of job ${args.id}...`) try { diff --git a/server/graph/schemas/page.graphql b/server/graph/schemas/page.graphql index 7a733a4d..1befeda7 100644 --- a/server/graph/schemas/page.graphql +++ b/server/graph/schemas/page.graphql @@ -65,6 +65,11 @@ extend type Query { limit: Int ): PageSearchResponse + searchPagesAutocomplete( + siteId: UUID! + query: String! + ): [String] + pages( limit: Int orderBy: PageOrderBy @@ -130,6 +135,7 @@ extend type Mutation { editor: String! icon: String isBrowsable: Boolean + isSearchable: Boolean locale: String! path: String! publishState: PagePublishState! @@ -231,6 +237,7 @@ type Page { icon: String id: UUID isBrowsable: Boolean + isSearchable: Boolean locale: String password: String path: String @@ -393,6 +400,7 @@ input PageUpdateInput { description: String icon: String isBrowsable: Boolean + isSearchable: Boolean locale: String password: String path: String diff --git a/server/graph/schemas/search.graphql b/server/graph/schemas/search.graphql deleted file mode 100644 index cb0a6db3..00000000 --- a/server/graph/schemas/search.graphql +++ /dev/null @@ -1,11 +0,0 @@ -# =============================================== -# SEARCH -# =============================================== - -extend type Mutation { - rebuildSearchIndex: DefaultResponse -} - -# ----------------------------------------------- -# TYPES -# ----------------------------------------------- diff --git a/server/graph/schemas/system.graphql b/server/graph/schemas/system.graphql index 28f0d12c..32564b1b 100644 --- a/server/graph/schemas/system.graphql +++ b/server/graph/schemas/system.graphql @@ -29,6 +29,8 @@ extend type Mutation { key: String! ): DefaultResponse + rebuildSearchIndex: DefaultResponse + retryJob( id: UUID! ): DefaultResponse diff --git a/server/locales/en.json b/server/locales/en.json index 11a2bd5a..ace207c5 100644 --- a/server/locales/en.json +++ b/server/locales/en.json @@ -525,7 +525,7 @@ "admin.scheduler.waitUntil": "Start", "admin.search.configSaveSuccess": "Search engine configuration saved successfully.", "admin.search.dictOverrides": "PostgreSQL Dictionary Mapping Overrides", - "admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. { \"en\": \"english\" }", + "admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. {0}", "admin.search.engineConfig": "Engine Configuration", "admin.search.engineNoConfig": "This engine has no configuration options you can modify.", "admin.search.highlighting": "Enable Term Highlighting", @@ -533,10 +533,12 @@ "admin.search.indexRebuildSuccess": "Index rebuilt successfully.", "admin.search.listRefreshSuccess": "List of search engines has been refreshed.", "admin.search.rebuildIndex": "Rebuild Index", + "admin.search.rebuildInitSuccess": "A search index rebuild has been initiated and will start shortly.", "admin.search.saveSuccess": "Search engine configuration saved successfully", "admin.search.searchEngine": "Search Engine", "admin.search.subtitle": "Configure the search capabilities of your wiki", "admin.search.title": "Search Engine", + "admin.searchRebuildIndex": "Rebuild Index", "admin.security.cors": "CORS (Cross-Origin Resource Sharing)", "admin.security.corsHostnames": "Hostnames Whitelist", "admin.security.corsHostnamesHint": "Enter one hostname per line", @@ -1535,6 +1537,7 @@ "editor.props.draftHint": "Visible to users with write access only.", "editor.props.icon": "Icon", "editor.props.info": "Info", + "editor.props.isSearchable": "Include in Search Results", "editor.props.jsLoad": "Javascript - On Load", "editor.props.jsLoadHint": "Execute javascript once the page is loaded", "editor.props.jsUnload": "Javascript - On Unload", diff --git a/server/models/pages.mjs b/server/models/pages.mjs index 1aff0f17..d298ebe2 100644 --- a/server/models/pages.mjs +++ b/server/models/pages.mjs @@ -344,6 +344,7 @@ export class Page extends Model { hash: generateHash({ path: opts.path, locale: opts.locale }), icon: opts.icon, isBrowsable: opts.isBrowsable ?? true, + isSearchable: opts.isSearchable ?? true, localeCode: opts.locale, ownerId: opts.user.id, path: opts.path, @@ -388,7 +389,7 @@ export class Page extends Model { }) // -> Update search vector - WIKI.db.pages.updatePageSearchVector(page.id) + WIKI.db.pages.updatePageSearchVector({ id: page.id }) // // -> Add to Storage // if (!opts.skipStorage) { @@ -507,14 +508,17 @@ export class Page extends Model { historyData.affectedFields.push('publishEndDate') } - // -> Page Config + // -> Browsable / Searchable Flags if ('isBrowsable' in opts.patch) { - patch.config = { - ...patch.config ?? ogPage.config ?? {}, - isBrowsable: opts.patch.isBrowsable - } + patch.isBrowsable = opts.patch.isBrowsable historyData.affectedFields.push('isBrowsable') } + if ('isSearchable' in opts.patch) { + patch.isSearchable = opts.patch.isSearchable + historyData.affectedFields.push('isSearchable') + } + + // -> Page Config if ('allowComments' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, @@ -644,7 +648,7 @@ export class Page extends Model { // -> Save Tags if (opts.patch.tags) { - await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page }) + // await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page }) } // -> Render page to HTML @@ -672,7 +676,7 @@ export class Page extends Model { // -> Update search vector if (shouldUpdateSearch) { - WIKI.db.pages.updatePageSearchVector(page.id) + WIKI.db.pages.updatePageSearchVector({ id: page.id }) } // -> Update on Storage @@ -710,13 +714,21 @@ export class Page extends Model { /** * Update a page text search vector value * - * @param {String} id Page UUID + * @param {Object} opts - Options + * @param {string} [opts.id] - Page ID to update (fetch from DB) + * @param {Object} [opts.page] - Page object to update (use directly) */ - static async updatePageSearchVector (id) { - const page = await WIKI.db.pages.query().findById(id).select('localeCode', 'render') - const safeContent = WIKI.db.pages.cleanHTML(page.render) + static async updatePageSearchVector ({ id, page }) { + if (!page) { + if (!id) { + throw new Error('Must provide either the page ID or the page object.') + } + page = await WIKI.db.pages.query().findById(id).select('id', 'localeCode', 'render', 'password') + } + // -> Exclude password-protected content from being indexed + const safeContent = page.password ? '' : WIKI.db.pages.cleanHTML(page.render) const dictName = getDictNameFromLocale(page.localeCode) - return WIKI.db.knex('pages').where('id', id).update({ + return WIKI.db.knex('pages').where('id', page.id).update({ searchContent: safeContent, ts: WIKI.db.knex.raw(` setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') || @@ -725,6 +737,19 @@ export class Page extends Model { }) } + /** + * Refresh Autocomplete Index + */ + static async refreshAutocompleteIndex () { + await WIKI.db.knex('autocomplete').truncate() + await WIKI.db.knex.raw(` + INSERT INTO "autocomplete" (word) + SELECT word FROM ts_stat( + 'SELECT to_tsvector(''simple'', "title") || to_tsvector(''simple'', "description") || to_tsvector(''simple'', "searchContent") FROM "pages" WHERE isSearchableComputed IS TRUE' + ) + `) + } + /** * Convert an Existing Page * diff --git a/server/tasks/simple/refresh-autocomplete.mjs b/server/tasks/simple/refresh-autocomplete.mjs new file mode 100644 index 00000000..332550e9 --- /dev/null +++ b/server/tasks/simple/refresh-autocomplete.mjs @@ -0,0 +1,13 @@ +export async function task (payload) { + WIKI.logger.info('Refreshing autocomplete word index...') + + try { + await WIKI.db.pages.refreshAutocompleteIndex() + + WIKI.logger.info('Refreshed autocomplete word index: [ COMPLETED ]') + } catch (err) { + WIKI.logger.error('Refreshing autocomplete word index: [ FAILED ]') + WIKI.logger.error(err.message) + throw err + } +} diff --git a/server/tasks/workers/purge-uploads.mjs b/server/tasks/workers/purge-uploads.mjs index 4fc4fa8f..2541e74d 100644 --- a/server/tasks/workers/purge-uploads.mjs +++ b/server/tasks/workers/purge-uploads.mjs @@ -22,5 +22,6 @@ export async function task ({ payload }) { } catch (err) { WIKI.logger.error('Purging orphaned upload files: [ FAILED ]') WIKI.logger.error(err.message) + throw err } } diff --git a/server/tasks/workers/rebuild-search-index.mjs b/server/tasks/workers/rebuild-search-index.mjs new file mode 100644 index 00000000..b8d8075b --- /dev/null +++ b/server/tasks/workers/rebuild-search-index.mjs @@ -0,0 +1,35 @@ +import { pipeline } from 'node:stream/promises' +import { Transform } from 'node:stream' + +export async function task ({ payload }) { + WIKI.logger.info('Rebuilding search index...') + + try { + await WIKI.ensureDb() + + let idx = 0 + await pipeline( + WIKI.db.knex.select('id', 'title', 'description', 'localeCode', 'render', 'password').from('pages').stream(), + new Transform({ + objectMode: true, + transform: async (page, enc, cb) => { + idx++ + await WIKI.db.pages.updatePageSearchVector({ page }) + if (idx % 50 === 0) { + WIKI.logger.info(`Rebuilding search index... (${idx} processed)`) + } + cb() + } + }) + ) + + WIKI.logger.info('Refreshing autocomplete index...') + await WIKI.db.pages.refreshAutocompleteIndex() + + WIKI.logger.info('Rebuilt search index: [ COMPLETED ]') + } catch (err) { + WIKI.logger.error('Rebuilding search index: [ FAILED ]') + WIKI.logger.error(err.message) + throw err + } +} diff --git a/ux/src/components/PagePropertiesDialog.vue b/ux/src/components/PagePropertiesDialog.vue index b585c239..bd0a6cef 100644 --- a/ux/src/components/PagePropertiesDialog.vue +++ b/ux/src/components/PagePropertiesDialog.vue @@ -265,6 +265,15 @@ q-card.page-properties-dialog checked-icon='las la-check' unchecked-icon='las la-times' ) + div + q-toggle( + v-model='pageStore.isSearchable' + dense + :label='$t(`editor.props.isSearchable`)' + color='primary' + checked-icon='las la-check' + unchecked-icon='las la-times' + ) div q-toggle( v-model='state.requirePassword' diff --git a/ux/src/pages/AdminSearch.vue b/ux/src/pages/AdminSearch.vue index 00592eb8..c8031b9d 100644 --- a/ux/src/pages/AdminSearch.vue +++ b/ux/src/pages/AdminSearch.vue @@ -6,7 +6,16 @@ q-page.admin-flags .col.q-pl-md .text-h5.text-primary.animated.fadeInLeft {{ t('admin.search.title') }} .text-subtitle1.text-grey.animated.fadeInLeft.wait-p2s {{ t('admin.search.subtitle') }} - .col-auto + .col-auto.flex + q-btn.q-mr-sm.acrylic-btn( + flat + icon='mdi-database-refresh' + :label='t(`admin.searchRebuildIndex`)' + color='purple' + @click='rebuild' + :loading='state.rebuildLoading' + ) + q-separator.q-mr-sm(vertical) q-btn.q-mr-sm.acrylic-btn( icon='las la-question-circle' flat @@ -62,7 +71,9 @@ q-page.admin-flags language='json' :min-height='250' ) - q-item-label(caption) {{ t('admin.search.dictOverridesHint') }} + q-item-label(caption) + i18n-t(keypath='admin.search.dictOverridesHint' tag='span') + span { "en": "english" } .col-12.col-lg-5.gt-md .q-pa-md.text-center @@ -104,6 +115,7 @@ useMeta({ const state = reactive({ loading: 0, + rebuildLoading: false, config: { termHighlighting: false, dictOverrides: '' @@ -181,6 +193,41 @@ async function save () { state.loading-- } +async function rebuild () { + state.rebuildLoading = true + try { + const respRaw = await APOLLO_CLIENT.mutate({ + mutation: gql` + mutation rebuildSearchIndex { + rebuildSearchIndex { + operation { + succeeded + slug + message + } + } + } + ` + }) + const resp = respRaw?.data?.rebuildSearchIndex?.operation || {} + if (resp.succeeded) { + $q.notify({ + type: 'positive', + message: t('admin.search.rebuildInitSuccess') + }) + } else { + throw new Error(resp.message) + } + } catch (err) { + $q.notify({ + type: 'negative', + message: 'Failed to initiate a search index rebuild', + caption: err.message + }) + } + state.rebuildLoading = false +} + // MOUNTED onMounted(async () => { diff --git a/ux/src/pages/Search.vue b/ux/src/pages/Search.vue index 7c8f8d00..7983b17e 100644 --- a/ux/src/pages/Search.vue +++ b/ux/src/pages/Search.vue @@ -21,7 +21,7 @@ q-layout(view='hHh Lpr lff') side ) q-icon( - :name='state.params.orderByDirection === `desc` ? `mdi-chevron-double-down` : `mdi-chevron-double-up`' + :name='state.params.orderByDirection === `desc` ? `mdi-transfer-down` : `mdi-transfer-up`' size='sm' color='primary' ) diff --git a/ux/src/stores/page.js b/ux/src/stores/page.js index 0600f6e5..c6528c54 100644 --- a/ux/src/stores/page.js +++ b/ux/src/stores/page.js @@ -19,6 +19,7 @@ const pagePropsFragment = gql` icon id isBrowsable + isSearchable locale password path @@ -122,6 +123,7 @@ const gqlMutations = { $editor: String! $icon: String $isBrowsable: Boolean + $isSearchable: Boolean $locale: String! $path: String! $publishState: PagePublishState! @@ -149,6 +151,7 @@ const gqlMutations = { editor: $editor icon: $icon isBrowsable: $isBrowsable + isSearchable: $isSearchable locale: $locale path: $path publishState: $publishState @@ -195,6 +198,7 @@ export const usePageStore = defineStore('page', { icon: 'las la-file-alt', id: '', isBrowsable: true, + isSearchable: true, locale: 'en', password: '', path: '', @@ -367,6 +371,8 @@ export const usePageStore = defineStore('page', { tags: [], content: content ?? '', render: '', + isBrowsable: true, + isSearchable: true, mode: 'edit' }) }, @@ -420,6 +426,7 @@ export const usePageStore = defineStore('page', { 'description', 'icon', 'isBrowsable', + 'isSearchable', 'locale', 'password', 'path', @@ -491,6 +498,7 @@ export const usePageStore = defineStore('page', { 'description', 'icon', 'isBrowsable', + 'isSearchable', 'locale', 'password', 'path',