feat: rebuild search index + isSearchable flag

server/db/migrations/3.0.0.mjs

@@ -12,6 +12,7 @@ export async function up (knex) {
   // =====================================
   await knex.raw('CREATE EXTENSION IF NOT EXISTS ltree;')
   await knex.raw('CREATE EXTENSION IF NOT EXISTS pgcrypto;')
+  await knex.raw('CREATE EXTENSION IF NOT EXISTS pg_trgm;')
 
   await knex.schema
     // =====================================
@@ -235,6 +236,8 @@ export async function up (knex) {
       table.string('editor').notNullable()
       table.string('contentType').notNullable()
       table.boolean('isBrowsable').notNullable().defaultTo(true)
+      table.boolean('isSearchable').notNullable().defaultTo(true)
+      table.specificType('isSearchableComputed', `boolean GENERATED ALWAYS AS ("publishState" != 'draft' AND "isSearchable") STORED`).index()
       table.string('password')
       table.integer('ratingScore').notNullable().defaultTo(0)
       table.integer('ratingCount').notNullable().defaultTo(0)
@@ -393,6 +396,13 @@ export async function up (knex) {
     .table('userKeys', table => {
       table.uuid('userId').notNullable().references('id').inTable('users')
     })
+    // =====================================
+    // TS WORD SUGGESTION TABLE
+    // =====================================
+    .createTable('autocomplete', table => {
+      table.text('word')
+    })
+    .raw(`CREATE INDEX "autocomplete_idx" ON "autocomplete" USING GIN (word gin_trgm_ops)`)
 
   // =====================================
   // DEFAULT DATA
@@ -518,8 +528,8 @@ export async function up (knex) {
       key: 'update',
       value: {
         lastCheckedAt: null,
-        version: null,
+        version: WIKI.version,
-        versionDate: null
+        versionDate: WIKI.releaseDate
       }
     },
     {
@@ -771,6 +781,11 @@ export async function up (knex) {
       cron: '5 0 * * *',
       type: 'system'
     },
+    {
+      task: 'refreshAutocomplete',
+      cron: '0 */3 * * *',
+      type: 'system'
+    },
     {
       task: 'updateLocales',
       cron: '0 0 * * *',

server/graph/resolvers/page.mjs

@@ -77,6 +77,7 @@ export default {
           .select(searchCols)
           .fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query])
           .where('siteId', args.siteId)
+          .where('isSearchableComputed', true)
           .where(builder => {
             if (args.path) {
               builder.where('path', 'ILIKE', `${args.path}%`)

server/graph/resolvers/search.mjs

@@ -1,16 +0,0 @@
-import { generateError, generateSuccess } from '../../helpers/graph.mjs'
-
-export default {
-  Mutation: {
-    async rebuildSearchIndex (obj, args, context) {
-      try {
-        await WIKI.data.searchEngine.rebuild()
-        return {
-          responseResult: generateSuccess('Index rebuilt successfully')
-        }
-      } catch (err) {
-        return generateError(err)
-      }
-    }
-  }
-}

server/graph/resolvers/system.mjs

@@ -141,6 +141,19 @@ export default {
         return generateError(err)
       }
     },
+    async rebuildSearchIndex (obj, args, context) {
+      try {
+        await WIKI.scheduler.addJob({
+          task: 'rebuildSearchIndex',
+          maxRetries: 0
+        })
+        return {
+          operation: generateSuccess('Search index rebuild has been scheduled and will start shortly.')
+        }
+      } catch (err) {
+        return generateError(err)
+      }
+    },
     async retryJob (obj, args, context) {
       WIKI.logger.info(`Admin requested rescheduling of job ${args.id}...`)
       try {

server/graph/schemas/page.graphql

@@ -65,6 +65,11 @@ extend type Query {
     limit: Int
   ): PageSearchResponse
 
+  searchPagesAutocomplete(
+    siteId: UUID!
+    query: String!
+  ): [String]
+
   pages(
     limit: Int
     orderBy: PageOrderBy
@@ -130,6 +135,7 @@ extend type Mutation {
     editor: String!
     icon: String
     isBrowsable: Boolean
+    isSearchable: Boolean
     locale: String!
     path: String!
     publishState: PagePublishState!
@@ -231,6 +237,7 @@ type Page {
   icon: String
   id: UUID
   isBrowsable: Boolean
+  isSearchable: Boolean
   locale: String
   password: String
   path: String
@@ -393,6 +400,7 @@ input PageUpdateInput {
   description: String
   icon: String
   isBrowsable: Boolean
+  isSearchable: Boolean
   locale: String
   password: String
   path: String

server/graph/schemas/search.graphql

@@ -1,11 +0,0 @@
-# ===============================================
-# SEARCH
-# ===============================================
-
-extend type Mutation {
-  rebuildSearchIndex: DefaultResponse
-}
-
-# -----------------------------------------------
-# TYPES
-# -----------------------------------------------

server/graph/schemas/system.graphql

@@ -29,6 +29,8 @@ extend type Mutation {
     key: String!
   ): DefaultResponse
 
+  rebuildSearchIndex: DefaultResponse
+
   retryJob(
     id: UUID!
   ): DefaultResponse

server/locales/en.json

@@ -525,7 +525,7 @@
   "admin.scheduler.waitUntil": "Start",
   "admin.search.configSaveSuccess": "Search engine configuration saved successfully.",
   "admin.search.dictOverrides": "PostgreSQL Dictionary Mapping Overrides",
-  "admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. { \"en\": \"english\" }",
+  "admin.search.dictOverridesHint": "JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. {0}",
   "admin.search.engineConfig": "Engine Configuration",
   "admin.search.engineNoConfig": "This engine has no configuration options you can modify.",
   "admin.search.highlighting": "Enable Term Highlighting",
@@ -533,10 +533,12 @@
   "admin.search.indexRebuildSuccess": "Index rebuilt successfully.",
   "admin.search.listRefreshSuccess": "List of search engines has been refreshed.",
   "admin.search.rebuildIndex": "Rebuild Index",
+  "admin.search.rebuildInitSuccess": "A search index rebuild has been initiated and will start shortly.",
   "admin.search.saveSuccess": "Search engine configuration saved successfully",
   "admin.search.searchEngine": "Search Engine",
   "admin.search.subtitle": "Configure the search capabilities of your wiki",
   "admin.search.title": "Search Engine",
+  "admin.searchRebuildIndex": "Rebuild Index",
   "admin.security.cors": "CORS (Cross-Origin Resource Sharing)",
   "admin.security.corsHostnames": "Hostnames Whitelist",
   "admin.security.corsHostnamesHint": "Enter one hostname per line",
@@ -1535,6 +1537,7 @@
   "editor.props.draftHint": "Visible to users with write access only.",
   "editor.props.icon": "Icon",
   "editor.props.info": "Info",
+  "editor.props.isSearchable": "Include in Search Results",
   "editor.props.jsLoad": "Javascript - On Load",
   "editor.props.jsLoadHint": "Execute javascript once the page is loaded",
   "editor.props.jsUnload": "Javascript - On Unload",

server/models/pages.mjs

@@ -344,6 +344,7 @@ export class Page extends Model {
       hash: generateHash({ path: opts.path, locale: opts.locale }),
       icon: opts.icon,
       isBrowsable: opts.isBrowsable ?? true,
+      isSearchable: opts.isSearchable ?? true,
       localeCode: opts.locale,
       ownerId: opts.user.id,
       path: opts.path,
@@ -388,7 +389,7 @@ export class Page extends Model {
     })
 
     // -> Update search vector
-    WIKI.db.pages.updatePageSearchVector(page.id)
+    WIKI.db.pages.updatePageSearchVector({ id: page.id })
 
     // // -> Add to Storage
     // if (!opts.skipStorage) {
@@ -507,14 +508,17 @@ export class Page extends Model {
       historyData.affectedFields.push('publishEndDate')
     }
 
-    // -> Page Config
+    // -> Browsable / Searchable Flags
     if ('isBrowsable' in opts.patch) {
-      patch.config = {
+      patch.isBrowsable = opts.patch.isBrowsable
-        ...patch.config ?? ogPage.config ?? {},
-        isBrowsable: opts.patch.isBrowsable
-      }
       historyData.affectedFields.push('isBrowsable')
     }
+    if ('isSearchable' in opts.patch) {
+      patch.isSearchable = opts.patch.isSearchable
+      historyData.affectedFields.push('isSearchable')
+    }
+
+    // -> Page Config
     if ('allowComments' in opts.patch) {
       patch.config = {
         ...patch.config ?? ogPage.config ?? {},
@@ -644,7 +648,7 @@ export class Page extends Model {
 
     // -> Save Tags
     if (opts.patch.tags) {
-      await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page })
+      // await WIKI.db.tags.associateTags({ tags: opts.patch.tags, page })
     }
 
     // -> Render page to HTML
@@ -672,7 +676,7 @@ export class Page extends Model {
 
     // -> Update search vector
     if (shouldUpdateSearch) {
-      WIKI.db.pages.updatePageSearchVector(page.id)
+      WIKI.db.pages.updatePageSearchVector({ id: page.id })
     }
 
     // -> Update on Storage
@@ -710,13 +714,21 @@ export class Page extends Model {
   /**
    * Update a page text search vector value
    *
-   * @param {String} id Page UUID
+   * @param {Object} opts - Options
+   * @param {string} [opts.id] - Page ID to update (fetch from DB)
+   * @param {Object} [opts.page] - Page object to update (use directly)
    */
-  static async updatePageSearchVector (id) {
+  static async updatePageSearchVector ({ id, page }) {
-    const page = await WIKI.db.pages.query().findById(id).select('localeCode', 'render')
+    if (!page) {
-    const safeContent = WIKI.db.pages.cleanHTML(page.render)
+      if (!id) {
+        throw new Error('Must provide either the page ID or the page object.')
+      }
+      page = await WIKI.db.pages.query().findById(id).select('id', 'localeCode', 'render', 'password')
+    }
+    // -> Exclude password-protected content from being indexed
+    const safeContent = page.password ? '' : WIKI.db.pages.cleanHTML(page.render)
     const dictName = getDictNameFromLocale(page.localeCode)
-    return WIKI.db.knex('pages').where('id', id).update({
+    return WIKI.db.knex('pages').where('id', page.id).update({
       searchContent: safeContent,
       ts: WIKI.db.knex.raw(`
         setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') ||
@@ -725,6 +737,19 @@ export class Page extends Model {
     })
   }
 
+  /**
+   * Refresh Autocomplete Index
+   */
+  static async refreshAutocompleteIndex () {
+    await WIKI.db.knex('autocomplete').truncate()
+    await WIKI.db.knex.raw(`
+      INSERT INTO "autocomplete" (word)
+        SELECT word FROM ts_stat(
+          'SELECT to_tsvector(''simple'', "title") || to_tsvector(''simple'', "description") || to_tsvector(''simple'', "searchContent") FROM "pages" WHERE isSearchableComputed IS TRUE'
+        )
+    `)
+  }
+
   /**
    * Convert an Existing Page
    *

server/tasks/simple/refresh-autocomplete.mjs

@@ -0,0 +1,13 @@
+export async function task (payload) {
+  WIKI.logger.info('Refreshing autocomplete word index...')
+
+  try {
+    await WIKI.db.pages.refreshAutocompleteIndex()
+
+    WIKI.logger.info('Refreshed autocomplete word index: [ COMPLETED ]')
+  } catch (err) {
+    WIKI.logger.error('Refreshing autocomplete word index: [ FAILED ]')
+    WIKI.logger.error(err.message)
+    throw err
+  }
+}

server/tasks/workers/purge-uploads.mjs

@@ -22,5 +22,6 @@ export async function task ({ payload }) {
   } catch (err) {
     WIKI.logger.error('Purging orphaned upload files: [ FAILED ]')
     WIKI.logger.error(err.message)
+    throw err
   }
 }

server/tasks/workers/rebuild-search-index.mjs

@@ -0,0 +1,35 @@
+import { pipeline } from 'node:stream/promises'
+import { Transform } from 'node:stream'
+
+export async function task ({ payload }) {
+  WIKI.logger.info('Rebuilding search index...')
+
+  try {
+    await WIKI.ensureDb()
+
+    let idx = 0
+    await pipeline(
+      WIKI.db.knex.select('id', 'title', 'description', 'localeCode', 'render', 'password').from('pages').stream(),
+      new Transform({
+        objectMode: true,
+        transform: async (page, enc, cb) => {
+          idx++
+          await WIKI.db.pages.updatePageSearchVector({ page })
+          if (idx % 50 === 0) {
+            WIKI.logger.info(`Rebuilding search index... (${idx} processed)`)
+          }
+          cb()
+        }
+      })
+    )
+
+    WIKI.logger.info('Refreshing autocomplete index...')
+    await WIKI.db.pages.refreshAutocompleteIndex()
+
+    WIKI.logger.info('Rebuilt search index: [ COMPLETED ]')
+  } catch (err) {
+    WIKI.logger.error('Rebuilding search index: [ FAILED ]')
+    WIKI.logger.error(err.message)
+    throw err
+  }
+}

ux/src/components/PagePropertiesDialog.vue

@@ -265,6 +265,15 @@ q-card.page-properties-dialog
             checked-icon='las la-check'
             unchecked-icon='las la-times'
           )
+        div
+          q-toggle(
+            v-model='pageStore.isSearchable'
+            dense
+            :label='$t(`editor.props.isSearchable`)'
+            color='primary'
+            checked-icon='las la-check'
+            unchecked-icon='las la-times'
+          )
         div
           q-toggle(
             v-model='state.requirePassword'

ux/src/pages/AdminSearch.vue

@@ -6,7 +6,16 @@ q-page.admin-flags
     .col.q-pl-md
       .text-h5.text-primary.animated.fadeInLeft {{ t('admin.search.title') }}
       .text-subtitle1.text-grey.animated.fadeInLeft.wait-p2s {{ t('admin.search.subtitle') }}
-    .col-auto
+    .col-auto.flex
+      q-btn.q-mr-sm.acrylic-btn(
+        flat
+        icon='mdi-database-refresh'
+        :label='t(`admin.searchRebuildIndex`)'
+        color='purple'
+        @click='rebuild'
+        :loading='state.rebuildLoading'
+      )
+      q-separator.q-mr-sm(vertical)
       q-btn.q-mr-sm.acrylic-btn(
         icon='las la-question-circle'
         flat
@@ -62,7 +71,9 @@ q-page.admin-flags
                 language='json'
                 :min-height='250'
               )
-              q-item-label(caption) {{ t('admin.search.dictOverridesHint') }}
+              q-item-label(caption)
+                i18n-t(keypath='admin.search.dictOverridesHint' tag='span')
+                  span { "en": "english" }
 
     .col-12.col-lg-5.gt-md
       .q-pa-md.text-center
@@ -104,6 +115,7 @@ useMeta({
 
 const state = reactive({
   loading: 0,
+  rebuildLoading: false,
   config: {
     termHighlighting: false,
     dictOverrides: ''
@@ -181,6 +193,41 @@ async function save () {
   state.loading--
 }
 
+async function rebuild () {
+  state.rebuildLoading = true
+  try {
+    const respRaw = await APOLLO_CLIENT.mutate({
+      mutation: gql`
+        mutation rebuildSearchIndex {
+          rebuildSearchIndex {
+            operation {
+              succeeded
+              slug
+              message
+            }
+          }
+        }
+      `
+    })
+    const resp = respRaw?.data?.rebuildSearchIndex?.operation || {}
+    if (resp.succeeded) {
+      $q.notify({
+        type: 'positive',
+        message: t('admin.search.rebuildInitSuccess')
+      })
+    } else {
+      throw new Error(resp.message)
+    }
+  } catch (err) {
+    $q.notify({
+      type: 'negative',
+      message: 'Failed to initiate a search index rebuild',
+      caption: err.message
+    })
+  }
+  state.rebuildLoading = false
+}
+
 // MOUNTED
 
 onMounted(async () => {

ux/src/pages/Search.vue

@@ -21,7 +21,7 @@ q-layout(view='hHh Lpr lff')
               side
               )
               q-icon(
-                :name='state.params.orderByDirection === `desc` ? `mdi-chevron-double-down` : `mdi-chevron-double-up`'
+                :name='state.params.orderByDirection === `desc` ? `mdi-transfer-down` : `mdi-transfer-up`'
                 size='sm'
                 color='primary'
                 )

ux/src/stores/page.js

@@ -19,6 +19,7 @@ const pagePropsFragment = gql`
     icon
     id
     isBrowsable
+    isSearchable
     locale
     password
     path
@@ -122,6 +123,7 @@ const gqlMutations = {
       $editor: String!
       $icon: String
       $isBrowsable: Boolean
+      $isSearchable: Boolean
       $locale: String!
       $path: String!
       $publishState: PagePublishState!
@@ -149,6 +151,7 @@ const gqlMutations = {
         editor: $editor
         icon: $icon
         isBrowsable: $isBrowsable
+        isSearchable: $isSearchable
         locale: $locale
         path: $path
         publishState: $publishState
@@ -195,6 +198,7 @@ export const usePageStore = defineStore('page', {
     icon: 'las la-file-alt',
     id: '',
     isBrowsable: true,
+    isSearchable: true,
     locale: 'en',
     password: '',
     path: '',
@@ -367,6 +371,8 @@ export const usePageStore = defineStore('page', {
         tags: [],
         content: content ?? '',
         render: '',
+        isBrowsable: true,
+        isSearchable: true,
         mode: 'edit'
       })
     },
@@ -420,6 +426,7 @@ export const usePageStore = defineStore('page', {
                 'description',
                 'icon',
                 'isBrowsable',
+                'isSearchable',
                 'locale',
                 'password',
                 'path',
@@ -491,6 +498,7 @@ export const usePageStore = defineStore('page', {
                   'description',
                   'icon',
                   'isBrowsable',
+                  'isSearchable',
                   'locale',
                   'password',
                   'path',