feat: search results + dict override + remove tags relation table (wip)

server/app/data.yml

@@ -85,9 +85,36 @@ defaults:
       maxAge: 600
       methods: 'GET,POST'
       origin: true
-    search:
-      maxHits: 100
     maintainerEmail: security@requarks.io
+tsDictMappings:
+  ar: arabic
+  hy: armenian
+  eu: basque
+  ca: catalan
+  da: danish
+  nl: dutch
+  en: english
+  fi: finnish
+  fr: french
+  de: german
+  el: greek
+  hi: hindi
+  hu: hungarian
+  id: indonesian
+  ga: irish
+  it: italian
+  lt: lithuanian
+  ne: nepali
+  no: norwegian
+  pt: portuguese
+  ro: romanian
+  ru: russian
+  sr: serbian
+  es: spanish
+  sv: swedish
+  ta: tamil
+  tr: turkish
+  yi: yiddish
 editors:
   asciidoc:
     contentType: html

server/db/migrations/3.0.0.mjs

@@ -228,7 +228,9 @@ export async function up (knex) {
       table.jsonb('relations').notNullable().defaultTo('[]')
       table.text('content')
       table.text('render')
+      table.text('searchContent')
       table.specificType('ts', 'tsvector').index('ts_idx', { indexType: 'GIN' })
+      table.specificType('tags', 'int[]').index('tags_idx', { indexType: 'GIN' })
       table.jsonb('toc')
       table.string('editor').notNullable()
       table.string('contentType').notNullable()
@@ -277,7 +279,6 @@ export async function up (knex) {
     .createTable('tags', table => {
       table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
       table.string('tag').notNullable()
-      table.jsonb('display').notNullable().defaultTo('{}')
       table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
       table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
     })
@@ -334,12 +335,6 @@ export async function up (knex) {
     // =====================================
     // RELATION TABLES
     // =====================================
-    // PAGE TAGS ---------------------------
-    .createTable('pageTags', table => {
-      table.increments('id').primary()
-      table.uuid('pageId').references('id').inTable('pages').onDelete('CASCADE')
-      table.uuid('tagId').references('id').inTable('tags').onDelete('CASCADE')
-    })
     // USER GROUPS -------------------------
     .createTable('userGroups', table => {
       table.increments('id').primary()
@@ -493,7 +488,7 @@ export async function up (knex) {
       key: 'search',
       value: {
         termHighlighting: true,
-        dictOverrides: []
+        dictOverrides: {}
       }
     },
     {

server/graph/resolvers/page.mjs

@@ -43,24 +43,61 @@ export default {
      * SEARCH PAGES
      */
     async searchPages (obj, args, context) {
-      if (WIKI.data.searchEngine) {
+      if (!args.siteId) {
-        const resp = await WIKI.data.searchEngine.query(args.query, args)
+        throw new Error('Missing Site ID')
-        return {
-          ...resp,
-          results: _.filter(resp.results, r => {
-            return WIKI.auth.checkAccess(context.req.user, ['read:pages'], {
-              path: r.path,
-              locale: r.locale,
-              tags: r.tags // Tags are needed since access permissions can be limited by page tags too
-            })
-          })
       }
-      } else {
+      if (!args.query?.trim()) {
+        throw new Error('Missing Query')
+      }
+      if (args.offset && args.offset < 0) {
+        throw new Error('Invalid offset value.')
+      }
+      if (args.limit && (args.limit < 1 || args.limit > 100)) {
+        throw new Error('Limit must be between 1 and 100.')
+      }
+      try {
+        const dictName = 'english'
+        const results = await WIKI.db.knex
+          .select(
+            'id',
+            'path',
+            'localeCode AS locale',
+            'title',
+            'description',
+            'icon',
+            'updatedAt',
+            WIKI.db.knex.raw('ts_rank_cd(ts, query) AS relevancy'),
+            WIKI.db.knex.raw(`ts_headline(?, "searchContent", query, 'MaxWords=5, MinWords=3, MaxFragments=5') AS highlight`, [dictName]),
+            WIKI.db.knex.raw('count(*) OVER() AS total')
+          )
+          .fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query])
+          .where('siteId', args.siteId)
+          .where(builder => {
+            if (args.path) {
+              builder.where('path', 'ILIKE', `${path}%`)
+            }
+            if (args.locale?.length > 0) {
+              builder.whereIn('localeCode', args.locale)
+            }
+            if (args.editor) {
+              builder.where('editor', args.editor)
+            }
+            if (args.publishState) {
+              builder.where('publishState', args.publishState)
+            }
+          })
+          .whereRaw('query @@ ts')
+          .orderBy(args.orderBy || 'relevancy', args.orderByDirection || 'desc')
+          .offset(args.offset || 0)
+          .limit(args.limit || 25)
+
         return {
-          results: [],
+          results,
-          suggestions: [],
+          totalHits: results?.length > 0 ? results[0].total : 0
-          totalHits: 0
         }
+      } catch (err) {
+        WIKI.logger.warn(`Search Query Error: ${err.message}`)
+        throw err
       }
     },
     /**
@@ -645,9 +682,9 @@ export default {
     password (obj) {
       return obj.password ? '********' : ''
     },
-    async tags (obj) {
+    // async tags (obj) {
-      return WIKI.db.pages.relatedQuery('tags').for(obj.id)
+    //   return WIKI.db.pages.relatedQuery('tags').for(obj.id)
-    },
+    // },
     tocDepth (obj) {
       return {
         min: obj.extra?.tocDepth?.min ?? 1,

server/graph/resolvers/system.mjs

@@ -78,7 +78,10 @@ export default {
       ])
     },
     systemSearch () {
-      return WIKI.config.search
+      return {
+        ...WIKI.config.search,
+        dictOverrides: JSON.stringify(WIKI.config.search.dictOverrides, null, 2)
+      }
     }
   },
   Mutation: {
@@ -183,7 +186,11 @@ export default {
       }
     },
     async updateSystemSearch (obj, args, context) {
-      WIKI.config.search = _.defaultsDeep(_.omit(args, ['__typename']), WIKI.config.search)
+      WIKI.config.search = {
+        ...WIKI.config.search,
+        termHighlighting: args.termHighlighting ?? WIKI.config.search.termHighlighting,
+        dictOverrides: args.dictOverrides ? JSON.parse(args.dictOverrides) : WIKI.config.search.dictOverrides
+      }
       // TODO: broadcast config update
       await WIKI.configSvc.saveToDb(['search'])
       return {

server/graph/schemas/page.graphql

@@ -15,16 +15,53 @@ extend type Query {
   ): PageVersion
 
   searchPages(
+    """
+    Site ID to search in (required)
+    """
     siteId: UUID!
+    """
+    Search Query (required)
+    """
     query: String!
+    """
+    The locale to perform the query as. Affects how the query is parsed by the search engine.
+    """
+    queryLocale: String
+    """
+    Only match pages that starts with the provided path.
+    """
     path: String
+    """
+    Only match pages having one of the provided locales.
+    """
     locale: [String]
+    """
+    Only match pages having one of the provided tags.
+    """
     tags: [String]
+    """
+    Only match pages using the provided editor.
+    """
     editor: String
+    """
+    Only match pages is the provided state.
+    """
     publishState: PagePublishState
+    """
+    Result ordering. Defaults to relevancy.
+    """
     orderBy: PageSearchSort
+    """
+    Result ordering direction. Defaults to descending.
+    """
     orderByDirection: OrderByDirection
+    """
+    Result offset. Defaults to 0.
+    """
     offset: Int
+    """
+    Results amount to return. Defaults to 25. Maximum 100.
+    """
     limit: Int
   ): PageSearchResponse
 
@@ -264,19 +301,20 @@ type PageHistoryResult {
 
 type PageSearchResponse {
   results: [PageSearchResult]
-  suggestions: [String]
   totalHits: Int
 }
 
 type PageSearchResult {
-  id: UUID
-  title: String
   description: String
+  highlight: String
   icon: String
+  id: UUID
+  locale: String
   path: String
+  relevancy: Float
   tags: [String]
+  title: String
   updatedAt: Date
-  locale: String
 }
 
 type PageListItem {
@@ -392,7 +430,7 @@ input PageTocDepthInput {
 enum PageSearchSort {
   relevancy
   title
-  updated
+  updatedAt
 }
 
 enum PageOrderBy {

server/helpers/common.mjs

@@ -120,3 +120,12 @@ export function parseModuleProps (props) {
     return result
   }, {})
 }
+
+export function getDictNameFromLocale (locale) {
+  const localeCode = locale.length > 2 ? locale.substring(0, 2) : locale
+  if (localeCode in WIKI.config.search.dictOverrides) {
+    return WIKI.config.search.dictOverrides[localeCode]
+  } else {
+    return WIKI.data.tsDictMappings[localeCode] ?? 'simple'
+  }
+}

server/models/pages.mjs

@@ -1,6 +1,7 @@
 import { Model } from 'objection'
 import { find, get, has, initial, isEmpty, isString, last, pick } from 'lodash-es'
 import { Type as JSBinType } from 'js-binary'
+import { getDictNameFromLocale } from '../helpers/common.mjs'
 import { generateHash, getFileExtension, injectPageMetadata } from '../helpers/page.mjs'
 import path from 'node:path'
 import fse from 'fs-extra'
@@ -27,9 +28,6 @@ const frontmatterRegex = {
   markdown: /^(-{3}(?:\n|\r)([\w\W]+?)(?:\n|\r)-{3})?(?:\n|\r)*([\w\W]*)*/
 }
 
-const punctuationRegex = /[!,:;/\\_+\-=()&#@<>$~%^*[\]{}"'|]+|(\.\s)|(\s\.)/ig
-// const htmlEntitiesRegex = /(&#[0-9]{3};)|(&#x[a-zA-Z0-9]{2};)/ig
-
 /**
  * Pages model
  */
@@ -66,18 +64,18 @@ export class Page extends Model {
 
   static get relationMappings() {
     return {
-      tags: {
+      // tags: {
-        relation: Model.ManyToManyRelation,
+      //   relation: Model.ManyToManyRelation,
-        modelClass: Tag,
+      //   modelClass: Tag,
-        join: {
+      //   join: {
-          from: 'pages.id',
+      //     from: 'pages.id',
-          through: {
+      //     through: {
-            from: 'pageTags.pageId',
+      //       from: 'pageTags.pageId',
-            to: 'pageTags.tagId'
+      //       to: 'pageTags.tagId'
-          },
+      //     },
-          to: 'tags.id'
+      //     to: 'tags.id'
-        }
+      //   }
-      },
+      // },
       links: {
         relation: Model.HasManyRelation,
         modelClass: PageLink,
@@ -319,6 +317,12 @@ export class Page extends Model {
       scriptJsUnload = opts.scriptJsUnload || ''
     }
 
+    // -> Get Tags
+    let tags = []
+    if (opts.tags && opts.tags.length > 0) {
+      tags = await WIKI.db.tags.fetchIds({ tags: opts.tags, siteId: opts.siteId })
+    }
+
     // -> Create page
     const page = await WIKI.db.pages.query().insert({
       alias: opts.alias,
@@ -348,6 +352,7 @@ export class Page extends Model {
       publishStartDate: opts.publishStartDate?.toISO(),
       relations: opts.relations ?? [],
       siteId: opts.siteId,
+      tags,
       title: opts.title,
       toc: '[]',
       scripts: JSON.stringify({
@@ -357,11 +362,6 @@ export class Page extends Model {
       })
     }).returning('*')
 
-    // -> Save Tags
-    if (opts.tags && opts.tags.length > 0) {
-      await WIKI.db.tags.associateTags({ tags: opts.tags, page })
-    }
-
     // -> Render page to HTML
     await WIKI.db.pages.renderPage(page)
 
@@ -387,31 +387,23 @@ export class Page extends Model {
       siteId: page.siteId
     })
 
-    return page
+    // -> Update search vector
-    // TODO: Handle remaining flow
+    WIKI.db.pages.updatePageSearchVector(page.id)
-
-    // -> Rebuild page tree
-    await WIKI.db.pages.rebuildTree()
-
-    // -> Add to Search Index
-    const pageContents = await WIKI.db.pages.query().findById(page.id).select('render')
-    page.safeContent = WIKI.db.pages.cleanHTML(pageContents.render)
-    await WIKI.data.searchEngine.created(page)
 
-    // -> Add to Storage
+    // // -> Add to Storage
-    if (!opts.skipStorage) {
+    // if (!opts.skipStorage) {
-      await WIKI.db.storage.pageEvent({
+    //   await WIKI.db.storage.pageEvent({
-        event: 'created',
+    //     event: 'created',
-        page
+    //     page
-      })
+    //   })
-    }
+    // }
 
-    // -> Reconnect Links
+    // // -> Reconnect Links
-    await WIKI.db.pages.reconnectLinks({
+    // await WIKI.db.pages.reconnectLinks({
-      locale: page.localeCode,
+    //   locale: page.localeCode,
-      path: page.path,
+    //   path: page.path,
-      mode: 'create'
+    //   mode: 'create'
-    })
+    // })
 
     // -> Get latest updatedAt
     page.updatedAt = await WIKI.db.pages.query().findById(page.id).select('updatedAt').then(r => r.updatedAt)
@@ -445,6 +437,7 @@ export class Page extends Model {
       action: 'updated',
       affectedFields: []
     }
+    let shouldUpdateSearch = false
 
     // -> Create version snapshot
     await WIKI.db.pageHistory.addVersion(ogPage)
@@ -453,6 +446,7 @@ export class Page extends Model {
     if ('title' in opts.patch) {
       patch.title = opts.patch.title.trim()
       historyData.affectedFields.push('title')
+      shouldUpdateSearch = true
 
       if (patch.title.length < 1) {
         throw new Error('ERR_PAGE_TITLE_MISSING')
@@ -462,6 +456,7 @@ export class Page extends Model {
     if ('description' in opts.patch) {
       patch.description = opts.patch.description.trim()
       historyData.affectedFields.push('description')
+      shouldUpdateSearch = true
     }
 
     if ('icon' in opts.patch) {
@@ -488,9 +483,10 @@ export class Page extends Model {
       }
     }
 
-    if ('content' in opts.patch) {
+    if ('content' in opts.patch && opts.patch.content) {
       patch.content = opts.patch.content
       historyData.affectedFields.push('content')
+      shouldUpdateSearch = true
     }
 
     // -> Publish State
@@ -674,10 +670,10 @@ export class Page extends Model {
       updatedAt: page.updatedAt
     })
 
-    // // -> Update Search Index
+    // -> Update search vector
-    // const pageContents = await WIKI.db.pages.query().findById(page.id).select('render')
+    if (shouldUpdateSearch) {
-    // page.safeContent = WIKI.db.pages.cleanHTML(pageContents.render)
+      WIKI.db.pages.updatePageSearchVector(page.id)
-    // await WIKI.data.searchEngine.updated(page)
+    }
 
     // -> Update on Storage
     // if (!opts.skipStorage) {
@@ -711,6 +707,24 @@ export class Page extends Model {
     return page
   }
 
+  /**
+   * Update a page text search vector value
+   *
+   * @param {String} id Page UUID
+   */
+  static async updatePageSearchVector (id) {
+    const page = await WIKI.db.pages.query().findById(id).select('localeCode', 'render')
+    const safeContent = WIKI.db.pages.cleanHTML(page.render)
+    const dictName = getDictNameFromLocale(page.localeCode)
+    return WIKI.db.knex('pages').where('id', id).update({
+      searchContent: safeContent,
+      ts: WIKI.db.knex.raw(`
+        setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') ||
+        setweight(to_tsvector('${dictName}', coalesce(description,'')), 'B') ||
+        setweight(to_tsvector('${dictName}', coalesce(?,'')), 'C')`, [safeContent])
+    })
+  }
+
   /**
    * Convert an Existing Page
    *
@@ -1214,10 +1228,10 @@ export class Page extends Model {
         ])
         .joinRelated('author')
         .joinRelated('creator')
-        .withGraphJoined('tags')
+        // .withGraphJoined('tags')
-        .modifyGraph('tags', builder => {
+        // .modifyGraph('tags', builder => {
-          builder.select('tag')
+        //   builder.select('tag')
-        })
+        // })
         .where(queryModeID ? {
           'pages.id': opts
         } : {
@@ -1346,14 +1360,11 @@ export class Page extends Model {
    * @returns {string} Cleaned Content Text
    */
   static cleanHTML(rawHTML = '') {
-    let data = striptags(rawHTML || '', [], ' ')
+    const data = striptags(rawHTML || '', [], ' ')
       .replace(emojiRegex(), '')
-      // .replace(htmlEntitiesRegex, '')
     return he.decode(data)
-      .replace(punctuationRegex, ' ')
       .replace(/(\r\n|\n|\r)/gm, ' ')
       .replace(/\s\s+/g, ' ')
-      .split(' ').filter(w => w.length > 1).join(' ').toLowerCase()
   }
 
   /**

ux/src/pages/AdminSearch.vue

@@ -56,14 +56,13 @@ q-page.admin-flags
           blueprint-icon.self-start(icon='search')
           q-item-section
             q-item-label {{t(`admin.search.dictOverrides`)}}
-            q-input.q-mt-sm(
+            q-no-ssr(:placeholder='t(`common.loading`)')
-              type='textarea'
+              util-code-editor.admin-theme-cm.q-my-sm(
                 v-model='state.config.dictOverrides'
-              outlined
+                language='json'
-              :aria-label='t(`admin.search.dictOverrides`)'
+                :min-height='250'
-              :hint='t(`admin.search.dictOverridesHint`)'
-              input-style='min-height: 200px;'
               )
+              q-item-label(caption) JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. { "en": "english" }
 
     .col-12.col-lg-5.gt-md
       .q-pa-md.text-center
@@ -80,6 +79,8 @@ import { useI18n } from 'vue-i18n'
 import { useSiteStore } from 'src/stores/site'
 import { useFlagsStore } from 'src/stores/flags'
 
+import UtilCodeEditor from 'src/components/UtilCodeEditor.vue'
+
 // QUASAR
 
 const $q = useQuasar()

ux/src/pages/Search.vue

@@ -94,15 +94,19 @@ q-layout(view='hHh Lpr lff')
         .text-header.flex
           span {{t('search.results')}}
           q-space
-          span.text-caption #[strong {{ state.items }}] results
+          span.text-caption #[strong {{ state.total }}] results
-        q-list(separator, padding)
+        q-list(separator)
-          q-item(v-for='item of state.items', clickable)
+          q-item(
+            v-for='item of state.results'
+            clickable
+            :to='`/` + item.path'
+            )
             q-item-section(avatar)
-              q-avatar(color='primary' text-color='white' rounded icon='las la-file-alt')
+              q-avatar(color='primary' text-color='white' rounded :icon='item.icon')
             q-item-section
-              q-item-label Page ABC def {{ item }}
+              q-item-label {{ item.title }}
-              q-item-label(caption) Lorem ipsum beep boop foo bar
+              q-item-label(caption) {{ item.description }}
-              q-item-label(caption) ...Abc def #[span.text-highlight home] efg hig klm...
+              q-item-label.text-highlight(caption, v-html='item.highlight')
             q-item-section(side)
               .flex
                 q-chip(
@@ -114,8 +118,8 @@ q-layout(view='hHh Lpr lff')
                   size='sm'
                   ) tag {{ tag }}
               .flex
-                .text-caption.q-mr-sm.text-grey /beep/boop/hello
+                .text-caption.q-mr-sm.text-grey /{{ item.path }}
-                .text-caption 2023-01-25
+                .text-caption {{ humanizeDate(item.updatedAt) }}
 
       q-inner-loading(:showing='state.loading > 0')
   main-overlay-dialog
@@ -127,6 +131,9 @@ import { useI18n } from 'vue-i18n'
 import { useMeta, useQuasar } from 'quasar'
 import { computed, onMounted, reactive, watch } from 'vue'
 import { useRouter, useRoute } from 'vue-router'
+import gql from 'graphql-tag'
+import { cloneDeep } from 'lodash-es'
+import { DateTime } from 'luxon'
 
 import { useFlagsStore } from 'src/stores/flags'
 import { useSiteStore } from 'src/stores/site'
@@ -170,7 +177,8 @@ const state = reactive({
   filterLocale: ['en'],
   filterEditor: '',
   filterPublishState: '',
-  items: 25
+  results: [],
+  total: 0
 })
 
 const editors = computed(() => {
@@ -196,6 +204,7 @@ const publishStates = computed(() => {
 watch(() => route.query, async (newQueryObj) => {
   if (newQueryObj.q) {
     siteStore.search = newQueryObj.q
+    performSearch()
   }
 }, { immediate: true })
 
@@ -207,10 +216,85 @@ function pageStyle (offset, height) {
   }
 }
 
+function humanizeDate (val) {
+  return DateTime.fromISO(val).toFormat(userStore.preferredDateFormat)
+}
+
+async function performSearch () {
+  siteStore.searchIsLoading = true
+  try {
+    const resp = await APOLLO_CLIENT.query({
+      query: gql`
+        query searchPages (
+          $siteId: UUID!
+          $query: String!
+          $path: String
+          $locale: [String]
+          $tags: [String]
+          $editor: String
+          $publishState: PagePublishState
+          $orderBy: PageSearchSort
+          $orderByDirection: OrderByDirection
+          $offset: Int
+          $limit: Int
+        ) {
+          searchPages(
+            siteId: $siteId
+            query: $query
+            path: $path
+            locale: $locale
+            tags: $tags
+            editor: $editor
+            publishState: $publishState
+            orderBy: $orderBy
+            orderByDirection: $orderByDirection
+            offset: $offset
+            limit: $limit
+          ) {
+            results {
+              id
+              path
+              locale
+              title
+              description
+              icon
+              updatedAt
+              relevancy
+              highlight
+            }
+            totalHits
+          }
+        }
+      `,
+      variables: {
+        siteId: siteStore.id,
+        query: siteStore.search
+      },
+      fetchPolicy: 'network-only'
+    })
+    if (!resp?.data?.searchPages) {
+      throw new Error('Unexpected error')
+    }
+    state.results = cloneDeep(resp.data.searchPages.results)
+    state.total = resp.data.searchPages.totalHits
+  } catch (err) {
+    $q.notify({
+      type: 'negative',
+      message: 'Failed to perform search query.',
+      caption: err.message
+    })
+  }
+  siteStore.searchIsLoading = false
+}
+
 // MOUNTED
 
 onMounted(() => {
+  if (siteStore.search) {
+    // performSearch()
+  } else {
     siteStore.searchIsLoading = false
+  }
 })
 
 </script>
@@ -298,10 +382,14 @@ onMounted(() => {
   }
 
   .text-highlight {
+    font-style: italic;
+
+    > b {
       background-color: rgba($yellow-7, .5);
       padding: 0 3px;
       border-radius: 3px;
     }
+  }
 
   .q-page {
     flex: 1 1;