msb-public
/
wiki
mirror of https://github.com/requarks/wiki
1
0
Code Issues Projects Releases Wiki Activity

feat: search results + dict override + remove tags relation table (wip)

Browse Source
pull/6775/head
NGPixel 1 year ago
parent cbbc10dadb
commit b75275d748
No known key found for this signature in database
GPG Key ID: B755FB6870B30F63
9 changed files with 325 additions and 112 deletions
Show Stats Download Patch File Download Diff File

31
server/app/data.yml
Unescape View File

@ -85,9 +85,36 @@ defaults:
maxAge: 600
methods: 'GET,POST'
origin: true
search:
maxHits: 100
maintainerEmail: security@requarks.io
tsDictMappings:
ar: arabic
hy: armenian
eu: basque
ca: catalan
da: danish
nl: dutch
en: english
fi: finnish
fr: french
de: german
el: greek
hi: hindi
hu: hungarian
id: indonesian
ga: irish
it: italian
lt: lithuanian
ne: nepali
no: norwegian
pt: portuguese
ro: romanian
ru: russian
sr: serbian
es: spanish
sv: swedish
ta: tamil
tr: turkish
yi: yiddish
editors:
asciidoc:
contentType: html

11
server/db/migrations/3.0.0.mjs
Unescape View File

@ -228,7 +228,9 @@ export async function up (knex) {
table.jsonb('relations').notNullable().defaultTo('[]')
table.text('content')
table.text('render')
table.text('searchContent')
table.specificType('ts', 'tsvector').index('ts_idx', { indexType: 'GIN' })
table.specificType('tags', 'int[]').index('tags_idx', { indexType: 'GIN' })
table.jsonb('toc')
table.string('editor').notNullable()
table.string('contentType').notNullable()
@ -277,7 +279,6 @@ export async function up (knex) {
.createTable('tags', table => {
table.uuid('id').notNullable().primary().defaultTo(knex.raw('gen_random_uuid()'))
table.string('tag').notNullable()
table.jsonb('display').notNullable().defaultTo('{}')
table.timestamp('createdAt').notNullable().defaultTo(knex.fn.now())
table.timestamp('updatedAt').notNullable().defaultTo(knex.fn.now())
})
@ -334,12 +335,6 @@ export async function up (knex) {
// =====================================
// RELATION TABLES
// =====================================
// PAGE TAGS ---------------------------
.createTable('pageTags', table => {
table.increments('id').primary()
table.uuid('pageId').references('id').inTable('pages').onDelete('CASCADE')
table.uuid('tagId').references('id').inTable('tags').onDelete('CASCADE')
})
// USER GROUPS -------------------------
.createTable('userGroups', table => {
table.increments('id').primary()
@ -493,7 +488,7 @@ export async function up (knex) {
key: 'search',
value: {
termHighlighting: true,
dictOverrides: []
dictOverrides: {}
}
},
{

73
server/graph/resolvers/page.mjs
Unescape View File

@ -43,24 +43,61 @@ export default {
* SEARCH PAGES
*/
async searchPages (obj, args, context) {
if (WIKI.data.searchEngine) {
const resp = await WIKI.data.searchEngine.query(args.query, args)
return {
...resp,
results: _.filter(resp.results, r => {
return WIKI.auth.checkAccess(context.req.user, ['read:pages'], {
path: r.path,
locale: r.locale,
tags: r.tags // Tags are needed since access permissions can be limited by page tags too
})
})
if (!args.siteId) {
throw new Error('Missing Site ID')
}
} else {
if (!args.query?.trim()) {
throw new Error('Missing Query')
}
if (args.offset && args.offset < 0) {
throw new Error('Invalid offset value.')
}
if (args.limit && (args.limit < 1 || args.limit > 100)) {
throw new Error('Limit must be between 1 and 100.')
}
try {
const dictName = 'english'
const results = await WIKI.db.knex
.select(
'id',
'path',
'localeCode AS locale',
'title',
'description',
'icon',
'updatedAt',
WIKI.db.knex.raw('ts_rank_cd(ts, query) AS relevancy'),
WIKI.db.knex.raw(`ts_headline(?, "searchContent", query, 'MaxWords=5, MinWords=3, MaxFragments=5') AS highlight`, [dictName]),
WIKI.db.knex.raw('count(*) OVER() AS total')
)
.fromRaw('pages, websearch_to_tsquery(?, ?) query', [dictName, args.query])
.where('siteId', args.siteId)
.where(builder => {
if (args.path) {
builder.where('path', 'ILIKE', `${path}%`)
}
if (args.locale?.length > 0) {
builder.whereIn('localeCode', args.locale)
}
if (args.editor) {
builder.where('editor', args.editor)
}
if (args.publishState) {
builder.where('publishState', args.publishState)
}
})
.whereRaw('query @@ ts')
.orderBy(args.orderBy || 'relevancy', args.orderByDirection || 'desc')
.offset(args.offset || 0)
.limit(args.limit || 25)
return {
results: [],
suggestions: [],
totalHits: 0
results,
totalHits: results?.length > 0 ? results[0].total : 0
}
} catch (err) {
WIKI.logger.warn(`Search Query Error: ${err.message}`)
throw err
}
},
/**
@ -645,9 +682,9 @@ export default {
password (obj) {
return obj.password ? '********' : ''
},
async tags (obj) {
return WIKI.db.pages.relatedQuery('tags').for(obj.id)
},
// async tags (obj) {
// return WIKI.db.pages.relatedQuery('tags').for(obj.id)
// },
tocDepth (obj) {
return {
min: obj.extra?.tocDepth?.min ?? 1,

11
server/graph/resolvers/system.mjs
Unescape View File

@ -78,7 +78,10 @@ export default {
])
},
systemSearch () {
return WIKI.config.search
return {
...WIKI.config.search,
dictOverrides: JSON.stringify(WIKI.config.search.dictOverrides, null, 2)
}
}
},
Mutation: {
@ -183,7 +186,11 @@ export default {
}
},
async updateSystemSearch (obj, args, context) {
WIKI.config.search = _.defaultsDeep(_.omit(args, ['__typename']), WIKI.config.search)
WIKI.config.search = {
...WIKI.config.search,
termHighlighting: args.termHighlighting ?? WIKI.config.search.termHighlighting,
dictOverrides: args.dictOverrides ? JSON.parse(args.dictOverrides) : WIKI.config.search.dictOverrides
}
// TODO: broadcast config update
await WIKI.configSvc.saveToDb(['search'])
return {

48
server/graph/schemas/page.graphql
Unescape View File

@ -15,16 +15,53 @@ extend type Query {
): PageVersion
searchPages(
"""
Site ID to search in (required)
"""
siteId: UUID!
"""
Search Query (required)
"""
query: String!
"""
The locale to perform the query as. Affects how the query is parsed by the search engine.
"""
queryLocale: String
"""
Only match pages that starts with the provided path.
"""
path: String
"""
Only match pages having one of the provided locales.
"""
locale: [String]
"""
Only match pages having one of the provided tags.
"""
tags: [String]
"""
Only match pages using the provided editor.
"""
editor: String
"""
Only match pages is the provided state.
"""
publishState: PagePublishState
"""
Result ordering. Defaults to relevancy.
"""
orderBy: PageSearchSort
"""
Result ordering direction. Defaults to descending.
"""
orderByDirection: OrderByDirection
"""
Result offset. Defaults to 0.
"""
offset: Int
"""
Results amount to return. Defaults to 25. Maximum 100.
"""
limit: Int
): PageSearchResponse
@ -264,19 +301,20 @@ type PageHistoryResult {
type PageSearchResponse {
results: [PageSearchResult]
suggestions: [String]
totalHits: Int
}
type PageSearchResult {
id: UUID
title: String
description: String
highlight: String
icon: String
id: UUID
locale: String
path: String
relevancy: Float
tags: [String]
title: String
updatedAt: Date
locale: String
}
type PageListItem {
@ -392,7 +430,7 @@ input PageTocDepthInput {
enum PageSearchSort {
relevancy
title
updated
updatedAt
}
enum PageOrderBy {

9
server/helpers/common.mjs
Unescape View File

@ -120,3 +120,12 @@ export function parseModuleProps (props) {
return result
}, {})
}
export function getDictNameFromLocale (locale) {
const localeCode = locale.length > 2 ? locale.substring(0, 2) : locale
if (localeCode in WIKI.config.search.dictOverrides) {
return WIKI.config.search.dictOverrides[localeCode]
} else {
return WIKI.data.tsDictMappings[localeCode] ?? 'simple'
}
}

123
server/models/pages.mjs
Unescape View File

@ -1,6 +1,7 @@
import { Model } from 'objection'
import { find, get, has, initial, isEmpty, isString, last, pick } from 'lodash-es'
import { Type as JSBinType } from 'js-binary'
import { getDictNameFromLocale } from '../helpers/common.mjs'
import { generateHash, getFileExtension, injectPageMetadata } from '../helpers/page.mjs'
import path from 'node:path'
import fse from 'fs-extra'
@ -27,9 +28,6 @@ const frontmatterRegex = {
markdown: /^(-{3}(?:\n|\r)([\w\W]+?)(?:\n|\r)-{3})?(?:\n|\r)*([\w\W]*)*/
}
const punctuationRegex = /[!,:;/\\_+\-=()&#@<>$~%^*[\]{}"'|]+|(\.\s)|(\s\.)/ig
// const htmlEntitiesRegex = /(&#[0-9]{3};)|(&#x[a-zA-Z0-9]{2};)/ig
/**
* Pages model
*/
@ -66,18 +64,18 @@ export class Page extends Model {
static get relationMappings() {
return {
tags: {
relation: Model.ManyToManyRelation,
modelClass: Tag,
join: {
from: 'pages.id',
through: {
from: 'pageTags.pageId',
to: 'pageTags.tagId'
},
to: 'tags.id'
}
},
// tags: {
// relation: Model.ManyToManyRelation,
// modelClass: Tag,
// join: {
// from: 'pages.id',
// through: {
// from: 'pageTags.pageId',
// to: 'pageTags.tagId'
// },
// to: 'tags.id'
// }
// },
links: {
relation: Model.HasManyRelation,
modelClass: PageLink,
@ -319,6 +317,12 @@ export class Page extends Model {
scriptJsUnload = opts.scriptJsUnload || ''
}
// -> Get Tags
let tags = []
if (opts.tags && opts.tags.length > 0) {
tags = await WIKI.db.tags.fetchIds({ tags: opts.tags, siteId: opts.siteId })
}
// -> Create page
const page = await WIKI.db.pages.query().insert({
alias: opts.alias,
@ -348,6 +352,7 @@ export class Page extends Model {
publishStartDate: opts.publishStartDate?.toISO(),
relations: opts.relations ?? [],
siteId: opts.siteId,
tags,
title: opts.title,
toc: '[]',
scripts: JSON.stringify({
@ -357,11 +362,6 @@ export class Page extends Model {
})
}).returning('*')
// -> Save Tags
if (opts.tags && opts.tags.length > 0) {
await WIKI.db.tags.associateTags({ tags: opts.tags, page })
}
// -> Render page to HTML
await WIKI.db.pages.renderPage(page)
@ -387,31 +387,23 @@ export class Page extends Model {
siteId: page.siteId
})
return page
// TODO: Handle remaining flow
// -> Rebuild page tree
await WIKI.db.pages.rebuildTree()
// -> Add to Search Index
const pageContents = await WIKI.db.pages.query().findById(page.id).select('render')
page.safeContent = WIKI.db.pages.cleanHTML(pageContents.render)
await WIKI.data.searchEngine.created(page)
// -> Update search vector
WIKI.db.pages.updatePageSearchVector(page.id)
// -> Add to Storage
if (!opts.skipStorage) {
await WIKI.db.storage.pageEvent({
event: 'created',
page
})
}
// // -> Add to Storage
// if (!opts.skipStorage) {
// await WIKI.db.storage.pageEvent({
// event: 'created',
// page
// })
// }
// -> Reconnect Links
await WIKI.db.pages.reconnectLinks({
locale: page.localeCode,
path: page.path,
mode: 'create'
})
// // -> Reconnect Links
// await WIKI.db.pages.reconnectLinks({
// locale: page.localeCode,
// path: page.path,
// mode: 'create'
// })
// -> Get latest updatedAt
page.updatedAt = await WIKI.db.pages.query().findById(page.id).select('updatedAt').then(r => r.updatedAt)
@ -445,6 +437,7 @@ export class Page extends Model {
action: 'updated',
affectedFields: []
}
let shouldUpdateSearch = false
// -> Create version snapshot
await WIKI.db.pageHistory.addVersion(ogPage)
@ -453,6 +446,7 @@ export class Page extends Model {
if ('title' in opts.patch) {
patch.title = opts.patch.title.trim()
historyData.affectedFields.push('title')
shouldUpdateSearch = true
if (patch.title.length < 1) {
throw new Error('ERR_PAGE_TITLE_MISSING')
@ -462,6 +456,7 @@ export class Page extends Model {
if ('description' in opts.patch) {
patch.description = opts.patch.description.trim()
historyData.affectedFields.push('description')
shouldUpdateSearch = true
}
if ('icon' in opts.patch) {
@ -488,9 +483,10 @@ export class Page extends Model {
}
}
if ('content' in opts.patch) {
if ('content' in opts.patch && opts.patch.content) {
patch.content = opts.patch.content
historyData.affectedFields.push('content')
shouldUpdateSearch = true
}
// -> Publish State
@ -674,10 +670,10 @@ export class Page extends Model {
updatedAt: page.updatedAt
})
// // -> Update Search Index
// const pageContents = await WIKI.db.pages.query().findById(page.id).select('render')
// page.safeContent = WIKI.db.pages.cleanHTML(pageContents.render)
// await WIKI.data.searchEngine.updated(page)
// -> Update search vector
if (shouldUpdateSearch) {
WIKI.db.pages.updatePageSearchVector(page.id)
}
// -> Update on Storage
// if (!opts.skipStorage) {
@ -711,6 +707,24 @@ export class Page extends Model {
return page
}
/**
* Update a page text search vector value
*
* @param {String} id Page UUID
*/
static async updatePageSearchVector (id) {
const page = await WIKI.db.pages.query().findById(id).select('localeCode', 'render')
const safeContent = WIKI.db.pages.cleanHTML(page.render)
const dictName = getDictNameFromLocale(page.localeCode)
return WIKI.db.knex('pages').where('id', id).update({
searchContent: safeContent,
ts: WIKI.db.knex.raw(`
setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') ||
setweight(to_tsvector('${dictName}', coalesce(description,'')), 'B') ||
setweight(to_tsvector('${dictName}', coalesce(?,'')), 'C')`, [safeContent])
})
}
/**
* Convert an Existing Page
*
@ -1214,10 +1228,10 @@ export class Page extends Model {
])
.joinRelated('author')
.joinRelated('creator')
.withGraphJoined('tags')
.modifyGraph('tags', builder => {
builder.select('tag')
})
// .withGraphJoined('tags')
// .modifyGraph('tags', builder => {
// builder.select('tag')
// })
.where(queryModeID ? {
'pages.id': opts
} : {
@ -1346,14 +1360,11 @@ export class Page extends Model {
* @returns {string} Cleaned Content Text
*/
static cleanHTML(rawHTML = '') {
let data = striptags(rawHTML || '', [], ' ')
const data = striptags(rawHTML || '', [], ' ')
.replace(emojiRegex(), '')
// .replace(htmlEntitiesRegex, '')
return he.decode(data)
.replace(punctuationRegex, ' ')
.replace(/(\r\n|\n|\r)/gm, ' ')
.replace(/\s\s+/g, ' ')
.split(' ').filter(w => w.length > 1).join(' ').toLowerCase()
}
/**

13
ux/src/pages/AdminSearch.vue
Unescape View File

@ -56,14 +56,13 @@ q-page.admin-flags
blueprint-icon.self-start(icon='search')
q-item-section
q-item-label {{t(`admin.search.dictOverrides`)}}
q-input.q-mt-sm(
type='textarea'
q-no-ssr(:placeholder='t(`common.loading`)')
util-code-editor.admin-theme-cm.q-my-sm(
v-model='state.config.dictOverrides'
outlined
:aria-label='t(`admin.search.dictOverrides`)'
:hint='t(`admin.search.dictOverridesHint`)'
input-style='min-height: 200px;'
language='json'
:min-height='250'
)
q-item-label(caption) JSON object of 2 letters locale codes and their PostgreSQL dictionary association. e.g. { "en": "english" }
.col-12.col-lg-5.gt-md
.q-pa-md.text-center
@ -80,6 +79,8 @@ import { useI18n } from 'vue-i18n'
import { useSiteStore } from 'src/stores/site'
import { useFlagsStore } from 'src/stores/flags'
import UtilCodeEditor from 'src/components/UtilCodeEditor.vue'
// QUASAR
const $q = useQuasar()

108
ux/src/pages/Search.vue
Unescape View File

@ -94,15 +94,19 @@ q-layout(view='hHh Lpr lff')
.text-header.flex
span {{t('search.results')}}
q-space
span.text-caption #[strong {{ state.items }}] results
q-list(separator, padding)
q-item(v-for='item of state.items', clickable)
span.text-caption #[strong {{ state.total }}] results
q-list(separator)
q-item(
v-for='item of state.results'
clickable
:to='`/` + item.path'
)
q-item-section(avatar)
q-avatar(color='primary' text-color='white' rounded icon='las la-file-alt')
q-avatar(color='primary' text-color='white' rounded :icon='item.icon')
q-item-section
q-item-label Page ABC def {{ item }}
q-item-label(caption) Lorem ipsum beep boop foo bar
q-item-label(caption) ...Abc def #[span.text-highlight home] efg hig klm...
q-item-label {{ item.title }}
q-item-label(caption) {{ item.description }}
q-item-label.text-highlight(caption, v-html='item.highlight')
q-item-section(side)
.flex
q-chip(
@ -114,8 +118,8 @@ q-layout(view='hHh Lpr lff')
size='sm'
) tag {{ tag }}
.flex
.text-caption.q-mr-sm.text-grey /beep/boop/hello
.text-caption 2023-01-25
.text-caption.q-mr-sm.text-grey /{{ item.path }}
.text-caption {{ humanizeDate(item.updatedAt) }}
q-inner-loading(:showing='state.loading > 0')
main-overlay-dialog
@ -127,6 +131,9 @@ import { useI18n } from 'vue-i18n'
import { useMeta, useQuasar } from 'quasar'
import { computed, onMounted, reactive, watch } from 'vue'
import { useRouter, useRoute } from 'vue-router'
import gql from 'graphql-tag'
import { cloneDeep } from 'lodash-es'
import { DateTime } from 'luxon'
import { useFlagsStore } from 'src/stores/flags'
import { useSiteStore } from 'src/stores/site'
@ -170,7 +177,8 @@ const state = reactive({
filterLocale: ['en'],
filterEditor: '',
filterPublishState: '',
items: 25
results: [],
total: 0
})
const editors = computed(() => {
@ -196,6 +204,7 @@ const publishStates = computed(() => {
watch(() => route.query, async (newQueryObj) => {
if (newQueryObj.q) {
siteStore.search = newQueryObj.q
performSearch()
}
}, { immediate: true })
@ -207,10 +216,85 @@ function pageStyle (offset, height) {
}
}
function humanizeDate (val) {
return DateTime.fromISO(val).toFormat(userStore.preferredDateFormat)
}
async function performSearch () {
siteStore.searchIsLoading = true
try {
const resp = await APOLLO_CLIENT.query({
query: gql`
query searchPages (
$siteId: UUID!
$query: String!
$path: String
$locale: [String]
$tags: [String]
$editor: String
$publishState: PagePublishState
$orderBy: PageSearchSort
$orderByDirection: OrderByDirection
$offset: Int
$limit: Int
) {
searchPages(
siteId: $siteId
query: $query
path: $path
locale: $locale
tags: $tags
editor: $editor
publishState: $publishState
orderBy: $orderBy
orderByDirection: $orderByDirection
offset: $offset
limit: $limit
) {
results {
id
path
locale
title
description
icon
updatedAt
relevancy
highlight
}
totalHits
}
}
`,
variables: {
siteId: siteStore.id,
query: siteStore.search
},
fetchPolicy: 'network-only'
})
if (!resp?.data?.searchPages) {
throw new Error('Unexpected error')
}
state.results = cloneDeep(resp.data.searchPages.results)
state.total = resp.data.searchPages.totalHits
} catch (err) {
$q.notify({
type: 'negative',
message: 'Failed to perform search query.',
caption: err.message
})
}
siteStore.searchIsLoading = false
}
// MOUNTED
onMounted(() => {
if (siteStore.search) {
// performSearch()
} else {
siteStore.searchIsLoading = false
}
})
</script>
@ -298,10 +382,14 @@ onMounted(() => {
}
.text-highlight {
font-style: italic;
> b {
background-color: rgba($yellow-7, .5);
padding: 0 3px;
border-radius: 3px;
}
}
.q-page {
flex: 1 1;

Write Preview
Loading…
Cancel
Save