import { Model } from 'objection' import { find, get, has, initial, isEmpty, isString, last, pick } from 'lodash-es' import { Type as JSBinType } from 'js-binary' import { getDictNameFromLocale } from '../helpers/common.mjs' import { generateHash, getFileExtension, injectPageMetadata } from '../helpers/page.mjs' import path from 'node:path' import fse from 'fs-extra' import yaml from 'js-yaml' import striptags from 'striptags' import emojiRegex from 'emoji-regex' import he from 'he' import CleanCSS from 'clean-css' import TurndownService from 'turndown' import { gfm as turndownPluginGfm } from '@joplin/turndown-plugin-gfm' import * as cheerio from 'cheerio' import matter from 'gray-matter' import { PageLink } from './pageLinks.mjs' import { User } from './users.mjs' const pageRegex = /^[a-zA-Z0-9-_/]*$/ const aliasRegex = /^[a-zA-Z0-9-_]*$/ const frontmatterRegex = { html: /^()?(?:\n|\r)*([\w\W]*)*/ } /** * Pages model */ export class Page extends Model { static get tableName () { return 'pages' } static get jsonSchema () { return { type: 'object', required: ['path', 'title'], properties: { id: { type: 'string' }, path: { type: 'string' }, hash: { type: 'string' }, title: { type: 'string' }, description: { type: 'string' }, publishState: { type: 'string' }, publishStartDate: { type: 'string' }, publishEndDate: { type: 'string' }, content: { type: 'string' }, contentType: { type: 'string' }, render: { type: 'string' }, siteId: { type: 'string' }, createdAt: { type: 'string' }, updatedAt: { type: 'string' } } } } static get jsonAttributes () { return ['config', 'historyData', 'relations', 'scripts', 'toc'] } static get relationMappings () { return { // tags: { // relation: Model.ManyToManyRelation, // modelClass: Tag, // join: { // from: 'pages.id', // through: { // from: 'pageTags.pageId', // to: 'pageTags.tagId' // }, // to: 'tags.id' // } // }, links: { relation: Model.HasManyRelation, modelClass: PageLink, join: { from: 'pages.id', to: 'pageLinks.pageId' } }, author: { relation: Model.BelongsToOneRelation, modelClass: User, join: { from: 'pages.authorId', to: 'users.id' } }, creator: { relation: Model.BelongsToOneRelation, modelClass: User, join: { from: 'pages.creatorId', to: 'users.id' } } } } $beforeUpdate () { this.updatedAt = new Date().toISOString() } $beforeInsert () { this.createdAt = new Date().toISOString() this.updatedAt = new Date().toISOString() } /** * Solving the violates foreign key constraint using cascade strategy * using static hooks * @see https://vincit.github.io/objection.js/api/types/#type-statichookarguments */ static async beforeDelete ({ asFindQuery }) { const page = await asFindQuery().select('id') await WIKI.db.comments.query().delete().where('pageId', page[0].id) } /** * Cache Schema */ static get cacheSchema () { return new JSBinType({ id: 'string', authorId: 'string', authorName: 'string', createdAt: 'string', creatorId: 'string', creatorName: 'string', description: 'string', editor: 'string', publishState: 'string', publishEndDate: 'string', publishStartDate: 'string', render: 'string', siteId: 'string', tags: [ { tag: 'string' } ], extra: { js: 'string', css: 'string' }, title: 'string', toc: 'string', updatedAt: 'string' }) } /** * Inject page metadata into contents * * @returns {string} Page Contents with Injected Metadata */ injectMetadata () { return injectPageMetadata(this) } /** * Get the page's file extension based on content type * * @returns {string} File Extension */ getFileExtension () { return getFileExtension(this.contentType) } /** * Parse injected page metadata from raw content * * @param {String} raw Raw file contents * @param {String} contentType Content Type * @returns {Object} Parsed Page Metadata with Raw Content */ static parseMetadata (raw, contentType) { try { switch (contentType) { case 'markdown': { const result = matter(raw) if (!result?.isEmpty) { return { content: result.content, ...result.data } } break } case 'html': { const result = frontmatterRegex.html.exec(raw) if (result[2]) { return { ...yaml.safeLoad(result[2]), content: result[3] } } break } } } catch (err) { WIKI.logger.warn('Failed to parse page metadata. Invalid syntax.') } return { content: raw } } /** * Create a New Page * * @param {Object} opts Page Properties * @returns {Promise} Promise of the Page Model Instance */ static async createPage (opts) { // -> Validate site if (!WIKI.sites[opts.siteId]) { throw new Error('ERR_INVALID_SITE') } // -> Remove trailing slash if (opts.path.endsWith('/')) { opts.path = opts.path.slice(0, -1) } // -> Remove starting slash if (opts.path.startsWith('/')) { opts.path = opts.path.slice(1) } // -> Validate path if (!pageRegex.test(opts.path)) { throw new Error('ERR_INVALID_PATH') } opts.path = opts.path.toLowerCase() // const dotPath = opts.path.replaceAll('/', '.').replaceAll('-', '_') // -> Check for page access if (!WIKI.auth.checkAccess(opts.user, ['write:pages'], { locale: opts.locale, path: opts.path })) { throw new Error('ERR_FORBIDDEN') } // -> Check for duplicate const dupCheck = await WIKI.db.pages.query().findOne({ siteId: opts.siteId, locale: opts.locale, path: opts.path }).select('id') if (dupCheck) { throw new Error('ERR_PAGE_DUPLICATE_PATH') } // -> Check for alias if (opts.alias) { if (!aliasRegex.test(opts.alias)) { throw new Error('ERR_PAGE_INVALID_ALIAS') } const dupAliasCheck = await WIKI.db.pages.query().findOne({ siteId: opts.siteId, alias: opts.alias }).select('id') if (dupAliasCheck) { throw new Error('ERR_PAGE_DUPLICATE_ALIAS') } } // -> Check for empty content if (!opts.content || opts.content.trim().length < 1) { throw new WIKI.Error.PageEmptyContent() } // -> Format CSS Scripts let scriptCss = '' if (WIKI.auth.checkAccess(opts.user, ['write:styles'], { locale: opts.locale, path: opts.path })) { if (!isEmpty(opts.scriptCss)) { scriptCss = new CleanCSS({ inline: false }).minify(opts.scriptCss).styles } else { scriptCss = '' } } // -> Format JS Scripts let scriptJsLoad = '' let scriptJsUnload = '' if (WIKI.auth.checkAccess(opts.user, ['write:scripts'], { locale: opts.locale, path: opts.path })) { scriptJsLoad = opts.scriptJsLoad || '' scriptJsUnload = opts.scriptJsUnload || '' } // -> Get Tags let tags = [] if (opts.tags && opts.tags.length > 0) { tags = await WIKI.db.tags.processNewTags(opts.tags, opts.siteId) } // -> Create page const page = await WIKI.db.pages.query().insert({ alias: opts.alias, authorId: opts.user.id, content: opts.content, creatorId: opts.user.id, config: { allowComments: opts.allowComments ?? true, allowContributions: opts.allowContributions ?? true, allowRatings: opts.allowRatings ?? true, showSidebar: opts.showSidebar ?? true, showTags: opts.showTags ?? true, showToc: opts.showToc ?? true, tocDepth: opts.tocDepth ?? WIKI.sites[opts.siteId].config?.defaults.tocDepth }, contentType: WIKI.data.editors[opts.editor]?.contentType ?? 'text', description: opts.description, editor: opts.editor, hash: generateHash({ path: opts.path, locale: opts.locale }), icon: opts.icon, isBrowsable: opts.isBrowsable ?? true, isSearchable: opts.isSearchable ?? true, locale: opts.locale, ownerId: opts.user.id, path: opts.path, publishState: opts.publishState, publishEndDate: opts.publishEndDate?.toISO(), publishStartDate: opts.publishStartDate?.toISO(), relations: opts.relations ?? [], siteId: opts.siteId, tags, title: opts.title, toc: '[]', scripts: JSON.stringify({ jsLoad: scriptJsLoad, jsUnload: scriptJsUnload, css: scriptCss }) }).returning('*') // -> Render page to HTML await WIKI.db.pages.renderPage(page) // -> Add to tree const pathParts = page.path.split('/') await WIKI.db.tree.addPage({ id: page.id, parentPath: initial(pathParts).join('/'), fileName: last(pathParts), locale: page.locale, title: page.title, tags, meta: { authorId: page.authorId, contentType: page.contentType, creatorId: page.creatorId, description: page.description, isBrowsable: page.isBrowsable, ownerId: page.ownerId, publishState: page.publishState, publishEndDate: page.publishEndDate, publishStartDate: page.publishStartDate }, siteId: page.siteId }) // -> Update search vector WIKI.db.pages.updatePageSearchVector({ id: page.id }) // // -> Add to Storage // if (!opts.skipStorage) { // await WIKI.db.storage.pageEvent({ // event: 'created', // page // }) // } // // -> Reconnect Links // await WIKI.db.pages.reconnectLinks({ // locale: page.locale, // path: page.path, // mode: 'create' // }) // -> Get latest updatedAt page.updatedAt = await WIKI.db.pages.query().findById(page.id).select('updatedAt').then(r => r.updatedAt) return page } /** * Update an Existing Page * * @param {Object} opts Page Properties * @returns {Promise} Promise of the Page Model Instance */ static async updatePage (opts) { // -> Fetch original page const ogPage = await WIKI.db.pages.query().findById(opts.id) if (!ogPage) { throw new Error('ERR_PAGE_NOT_FOUND') } // -> Check for page access if (!WIKI.auth.checkAccess(opts.user, ['write:pages'], { locale: ogPage.locale, path: ogPage.path })) { throw new Error('ERR_PAGE_UPDATE_FORBIDDEN') } const patch = {} const historyData = { action: 'updated', reason: opts.reasonForChange, affectedFields: [] } let shouldUpdateSearch = false // -> Create version snapshot await WIKI.db.pageHistory.addVersion(ogPage) // -> Basic fields if ('title' in opts.patch) { patch.title = opts.patch.title.trim() historyData.affectedFields.push('title') shouldUpdateSearch = true if (patch.title.length < 1) { throw new Error('ERR_PAGE_TITLE_MISSING') } } if ('description' in opts.patch) { patch.description = opts.patch.description.trim() historyData.affectedFields.push('description') shouldUpdateSearch = true } if ('icon' in opts.patch) { patch.icon = opts.patch.icon.trim() historyData.affectedFields.push('icon') } if ('alias' in opts.patch) { patch.alias = opts.patch.alias.trim() historyData.affectedFields.push('alias') if (patch.alias.length > 255) { throw new Error('ERR_PAGE_ALIAS_TOO_LONG') } else if (!aliasRegex.test(patch.alias)) { throw new Error('ERR_PAGE_INVALID_ALIAS') } else if (patch.alias.length > 0) { const dupAliasCheck = await WIKI.db.pages.query().where({ siteId: ogPage.siteId, alias: patch.alias }).andWhereNot('id', ogPage.id).select('id').first() if (dupAliasCheck) { throw new Error('ERR_PAGE_DUPLICATE_ALIAS') } } } if ('content' in opts.patch && opts.patch.content) { patch.content = opts.patch.content historyData.affectedFields.push('content') shouldUpdateSearch = true } // -> Publish State if (opts.patch.publishState) { patch.publishState = opts.patch.publishState historyData.affectedFields.push('publishState') if (patch.publishState === 'scheduled' && (!opts.patch.publishStartDate || !opts.patch.publishEndDate)) { throw new Error('ERR_PAGE_MISSING_SCHEDULED_DATES') } } if (opts.patch.publishStartDate) { patch.publishStartDate = opts.patch.publishStartDate historyData.affectedFields.push('publishStartDate') } if (opts.patch.publishEndDate) { patch.publishEndDate = opts.patch.publishEndDate historyData.affectedFields.push('publishEndDate') } // -> Browsable / Searchable Flags if ('isBrowsable' in opts.patch) { patch.isBrowsable = opts.patch.isBrowsable historyData.affectedFields.push('isBrowsable') } if ('isSearchable' in opts.patch) { patch.isSearchable = opts.patch.isSearchable historyData.affectedFields.push('isSearchable') } // -> Page Config if ('allowComments' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, allowComments: opts.patch.allowComments } historyData.affectedFields.push('allowComments') } if ('allowContributions' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, allowContributions: opts.patch.allowContributions } historyData.affectedFields.push('allowContributions') } if ('allowRatings' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, allowRatings: opts.patch.allowRatings } historyData.affectedFields.push('allowRatings') } if ('showSidebar' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, showSidebar: opts.patch.showSidebar } historyData.affectedFields.push('showSidebar') } if ('showTags' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, showTags: opts.patch.showTags } historyData.affectedFields.push('showTags') } if ('showToc' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, showToc: opts.patch.showToc } historyData.affectedFields.push('showToc') } if ('tocDepth' in opts.patch) { patch.config = { ...patch.config ?? ogPage.config ?? {}, tocDepth: opts.patch.tocDepth } historyData.affectedFields.push('tocDepth') if (patch.config.tocDepth?.min < 1 || patch.config.tocDepth?.min > 6) { throw new Error('ERR_PAGE_INVALID_TOC_DEPTH') } if (patch.config.tocDepth?.max < 1 || patch.config.tocDepth?.max > 6) { throw new Error('ERR_PAGE_INVALID_TOC_DEPTH') } } // -> Relations if ('relations' in opts.patch) { patch.relations = opts.patch.relations.map(r => { if (r.label.length < 1) { throw new Error('ERR_PAGE_RELATION_LABEL_MISSING') } else if (r.label.length > 255) { throw new Error('ERR_PAGE_RELATION_LABEL_TOOLONG') } else if (r.icon.length > 255) { throw new Error('ERR_PAGE_RELATION_ICON_INVALID') } else if (r.target.length > 1024) { throw new Error('ERR_PAGE_RELATION_TARGET_INVALID') } return r }) historyData.affectedFields.push('relations') } // -> Format CSS Scripts if (opts.patch.scriptCss) { if (WIKI.auth.checkAccess(opts.user, ['write:styles'], { locale: ogPage.locale, path: ogPage.path })) { patch.scripts = { ...patch.scripts ?? ogPage.scripts ?? {}, css: !isEmpty(opts.patch.scriptCss) ? new CleanCSS({ inline: false }).minify(opts.patch.scriptCss).styles : '' } historyData.affectedFields.push('scripts.css') } } // -> Format JS Scripts if (opts.patch.scriptJsLoad) { if (WIKI.auth.checkAccess(opts.user, ['write:scripts'], { locale: ogPage.locale, path: ogPage.path })) { patch.scripts = { ...patch.scripts ?? ogPage.scripts ?? {}, jsLoad: opts.patch.scriptJsLoad ?? '' } historyData.affectedFields.push('scripts.jsLoad') } } if (opts.patch.scriptJsUnload) { if (WIKI.auth.checkAccess(opts.user, ['write:scripts'], { locale: ogPage.locale, path: ogPage.path })) { patch.scripts = { ...patch.scripts ?? ogPage.scripts ?? {}, jsUnload: opts.patch.scriptJsUnload ?? '' } historyData.affectedFields.push('scripts.jsUnload') } } // -> Tags if ('tags' in opts.patch) { patch.tags = await WIKI.db.tags.processNewTags(opts.patch.tags, ogPage.siteId) historyData.affectedFields.push('tags') } // -> Update page await WIKI.db.pages.query().patch({ ...patch, authorId: opts.user.id, historyData }).where('id', ogPage.id) const page = await WIKI.db.pages.getPageFromDb(ogPage.id) // -> Render page to HTML if (opts.patch.content) { await WIKI.db.pages.renderPage(page) } WIKI.events.outbound.emit('deletePageFromCache', page.hash) // -> Update tree await WIKI.db.knex('tree').where('id', page.id).update({ title: page.title, tags: page.tags, meta: { authorId: page.authorId, contentType: page.contentType, creatorId: page.creatorId, description: page.description, isBrowsable: page.isBrowsable, ownerId: page.ownerId, publishState: page.publishState, publishEndDate: page.publishEndDate, publishStartDate: page.publishStartDate }, updatedAt: page.updatedAt }) // -> Update search vector if (shouldUpdateSearch) { WIKI.db.pages.updatePageSearchVector({ id: page.id }) } // -> Update on Storage // if (!opts.skipStorage) { // await WIKI.db.storage.pageEvent({ // event: 'updated', // page // }) // } // -> Get latest updatedAt page.updatedAt = await WIKI.db.pages.query().findById(page.id).select('updatedAt').then(r => r.updatedAt) return page } /** * Update a page text search vector value * * @param {Object} opts - Options * @param {string} [opts.id] - Page ID to update (fetch from DB) * @param {Object} [opts.page] - Page object to update (use directly) */ static async updatePageSearchVector ({ id, page }) { if (!page) { if (!id) { throw new Error('Must provide either the page ID or the page object.') } page = await WIKI.db.pages.query().findById(id).select('id', 'locale', 'render', 'password') } // -> Exclude password-protected content from being indexed const safeContent = page.password ? '' : WIKI.db.pages.cleanHTML(page.render) const dictName = getDictNameFromLocale(page.locale) return WIKI.db.knex('pages').where('id', page.id).update({ searchContent: safeContent, ts: WIKI.db.knex.raw(` setweight(to_tsvector('${dictName}', coalesce(title,'')), 'A') || setweight(to_tsvector('${dictName}', coalesce(description,'')), 'B') || setweight(to_tsvector('${dictName}', coalesce(?,'')), 'C')`, [safeContent]) }) } /** * Refresh Autocomplete Index */ static async refreshAutocompleteIndex () { await WIKI.db.knex('autocomplete').truncate() await WIKI.db.knex.raw(` INSERT INTO "autocomplete" (word) SELECT word FROM ts_stat( 'SELECT to_tsvector(''simple'', "title") || to_tsvector(''simple'', "description") || to_tsvector(''simple'', "searchContent") FROM "pages" WHERE "isSearchableComputed" IS TRUE' ) `) } /** * Convert an Existing Page * * @param {Object} opts Page Properties * @returns {Promise} Promise of the Page Model Instance */ static async convertPage (opts) { // -> Fetch original page const ogPage = await WIKI.db.pages.query().findById(opts.id) if (!ogPage) { throw new Error('Invalid Page Id') } if (ogPage.editor === opts.editor) { throw new Error('Page is already using this editor. Nothing to convert.') } // -> Check for page access if (!WIKI.auth.checkAccess(opts.user, ['write:pages'], { locale: ogPage.locale, path: ogPage.path })) { throw new WIKI.Error.PageUpdateForbidden() } // -> Check content type const sourceContentType = ogPage.contentType const targetContentType = get(find(WIKI.data.editors, ['key', opts.editor]), 'contentType', 'text') const shouldConvert = sourceContentType !== targetContentType let convertedContent = null // -> Convert content if (shouldConvert) { // -> Markdown => HTML if (sourceContentType === 'markdown' && targetContentType === 'html') { if (!ogPage.render) { throw new Error('Aborted conversion because rendered page content is empty!') } convertedContent = ogPage.render const $ = cheerio.load(convertedContent, { decodeEntities: true }) if ($.root().children().length > 0) { // Remove header anchors $('.toc-anchor').remove() // Attempt to convert tabsets $('tabset').each((tabI, tabElm) => { const tabHeaders = [] // -> Extract templates $(tabElm).children('template').each((tmplI, tmplElm) => { if ($(tmplElm).attr('v-slot:tabs') === '') { $(tabElm).before('