feat: rendering pipeline logic

pull/662/head
Nicolas Giard 6 years ago
parent 0b93544677
commit 09d1f580d9

@ -92,7 +92,7 @@
v-btn(icon, slot='activator')
v-icon(color='grey') account_circle
span Account
v-list.py-0
v-list.py-0(:light='!$vuetify.dark')
v-list-tile.py-3(avatar)
v-list-tile-avatar
v-avatar.red(:size='40'): span.white--text.subheading JD

@ -13,7 +13,7 @@ module.exports = {
LocalizationQuery: {
async locales(obj, args, context, info) {
let remoteLocales = await WIKI.redis.get('locales')
let localLocales = await WIKI.models.locales.query().select('id', 'code', 'isRTL', 'name', 'nativeName', 'createdAt', 'updatedAt')
let localLocales = await WIKI.models.locales.query().select('code', 'isRTL', 'name', 'nativeName', 'createdAt', 'updatedAt')
remoteLocales = (remoteLocales) ? JSON.parse(remoteLocales) : localLocales
return _.map(remoteLocales, rl => {
let isInstalled = _.some(localLocales, ['code', rl.code])

@ -1,16 +1,30 @@
require('../core/worker')
const _ = require('lodash')
/* global WIKI */
WIKI.models = require('../core/db').init()
module.exports = async (job) => {
WIKI.logger.info(`Rendering page ${job.data.path}...`)
WIKI.logger.info(`Rendering page ${job.data.page.path}...`)
try {
WIKI.logger.info(`Rendering page ${job.data.path}: [ COMPLETED ]`)
let output = job.data.page.content
for (let core of job.data.pipeline) {
const renderer = require(`../modules/rendering/${_.kebabCase(core.key)}/renderer.js`)
output = await renderer.render.call({
config: core.config,
children: core.children,
page: job.data.page,
input: output
})
}
console.info(output)
WIKI.logger.info(`Rendering page ${job.data.page.path}: [ COMPLETED ]`)
} catch (err) {
WIKI.logger.error(`Rendering page ${job.data.path}: [ FAILED ]`)
WIKI.logger.error(`Rendering page ${job.data.page.path}: [ FAILED ]`)
WIKI.logger.error(err.message)
}
}

@ -111,7 +111,6 @@ module.exports = class Page extends Model {
}
static async createPage(opts) {
await WIKI.models.pages.renderPage(opts)
const page = await WIKI.models.pages.query().insertAndFetch({
authorId: opts.authorId,
content: opts.content,
@ -127,6 +126,7 @@ module.exports = class Page extends Model {
publishStartDate: opts.publishStartDate,
title: opts.title
})
await WIKI.models.pages.renderPage(page)
await WIKI.models.storage.pageEvent({
event: 'created',
page
@ -149,6 +149,7 @@ module.exports = class Page extends Model {
publishStartDate: opts.publishStartDate,
title: opts.title
})
await WIKI.models.pages.renderPage(page)
await WIKI.models.storage.pageEvent({
event: 'updated',
page
@ -156,8 +157,12 @@ module.exports = class Page extends Model {
return page
}
static async renderPage(opts) {
WIKI.queue.job.renderPage.add(opts, {
static async renderPage(page) {
const pipeline = await WIKI.models.renderers.getRenderingPipeline(page.contentType)
WIKI.queue.job.renderPage.add({
page,
pipeline
}, {
removeOnComplete: true,
removeOnFail: true
})

@ -3,6 +3,7 @@ const path = require('path')
const fs = require('fs-extra')
const _ = require('lodash')
const yaml = require('js-yaml')
const DepGraph = require('dependency-graph').DepGraph
const commonHelper = require('../helpers/common')
/* global WIKI */
@ -37,10 +38,10 @@ module.exports = class Renderer extends Model {
const dbRenderers = await WIKI.models.renderers.query()
// -> Fetch definitions from disk
const rendererDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/renderer'))
const rendererDirs = await fs.readdir(path.join(WIKI.SERVERPATH, 'modules/rendering'))
let diskRenderers = []
for (let dir of rendererDirs) {
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/renderer', dir, 'definition.yml'), 'utf8')
const def = await fs.readFile(path.join(WIKI.SERVERPATH, 'modules/rendering', dir, 'definition.yml'), 'utf8')
diskRenderers.push(yaml.safeLoad(def))
}
WIKI.data.renderers = diskRenderers.map(renderer => ({
@ -91,18 +92,66 @@ module.exports = class Renderer extends Model {
}
}
static async pageEvent({ event, page }) {
const targets = await WIKI.models.storage.query().where('isEnabled', true)
if (targets && targets.length > 0) {
_.forEach(targets, target => {
WIKI.queue.job.syncStorage.add({
event,
target,
page
}, {
removeOnComplete: true
static async getRenderingPipeline(contentType) {
const renderersDb = await WIKI.models.renderers.query().where('isEnabled', true)
if (renderersDb && renderersDb.length > 0) {
const renderers = renderersDb.map(rdr => {
const renderer = _.find(WIKI.data.renderers, ['key', rdr.key])
return {
...renderer,
config: rdr.config
}
})
// Build tree
const rawCores = _.filter(renderers, renderer => !_.has(renderer, 'dependsOn')).map(core => {
core.children = _.concat([_.cloneDeep(core)], _.filter(renderers, ['dependsOn', core.key]))
return core
})
// Build dependency graph
const graph = new DepGraph({ circular: true })
rawCores.map(core => { graph.addNode(core.key) })
rawCores.map(core => {
rawCores.map(coreTarget => {
if (core.key !== coreTarget.key) {
if (core.output === coreTarget.input) {
graph.addDependency(core.key, coreTarget.key)
}
}
})
})
// Filter unused cores
let activeCoreKeys = _.filter(rawCores, ['input', contentType]).map(core => core.key)
_.clone(activeCoreKeys).map(coreKey => {
activeCoreKeys = _.union(activeCoreKeys, graph.dependenciesOf(coreKey))
})
const activeCores = _.filter(rawCores, core => _.includes(activeCoreKeys, core.key))
// Rebuild dependency graph with active cores
const graphActive = new DepGraph({ circular: true })
activeCores.map(core => { graphActive.addNode(core.key) })
activeCores.map(core => {
activeCores.map(coreTarget => {
if (core.key !== coreTarget.key) {
if (core.output === coreTarget.input) {
graphActive.addDependency(core.key, coreTarget.key)
}
}
})
})
// Reorder cores in reverse dependency order
let orderedCores = []
_.reverse(graphActive.overallOrder()).map(coreKey => {
orderedCores.push(_.find(rawCores, ['key', coreKey]))
})
return orderedCores
} else {
WIKI.logger.error(`Rendering pipeline is empty!`)
return false
}
}
}

@ -1,8 +0,0 @@
key: htmlSecurity
title: Security
description: Filter and strips potentially dangerous content
author: requarks.io
icon: whatshot
enabledDefault: true
dependsOn: htmlCore
props: {}

@ -1,31 +0,0 @@
key: markdownCore
title: Core
description: Basic Markdown Parser
author: requarks.io
input: markdown
output: html
icon: crop_free
props:
linkify:
type: Boolean
default: true
title: Automatically convert links
hint: Links will automatically be converted to clickable links.
linebreaks:
type: Boolean
default: true
title: Automatically convert line breaks
hint: Add linebreaks within paragraphs.
highlightCode:
type: Boolean
default: true
title: Highlight code blocks
hint: Add syntax coloring to code blocks.
codeTheme:
type: String
default: light
title: Code Color Theme
hint: Color theme for code blocks
enum:
- light
- dark

@ -0,0 +1,5 @@
module.exports = {
async render() {
return this.input
}
}

@ -0,0 +1,18 @@
key: htmlSecurity
title: Security
description: Filter and strips potentially dangerous content
author: requarks.io
icon: whatshot
enabledDefault: true
dependsOn: htmlCore
props:
stripJS:
type: Boolean
title: Strip Javascript
default: false
hint: Javascript code within code blocks won't be affected
filterBadWords:
type: Boolean
title: Filter Bad Words
default: false
hint: Replace bad words with asterisks

@ -5,10 +5,6 @@ const mdAbbr = require('markdown-it-abbr')
// ------------------------------------
module.exports = {
key: 'markdown/abbreviations',
title: 'Abbreviations',
dependsOn: [],
props: [],
init (md, conf) {
md.use(mdAbbr)
}

@ -0,0 +1,51 @@
key: markdownCore
title: Core
description: Basic Markdown Parser
author: requarks.io
input: markdown
output: html
icon: crop_free
props:
allowHTML:
type: Boolean
default: true
title: Allow HTML
hint: Enable HTML tags in content
linkify:
type: Boolean
default: true
title: Automatically convert links
hint: Links will automatically be converted to clickable links.
linebreaks:
type: Boolean
default: true
title: Automatically convert line breaks
hint: Add linebreaks within paragraphs.
typographer:
type: Boolean
default: false
title: Typographer
hint: Enable some language-neutral replacement + quotes beautification
quotes:
type: String
default: English
title: Quotes style
hint: When typographer is enabled. Double + single quotes replacement pairs. e.g. «»„“ for Russian, „“‚‘ for German, etc.
enum:
- Chinese
- English
- French
- German
- Greek
- Japanese
- Hungarian
- Polish
- Portuguese
- Russian
- Spanish
- Swedish
highlightCode:
type: Boolean
default: true
title: Highlight code blocks
hint: Add syntax coloring to code blocks.

@ -0,0 +1,42 @@
const md = require('markdown-it')
// const hljs = require('highlight.js')
const _ = require('lodash')
const quoteStyles = {
Chinese: '””‘’',
English: '“”‘’',
French: ['«\xA0', '\xA0»', '\xA0', '\xA0'],
German: '„“‚‘',
Greek: '«»‘’',
Japanese: '「」「」',
Hungarian: '„”’’',
Polish: '„”‚‘',
Portuguese: '«»‘’',
Russian: '«»„“',
Spanish: '«»‘’',
Swedish: '””’’'
}
module.exports = {
async render() {
const mkdown = md({
html: this.config.allowHTML,
breaks: this.config.linebreaks,
linkify: this.config.linkify,
typographer: this.config.typographer,
quotes: _.get(quoteStyles, this.config.quotes, quoteStyles.English),
highlight(str, lang) {
// if (this.config.highlightCode && lang && hljs.getLanguage(lang)) {
// try {
// return '<pre class="hljs"><code>' + hljs.highlight(lang, str, true).value + '</code></pre>'
// } catch (err) {
// return '<pre><code>' + _.escape(str) + '</code></pre>'
// }
// }
return '<pre><code>' + _.escape(str) + '</code></pre>'
}
})
return mkdown.render(this.input)
}
}

@ -5,10 +5,6 @@ const mdEmoji = require('markdown-it-emoji')
// ------------------------------------
module.exports = {
key: 'markdown/emoji',
title: 'Emoji',
dependsOn: [],
props: [],
init (md, conf) {
md.use(mdEmoji)
}

@ -5,4 +5,9 @@ author: requarks.io
icon: space_bar
enabledDefault: true
dependsOn: markdownCore
props: {}
props:
tabWidth:
type: Number
title: Tab Width
hint: Amount of spaces for each tab
default: 4

@ -6,10 +6,6 @@ const _ = require('lodash')
// ------------------------------------
module.exports = {
key: 'markdown/expand-tabs',
title: 'Expand Tabs',
dependsOn: [],
props: ['tabWidth'],
init (md, conf) {
md.use(mdExpandTabs, {
tabWidth: _.toInteger(conf.tabWidth || 4)

@ -5,10 +5,6 @@ const mdFootnote = require('markdown-it-footnote')
// ------------------------------------
module.exports = {
key: 'markdown/footnotes',
title: 'Footnotes',
dependsOn: [],
props: [],
init (md, conf) {
md.use(mdFootnote)
}

@ -5,10 +5,6 @@ const mdMathjax = require('markdown-it-mathjax')()
// ------------------------------------
module.exports = {
key: 'markdown/mathjax',
title: 'Mathjax Preprocessor',
dependsOn: [],
props: [],
init (md, conf) {
md.use(mdMathjax)
}

@ -5,10 +5,6 @@ const mdTaskLists = require('markdown-it-task-lists')
// ------------------------------------
module.exports = {
key: 'markdown/task-lists',
title: 'Task Lists',
dependsOn: [],
props: [],
init (md, conf) {
md.use(mdTaskLists)
}
Loading…
Cancel
Save