fix: cache markdown it instance and properly dispose shiki on config reload (#4321)

This results in over 5x speedup in build times of certain projects. But this comes at the cost of correctness. `createMarkdownRenderer` now ignores any arguments passed by user. But from our GitHub code search we didn't find any user passing options different than their siteConfig to this function. If you are doing that, please open an issue and we can discuss the best way forward.

---
Co-authored-by: Divyansh Singh <40380293+brc-dd@users.noreply.github.com>
pull/4322/head
Estéban 11 months ago committed by GitHub
parent 81fc148198
commit 45968cdc50
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -100,20 +100,20 @@
"dependencies": { "dependencies": {
"@docsearch/css": "^3.6.2", "@docsearch/css": "^3.6.2",
"@docsearch/js": "^3.6.2", "@docsearch/js": "^3.6.2",
"@shikijs/core": "^1.22.0", "@shikijs/core": "^1.22.2",
"@shikijs/transformers": "^1.22.0", "@shikijs/transformers": "^1.22.2",
"@shikijs/types": "^1.22.0", "@shikijs/types": "^1.22.2",
"@types/markdown-it": "^14.1.2", "@types/markdown-it": "^14.1.2",
"@vitejs/plugin-vue": "^5.1.4", "@vitejs/plugin-vue": "^5.1.4",
"@vue/devtools-api": "^7.4.6", "@vue/devtools-api": "^7.5.4",
"@vue/shared": "^3.5.12", "@vue/shared": "^3.5.12",
"@vueuse/core": "^11.1.0", "@vueuse/core": "^11.1.0",
"@vueuse/integrations": "^11.1.0", "@vueuse/integrations": "^11.1.0",
"focus-trap": "^7.6.0", "focus-trap": "^7.6.0",
"mark.js": "8.11.1", "mark.js": "8.11.1",
"minisearch": "^7.1.0", "minisearch": "^7.1.0",
"shiki": "^1.22.0", "shiki": "^1.22.2",
"vite": "^5.4.8", "vite": "^5.4.10",
"vue": "^3.5.12" "vue": "^3.5.12"
}, },
"devDependencies": { "devDependencies": {
@ -127,7 +127,7 @@
"@mdit-vue/shared": "^2.1.3", "@mdit-vue/shared": "^2.1.3",
"@polka/compression": "^1.0.0-next.28", "@polka/compression": "^1.0.0-next.28",
"@rollup/plugin-alias": "^5.1.1", "@rollup/plugin-alias": "^5.1.1",
"@rollup/plugin-commonjs": "^28.0.0", "@rollup/plugin-commonjs": "^28.0.1",
"@rollup/plugin-json": "^6.1.0", "@rollup/plugin-json": "^6.1.0",
"@rollup/plugin-node-resolve": "^15.3.0", "@rollup/plugin-node-resolve": "^15.3.0",
"@rollup/plugin-replace": "^6.0.1", "@rollup/plugin-replace": "^6.0.1",
@ -141,7 +141,7 @@
"@types/markdown-it-emoji": "^3.0.1", "@types/markdown-it-emoji": "^3.0.1",
"@types/micromatch": "^4.0.9", "@types/micromatch": "^4.0.9",
"@types/minimist": "^1.2.5", "@types/minimist": "^1.2.5",
"@types/node": "^22.7.5", "@types/node": "^22.8.2",
"@types/postcss-prefix-selector": "^1.16.3", "@types/postcss-prefix-selector": "^1.16.3",
"@types/prompts": "^2.4.9", "@types/prompts": "^2.4.9",
"chokidar": "^3.6.0", "chokidar": "^3.6.0",
@ -149,7 +149,7 @@
"cross-spawn": "^7.0.3", "cross-spawn": "^7.0.3",
"debug": "^4.3.7", "debug": "^4.3.7",
"esbuild": "^0.24.0", "esbuild": "^0.24.0",
"execa": "^9.4.0", "execa": "^9.5.1",
"fs-extra": "^11.2.0", "fs-extra": "^11.2.0",
"get-port": "^7.1.0", "get-port": "^7.1.0",
"gray-matter": "^4.0.3", "gray-matter": "^4.0.3",
@ -164,20 +164,20 @@
"markdown-it-mathjax3": "^4.3.2", "markdown-it-mathjax3": "^4.3.2",
"micromatch": "^4.0.8", "micromatch": "^4.0.8",
"minimist": "^1.2.8", "minimist": "^1.2.8",
"nanoid": "^5.0.7", "nanoid": "^5.0.8",
"ora": "^8.1.0", "ora": "^8.1.0",
"p-map": "^7.0.2", "p-map": "^7.0.2",
"path-to-regexp": "^6.3.0", "path-to-regexp": "^6.3.0",
"picocolors": "^1.1.0", "picocolors": "^1.1.1",
"pkg-dir": "^8.0.0", "pkg-dir": "^8.0.0",
"playwright-chromium": "^1.48.0", "playwright-chromium": "^1.48.2",
"polka": "^1.0.0-next.28", "polka": "^1.0.0-next.28",
"postcss-prefix-selector": "^2.1.0", "postcss-prefix-selector": "^2.1.0",
"prettier": "^3.3.3", "prettier": "^3.3.3",
"prompts": "^2.4.2", "prompts": "^2.4.2",
"punycode": "^2.3.1", "punycode": "^2.3.1",
"rimraf": "^6.0.1", "rimraf": "^6.0.1",
"rollup": "^4.24.0", "rollup": "^4.24.2",
"rollup-plugin-dts": "^6.1.1", "rollup-plugin-dts": "^6.1.1",
"rollup-plugin-esbuild": "^6.1.1", "rollup-plugin-esbuild": "^6.1.1",
"semver": "^7.6.3", "semver": "^7.6.3",
@ -185,10 +185,10 @@
"sirv": "^3.0.0", "sirv": "^3.0.0",
"sitemap": "^8.0.0", "sitemap": "^8.0.0",
"supports-color": "^9.4.0", "supports-color": "^9.4.0",
"tinyglobby": "^0.2.9", "tinyglobby": "^0.2.10",
"typescript": "^5.6.3", "typescript": "^5.6.3",
"vitest": "^2.1.2", "vitest": "^2.1.4",
"vue-tsc": "^2.1.6", "vue-tsc": "^2.1.8",
"wait-on": "^8.0.1" "wait-on": "^8.0.1"
}, },
"peerDependencies": { "peerDependencies": {
@ -203,7 +203,7 @@
"optional": true "optional": true
} }
}, },
"packageManager": "pnpm@9.12.1", "packageManager": "pnpm@9.12.3",
"pnpm": { "pnpm": {
"peerDependencyRules": { "peerDependencyRules": {
"ignoreMissing": [ "ignoreMissing": [

File diff suppressed because it is too large Load Diff

@ -4,10 +4,7 @@ import { glob, type GlobOptions } from 'tinyglobby'
import type { SiteConfig } from './config' import type { SiteConfig } from './config'
import matter from 'gray-matter' import matter from 'gray-matter'
import { normalizePath } from 'vite' import { normalizePath } from 'vite'
import { import { createMarkdownRenderer } from './markdown/markdown'
createMarkdownRenderer,
type MarkdownRenderer
} from './markdown/markdown'
export interface ContentOptions<T = ContentData[]> { export interface ContentOptions<T = ContentData[]> {
/** /**
@ -100,15 +97,7 @@ export function createContentLoader<T = ContentData[]>(
if (typeof pattern === 'string') pattern = [pattern] if (typeof pattern === 'string') pattern = [pattern]
pattern = pattern.map((p) => normalizePath(path.join(config.srcDir, p))) pattern = pattern.map((p) => normalizePath(path.join(config.srcDir, p)))
let md: MarkdownRenderer const cache = new Map<string, { data: any; timestamp: number }>()
const cache = new Map<
string,
{
data: any
timestamp: number
}
>()
return { return {
watch: pattern, watch: pattern,
@ -124,14 +113,12 @@ export function createContentLoader<T = ContentData[]>(
).sort() ).sort()
} }
md = const md = await createMarkdownRenderer(
md || config.srcDir,
(await createMarkdownRenderer( config.markdown,
config.srcDir, config.site.base,
config.markdown, config.logger
config.site.base, )
config.logger
))
const raw: ContentData[] = [] const raw: ContentData[] = []

@ -28,7 +28,7 @@ import type {
import type { Logger } from 'vite' import type { Logger } from 'vite'
import { containerPlugin, type ContainerOptions } from './plugins/containers' import { containerPlugin, type ContainerOptions } from './plugins/containers'
import { gitHubAlertsPlugin } from './plugins/githubAlerts' import { gitHubAlertsPlugin } from './plugins/githubAlerts'
import { highlight } from './plugins/highlight' import { highlight as createHighlighter } from './plugins/highlight'
import { highlightLinePlugin } from './plugins/highlightLines' import { highlightLinePlugin } from './plugins/highlightLines'
import { imagePlugin, type Options as ImageOptions } from './plugins/image' import { imagePlugin, type Options as ImageOptions } from './plugins/image'
import { lineNumberPlugin } from './plugins/lineNumbers' import { lineNumberPlugin } from './plugins/lineNumbers'
@ -173,7 +173,7 @@ export interface MarkdownOptions extends Options {
*/ */
container?: ContainerOptions container?: ContainerOptions
/** /**
* Math support (experimental) * Math support
* *
* You need to install `markdown-it-mathjax3` and set `math` to `true` to enable it. * You need to install `markdown-it-mathjax3` and set `math` to `true` to enable it.
* You can also pass options to `markdown-it-mathjax3` here. * You can also pass options to `markdown-it-mathjax3` here.
@ -192,22 +192,38 @@ export interface MarkdownOptions extends Options {
export type MarkdownRenderer = MarkdownIt export type MarkdownRenderer = MarkdownIt
export const createMarkdownRenderer = async ( let md: MarkdownRenderer | undefined
let _disposeHighlighter: (() => void) | undefined
export function disposeMdItInstance() {
if (md) {
md = undefined
_disposeHighlighter?.()
}
}
/**
* @experimental
*/
export async function createMarkdownRenderer(
srcDir: string, srcDir: string,
options: MarkdownOptions = {}, options: MarkdownOptions = {},
base = '/', base = '/',
logger: Pick<Logger, 'warn'> = console logger: Pick<Logger, 'warn'> = console
): Promise<MarkdownRenderer> => { ): Promise<MarkdownRenderer> {
if (md) return md
const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' } const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' }
const codeCopyButtonTitle = options.codeCopyButtonTitle || 'Copy Code' const codeCopyButtonTitle = options.codeCopyButtonTitle || 'Copy Code'
const hasSingleTheme = typeof theme === 'string' || 'name' in theme const hasSingleTheme = typeof theme === 'string' || 'name' in theme
const md = MarkdownIt({ let [highlight, dispose] = options.highlight
html: true, ? [options.highlight, () => {}]
linkify: true, : await createHighlighter(theme, options, logger)
highlight: options.highlight || (await highlight(theme, options, logger)),
...options _disposeHighlighter = dispose
})
md = MarkdownIt({ html: true, linkify: true, highlight, ...options })
md.linkify.set({ fuzzyLink: false }) md.linkify.set({ fuzzyLink: false })
md.use(restoreEntities) md.use(restoreEntities)

@ -23,7 +23,7 @@ const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
* 2. convert line numbers into line options: * 2. convert line numbers into line options:
* [{ line: number, classes: string[] }] * [{ line: number, classes: string[] }]
*/ */
const attrsToLines = (attrs: string): TransformerCompactLineOption[] => { function attrsToLines(attrs: string): TransformerCompactLineOption[] {
attrs = attrs.replace(/^(?:\[.*?\])?.*?([\d,-]+).*/, '$1').trim() attrs = attrs.replace(/^(?:\[.*?\])?.*?([\d,-]+).*/, '$1').trim()
const result: number[] = [] const result: number[] = []
if (!attrs) { if (!attrs) {
@ -51,7 +51,7 @@ export async function highlight(
theme: ThemeOptions, theme: ThemeOptions,
options: MarkdownOptions, options: MarkdownOptions,
logger: Pick<Logger, 'warn'> = console logger: Pick<Logger, 'warn'> = console
): Promise<(str: string, lang: string, attrs: string) => string> { ): Promise<[(str: string, lang: string, attrs: string) => string, () => void]> {
const { const {
defaultHighlightLang: defaultLang = '', defaultHighlightLang: defaultLang = '',
codeTransformers: userTransformers = [] codeTransformers: userTransformers = []
@ -95,93 +95,96 @@ export async function highlight(
const lineNoRE = /:(no-)?line-numbers(=\d*)?$/ const lineNoRE = /:(no-)?line-numbers(=\d*)?$/
const mustacheRE = /\{\{.*?\}\}/g const mustacheRE = /\{\{.*?\}\}/g
return (str: string, lang: string, attrs: string) => { return [
const vPre = vueRE.test(lang) ? '' : 'v-pre' (str: string, lang: string, attrs: string) => {
lang = const vPre = vueRE.test(lang) ? '' : 'v-pre'
lang lang =
.replace(lineNoStartRE, '') lang
.replace(lineNoRE, '') .replace(lineNoStartRE, '')
.replace(vueRE, '') .replace(lineNoRE, '')
.toLowerCase() || defaultLang .replace(vueRE, '')
.toLowerCase() || defaultLang
if (lang) { if (lang) {
const langLoaded = highlighter.getLoadedLanguages().includes(lang as any) const langLoaded = highlighter.getLoadedLanguages().includes(lang)
if (!langLoaded && !isSpecialLang(lang)) { if (!langLoaded && !isSpecialLang(lang)) {
logger.warn( logger.warn(
c.yellow( c.yellow(
`\nThe language '${lang}' is not loaded, falling back to '${ `\nThe language '${lang}' is not loaded, falling back to '${
defaultLang || 'txt' defaultLang || 'txt'
}' for syntax highlighting.` }' for syntax highlighting.`
)
) )
) lang = defaultLang
lang = defaultLang }
} }
}
const lineOptions = attrsToLines(attrs) const lineOptions = attrsToLines(attrs)
const mustaches = new Map<string, string>() const mustaches = new Map<string, string>()
const removeMustache = (s: string) => { const removeMustache = (s: string) => {
if (vPre) return s if (vPre) return s
return s.replace(mustacheRE, (match) => { return s.replace(mustacheRE, (match) => {
let marker = mustaches.get(match) let marker = mustaches.get(match)
if (!marker) { if (!marker) {
marker = nanoid() marker = nanoid()
mustaches.set(match, marker) mustaches.set(match, marker)
} }
return marker return marker
}) })
} }
const restoreMustache = (s: string) => { const restoreMustache = (s: string) => {
mustaches.forEach((marker, match) => { mustaches.forEach((marker, match) => {
s = s.replaceAll(marker, match) s = s.replaceAll(marker, match)
}) })
return s return s
} }
str = removeMustache(str).trimEnd() str = removeMustache(str).trimEnd()
const highlighted = highlighter.codeToHtml(str, { const highlighted = highlighter.codeToHtml(str, {
lang, lang,
transformers: [ transformers: [
...transformers, ...transformers,
transformerCompactLineOptions(lineOptions), transformerCompactLineOptions(lineOptions),
{ {
name: 'vitepress:v-pre', name: 'vitepress:v-pre',
pre(node) { pre(node) {
if (vPre) node.properties['v-pre'] = '' if (vPre) node.properties['v-pre'] = ''
} }
}, },
{ {
name: 'vitepress:empty-line', name: 'vitepress:empty-line',
code(hast) { code(hast) {
hast.children.forEach((span) => { hast.children.forEach((span) => {
if ( if (
span.type === 'element' && span.type === 'element' &&
span.tagName === 'span' && span.tagName === 'span' &&
Array.isArray(span.properties.class) && Array.isArray(span.properties.class) &&
span.properties.class.includes('line') && span.properties.class.includes('line') &&
span.children.length === 0 span.children.length === 0
) { ) {
span.children.push({ span.children.push({
type: 'element', type: 'element',
tagName: 'wbr', tagName: 'wbr',
properties: {}, properties: {},
children: [] children: []
}) })
} }
}) })
} }
}, },
...userTransformers ...userTransformers
], ],
meta: { __raw: attrs }, meta: { __raw: attrs },
...(typeof theme === 'object' && 'light' in theme && 'dark' in theme ...(typeof theme === 'object' && 'light' in theme && 'dark' in theme
? { themes: theme, defaultColor: false } ? { themes: theme, defaultColor: false }
: { theme }) : { theme })
}) })
return restoreMustache(highlighted) return restoreMustache(highlighted)
} },
highlighter.dispose
]
} }

@ -16,6 +16,7 @@ import {
resolveAliases resolveAliases
} from './alias' } from './alias'
import { resolvePages, resolveUserConfig, type SiteConfig } from './config' import { resolvePages, resolveUserConfig, type SiteConfig } from './config'
import { disposeMdItInstance } from './markdown/markdown'
import { import {
clearCache, clearCache,
createMarkdownToVueRenderFn, createMarkdownToVueRenderFn,
@ -388,6 +389,7 @@ export async function createVitePressPlugin(
return return
} }
disposeMdItInstance()
clearCache() clearCache()
await recreateServer?.() await recreateServer?.()
return return

Loading…
Cancel
Save