fix: cache markdown it instance and properly dispose shiki on config reload (#4321)

This results in over 5x speedup in build times of certain projects. But this comes at the cost of correctness. `createMarkdownRenderer` now ignores any arguments passed by user. But from our GitHub code search we didn't find any user passing options different than their siteConfig to this function. If you are doing that, please open an issue and we can discuss the best way forward. --- Co-authored-by: Divyansh Singh <40380293+brc-dd@users.noreply.github.com>
11 months ago · 45968cdc50
parent 81fc148198
commit 45968cdc50
6 changed files with 469 additions and 423 deletions
--- a/package.json
+++ b/package.json
@ -100,20 +100,20 @@
  "dependencies": {
    "@docsearch/css": "^3.6.2",
    "@docsearch/js": "^3.6.2",
-    "@shikijs/core": "^1.22.0",
-    "@shikijs/transformers": "^1.22.0",
-    "@shikijs/types": "^1.22.0",
+    "@shikijs/core": "^1.22.2",
+    "@shikijs/transformers": "^1.22.2",
+    "@shikijs/types": "^1.22.2",
    "@types/markdown-it": "^14.1.2",
    "@vitejs/plugin-vue": "^5.1.4",
-    "@vue/devtools-api": "^7.4.6",
+    "@vue/devtools-api": "^7.5.4",
    "@vue/shared": "^3.5.12",
    "@vueuse/core": "^11.1.0",
    "@vueuse/integrations": "^11.1.0",
    "focus-trap": "^7.6.0",
    "mark.js": "8.11.1",
    "minisearch": "^7.1.0",
-    "shiki": "^1.22.0",
-    "vite": "^5.4.8",
+    "shiki": "^1.22.2",
+    "vite": "^5.4.10",
    "vue": "^3.5.12"
  },
  "devDependencies": {
@ -127,7 +127,7 @@
    "@mdit-vue/shared": "^2.1.3",
    "@polka/compression": "^1.0.0-next.28",
    "@rollup/plugin-alias": "^5.1.1",
-    "@rollup/plugin-commonjs": "^28.0.0",
+    "@rollup/plugin-commonjs": "^28.0.1",
    "@rollup/plugin-json": "^6.1.0",
    "@rollup/plugin-node-resolve": "^15.3.0",
    "@rollup/plugin-replace": "^6.0.1",
@ -141,7 +141,7 @@
    "@types/markdown-it-emoji": "^3.0.1",
    "@types/micromatch": "^4.0.9",
    "@types/minimist": "^1.2.5",
-    "@types/node": "^22.7.5",
+    "@types/node": "^22.8.2",
    "@types/postcss-prefix-selector": "^1.16.3",
    "@types/prompts": "^2.4.9",
    "chokidar": "^3.6.0",
@ -149,7 +149,7 @@
    "cross-spawn": "^7.0.3",
    "debug": "^4.3.7",
    "esbuild": "^0.24.0",
-    "execa": "^9.4.0",
+    "execa": "^9.5.1",
    "fs-extra": "^11.2.0",
    "get-port": "^7.1.0",
    "gray-matter": "^4.0.3",
@ -164,20 +164,20 @@
    "markdown-it-mathjax3": "^4.3.2",
    "micromatch": "^4.0.8",
    "minimist": "^1.2.8",
-    "nanoid": "^5.0.7",
+    "nanoid": "^5.0.8",
    "ora": "^8.1.0",
    "p-map": "^7.0.2",
    "path-to-regexp": "^6.3.0",
-    "picocolors": "^1.1.0",
+    "picocolors": "^1.1.1",
    "pkg-dir": "^8.0.0",
-    "playwright-chromium": "^1.48.0",
+    "playwright-chromium": "^1.48.2",
    "polka": "^1.0.0-next.28",
    "postcss-prefix-selector": "^2.1.0",
    "prettier": "^3.3.3",
    "prompts": "^2.4.2",
    "punycode": "^2.3.1",
    "rimraf": "^6.0.1",
-    "rollup": "^4.24.0",
+    "rollup": "^4.24.2",
    "rollup-plugin-dts": "^6.1.1",
    "rollup-plugin-esbuild": "^6.1.1",
    "semver": "^7.6.3",
@ -185,10 +185,10 @@
    "sirv": "^3.0.0",
    "sitemap": "^8.0.0",
    "supports-color": "^9.4.0",
-    "tinyglobby": "^0.2.9",
+    "tinyglobby": "^0.2.10",
    "typescript": "^5.6.3",
-    "vitest": "^2.1.2",
-    "vue-tsc": "^2.1.6",
+    "vitest": "^2.1.4",
+    "vue-tsc": "^2.1.8",
    "wait-on": "^8.0.1"
  },
  "peerDependencies": {
@ -203,7 +203,7 @@
      "optional": true
    }
  },
-  "packageManager": "pnpm@9.12.1",
+  "packageManager": "pnpm@9.12.3",
  "pnpm": {
    "peerDependencyRules": {
      "ignoreMissing": [
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/src/node/contentLoader.ts
+++ b/src/node/contentLoader.ts
@ -4,10 +4,7 @@ import { glob, type GlobOptions } from 'tinyglobby'
 import type { SiteConfig } from './config'
 import matter from 'gray-matter'
 import { normalizePath } from 'vite'
-import {
-  createMarkdownRenderer,
-  type MarkdownRenderer
-} from './markdown/markdown'
+import { createMarkdownRenderer } from './markdown/markdown'

 export interface ContentOptions<T = ContentData[]> {
  /**
@ -100,15 +97,7 @@ export function createContentLoader<T = ContentData[]>(
  if (typeof pattern === 'string') pattern = [pattern]
  pattern = pattern.map((p) => normalizePath(path.join(config.srcDir, p)))

-  let md: MarkdownRenderer
-
-  const cache = new Map<
-    string,
-    {
-      data: any
-      timestamp: number
-    }
-  >()
+  const cache = new Map<string, { data: any; timestamp: number }>()

  return {
    watch: pattern,
@ -124,14 +113,12 @@ export function createContentLoader<T = ContentData[]>(
        ).sort()
      }

-      md =
-        md ||
-        (await createMarkdownRenderer(
+      const md = await createMarkdownRenderer(
        config.srcDir,
        config.markdown,
        config.site.base,
        config.logger
-        ))
+      )

      const raw: ContentData[] = []

--- a/src/node/markdown/markdown.ts
+++ b/src/node/markdown/markdown.ts
@ -28,7 +28,7 @@ import type {
 import type { Logger } from 'vite'
 import { containerPlugin, type ContainerOptions } from './plugins/containers'
 import { gitHubAlertsPlugin } from './plugins/githubAlerts'
-import { highlight } from './plugins/highlight'
+import { highlight as createHighlighter } from './plugins/highlight'
 import { highlightLinePlugin } from './plugins/highlightLines'
 import { imagePlugin, type Options as ImageOptions } from './plugins/image'
 import { lineNumberPlugin } from './plugins/lineNumbers'
@ -173,7 +173,7 @@ export interface MarkdownOptions extends Options {
   */
  container?: ContainerOptions
  /**
-   * Math support (experimental)
+   * Math support
   *
   * You need to install `markdown-it-mathjax3` and set `math` to `true` to enable it.
   * You can also pass options to `markdown-it-mathjax3` here.
@ -192,22 +192,38 @@ export interface MarkdownOptions extends Options {

 export type MarkdownRenderer = MarkdownIt

-export const createMarkdownRenderer = async (
+let md: MarkdownRenderer | undefined
+let _disposeHighlighter: (() => void) | undefined
+
+export function disposeMdItInstance() {
+  if (md) {
+    md = undefined
+    _disposeHighlighter?.()
+  }
+}
+
+/**
+ * @experimental
+ */
+export async function createMarkdownRenderer(
  srcDir: string,
  options: MarkdownOptions = {},
  base = '/',
  logger: Pick<Logger, 'warn'> = console
-): Promise<MarkdownRenderer> => {
+): Promise<MarkdownRenderer> {
+  if (md) return md
+
  const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' }
  const codeCopyButtonTitle = options.codeCopyButtonTitle || 'Copy Code'
  const hasSingleTheme = typeof theme === 'string' || 'name' in theme

-  const md = MarkdownIt({
-    html: true,
-    linkify: true,
-    highlight: options.highlight || (await highlight(theme, options, logger)),
-    ...options
-  })
+  let [highlight, dispose] = options.highlight
+    ? [options.highlight, () => {}]
+    : await createHighlighter(theme, options, logger)
+
+  _disposeHighlighter = dispose
+
+  md = MarkdownIt({ html: true, linkify: true, highlight, ...options })

  md.linkify.set({ fuzzyLink: false })
  md.use(restoreEntities)
--- a/src/node/markdown/plugins/highlight.ts
+++ b/src/node/markdown/plugins/highlight.ts
@ -23,7 +23,7 @@ const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
 * 2. convert line numbers into line options:
 *    [{ line: number, classes: string[] }]
 */
-const attrsToLines = (attrs: string): TransformerCompactLineOption[] => {
+function attrsToLines(attrs: string): TransformerCompactLineOption[] {
  attrs = attrs.replace(/^(?:\[.*?\])?.*?([\d,-]+).*/, '$1').trim()
  const result: number[] = []
  if (!attrs) {
@ -51,7 +51,7 @@ export async function highlight(
  theme: ThemeOptions,
  options: MarkdownOptions,
  logger: Pick<Logger, 'warn'> = console
-): Promise<(str: string, lang: string, attrs: string) => string> {
+): Promise<[(str: string, lang: string, attrs: string) => string, () => void]> {
  const {
    defaultHighlightLang: defaultLang = '',
    codeTransformers: userTransformers = []
@ -95,7 +95,8 @@ export async function highlight(
  const lineNoRE = /:(no-)?line-numbers(=\d*)?$/
  const mustacheRE = /\{\{.*?\}\}/g

-  return (str: string, lang: string, attrs: string) => {
+  return [
+    (str: string, lang: string, attrs: string) => {
      const vPre = vueRE.test(lang) ? '' : 'v-pre'
      lang =
        lang
@ -105,7 +106,7 @@ export async function highlight(
          .toLowerCase() || defaultLang

      if (lang) {
-      const langLoaded = highlighter.getLoadedLanguages().includes(lang as any)
+        const langLoaded = highlighter.getLoadedLanguages().includes(lang)
        if (!langLoaded && !isSpecialLang(lang)) {
          logger.warn(
            c.yellow(
@ -183,5 +184,7 @@ export async function highlight(
      })

      return restoreMustache(highlighted)
-  }
+    },
+    highlighter.dispose
+  ]
 }
--- a/src/node/plugin.ts
+++ b/src/node/plugin.ts
@ -16,6 +16,7 @@ import {
  resolveAliases
 } from './alias'
 import { resolvePages, resolveUserConfig, type SiteConfig } from './config'
+import { disposeMdItInstance } from './markdown/markdown'
 import {
  clearCache,
  createMarkdownToVueRenderFn,
@ -388,6 +389,7 @@ export async function createVitePressPlugin(
          return
        }

+        disposeMdItInstance()
        clearCache()
        await recreateServer?.()
        return