perf: lazy load shiki languages (#4326)

package.json

@@ -185,6 +185,7 @@
     "sirv": "^3.0.0",
     "sitemap": "^8.0.0",
     "supports-color": "^9.4.0",
+    "synckit": "^0.9.2",
     "tinyglobby": "^0.2.10",
     "typescript": "^5.6.3",
     "vitest": "^2.1.4",

pnpm-lock.yaml

@@ -272,6 +272,9 @@ importers:
       supports-color:
         specifier: ^9.4.0
         version: 9.4.0
+      synckit:
+        specifier: ^0.9.2
+        version: 0.9.2
       tinyglobby:
         specifier: ^0.2.10
         version: 0.2.10
@@ -796,6 +799,10 @@ packages:
     resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
     engines: {node: '>= 8'}
 
+  '@pkgr/core@0.1.1':
+    resolution: {integrity: sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==}
+    engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0}
+
   '@polka/compression@1.0.0-next.28':
     resolution: {integrity: sha512-aDmrBhgHJtxE+jy145WfhW9WmTAFmES/dNnn1LAs8UnnkFgBUj4T8I4ScQ9+rOkpDZStvnVP5iqhN3tvt7O1NA==}
     engines: {node: '>=6'}
@@ -2569,6 +2576,10 @@ packages:
     resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
     engines: {node: '>= 0.4'}
 
+  synckit@0.9.2:
+    resolution: {integrity: sha512-vrozgXDQwYO72vHjUb/HnFbQx1exDjoKzqx23aXEg2a9VIg2TSFZ8FmeZpTjUCFMYw7mpX4BE2SFu8wI7asYsw==}
+    engines: {node: ^14.18.0 || >=16.0.0}
+
   tabbable@6.2.0:
     resolution: {integrity: sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==}
 
@@ -3278,6 +3289,8 @@ snapshots:
       '@nodelib/fs.scandir': 2.1.5
       fastq: 1.17.1
 
+  '@pkgr/core@0.1.1': {}
+
   '@polka/compression@1.0.0-next.28': {}
 
   '@polka/url@1.0.0-next.28': {}
@@ -5095,6 +5108,11 @@ snapshots:
 
   supports-preserve-symlinks-flag@1.0.0: {}
 
+  synckit@0.9.2:
+    dependencies:
+      '@pkgr/core': 0.1.1
+      tslib: 2.7.0
+
   tabbable@6.2.0: {}
 
   temp-dir@3.0.0: {}

rollup.config.ts

@@ -1,7 +1,7 @@
-import { promises as fs } from 'fs'
-import { builtinModules, createRequire } from 'module'
-import { resolve } from 'path'
-import { fileURLToPath } from 'url'
+import * as fs from 'node:fs/promises'
+import { builtinModules, createRequire } from 'node:module'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
 import { type RollupOptions, defineConfig } from 'rollup'
 import { nodeResolve } from '@rollup/plugin-node-resolve'
 import commonjs from '@rollup/plugin-commonjs'
@@ -10,6 +10,7 @@ import json from '@rollup/plugin-json'
 import replace from '@rollup/plugin-replace'
 import alias from '@rollup/plugin-alias'
 import dts from 'rollup-plugin-dts'
+import { globSync } from 'tinyglobby'
 
 const ROOT = fileURLToPath(import.meta.url)
 const r = (p: string) => resolve(ROOT, '..', p)
@@ -43,11 +44,15 @@ const plugins = [
 ]
 
 const esmBuild: RollupOptions = {
-  input: [r('src/node/index.ts'), r('src/node/cli.ts')],
+  input: [
+    r('src/node/index.ts'),
+    r('src/node/cli.ts'),
+    ...globSync(r('src/node/worker_*.ts'))
+  ],
   output: {
     format: 'esm',
     entryFileNames: `[name].js`,
-    chunkFileNames: 'serve-[hash].js',
+    chunkFileNames: 'chunk-[hash].js',
     dir: r('dist/node'),
     sourcemap: DEV
   },

src/node/markdown/plugins/highlight.ts

@@ -1,7 +1,3 @@
-import { customAlphabet } from 'nanoid'
-import c from 'picocolors'
-import type { ShikiTransformer } from 'shiki'
-import { bundledLanguages, createHighlighter, isSpecialLang } from 'shiki'
 import {
   transformerCompactLineOptions,
   transformerNotationDiff,
@@ -10,9 +6,21 @@ import {
   transformerNotationHighlight,
   type TransformerCompactLineOption
 } from '@shikijs/transformers'
+import { customAlphabet } from 'nanoid'
+import { createRequire } from 'node:module'
+import c from 'picocolors'
+import type { ShikiTransformer } from 'shiki'
+import { createHighlighter, isSpecialLang } from 'shiki'
+import { createSyncFn } from 'synckit'
 import type { Logger } from 'vite'
+import type { ShikiResolveLang } from 'worker_shikiResolveLang'
 import type { MarkdownOptions, ThemeOptions } from '../markdown'
 
+const require = createRequire(import.meta.url)
+
+const resolveLangSync = createSyncFn<ShikiResolveLang>(
+  require.resolve('vitepress/dist/node/worker_shikiResolveLang.js')
+)
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
 
 /**
@@ -62,7 +70,10 @@ export async function highlight(
       typeof theme === 'object' && 'light' in theme && 'dark' in theme
         ? [theme.light, theme.dark]
         : [theme],
-    langs: [...Object.keys(bundledLanguages), ...(options.languages || [])],
+    langs: [
+      ...(options.languages || []),
+      ...Object.values(options.languageAlias || {})
+    ],
     langAlias: options.languageAlias
   })
 
@@ -108,14 +119,19 @@ export async function highlight(
       if (lang) {
         const langLoaded = highlighter.getLoadedLanguages().includes(lang)
         if (!langLoaded && !isSpecialLang(lang)) {
-          logger.warn(
-            c.yellow(
-              `\nThe language '${lang}' is not loaded, falling back to '${
-                defaultLang || 'txt'
-              }' for syntax highlighting.`
+          const resolvedLang = resolveLangSync(lang)
+          if (!resolvedLang) {
+            logger.warn(
+              c.yellow(
+                `\nThe language '${lang}' is not loaded, falling back to '${
+                  defaultLang || 'txt'
+                }' for syntax highlighting.`
+              )
             )
-          )
-          lang = defaultLang
+            lang = defaultLang
+          } else {
+            highlighter.loadLanguageSync(resolvedLang)
+          }
         }
       }
 

src/node/worker_shikiResolveLang.ts

@@ -0,0 +1,14 @@
+import { bundledLanguages, type DynamicImportLanguageRegistration } from 'shiki'
+import { runAsWorker } from 'synckit'
+
+async function resolveLang(lang: string) {
+  return (
+    (bundledLanguages as Record<string, DynamicImportLanguageRegistration>)
+      [lang]?.()
+      .then((m) => m.default) || []
+  )
+}
+
+runAsWorker(resolveLang)
+
+export type ShikiResolveLang = typeof resolveLang