feat: allow customizing markdown renderer used for local search indexing (#2770)

BREAKING CHANGES: `search.options.exclude` for local search is removed in favor of more flexible `search.options._render`

Co-authored-by: Divyansh Singh <40380293+brc-dd@users.noreply.github.com>

__tests__/e2e/.vitepress/config.ts

@@ -91,8 +91,11 @@ export default defineConfig({
     search: {
       provider: 'local',
       options: {
-        exclude(relativePath) {
+        _render(src, env, md) {
-          return relativePath.startsWith('local-search/excluded')
+          const html = md.render(src, env)
+          if (env.frontmatter?.search === false) return ''
+          if (env.relativePath.startsWith('local-search/excluded')) return ''
+          return html
         }
       }
     }

docs/reference/default-theme-search.md

@@ -98,9 +98,9 @@ export default defineConfig({
 
 Learn more in [MiniSearch docs](https://lucaong.github.io/minisearch/classes/_minisearch_.minisearch.html).
 
-### Excluding pages from search
+### Custom content renderer
 
-You can exclude pages from search by adding `search: false` to the frontmatter of the page. Alternatively, you can also pass `exclude` function to `themeConfig.search.options` to exclude pages based on their path relative to `srcDir`:
+You can customize the function used to render the markdown content before indexing it:
 
 ```ts
 import { defineConfig } from 'vitepress'
@@ -108,8 +108,68 @@ import { defineConfig } from 'vitepress'
 export default defineConfig({
   themeConfig: {
     search: {
+      provider: 'local',
+      options: {
+        /**
+         * @param {string} src
+         * @param {import('vitepress').MarkdownEnv} env
+         * @param {import('markdown-it')} md
+         */
+        _render(src, env, md) {
+          // return html string
+        }
+      }
+    }
+  }
+})
+```
+
+This function will be stripped from client-side site data, so you can use Node.js APIs in it.
+
+#### Example: Excluding pages from search
+
+You can exclude pages from search by adding `search: false` to the frontmatter of the page. Alternatively:
+
+```ts
+import { defineConfig } from 'vitepress'
+
+export default defineConfig({
+  themeConfig: {
+    search: {
+      provider: 'local',
       options: {
-        exclude: (path) => path.startsWith('/some/path')
+        _render(src, env, md) {
+          const html = md.render(src, env)
+          if (env.frontmatter?.search === false) return ''
+          if (env.relativePath.startsWith('some/path')) return ''
+          return html
+        }
+      }
+    }
+  }
+})
+```
+
+::: warning Note
+In case a custom `_render` function is provided, you need to handle the `search: false` frontmatter yourself. Also, the `env` object won't be completely populated before `md.render` is called, so any checks on optional `env` properties like `frontmatter` should be done after that.
+:::
+
+#### Example: Transforming content - adding anchors
+
+```ts
+import { defineConfig } from 'vitepress'
+
+export default defineConfig({
+  themeConfig: {
+    search: {
+      provider: 'local',
+      options: {
+        _render(src, env, md) {
+          const html = md.render(src, env)
+          if (env.frontmatter?.title)
+            return md.render(`# ${env.frontmatter.title}`) + html
+          return html
+        }
       }
     }
   }

src/node/markdown/env.ts

@@ -1,40 +0,0 @@
-import type { MarkdownSfcBlocks } from '@mdit-vue/plugin-sfc'
-import type { Header } from '../shared'
-
-// Manually declaring all properties as rollup-plugin-dts
-// is unable to merge augmented module declarations
-
-export interface MarkdownEnv {
-  /**
-   * The raw Markdown content without frontmatter
-   */
-  content?: string
-  /**
-   * The excerpt that extracted by `@mdit-vue/plugin-frontmatter`
-   *
-   * - Would be the rendered HTML when `renderExcerpt` is enabled
-   * - Would be the raw Markdown when `renderExcerpt` is disabled
-   */
-  excerpt?: string
-  /**
-   * The frontmatter that extracted by `@mdit-vue/plugin-frontmatter`
-   */
-  frontmatter?: Record<string, unknown>
-  /**
-   * The headers that extracted by `@mdit-vue/plugin-headers`
-   */
-  headers?: Header[]
-  /**
-   * SFC blocks that extracted by `@mdit-vue/plugin-sfc`
-   */
-  sfcBlocks?: MarkdownSfcBlocks
-  /**
-   * The title that extracted by `@mdit-vue/plugin-title`
-   */
-  title?: string
-  path: string
-  relativePath: string
-  cleanUrls: boolean
-  links?: string[]
-  includes?: string[]
-}

src/node/markdown/index.ts

@@ -1,2 +1,154 @@
-export * from './env'
+import { componentPlugin } from '@mdit-vue/plugin-component'
-export * from './markdown'
+import {
+  frontmatterPlugin,
+  type FrontmatterPluginOptions
+} from '@mdit-vue/plugin-frontmatter'
+import {
+  headersPlugin,
+  type HeadersPluginOptions
+} from '@mdit-vue/plugin-headers'
+import { sfcPlugin, type SfcPluginOptions } from '@mdit-vue/plugin-sfc'
+import { titlePlugin } from '@mdit-vue/plugin-title'
+import { tocPlugin, type TocPluginOptions } from '@mdit-vue/plugin-toc'
+import { slugify } from '@mdit-vue/shared'
+import MarkdownIt from 'markdown-it'
+import anchorPlugin from 'markdown-it-anchor'
+import attrsPlugin from 'markdown-it-attrs'
+import emojiPlugin from 'markdown-it-emoji'
+import type { ILanguageRegistration, IThemeRegistration } from 'shiki'
+import type { Logger } from 'vite'
+import { containerPlugin } from './plugins/containers'
+import { highlight } from './plugins/highlight'
+import { highlightLinePlugin } from './plugins/highlightLines'
+import { imagePlugin } from './plugins/image'
+import { lineNumberPlugin } from './plugins/lineNumbers'
+import { linkPlugin } from './plugins/link'
+import { preWrapperPlugin } from './plugins/preWrapper'
+import { snippetPlugin } from './plugins/snippet'
+
+export type { Header } from '../shared'
+
+export type ThemeOptions =
+  | IThemeRegistration
+  | { light: IThemeRegistration; dark: IThemeRegistration }
+
+export interface MarkdownOptions extends MarkdownIt.Options {
+  lineNumbers?: boolean
+  preConfig?: (md: MarkdownIt) => void
+  config?: (md: MarkdownIt) => void
+  anchor?: anchorPlugin.AnchorOptions
+  attrs?: {
+    leftDelimiter?: string
+    rightDelimiter?: string
+    allowedAttributes?: string[]
+    disable?: boolean
+  }
+  defaultHighlightLang?: string
+  frontmatter?: FrontmatterPluginOptions
+  headers?: HeadersPluginOptions | boolean
+  sfc?: SfcPluginOptions
+  theme?: ThemeOptions
+  languages?: ILanguageRegistration[]
+  toc?: TocPluginOptions
+  externalLinks?: Record<string, string>
+  cache?: boolean
+}
+
+export type MarkdownRenderer = MarkdownIt
+
+export const createMarkdownRenderer = async (
+  srcDir: string,
+  options: MarkdownOptions = {},
+  base = '/',
+  logger: Pick<Logger, 'warn'> = console
+): Promise<MarkdownRenderer> => {
+  const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' }
+  const hasSingleTheme = typeof theme === 'string' || 'name' in theme
+
+  const md = MarkdownIt({
+    html: true,
+    linkify: true,
+    highlight:
+      options.highlight ||
+      (await highlight(
+        theme,
+        options.languages,
+        options.defaultHighlightLang,
+        logger
+      )),
+    ...options
+  })
+
+  md.linkify.set({ fuzzyLink: false })
+
+  if (options.preConfig) {
+    options.preConfig(md)
+  }
+
+  // custom plugins
+  md.use(componentPlugin)
+    .use(highlightLinePlugin)
+    .use(preWrapperPlugin, { hasSingleTheme })
+    .use(snippetPlugin, srcDir)
+    .use(containerPlugin, { hasSingleTheme })
+    .use(imagePlugin)
+    .use(
+      linkPlugin,
+      { target: '_blank', rel: 'noreferrer', ...options.externalLinks },
+      base
+    )
+    .use(lineNumberPlugin, options.lineNumbers)
+
+  // 3rd party plugins
+  if (!options.attrs?.disable) {
+    md.use(attrsPlugin, options.attrs)
+  }
+  md.use(emojiPlugin)
+
+  // mdit-vue plugins
+  md.use(anchorPlugin, {
+    slugify,
+    permalink: anchorPlugin.permalink.linkInsideHeader({
+      symbol: '&ZeroWidthSpace;',
+      renderAttrs: (slug, state) => {
+        // Find `heading_open` with the id identical to slug
+        const idx = state.tokens.findIndex((token) => {
+          const attrs = token.attrs
+          const id = attrs?.find((attr) => attr[0] === 'id')
+          return id && slug === id[1]
+        })
+        // Get the actual heading content
+        const title = state.tokens[idx + 1].content
+        return {
+          'aria-label': `Permalink to "${title}"`
+        }
+      }
+    }),
+    ...options.anchor
+  } as anchorPlugin.AnchorOptions).use(frontmatterPlugin, {
+    ...options.frontmatter
+  } as FrontmatterPluginOptions)
+
+  if (options.headers) {
+    md.use(headersPlugin, {
+      level: [2, 3, 4, 5, 6],
+      slugify,
+      ...(typeof options.headers === 'boolean' ? undefined : options.headers)
+    } as HeadersPluginOptions)
+  }
+
+  md.use(sfcPlugin, {
+    ...options.sfc
+  } as SfcPluginOptions)
+    .use(titlePlugin)
+    .use(tocPlugin, {
+      ...options.toc
+    } as TocPluginOptions)
+
+  // apply user config
+  if (options.config) {
+    options.config(md)
+  }
+
+  return md
+}

src/node/markdown/markdown.ts

@@ -1,154 +0,0 @@
-import { componentPlugin } from '@mdit-vue/plugin-component'
-import {
-  frontmatterPlugin,
-  type FrontmatterPluginOptions
-} from '@mdit-vue/plugin-frontmatter'
-import {
-  headersPlugin,
-  type HeadersPluginOptions
-} from '@mdit-vue/plugin-headers'
-import { sfcPlugin, type SfcPluginOptions } from '@mdit-vue/plugin-sfc'
-import { titlePlugin } from '@mdit-vue/plugin-title'
-import { tocPlugin, type TocPluginOptions } from '@mdit-vue/plugin-toc'
-import { slugify } from '@mdit-vue/shared'
-import MarkdownIt from 'markdown-it'
-import anchorPlugin from 'markdown-it-anchor'
-import attrsPlugin from 'markdown-it-attrs'
-import emojiPlugin from 'markdown-it-emoji'
-import type { ILanguageRegistration, IThemeRegistration } from 'shiki'
-import type { Logger } from 'vite'
-import { containerPlugin } from './plugins/containers'
-import { highlight } from './plugins/highlight'
-import { highlightLinePlugin } from './plugins/highlightLines'
-import { imagePlugin } from './plugins/image'
-import { lineNumberPlugin } from './plugins/lineNumbers'
-import { linkPlugin } from './plugins/link'
-import { preWrapperPlugin } from './plugins/preWrapper'
-import { snippetPlugin } from './plugins/snippet'
-
-export type { Header } from '../shared'
-
-export type ThemeOptions =
-  | IThemeRegistration
-  | { light: IThemeRegistration; dark: IThemeRegistration }
-
-export interface MarkdownOptions extends MarkdownIt.Options {
-  lineNumbers?: boolean
-  preConfig?: (md: MarkdownIt) => void
-  config?: (md: MarkdownIt) => void
-  anchor?: anchorPlugin.AnchorOptions
-  attrs?: {
-    leftDelimiter?: string
-    rightDelimiter?: string
-    allowedAttributes?: string[]
-    disable?: boolean
-  }
-  defaultHighlightLang?: string
-  frontmatter?: FrontmatterPluginOptions
-  headers?: HeadersPluginOptions | boolean
-  sfc?: SfcPluginOptions
-  theme?: ThemeOptions
-  languages?: ILanguageRegistration[]
-  toc?: TocPluginOptions
-  externalLinks?: Record<string, string>
-  cache?: boolean
-}
-
-export type MarkdownRenderer = MarkdownIt
-
-export const createMarkdownRenderer = async (
-  srcDir: string,
-  options: MarkdownOptions = {},
-  base = '/',
-  logger: Pick<Logger, 'warn'> = console
-): Promise<MarkdownRenderer> => {
-  const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' }
-  const hasSingleTheme = typeof theme === 'string' || 'name' in theme
-
-  const md = MarkdownIt({
-    html: true,
-    linkify: true,
-    highlight:
-      options.highlight ||
-      (await highlight(
-        theme,
-        options.languages,
-        options.defaultHighlightLang,
-        logger
-      )),
-    ...options
-  })
-
-  md.linkify.set({ fuzzyLink: false })
-
-  if (options.preConfig) {
-    options.preConfig(md)
-  }
-
-  // custom plugins
-  md.use(componentPlugin)
-    .use(highlightLinePlugin)
-    .use(preWrapperPlugin, { hasSingleTheme })
-    .use(snippetPlugin, srcDir)
-    .use(containerPlugin, { hasSingleTheme })
-    .use(imagePlugin)
-    .use(
-      linkPlugin,
-      { target: '_blank', rel: 'noreferrer', ...options.externalLinks },
-      base
-    )
-    .use(lineNumberPlugin, options.lineNumbers)
-
-  // 3rd party plugins
-  if (!options.attrs?.disable) {
-    md.use(attrsPlugin, options.attrs)
-  }
-  md.use(emojiPlugin)
-
-  // mdit-vue plugins
-  md.use(anchorPlugin, {
-    slugify,
-    permalink: anchorPlugin.permalink.linkInsideHeader({
-      symbol: '&ZeroWidthSpace;',
-      renderAttrs: (slug, state) => {
-        // Find `heading_open` with the id identical to slug
-        const idx = state.tokens.findIndex((token) => {
-          const attrs = token.attrs
-          const id = attrs?.find((attr) => attr[0] === 'id')
-          return id && slug === id[1]
-        })
-        // Get the actual heading content
-        const title = state.tokens[idx + 1].content
-        return {
-          'aria-label': `Permalink to "${title}"`
-        }
-      }
-    }),
-    ...options.anchor
-  } as anchorPlugin.AnchorOptions).use(frontmatterPlugin, {
-    ...options.frontmatter
-  } as FrontmatterPluginOptions)
-
-  if (options.headers) {
-    md.use(headersPlugin, {
-      level: [2, 3, 4, 5, 6],
-      slugify,
-      ...(typeof options.headers === 'boolean' ? undefined : options.headers)
-    } as HeadersPluginOptions)
-  }
-
-  md.use(sfcPlugin, {
-    ...options.sfc
-  } as SfcPluginOptions)
-    .use(titlePlugin)
-    .use(tocPlugin, {
-      ...options.toc
-    } as TocPluginOptions)
-
-  // apply user config
-  if (options.config) {
-    options.config(md)
-  }
-
-  return md
-}

src/node/markdown/plugins/highlight.ts

@@ -17,7 +17,7 @@ import {
   type Processor
 } from 'shiki-processor'
 import type { Logger } from 'vite'
-import type { ThemeOptions } from '../markdown'
+import type { ThemeOptions } from '..'
 
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)
 

src/node/markdown/plugins/link.ts

@@ -3,9 +3,13 @@
 // 2. normalize internal links to end with `.html`
 
 import type MarkdownIt from 'markdown-it'
-import type { MarkdownEnv } from '../env'
 import { URL } from 'url'
-import { EXTERNAL_URL_RE, PATHNAME_PROTOCOL_RE, isExternal } from '../../shared'
+import {
+  EXTERNAL_URL_RE,
+  PATHNAME_PROTOCOL_RE,
+  isExternal,
+  type MarkdownEnv
+} from '../../shared'
 
 const indexRE = /(^|.*\/)index.md(#?.*)$/i
 

src/node/markdown/plugins/snippet.ts

@@ -1,8 +1,8 @@
 import fs from 'fs-extra'
-import path from 'path'
 import type MarkdownIt from 'markdown-it'
 import type { RuleBlock } from 'markdown-it/lib/parser_block'
-import type { MarkdownEnv } from '../env'
+import path from 'path'
+import type { MarkdownEnv } from '../../shared'
 
 export function dedent(text: string): string {
   const lines = text.split('\n')

src/node/markdownToVue.ts

@@ -6,7 +6,6 @@ import path from 'path'
 import type { SiteConfig } from './config'
 import {
   createMarkdownRenderer,
-  type MarkdownEnv,
   type MarkdownOptions,
   type MarkdownRenderer
 } from './markdown'
@@ -14,6 +13,7 @@ import {
   EXTERNAL_URL_RE,
   slash,
   type HeadConfig,
+  type MarkdownEnv,
   type PageData
 } from './shared'
 import { getGitTimestamp } from './utils/getGitTimestamp'

src/node/plugins/localSearchPlugin.ts

@@ -4,8 +4,13 @@ import MiniSearch from 'minisearch'
 import path from 'path'
 import type { Plugin, ViteDevServer } from 'vite'
 import type { SiteConfig } from '../config'
-import { createMarkdownRenderer, type MarkdownEnv } from '../markdown'
+import { createMarkdownRenderer } from '../markdown'
-import { resolveSiteDataByRoute, slash, type DefaultTheme } from '../shared'
+import {
+  resolveSiteDataByRoute,
+  slash,
+  type DefaultTheme,
+  type MarkdownEnv
+} from '../shared'
 
 const debug = _debug('vitepress:local-search')
 
@@ -45,23 +50,16 @@ export async function localSearchPlugin(
     siteConfig.logger
   )
 
+  const options = siteConfig.site.themeConfig.search.options || {}
+
   function render(file: string) {
-    const { srcDir, cleanUrls = false, site } = siteConfig
+    const { srcDir, cleanUrls = false } = siteConfig
     const relativePath = slash(path.relative(srcDir, file))
-    const env: MarkdownEnv = {
+    const env: MarkdownEnv = { path: file, relativePath, cleanUrls }
-      path: file,
+    const src = fs.readFileSync(file, 'utf-8')
-      relativePath,
+    if (options._render) return options._render(src, env, md)
-      cleanUrls
+    const html = md.render(src, env)
-    }
+    return env.frontmatter?.search === false ? '' : html
-    const html = md.render(fs.readFileSync(file, 'utf-8'), env)
-    if (
-      env.frontmatter?.search === false ||
-      (site.themeConfig.search?.provider === 'local' &&
-        site.themeConfig.search.options?.exclude?.(relativePath))
-    ) {
-      return ''
-    }
-    return html
   }
 
   const indexByLocales = new Map<string, MiniSearch<IndexObject>>()
@@ -72,8 +70,7 @@ export async function localSearchPlugin(
       index = new MiniSearch<IndexObject>({
         fields: ['title', 'titles', 'text'],
         storeFields: ['title', 'titles'],
-        ...(siteConfig.site.themeConfig?.search?.provider === 'local' &&
+        ...options.miniSearch?.options
-          siteConfig.site.themeConfig.search.options?.miniSearch?.options)
       })
       indexByLocales.set(locale, index)
     }

src/node/utils/fnSerialize.ts

@@ -3,6 +3,7 @@ export function serializeFunctions(value: any, key?: string): any {
     return value.map((v) => serializeFunctions(v))
   } else if (typeof value === 'object' && value !== null) {
     return Object.keys(value).reduce((acc, key) => {
+      if (key[0] === '_') return acc
       acc[key] = serializeFunctions(value[key], key)
       return acc
     }, {} as any)

src/shared/shared.ts

@@ -7,10 +7,11 @@ export type {
   Header,
   LocaleConfig,
   LocaleSpecificConfig,
+  MarkdownEnv,
   PageData,
   PageDataPayload,
-  SiteData,
+  SSGContext,
-  SSGContext
+  SiteData
 } from '../../types/shared'
 
 export const EXTERNAL_URL_RE = /^[a-z]+:/i

types/default-theme.d.ts

@@ -1,8 +1,9 @@
+import type MarkdownIt from 'markdown-it'
 import type { Options as MiniSearchOptions } from 'minisearch'
 import type { ComputedRef, Ref } from 'vue'
 import type { DocSearchProps } from './docsearch.js'
 import type { LocalSearchTranslations } from './local-search.js'
-import type { PageData } from './shared.js'
+import type { MarkdownEnv, PageData } from './shared.js'
 
 export namespace DefaultTheme {
   export interface Config {
@@ -383,15 +384,16 @@ export namespace DefaultTheme {
     }
 
     /**
-     * exclude content from search results
+     * Allows transformation of content before indexing (node only)
+     * Return empty string to skip indexing
      */
-    exclude?: (relativePath: string) => boolean
+    _render?: (src: string, env: MarkdownEnv, md: MarkdownIt) => string
   }
 
   // algolia -------------------------------------------------------------------
 
   /**
-   * The Algolia search options. Partially copied from
+   * Algolia search options. Partially copied from
    * `@docsearch/react/dist/esm/DocSearch.d.ts`
    */
   export interface AlgoliaSearchOptions extends DocSearchProps {

types/shared.d.ts

@@ -1,4 +1,5 @@
 // types shared between server and client
+import type { MarkdownSfcBlocks } from '@mdit-vue/plugin-sfc'
 import type { UseDarkOptions } from '@vueuse/core'
 import type { SSRContext } from 'vue/server-renderer'
 export type { DefaultTheme } from './default-theme.js'
@@ -98,3 +99,41 @@ export type LocaleConfig<ThemeConfig = any> = Record<
   string,
   LocaleSpecificConfig<ThemeConfig> & { label: string; link?: string }
 >
+
+// Manually declaring all properties as rollup-plugin-dts
+// is unable to merge augmented module declarations
+
+export interface MarkdownEnv {
+  /**
+   * The raw Markdown content without frontmatter
+   */
+  content?: string
+  /**
+   * The excerpt that extracted by `@mdit-vue/plugin-frontmatter`
+   *
+   * - Would be the rendered HTML when `renderExcerpt` is enabled
+   * - Would be the raw Markdown when `renderExcerpt` is disabled
+   */
+  excerpt?: string
+  /**
+   * The frontmatter that extracted by `@mdit-vue/plugin-frontmatter`
+   */
+  frontmatter?: Record<string, unknown>
+  /**
+   * The headers that extracted by `@mdit-vue/plugin-headers`
+   */
+  headers?: Header[]
+  /**
+   * SFC blocks that extracted by `@mdit-vue/plugin-sfc`
+   */
+  sfcBlocks?: MarkdownSfcBlocks
+  /**
+   * The title that extracted by `@mdit-vue/plugin-title`
+   */
+  title?: string
+  path: string
+  relativePath: string
+  cleanUrls: boolean
+  links?: string[]
+  includes?: string[]
+}