feat: allow customizing markdown renderer used for local search indexing (#2770)

BREAKING CHANGES: `search.options.exclude` for local search is removed in favor of more flexible `search.options._render`

Co-authored-by: Divyansh Singh <40380293+brc-dd@users.noreply.github.com>
pull/2495/merge
Damien Guard 11 months ago committed by GitHub
parent e8edd0a05f
commit 00dc1e6742
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -91,8 +91,11 @@ export default defineConfig({
search: {
provider: 'local',
options: {
exclude(relativePath) {
return relativePath.startsWith('local-search/excluded')
_render(src, env, md) {
const html = md.render(src, env)
if (env.frontmatter?.search === false) return ''
if (env.relativePath.startsWith('local-search/excluded')) return ''
return html
}
}
}

@ -98,9 +98,9 @@ export default defineConfig({
Learn more in [MiniSearch docs](https://lucaong.github.io/minisearch/classes/_minisearch_.minisearch.html).
### Excluding pages from search
### Custom content renderer
You can exclude pages from search by adding `search: false` to the frontmatter of the page. Alternatively, you can also pass `exclude` function to `themeConfig.search.options` to exclude pages based on their path relative to `srcDir`:
You can customize the function used to render the markdown content before indexing it:
```ts
import { defineConfig } from 'vitepress'
@ -108,8 +108,68 @@ import { defineConfig } from 'vitepress'
export default defineConfig({
themeConfig: {
search: {
provider: 'local',
options: {
/**
* @param {string} src
* @param {import('vitepress').MarkdownEnv} env
* @param {import('markdown-it')} md
*/
_render(src, env, md) {
// return html string
}
}
}
}
})
```
This function will be stripped from client-side site data, so you can use Node.js APIs in it.
#### Example: Excluding pages from search
You can exclude pages from search by adding `search: false` to the frontmatter of the page. Alternatively:
```ts
import { defineConfig } from 'vitepress'
export default defineConfig({
themeConfig: {
search: {
provider: 'local',
options: {
exclude: (path) => path.startsWith('/some/path')
_render(src, env, md) {
const html = md.render(src, env)
if (env.frontmatter?.search === false) return ''
if (env.relativePath.startsWith('some/path')) return ''
return html
}
}
}
}
})
```
::: warning Note
In case a custom `_render` function is provided, you need to handle the `search: false` frontmatter yourself. Also, the `env` object won't be completely populated before `md.render` is called, so any checks on optional `env` properties like `frontmatter` should be done after that.
:::
#### Example: Transforming content - adding anchors
```ts
import { defineConfig } from 'vitepress'
export default defineConfig({
themeConfig: {
search: {
provider: 'local',
options: {
_render(src, env, md) {
const html = md.render(src, env)
if (env.frontmatter?.title)
return md.render(`# ${env.frontmatter.title}`) + html
return html
}
}
}
}

@ -1,40 +0,0 @@
import type { MarkdownSfcBlocks } from '@mdit-vue/plugin-sfc'
import type { Header } from '../shared'
// Manually declaring all properties as rollup-plugin-dts
// is unable to merge augmented module declarations
export interface MarkdownEnv {
/**
* The raw Markdown content without frontmatter
*/
content?: string
/**
* The excerpt that extracted by `@mdit-vue/plugin-frontmatter`
*
* - Would be the rendered HTML when `renderExcerpt` is enabled
* - Would be the raw Markdown when `renderExcerpt` is disabled
*/
excerpt?: string
/**
* The frontmatter that extracted by `@mdit-vue/plugin-frontmatter`
*/
frontmatter?: Record<string, unknown>
/**
* The headers that extracted by `@mdit-vue/plugin-headers`
*/
headers?: Header[]
/**
* SFC blocks that extracted by `@mdit-vue/plugin-sfc`
*/
sfcBlocks?: MarkdownSfcBlocks
/**
* The title that extracted by `@mdit-vue/plugin-title`
*/
title?: string
path: string
relativePath: string
cleanUrls: boolean
links?: string[]
includes?: string[]
}

@ -1,2 +1,154 @@
export * from './env'
export * from './markdown'
import { componentPlugin } from '@mdit-vue/plugin-component'
import {
frontmatterPlugin,
type FrontmatterPluginOptions
} from '@mdit-vue/plugin-frontmatter'
import {
headersPlugin,
type HeadersPluginOptions
} from '@mdit-vue/plugin-headers'
import { sfcPlugin, type SfcPluginOptions } from '@mdit-vue/plugin-sfc'
import { titlePlugin } from '@mdit-vue/plugin-title'
import { tocPlugin, type TocPluginOptions } from '@mdit-vue/plugin-toc'
import { slugify } from '@mdit-vue/shared'
import MarkdownIt from 'markdown-it'
import anchorPlugin from 'markdown-it-anchor'
import attrsPlugin from 'markdown-it-attrs'
import emojiPlugin from 'markdown-it-emoji'
import type { ILanguageRegistration, IThemeRegistration } from 'shiki'
import type { Logger } from 'vite'
import { containerPlugin } from './plugins/containers'
import { highlight } from './plugins/highlight'
import { highlightLinePlugin } from './plugins/highlightLines'
import { imagePlugin } from './plugins/image'
import { lineNumberPlugin } from './plugins/lineNumbers'
import { linkPlugin } from './plugins/link'
import { preWrapperPlugin } from './plugins/preWrapper'
import { snippetPlugin } from './plugins/snippet'
export type { Header } from '../shared'
export type ThemeOptions =
| IThemeRegistration
| { light: IThemeRegistration; dark: IThemeRegistration }
export interface MarkdownOptions extends MarkdownIt.Options {
lineNumbers?: boolean
preConfig?: (md: MarkdownIt) => void
config?: (md: MarkdownIt) => void
anchor?: anchorPlugin.AnchorOptions
attrs?: {
leftDelimiter?: string
rightDelimiter?: string
allowedAttributes?: string[]
disable?: boolean
}
defaultHighlightLang?: string
frontmatter?: FrontmatterPluginOptions
headers?: HeadersPluginOptions | boolean
sfc?: SfcPluginOptions
theme?: ThemeOptions
languages?: ILanguageRegistration[]
toc?: TocPluginOptions
externalLinks?: Record<string, string>
cache?: boolean
}
export type MarkdownRenderer = MarkdownIt
export const createMarkdownRenderer = async (
srcDir: string,
options: MarkdownOptions = {},
base = '/',
logger: Pick<Logger, 'warn'> = console
): Promise<MarkdownRenderer> => {
const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' }
const hasSingleTheme = typeof theme === 'string' || 'name' in theme
const md = MarkdownIt({
html: true,
linkify: true,
highlight:
options.highlight ||
(await highlight(
theme,
options.languages,
options.defaultHighlightLang,
logger
)),
...options
})
md.linkify.set({ fuzzyLink: false })
if (options.preConfig) {
options.preConfig(md)
}
// custom plugins
md.use(componentPlugin)
.use(highlightLinePlugin)
.use(preWrapperPlugin, { hasSingleTheme })
.use(snippetPlugin, srcDir)
.use(containerPlugin, { hasSingleTheme })
.use(imagePlugin)
.use(
linkPlugin,
{ target: '_blank', rel: 'noreferrer', ...options.externalLinks },
base
)
.use(lineNumberPlugin, options.lineNumbers)
// 3rd party plugins
if (!options.attrs?.disable) {
md.use(attrsPlugin, options.attrs)
}
md.use(emojiPlugin)
// mdit-vue plugins
md.use(anchorPlugin, {
slugify,
permalink: anchorPlugin.permalink.linkInsideHeader({
symbol: '&ZeroWidthSpace;',
renderAttrs: (slug, state) => {
// Find `heading_open` with the id identical to slug
const idx = state.tokens.findIndex((token) => {
const attrs = token.attrs
const id = attrs?.find((attr) => attr[0] === 'id')
return id && slug === id[1]
})
// Get the actual heading content
const title = state.tokens[idx + 1].content
return {
'aria-label': `Permalink to "${title}"`
}
}
}),
...options.anchor
} as anchorPlugin.AnchorOptions).use(frontmatterPlugin, {
...options.frontmatter
} as FrontmatterPluginOptions)
if (options.headers) {
md.use(headersPlugin, {
level: [2, 3, 4, 5, 6],
slugify,
...(typeof options.headers === 'boolean' ? undefined : options.headers)
} as HeadersPluginOptions)
}
md.use(sfcPlugin, {
...options.sfc
} as SfcPluginOptions)
.use(titlePlugin)
.use(tocPlugin, {
...options.toc
} as TocPluginOptions)
// apply user config
if (options.config) {
options.config(md)
}
return md
}

@ -1,154 +0,0 @@
import { componentPlugin } from '@mdit-vue/plugin-component'
import {
frontmatterPlugin,
type FrontmatterPluginOptions
} from '@mdit-vue/plugin-frontmatter'
import {
headersPlugin,
type HeadersPluginOptions
} from '@mdit-vue/plugin-headers'
import { sfcPlugin, type SfcPluginOptions } from '@mdit-vue/plugin-sfc'
import { titlePlugin } from '@mdit-vue/plugin-title'
import { tocPlugin, type TocPluginOptions } from '@mdit-vue/plugin-toc'
import { slugify } from '@mdit-vue/shared'
import MarkdownIt from 'markdown-it'
import anchorPlugin from 'markdown-it-anchor'
import attrsPlugin from 'markdown-it-attrs'
import emojiPlugin from 'markdown-it-emoji'
import type { ILanguageRegistration, IThemeRegistration } from 'shiki'
import type { Logger } from 'vite'
import { containerPlugin } from './plugins/containers'
import { highlight } from './plugins/highlight'
import { highlightLinePlugin } from './plugins/highlightLines'
import { imagePlugin } from './plugins/image'
import { lineNumberPlugin } from './plugins/lineNumbers'
import { linkPlugin } from './plugins/link'
import { preWrapperPlugin } from './plugins/preWrapper'
import { snippetPlugin } from './plugins/snippet'
export type { Header } from '../shared'
export type ThemeOptions =
| IThemeRegistration
| { light: IThemeRegistration; dark: IThemeRegistration }
export interface MarkdownOptions extends MarkdownIt.Options {
lineNumbers?: boolean
preConfig?: (md: MarkdownIt) => void
config?: (md: MarkdownIt) => void
anchor?: anchorPlugin.AnchorOptions
attrs?: {
leftDelimiter?: string
rightDelimiter?: string
allowedAttributes?: string[]
disable?: boolean
}
defaultHighlightLang?: string
frontmatter?: FrontmatterPluginOptions
headers?: HeadersPluginOptions | boolean
sfc?: SfcPluginOptions
theme?: ThemeOptions
languages?: ILanguageRegistration[]
toc?: TocPluginOptions
externalLinks?: Record<string, string>
cache?: boolean
}
export type MarkdownRenderer = MarkdownIt
export const createMarkdownRenderer = async (
srcDir: string,
options: MarkdownOptions = {},
base = '/',
logger: Pick<Logger, 'warn'> = console
): Promise<MarkdownRenderer> => {
const theme = options.theme ?? { light: 'github-light', dark: 'github-dark' }
const hasSingleTheme = typeof theme === 'string' || 'name' in theme
const md = MarkdownIt({
html: true,
linkify: true,
highlight:
options.highlight ||
(await highlight(
theme,
options.languages,
options.defaultHighlightLang,
logger
)),
...options
})
md.linkify.set({ fuzzyLink: false })
if (options.preConfig) {
options.preConfig(md)
}
// custom plugins
md.use(componentPlugin)
.use(highlightLinePlugin)
.use(preWrapperPlugin, { hasSingleTheme })
.use(snippetPlugin, srcDir)
.use(containerPlugin, { hasSingleTheme })
.use(imagePlugin)
.use(
linkPlugin,
{ target: '_blank', rel: 'noreferrer', ...options.externalLinks },
base
)
.use(lineNumberPlugin, options.lineNumbers)
// 3rd party plugins
if (!options.attrs?.disable) {
md.use(attrsPlugin, options.attrs)
}
md.use(emojiPlugin)
// mdit-vue plugins
md.use(anchorPlugin, {
slugify,
permalink: anchorPlugin.permalink.linkInsideHeader({
symbol: '&ZeroWidthSpace;',
renderAttrs: (slug, state) => {
// Find `heading_open` with the id identical to slug
const idx = state.tokens.findIndex((token) => {
const attrs = token.attrs
const id = attrs?.find((attr) => attr[0] === 'id')
return id && slug === id[1]
})
// Get the actual heading content
const title = state.tokens[idx + 1].content
return {
'aria-label': `Permalink to "${title}"`
}
}
}),
...options.anchor
} as anchorPlugin.AnchorOptions).use(frontmatterPlugin, {
...options.frontmatter
} as FrontmatterPluginOptions)
if (options.headers) {
md.use(headersPlugin, {
level: [2, 3, 4, 5, 6],
slugify,
...(typeof options.headers === 'boolean' ? undefined : options.headers)
} as HeadersPluginOptions)
}
md.use(sfcPlugin, {
...options.sfc
} as SfcPluginOptions)
.use(titlePlugin)
.use(tocPlugin, {
...options.toc
} as TocPluginOptions)
// apply user config
if (options.config) {
options.config(md)
}
return md
}

@ -17,7 +17,7 @@ import {
type Processor
} from 'shiki-processor'
import type { Logger } from 'vite'
import type { ThemeOptions } from '../markdown'
import type { ThemeOptions } from '..'
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)

@ -3,9 +3,13 @@
// 2. normalize internal links to end with `.html`
import type MarkdownIt from 'markdown-it'
import type { MarkdownEnv } from '../env'
import { URL } from 'url'
import { EXTERNAL_URL_RE, PATHNAME_PROTOCOL_RE, isExternal } from '../../shared'
import {
EXTERNAL_URL_RE,
PATHNAME_PROTOCOL_RE,
isExternal,
type MarkdownEnv
} from '../../shared'
const indexRE = /(^|.*\/)index.md(#?.*)$/i

@ -1,8 +1,8 @@
import fs from 'fs-extra'
import path from 'path'
import type MarkdownIt from 'markdown-it'
import type { RuleBlock } from 'markdown-it/lib/parser_block'
import type { MarkdownEnv } from '../env'
import path from 'path'
import type { MarkdownEnv } from '../../shared'
export function dedent(text: string): string {
const lines = text.split('\n')

@ -6,7 +6,6 @@ import path from 'path'
import type { SiteConfig } from './config'
import {
createMarkdownRenderer,
type MarkdownEnv,
type MarkdownOptions,
type MarkdownRenderer
} from './markdown'
@ -14,6 +13,7 @@ import {
EXTERNAL_URL_RE,
slash,
type HeadConfig,
type MarkdownEnv,
type PageData
} from './shared'
import { getGitTimestamp } from './utils/getGitTimestamp'

@ -4,8 +4,13 @@ import MiniSearch from 'minisearch'
import path from 'path'
import type { Plugin, ViteDevServer } from 'vite'
import type { SiteConfig } from '../config'
import { createMarkdownRenderer, type MarkdownEnv } from '../markdown'
import { resolveSiteDataByRoute, slash, type DefaultTheme } from '../shared'
import { createMarkdownRenderer } from '../markdown'
import {
resolveSiteDataByRoute,
slash,
type DefaultTheme,
type MarkdownEnv
} from '../shared'
const debug = _debug('vitepress:local-search')
@ -45,23 +50,16 @@ export async function localSearchPlugin(
siteConfig.logger
)
const options = siteConfig.site.themeConfig.search.options || {}
function render(file: string) {
const { srcDir, cleanUrls = false, site } = siteConfig
const { srcDir, cleanUrls = false } = siteConfig
const relativePath = slash(path.relative(srcDir, file))
const env: MarkdownEnv = {
path: file,
relativePath,
cleanUrls
}
const html = md.render(fs.readFileSync(file, 'utf-8'), env)
if (
env.frontmatter?.search === false ||
(site.themeConfig.search?.provider === 'local' &&
site.themeConfig.search.options?.exclude?.(relativePath))
) {
return ''
}
return html
const env: MarkdownEnv = { path: file, relativePath, cleanUrls }
const src = fs.readFileSync(file, 'utf-8')
if (options._render) return options._render(src, env, md)
const html = md.render(src, env)
return env.frontmatter?.search === false ? '' : html
}
const indexByLocales = new Map<string, MiniSearch<IndexObject>>()
@ -72,8 +70,7 @@ export async function localSearchPlugin(
index = new MiniSearch<IndexObject>({
fields: ['title', 'titles', 'text'],
storeFields: ['title', 'titles'],
...(siteConfig.site.themeConfig?.search?.provider === 'local' &&
siteConfig.site.themeConfig.search.options?.miniSearch?.options)
...options.miniSearch?.options
})
indexByLocales.set(locale, index)
}

@ -3,6 +3,7 @@ export function serializeFunctions(value: any, key?: string): any {
return value.map((v) => serializeFunctions(v))
} else if (typeof value === 'object' && value !== null) {
return Object.keys(value).reduce((acc, key) => {
if (key[0] === '_') return acc
acc[key] = serializeFunctions(value[key], key)
return acc
}, {} as any)

@ -7,10 +7,11 @@ export type {
Header,
LocaleConfig,
LocaleSpecificConfig,
MarkdownEnv,
PageData,
PageDataPayload,
SiteData,
SSGContext
SSGContext,
SiteData
} from '../../types/shared'
export const EXTERNAL_URL_RE = /^[a-z]+:/i

@ -1,8 +1,9 @@
import type MarkdownIt from 'markdown-it'
import type { Options as MiniSearchOptions } from 'minisearch'
import type { ComputedRef, Ref } from 'vue'
import type { DocSearchProps } from './docsearch.js'
import type { LocalSearchTranslations } from './local-search.js'
import type { PageData } from './shared.js'
import type { MarkdownEnv, PageData } from './shared.js'
export namespace DefaultTheme {
export interface Config {
@ -383,15 +384,16 @@ export namespace DefaultTheme {
}
/**
* exclude content from search results
* Allows transformation of content before indexing (node only)
* Return empty string to skip indexing
*/
exclude?: (relativePath: string) => boolean
_render?: (src: string, env: MarkdownEnv, md: MarkdownIt) => string
}
// algolia -------------------------------------------------------------------
/**
* The Algolia search options. Partially copied from
* Algolia search options. Partially copied from
* `@docsearch/react/dist/esm/DocSearch.d.ts`
*/
export interface AlgoliaSearchOptions extends DocSearchProps {

39
types/shared.d.ts vendored

@ -1,4 +1,5 @@
// types shared between server and client
import type { MarkdownSfcBlocks } from '@mdit-vue/plugin-sfc'
import type { UseDarkOptions } from '@vueuse/core'
import type { SSRContext } from 'vue/server-renderer'
export type { DefaultTheme } from './default-theme.js'
@ -98,3 +99,41 @@ export type LocaleConfig<ThemeConfig = any> = Record<
string,
LocaleSpecificConfig<ThemeConfig> & { label: string; link?: string }
>
// Manually declaring all properties as rollup-plugin-dts
// is unable to merge augmented module declarations
export interface MarkdownEnv {
/**
* The raw Markdown content without frontmatter
*/
content?: string
/**
* The excerpt that extracted by `@mdit-vue/plugin-frontmatter`
*
* - Would be the rendered HTML when `renderExcerpt` is enabled
* - Would be the raw Markdown when `renderExcerpt` is disabled
*/
excerpt?: string
/**
* The frontmatter that extracted by `@mdit-vue/plugin-frontmatter`
*/
frontmatter?: Record<string, unknown>
/**
* The headers that extracted by `@mdit-vue/plugin-headers`
*/
headers?: Header[]
/**
* SFC blocks that extracted by `@mdit-vue/plugin-sfc`
*/
sfcBlocks?: MarkdownSfcBlocks
/**
* The title that extracted by `@mdit-vue/plugin-title`
*/
title?: string
path: string
relativePath: string
cleanUrls: boolean
links?: string[]
includes?: string[]
}

Loading…
Cancel
Save