feat!: consistent glob options across content, data, and path loaders (#4808)

BREAKING CHANGES: Only `cwd`, `ignore`, `dot` and `debug` are supported in `globOptions` of `createContentLoader`. If you want to pass other options, you still can but you might need to suppress type errors.
pull/4813/head
Divyansh Singh 3 months ago committed by GitHub
parent 56ba65e130
commit 7619521259
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -2248,7 +2248,6 @@ This version uses Vue 3.2.0.
### BREAKING CHANGES
- Some config options have changed.
- `vueOptions` renamed to `vue`
- `alias` option has been removed. Use `vite.resovle.alias` instead.
@ -2266,7 +2265,6 @@ This version uses Vue 3.2.0.
### BREAKING CHANGES
- The following methods are removed.
- `useSiteData`
- `useSiteDataByRoute`
- `usePageData`

@ -16,6 +16,6 @@
"postcss-rtlcss": "^5.7.1",
"vitepress": "workspace:*",
"vitepress-plugin-group-icons": "^1.6.0",
"vitepress-plugin-llms": "^1.5.0"
"vitepress-plugin-llms": "^1.5.1"
}
}

@ -97,21 +97,21 @@
"dependencies": {
"@docsearch/css": "^3.9.0",
"@docsearch/js": "^3.9.0",
"@iconify-json/simple-icons": "^1.2.38",
"@shikijs/core": "^3.6.0",
"@shikijs/transformers": "^3.6.0",
"@shikijs/types": "^3.6.0",
"@iconify-json/simple-icons": "^1.2.39",
"@shikijs/core": "^3.7.0",
"@shikijs/transformers": "^3.7.0",
"@shikijs/types": "^3.7.0",
"@vitejs/plugin-vue": "^5.2.4",
"@vue/devtools-api": "^7.7.6",
"@vue/shared": "^3.5.16",
"@vueuse/core": "^13.3.0",
"@vueuse/integrations": "^13.3.0",
"@vue/devtools-api": "^7.7.7",
"@vue/shared": "^3.5.17",
"@vueuse/core": "^13.4.0",
"@vueuse/integrations": "^13.4.0",
"focus-trap": "^7.6.5",
"mark.js": "8.11.1",
"minisearch": "^7.1.2",
"shiki": "^3.6.0",
"shiki": "^3.7.0",
"vite": "^6.3.5",
"vue": "^3.5.16"
"vue": "^3.5.17"
},
"devDependencies": {
"@clack/prompts": "^1.0.0-alpha.1",
@ -125,7 +125,7 @@
"@mdit-vue/shared": "^2.1.4",
"@polka/compression": "^1.0.0-next.28",
"@rollup/plugin-alias": "^5.1.1",
"@rollup/plugin-commonjs": "^28.0.3",
"@rollup/plugin-commonjs": "^28.0.6",
"@rollup/plugin-json": "^6.1.0",
"@rollup/plugin-node-resolve": "^16.0.1",
"@rollup/plugin-replace": "^6.0.2",
@ -139,7 +139,7 @@
"@types/markdown-it-container": "^2.0.10",
"@types/markdown-it-emoji": "^3.0.1",
"@types/minimist": "^1.2.5",
"@types/node": "^24.0.1",
"@types/node": "^24.0.3",
"@types/picomatch": "^4.0.0",
"@types/postcss-prefix-selector": "^1.16.3",
"@types/prompts": "^2.4.9",
@ -152,7 +152,7 @@
"fs-extra": "^11.3.0",
"get-port": "^7.1.0",
"gray-matter": "^4.0.3",
"lint-staged": "^16.1.0",
"lint-staged": "^16.1.2",
"lodash.template": "^4.5.0",
"lru-cache": "^11.1.0",
"markdown-it": "^14.1.0",
@ -165,20 +165,20 @@
"minimist": "^1.2.8",
"nanoid": "^5.1.5",
"ora": "^8.2.0",
"oxc-minify": "^0.72.3",
"oxc-minify": "^0.74.0",
"p-map": "^7.0.3",
"path-to-regexp": "^6.3.0",
"picocolors": "^1.1.1",
"picomatch": "^4.0.2",
"package-directory": "^8.1.0",
"playwright-chromium": "^1.53.0",
"playwright-chromium": "^1.53.1",
"polka": "^1.0.0-next.28",
"postcss-prefix-selector": "^2.1.1",
"prettier": "^3.5.3",
"prettier": "^3.6.0",
"prompts": "^2.4.2",
"punycode": "^2.3.1",
"rimraf": "^6.0.1",
"rollup": "^4.43.0",
"rollup": "^4.44.0",
"rollup-plugin-dts": "6.1.1",
"rollup-plugin-esbuild": "^6.2.1",
"semver": "^7.7.2",
@ -187,13 +187,13 @@
"sitemap": "^8.0.0",
"tinyglobby": "^0.2.14",
"typescript": "^5.8.3",
"vitest": "^3.2.3",
"vue-tsc": "^3.0.0-alpha.10",
"vitest": "^3.2.4",
"vue-tsc": "^3.0.0-beta.3",
"wait-on": "^8.0.3"
},
"peerDependencies": {
"markdown-it-mathjax3": "^4",
"oxc-minify": "^0.72.3",
"oxc-minify": "^0.74.0",
"postcss": "^8"
},
"peerDependenciesMeta": {
@ -207,7 +207,7 @@
"optional": true
}
},
"packageManager": "pnpm@10.12.1",
"packageManager": "pnpm@10.12.2",
"pnpm": {
"peerDependencyRules": {
"ignoreMissing": [

File diff suppressed because it is too large Load Diff

@ -2,7 +2,6 @@ import _debug from 'debug'
import fs from 'fs-extra'
import path from 'node:path'
import c from 'picocolors'
import { glob } from 'tinyglobby'
import {
createLogger,
loadConfigFromFile,
@ -24,6 +23,7 @@ import {
type SiteData
} from './shared'
import type { RawConfigExports, SiteConfig, UserConfig } from './siteConfig'
import { glob } from './utils/glob'
export { resolvePages } from './plugins/dynamicRoutesPlugin'
export { resolveSiteDataByRoute } from './shared'
@ -190,11 +190,9 @@ async function gatherAdditionalConfig(
) {
//
const candidates = await glob(additionalConfigGlob, {
const candidates = await glob([additionalConfigGlob], {
cwd: path.resolve(root, srcDir),
dot: false, // conveniently ignores .vitepress/*
ignore: ['**/node_modules/**', ...srcExclude],
expandDirectories: false
ignore: srcExclude
})
const deps: string[][] = []

@ -1,10 +1,11 @@
import fs from 'fs-extra'
import matter from 'gray-matter'
import path from 'node:path'
import { glob, type GlobOptions } from 'tinyglobby'
import { normalizePath } from 'vite'
import type { SiteConfig } from './config'
import { createMarkdownRenderer } from './markdown/markdown'
import type { Awaitable } from './shared'
import { glob, normalizeGlob, type GlobOptions } from './utils/glob'
export interface ContentOptions<T = ContentData[]> {
/**
@ -48,12 +49,10 @@ export interface ContentOptions<T = ContentData[]> {
* Transform the data. Note the data will be inlined as JSON in the client
* bundle if imported from components or markdown files.
*/
transform?: (data: ContentData[]) => T | Promise<T>
transform?: (data: ContentData[]) => Awaitable<T>
/**
* Options to pass to `tinyglobby`.
* You'll need to manually specify `node_modules` and `dist` in
* `globOptions.ignore` if you've overridden it.
* Options to pass to `tinyglobby` and `picomatch` for globbing.
*/
globOptions?: GlobOptions
}
@ -74,18 +73,9 @@ export function createContentLoader<T = ContentData[]>(
/**
* files to glob / watch - relative to srcDir
*/
pattern: string | string[],
{
includeSrc,
render,
excerpt: renderExcerpt,
transform,
globOptions
}: ContentOptions<T> = {}
): {
watch: string | string[]
load: () => Promise<T>
} {
watch: string | string[],
options: ContentOptions<T> = {}
) {
const config: SiteConfig = (global as any).VITEPRESS_CONFIG
if (!config) {
throw new Error(
@ -94,24 +84,17 @@ export function createContentLoader<T = ContentData[]>(
)
}
if (typeof pattern === 'string') pattern = [pattern]
pattern = pattern.map((p) => normalizePath(path.join(config.srcDir, p)))
const cache = new Map<string, { data: any; timestamp: number }>()
watch = normalizeGlob(watch, config.srcDir)
return {
watch: pattern,
watch,
options: { globOptions: options.globOptions },
async load(files?: string[]) {
if (!files) {
// the loader is being called directly, do a fresh glob
files = (
await glob(pattern, {
ignore: ['**/node_modules/**', '**/dist/**'],
expandDirectories: false,
...globOptions
})
).sort()
}
files = files ?? (await glob(watch, options.globOptions))
const md = await createMarkdownRenderer(
config.srcDir,
@ -123,43 +106,51 @@ export function createContentLoader<T = ContentData[]>(
const raw: ContentData[] = []
for (const file of files) {
if (!file.endsWith('.md')) {
continue
}
if (!file.endsWith('.md')) continue
const timestamp = fs.statSync(file).mtimeMs
const cached = cache.get(file)
if (cached && timestamp === cached.timestamp) {
raw.push(cached.data)
} else {
//
const src = fs.readFileSync(file, 'utf-8')
const renderExcerpt = options.excerpt
const { data: frontmatter, excerpt } = matter(
src,
// @ts-expect-error gray-matter types are wrong
typeof renderExcerpt === 'string'
? { excerpt_separator: renderExcerpt }
: { excerpt: renderExcerpt }
: { excerpt: renderExcerpt as any } // gray-matter types are wrong
)
const url =
'/' +
normalizePath(path.relative(config.srcDir, file))
.replace(/(^|\/)index\.md$/, '$1')
.replace(/\.md$/, config.cleanUrls ? '' : '.html')
const html = render ? await md.renderAsync(src) : undefined
const html = options.render ? await md.renderAsync(src) : undefined
const renderedExcerpt = renderExcerpt
? excerpt && (await md.renderAsync(excerpt))
: undefined
const data: ContentData = {
src: includeSrc ? src : undefined,
src: options.includeSrc ? src : undefined,
html,
frontmatter,
excerpt: renderedExcerpt,
url
}
cache.set(file, { data, timestamp })
raw.push(data)
}
}
return (transform ? transform(raw) : raw) as any
return options.transform?.(raw) ?? (raw as T)
}
}
}

@ -1,8 +1,7 @@
import fs from 'fs-extra'
import path from 'node:path'
import c from 'picocolors'
import { isMatch } from 'picomatch'
import { glob } from 'tinyglobby'
import pm from 'picomatch'
import {
loadConfigFromFile,
normalizePath,
@ -12,6 +11,7 @@ import {
} from 'vite'
import type { Awaitable } from '../shared'
import { type SiteConfig, type UserConfig } from '../siteConfig'
import { glob, normalizeGlob, type GlobOptions } from '../utils/glob'
import { ModuleGraph } from '../utils/moduleGraph'
import { resolveRewrites } from './rewritesPlugin'
@ -45,13 +45,15 @@ export interface RouteModule {
| UserRouteConfig[]
| ((watchedFiles: string[]) => Awaitable<UserRouteConfig[]>)
transformPageData?: UserConfig['transformPageData']
options?: { globOptions?: GlobOptions }
}
interface ResolvedRouteModule {
watch: string[] | undefined
routes: ResolvedRouteConfig[] | undefined
watch: string[]
routes?: ResolvedRouteConfig[]
loader: RouteModule['paths']
transformPageData?: RouteModule['transformPageData']
options: NonNullable<RouteModule['options']>
}
const dynamicRouteRE = /\[(\w+?)\]/g
@ -63,7 +65,7 @@ let moduleGraph = new ModuleGraph()
/**
* Helper for defining routes with type inference
*/
export function defineRoutes(loader: RouteModule) {
export function defineRoutes(loader: RouteModule): RouteModule {
return loader
}
@ -78,23 +80,10 @@ export async function resolvePages(
routeModuleCache.clear()
}
// Important: tinyglobby doesn't guarantee order of the returned files.
// We must sort the pages so the input list to rollup is stable across
// builds - otherwise different input order could result in different exports
// order in shared chunks which in turns invalidates the hash of every chunk!
// JavaScript built-in sort() is mandated to be stable as of ES2019 and
// supported in Node 12+, which is required by Vite.
const allMarkdownFiles = (
await glob(['**/*.md'], {
const allMarkdownFiles = await glob(['**/*.md'], {
cwd: srcDir,
ignore: [
'**/node_modules/**',
'**/dist/**',
...(userConfig.srcExclude || [])
],
expandDirectories: false
ignore: userConfig.srcExclude
})
).sort()
const pages: string[] = []
const dynamicRouteFiles: string[] = []
@ -137,9 +126,7 @@ export const dynamicRoutesPlugin = async (
const matched = config.dynamicRoutes.find(
(r) => r.fullPath === normalizedId
)
if (matched) {
return normalizedId
}
if (matched) return normalizedId
},
load(id) {
@ -180,23 +167,22 @@ export const dynamicRoutesPlugin = async (
for (const id of moduleGraph.delete(normalizedFile)) {
routeModuleCache.delete(id)
const mod = this.environment.moduleGraph.getModuleById(id)
if (mod) {
modules.push(mod)
}
if (mod) modules.push(mod)
}
// Also check if the file matches any custom watch patterns.
let watchedFileChanged = false
for (const [file, route] of routeModuleCache) {
if (route.watch && isMatch(normalizedFile, route.watch)) {
if (
route.watch?.length &&
pm(route.watch, route.options.globOptions)(normalizedFile)
) {
route.routes = undefined
watchedFileChanged = true
for (const id of moduleGraph.delete(file)) {
const mod = this.environment.moduleGraph.getModuleById(id)
if (mod) {
modules.push(mod)
}
if (mod) modules.push(mod)
}
}
}
@ -255,7 +241,8 @@ async function resolveDynamicRoutes(
// load the paths loader module
let watch: ResolvedRouteModule['watch']
let loader: ResolvedRouteModule['loader']
let extras: Partial<ResolvedRouteModule>
let transformPageData: ResolvedRouteModule['transformPageData']
let options: ResolvedRouteModule['options']
const loaderPath = normalizePath(pathsFile)
const existing = routeModuleCache.get(loaderPath)
@ -267,7 +254,7 @@ async function resolveDynamicRoutes(
continue
}
;({ watch, loader, ...extras } = existing)
;({ watch, loader, transformPageData, options } = existing)
} else {
let mod
try {
@ -294,8 +281,11 @@ async function resolveDynamicRoutes(
continue
}
// @ts-ignore
;({ paths: loader, watch, ...extras } = mod.config)
const loaderModule = mod.config as RouteModule
watch = normalizeGlob(loaderModule.watch, path.dirname(pathsFile))
loader = loaderModule.paths
transformPageData = loaderModule.transformPageData
options = loaderModule.options || {}
if (!loader) {
logger.warn(
@ -307,15 +297,6 @@ async function resolveDynamicRoutes(
continue
}
watch = typeof watch === 'string' ? [watch] : watch
if (watch) {
watch = watch.map((p) =>
p.startsWith('.')
? normalizePath(path.resolve(path.dirname(pathsFile), p))
: normalizePath(p)
)
}
// record deps for hmr
newModuleGraph.add(
loaderPath,
@ -327,15 +308,7 @@ async function resolveDynamicRoutes(
let pathsData: UserRouteConfig[]
if (typeof loader === 'function') {
let watchedFiles: string[] = []
if (watch) {
watchedFiles = (
await glob(watch, {
ignore: ['**/node_modules/**', '**/dist/**'],
expandDirectories: false
})
).sort()
}
const watchedFiles = await glob(watch, options.globOptions)
pathsData = await loader(watchedFiles)
} else {
pathsData = loader
@ -355,7 +328,8 @@ async function resolveDynamicRoutes(
}
})
routeModuleCache.set(loaderPath, { ...extras, watch, routes, loader })
const mod = { watch, routes, loader, transformPageData, options }
routeModuleCache.set(loaderPath, mod)
return routes
}

@ -1,33 +1,37 @@
import path from 'node:path'
import { isMatch } from 'picomatch'
import { glob } from 'tinyglobby'
import pm from 'picomatch'
import {
loadConfigFromFile,
normalizePath,
type EnvironmentModuleNode,
type Plugin,
type ViteDevServer,
loadConfigFromFile,
normalizePath
type ViteDevServer
} from 'vite'
import type { Awaitable } from '../shared'
import { glob, normalizeGlob, type GlobOptions } from '../utils/glob'
const loaderMatch = /\.data\.m?(j|t)s($|\?)/
let server: ViteDevServer
export interface LoaderModule {
export interface LoaderModule<T = any> {
watch?: string[] | string
load: (watchedFiles: string[]) => any
load: (watchedFiles: string[]) => Awaitable<T>
options?: { globOptions?: GlobOptions }
}
/**
* Helper for defining loaders with type inference
*/
export function defineLoader(loader: LoaderModule) {
export function defineLoader<T>(loader: LoaderModule<T>): LoaderModule<T> {
return loader
}
// Map from loader module id to its module info
const idToLoaderModulesMap: Record<string, LoaderModule | undefined> =
Object.create(null)
const idToLoaderModulesMap: Record<
string,
(Required<Omit<LoaderModule, 'watch'>> & { watch: string[] }) | undefined
> = Object.create(null)
// Map from dependency file to a set of loader module ids
const depToLoaderModuleIdsMap: Record<string, Set<string>> = Object.create(null)
@ -56,9 +60,7 @@ export const staticDataPlugin: Plugin = {
if (loaderMatch.test(id)) {
let _resolve: ((res: any) => void) | undefined
if (isBuild) {
if (idToPendingPromiseMap[id]) {
return idToPendingPromiseMap[id]
}
if (idToPendingPromiseMap[id]) return idToPendingPromiseMap[id]
idToPendingPromiseMap[id] = new Promise((r) => {
_resolve = r
})
@ -67,10 +69,11 @@ export const staticDataPlugin: Plugin = {
const base = path.dirname(id)
let watch: LoaderModule['watch']
let load: LoaderModule['load']
let options: LoaderModule['options']
const existing = idToLoaderModulesMap[id]
if (existing) {
;({ watch, load } = existing)
;({ watch, load, options } = existing)
} else {
// use vite's load config util as a way to load Node.js file with
// TS & native ESM support
@ -88,36 +91,17 @@ export const staticDataPlugin: Plugin = {
}
const loaderModule = res?.config as LoaderModule
watch =
typeof loaderModule.watch === 'string'
? [loaderModule.watch]
: loaderModule.watch
if (watch) {
watch = watch.map((p) => {
return p.startsWith('.')
? normalizePath(path.resolve(base, p))
: normalizePath(p)
})
}
watch = normalizeGlob(loaderModule.watch, base)
load = loaderModule.load
options = loaderModule.options || {}
}
// load the data
let watchedFiles: string[] = []
if (watch) {
watchedFiles = (
await glob(watch, {
ignore: ['**/node_modules/**', '**/dist/**'],
expandDirectories: false
})
).sort()
}
const watchedFiles = await glob(watch, options.globOptions)
const data = await load(watchedFiles)
// record loader module for HMR
if (server) {
idToLoaderModulesMap[id] = { watch, load }
}
if (server) idToLoaderModulesMap[id] = { watch, load, options }
const result = `export const data = JSON.parse(${JSON.stringify(JSON.stringify(data))})`
@ -139,20 +123,19 @@ export const staticDataPlugin: Plugin = {
)) {
delete idToLoaderModulesMap[id]
const mod = this.environment.moduleGraph.getModuleById(id)
if (mod) {
modules.push(mod)
}
if (mod) modules.push(mod)
}
}
// Also check if the file matches any custom watch patterns.
for (const id in idToLoaderModulesMap) {
const loader = idToLoaderModulesMap[id]
if (loader && loader.watch && isMatch(normalizedFile, loader.watch)) {
if (
loader?.watch?.length &&
pm(loader.watch, loader.options.globOptions)(normalizedFile)
) {
const mod = this.environment.moduleGraph.getModuleById(id)
if (mod) {
modules.push(mod)
}
if (mod) modules.push(mod)
}
}

@ -0,0 +1,35 @@
import path from 'node:path'
import { glob as _glob } from 'tinyglobby'
import { normalizePath } from 'vite'
export interface GlobOptions {
cwd?: string
ignore?: string | string[]
dot?: boolean
debug?: boolean
}
export function normalizeGlob(
patterns: string[] | string | undefined,
base: string
): string[] {
if (!patterns) return []
if (typeof patterns === 'string') patterns = [patterns]
return patterns.map((p) =>
p.startsWith('.') ? normalizePath(path.resolve(base, p)) : normalizePath(p)
)
}
export async function glob(
patterns: string[] | undefined,
options?: GlobOptions
): Promise<string[]> {
if (!patterns?.length) return []
return (
await _glob(patterns, {
expandDirectories: false,
...options,
ignore: ['**/node_modules/**', '**/dist/**', ...(options?.ignore || [])]
})
).sort()
}
Loading…
Cancel
Save