feat: createContentLoader

__tests__/e2e/.vitepress/config.ts

@@ -22,11 +22,11 @@ const sidebar: DefaultTheme.Config['sidebar'] = {
       ]
     },
     {
-      text: 'Static Data',
+      text: 'Data Loading',
       items: [
         {
           text: 'Test Page',
-          link: '/static-data/data'
+          link: '/data-loading/data'
         }
       ]
     },

__tests__/e2e/data-loading/content/bar.md

@@ -0,0 +1,9 @@
+---
+title: bar
+---
+
+Hello
+
+---
+
+world

__tests__/e2e/data-loading/content/foo.md

@@ -0,0 +1,9 @@
+---
+title: foo
+---
+
+Hello
+
+---
+
+world

__tests__/e2e/data-loading/contentLoader.data.ts

@@ -0,0 +1,13 @@
+import { createContentLoader } from 'vitepress'
+
+export default createContentLoader('data-loading/content/*.md', {
+  includeSrc: true,
+  excerpt: true,
+  render: true,
+  transform(data) {
+    return data.map((item) => ({
+      ...item,
+      transformed: true
+    }))
+  }
+})

__tests__/e2e/data-loading/data.md

@@ -0,0 +1,10 @@
+# Static Data
+
+<script setup lang="ts">
+import { data } from './basic.data.js'
+import { data as contentData } from './contentLoader.data.js'
+</script>
+
+<pre id="basic">{{ data }}</pre>
+
+<pre id="content">{{ contentData }}</pre>

__tests__/e2e/data-loading/data.test.ts

@@ -0,0 +1,42 @@
+describe('static data file support in vite 3', () => {
+  beforeAll(async () => {
+    await goto('/data-loading/data')
+  })
+
+  test('render correct content', async () => {
+    expect(await page.textContent('pre#basic')).toMatchInlineSnapshot(`
+      "[
+        {
+          \\"foo\\": true
+        },
+        {
+          \\"bar\\": true
+        }
+      ]"
+    `)
+    expect(await page.textContent('pre#content')).toMatchInlineSnapshot(`
+      "[
+        {
+          \\"src\\": \\"---\\\\ntitle: bar\\\\n---\\\\n\\\\nHello\\\\n\\\\n---\\\\n\\\\nworld\\\\n\\",
+          \\"html\\": \\"<p>Hello</p>\\\\n<hr>\\\\n<p>world</p>\\\\n\\",
+          \\"frontmatter\\": {
+            \\"title\\": \\"bar\\"
+          },
+          \\"excerpt\\": \\"<p>Hello</p>\\\\n\\",
+          \\"url\\": \\"/data-loading/content/bar.html\\",
+          \\"transformed\\": true
+        },
+        {
+          \\"src\\": \\"---\\\\ntitle: foo\\\\n---\\\\n\\\\nHello\\\\n\\\\n---\\\\n\\\\nworld\\\\n\\",
+          \\"html\\": \\"<p>Hello</p>\\\\n<hr>\\\\n<p>world</p>\\\\n\\",
+          \\"frontmatter\\": {
+            \\"title\\": \\"foo\\"
+          },
+          \\"excerpt\\": \\"<p>Hello</p>\\\\n\\",
+          \\"url\\": \\"/data-loading/content/foo.html\\",
+          \\"transformed\\": true
+        }
+      ]"
+    `)
+  })
+})

__tests__/e2e/multi-sidebar/index.test.ts

@@ -12,7 +12,7 @@ describe('test multi sidebar sort root', () => {
     expect(sidebarContent).toEqual([
       'Frontmatter',
       '& <Text Literals &> code',
-      'Static Data',
+      'Data Loading',
       'Multi Sidebar Test',
       'Dynamic Routes',
       'Markdown Extensions'

__tests__/e2e/static-data/__snapshots__/data.test.ts.snap

@@ -1,14 +0,0 @@
-// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
-
-exports[`static data file support in vite 3 > render correct content 1`] = `
-[
-  "[
-  {
-    \\"foo\\": true
-  },
-  {
-    \\"bar\\": true
-  }
-]",
-]
-`;

__tests__/e2e/static-data/data.md

@@ -1,7 +0,0 @@
-# Static Data
-
-<script setup lang="ts">
-import { data } from './static.data.js'
-</script>
-
-{{ data }}

__tests__/e2e/static-data/data.test.ts

@@ -1,12 +0,0 @@
-describe('static data file support in vite 3', () => {
-  beforeAll(async () => {
-    await goto('/static-data/data')
-  })
-
-  test('render correct content', async () => {
-    const pLocator = page.locator('.VPContent p')
-
-    const pContents = await pLocator.allTextContents()
-    expect(pContents).toMatchSnapshot()
-  })
-})

__tests__/tsconfig.json

@@ -1,6 +1,7 @@
 {
   "extends": "../tsconfig.json",
   "compilerOptions": {
+    "isolatedModules": false,
     "baseUrl": ".",
     "types": ["node", "vitest/globals"],
     "paths": {

docs/guide/data-loading.md

@@ -56,29 +56,118 @@ export default {
 
 When you need to generate data based on local files, you should use the `watch` option in the data loader so that changes made to these files can trigger hot updates.
 
-The `watch` option is also convenient in that you can use [glob patterns](https://github.com/mrmlnc/fast-glob#pattern-syntax) to match multiple files. The patterns can be relative to the loader file itself, and the `load()` function will receive the matched files as absolute paths:
+The `watch` option is also convenient in that you can use [glob patterns](https://github.com/mrmlnc/fast-glob#pattern-syntax) to match multiple files. The patterns can be relative to the loader file itself, and the `load()` function will receive the matched files as absolute paths.
+
+The following example shows loading CSV files and transforming them into JSON using [csv-parse](https://github.com/adaltas/node-csv/tree/master/packages/csv-parse/). Because this file only executes at build time, you will not be shipping the CSV parser to the client!
 
 ```js
 import fs from 'node:fs'
-import parseFrontmatter from 'gray-matter'
+import { parse } from 'csv-parse/sync'
 
 export default {
-  // watch all blog posts
+  watch: ['./data/*.csv'],
-  watch: ['./posts/*.md'],
   load(watchedFiles) {
     // watchedFiles will be an array of absolute paths of the matched files.
     // generate an array of blog post metadata that can be used to render
     // a list in the theme layout
     return watchedFiles.map(file => {
-      const content = fs.readFileSync(file, 'utf-8')
+      return parse(fs.readFileSync(file, 'utf-8'), {
-      const { data, excerpt } = parseFrontmatter(content)
+        columns: true,
-      return {
+        skip_empty_lines: true
-        file,
+      })
-        data,
+    })
-        excerpt
+  }
-      }
+}
+```
+
+## `createContentLoader`
+
+When building a content focused site, we often need to create an "archive" or "index" page: a page where we list all available entries in our content collection, for example blog posts or API pages. We **can** implement this directly with the data loader API, but since this is such a common use case, VitePress also provides a `createContentLoader` helper to simplify this:
+
+```js
+// posts.data.js
+import { createContentLoader } from 'vitepress'
+
+export default createContentLoader('posts/*.md', /* options */)
+```
+
+The helper takes a glob pattern relative to [project root](./routing#project-root), and returns a `{ watch, load }` data loader object that can be used as the default export in a data loader file. It also implements caching based on file modified timestamps to improve dev performance.
+
+Note the loader only works with Markdown files - matched non-Markdown files will be skipped.
+
+The loaded data will be an array with the type of `ContentData[]`:
+
+```ts
+interface ContentData {
+  // mapped absolute URL for the page. e.g. /posts/hello.html
+  url: string
+  // frontmatter data of the page
+  frontmatter: Record<string, any>
+
+  // the following are only present if relevant options are enabled
+  // we will discuss them below
+  src: string | undefined
+  html: string | undefined
+  excerpt: string | undefined
+}
+```
+
+By default, only `url` and `frontmatter` are provided. This is because the loaded data will be inlined as JSON in the client bundle, so we need to be cautious about its size. Here's an example using the data to build a minimal blog index page:
+
+```vue
+<script setup>
+import { data as posts } from './posts.data.js'
+</script>
+
+<template>
+  <h1>All Blog Posts</h1>
+  <ul>
+    <li v-for="post of posts">
+      <a :href="post.url">{{ post.frontmatter.title }}</a>
+      <span>by {{ post.frontmatter.author }}</span>
+    </li>
+  </ul>
+</template>
+```
+
+### Options
+
+The default data may not suit all needs - you can opt-in to transform the data using options:
+
+```js
+// posts.data.js
+import { createContentLoader } from 'vitepress'
+
+export default createContentLoader('posts/*.md', {
+  includeSrc: true, // include raw markdown source?
+  render: true,     // include rendered full page HTML?
+  excerpt: true,    // include excerpt?
+  transform(rawData) {
+    // map, sort, or filter the raw data as you wish.
+    // the final result is what will be shipped to the client.
+    return rawData.sort((a, b) => {
+      return +new Date(b.frontmatter.date) - +new Date(a.frontmatter.date)
+    }).map(page => {
+      page.src  // raw markdown source
+      page.html // rendered full page HTML
+      page.excerpt // rendered excerpt HTML (content above first `---`)
+      return {/* ... */}
     })
   }
+})
+```
+
+Check out how it is used in the [Vue.js blog](https://github.com/vuejs/blog/blob/main/.vitepress/theme/posts.data.ts).
+
+The `createContentLoader` API can also be used inside [build hooks](/reference/site-config#build-hooks):
+
+```js
+// .vitepress/config.js
+export default {
+  async buildEnd() {
+    const posts = await createContentLoader('posts/*.md').load()
+    // generate files based on posts metadata, e.g. RSS feed
+  }
 }
 ```
 

package.json

@@ -135,6 +135,7 @@
     "fast-glob": "^3.2.12",
     "fs-extra": "^11.1.0",
     "get-port": "^6.1.2",
+    "gray-matter": "^4.0.3",
     "lint-staged": "^13.2.0",
     "lodash.template": "^4.5.0",
     "lru-cache": "^7.18.3",

pnpm-lock.yaml

@@ -51,6 +51,7 @@ importers:
       fast-glob: ^3.2.12
       fs-extra: ^11.1.0
       get-port: ^6.1.2
+      gray-matter: ^4.0.3
       lint-staged: ^13.2.0
       lodash.template: ^4.5.0
       lru-cache: ^7.18.3
@@ -140,6 +141,7 @@ importers:
       fast-glob: 3.2.12
       fs-extra: 11.1.0
       get-port: 6.1.2
+      gray-matter: 4.0.3
       lint-staged: 13.2.0_dx6s57r75rxv5zregmjdjjgmei
       lodash.template: 4.5.0
       lru-cache: 7.18.3

src/node/config.ts

@@ -287,6 +287,10 @@ export async function resolveConfig(
     userConfig
   }
 
+  // to be shared with content loaders
+  // @ts-ignore
+  global.VITEPRESS_CONFIG = config
+
   return config
 }
 

src/node/cotentLoader.ts

@@ -0,0 +1,140 @@
+import fs from 'fs'
+import path from 'path'
+import glob from 'fast-glob'
+import type { SiteConfig } from './config'
+import matter from 'gray-matter'
+import { normalizePath } from 'vite'
+import { createMarkdownRenderer, type MarkdownRenderer } from './markdown'
+
+export interface ContentOptions<T = ContentData[]> {
+  /**
+   * Include src?
+   * default: false
+   */
+  includeSrc?: boolean
+  /**
+   * Render src to HTML and include in data?
+   * default: false
+   */
+  render?: boolean
+  /**
+   * Whether to parse and include excerpt (rendered as HTML)
+   * default: false
+   */
+  excerpt?: boolean
+  /**
+   * Transform the data. Note the data will be inlined as JSON in the client
+   * bundle if imported from components or markdown files.
+   */
+  transform?: (data: ContentData[]) => T | Promise<T>
+}
+
+export interface ContentData {
+  url: string
+  src: string | undefined
+  html: string | undefined
+  frontmatter: Record<string, any>
+  excerpt: string | undefined
+}
+
+/**
+ * Create a loader object that can be directly used as the default export
+ * of a data loader file.
+ */
+export function createContentLoader<T = ContentData[]>(
+  /**
+   * files to glob / watch - relative to <project root>
+   */
+  pattern: string | string[],
+  {
+    includeSrc,
+    render,
+    excerpt: renderExcerpt,
+    transform
+  }: ContentOptions<T> = {}
+): {
+  watch: string | string[]
+  load: () => Promise<T>
+} {
+  const config: SiteConfig = (global as any).VITEPRESS_CONFIG
+  if (!config) {
+    throw new Error(
+      'content loader invoked without an active vitepress process, ' +
+        'or before vitepress config is resolved.'
+    )
+  }
+
+  if (typeof pattern === 'string') pattern = [pattern]
+  pattern = pattern.map((p) => normalizePath(path.join(config.root, p)))
+
+  let md: MarkdownRenderer
+
+  const cache = new Map<
+    string,
+    {
+      data: any
+      timestamp: number
+    }
+  >()
+
+  return {
+    watch: pattern,
+    async load(files?: string[]) {
+      if (!files) {
+        // the loader is being called directly, do a fresh glob
+        files = (
+          await glob(pattern, {
+            ignore: ['**/node_modules/**', '**/dist/**']
+          })
+        ).sort()
+      }
+
+      md =
+        md ||
+        (await createMarkdownRenderer(
+          config.srcDir,
+          config.markdown,
+          config.site.base,
+          config.logger
+        ))
+
+      const raw: ContentData[] = []
+
+      for (const file of files) {
+        if (!file.endsWith('.md')) {
+          continue
+        }
+        const timestamp = fs.statSync(file).mtimeMs
+        const cached = cache.get(file)
+        if (cached && timestamp === cached.timestamp) {
+          raw.push(cached.data)
+        } else {
+          const src = fs.readFileSync(file, 'utf-8')
+          const { data: frontmatter, excerpt } = matter(src, {
+            excerpt: true
+          })
+          const url =
+            '/' +
+            normalizePath(path.relative(config.root, file)).replace(
+              /\.md$/,
+              config.cleanUrls ? '' : '.html'
+            )
+          const html = render ? md.render(src) : undefined
+          const renderedExcerpt = renderExcerpt
+            ? excerpt && md.render(excerpt)
+            : undefined
+          const data: ContentData = {
+            src: includeSrc ? src : undefined,
+            html,
+            frontmatter,
+            excerpt: renderedExcerpt,
+            url
+          }
+          cache.set(file, { data, timestamp })
+          raw.push(data)
+        }
+      }
+      return (transform ? transform(raw) : raw) as any
+    }
+  }
+}

src/node/index.ts

@@ -4,6 +4,7 @@ export * from './markdown'
 export * from './build/build'
 export * from './serve/serve'
 export * from './init/init'
+export * from './cotentLoader'
 export { defineLoader, type LoaderModule } from './plugins/staticDataPlugin'
 export { loadEnv } from 'vite'