feat: createContentLoader

pull/2040/head
Evan You 2 years ago
parent 905f58b2a8
commit d2838e3755

@ -22,11 +22,11 @@ const sidebar: DefaultTheme.Config['sidebar'] = {
]
},
{
text: 'Static Data',
text: 'Data Loading',
items: [
{
text: 'Test Page',
link: '/static-data/data'
link: '/data-loading/data'
}
]
},

@ -0,0 +1,9 @@
---
title: bar
---
Hello
---
world

@ -0,0 +1,9 @@
---
title: foo
---
Hello
---
world

@ -0,0 +1,13 @@
import { createContentLoader } from 'vitepress'
export default createContentLoader('data-loading/content/*.md', {
includeSrc: true,
excerpt: true,
render: true,
transform(data) {
return data.map((item) => ({
...item,
transformed: true
}))
}
})

@ -0,0 +1,10 @@
# Static Data
<script setup lang="ts">
import { data } from './basic.data.js'
import { data as contentData } from './contentLoader.data.js'
</script>
<pre id="basic">{{ data }}</pre>
<pre id="content">{{ contentData }}</pre>

@ -0,0 +1,42 @@
describe('static data file support in vite 3', () => {
beforeAll(async () => {
await goto('/data-loading/data')
})
test('render correct content', async () => {
expect(await page.textContent('pre#basic')).toMatchInlineSnapshot(`
"[
{
\\"foo\\": true
},
{
\\"bar\\": true
}
]"
`)
expect(await page.textContent('pre#content')).toMatchInlineSnapshot(`
"[
{
\\"src\\": \\"---\\\\ntitle: bar\\\\n---\\\\n\\\\nHello\\\\n\\\\n---\\\\n\\\\nworld\\\\n\\",
\\"html\\": \\"<p>Hello</p>\\\\n<hr>\\\\n<p>world</p>\\\\n\\",
\\"frontmatter\\": {
\\"title\\": \\"bar\\"
},
\\"excerpt\\": \\"<p>Hello</p>\\\\n\\",
\\"url\\": \\"/data-loading/content/bar.html\\",
\\"transformed\\": true
},
{
\\"src\\": \\"---\\\\ntitle: foo\\\\n---\\\\n\\\\nHello\\\\n\\\\n---\\\\n\\\\nworld\\\\n\\",
\\"html\\": \\"<p>Hello</p>\\\\n<hr>\\\\n<p>world</p>\\\\n\\",
\\"frontmatter\\": {
\\"title\\": \\"foo\\"
},
\\"excerpt\\": \\"<p>Hello</p>\\\\n\\",
\\"url\\": \\"/data-loading/content/foo.html\\",
\\"transformed\\": true
}
]"
`)
})
})

@ -12,7 +12,7 @@ describe('test multi sidebar sort root', () => {
expect(sidebarContent).toEqual([
'Frontmatter',
'& <Text Literals &> code',
'Static Data',
'Data Loading',
'Multi Sidebar Test',
'Dynamic Routes',
'Markdown Extensions'

@ -1,14 +0,0 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`static data file support in vite 3 > render correct content 1`] = `
[
"[
{
\\"foo\\": true
},
{
\\"bar\\": true
}
]",
]
`;

@ -1,7 +0,0 @@
# Static Data
<script setup lang="ts">
import { data } from './static.data.js'
</script>
{{ data }}

@ -1,12 +0,0 @@
describe('static data file support in vite 3', () => {
beforeAll(async () => {
await goto('/static-data/data')
})
test('render correct content', async () => {
const pLocator = page.locator('.VPContent p')
const pContents = await pLocator.allTextContents()
expect(pContents).toMatchSnapshot()
})
})

@ -1,6 +1,7 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"isolatedModules": false,
"baseUrl": ".",
"types": ["node", "vitest/globals"],
"paths": {

@ -56,29 +56,118 @@ export default {
When you need to generate data based on local files, you should use the `watch` option in the data loader so that changes made to these files can trigger hot updates.
The `watch` option is also convenient in that you can use [glob patterns](https://github.com/mrmlnc/fast-glob#pattern-syntax) to match multiple files. The patterns can be relative to the loader file itself, and the `load()` function will receive the matched files as absolute paths:
The `watch` option is also convenient in that you can use [glob patterns](https://github.com/mrmlnc/fast-glob#pattern-syntax) to match multiple files. The patterns can be relative to the loader file itself, and the `load()` function will receive the matched files as absolute paths.
The following example shows loading CSV files and transforming them into JSON using [csv-parse](https://github.com/adaltas/node-csv/tree/master/packages/csv-parse/). Because this file only executes at build time, you will not be shipping the CSV parser to the client!
```js
import fs from 'node:fs'
import parseFrontmatter from 'gray-matter'
import { parse } from 'csv-parse/sync'
export default {
// watch all blog posts
watch: ['./posts/*.md'],
watch: ['./data/*.csv'],
load(watchedFiles) {
// watchedFiles will be an array of absolute paths of the matched files.
// generate an array of blog post metadata that can be used to render
// a list in the theme layout
return watchedFiles.map(file => {
const content = fs.readFileSync(file, 'utf-8')
const { data, excerpt } = parseFrontmatter(content)
return {
file,
data,
excerpt
}
return parse(fs.readFileSync(file, 'utf-8'), {
columns: true,
skip_empty_lines: true
})
})
}
}
```
## `createContentLoader`
When building a content focused site, we often need to create an "archive" or "index" page: a page where we list all available entries in our content collection, for example blog posts or API pages. We **can** implement this directly with the data loader API, but since this is such a common use case, VitePress also provides a `createContentLoader` helper to simplify this:
```js
// posts.data.js
import { createContentLoader } from 'vitepress'
export default createContentLoader('posts/*.md', /* options */)
```
The helper takes a glob pattern relative to [project root](./routing#project-root), and returns a `{ watch, load }` data loader object that can be used as the default export in a data loader file. It also implements caching based on file modified timestamps to improve dev performance.
Note the loader only works with Markdown files - matched non-Markdown files will be skipped.
The loaded data will be an array with the type of `ContentData[]`:
```ts
interface ContentData {
// mapped absolute URL for the page. e.g. /posts/hello.html
url: string
// frontmatter data of the page
frontmatter: Record<string, any>
// the following are only present if relevant options are enabled
// we will discuss them below
src: string | undefined
html: string | undefined
excerpt: string | undefined
}
```
By default, only `url` and `frontmatter` are provided. This is because the loaded data will be inlined as JSON in the client bundle, so we need to be cautious about its size. Here's an example using the data to build a minimal blog index page:
```vue
<script setup>
import { data as posts } from './posts.data.js'
</script>
<template>
<h1>All Blog Posts</h1>
<ul>
<li v-for="post of posts">
<a :href="post.url">{{ post.frontmatter.title }}</a>
<span>by {{ post.frontmatter.author }}</span>
</li>
</ul>
</template>
```
### Options
The default data may not suit all needs - you can opt-in to transform the data using options:
```js
// posts.data.js
import { createContentLoader } from 'vitepress'
export default createContentLoader('posts/*.md', {
includeSrc: true, // include raw markdown source?
render: true, // include rendered full page HTML?
excerpt: true, // include excerpt?
transform(rawData) {
// map, sort, or filter the raw data as you wish.
// the final result is what will be shipped to the client.
return rawData.sort((a, b) => {
return +new Date(b.frontmatter.date) - +new Date(a.frontmatter.date)
}).map(page => {
page.src // raw markdown source
page.html // rendered full page HTML
page.excerpt // rendered excerpt HTML (content above first `---`)
return {/* ... */}
})
}
})
```
Check out how it is used in the [Vue.js blog](https://github.com/vuejs/blog/blob/main/.vitepress/theme/posts.data.ts).
The `createContentLoader` API can also be used inside [build hooks](/reference/site-config#build-hooks):
```js
// .vitepress/config.js
export default {
async buildEnd() {
const posts = await createContentLoader('posts/*.md').load()
// generate files based on posts metadata, e.g. RSS feed
}
}
```

@ -135,6 +135,7 @@
"fast-glob": "^3.2.12",
"fs-extra": "^11.1.0",
"get-port": "^6.1.2",
"gray-matter": "^4.0.3",
"lint-staged": "^13.2.0",
"lodash.template": "^4.5.0",
"lru-cache": "^7.18.3",

@ -51,6 +51,7 @@ importers:
fast-glob: ^3.2.12
fs-extra: ^11.1.0
get-port: ^6.1.2
gray-matter: ^4.0.3
lint-staged: ^13.2.0
lodash.template: ^4.5.0
lru-cache: ^7.18.3
@ -140,6 +141,7 @@ importers:
fast-glob: 3.2.12
fs-extra: 11.1.0
get-port: 6.1.2
gray-matter: 4.0.3
lint-staged: 13.2.0_dx6s57r75rxv5zregmjdjjgmei
lodash.template: 4.5.0
lru-cache: 7.18.3

@ -287,6 +287,10 @@ export async function resolveConfig(
userConfig
}
// to be shared with content loaders
// @ts-ignore
global.VITEPRESS_CONFIG = config
return config
}

@ -0,0 +1,140 @@
import fs from 'fs'
import path from 'path'
import glob from 'fast-glob'
import type { SiteConfig } from './config'
import matter from 'gray-matter'
import { normalizePath } from 'vite'
import { createMarkdownRenderer, type MarkdownRenderer } from './markdown'
export interface ContentOptions<T = ContentData[]> {
/**
* Include src?
* default: false
*/
includeSrc?: boolean
/**
* Render src to HTML and include in data?
* default: false
*/
render?: boolean
/**
* Whether to parse and include excerpt (rendered as HTML)
* default: false
*/
excerpt?: boolean
/**
* Transform the data. Note the data will be inlined as JSON in the client
* bundle if imported from components or markdown files.
*/
transform?: (data: ContentData[]) => T | Promise<T>
}
export interface ContentData {
url: string
src: string | undefined
html: string | undefined
frontmatter: Record<string, any>
excerpt: string | undefined
}
/**
* Create a loader object that can be directly used as the default export
* of a data loader file.
*/
export function createContentLoader<T = ContentData[]>(
/**
* files to glob / watch - relative to <project root>
*/
pattern: string | string[],
{
includeSrc,
render,
excerpt: renderExcerpt,
transform
}: ContentOptions<T> = {}
): {
watch: string | string[]
load: () => Promise<T>
} {
const config: SiteConfig = (global as any).VITEPRESS_CONFIG
if (!config) {
throw new Error(
'content loader invoked without an active vitepress process, ' +
'or before vitepress config is resolved.'
)
}
if (typeof pattern === 'string') pattern = [pattern]
pattern = pattern.map((p) => normalizePath(path.join(config.root, p)))
let md: MarkdownRenderer
const cache = new Map<
string,
{
data: any
timestamp: number
}
>()
return {
watch: pattern,
async load(files?: string[]) {
if (!files) {
// the loader is being called directly, do a fresh glob
files = (
await glob(pattern, {
ignore: ['**/node_modules/**', '**/dist/**']
})
).sort()
}
md =
md ||
(await createMarkdownRenderer(
config.srcDir,
config.markdown,
config.site.base,
config.logger
))
const raw: ContentData[] = []
for (const file of files) {
if (!file.endsWith('.md')) {
continue
}
const timestamp = fs.statSync(file).mtimeMs
const cached = cache.get(file)
if (cached && timestamp === cached.timestamp) {
raw.push(cached.data)
} else {
const src = fs.readFileSync(file, 'utf-8')
const { data: frontmatter, excerpt } = matter(src, {
excerpt: true
})
const url =
'/' +
normalizePath(path.relative(config.root, file)).replace(
/\.md$/,
config.cleanUrls ? '' : '.html'
)
const html = render ? md.render(src) : undefined
const renderedExcerpt = renderExcerpt
? excerpt && md.render(excerpt)
: undefined
const data: ContentData = {
src: includeSrc ? src : undefined,
html,
frontmatter,
excerpt: renderedExcerpt,
url
}
cache.set(file, { data, timestamp })
raw.push(data)
}
}
return (transform ? transform(raw) : raw) as any
}
}
}

@ -4,6 +4,7 @@ export * from './markdown'
export * from './build/build'
export * from './serve/serve'
export * from './init/init'
export * from './cotentLoader'
export { defineLoader, type LoaderModule } from './plugins/staticDataPlugin'
export { loadEnv } from 'vite'

Loading…
Cancel
Save