feat: add support for asset sharding (#5432)

pull/5434/head
Divyansh Singh 3 weeks ago committed by GitHub
parent b0452c5d8f
commit 5f69cc9d90
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -5,10 +5,11 @@ const mode = process.env.VP_TEST_MODE || 'relative'
export default defineConfig({
title: 'Base Fixture',
description: 'Fixture site for base/assetsBase behavior',
base: mode === 'plain' || mode === 'cdn' ? '/' : './',
base: ['plain', 'cdn', 'sharded'].includes(mode) ? '/' : './',
assetsBase:
mode === 'cdn' ? `http://localhost:${process.env.VP_CDN_PORT}/` : undefined,
mpa: mode === 'mpa',
assetsShards: mode === 'sharded' ? 3 : undefined,
outDir: `.vitepress/dist-${mode}`,
cleanUrls: false,
rewrites: { 'src-moved.md': 'moved/target.md' },

@ -0,0 +1,119 @@
import { readFileSync, readdirSync } from 'node:fs'
import { join, resolve, sep } from 'node:path'
import { fileURLToPath } from 'node:url'
import { newPage, realErrors, waitForHydration, type TestPage } from './helpers'
const dir = resolve(fileURLToPath(import.meta.url), '..')
const dist = (...p: string[]) =>
join(dir, 'fixture/.vitepress/dist-sharded', ...p)
const origin = () => `http://localhost:${process.env['SHARDED_PORT']}`
// output-relative paths of everything under assets/
const files = () =>
readdirSync(dist('assets'), { recursive: true, withFileTypes: true })
.filter((e) => e.isFile())
.map((e) =>
join(e.parentPath, e.name)
.slice(dist('assets').length + 1)
.split(sep)
.join('/')
)
const isShardedPath = (f: string) => /^[0-2]\/[^/]+$/.test(f)
describe('assetsShards emit', () => {
test('page chunks land in numbered subdirectories', () => {
const pages = files().filter((f) => /\.md\.[\w-]+(\.lean)?\.js$/.test(f))
expect(pages.length).toBeGreaterThan(0)
expect(pages.every(isShardedPath)).toBe(true)
})
test('imported assets are sharded too, shared chunks and the app are not', () => {
const all = files()
const assets = all.filter(
(f) => /\.(png|woff2|css)$/.test(f) && !/^vp-icons\./.test(f)
)
expect(assets.length).toBeGreaterThan(0)
expect(assets.every(isShardedPath)).toBe(true)
expect(all.some((f) => /^chunks\/framework\.[\w-]+\.js$/.test(f))).toBe(
true
)
expect(all.some((f) => /^app\.[\w-]+\.js$/.test(f))).toBe(true)
})
test('hash map entries carry the shard and resolve to real files', () => {
const map: Record<string, string> = JSON.parse(
readFileSync(dist('hashmap.json'), 'utf-8')
)
// hash map keys are lowercased page names, file names keep their case
const lower = new Set(files().map((f) => f.toLowerCase()))
expect(Object.keys(map).length).toBeGreaterThan(0)
for (const [page, entry] of Object.entries(map)) {
expect(entry).toMatch(/^[0-2]\/[\w-]+$/)
const [shard, hash] = entry.split('/')
const chunk = `${shard}/${page}.${hash}`.toLowerCase()
expect(lower.has(`${chunk}.js`)).toBe(true)
expect(lower.has(`${chunk}.lean.js`)).toBe(true)
}
})
test('preload links point at files that exist', () => {
const existing = new Set(files().map((f) => `assets/${f}`))
for (const page of [
'index',
'sub/page',
'sub/deep/page2',
'moved/target'
]) {
const links = [
...readFileSync(dist(`${page}.html`), 'utf-8').matchAll(
/<link rel="modulepreload" href="\/([^"]+)">/g
)
].map((m) => m[1]!)
expect(links.some((l) => l.includes('.md.'))).toBe(true)
for (const link of links) expect(existing.has(link)).toBe(true)
}
})
})
describe('assetsShards in the browser', () => {
let t: TestPage
beforeAll(async () => {
t = await newPage()
})
afterAll(async () => {
await t.page.close()
await t.browser.close()
})
test('pages hydrate with sharded chunks', async () => {
await t.page.goto(`${origin()}/`)
await waitForHydration(t.page)
expect(await t.page.textContent('h1')).toContain('Home')
})
test('client-side navigation loads page chunks from their shard', async () => {
await t.page.evaluate(() => ((window as any).__spa_marker = 1))
await t.page.click('.vp-doc a[href="/sub/page.html"]')
await t.page.waitForFunction(() =>
document.querySelector('h1')?.textContent?.includes('Sub page')
)
expect(
await t.page.evaluate(() => (window as any).__spa_marker === 1)
).toBe(true)
const chunk = await t.page.evaluate(() =>
performance
.getEntriesByType('resource')
.map((r) => r.name)
.find((n) => /\/assets\/\d+\/sub_page\.md\.[\w-]+\.js$/.test(n))
)
expect(chunk).toBeDefined()
})
test('no console or page errors across the whole flow', () => {
expect(realErrors(t.errors)).toEqual([])
})
})

@ -64,7 +64,7 @@ export async function setup() {
// one process per flavor: the markdown renderer is a module-level
// singleton, so in-process builds would leak the first base into the rest
for (const mode of ['plain', 'relative', 'cdn', 'mpa']) {
for (const mode of ['plain', 'relative', 'cdn', 'mpa', 'sharded']) {
// mpa builds never empty outDir, so stale assets would survive reruns
await rm(dist(mode), { recursive: true, force: true })
const res = spawnSync(process.execPath, [bin, 'build', 'fixture'], {
@ -91,7 +91,8 @@ export async function setup() {
false
),
await serveStatic([['/', dist('cdn')]], false),
cdnServer
cdnServer,
await serveStatic([['/', dist('sharded')]], false)
]
browserServer = await chromium.launchServer({
@ -105,6 +106,7 @@ export async function setup() {
process.env['SUB_PORT'] = String(portOf(servers[0]!))
process.env['PAGES_PORT'] = String(portOf(servers[1]!))
process.env['VP_CDN_PORT'] = String(cdnPort)
process.env['SHARDED_PORT'] = String(portOf(servers[3]!))
}
export async function teardown() {

@ -2,11 +2,29 @@ import {
isRelativeBase,
joinPath,
mergeHead,
pageChunkPath,
relativePathToRoot,
type HeadConfig
} from 'shared/shared'
describe('shared/shared', () => {
describe('pageChunkPath', () => {
test('keeps flat chunks directly in assetsDir', () => {
expect(pageChunkPath('guide_foo.md', 'Ab-12xyz')).toBe(
'guide_foo.md.Ab-12xyz.js'
)
})
test('prefixes the shard recorded in the hash map entry', () => {
expect(pageChunkPath('guide_foo.md', '3/Ab-12xyz')).toBe(
'3/guide_foo.md.Ab-12xyz.js'
)
expect(pageChunkPath('guide_foo.md', '3/Ab-12xyz', '.lean.js')).toBe(
'3/guide_foo.md.Ab-12xyz.lean.js'
)
})
})
describe('mergeHead', () => {
test('replaces meta tags with the same key in place', () => {
expect(

@ -489,6 +489,21 @@ When `assetsBase` points at another origin, VitePress adds `crossorigin` to the
Only production builds are affected. `vitepress preview` serves a root-absolute `assetsBase` (like `/cdn/`) from the local dist; an external one is requested from the real URL. Can also be set per build with `vitepress build --assetsBase https://cdn.example.com/`.
### assetsShards
- Type: `number`
- Default: `undefined`
Spreads the generated assets over this many subdirectories of [`assetsDir`](#assetsdir), `assets/0/` through `assets/N-1/`, instead of one flat directory. Use it when the host caps the number of files per directory; Netlify, for example, allows 54,000. Each page emits two JavaScript files, so a site with 60,000 pages needs at least three shards, plus some headroom because files are distributed by a hash of their name.
```ts
export default {
assetsShards: 4
}
```
Shared chunks stay in `assets/chunks/`. A file's shard depends only on its name, so unchanged files keep their URL between builds. Only production builds are affected.
### icons
- Type: `{ include?: string[] }`

@ -7,6 +7,7 @@ import {
inBrowser,
isRelativeBase,
joinPath,
pageChunkPath,
sanitizeFileName,
type Awaitable
} from '../shared'
@ -84,7 +85,7 @@ export function pathToFile(path: string) {
pageHash = __VP_HASH_MAP__[pagePath.toLowerCase()]
}
if (!pageHash) return null
pagePath = `${__ASSETS_BASE__ || base}${__ASSETS_DIR__}/${pagePath}.${pageHash}.js`
pagePath = `${__ASSETS_BASE__ || base}${__ASSETS_DIR__}/${pageChunkPath(pagePath, pageHash)}`
} else {
// ssr build uses much simpler name mapping
pagePath = `./${sanitizeFileName(

@ -1,3 +1,4 @@
import { createHash } from 'node:crypto'
import fs from 'node:fs'
import { cp, mkdir, readFile, writeFile } from 'node:fs/promises'
import path from 'node:path'
@ -83,6 +84,16 @@ export async function bundle(
const relativeBase = isRelativeBase(config.site.base)
// with assetsShards, page chunks and assets spread over `assetsDir/<n>/`
// (shared chunks stay in `chunks/`) for hosts that cap the files per
// directory. the shard depends only on the name, so unchanged files keep
// their url across builds. assets get the same names in the server build,
// which renders their urls into the html.
const shard = (name = '') =>
config.assetsShards
? `${createHash('sha256').update(name).digest().readUInt32BE(0) % config.assetsShards}/`
: ''
const resolveViteConfig = async (
ssr: boolean
): Promise<ViteInlineConfig> => ({
@ -122,14 +133,21 @@ export async function bundle(
output: {
sanitizeFileName,
...rolldownOptions?.output,
assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`,
assetFileNames: (asset) =>
`${config.assetsDir}/${shard(asset.names[0])}[name].[hash].[ext]`,
...(ssr
? {
entryFileNames: '[name].js',
chunkFileNames: '[name].[hash].js'
}
: {
entryFileNames: `${config.assetsDir}/[name].[hash].js`,
entryFileNames: (chunk) => {
// only page chunks are sharded; the app entry stays put
const dir = chunk.facadeModuleId?.endsWith('.md')
? shard(chunk.name)
: ''
return `${config.assetsDir}/${dir}[name].[hash].js`
},
chunkFileNames(chunk) {
// avoid ads chunk being intercepted by adblock
return /(?:Carbon|BuySell)Ads/.test(chunk.name)

@ -15,6 +15,7 @@ import {
isRelativeBase,
mergeHead,
notFoundPageData,
pageChunkPath,
relativePathToRoot,
resolveSiteDataByRoute,
sanitizeFileName,
@ -102,19 +103,17 @@ export async function renderPage(
const dir = pageData.frontmatter.dir || siteData.dir || 'ltr'
const isDefault404 = page === '404.md' && !hasCustom404
// the initial load only needs the lean page js — the static content is
// already in the HTML
const pageHash = pageToHashMap[pageName.toLowerCase()]
const pageClientJsFileName = `${config.assetsDir}/${pageName}.${pageHash}.lean.js`
let preloadLinks: string[] = []
if (result && appChunk && !config.mpa && !isDefault404) {
const pageHash = pageToHashMap[pageName.toLowerCase()]
preloadLinks = [
...new Set([
// the imports of index.js + page.md.js as well, so everything
// fetches without waiting for the entry chunks to parse
...(await resolvePageImports(config, page, result, appChunk)),
pageClientJsFileName
// the initial load only needs the lean page js — the static content is
// already in the HTML
`${config.assetsDir}/${pageChunkPath(pageName, pageHash, '.lean.js')}`
])
]
}

@ -177,6 +177,16 @@ export async function resolveConfig(
? normalizeAssetsBase(userConfig.assetsBase)
: undefined
const assetsShards = userConfig.assetsShards
if (
assetsShards !== undefined &&
(!Number.isInteger(assetsShards) || assetsShards < 2)
) {
throw new Error(
`assetsShards must be an integer greater than 1 (got: ${assetsShards})`
)
}
if (isRelativeBase(site.base) && site.cleanUrls && command === 'build') {
logger.warn(
c.yellow(
@ -193,6 +203,7 @@ export async function resolveConfig(
publicDir,
assetsDir,
assetsBase,
assetsShards,
site,
themeDir,
configPath,

@ -365,9 +365,14 @@ export async function createVitePressPlugin(
for (const name in bundle) {
const chunk = bundle[name]
if (isPageChunk(chunk)) {
// record page -> hash relations
// record page -> hash relations, keeping the subdirectory the
// chunk was sharded into so the client can locate it
const hash = chunk.fileName.match(hashRE)![1]
pageToHashMap![chunk.name.toLowerCase()] = hash
const dir = path.posix.dirname(
path.posix.relative(siteConfig.assetsDir, chunk.fileName)
)
pageToHashMap![chunk.name.toLowerCase()] =
dir === '.' ? hash : `${dir}/${hash}`
// inject another chunk with the content stripped
this.emitFile({

@ -145,6 +145,16 @@ export interface UserConfig<
* @example 'https://cdn.example.com/'
*/
assetsBase?: string
/**
* Number of subdirectories to spread the generated assets over
* (`assetsDir/0` ... `assetsDir/N-1`) for hosts that cap the files per
* directory. Page chunks and imported assets are distributed by a hash of
* their name; shared chunks stay in `assetsDir/chunks`.
*
* @experimental
* @example 4
*/
assetsShards?: number
/**
* Options for the generated icon stylesheet (`vp-icons.*.css`).
*/
@ -393,6 +403,10 @@ export interface SiteConfig<ThemeConfig = any> extends Pick<
* URL prefix for built assets, normalized to end with a slash.
*/
assetsBase?: string
/**
* Number of subdirectories the generated assets are spread over.
*/
assetsShards?: number
/**
* Absolute path of the cache directory.
*/

@ -294,6 +294,21 @@ export function sanitizeFileName(name: string): string {
)
}
/**
* Output path of a page's client chunk relative to `assetsDir`, from its hash
* map entry. The entry is the chunk's hash, prefixed with the subdirectory the
* build put the chunk in when `assetsShards` is set (`<shard>/<hash>`), so the
* client never has to guess the layout.
*/
export function pageChunkPath(
pageName: string,
hashEntry: string,
ext = '.js'
): string {
const dir = hashEntry.lastIndexOf('/') + 1
return `${hashEntry.slice(0, dir)}${pageName}.${hashEntry.slice(dir)}${ext}`
}
export function slash(p: string): string {
return p.replace(/\\/g, '/')
}

Loading…
Cancel
Save