diff --git a/__tests__/base/fixture/.vitepress/config.ts b/__tests__/base/fixture/.vitepress/config.ts index f135dd338..88b911af9 100644 --- a/__tests__/base/fixture/.vitepress/config.ts +++ b/__tests__/base/fixture/.vitepress/config.ts @@ -5,10 +5,11 @@ const mode = process.env.VP_TEST_MODE || 'relative' export default defineConfig({ title: 'Base Fixture', description: 'Fixture site for base/assetsBase behavior', - base: mode === 'plain' || mode === 'cdn' ? '/' : './', + base: ['plain', 'cdn', 'sharded'].includes(mode) ? '/' : './', assetsBase: mode === 'cdn' ? `http://localhost:${process.env.VP_CDN_PORT}/` : undefined, mpa: mode === 'mpa', + assetsShards: mode === 'sharded' ? 3 : undefined, outDir: `.vitepress/dist-${mode}`, cleanUrls: false, rewrites: { 'src-moved.md': 'moved/target.md' }, diff --git a/__tests__/base/sharded.test.ts b/__tests__/base/sharded.test.ts new file mode 100644 index 000000000..056cea95c --- /dev/null +++ b/__tests__/base/sharded.test.ts @@ -0,0 +1,119 @@ +import { readFileSync, readdirSync } from 'node:fs' +import { join, resolve, sep } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { newPage, realErrors, waitForHydration, type TestPage } from './helpers' + +const dir = resolve(fileURLToPath(import.meta.url), '..') +const dist = (...p: string[]) => + join(dir, 'fixture/.vitepress/dist-sharded', ...p) +const origin = () => `http://localhost:${process.env['SHARDED_PORT']}` + +// output-relative paths of everything under assets/ +const files = () => + readdirSync(dist('assets'), { recursive: true, withFileTypes: true }) + .filter((e) => e.isFile()) + .map((e) => + join(e.parentPath, e.name) + .slice(dist('assets').length + 1) + .split(sep) + .join('/') + ) + +const isShardedPath = (f: string) => /^[0-2]\/[^/]+$/.test(f) + +describe('assetsShards emit', () => { + test('page chunks land in numbered subdirectories', () => { + const pages = files().filter((f) => /\.md\.[\w-]+(\.lean)?\.js$/.test(f)) + expect(pages.length).toBeGreaterThan(0) + expect(pages.every(isShardedPath)).toBe(true) + }) + + test('imported assets are sharded too, shared chunks and the app are not', () => { + const all = files() + const assets = all.filter( + (f) => /\.(png|woff2|css)$/.test(f) && !/^vp-icons\./.test(f) + ) + expect(assets.length).toBeGreaterThan(0) + expect(assets.every(isShardedPath)).toBe(true) + expect(all.some((f) => /^chunks\/framework\.[\w-]+\.js$/.test(f))).toBe( + true + ) + expect(all.some((f) => /^app\.[\w-]+\.js$/.test(f))).toBe(true) + }) + + test('hash map entries carry the shard and resolve to real files', () => { + const map: Record = JSON.parse( + readFileSync(dist('hashmap.json'), 'utf-8') + ) + // hash map keys are lowercased page names, file names keep their case + const lower = new Set(files().map((f) => f.toLowerCase())) + expect(Object.keys(map).length).toBeGreaterThan(0) + for (const [page, entry] of Object.entries(map)) { + expect(entry).toMatch(/^[0-2]\/[\w-]+$/) + const [shard, hash] = entry.split('/') + const chunk = `${shard}/${page}.${hash}`.toLowerCase() + expect(lower.has(`${chunk}.js`)).toBe(true) + expect(lower.has(`${chunk}.lean.js`)).toBe(true) + } + }) + + test('preload links point at files that exist', () => { + const existing = new Set(files().map((f) => `assets/${f}`)) + for (const page of [ + 'index', + 'sub/page', + 'sub/deep/page2', + 'moved/target' + ]) { + const links = [ + ...readFileSync(dist(`${page}.html`), 'utf-8').matchAll( + //g + ) + ].map((m) => m[1]!) + expect(links.some((l) => l.includes('.md.'))).toBe(true) + for (const link of links) expect(existing.has(link)).toBe(true) + } + }) +}) + +describe('assetsShards in the browser', () => { + let t: TestPage + + beforeAll(async () => { + t = await newPage() + }) + + afterAll(async () => { + await t.page.close() + await t.browser.close() + }) + + test('pages hydrate with sharded chunks', async () => { + await t.page.goto(`${origin()}/`) + await waitForHydration(t.page) + expect(await t.page.textContent('h1')).toContain('Home') + }) + + test('client-side navigation loads page chunks from their shard', async () => { + await t.page.evaluate(() => ((window as any).__spa_marker = 1)) + await t.page.click('.vp-doc a[href="/sub/page.html"]') + await t.page.waitForFunction(() => + document.querySelector('h1')?.textContent?.includes('Sub page') + ) + expect( + await t.page.evaluate(() => (window as any).__spa_marker === 1) + ).toBe(true) + const chunk = await t.page.evaluate(() => + performance + .getEntriesByType('resource') + .map((r) => r.name) + .find((n) => /\/assets\/\d+\/sub_page\.md\.[\w-]+\.js$/.test(n)) + ) + expect(chunk).toBeDefined() + }) + + test('no console or page errors across the whole flow', () => { + expect(realErrors(t.errors)).toEqual([]) + }) +}) diff --git a/__tests__/base/vitestGlobalSetup.ts b/__tests__/base/vitestGlobalSetup.ts index 62953c6d6..8c566c66f 100644 --- a/__tests__/base/vitestGlobalSetup.ts +++ b/__tests__/base/vitestGlobalSetup.ts @@ -64,7 +64,7 @@ export async function setup() { // one process per flavor: the markdown renderer is a module-level // singleton, so in-process builds would leak the first base into the rest - for (const mode of ['plain', 'relative', 'cdn', 'mpa']) { + for (const mode of ['plain', 'relative', 'cdn', 'mpa', 'sharded']) { // mpa builds never empty outDir, so stale assets would survive reruns await rm(dist(mode), { recursive: true, force: true }) const res = spawnSync(process.execPath, [bin, 'build', 'fixture'], { @@ -91,7 +91,8 @@ export async function setup() { false ), await serveStatic([['/', dist('cdn')]], false), - cdnServer + cdnServer, + await serveStatic([['/', dist('sharded')]], false) ] browserServer = await chromium.launchServer({ @@ -105,6 +106,7 @@ export async function setup() { process.env['SUB_PORT'] = String(portOf(servers[0]!)) process.env['PAGES_PORT'] = String(portOf(servers[1]!)) process.env['VP_CDN_PORT'] = String(cdnPort) + process.env['SHARDED_PORT'] = String(portOf(servers[3]!)) } export async function teardown() { diff --git a/__tests__/unit/shared/shared.test.ts b/__tests__/unit/shared/shared.test.ts index 16b971ae9..4758ba3df 100644 --- a/__tests__/unit/shared/shared.test.ts +++ b/__tests__/unit/shared/shared.test.ts @@ -2,11 +2,29 @@ import { isRelativeBase, joinPath, mergeHead, + pageChunkPath, relativePathToRoot, type HeadConfig } from 'shared/shared' describe('shared/shared', () => { + describe('pageChunkPath', () => { + test('keeps flat chunks directly in assetsDir', () => { + expect(pageChunkPath('guide_foo.md', 'Ab-12xyz')).toBe( + 'guide_foo.md.Ab-12xyz.js' + ) + }) + + test('prefixes the shard recorded in the hash map entry', () => { + expect(pageChunkPath('guide_foo.md', '3/Ab-12xyz')).toBe( + '3/guide_foo.md.Ab-12xyz.js' + ) + expect(pageChunkPath('guide_foo.md', '3/Ab-12xyz', '.lean.js')).toBe( + '3/guide_foo.md.Ab-12xyz.lean.js' + ) + }) + }) + describe('mergeHead', () => { test('replaces meta tags with the same key in place', () => { expect( diff --git a/docs/en/reference/site-config.md b/docs/en/reference/site-config.md index 7f2141272..1b0e2f883 100644 --- a/docs/en/reference/site-config.md +++ b/docs/en/reference/site-config.md @@ -489,6 +489,21 @@ When `assetsBase` points at another origin, VitePress adds `crossorigin` to the Only production builds are affected. `vitepress preview` serves a root-absolute `assetsBase` (like `/cdn/`) from the local dist; an external one is requested from the real URL. Can also be set per build with `vitepress build --assetsBase https://cdn.example.com/`. +### assetsShards + +- Type: `number` +- Default: `undefined` + +Spreads the generated assets over this many subdirectories of [`assetsDir`](#assetsdir), `assets/0/` through `assets/N-1/`, instead of one flat directory. Use it when the host caps the number of files per directory; Netlify, for example, allows 54,000. Each page emits two JavaScript files, so a site with 60,000 pages needs at least three shards, plus some headroom because files are distributed by a hash of their name. + +```ts +export default { + assetsShards: 4 +} +``` + +Shared chunks stay in `assets/chunks/`. A file's shard depends only on its name, so unchanged files keep their URL between builds. Only production builds are affected. + ### icons - Type: `{ include?: string[] }` diff --git a/src/client/app/utils.ts b/src/client/app/utils.ts index 9683b9152..63e4708a6 100644 --- a/src/client/app/utils.ts +++ b/src/client/app/utils.ts @@ -7,6 +7,7 @@ import { inBrowser, isRelativeBase, joinPath, + pageChunkPath, sanitizeFileName, type Awaitable } from '../shared' @@ -84,7 +85,7 @@ export function pathToFile(path: string) { pageHash = __VP_HASH_MAP__[pagePath.toLowerCase()] } if (!pageHash) return null - pagePath = `${__ASSETS_BASE__ || base}${__ASSETS_DIR__}/${pagePath}.${pageHash}.js` + pagePath = `${__ASSETS_BASE__ || base}${__ASSETS_DIR__}/${pageChunkPath(pagePath, pageHash)}` } else { // ssr build uses much simpler name mapping pagePath = `./${sanitizeFileName( diff --git a/src/node/build/bundle.ts b/src/node/build/bundle.ts index f2d0ad085..b1e54d91c 100644 --- a/src/node/build/bundle.ts +++ b/src/node/build/bundle.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import fs from 'node:fs' import { cp, mkdir, readFile, writeFile } from 'node:fs/promises' import path from 'node:path' @@ -83,6 +84,16 @@ export async function bundle( const relativeBase = isRelativeBase(config.site.base) + // with assetsShards, page chunks and assets spread over `assetsDir//` + // (shared chunks stay in `chunks/`) for hosts that cap the files per + // directory. the shard depends only on the name, so unchanged files keep + // their url across builds. assets get the same names in the server build, + // which renders their urls into the html. + const shard = (name = '') => + config.assetsShards + ? `${createHash('sha256').update(name).digest().readUInt32BE(0) % config.assetsShards}/` + : '' + const resolveViteConfig = async ( ssr: boolean ): Promise => ({ @@ -122,14 +133,21 @@ export async function bundle( output: { sanitizeFileName, ...rolldownOptions?.output, - assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`, + assetFileNames: (asset) => + `${config.assetsDir}/${shard(asset.names[0])}[name].[hash].[ext]`, ...(ssr ? { entryFileNames: '[name].js', chunkFileNames: '[name].[hash].js' } : { - entryFileNames: `${config.assetsDir}/[name].[hash].js`, + entryFileNames: (chunk) => { + // only page chunks are sharded; the app entry stays put + const dir = chunk.facadeModuleId?.endsWith('.md') + ? shard(chunk.name) + : '' + return `${config.assetsDir}/${dir}[name].[hash].js` + }, chunkFileNames(chunk) { // avoid ads chunk being intercepted by adblock return /(?:Carbon|BuySell)Ads/.test(chunk.name) diff --git a/src/node/build/render.ts b/src/node/build/render.ts index 0e8864d8f..538e0238d 100644 --- a/src/node/build/render.ts +++ b/src/node/build/render.ts @@ -15,6 +15,7 @@ import { isRelativeBase, mergeHead, notFoundPageData, + pageChunkPath, relativePathToRoot, resolveSiteDataByRoute, sanitizeFileName, @@ -102,19 +103,17 @@ export async function renderPage( const dir = pageData.frontmatter.dir || siteData.dir || 'ltr' const isDefault404 = page === '404.md' && !hasCustom404 - // the initial load only needs the lean page js — the static content is - // already in the HTML - const pageHash = pageToHashMap[pageName.toLowerCase()] - const pageClientJsFileName = `${config.assetsDir}/${pageName}.${pageHash}.lean.js` - let preloadLinks: string[] = [] if (result && appChunk && !config.mpa && !isDefault404) { + const pageHash = pageToHashMap[pageName.toLowerCase()] preloadLinks = [ ...new Set([ // the imports of index.js + page.md.js as well, so everything // fetches without waiting for the entry chunks to parse ...(await resolvePageImports(config, page, result, appChunk)), - pageClientJsFileName + // the initial load only needs the lean page js — the static content is + // already in the HTML + `${config.assetsDir}/${pageChunkPath(pageName, pageHash, '.lean.js')}` ]) ] } diff --git a/src/node/config.ts b/src/node/config.ts index baea4a868..78d94c11d 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -177,6 +177,16 @@ export async function resolveConfig( ? normalizeAssetsBase(userConfig.assetsBase) : undefined + const assetsShards = userConfig.assetsShards + if ( + assetsShards !== undefined && + (!Number.isInteger(assetsShards) || assetsShards < 2) + ) { + throw new Error( + `assetsShards must be an integer greater than 1 (got: ${assetsShards})` + ) + } + if (isRelativeBase(site.base) && site.cleanUrls && command === 'build') { logger.warn( c.yellow( @@ -193,6 +203,7 @@ export async function resolveConfig( publicDir, assetsDir, assetsBase, + assetsShards, site, themeDir, configPath, diff --git a/src/node/plugin.ts b/src/node/plugin.ts index 886bae0d5..30578824f 100644 --- a/src/node/plugin.ts +++ b/src/node/plugin.ts @@ -365,9 +365,14 @@ export async function createVitePressPlugin( for (const name in bundle) { const chunk = bundle[name] if (isPageChunk(chunk)) { - // record page -> hash relations + // record page -> hash relations, keeping the subdirectory the + // chunk was sharded into so the client can locate it const hash = chunk.fileName.match(hashRE)![1] - pageToHashMap![chunk.name.toLowerCase()] = hash + const dir = path.posix.dirname( + path.posix.relative(siteConfig.assetsDir, chunk.fileName) + ) + pageToHashMap![chunk.name.toLowerCase()] = + dir === '.' ? hash : `${dir}/${hash}` // inject another chunk with the content stripped this.emitFile({ diff --git a/src/node/siteConfig.ts b/src/node/siteConfig.ts index 2b4d1eb40..627ea15b7 100644 --- a/src/node/siteConfig.ts +++ b/src/node/siteConfig.ts @@ -145,6 +145,16 @@ export interface UserConfig< * @example 'https://cdn.example.com/' */ assetsBase?: string + /** + * Number of subdirectories to spread the generated assets over + * (`assetsDir/0` ... `assetsDir/N-1`) for hosts that cap the files per + * directory. Page chunks and imported assets are distributed by a hash of + * their name; shared chunks stay in `assetsDir/chunks`. + * + * @experimental + * @example 4 + */ + assetsShards?: number /** * Options for the generated icon stylesheet (`vp-icons.*.css`). */ @@ -393,6 +403,10 @@ export interface SiteConfig extends Pick< * URL prefix for built assets, normalized to end with a slash. */ assetsBase?: string + /** + * Number of subdirectories the generated assets are spread over. + */ + assetsShards?: number /** * Absolute path of the cache directory. */ diff --git a/src/shared/shared.ts b/src/shared/shared.ts index 1dbb3a59f..12d4676e0 100644 --- a/src/shared/shared.ts +++ b/src/shared/shared.ts @@ -294,6 +294,21 @@ export function sanitizeFileName(name: string): string { ) } +/** + * Output path of a page's client chunk relative to `assetsDir`, from its hash + * map entry. The entry is the chunk's hash, prefixed with the subdirectory the + * build put the chunk in when `assetsShards` is set (`/`), so the + * client never has to guess the layout. + */ +export function pageChunkPath( + pageName: string, + hashEntry: string, + ext = '.js' +): string { + const dir = hashEntry.lastIndexOf('/') + 1 + return `${hashEntry.slice(0, dir)}${pageName}.${hashEntry.slice(dir)}${ext}` +} + export function slash(p: string): string { return p.replace(/\\/g, '/') }