revert changes for parallel bundling

pull/3386/head
Yuxuan Zhang 2 years ago
parent 1c880a83ab
commit 5493202cb2
No known key found for this signature in database
GPG Key ID: 6910B04F3351EF7D

@ -1,43 +1,185 @@
import fs from 'fs-extra' import fs from 'fs-extra'
import path from 'path' import path from 'path'
import { build, type BuildOptions, type PluginOption, type Rollup } from 'vite' import { fileURLToPath } from 'url'
import {
build,
normalizePath,
type BuildOptions,
type Rollup,
type InlineConfig as ViteInlineConfig
} from 'vite'
import { APP_PATH } from '../alias'
import type { SiteConfig } from '../config' import type { SiteConfig } from '../config'
import { updateCurrentTask } from '../utils/task'
import { buildMPAClient } from './buildMPAClient'
import { registerWorkload, shouldUseParallel } from '../worker'
import resolveViteConfig from './viteConfig'
import { type WorkerContext } from './build'
import { createVitePressPlugin } from '../plugin' import { createVitePressPlugin } from '../plugin'
import { escapeRegExp, sanitizeFileName, slash } from '../shared'
import { task } from '../utils/task'
import { buildMPAClient } from './buildMPAClient'
const dispatchBundleWorkload = registerWorkload( // https://github.com/vitejs/vite/blob/d2aa0969ee316000d3b957d7e879f001e85e369e/packages/vite/src/node/plugins/splitVendorChunk.ts#L14
'build:bundle', const CSS_LANGS_RE =
bundleWorkload, /\.(css|less|sass|scss|styl|stylus|pcss|postcss|sss)(?:$|\?)/
function (this: WorkerContext) {
// To make contentLoader happy const clientDir = normalizePath(
// @ts-ignore path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../client')
global.VITEPRESS_CONFIG = this.config
}
) )
async function bundleWorkload( // these deps are also being used in the client code (outside of the theme)
this: WorkerContext, // exclude them from the theme chunk so there is no circular dependency
ssr: boolean, const excludedModules = [
plugins: PluginOption[] '/@siteData',
) { 'node_modules/@vueuse/core/',
const config = await resolveViteConfig(ssr, { 'node_modules/@vueuse/shared/',
config: this.config, 'node_modules/vue/',
options: this.options, 'node_modules/vue-demi/',
plugins clientDir
]
// bundles the VitePress app for both client AND server.
export async function bundle(
config: SiteConfig,
options: BuildOptions
): Promise<{
clientResult: Rollup.RollupOutput | null
serverResult: Rollup.RollupOutput
pageToHashMap: Record<string, string>
}> {
const pageToHashMap = Object.create(null)
const clientJSMap = Object.create(null)
// define custom rollup input
// this is a multi-entry build - every page is considered an entry chunk
// the loading is done via filename conversion rules so that the
// metadata doesn't need to be included in the main chunk.
const input: Record<string, string> = {}
config.pages.forEach((file) => {
// page filename conversion
// foo/bar.md -> foo_bar.md
const alias = config.rewrites.map[file] || file
input[slash(alias).replace(/\//g, '_')] = path.resolve(config.srcDir, file)
}) })
return build(config) as Promise<Rollup.RollupOutput>
const themeEntryRE = new RegExp(
`^${escapeRegExp(
path.resolve(config.themeDir, 'index.js').replace(/\\/g, '/')
).slice(0, -2)}m?(j|t)s`
)
// resolve options to pass to vite
const { rollupOptions } = options
const resolveViteConfig = async (
ssr: boolean
): Promise<ViteInlineConfig> => ({
root: config.srcDir,
cacheDir: config.cacheDir,
base: config.site.base,
logLevel: config.vite?.logLevel ?? 'warn',
plugins: await createVitePressPlugin(
config,
ssr,
pageToHashMap,
clientJSMap
),
ssr: {
noExternal: ['vitepress', '@docsearch/css']
},
build: {
...options,
emptyOutDir: true,
ssr,
ssrEmitAssets: config.mpa,
// minify with esbuild in MPA mode (for CSS)
minify: ssr
? config.mpa
? 'esbuild'
: false
: typeof options.minify === 'boolean'
? options.minify
: !process.env.DEBUG,
outDir: ssr ? config.tempDir : config.outDir,
cssCodeSplit: false,
rollupOptions: {
...rollupOptions,
input: {
...input,
// use different entry based on ssr or not
app: path.resolve(APP_PATH, ssr ? 'ssr.js' : 'index.js')
},
// important so that each page chunk and the index export things for each
// other
preserveEntrySignatures: 'allow-extension',
output: {
sanitizeFileName,
...rollupOptions?.output,
assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`,
...(ssr
? {
entryFileNames: '[name].js',
chunkFileNames: '[name].[hash].js'
}
: {
entryFileNames: `${config.assetsDir}/[name].[hash].js`,
chunkFileNames(chunk) {
// avoid ads chunk being intercepted by adblock
return /(?:Carbon|BuySell)Ads/.test(chunk.name)
? `${config.assetsDir}/chunks/ui-custom.[hash].js`
: `${config.assetsDir}/chunks/[name].[hash].js`
},
manualChunks(id, ctx) {
// move known framework code into a stable chunk so that
// custom theme changes do not invalidate hash for all pages
if (id.startsWith('\0vite')) {
return 'framework'
}
if (id.includes('plugin-vue:export-helper')) {
return 'framework'
}
if (
id.includes(`${clientDir}/app`) &&
id !== `${clientDir}/app/index.js`
) {
return 'framework'
}
if (
isEagerChunk(id, ctx.getModuleInfo) &&
/@vue\/(runtime|shared|reactivity)/.test(id)
) {
return 'framework'
} }
async function bundleMPA( if (
config: SiteConfig, (id.startsWith(`${clientDir}/theme-default`) ||
serverResult: Rollup.RollupOutput, !excludedModules.some((i) => id.includes(i))) &&
clientJSMap: Record<string, string> staticImportedByEntry(
id,
ctx.getModuleInfo,
cacheTheme,
themeEntryRE
)
) { ) {
updateCurrentTask(0, 1, 'bundling MPA') return 'theme'
}
}
})
}
}
},
configFile: config.vite?.configFile
})
let clientResult!: Rollup.RollupOutput | null
let serverResult!: Rollup.RollupOutput
await task('building client + server bundles', async () => {
clientResult = config.mpa
? null
: ((await build(await resolveViteConfig(false))) as Rollup.RollupOutput)
serverResult = (await build(
await resolveViteConfig(true)
)) as Rollup.RollupOutput
})
if (config.mpa) {
// in MPA mode, we need to copy over the non-js asset files from the // in MPA mode, we need to copy over the non-js asset files from the
// server build since there is no client-side build. // server build since there is no client-side build.
await Promise.all( await Promise.all(
@ -54,47 +196,65 @@ async function bundleMPA(
if (fs.existsSync(publicDir)) { if (fs.existsSync(publicDir)) {
await fs.copy(publicDir, config.outDir) await fs.copy(publicDir, config.outDir)
} }
updateCurrentTask()
// build <script client> bundle // build <script client> bundle
if (Object.keys(clientJSMap).length) { if (Object.keys(clientJSMap).length) {
return buildMPAClient(clientJSMap, config) clientResult = await buildMPAClient(clientJSMap, config)
} else {
return null
} }
} }
// bundles the VitePress app for both client AND server. return { clientResult, serverResult, pageToHashMap }
export async function bundle( }
config: SiteConfig,
options: BuildOptions
): Promise<{
clientResult: Rollup.RollupOutput | null
serverResult: Rollup.RollupOutput
pageToHashMap: Record<string, string>
}> {
const pageToHashMap = Object.create(null)
const clientJSMap = Object.create(null)
const [serverResult, clientResult] = await Promise.all( const cache = new Map<string, boolean>()
[true, false].map(async (ssr) => { const cacheTheme = new Map<string, boolean>()
if (!ssr && config.mpa) return null
const plugins = await createVitePressPlugin( /**
config, * Check if a module is statically imported by at least one entry.
ssr, */
pageToHashMap, function isEagerChunk(id: string, getModuleInfo: Rollup.GetModuleInfo) {
clientJSMap if (
) id.includes('node_modules') &&
return shouldUseParallel(config, 'bundle') !CSS_LANGS_RE.test(id) &&
? dispatchBundleWorkload(ssr, plugins) staticImportedByEntry(id, getModuleInfo, cache)
: bundleWorkload.apply({ config, options }, [ssr, plugins]) ) {
}) return true
) }
}
return { function staticImportedByEntry(
clientResult: config.mpa id: string,
? await bundleMPA(config, serverResult!, clientJSMap) getModuleInfo: Rollup.GetModuleInfo,
: clientResult!, cache: Map<string, boolean>,
serverResult: serverResult!, entryRE: RegExp | null = null,
pageToHashMap importStack: string[] = []
): boolean {
if (cache.has(id)) {
return !!cache.get(id)
}
if (importStack.includes(id)) {
// circular deps!
cache.set(id, false)
return false
}
const mod = getModuleInfo(id)
if (!mod) {
cache.set(id, false)
return false
} }
if (entryRE ? entryRE.test(id) : mod.isEntry) {
cache.set(id, true)
return true
}
const someImporterIs = mod.importers.some((importer: string) =>
staticImportedByEntry(
importer,
getModuleInfo,
cache,
entryRE,
importStack.concat(id)
)
)
cache.set(id, someImporterIs)
return someImporterIs
} }

@ -32,7 +32,7 @@ export interface RenderPageContext {
export async function renderPage( export async function renderPage(
render: (path: string) => Promise<SSGContext>, render: (path: string) => Promise<SSGContext>,
pageNameRaw: string, page: string,
renderContext: RenderPageContext renderContext: RenderPageContext
) { ) {
const { const {
@ -46,7 +46,7 @@ export async function renderPage(
additionalHeadTags additionalHeadTags
} = renderContext } = renderContext
const page = config.rewrites.inv[pageNameRaw] || pageNameRaw page = config.rewrites.inv[page] ?? page
const routePath = `/${page.replace(/\.md$/, '')}` const routePath = `/${page.replace(/\.md$/, '')}`
const siteData = resolveSiteDataByRoute(config.site, routePath) const siteData = resolveSiteDataByRoute(config.site, routePath)
@ -170,34 +170,37 @@ export async function renderPage(
const dir = pageData.frontmatter.dir || siteData.dir || 'ltr' const dir = pageData.frontmatter.dir || siteData.dir || 'ltr'
const html = [ const html = `<!DOCTYPE html>
`<!DOCTYPE html>`, <html lang="${siteData.lang}" dir="${dir}">
`<html lang="${siteData.lang}" dir="${dir}">`, <head>
`<head>`, <meta charset="utf-8">
`<meta charset="utf-8">`, ${
isMetaViewportOverridden(head) isMetaViewportOverridden(head)
? '' ? ''
: '<meta name="viewport" content="width=device-width,initial-scale=1">', : '<meta name="viewport" content="width=device-width,initial-scale=1">'
`<title>${title}</title>`, }
<title>${title}</title>
${
isDescriptionOverridden(head) isDescriptionOverridden(head)
? '' ? ''
: `<meta name="description" content="${description}">`, : `<meta name="description" content="${description}">`
`<meta name="generator" content="VitePress v${version}">`, }
stylesheetLink, <meta name="generator" content="VitePress v${version}">
metadataScript.inHead ? metadataScript.html : '', ${stylesheetLink}
${metadataScript.inHead ? metadataScript.html : ''}
${
appChunk appChunk
? `<script type="module" src="${siteData.base}${appChunk.fileName}"></script>` ? `<script type="module" src="${siteData.base}${appChunk.fileName}"></script>`
: '', : ''
await renderHead(head), }
`</head>`, ${await renderHead(head)}
`<body>`, </head>
teleports?.body || '', <body>${teleports?.body || ''}
`<div id="app">${content}</div>`, <div id="app">${content}</div>
metadataScript.inHead ? '' : metadataScript.html, ${metadataScript.inHead ? '' : metadataScript.html}
inlinedScript, ${inlinedScript}
`</body>`, </body>
`</html>` </html>`
].join('')
const htmlFileName = path.join(config.outDir, page.replace(/\.md$/, '.html')) const htmlFileName = path.join(config.outDir, page.replace(/\.md$/, '.html'))
await fs.ensureDir(path.dirname(htmlFileName)) await fs.ensureDir(path.dirname(htmlFileName))

@ -1,212 +0,0 @@
import path from 'path'
import { fileURLToPath } from 'url'
import {
normalizePath,
type BuildOptions,
type Rollup,
type InlineConfig as ViteInlineConfig,
type PluginOption
} from 'vite'
import { APP_PATH } from '../alias'
import type { SiteConfig } from '../config'
import { slash } from '../shared'
import { escapeRegExp, sanitizeFileName } from '../shared'
// https://github.com/vitejs/vite/blob/d2aa0969ee316000d3b957d7e879f001e85e369e/packages/vite/src/node/plugins/splitVendorChunk.ts#L14
const CSS_LANGS_RE =
/\.(css|less|sass|scss|styl|stylus|pcss|postcss|sss)(?:$|\?)/
const clientDir = normalizePath(
path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../client')
)
// these deps are also being used in the client code (outside of the theme)
// exclude them from the theme chunk so there is no circular dependency
const excludedModules = [
'/@siteData',
'node_modules/@vueuse/core/',
'node_modules/@vueuse/shared/',
'node_modules/vue/',
'node_modules/vue-demi/',
clientDir
]
const themeEntryRE = (themeDir: string) =>
new RegExp(
`^${escapeRegExp(
path.resolve(themeDir, 'index.js').replace(/\\/g, '/')
).slice(0, -2)}m?(j|t)s`
)
const cache = new Map<string, boolean>()
const cacheTheme = new Map<string, boolean>()
/**
* Check if a module is statically imported by at least one entry.
*/
function isEagerChunk(id: string, getModuleInfo: Rollup.GetModuleInfo) {
if (
id.includes('node_modules') &&
!CSS_LANGS_RE.test(id) &&
staticImportedByEntry(id, getModuleInfo, cache)
) {
return true
}
}
function staticImportedByEntry(
id: string,
getModuleInfo: Rollup.GetModuleInfo,
cache: Map<string, boolean>,
entryRE: RegExp | null = null,
importStack: string[] = []
): boolean {
if (cache.has(id)) {
return !!cache.get(id)
}
if (importStack.includes(id)) {
// circular deps!
cache.set(id, false)
return false
}
const mod = getModuleInfo(id)
if (!mod) {
cache.set(id, false)
return false
}
if (entryRE ? entryRE.test(id) : mod.isEntry) {
cache.set(id, true)
return true
}
const someImporterIs = mod.importers.some((importer: string) =>
staticImportedByEntry(
importer,
getModuleInfo,
cache,
entryRE,
importStack.concat(id)
)
)
cache.set(id, someImporterIs)
return someImporterIs
}
// define custom rollup input
// this is a multi-entry build - every page is considered an entry chunk
// the loading is done via filename conversion rules so that the
// metadata doesn't need to be included in the main chunk.
const resolveInput = (config: SiteConfig) =>
Object.fromEntries(
config.pages.map((file) => {
// page filename conversion
// foo/bar.md -> foo_bar.md
const alias = config.rewrites.map[file] || file
return [
slash(alias).replace(/\//g, '_'),
path.resolve(config.srcDir, file)
]
})
) as Record<string, string>
export default async function resolveViteConfig(
ssr: boolean,
{
config,
options,
plugins
}: {
config: SiteConfig
options: BuildOptions
plugins: PluginOption[]
}
): Promise<ViteInlineConfig> {
return {
root: config.srcDir,
cacheDir: config.cacheDir,
base: config.site.base,
logLevel: config.vite?.logLevel ?? 'warn',
plugins,
ssr: {
noExternal: ['vitepress', '@docsearch/css']
},
build: {
...options,
emptyOutDir: true,
ssr,
ssrEmitAssets: config.mpa,
// minify with esbuild in MPA mode (for CSS)
minify: ssr
? config.mpa && 'esbuild'
: options.minify ?? !process.env.DEBUG,
outDir: ssr ? config.tempDir : config.outDir,
cssCodeSplit: false,
rollupOptions: {
...options.rollupOptions,
input: {
...resolveInput(config),
// use different entry based on ssr or not
app: path.resolve(APP_PATH, ssr ? 'ssr.js' : 'index.js')
},
// important so that each page chunk and the index export things for each
// other
preserveEntrySignatures: 'allow-extension',
output: {
sanitizeFileName,
...options.rollupOptions?.output,
assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`,
...(ssr
? {
entryFileNames: '[name].js',
chunkFileNames: '[name].[hash].js'
}
: {
entryFileNames: `${config.assetsDir}/[name].[hash].js`,
chunkFileNames(chunk) {
// avoid ads chunk being intercepted by adblock
return /(?:Carbon|BuySell)Ads/.test(chunk.name)
? `${config.assetsDir}/chunks/ui-custom.[hash].js`
: `${config.assetsDir}/chunks/[name].[hash].js`
},
manualChunks(id, ctx) {
// move known framework code into a stable chunk so that
// custom theme changes do not invalidate hash for all pages
if (id.startsWith('\0vite')) {
return 'framework'
}
if (id.includes('plugin-vue:export-helper')) {
return 'framework'
}
if (
id.includes(`${clientDir}/app`) &&
id !== `${clientDir}/app/index.js`
) {
return 'framework'
}
if (
isEagerChunk(id, ctx.getModuleInfo) &&
/@vue\/(runtime|shared|reactivity)/.test(id)
) {
return 'framework'
}
if (
(id.startsWith(`${clientDir}/theme-default`) ||
!excludedModules.some((i) => id.includes(i))) &&
staticImportedByEntry(
id,
ctx.getModuleInfo,
cacheTheme,
themeEntryRE(config.themeDir)
)
) {
return 'theme'
}
}
})
}
}
},
configFile: config.vite?.configFile
}
}

@ -32,9 +32,11 @@ interface IndexObject {
titles: string[] titles: string[]
} }
// SSR being `true` or `false` dose not affect the config related to markdown
// renderer, so it can be reused.
let md: MarkdownRenderer let md: MarkdownRenderer
let flagAllIndexed = false let flagScanned = false
const taskByLocale = new Map<string, Promise<void>>() const taskByLocale = new Map<string, Promise<void>>()
const filesByLocale = new Map<string, Set<string>>() const filesByLocale = new Map<string, Set<string>>()
const indexByLocale = new Map<string, MiniSearch<IndexObject>>() const indexByLocale = new Map<string, MiniSearch<IndexObject>>()
@ -160,9 +162,10 @@ export async function localSearchPlugin(
} }
} }
function indexAll() { // scan all pages, group by locale and create empty indexes accordingly.
if (flagAllIndexed) return function scanForIndex() {
flagAllIndexed = true if (flagScanned) return
flagScanned = true
for (const page of siteConfig.pages) { for (const page of siteConfig.pages) {
const file = path.join(siteConfig.srcDir, page) const file = path.join(siteConfig.srcDir, page)
@ -209,7 +212,7 @@ export async function localSearchPlugin(
async load(id) { async load(id) {
if (id === LOCAL_SEARCH_INDEX_REQUEST_PATH) { if (id === LOCAL_SEARCH_INDEX_REQUEST_PATH) {
indexAll() scanForIndex()
let records: string[] = [] let records: string[] = []
for (const [locale] of filesByLocale) { for (const [locale] of filesByLocale) {
records.push( records.push(

@ -28,7 +28,10 @@ export async function createServer(
configFile: config.vite?.configFile configFile: config.vite?.configFile
}).then((server) => }).then((server) =>
Object.assign({}, server, { Object.assign({}, server, {
close: () => server.close().then(() => stopWorkers('server.close()')) close() {
stopWorkers('server.close()')
return server.close()
}
}) })
) )
} }

@ -19,6 +19,7 @@ export default class Queue<T> {
enqueue(data: T) { enqueue(data: T) {
if (this.closed) if (this.closed)
throw new Error(`Failed to enqueue ${data}, queue already closed`) throw new Error(`Failed to enqueue ${data}, queue already closed`)
if (data === null) return this.close()
if (this.pending.length) this.pending.shift()!(data) if (this.pending.length) this.pending.shift()!(data)
else this.queue.push(data) else this.queue.push(data)
} }

@ -1,5 +1,8 @@
import { Worker, isMainThread, parentPort, workerData } from 'worker_threads' import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'
import RPCContext, { deferPromise } from 'rpc-magic-proxy' import RPCContext, {
deferPromise,
type RPCContextOptions
} from 'rpc-magic-proxy'
import { task, updateCurrentTask } from './utils/task' import { task, updateCurrentTask } from './utils/task'
import c from 'picocolors' import c from 'picocolors'
import Queue from './utils/queue' import Queue from './utils/queue'
@ -7,8 +10,12 @@ import _debug from 'debug'
import type { SiteConfig } from 'siteConfig' import type { SiteConfig } from 'siteConfig'
import humanizeDuration from 'humanize-duration' import humanizeDuration from 'humanize-duration'
export type SupportsParallel = 'bundle' | 'render' | 'local-search' export type SupportsParallel = 'render' | 'local-search'
const options: RPCContextOptions = {
carryThis: false,
carrySideEffect: false
}
/** /**
* Checks if the given task should be run in parallel. * Checks if the given task should be run in parallel.
* If task is omitted, checks if any task should be run in parallel. * If task is omitted, checks if any task should be run in parallel.
@ -61,7 +68,7 @@ export async function launchWorkers(numWorkers: number, context: Object) {
debug(`launching ${numWorkers} workers`) debug(`launching ${numWorkers} workers`)
taskQueue = new Queue<WorkerTask>() taskQueue = new Queue<WorkerTask>()
const allInitialized: Array<Promise<void>> = [] const allInitialized: Array<Promise<void>> = []
const ctx = new RPCContext() const ctx = new RPCContext(options)
const getNextTask = () => taskQueue?.dequeue() ?? null const getNextTask = () => taskQueue?.dequeue() ?? null
for (let i = 0; i < numWorkers; i++) { for (let i = 0; i < numWorkers; i++) {
const workerId = (i + 1).toString().padStart(2, '0') const workerId = (i + 1).toString().padStart(2, '0')
@ -99,7 +106,8 @@ export function updateContext(context: Object) {
return Promise.all(workers.map(({ hooks }) => hooks.updateContext(context))) return Promise.all(workers.map(({ hooks }) => hooks.updateContext(context)))
} }
// Wait for workers to drain the taskQueue and exit. // Wait for workers to finish and exit.
// Will return immediately if no worker exists.
export async function stopWorkers(reason: string = 'exit') { export async function stopWorkers(reason: string = 'exit') {
debug('stopping workers:', reason) debug('stopping workers:', reason)
const allClosed = workers.map((w) => const allClosed = workers.map((w) =>
@ -111,10 +119,10 @@ export async function stopWorkers(reason: string = 'exit') {
taskQueue = null taskQueue = null
const success = await Promise.any([ const success = await Promise.any([
Promise.all(allClosed).then(() => true), Promise.all(allClosed).then(() => true),
new Promise<false>((res) => setTimeout(() => res(false), 2000)) new Promise<false>((res) => setTimeout(() => res(false), 1500))
]) ])
if (!success) { if (!success) {
console.warn('forcefully terminating workers') debug('forcefully terminating workers')
for (const w of workers) { for (const w of workers) {
try { try {
w.terminate() w.terminate()
@ -152,7 +160,7 @@ export function registerWorkload<T extends Object, K extends any[], V>(
// Will keep querying next workload from main thread // Will keep querying next workload from main thread
async function workerMainLoop() { async function workerMainLoop() {
const ctx = new RPCContext().bind(parentPort! as any) const ctx = new RPCContext(options).bind(parentPort! as any)
const { const {
workerMeta: _workerMeta, workerMeta: _workerMeta,
initWorkerHooks, initWorkerHooks,

Loading…
Cancel
Save