revert changes for parallel bundling

pull/3386/head
Yuxuan Zhang 2 years ago
parent 1c880a83ab
commit 5493202cb2
No known key found for this signature in database
GPG Key ID: 6910B04F3351EF7D

@ -1,67 +1,38 @@
import fs from 'fs-extra' import fs from 'fs-extra'
import path from 'path' import path from 'path'
import { build, type BuildOptions, type PluginOption, type Rollup } from 'vite' import { fileURLToPath } from 'url'
import {
build,
normalizePath,
type BuildOptions,
type Rollup,
type InlineConfig as ViteInlineConfig
} from 'vite'
import { APP_PATH } from '../alias'
import type { SiteConfig } from '../config' import type { SiteConfig } from '../config'
import { updateCurrentTask } from '../utils/task'
import { buildMPAClient } from './buildMPAClient'
import { registerWorkload, shouldUseParallel } from '../worker'
import resolveViteConfig from './viteConfig'
import { type WorkerContext } from './build'
import { createVitePressPlugin } from '../plugin' import { createVitePressPlugin } from '../plugin'
import { escapeRegExp, sanitizeFileName, slash } from '../shared'
import { task } from '../utils/task'
import { buildMPAClient } from './buildMPAClient'
const dispatchBundleWorkload = registerWorkload( // https://github.com/vitejs/vite/blob/d2aa0969ee316000d3b957d7e879f001e85e369e/packages/vite/src/node/plugins/splitVendorChunk.ts#L14
'build:bundle', const CSS_LANGS_RE =
bundleWorkload, /\.(css|less|sass|scss|styl|stylus|pcss|postcss|sss)(?:$|\?)/
function (this: WorkerContext) {
// To make contentLoader happy
// @ts-ignore
global.VITEPRESS_CONFIG = this.config
}
)
async function bundleWorkload( const clientDir = normalizePath(
this: WorkerContext, path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../client')
ssr: boolean, )
plugins: PluginOption[]
) {
const config = await resolveViteConfig(ssr, {
config: this.config,
options: this.options,
plugins
})
return build(config) as Promise<Rollup.RollupOutput>
}
async function bundleMPA( // these deps are also being used in the client code (outside of the theme)
config: SiteConfig, // exclude them from the theme chunk so there is no circular dependency
serverResult: Rollup.RollupOutput, const excludedModules = [
clientJSMap: Record<string, string> '/@siteData',
) { 'node_modules/@vueuse/core/',
updateCurrentTask(0, 1, 'bundling MPA') 'node_modules/@vueuse/shared/',
// in MPA mode, we need to copy over the non-js asset files from the 'node_modules/vue/',
// server build since there is no client-side build. 'node_modules/vue-demi/',
await Promise.all( clientDir
serverResult.output.map(async (chunk) => { ]
if (!chunk.fileName.endsWith('.js')) {
const tempPath = path.resolve(config.tempDir, chunk.fileName)
const outPath = path.resolve(config.outDir, chunk.fileName)
await fs.copy(tempPath, outPath)
}
})
)
// also copy over public dir
const publicDir = path.resolve(config.srcDir, 'public')
if (fs.existsSync(publicDir)) {
await fs.copy(publicDir, config.outDir)
}
updateCurrentTask()
// build <script client> bundle
if (Object.keys(clientJSMap).length) {
return buildMPAClient(clientJSMap, config)
} else {
return null
}
}
// bundles the VitePress app for both client AND server. // bundles the VitePress app for both client AND server.
export async function bundle( export async function bundle(
@ -75,26 +46,215 @@ export async function bundle(
const pageToHashMap = Object.create(null) const pageToHashMap = Object.create(null)
const clientJSMap = Object.create(null) const clientJSMap = Object.create(null)
const [serverResult, clientResult] = await Promise.all( // define custom rollup input
[true, false].map(async (ssr) => { // this is a multi-entry build - every page is considered an entry chunk
if (!ssr && config.mpa) return null // the loading is done via filename conversion rules so that the
const plugins = await createVitePressPlugin( // metadata doesn't need to be included in the main chunk.
config, const input: Record<string, string> = {}
ssr, config.pages.forEach((file) => {
pageToHashMap, // page filename conversion
clientJSMap // foo/bar.md -> foo_bar.md
) const alias = config.rewrites.map[file] || file
return shouldUseParallel(config, 'bundle') input[slash(alias).replace(/\//g, '_')] = path.resolve(config.srcDir, file)
? dispatchBundleWorkload(ssr, plugins) })
: bundleWorkload.apply({ config, options }, [ssr, plugins])
}) const themeEntryRE = new RegExp(
`^${escapeRegExp(
path.resolve(config.themeDir, 'index.js').replace(/\\/g, '/')
).slice(0, -2)}m?(j|t)s`
) )
return { // resolve options to pass to vite
clientResult: config.mpa const { rollupOptions } = options
? await bundleMPA(config, serverResult!, clientJSMap)
: clientResult!, const resolveViteConfig = async (
serverResult: serverResult!, ssr: boolean
pageToHashMap ): Promise<ViteInlineConfig> => ({
root: config.srcDir,
cacheDir: config.cacheDir,
base: config.site.base,
logLevel: config.vite?.logLevel ?? 'warn',
plugins: await createVitePressPlugin(
config,
ssr,
pageToHashMap,
clientJSMap
),
ssr: {
noExternal: ['vitepress', '@docsearch/css']
},
build: {
...options,
emptyOutDir: true,
ssr,
ssrEmitAssets: config.mpa,
// minify with esbuild in MPA mode (for CSS)
minify: ssr
? config.mpa
? 'esbuild'
: false
: typeof options.minify === 'boolean'
? options.minify
: !process.env.DEBUG,
outDir: ssr ? config.tempDir : config.outDir,
cssCodeSplit: false,
rollupOptions: {
...rollupOptions,
input: {
...input,
// use different entry based on ssr or not
app: path.resolve(APP_PATH, ssr ? 'ssr.js' : 'index.js')
},
// important so that each page chunk and the index export things for each
// other
preserveEntrySignatures: 'allow-extension',
output: {
sanitizeFileName,
...rollupOptions?.output,
assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`,
...(ssr
? {
entryFileNames: '[name].js',
chunkFileNames: '[name].[hash].js'
}
: {
entryFileNames: `${config.assetsDir}/[name].[hash].js`,
chunkFileNames(chunk) {
// avoid ads chunk being intercepted by adblock
return /(?:Carbon|BuySell)Ads/.test(chunk.name)
? `${config.assetsDir}/chunks/ui-custom.[hash].js`
: `${config.assetsDir}/chunks/[name].[hash].js`
},
manualChunks(id, ctx) {
// move known framework code into a stable chunk so that
// custom theme changes do not invalidate hash for all pages
if (id.startsWith('\0vite')) {
return 'framework'
}
if (id.includes('plugin-vue:export-helper')) {
return 'framework'
}
if (
id.includes(`${clientDir}/app`) &&
id !== `${clientDir}/app/index.js`
) {
return 'framework'
}
if (
isEagerChunk(id, ctx.getModuleInfo) &&
/@vue\/(runtime|shared|reactivity)/.test(id)
) {
return 'framework'
}
if (
(id.startsWith(`${clientDir}/theme-default`) ||
!excludedModules.some((i) => id.includes(i))) &&
staticImportedByEntry(
id,
ctx.getModuleInfo,
cacheTheme,
themeEntryRE
)
) {
return 'theme'
}
}
})
}
}
},
configFile: config.vite?.configFile
})
let clientResult!: Rollup.RollupOutput | null
let serverResult!: Rollup.RollupOutput
await task('building client + server bundles', async () => {
clientResult = config.mpa
? null
: ((await build(await resolveViteConfig(false))) as Rollup.RollupOutput)
serverResult = (await build(
await resolveViteConfig(true)
)) as Rollup.RollupOutput
})
if (config.mpa) {
// in MPA mode, we need to copy over the non-js asset files from the
// server build since there is no client-side build.
await Promise.all(
serverResult.output.map(async (chunk) => {
if (!chunk.fileName.endsWith('.js')) {
const tempPath = path.resolve(config.tempDir, chunk.fileName)
const outPath = path.resolve(config.outDir, chunk.fileName)
await fs.copy(tempPath, outPath)
}
})
)
// also copy over public dir
const publicDir = path.resolve(config.srcDir, 'public')
if (fs.existsSync(publicDir)) {
await fs.copy(publicDir, config.outDir)
}
// build <script client> bundle
if (Object.keys(clientJSMap).length) {
clientResult = await buildMPAClient(clientJSMap, config)
}
}
return { clientResult, serverResult, pageToHashMap }
}
const cache = new Map<string, boolean>()
const cacheTheme = new Map<string, boolean>()
/**
* Check if a module is statically imported by at least one entry.
*/
function isEagerChunk(id: string, getModuleInfo: Rollup.GetModuleInfo) {
if (
id.includes('node_modules') &&
!CSS_LANGS_RE.test(id) &&
staticImportedByEntry(id, getModuleInfo, cache)
) {
return true
}
}
function staticImportedByEntry(
id: string,
getModuleInfo: Rollup.GetModuleInfo,
cache: Map<string, boolean>,
entryRE: RegExp | null = null,
importStack: string[] = []
): boolean {
if (cache.has(id)) {
return !!cache.get(id)
}
if (importStack.includes(id)) {
// circular deps!
cache.set(id, false)
return false
} }
const mod = getModuleInfo(id)
if (!mod) {
cache.set(id, false)
return false
}
if (entryRE ? entryRE.test(id) : mod.isEntry) {
cache.set(id, true)
return true
}
const someImporterIs = mod.importers.some((importer: string) =>
staticImportedByEntry(
importer,
getModuleInfo,
cache,
entryRE,
importStack.concat(id)
)
)
cache.set(id, someImporterIs)
return someImporterIs
} }

@ -32,7 +32,7 @@ export interface RenderPageContext {
export async function renderPage( export async function renderPage(
render: (path: string) => Promise<SSGContext>, render: (path: string) => Promise<SSGContext>,
pageNameRaw: string, page: string,
renderContext: RenderPageContext renderContext: RenderPageContext
) { ) {
const { const {
@ -46,7 +46,7 @@ export async function renderPage(
additionalHeadTags additionalHeadTags
} = renderContext } = renderContext
const page = config.rewrites.inv[pageNameRaw] || pageNameRaw page = config.rewrites.inv[page] ?? page
const routePath = `/${page.replace(/\.md$/, '')}` const routePath = `/${page.replace(/\.md$/, '')}`
const siteData = resolveSiteDataByRoute(config.site, routePath) const siteData = resolveSiteDataByRoute(config.site, routePath)
@ -170,34 +170,37 @@ export async function renderPage(
const dir = pageData.frontmatter.dir || siteData.dir || 'ltr' const dir = pageData.frontmatter.dir || siteData.dir || 'ltr'
const html = [ const html = `<!DOCTYPE html>
`<!DOCTYPE html>`, <html lang="${siteData.lang}" dir="${dir}">
`<html lang="${siteData.lang}" dir="${dir}">`, <head>
`<head>`, <meta charset="utf-8">
`<meta charset="utf-8">`, ${
isMetaViewportOverridden(head) isMetaViewportOverridden(head)
? '' ? ''
: '<meta name="viewport" content="width=device-width,initial-scale=1">', : '<meta name="viewport" content="width=device-width,initial-scale=1">'
`<title>${title}</title>`, }
isDescriptionOverridden(head) <title>${title}</title>
? '' ${
: `<meta name="description" content="${description}">`, isDescriptionOverridden(head)
`<meta name="generator" content="VitePress v${version}">`, ? ''
stylesheetLink, : `<meta name="description" content="${description}">`
metadataScript.inHead ? metadataScript.html : '', }
appChunk <meta name="generator" content="VitePress v${version}">
? `<script type="module" src="${siteData.base}${appChunk.fileName}"></script>` ${stylesheetLink}
: '', ${metadataScript.inHead ? metadataScript.html : ''}
await renderHead(head), ${
`</head>`, appChunk
`<body>`, ? `<script type="module" src="${siteData.base}${appChunk.fileName}"></script>`
teleports?.body || '', : ''
`<div id="app">${content}</div>`, }
metadataScript.inHead ? '' : metadataScript.html, ${await renderHead(head)}
inlinedScript, </head>
`</body>`, <body>${teleports?.body || ''}
`</html>` <div id="app">${content}</div>
].join('') ${metadataScript.inHead ? '' : metadataScript.html}
${inlinedScript}
</body>
</html>`
const htmlFileName = path.join(config.outDir, page.replace(/\.md$/, '.html')) const htmlFileName = path.join(config.outDir, page.replace(/\.md$/, '.html'))
await fs.ensureDir(path.dirname(htmlFileName)) await fs.ensureDir(path.dirname(htmlFileName))

@ -1,212 +0,0 @@
import path from 'path'
import { fileURLToPath } from 'url'
import {
normalizePath,
type BuildOptions,
type Rollup,
type InlineConfig as ViteInlineConfig,
type PluginOption
} from 'vite'
import { APP_PATH } from '../alias'
import type { SiteConfig } from '../config'
import { slash } from '../shared'
import { escapeRegExp, sanitizeFileName } from '../shared'
// https://github.com/vitejs/vite/blob/d2aa0969ee316000d3b957d7e879f001e85e369e/packages/vite/src/node/plugins/splitVendorChunk.ts#L14
const CSS_LANGS_RE =
/\.(css|less|sass|scss|styl|stylus|pcss|postcss|sss)(?:$|\?)/
const clientDir = normalizePath(
path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../client')
)
// these deps are also being used in the client code (outside of the theme)
// exclude them from the theme chunk so there is no circular dependency
const excludedModules = [
'/@siteData',
'node_modules/@vueuse/core/',
'node_modules/@vueuse/shared/',
'node_modules/vue/',
'node_modules/vue-demi/',
clientDir
]
const themeEntryRE = (themeDir: string) =>
new RegExp(
`^${escapeRegExp(
path.resolve(themeDir, 'index.js').replace(/\\/g, '/')
).slice(0, -2)}m?(j|t)s`
)
const cache = new Map<string, boolean>()
const cacheTheme = new Map<string, boolean>()
/**
* Check if a module is statically imported by at least one entry.
*/
function isEagerChunk(id: string, getModuleInfo: Rollup.GetModuleInfo) {
if (
id.includes('node_modules') &&
!CSS_LANGS_RE.test(id) &&
staticImportedByEntry(id, getModuleInfo, cache)
) {
return true
}
}
function staticImportedByEntry(
id: string,
getModuleInfo: Rollup.GetModuleInfo,
cache: Map<string, boolean>,
entryRE: RegExp | null = null,
importStack: string[] = []
): boolean {
if (cache.has(id)) {
return !!cache.get(id)
}
if (importStack.includes(id)) {
// circular deps!
cache.set(id, false)
return false
}
const mod = getModuleInfo(id)
if (!mod) {
cache.set(id, false)
return false
}
if (entryRE ? entryRE.test(id) : mod.isEntry) {
cache.set(id, true)
return true
}
const someImporterIs = mod.importers.some((importer: string) =>
staticImportedByEntry(
importer,
getModuleInfo,
cache,
entryRE,
importStack.concat(id)
)
)
cache.set(id, someImporterIs)
return someImporterIs
}
// define custom rollup input
// this is a multi-entry build - every page is considered an entry chunk
// the loading is done via filename conversion rules so that the
// metadata doesn't need to be included in the main chunk.
const resolveInput = (config: SiteConfig) =>
Object.fromEntries(
config.pages.map((file) => {
// page filename conversion
// foo/bar.md -> foo_bar.md
const alias = config.rewrites.map[file] || file
return [
slash(alias).replace(/\//g, '_'),
path.resolve(config.srcDir, file)
]
})
) as Record<string, string>
export default async function resolveViteConfig(
ssr: boolean,
{
config,
options,
plugins
}: {
config: SiteConfig
options: BuildOptions
plugins: PluginOption[]
}
): Promise<ViteInlineConfig> {
return {
root: config.srcDir,
cacheDir: config.cacheDir,
base: config.site.base,
logLevel: config.vite?.logLevel ?? 'warn',
plugins,
ssr: {
noExternal: ['vitepress', '@docsearch/css']
},
build: {
...options,
emptyOutDir: true,
ssr,
ssrEmitAssets: config.mpa,
// minify with esbuild in MPA mode (for CSS)
minify: ssr
? config.mpa && 'esbuild'
: options.minify ?? !process.env.DEBUG,
outDir: ssr ? config.tempDir : config.outDir,
cssCodeSplit: false,
rollupOptions: {
...options.rollupOptions,
input: {
...resolveInput(config),
// use different entry based on ssr or not
app: path.resolve(APP_PATH, ssr ? 'ssr.js' : 'index.js')
},
// important so that each page chunk and the index export things for each
// other
preserveEntrySignatures: 'allow-extension',
output: {
sanitizeFileName,
...options.rollupOptions?.output,
assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`,
...(ssr
? {
entryFileNames: '[name].js',
chunkFileNames: '[name].[hash].js'
}
: {
entryFileNames: `${config.assetsDir}/[name].[hash].js`,
chunkFileNames(chunk) {
// avoid ads chunk being intercepted by adblock
return /(?:Carbon|BuySell)Ads/.test(chunk.name)
? `${config.assetsDir}/chunks/ui-custom.[hash].js`
: `${config.assetsDir}/chunks/[name].[hash].js`
},
manualChunks(id, ctx) {
// move known framework code into a stable chunk so that
// custom theme changes do not invalidate hash for all pages
if (id.startsWith('\0vite')) {
return 'framework'
}
if (id.includes('plugin-vue:export-helper')) {
return 'framework'
}
if (
id.includes(`${clientDir}/app`) &&
id !== `${clientDir}/app/index.js`
) {
return 'framework'
}
if (
isEagerChunk(id, ctx.getModuleInfo) &&
/@vue\/(runtime|shared|reactivity)/.test(id)
) {
return 'framework'
}
if (
(id.startsWith(`${clientDir}/theme-default`) ||
!excludedModules.some((i) => id.includes(i))) &&
staticImportedByEntry(
id,
ctx.getModuleInfo,
cacheTheme,
themeEntryRE(config.themeDir)
)
) {
return 'theme'
}
}
})
}
}
},
configFile: config.vite?.configFile
}
}

@ -32,9 +32,11 @@ interface IndexObject {
titles: string[] titles: string[]
} }
// SSR being `true` or `false` dose not affect the config related to markdown
// renderer, so it can be reused.
let md: MarkdownRenderer let md: MarkdownRenderer
let flagAllIndexed = false let flagScanned = false
const taskByLocale = new Map<string, Promise<void>>() const taskByLocale = new Map<string, Promise<void>>()
const filesByLocale = new Map<string, Set<string>>() const filesByLocale = new Map<string, Set<string>>()
const indexByLocale = new Map<string, MiniSearch<IndexObject>>() const indexByLocale = new Map<string, MiniSearch<IndexObject>>()
@ -160,9 +162,10 @@ export async function localSearchPlugin(
} }
} }
function indexAll() { // scan all pages, group by locale and create empty indexes accordingly.
if (flagAllIndexed) return function scanForIndex() {
flagAllIndexed = true if (flagScanned) return
flagScanned = true
for (const page of siteConfig.pages) { for (const page of siteConfig.pages) {
const file = path.join(siteConfig.srcDir, page) const file = path.join(siteConfig.srcDir, page)
@ -209,7 +212,7 @@ export async function localSearchPlugin(
async load(id) { async load(id) {
if (id === LOCAL_SEARCH_INDEX_REQUEST_PATH) { if (id === LOCAL_SEARCH_INDEX_REQUEST_PATH) {
indexAll() scanForIndex()
let records: string[] = [] let records: string[] = []
for (const [locale] of filesByLocale) { for (const [locale] of filesByLocale) {
records.push( records.push(

@ -28,7 +28,10 @@ export async function createServer(
configFile: config.vite?.configFile configFile: config.vite?.configFile
}).then((server) => }).then((server) =>
Object.assign({}, server, { Object.assign({}, server, {
close: () => server.close().then(() => stopWorkers('server.close()')) close() {
stopWorkers('server.close()')
return server.close()
}
}) })
) )
} }

@ -19,6 +19,7 @@ export default class Queue<T> {
enqueue(data: T) { enqueue(data: T) {
if (this.closed) if (this.closed)
throw new Error(`Failed to enqueue ${data}, queue already closed`) throw new Error(`Failed to enqueue ${data}, queue already closed`)
if (data === null) return this.close()
if (this.pending.length) this.pending.shift()!(data) if (this.pending.length) this.pending.shift()!(data)
else this.queue.push(data) else this.queue.push(data)
} }

@ -1,5 +1,8 @@
import { Worker, isMainThread, parentPort, workerData } from 'worker_threads' import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'
import RPCContext, { deferPromise } from 'rpc-magic-proxy' import RPCContext, {
deferPromise,
type RPCContextOptions
} from 'rpc-magic-proxy'
import { task, updateCurrentTask } from './utils/task' import { task, updateCurrentTask } from './utils/task'
import c from 'picocolors' import c from 'picocolors'
import Queue from './utils/queue' import Queue from './utils/queue'
@ -7,8 +10,12 @@ import _debug from 'debug'
import type { SiteConfig } from 'siteConfig' import type { SiteConfig } from 'siteConfig'
import humanizeDuration from 'humanize-duration' import humanizeDuration from 'humanize-duration'
export type SupportsParallel = 'bundle' | 'render' | 'local-search' export type SupportsParallel = 'render' | 'local-search'
const options: RPCContextOptions = {
carryThis: false,
carrySideEffect: false
}
/** /**
* Checks if the given task should be run in parallel. * Checks if the given task should be run in parallel.
* If task is omitted, checks if any task should be run in parallel. * If task is omitted, checks if any task should be run in parallel.
@ -61,7 +68,7 @@ export async function launchWorkers(numWorkers: number, context: Object) {
debug(`launching ${numWorkers} workers`) debug(`launching ${numWorkers} workers`)
taskQueue = new Queue<WorkerTask>() taskQueue = new Queue<WorkerTask>()
const allInitialized: Array<Promise<void>> = [] const allInitialized: Array<Promise<void>> = []
const ctx = new RPCContext() const ctx = new RPCContext(options)
const getNextTask = () => taskQueue?.dequeue() ?? null const getNextTask = () => taskQueue?.dequeue() ?? null
for (let i = 0; i < numWorkers; i++) { for (let i = 0; i < numWorkers; i++) {
const workerId = (i + 1).toString().padStart(2, '0') const workerId = (i + 1).toString().padStart(2, '0')
@ -99,7 +106,8 @@ export function updateContext(context: Object) {
return Promise.all(workers.map(({ hooks }) => hooks.updateContext(context))) return Promise.all(workers.map(({ hooks }) => hooks.updateContext(context)))
} }
// Wait for workers to drain the taskQueue and exit. // Wait for workers to finish and exit.
// Will return immediately if no worker exists.
export async function stopWorkers(reason: string = 'exit') { export async function stopWorkers(reason: string = 'exit') {
debug('stopping workers:', reason) debug('stopping workers:', reason)
const allClosed = workers.map((w) => const allClosed = workers.map((w) =>
@ -111,10 +119,10 @@ export async function stopWorkers(reason: string = 'exit') {
taskQueue = null taskQueue = null
const success = await Promise.any([ const success = await Promise.any([
Promise.all(allClosed).then(() => true), Promise.all(allClosed).then(() => true),
new Promise<false>((res) => setTimeout(() => res(false), 2000)) new Promise<false>((res) => setTimeout(() => res(false), 1500))
]) ])
if (!success) { if (!success) {
console.warn('forcefully terminating workers') debug('forcefully terminating workers')
for (const w of workers) { for (const w of workers) {
try { try {
w.terminate() w.terminate()
@ -152,7 +160,7 @@ export function registerWorkload<T extends Object, K extends any[], V>(
// Will keep querying next workload from main thread // Will keep querying next workload from main thread
async function workerMainLoop() { async function workerMainLoop() {
const ctx = new RPCContext().bind(parentPort! as any) const ctx = new RPCContext(options).bind(parentPort! as any)
const { const {
workerMeta: _workerMeta, workerMeta: _workerMeta,
initWorkerHooks, initWorkerHooks,

Loading…
Cancel
Save