revert changes for parallel bundling

pull/3386/head
Yuxuan Zhang 2 years ago
parent 1c880a83ab
commit 5493202cb2
No known key found for this signature in database
GPG Key ID: 6910B04F3351EF7D

@ -1,43 +1,185 @@
import fs from 'fs-extra'
import path from 'path'
import { build, type BuildOptions, type PluginOption, type Rollup } from 'vite'
import { fileURLToPath } from 'url'
import {
build,
normalizePath,
type BuildOptions,
type Rollup,
type InlineConfig as ViteInlineConfig
} from 'vite'
import { APP_PATH } from '../alias'
import type { SiteConfig } from '../config'
import { updateCurrentTask } from '../utils/task'
import { buildMPAClient } from './buildMPAClient'
import { registerWorkload, shouldUseParallel } from '../worker'
import resolveViteConfig from './viteConfig'
import { type WorkerContext } from './build'
import { createVitePressPlugin } from '../plugin'
import { escapeRegExp, sanitizeFileName, slash } from '../shared'
import { task } from '../utils/task'
import { buildMPAClient } from './buildMPAClient'
const dispatchBundleWorkload = registerWorkload(
'build:bundle',
bundleWorkload,
function (this: WorkerContext) {
// To make contentLoader happy
// @ts-ignore
global.VITEPRESS_CONFIG = this.config
}
// https://github.com/vitejs/vite/blob/d2aa0969ee316000d3b957d7e879f001e85e369e/packages/vite/src/node/plugins/splitVendorChunk.ts#L14
const CSS_LANGS_RE =
/\.(css|less|sass|scss|styl|stylus|pcss|postcss|sss)(?:$|\?)/
const clientDir = normalizePath(
path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../client')
)
async function bundleWorkload(
this: WorkerContext,
ssr: boolean,
plugins: PluginOption[]
) {
const config = await resolveViteConfig(ssr, {
config: this.config,
options: this.options,
plugins
})
return build(config) as Promise<Rollup.RollupOutput>
}
// these deps are also being used in the client code (outside of the theme)
// exclude them from the theme chunk so there is no circular dependency
const excludedModules = [
'/@siteData',
'node_modules/@vueuse/core/',
'node_modules/@vueuse/shared/',
'node_modules/vue/',
'node_modules/vue-demi/',
clientDir
]
async function bundleMPA(
// bundles the VitePress app for both client AND server.
export async function bundle(
config: SiteConfig,
serverResult: Rollup.RollupOutput,
clientJSMap: Record<string, string>
) {
updateCurrentTask(0, 1, 'bundling MPA')
options: BuildOptions
): Promise<{
clientResult: Rollup.RollupOutput | null
serverResult: Rollup.RollupOutput
pageToHashMap: Record<string, string>
}> {
const pageToHashMap = Object.create(null)
const clientJSMap = Object.create(null)
// define custom rollup input
// this is a multi-entry build - every page is considered an entry chunk
// the loading is done via filename conversion rules so that the
// metadata doesn't need to be included in the main chunk.
const input: Record<string, string> = {}
config.pages.forEach((file) => {
// page filename conversion
// foo/bar.md -> foo_bar.md
const alias = config.rewrites.map[file] || file
input[slash(alias).replace(/\//g, '_')] = path.resolve(config.srcDir, file)
})
const themeEntryRE = new RegExp(
`^${escapeRegExp(
path.resolve(config.themeDir, 'index.js').replace(/\\/g, '/')
).slice(0, -2)}m?(j|t)s`
)
// resolve options to pass to vite
const { rollupOptions } = options
const resolveViteConfig = async (
ssr: boolean
): Promise<ViteInlineConfig> => ({
root: config.srcDir,
cacheDir: config.cacheDir,
base: config.site.base,
logLevel: config.vite?.logLevel ?? 'warn',
plugins: await createVitePressPlugin(
config,
ssr,
pageToHashMap,
clientJSMap
),
ssr: {
noExternal: ['vitepress', '@docsearch/css']
},
build: {
...options,
emptyOutDir: true,
ssr,
ssrEmitAssets: config.mpa,
// minify with esbuild in MPA mode (for CSS)
minify: ssr
? config.mpa
? 'esbuild'
: false
: typeof options.minify === 'boolean'
? options.minify
: !process.env.DEBUG,
outDir: ssr ? config.tempDir : config.outDir,
cssCodeSplit: false,
rollupOptions: {
...rollupOptions,
input: {
...input,
// use different entry based on ssr or not
app: path.resolve(APP_PATH, ssr ? 'ssr.js' : 'index.js')
},
// important so that each page chunk and the index export things for each
// other
preserveEntrySignatures: 'allow-extension',
output: {
sanitizeFileName,
...rollupOptions?.output,
assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`,
...(ssr
? {
entryFileNames: '[name].js',
chunkFileNames: '[name].[hash].js'
}
: {
entryFileNames: `${config.assetsDir}/[name].[hash].js`,
chunkFileNames(chunk) {
// avoid ads chunk being intercepted by adblock
return /(?:Carbon|BuySell)Ads/.test(chunk.name)
? `${config.assetsDir}/chunks/ui-custom.[hash].js`
: `${config.assetsDir}/chunks/[name].[hash].js`
},
manualChunks(id, ctx) {
// move known framework code into a stable chunk so that
// custom theme changes do not invalidate hash for all pages
if (id.startsWith('\0vite')) {
return 'framework'
}
if (id.includes('plugin-vue:export-helper')) {
return 'framework'
}
if (
id.includes(`${clientDir}/app`) &&
id !== `${clientDir}/app/index.js`
) {
return 'framework'
}
if (
isEagerChunk(id, ctx.getModuleInfo) &&
/@vue\/(runtime|shared|reactivity)/.test(id)
) {
return 'framework'
}
if (
(id.startsWith(`${clientDir}/theme-default`) ||
!excludedModules.some((i) => id.includes(i))) &&
staticImportedByEntry(
id,
ctx.getModuleInfo,
cacheTheme,
themeEntryRE
)
) {
return 'theme'
}
}
})
}
}
},
configFile: config.vite?.configFile
})
let clientResult!: Rollup.RollupOutput | null
let serverResult!: Rollup.RollupOutput
await task('building client + server bundles', async () => {
clientResult = config.mpa
? null
: ((await build(await resolveViteConfig(false))) as Rollup.RollupOutput)
serverResult = (await build(
await resolveViteConfig(true)
)) as Rollup.RollupOutput
})
if (config.mpa) {
// in MPA mode, we need to copy over the non-js asset files from the
// server build since there is no client-side build.
await Promise.all(
@ -54,47 +196,65 @@ async function bundleMPA(
if (fs.existsSync(publicDir)) {
await fs.copy(publicDir, config.outDir)
}
updateCurrentTask()
// build <script client> bundle
if (Object.keys(clientJSMap).length) {
return buildMPAClient(clientJSMap, config)
} else {
return null
clientResult = await buildMPAClient(clientJSMap, config)
}
}
return { clientResult, serverResult, pageToHashMap }
}
// bundles the VitePress app for both client AND server.
export async function bundle(
config: SiteConfig,
options: BuildOptions
): Promise<{
clientResult: Rollup.RollupOutput | null
serverResult: Rollup.RollupOutput
pageToHashMap: Record<string, string>
}> {
const pageToHashMap = Object.create(null)
const clientJSMap = Object.create(null)
const cache = new Map<string, boolean>()
const cacheTheme = new Map<string, boolean>()
const [serverResult, clientResult] = await Promise.all(
[true, false].map(async (ssr) => {
if (!ssr && config.mpa) return null
const plugins = await createVitePressPlugin(
config,
ssr,
pageToHashMap,
clientJSMap
)
return shouldUseParallel(config, 'bundle')
? dispatchBundleWorkload(ssr, plugins)
: bundleWorkload.apply({ config, options }, [ssr, plugins])
})
)
/**
* Check if a module is statically imported by at least one entry.
*/
function isEagerChunk(id: string, getModuleInfo: Rollup.GetModuleInfo) {
if (
id.includes('node_modules') &&
!CSS_LANGS_RE.test(id) &&
staticImportedByEntry(id, getModuleInfo, cache)
) {
return true
}
}
return {
clientResult: config.mpa
? await bundleMPA(config, serverResult!, clientJSMap)
: clientResult!,
serverResult: serverResult!,
pageToHashMap
function staticImportedByEntry(
id: string,
getModuleInfo: Rollup.GetModuleInfo,
cache: Map<string, boolean>,
entryRE: RegExp | null = null,
importStack: string[] = []
): boolean {
if (cache.has(id)) {
return !!cache.get(id)
}
if (importStack.includes(id)) {
// circular deps!
cache.set(id, false)
return false
}
const mod = getModuleInfo(id)
if (!mod) {
cache.set(id, false)
return false
}
if (entryRE ? entryRE.test(id) : mod.isEntry) {
cache.set(id, true)
return true
}
const someImporterIs = mod.importers.some((importer: string) =>
staticImportedByEntry(
importer,
getModuleInfo,
cache,
entryRE,
importStack.concat(id)
)
)
cache.set(id, someImporterIs)
return someImporterIs
}

@ -32,7 +32,7 @@ export interface RenderPageContext {
export async function renderPage(
render: (path: string) => Promise<SSGContext>,
pageNameRaw: string,
page: string,
renderContext: RenderPageContext
) {
const {
@ -46,7 +46,7 @@ export async function renderPage(
additionalHeadTags
} = renderContext
const page = config.rewrites.inv[pageNameRaw] || pageNameRaw
page = config.rewrites.inv[page] ?? page
const routePath = `/${page.replace(/\.md$/, '')}`
const siteData = resolveSiteDataByRoute(config.site, routePath)
@ -170,34 +170,37 @@ export async function renderPage(
const dir = pageData.frontmatter.dir || siteData.dir || 'ltr'
const html = [
`<!DOCTYPE html>`,
`<html lang="${siteData.lang}" dir="${dir}">`,
`<head>`,
`<meta charset="utf-8">`,
const html = `<!DOCTYPE html>
<html lang="${siteData.lang}" dir="${dir}">
<head>
<meta charset="utf-8">
${
isMetaViewportOverridden(head)
? ''
: '<meta name="viewport" content="width=device-width,initial-scale=1">',
`<title>${title}</title>`,
: '<meta name="viewport" content="width=device-width,initial-scale=1">'
}
<title>${title}</title>
${
isDescriptionOverridden(head)
? ''
: `<meta name="description" content="${description}">`,
`<meta name="generator" content="VitePress v${version}">`,
stylesheetLink,
metadataScript.inHead ? metadataScript.html : '',
: `<meta name="description" content="${description}">`
}
<meta name="generator" content="VitePress v${version}">
${stylesheetLink}
${metadataScript.inHead ? metadataScript.html : ''}
${
appChunk
? `<script type="module" src="${siteData.base}${appChunk.fileName}"></script>`
: '',
await renderHead(head),
`</head>`,
`<body>`,
teleports?.body || '',
`<div id="app">${content}</div>`,
metadataScript.inHead ? '' : metadataScript.html,
inlinedScript,
`</body>`,
`</html>`
].join('')
: ''
}
${await renderHead(head)}
</head>
<body>${teleports?.body || ''}
<div id="app">${content}</div>
${metadataScript.inHead ? '' : metadataScript.html}
${inlinedScript}
</body>
</html>`
const htmlFileName = path.join(config.outDir, page.replace(/\.md$/, '.html'))
await fs.ensureDir(path.dirname(htmlFileName))

@ -1,212 +0,0 @@
import path from 'path'
import { fileURLToPath } from 'url'
import {
normalizePath,
type BuildOptions,
type Rollup,
type InlineConfig as ViteInlineConfig,
type PluginOption
} from 'vite'
import { APP_PATH } from '../alias'
import type { SiteConfig } from '../config'
import { slash } from '../shared'
import { escapeRegExp, sanitizeFileName } from '../shared'
// https://github.com/vitejs/vite/blob/d2aa0969ee316000d3b957d7e879f001e85e369e/packages/vite/src/node/plugins/splitVendorChunk.ts#L14
const CSS_LANGS_RE =
/\.(css|less|sass|scss|styl|stylus|pcss|postcss|sss)(?:$|\?)/
const clientDir = normalizePath(
path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../client')
)
// these deps are also being used in the client code (outside of the theme)
// exclude them from the theme chunk so there is no circular dependency
const excludedModules = [
'/@siteData',
'node_modules/@vueuse/core/',
'node_modules/@vueuse/shared/',
'node_modules/vue/',
'node_modules/vue-demi/',
clientDir
]
const themeEntryRE = (themeDir: string) =>
new RegExp(
`^${escapeRegExp(
path.resolve(themeDir, 'index.js').replace(/\\/g, '/')
).slice(0, -2)}m?(j|t)s`
)
const cache = new Map<string, boolean>()
const cacheTheme = new Map<string, boolean>()
/**
* Check if a module is statically imported by at least one entry.
*/
function isEagerChunk(id: string, getModuleInfo: Rollup.GetModuleInfo) {
if (
id.includes('node_modules') &&
!CSS_LANGS_RE.test(id) &&
staticImportedByEntry(id, getModuleInfo, cache)
) {
return true
}
}
function staticImportedByEntry(
id: string,
getModuleInfo: Rollup.GetModuleInfo,
cache: Map<string, boolean>,
entryRE: RegExp | null = null,
importStack: string[] = []
): boolean {
if (cache.has(id)) {
return !!cache.get(id)
}
if (importStack.includes(id)) {
// circular deps!
cache.set(id, false)
return false
}
const mod = getModuleInfo(id)
if (!mod) {
cache.set(id, false)
return false
}
if (entryRE ? entryRE.test(id) : mod.isEntry) {
cache.set(id, true)
return true
}
const someImporterIs = mod.importers.some((importer: string) =>
staticImportedByEntry(
importer,
getModuleInfo,
cache,
entryRE,
importStack.concat(id)
)
)
cache.set(id, someImporterIs)
return someImporterIs
}
// define custom rollup input
// this is a multi-entry build - every page is considered an entry chunk
// the loading is done via filename conversion rules so that the
// metadata doesn't need to be included in the main chunk.
const resolveInput = (config: SiteConfig) =>
Object.fromEntries(
config.pages.map((file) => {
// page filename conversion
// foo/bar.md -> foo_bar.md
const alias = config.rewrites.map[file] || file
return [
slash(alias).replace(/\//g, '_'),
path.resolve(config.srcDir, file)
]
})
) as Record<string, string>
export default async function resolveViteConfig(
ssr: boolean,
{
config,
options,
plugins
}: {
config: SiteConfig
options: BuildOptions
plugins: PluginOption[]
}
): Promise<ViteInlineConfig> {
return {
root: config.srcDir,
cacheDir: config.cacheDir,
base: config.site.base,
logLevel: config.vite?.logLevel ?? 'warn',
plugins,
ssr: {
noExternal: ['vitepress', '@docsearch/css']
},
build: {
...options,
emptyOutDir: true,
ssr,
ssrEmitAssets: config.mpa,
// minify with esbuild in MPA mode (for CSS)
minify: ssr
? config.mpa && 'esbuild'
: options.minify ?? !process.env.DEBUG,
outDir: ssr ? config.tempDir : config.outDir,
cssCodeSplit: false,
rollupOptions: {
...options.rollupOptions,
input: {
...resolveInput(config),
// use different entry based on ssr or not
app: path.resolve(APP_PATH, ssr ? 'ssr.js' : 'index.js')
},
// important so that each page chunk and the index export things for each
// other
preserveEntrySignatures: 'allow-extension',
output: {
sanitizeFileName,
...options.rollupOptions?.output,
assetFileNames: `${config.assetsDir}/[name].[hash].[ext]`,
...(ssr
? {
entryFileNames: '[name].js',
chunkFileNames: '[name].[hash].js'
}
: {
entryFileNames: `${config.assetsDir}/[name].[hash].js`,
chunkFileNames(chunk) {
// avoid ads chunk being intercepted by adblock
return /(?:Carbon|BuySell)Ads/.test(chunk.name)
? `${config.assetsDir}/chunks/ui-custom.[hash].js`
: `${config.assetsDir}/chunks/[name].[hash].js`
},
manualChunks(id, ctx) {
// move known framework code into a stable chunk so that
// custom theme changes do not invalidate hash for all pages
if (id.startsWith('\0vite')) {
return 'framework'
}
if (id.includes('plugin-vue:export-helper')) {
return 'framework'
}
if (
id.includes(`${clientDir}/app`) &&
id !== `${clientDir}/app/index.js`
) {
return 'framework'
}
if (
isEagerChunk(id, ctx.getModuleInfo) &&
/@vue\/(runtime|shared|reactivity)/.test(id)
) {
return 'framework'
}
if (
(id.startsWith(`${clientDir}/theme-default`) ||
!excludedModules.some((i) => id.includes(i))) &&
staticImportedByEntry(
id,
ctx.getModuleInfo,
cacheTheme,
themeEntryRE(config.themeDir)
)
) {
return 'theme'
}
}
})
}
}
},
configFile: config.vite?.configFile
}
}

@ -32,9 +32,11 @@ interface IndexObject {
titles: string[]
}
// SSR being `true` or `false` dose not affect the config related to markdown
// renderer, so it can be reused.
let md: MarkdownRenderer
let flagAllIndexed = false
let flagScanned = false
const taskByLocale = new Map<string, Promise<void>>()
const filesByLocale = new Map<string, Set<string>>()
const indexByLocale = new Map<string, MiniSearch<IndexObject>>()
@ -160,9 +162,10 @@ export async function localSearchPlugin(
}
}
function indexAll() {
if (flagAllIndexed) return
flagAllIndexed = true
// scan all pages, group by locale and create empty indexes accordingly.
function scanForIndex() {
if (flagScanned) return
flagScanned = true
for (const page of siteConfig.pages) {
const file = path.join(siteConfig.srcDir, page)
@ -209,7 +212,7 @@ export async function localSearchPlugin(
async load(id) {
if (id === LOCAL_SEARCH_INDEX_REQUEST_PATH) {
indexAll()
scanForIndex()
let records: string[] = []
for (const [locale] of filesByLocale) {
records.push(

@ -28,7 +28,10 @@ export async function createServer(
configFile: config.vite?.configFile
}).then((server) =>
Object.assign({}, server, {
close: () => server.close().then(() => stopWorkers('server.close()'))
close() {
stopWorkers('server.close()')
return server.close()
}
})
)
}

@ -19,6 +19,7 @@ export default class Queue<T> {
enqueue(data: T) {
if (this.closed)
throw new Error(`Failed to enqueue ${data}, queue already closed`)
if (data === null) return this.close()
if (this.pending.length) this.pending.shift()!(data)
else this.queue.push(data)
}

@ -1,5 +1,8 @@
import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'
import RPCContext, { deferPromise } from 'rpc-magic-proxy'
import RPCContext, {
deferPromise,
type RPCContextOptions
} from 'rpc-magic-proxy'
import { task, updateCurrentTask } from './utils/task'
import c from 'picocolors'
import Queue from './utils/queue'
@ -7,8 +10,12 @@ import _debug from 'debug'
import type { SiteConfig } from 'siteConfig'
import humanizeDuration from 'humanize-duration'
export type SupportsParallel = 'bundle' | 'render' | 'local-search'
export type SupportsParallel = 'render' | 'local-search'
const options: RPCContextOptions = {
carryThis: false,
carrySideEffect: false
}
/**
* Checks if the given task should be run in parallel.
* If task is omitted, checks if any task should be run in parallel.
@ -61,7 +68,7 @@ export async function launchWorkers(numWorkers: number, context: Object) {
debug(`launching ${numWorkers} workers`)
taskQueue = new Queue<WorkerTask>()
const allInitialized: Array<Promise<void>> = []
const ctx = new RPCContext()
const ctx = new RPCContext(options)
const getNextTask = () => taskQueue?.dequeue() ?? null
for (let i = 0; i < numWorkers; i++) {
const workerId = (i + 1).toString().padStart(2, '0')
@ -99,7 +106,8 @@ export function updateContext(context: Object) {
return Promise.all(workers.map(({ hooks }) => hooks.updateContext(context)))
}
// Wait for workers to drain the taskQueue and exit.
// Wait for workers to finish and exit.
// Will return immediately if no worker exists.
export async function stopWorkers(reason: string = 'exit') {
debug('stopping workers:', reason)
const allClosed = workers.map((w) =>
@ -111,10 +119,10 @@ export async function stopWorkers(reason: string = 'exit') {
taskQueue = null
const success = await Promise.any([
Promise.all(allClosed).then(() => true),
new Promise<false>((res) => setTimeout(() => res(false), 2000))
new Promise<false>((res) => setTimeout(() => res(false), 1500))
])
if (!success) {
console.warn('forcefully terminating workers')
debug('forcefully terminating workers')
for (const w of workers) {
try {
w.terminate()
@ -152,7 +160,7 @@ export function registerWorkload<T extends Object, K extends any[], V>(
// Will keep querying next workload from main thread
async function workerMainLoop() {
const ctx = new RPCContext().bind(parentPort! as any)
const ctx = new RPCContext(options).bind(parentPort! as any)
const {
workerMeta: _workerMeta,
initWorkerHooks,

Loading…
Cancel
Save