pull/3366/head
Divyansh Singh 2 years ago
parent b5e25f949c
commit d16e67a02a

@ -174,6 +174,7 @@
"nanoid": "^5.0.4",
"npm-run-all": "^4.1.5",
"ora": "^8.0.1",
"p-map": "^7.0.0",
"path-to-regexp": "^6.2.1",
"picocolors": "^1.0.0",
"pkg-dir": "^8.0.0",

@ -222,6 +222,9 @@ importers:
ora:
specifier: ^8.0.1
version: 8.0.1
p-map:
specifier: ^7.0.0
version: 7.0.0
path-to-regexp:
specifier: ^6.2.1
version: 6.2.1
@ -3540,6 +3543,11 @@ packages:
p-limit: 4.0.0
dev: true
/p-map@7.0.0:
resolution: {integrity: sha512-EZl03dLKv3RypkrjlevZoNwQMSy4bAblWcR18zhonktnN4fUs3asFQKSe0awn982omGxamvbejqQKQYDJYHCEg==}
engines: {node: '>=18'}
dev: true
/parse-json@4.0.0:
resolution: {integrity: sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==}
engines: {node: '>=4'}

@ -1,6 +1,7 @@
import { createHash } from 'crypto'
import fs from 'fs-extra'
import { createRequire } from 'module'
import pMap from 'p-map'
import path from 'path'
import { packageDirectorySync } from 'pkg-dir'
import { rimraf } from 'rimraf'
@ -10,7 +11,6 @@ import { resolveConfig, type SiteConfig } from '../config'
import { slash, type HeadConfig } from '../shared'
import { deserializeFunctions, serializeFunctions } from '../utils/fnSerialize'
import { task } from '../utils/task'
import Pool from '../utils/pool'
import { bundle } from './bundle'
import { generateSitemap } from './generateSitemap'
import { renderPage } from './render'
@ -107,24 +107,10 @@ export async function build(
}
}
/**
* Since this pool is not a thread pool, we can give it a "thread count"
* larger than the actual number of CPUs available.
* ---
* https://github.com/vuejs/vitepress/issues/3362
* This fix will defer the memory allocation of render tasks.
* ---
* By limiting the max number of dispatched promises, it allows NodeJS to
* reclaim memory associated to each promise, avoiding memory allocation
* failures when handling large volume of pages.
* ---
* Fixes: #3362
*/
const renderPageTaskPool = new Pool(64)
// Pool all rendering tasks
for (const page of ['404.md', ...siteConfig.pages])
renderPageTaskPool.add(() =>
renderPage(
await pMap(
['404.md', ...siteConfig.pages],
async (page) => {
await renderPage(
render,
siteConfig,
siteConfig.rewrites.map[page] || page,
@ -136,9 +122,9 @@ export async function build(
metadataScript,
additionalHeadTags
)
},
{ concurrency: 64 }
)
// Wait for all rendering tasks to finish
await renderPageTaskPool.drain()
})
// emit page hash map for the case where a user session is open

@ -1,96 +0,0 @@
/* ---------------------------------------------------------
* Copyright (c) 2023 Yuxuan Zhang, web-dev@z-yx.cc
* This source code is licensed under the MIT license.
* You may find the full license in project root directory.
* ------------------------------------------------------ */
import os from 'os'
/**
* Create a callback pool, with a maximum number of parallel tasks.
* Queued tasks will be executed directly when the pool is not full.
* Otherwise, they will be executed in a first-in-first-out manner
* when a dispatched task finishes.
* ---
* When used with 'node:child_process', you can control the pooling
* of parallel tasks, and avoid spawning too many child processes.
*/
export default class Pool extends EventTarget {
// Number of dispatched tasks (currently in progress)
private numExecutors: number = 0
// The pending task queue. Promise return type <T> varies by task
private pendingTaskQueue: Array<() => Promise<any>> = []
/**
* This executor will keep executing next available task in queue.
* Therefore, it always owns a running task until the queue drains.
* You can think of it as a "worker thread" in a thread pool.
*/
private executor() {
// Only launch new executor when the pool is not full
if (this.numExecutors >= this.maxThreads) return
// Register new executor
this.numExecutors++
;(async () => {
while (this.pendingTaskQueue.length > 0) {
try {
const nextTaskInQueue = this.pendingTaskQueue.shift()!
await nextTaskInQueue()
} catch (e) {}
}
// Unregister the executor
this.numExecutors--
// Dispatch "drain" event when the last executor finishes
if (this.numExecutors === 0) this.dispatchEvent(new Event('drain'))
})()
}
/**
* @param maxThreads Maximum number of parallel tasks
*/
constructor(private maxThreads: number = os.cpus().length) {
super()
}
/**
* Dispatch a task to the pool. Use it like `new Promise<T>(task)`
* @param task The task call back, just like the one in Promise constructor
* @returns
* The constructed promise object, will resolve to the value
* passed to resolve callback
*/
add<T = any>(task: (...args: any[]) => T, ...args: any[]): Promise<T> {
// Create deferred promise handlers
let resolve: (v: T) => void, reject: (reason?: any) => void
const promise = new Promise<T>(
(...callbacks) => ([resolve, reject] = callbacks)
)
// Push the task to queue for deferred resolution
// Notice the task may NOT be invoked right away
this.pendingTaskQueue.push(async () => {
try {
resolve(await task(...args))
} catch (error: any) {
reject(error)
}
})
// Try to launch a new executor
this.executor()
// Return the promise
return promise
}
/**
* Wait for all dispatched executors to finish.
* Return immediately if no executor is running.
* @returns A promise that resolves when the pool drains
*/
async drain() {
if (this.numExecutors > 0)
await new Promise<void>((resolve) =>
this.addEventListener('drain', () => resolve(), { once: true })
)
return
}
}
Loading…
Cancel
Save