From d16e67a02a01f7a145ff8a91020eaf67eb9b8eb9 Mon Sep 17 00:00:00 2001 From: Divyansh Singh <40380293+brc-dd@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:37:47 +0530 Subject: [PATCH] use p-map --- package.json | 1 + pnpm-lock.yaml | 8 ++++ src/node/build/build.ts | 30 ++++--------- src/node/utils/pool.ts | 96 ----------------------------------------- 4 files changed, 17 insertions(+), 118 deletions(-) delete mode 100644 src/node/utils/pool.ts diff --git a/package.json b/package.json index 983c60dc..80624d72 100644 --- a/package.json +++ b/package.json @@ -174,6 +174,7 @@ "nanoid": "^5.0.4", "npm-run-all": "^4.1.5", "ora": "^8.0.1", + "p-map": "^7.0.0", "path-to-regexp": "^6.2.1", "picocolors": "^1.0.0", "pkg-dir": "^8.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 059f348b..a1118b09 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -222,6 +222,9 @@ importers: ora: specifier: ^8.0.1 version: 8.0.1 + p-map: + specifier: ^7.0.0 + version: 7.0.0 path-to-regexp: specifier: ^6.2.1 version: 6.2.1 @@ -3540,6 +3543,11 @@ packages: p-limit: 4.0.0 dev: true + /p-map@7.0.0: + resolution: {integrity: sha512-EZl03dLKv3RypkrjlevZoNwQMSy4bAblWcR18zhonktnN4fUs3asFQKSe0awn982omGxamvbejqQKQYDJYHCEg==} + engines: {node: '>=18'} + dev: true + /parse-json@4.0.0: resolution: {integrity: sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==} engines: {node: '>=4'} diff --git a/src/node/build/build.ts b/src/node/build/build.ts index 781ded11..da9b6593 100644 --- a/src/node/build/build.ts +++ b/src/node/build/build.ts @@ -1,6 +1,7 @@ import { createHash } from 'crypto' import fs from 'fs-extra' import { createRequire } from 'module' +import pMap from 'p-map' import path from 'path' import { packageDirectorySync } from 'pkg-dir' import { rimraf } from 'rimraf' @@ -10,7 +11,6 @@ import { resolveConfig, type SiteConfig } from '../config' import { slash, type HeadConfig } from '../shared' import { deserializeFunctions, serializeFunctions } from '../utils/fnSerialize' import { task } from '../utils/task' -import Pool from '../utils/pool' import { bundle } from './bundle' import { generateSitemap } from './generateSitemap' import { renderPage } from './render' @@ -107,24 +107,10 @@ export async function build( } } - /** - * Since this pool is not a thread pool, we can give it a "thread count" - * larger than the actual number of CPUs available. - * --- - * https://github.com/vuejs/vitepress/issues/3362 - * This fix will defer the memory allocation of render tasks. - * --- - * By limiting the max number of dispatched promises, it allows NodeJS to - * reclaim memory associated to each promise, avoiding memory allocation - * failures when handling large volume of pages. - * --- - * Fixes: #3362 - */ - const renderPageTaskPool = new Pool(64) - // Pool all rendering tasks - for (const page of ['404.md', ...siteConfig.pages]) - renderPageTaskPool.add(() => - renderPage( + await pMap( + ['404.md', ...siteConfig.pages], + async (page) => { + await renderPage( render, siteConfig, siteConfig.rewrites.map[page] || page, @@ -136,9 +122,9 @@ export async function build( metadataScript, additionalHeadTags ) - ) - // Wait for all rendering tasks to finish - await renderPageTaskPool.drain() + }, + { concurrency: 64 } + ) }) // emit page hash map for the case where a user session is open diff --git a/src/node/utils/pool.ts b/src/node/utils/pool.ts deleted file mode 100644 index 13bcba49..00000000 --- a/src/node/utils/pool.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* --------------------------------------------------------- - * Copyright (c) 2023 Yuxuan Zhang, web-dev@z-yx.cc - * This source code is licensed under the MIT license. - * You may find the full license in project root directory. - * ------------------------------------------------------ */ - -import os from 'os' - -/** - * Create a callback pool, with a maximum number of parallel tasks. - * Queued tasks will be executed directly when the pool is not full. - * Otherwise, they will be executed in a first-in-first-out manner - * when a dispatched task finishes. - * --- - * When used with 'node:child_process', you can control the pooling - * of parallel tasks, and avoid spawning too many child processes. - */ -export default class Pool extends EventTarget { - // Number of dispatched tasks (currently in progress) - private numExecutors: number = 0 - - // The pending task queue. Promise return type varies by task - private pendingTaskQueue: Array<() => Promise> = [] - - /** - * This executor will keep executing next available task in queue. - * Therefore, it always owns a running task until the queue drains. - * You can think of it as a "worker thread" in a thread pool. - */ - private executor() { - // Only launch new executor when the pool is not full - if (this.numExecutors >= this.maxThreads) return - // Register new executor - this.numExecutors++ - ;(async () => { - while (this.pendingTaskQueue.length > 0) { - try { - const nextTaskInQueue = this.pendingTaskQueue.shift()! - await nextTaskInQueue() - } catch (e) {} - } - // Unregister the executor - this.numExecutors-- - // Dispatch "drain" event when the last executor finishes - if (this.numExecutors === 0) this.dispatchEvent(new Event('drain')) - })() - } - - /** - * @param maxThreads Maximum number of parallel tasks - */ - constructor(private maxThreads: number = os.cpus().length) { - super() - } - - /** - * Dispatch a task to the pool. Use it like `new Promise(task)` - * @param task The task call back, just like the one in Promise constructor - * @returns - * The constructed promise object, will resolve to the value - * passed to resolve callback - */ - add(task: (...args: any[]) => T, ...args: any[]): Promise { - // Create deferred promise handlers - let resolve: (v: T) => void, reject: (reason?: any) => void - const promise = new Promise( - (...callbacks) => ([resolve, reject] = callbacks) - ) - // Push the task to queue for deferred resolution - // Notice the task may NOT be invoked right away - this.pendingTaskQueue.push(async () => { - try { - resolve(await task(...args)) - } catch (error: any) { - reject(error) - } - }) - // Try to launch a new executor - this.executor() - // Return the promise - return promise - } - - /** - * Wait for all dispatched executors to finish. - * Return immediately if no executor is running. - * @returns A promise that resolves when the pool drains - */ - async drain() { - if (this.numExecutors > 0) - await new Promise((resolve) => - this.addEventListener('drain', () => resolve(), { once: true }) - ) - return - } -}