refactor: add concurrent promise pooling for render task (#3366)

fixes #3362
closes #3285

---------

Co-authored-by: Yuxuan Zhang <yuxuan@yuxuanzhang.net>
Co-authored-by: Divyansh Singh <40380293+brc-dd@users.noreply.github.com>
Co-authored-by: David Silva <srdavidsilva@gmail.com>

package.json

@@ -174,6 +174,7 @@
     "nanoid": "^5.0.4",
     "npm-run-all": "^4.1.5",
     "ora": "^8.0.1",
+    "p-map": "^7.0.0",
     "path-to-regexp": "^6.2.1",
     "picocolors": "^1.0.0",
     "pkg-dir": "^8.0.0",

pnpm-lock.yaml

@@ -222,6 +222,9 @@ importers:
       ora:
         specifier: ^8.0.1
         version: 8.0.1
+      p-map:
+        specifier: ^7.0.0
+        version: 7.0.0
       path-to-regexp:
         specifier: ^6.2.1
         version: 6.2.1
@@ -3540,6 +3543,11 @@ packages:
       p-limit: 4.0.0
     dev: true
 
+  /p-map@7.0.0:
+    resolution: {integrity: sha512-EZl03dLKv3RypkrjlevZoNwQMSy4bAblWcR18zhonktnN4fUs3asFQKSe0awn982omGxamvbejqQKQYDJYHCEg==}
+    engines: {node: '>=18'}
+    dev: true
+
   /parse-json@4.0.0:
     resolution: {integrity: sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==}
     engines: {node: '>=4'}

src/node/build/build.ts

@@ -1,6 +1,7 @@
 import { createHash } from 'crypto'
 import fs from 'fs-extra'
 import { createRequire } from 'module'
+import pMap from 'p-map'
 import path from 'path'
 import { packageDirectorySync } from 'pkg-dir'
 import { rimraf } from 'rimraf'
@@ -106,23 +107,23 @@ export async function build(
         }
       }
 
-      await Promise.all(
+      await pMap(
-        ['404.md', ...siteConfig.pages]
+        ['404.md', ...siteConfig.pages],
-          .map((page) => siteConfig.rewrites.map[page] || page)
+        async (page) => {
-          .map((page) =>
+          await renderPage(
-            renderPage(
+            render,
-              render,
+            siteConfig,
-              siteConfig,
+            siteConfig.rewrites.map[page] || page,
-              page,
+            clientResult,
-              clientResult,
+            appChunk,
-              appChunk,
+            cssChunk,
-              cssChunk,
+            assets,
-              assets,
+            pageToHashMap,
-              pageToHashMap,
+            metadataScript,
-              metadataScript,
+            additionalHeadTags
-              additionalHeadTags
-            )
           )
+        },
+        { concurrency: siteConfig.buildConcurrency }
       )
     })
 

src/node/config.ts

@@ -141,7 +141,8 @@ export async function resolveConfig(
     transformPageData: userConfig.transformPageData,
     rewrites,
     userConfig,
-    sitemap: userConfig.sitemap
+    sitemap: userConfig.sitemap,
+    buildConcurrency: userConfig.buildConcurrency ?? 64
   }
 
   // to be shared with content loaders

src/node/siteConfig.ts

@@ -147,6 +147,15 @@ export interface UserConfig<ThemeConfig = any>
    */
   useWebFonts?: boolean
 
+  /**
+   * This option allows you to configure the concurrency of the build.
+   * A lower number will reduce the memory usage but will increase the build time.
+   *
+   * @experimental
+   * @default 64
+   */
+  buildConcurrency?: number
+
   /**
    * @experimental
    *
@@ -240,4 +249,5 @@ export interface SiteConfig<ThemeConfig = any>
   }
   logger: Logger
   userConfig: UserConfig
+  buildConcurrency: number
 }