From da71173afc8eece9af49ff25ad1bff9bbd676096 Mon Sep 17 00:00:00 2001 From: Divyansh Singh <40380293+brc-dd@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:18:02 +0530 Subject: [PATCH] fix: retry file reads when out of file descriptors Content, include, route-template and search reads now run concurrently, so large sites can hit EMFILE/ENFILE. Route all async reads through a shared readFile util that retries with backoff. Co-Authored-By: Claude Fable 5 --- src/node/contentLoader.ts | 5 +++-- src/node/plugins/dynamicRoutesPlugin.ts | 4 ++-- src/node/plugins/localSearchPlugin.ts | 4 ++-- src/node/serve/serve.ts | 7 ++----- src/node/utils/fs.ts | 19 +++++++++++++++++++ src/node/utils/processIncludes.ts | 4 ++-- 6 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 src/node/utils/fs.ts diff --git a/src/node/contentLoader.ts b/src/node/contentLoader.ts index 7f4d1cb2..8396b35c 100644 --- a/src/node/contentLoader.ts +++ b/src/node/contentLoader.ts @@ -1,5 +1,5 @@ import matter from 'gray-matter' -import { readFile, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' import path from 'node:path' import pMap from 'p-map' import { normalizePath } from 'vite' @@ -9,6 +9,7 @@ import { mergeMarkdownLocales } from './markdown/markdown' import type { Awaitable, MarkdownEnv } from './shared' +import { readFile } from './utils/fs' import { glob, normalizeGlob, type GlobOptions } from './utils/glob' export interface ContentOptions { @@ -121,7 +122,7 @@ export function createContentLoader( if (cached && timestamp === cached.timestamp) return cached.data - const src = await readFile(file, 'utf8') + const src = await readFile(file) const renderExcerpt = options.excerpt const { data: frontmatter, excerpt } = matter( diff --git a/src/node/plugins/dynamicRoutesPlugin.ts b/src/node/plugins/dynamicRoutesPlugin.ts index 69f7c03d..3f29a27a 100644 --- a/src/node/plugins/dynamicRoutesPlugin.ts +++ b/src/node/plugins/dynamicRoutesPlugin.ts @@ -1,5 +1,4 @@ import fs from 'node:fs' -import { readFile } from 'node:fs/promises' import path from 'node:path' import c from 'picocolors' import pm from 'picomatch' @@ -13,6 +12,7 @@ import { } from 'vite' import type { Awaitable } from '../shared' import { type SiteConfig, type UserConfig } from '../siteConfig' +import { readFile } from '../utils/fs' import { glob, normalizeGlob, type GlobOptions } from '../utils/glob' import { ModuleGraph } from '../utils/moduleGraph' import { resolveRewrites } from './rewritesPlugin' @@ -158,7 +158,7 @@ export const dynamicRoutesPlugin = async ( moduleGraph.add(id, [routeFile]) moduleGraph.add(routeFile, [matched.loaderPath]) - let baseContent = await readFile(routeFile, 'utf8') + let baseContent = await readFile(routeFile) // inject raw content // this is intended for integration with CMS diff --git a/src/node/plugins/localSearchPlugin.ts b/src/node/plugins/localSearchPlugin.ts index 65a9b1d8..1d6b8a3b 100644 --- a/src/node/plugins/localSearchPlugin.ts +++ b/src/node/plugins/localSearchPlugin.ts @@ -1,6 +1,5 @@ import { prefixRegex } from '@rolldown/pluginutils' import MiniSearch from 'minisearch' -import { readFile } from 'node:fs/promises' import path from 'node:path' import { createDebug } from 'obug' import type { Plugin, ViteDevServer } from 'vite' @@ -8,6 +7,7 @@ import type { SiteConfig } from '../config' import type { DefaultTheme } from '../defaultTheme' import { createMarkdownRenderer } from '../markdown/markdown' import { getLocaleForPath, slash, type MarkdownEnv } from '../shared' +import { readFile } from '../utils/fs' import { processIncludes } from '../utils/processIncludes' const debug = createDebug('vitepress:local-search') @@ -52,7 +52,7 @@ export async function localSearchPlugin( const { srcDir, cleanUrls = false } = siteConfig const relativePath = slash(path.relative(srcDir, file)) const env: MarkdownEnv = { path: file, relativePath, cleanUrls } - const raw = await readFile(file, 'utf8').catch((e) => { + const raw = await readFile(file).catch((e) => { if (e.code === 'ENOENT') { debug(`File not found: ${file}`) return '' diff --git a/src/node/serve/serve.ts b/src/node/serve/serve.ts index 7483702b..e46e5522 100644 --- a/src/node/serve/serve.ts +++ b/src/node/serve/serve.ts @@ -1,9 +1,9 @@ import compression from '@polka/compression' -import { readFile } from 'node:fs/promises' import path from 'node:path' import polka, { type IOptions } from 'polka' import sirv from 'sirv' import { resolveConfig } from '../config' +import { readFile } from '../utils/fs' export interface ServeOptions { base?: string @@ -21,10 +21,7 @@ export async function serve(options: ServeOptions = {}) { const notAnAsset = (pathname: string) => !pathname.includes(`/${config.assetsDir}/`) - const notFound = await readFile( - path.resolve(config.outDir, './404.html'), - 'utf8' - ) + const notFound = await readFile(path.resolve(config.outDir, './404.html')) const onNoMatch: IOptions['onNoMatch'] = (req, res) => { res.statusCode = 404 if (notAnAsset(req.path)) res.write(notFound) diff --git a/src/node/utils/fs.ts b/src/node/utils/fs.ts new file mode 100644 index 00000000..d623a969 --- /dev/null +++ b/src/node/utils/fs.ts @@ -0,0 +1,19 @@ +import { readFile as fsReadFile } from 'node:fs/promises' + +const retryCodes = new Set(['EMFILE', 'ENFILE']) + +/** + * Reads a file as utf8, retrying with backoff when the process is + * temporarily out of file descriptors (EMFILE/ENFILE). + */ +export async function readFile(file: string): Promise { + for (let attempt = 0; ; attempt++) { + try { + return await fsReadFile(file, 'utf8') + } catch (e) { + const code = (e as NodeJS.ErrnoException).code + if (attempt >= 9 || !code || !retryCodes.has(code)) throw e + await new Promise((resolve) => setTimeout(resolve, 2 ** attempt * 10)) + } + } +} diff --git a/src/node/utils/processIncludes.ts b/src/node/utils/processIncludes.ts index e111648e..afc86b1a 100644 --- a/src/node/utils/processIncludes.ts +++ b/src/node/utils/processIncludes.ts @@ -1,9 +1,9 @@ import matter from 'gray-matter' import { replaceAsync, type MarkdownItAsync } from 'markdown-it-async' -import { readFile } from 'node:fs/promises' import path from 'node:path' import { findRegion } from '../markdown/plugins/snippet' import { slash, type MarkdownEnv } from '../shared' +import { readFile } from './fs' export function processIncludes( md: MarkdownItAsync, @@ -41,7 +41,7 @@ export function processIncludes( // chain are cycles, the same file may still be included by siblings if (includePath === file || ancestors.includes(includePath)) return m - let content = await readFile(includePath, 'utf8') + let content = await readFile(includePath) if (region) { const [regionName] = region