From 5c8325989614042d373e5488102f15bff34122d9 Mon Sep 17 00:00:00 2001 From: Okinea Dev Date: Tue, 15 Apr 2025 15:31:52 +0200 Subject: [PATCH] format files --- docs/.vitepress/config/shared.ts | 2 +- src/node/config.ts | 2 +- src/node/plugins/llmstxt/helpers/index.ts | 208 +++---- src/node/plugins/llmstxt/helpers/logger.ts | 54 +- src/node/plugins/llmstxt/helpers/toc.ts | 416 ++++++------- src/node/plugins/llmstxt/helpers/utils.ts | 118 ++-- src/node/plugins/llmstxt/index.ts | 658 ++++++++++----------- src/node/plugins/llmstxt/types.d.ts | 438 +++++++------- 8 files changed, 948 insertions(+), 948 deletions(-) diff --git a/docs/.vitepress/config/shared.ts b/docs/.vitepress/config/shared.ts index 6ade0428..f2501caa 100644 --- a/docs/.vitepress/config/shared.ts +++ b/docs/.vitepress/config/shared.ts @@ -122,7 +122,7 @@ export const shared = defineConfig({ ), firebase: 'logos:firebase' } - }), + }) ] }, llms: { diff --git a/src/node/config.ts b/src/node/config.ts index 7aba01be..cf980952 100644 --- a/src/node/config.ts +++ b/src/node/config.ts @@ -243,7 +243,7 @@ export async function resolveSiteData( scrollOffset: userConfig.scrollOffset ?? 134, cleanUrls: !!userConfig.cleanUrls, contentProps: userConfig.contentProps, - llms: userConfig.llms ?? false, + llms: userConfig.llms ?? false } } diff --git a/src/node/plugins/llmstxt/helpers/index.ts b/src/node/plugins/llmstxt/helpers/index.ts index c6ca250d..66dab630 100644 --- a/src/node/plugins/llmstxt/helpers/index.ts +++ b/src/node/plugins/llmstxt/helpers/index.ts @@ -6,10 +6,10 @@ import matter from 'gray-matter' import type { DefaultTheme } from 'vitepress' import { defaultLLMsTxtTemplate } from '../constants' import type { - LinksExtension, - LlmstxtSettings, - PreparedFile, - VitePressConfig, + LinksExtension, + LlmstxtSettings, + PreparedFile, + VitePressConfig } from '../types' import { generateTOC } from './toc' import { expandTemplate, extractTitle, generateMetadata } from './utils' @@ -18,32 +18,32 @@ import { expandTemplate, extractTitle, generateMetadata } from './utils' * Options for generating the `llms.txt` file. */ export interface GenerateLLMsTxtOptions { - /** Path to the main documentation file `index.md`.*/ - indexMd: string + /** Path to the main documentation file `index.md`.*/ + indexMd: string - /** The source directory for the files. */ - srcDir: VitePressConfig['vitepress']['srcDir'] + /** The source directory for the files. */ + srcDir: VitePressConfig['vitepress']['srcDir'] - /** Template to use for generating `llms.txt`. */ - LLMsTxtTemplate?: LlmstxtSettings['customLLMsTxtTemplate'] + /** Template to use for generating `llms.txt`. */ + LLMsTxtTemplate?: LlmstxtSettings['customLLMsTxtTemplate'] - /** Template variables for `customLLMsTxtTemplate`. */ - templateVariables?: LlmstxtSettings['customTemplateVariables'] + /** Template variables for `customLLMsTxtTemplate`. */ + templateVariables?: LlmstxtSettings['customTemplateVariables'] - /** The VitePress configuration. */ - vitepressConfig?: VitePressConfig['vitepress']['userConfig'] + /** The VitePress configuration. */ + vitepressConfig?: VitePressConfig['vitepress']['userConfig'] - /** The base domain for the generated links. */ - domain?: LlmstxtSettings['domain'] + /** The base domain for the generated links. */ + domain?: LlmstxtSettings['domain'] - /** The link extension for generated links. */ - linksExtension?: LinksExtension + /** The link extension for generated links. */ + linksExtension?: LinksExtension - /** Whether to use clean URLs (without the extension). */ - cleanUrls?: VitePressConfig['cleanUrls'] + /** Whether to use clean URLs (without the extension). */ + cleanUrls?: VitePressConfig['cleanUrls'] - /** Optional sidebar configuration for organizing the TOC. */ - sidebar?: DefaultTheme.Sidebar + /** Optional sidebar configuration for organizing the TOC. */ + sidebar?: DefaultTheme.Sidebar } /** @@ -69,77 +69,77 @@ export interface GenerateLLMsTxtOptions { * @see https://llmstxt.org/#format */ export async function generateLLMsTxt( - preparedFiles: PreparedFile[], - options: GenerateLLMsTxtOptions, + preparedFiles: PreparedFile[], + options: GenerateLLMsTxtOptions ): Promise { - const { - indexMd, - srcDir, - LLMsTxtTemplate = defaultLLMsTxtTemplate, - templateVariables = {}, - vitepressConfig, - domain, - sidebar, - linksExtension, - cleanUrls, - } = options - - // @ts-expect-error - matter.clearCache() - - const indexMdContent = await fs.readFile(indexMd, 'utf-8') - const indexMdFile = matter(indexMdContent as string) - - templateVariables.title ??= - indexMdFile.data?.hero?.name || - indexMdFile.data?.title || - vitepressConfig?.title || - vitepressConfig?.titleTemplate || - extractTitle(indexMdFile) || - 'LLMs Documentation' - - templateVariables.description ??= - indexMdFile.data?.hero?.text || - vitepressConfig?.description || - indexMdFile?.data?.description || - indexMdFile.data?.titleTemplate - - if (templateVariables.description) { - templateVariables.description = `> ${templateVariables.description}` - } - - templateVariables.details ??= - indexMdFile.data?.hero?.tagline || - indexMdFile.data?.tagline || - (!templateVariables.description && - 'This file contains links to all documentation sections.') - - templateVariables.toc ??= await generateTOC(preparedFiles, { - srcDir, - domain, - sidebarConfig: sidebar || vitepressConfig?.themeConfig?.sidebar, - linksExtension, - cleanUrls, - }) - - return expandTemplate(LLMsTxtTemplate, templateVariables) + const { + indexMd, + srcDir, + LLMsTxtTemplate = defaultLLMsTxtTemplate, + templateVariables = {}, + vitepressConfig, + domain, + sidebar, + linksExtension, + cleanUrls + } = options + + // @ts-expect-error + matter.clearCache() + + const indexMdContent = await fs.readFile(indexMd, 'utf-8') + const indexMdFile = matter(indexMdContent as string) + + templateVariables.title ??= + indexMdFile.data?.hero?.name || + indexMdFile.data?.title || + vitepressConfig?.title || + vitepressConfig?.titleTemplate || + extractTitle(indexMdFile) || + 'LLMs Documentation' + + templateVariables.description ??= + indexMdFile.data?.hero?.text || + vitepressConfig?.description || + indexMdFile?.data?.description || + indexMdFile.data?.titleTemplate + + if (templateVariables.description) { + templateVariables.description = `> ${templateVariables.description}` + } + + templateVariables.details ??= + indexMdFile.data?.hero?.tagline || + indexMdFile.data?.tagline || + (!templateVariables.description && + 'This file contains links to all documentation sections.') + + templateVariables.toc ??= await generateTOC(preparedFiles, { + srcDir, + domain, + sidebarConfig: sidebar || vitepressConfig?.themeConfig?.sidebar, + linksExtension, + cleanUrls + }) + + return expandTemplate(LLMsTxtTemplate, templateVariables) } /** * Options for generating the `llms-full.txt` file. */ export interface GenerateLLMsFullTxtOptions { - /** The source directory for the files. */ - srcDir: VitePressConfig['vitepress']['srcDir'] + /** The source directory for the files. */ + srcDir: VitePressConfig['vitepress']['srcDir'] - /** The base domain for the generated links. */ - domain?: LlmstxtSettings['domain'] + /** The base domain for the generated links. */ + domain?: LlmstxtSettings['domain'] - /** The link extension for generated links. */ - linksExtension?: LinksExtension + /** The link extension for generated links. */ + linksExtension?: LinksExtension - /** Whether to use clean URLs (without the extension). */ - cleanUrls?: VitePressConfig['cleanUrls'] + /** Whether to use clean URLs (without the extension). */ + cleanUrls?: VitePressConfig['cleanUrls'] } /** @@ -150,26 +150,26 @@ export interface GenerateLLMsFullTxtOptions { * @returns A string representing the full content of the LLMs.txt file. */ export function generateLLMsFullTxt( - preparedFiles: PreparedFile[], - options: GenerateLLMsFullTxtOptions, + preparedFiles: PreparedFile[], + options: GenerateLLMsFullTxtOptions ) { - const { srcDir, domain, linksExtension, cleanUrls } = options - - const llmsFullTxtContent = preparedFiles - .map((preparedFile) => { - const relativePath = path.relative(srcDir, preparedFile.path) - - return matter.stringify( - preparedFile.file.content, - generateMetadata(preparedFile.file, { - domain, - filePath: relativePath, - linksExtension, - cleanUrls, - }), - ) - }) - .join('\n---\n\n') - - return llmsFullTxtContent + const { srcDir, domain, linksExtension, cleanUrls } = options + + const llmsFullTxtContent = preparedFiles + .map((preparedFile) => { + const relativePath = path.relative(srcDir, preparedFile.path) + + return matter.stringify( + preparedFile.file.content, + generateMetadata(preparedFile.file, { + domain, + filePath: relativePath, + linksExtension, + cleanUrls + }) + ) + }) + .join('\n---\n\n') + + return llmsFullTxtContent } diff --git a/src/node/plugins/llmstxt/helpers/logger.ts b/src/node/plugins/llmstxt/helpers/logger.ts index f08aed46..8d7baf99 100644 --- a/src/node/plugins/llmstxt/helpers/logger.ts +++ b/src/node/plugins/llmstxt/helpers/logger.ts @@ -8,36 +8,36 @@ const logPrefix = pc.blue('llmstxt') + pc.dim(' » ') /** Logger object with standardized logging methods. */ const log = { - /** - * Logs an informational message to the console. - * - * @param message - The message to log. - */ - info: (message: string) => console.log(`${logPrefix} ${message}`), + /** + * Logs an informational message to the console. + * + * @param message - The message to log. + */ + info: (message: string) => console.log(`${logPrefix} ${message}`), - /** - * Logs a success message to the console. - * - * @param message - The message to log. - */ - success: (message: string) => - console.log(`${logPrefix}${pc.green('✓')} ${message}`), + /** + * Logs a success message to the console. + * + * @param message - The message to log. + */ + success: (message: string) => + console.log(`${logPrefix}${pc.green('✓')} ${message}`), - /** - * Logs a warning message to the console. - * - * @param message - The message to log. - */ - warn: (message: string) => - console.warn(`${logPrefix}${pc.yellow('⚠')} ${pc.yellow(message)}`), + /** + * Logs a warning message to the console. + * + * @param message - The message to log. + */ + warn: (message: string) => + console.warn(`${logPrefix}${pc.yellow('⚠')} ${pc.yellow(message)}`), - /** - * Logs an error message to the console. - * - * @param message - The message to log. - */ - error: (message: string) => - console.error(`${logPrefix}${pc.red('✗')} ${pc.red(message)}`), + /** + * Logs an error message to the console. + * + * @param message - The message to log. + */ + error: (message: string) => + console.error(`${logPrefix}${pc.red('✗')} ${pc.red(message)}`) } export default log diff --git a/src/node/plugins/llmstxt/helpers/toc.ts b/src/node/plugins/llmstxt/helpers/toc.ts index a4cac54b..324a13c8 100644 --- a/src/node/plugins/llmstxt/helpers/toc.ts +++ b/src/node/plugins/llmstxt/helpers/toc.ts @@ -1,10 +1,10 @@ import path from 'node:path' import type { DefaultTheme } from 'vitepress' import type { - LinksExtension, - LlmstxtSettings, - PreparedFile, - VitePressConfig, + LinksExtension, + LlmstxtSettings, + PreparedFile, + VitePressConfig } from '../types' import { generateLink, stripExtPosix } from './utils' @@ -19,14 +19,14 @@ import { generateLink, stripExtPosix } from './utils' * @returns The formatted TOC entry as a Markdown list item. */ export const generateTOCLink = ( - file: PreparedFile, - domain: LlmstxtSettings['domain'], - relativePath: string, - extension?: LinksExtension, - cleanUrls: VitePressConfig['cleanUrls'] = false, + file: PreparedFile, + domain: LlmstxtSettings['domain'], + relativePath: string, + extension?: LinksExtension, + cleanUrls: VitePressConfig['cleanUrls'] = false ) => { - const description: string = file.file.data.description - return `- [${file.title}](${generateLink(stripExtPosix(relativePath), domain, extension ?? '.md', cleanUrls)})${description ? `: ${description.trim()}` : ''}\n` + const description: string = file.file.data.description + return `- [${file.title}](${generateLink(stripExtPosix(relativePath), domain, extension ?? '.md', cleanUrls)})${description ? `: ${description.trim()}` : ''}\n` } /** @@ -36,23 +36,23 @@ export const generateTOCLink = ( * @returns Array of paths collected from the sidebar items. */ function collectPathsFromSidebarItems( - items: DefaultTheme.SidebarItem[], + items: DefaultTheme.SidebarItem[] ): string[] { - const paths: string[] = [] + const paths: string[] = [] - for (const item of items) { - // Add the current item's path if it exists - if (item.link) { - paths.push(item.link) - } + for (const item of items) { + // Add the current item's path if it exists + if (item.link) { + paths.push(item.link) + } - // Recursively add paths from nested items - if (item.items && Array.isArray(item.items)) { - paths.push(...collectPathsFromSidebarItems(item.items)) - } - } + // Recursively add paths from nested items + if (item.items && Array.isArray(item.items)) { + paths.push(...collectPathsFromSidebarItems(item.items)) + } + } - return paths + return paths } /** @@ -62,13 +62,13 @@ function collectPathsFromSidebarItems( * @returns Normalized link path for consistent comparison. */ export function normalizeLinkPath(link: string): string { - const normalizedPath = stripExtPosix(link) + const normalizedPath = stripExtPosix(link) - if (path.basename(normalizedPath) === 'index') { - return path.dirname(normalizedPath) - } + if (path.basename(normalizedPath) === 'index') { + return path.dirname(normalizedPath) + } - return normalizedPath + return normalizedPath } /** @@ -79,13 +79,13 @@ export function normalizeLinkPath(link: string): string { * @returns True if paths match, false otherwise */ export function isPathMatch(filePath: string, sidebarPath: string): boolean { - const normalizedFilePath = normalizeLinkPath(filePath) - const normalizedSidebarPath = normalizeLinkPath(sidebarPath) + const normalizedFilePath = normalizeLinkPath(filePath) + const normalizedSidebarPath = normalizeLinkPath(sidebarPath) - return ( - normalizedFilePath === normalizedSidebarPath || - normalizedFilePath === `${normalizedSidebarPath}.md` - ) + return ( + normalizedFilePath === normalizedSidebarPath || + normalizedFilePath === `${normalizedSidebarPath}.md` + ) } /** @@ -100,86 +100,86 @@ export function isPathMatch(filePath: string, sidebarPath: string): boolean { * @returns A string representing the formatted section of the TOC */ async function processSidebarSection( - section: DefaultTheme.SidebarItem, - preparedFiles: PreparedFile[], - srcDir: VitePressConfig['vitepress']['srcDir'], - domain?: LlmstxtSettings['domain'], - linksExtension?: LinksExtension, - cleanUrls?: VitePressConfig['cleanUrls'], - depth = 3, + section: DefaultTheme.SidebarItem, + preparedFiles: PreparedFile[], + srcDir: VitePressConfig['vitepress']['srcDir'], + domain?: LlmstxtSettings['domain'], + linksExtension?: LinksExtension, + cleanUrls?: VitePressConfig['cleanUrls'], + depth = 3 ): Promise { - let sectionTOC = '' - - // Add section header only if it has text and is not just a link container - if (section.text) { - sectionTOC += `${'#'.repeat(depth)} ${section.text}\n\n` - } - - // Process items in this section - if (section.items && Array.isArray(section.items)) { - const linkItems: string[] = [] - const nestedSections: string[] = [] - - // First pass: separate link items and nested sections - await Promise.all( - section.items.map(async (item) => { - // Process nested sections - if (item.items && item.items.length > 0) { - const processedSection = await processSidebarSection( - item, - preparedFiles, - srcDir, - domain, - linksExtension, - cleanUrls, - // Increase depth for nested sections to maintain proper heading levels - depth + 1, - ) - nestedSections.push(processedSection) - } - // Process link items - else if (item.link) { - // Normalize the link for matching - const normalizedItemLink = normalizeLinkPath(item.link) - - const matchingFile = preparedFiles.find((file) => { - const relativePath = `/${stripExtPosix(path.relative(srcDir, file.path))}` - return isPathMatch(relativePath, normalizedItemLink) - }) - - if (matchingFile) { - const relativePath = path.relative(srcDir, matchingFile.path) - linkItems.push( - generateTOCLink( - matchingFile, - domain, - relativePath, - linksExtension, - cleanUrls, - ), - ) - } - } - }), - ) - - // Add link items if any - if (linkItems.length > 0) { - sectionTOC += linkItems.join('') - } - - // Add a blank line before nested sections if we have link items - if (linkItems.length > 0 && nestedSections.length > 0) { - sectionTOC += '\n' - } - - // Add nested sections with appropriate spacing - if (nestedSections.length > 0) { - sectionTOC += nestedSections.join('\n') - } - } - - return sectionTOC + let sectionTOC = '' + + // Add section header only if it has text and is not just a link container + if (section.text) { + sectionTOC += `${'#'.repeat(depth)} ${section.text}\n\n` + } + + // Process items in this section + if (section.items && Array.isArray(section.items)) { + const linkItems: string[] = [] + const nestedSections: string[] = [] + + // First pass: separate link items and nested sections + await Promise.all( + section.items.map(async (item) => { + // Process nested sections + if (item.items && item.items.length > 0) { + const processedSection = await processSidebarSection( + item, + preparedFiles, + srcDir, + domain, + linksExtension, + cleanUrls, + // Increase depth for nested sections to maintain proper heading levels + depth + 1 + ) + nestedSections.push(processedSection) + } + // Process link items + else if (item.link) { + // Normalize the link for matching + const normalizedItemLink = normalizeLinkPath(item.link) + + const matchingFile = preparedFiles.find((file) => { + const relativePath = `/${stripExtPosix(path.relative(srcDir, file.path))}` + return isPathMatch(relativePath, normalizedItemLink) + }) + + if (matchingFile) { + const relativePath = path.relative(srcDir, matchingFile.path) + linkItems.push( + generateTOCLink( + matchingFile, + domain, + relativePath, + linksExtension, + cleanUrls + ) + ) + } + } + }) + ) + + // Add link items if any + if (linkItems.length > 0) { + sectionTOC += linkItems.join('') + } + + // Add a blank line before nested sections if we have link items + if (linkItems.length > 0 && nestedSections.length > 0) { + sectionTOC += '\n' + } + + // Add nested sections with appropriate spacing + if (nestedSections.length > 0) { + sectionTOC += nestedSections.join('\n') + } + } + + return sectionTOC } /** @@ -189,46 +189,46 @@ async function processSidebarSection( * @returns An array of sidebar items. */ function flattenSidebarConfig( - sidebarConfig: DefaultTheme.Sidebar, + sidebarConfig: DefaultTheme.Sidebar ): DefaultTheme.SidebarItem[] { - // If it's already an array, return as is - if (Array.isArray(sidebarConfig)) { - return sidebarConfig - } - - // If it's an object with path keys, flatten it - if (typeof sidebarConfig === 'object') { - return Object.values(sidebarConfig).flat() - } - - // If it's neither, return an empty array - return [] + // If it's already an array, return as is + if (Array.isArray(sidebarConfig)) { + return sidebarConfig + } + + // If it's an object with path keys, flatten it + if (typeof sidebarConfig === 'object') { + return Object.values(sidebarConfig).flat() + } + + // If it's neither, return an empty array + return [] } /** * Options for generating a Table of Contents (TOC). */ export interface GenerateTOCOptions { - /** - * The VitePress source directory. - */ - srcDir: VitePressConfig['vitepress']['srcDir'] - - /** - * Optional domain to prefix URLs with. - */ - domain?: LlmstxtSettings['domain'] - - /** - * Optional VitePress sidebar configuration. - */ - sidebarConfig?: DefaultTheme.Sidebar - - /** The link extension for generated links. */ - linksExtension?: LinksExtension - - /** Whether to use clean URLs (without the extension). */ - cleanUrls?: VitePressConfig['cleanUrls'] + /** + * The VitePress source directory. + */ + srcDir: VitePressConfig['vitepress']['srcDir'] + + /** + * Optional domain to prefix URLs with. + */ + domain?: LlmstxtSettings['domain'] + + /** + * Optional VitePress sidebar configuration. + */ + sidebarConfig?: DefaultTheme.Sidebar + + /** The link extension for generated links. */ + linksExtension?: LinksExtension + + /** Whether to use clean URLs (without the extension). */ + cleanUrls?: VitePressConfig['cleanUrls'] } /** @@ -244,66 +244,66 @@ export interface GenerateTOCOptions { * @returns A string representing the formatted Table of Contents. */ export async function generateTOC( - preparedFiles: PreparedFile[], - options: GenerateTOCOptions, + preparedFiles: PreparedFile[], + options: GenerateTOCOptions ): Promise { - const { srcDir, domain, sidebarConfig, linksExtension, cleanUrls } = options - let tableOfContent = '' - - let filesToProcess = preparedFiles - - // If sidebar configuration exists - if (sidebarConfig) { - // Flatten sidebar config if it's an object with path keys - const flattenedSidebarConfig = flattenSidebarConfig(sidebarConfig) - - // Process each top-level section in the flattened sidebar - if (flattenedSidebarConfig.length > 0) { - for (const section of flattenedSidebarConfig) { - tableOfContent += await processSidebarSection( - section, - filesToProcess, - srcDir, - domain, - linksExtension, - cleanUrls, - ) - - // tableOfContent = `${tableOfContent.trimEnd()}\n\n` - tableOfContent += '\n' - } - - // Find files that didn't match any section - const allSidebarPaths = collectPathsFromSidebarItems( - flattenedSidebarConfig, - ) - const unsortedFiles = filesToProcess.filter((file) => { - const relativePath = `/${stripExtPosix(path.relative(srcDir, file.path))}` - return !allSidebarPaths.some((sidebarPath) => - isPathMatch(relativePath, sidebarPath), - ) - }) - - // Add files that didn't match any section - if (unsortedFiles.length > 0) { - tableOfContent += '### Other\n\n' - filesToProcess = unsortedFiles - } - } - } - - const tocEntries: string[] = [] - - await Promise.all( - filesToProcess.map(async (file) => { - const relativePath = path.relative(srcDir, file.path) - tocEntries.push( - generateTOCLink(file, domain, relativePath, linksExtension, cleanUrls), - ) - }), - ) - - tableOfContent += tocEntries.join('') - - return tableOfContent + const { srcDir, domain, sidebarConfig, linksExtension, cleanUrls } = options + let tableOfContent = '' + + let filesToProcess = preparedFiles + + // If sidebar configuration exists + if (sidebarConfig) { + // Flatten sidebar config if it's an object with path keys + const flattenedSidebarConfig = flattenSidebarConfig(sidebarConfig) + + // Process each top-level section in the flattened sidebar + if (flattenedSidebarConfig.length > 0) { + for (const section of flattenedSidebarConfig) { + tableOfContent += await processSidebarSection( + section, + filesToProcess, + srcDir, + domain, + linksExtension, + cleanUrls + ) + + // tableOfContent = `${tableOfContent.trimEnd()}\n\n` + tableOfContent += '\n' + } + + // Find files that didn't match any section + const allSidebarPaths = collectPathsFromSidebarItems( + flattenedSidebarConfig + ) + const unsortedFiles = filesToProcess.filter((file) => { + const relativePath = `/${stripExtPosix(path.relative(srcDir, file.path))}` + return !allSidebarPaths.some((sidebarPath) => + isPathMatch(relativePath, sidebarPath) + ) + }) + + // Add files that didn't match any section + if (unsortedFiles.length > 0) { + tableOfContent += '### Other\n\n' + filesToProcess = unsortedFiles + } + } + } + + const tocEntries: string[] = [] + + await Promise.all( + filesToProcess.map(async (file) => { + const relativePath = path.relative(srcDir, file.path) + tocEntries.push( + generateTOCLink(file, domain, relativePath, linksExtension, cleanUrls) + ) + }) + ) + + tableOfContent += tocEntries.join('') + + return tableOfContent } diff --git a/src/node/plugins/llmstxt/helpers/utils.ts b/src/node/plugins/llmstxt/helpers/utils.ts index 93527087..e346204a 100644 --- a/src/node/plugins/llmstxt/helpers/utils.ts +++ b/src/node/plugins/llmstxt/helpers/utils.ts @@ -12,8 +12,8 @@ import type { LinksExtension, LlmstxtSettings, VitePressConfig } from '../types' * @returns An object containing the directory and file name. */ export const splitDirAndFile = (filepath: string) => ({ - dir: path.dirname(filepath), - file: path.basename(filepath), + dir: path.dirname(filepath), + file: path.basename(filepath) }) /** @@ -23,9 +23,9 @@ export const splitDirAndFile = (filepath: string) => ({ * @returns The filename without the extension. */ export const stripExt = (filepath: string) => { - const { dir, file } = splitDirAndFile(filepath) + const { dir, file } = splitDirAndFile(filepath) - return path.join(dir, path.basename(file, path.extname(file))) + return path.join(dir, path.basename(file, path.extname(file))) } /** @@ -35,9 +35,9 @@ export const stripExt = (filepath: string) => { * @returns The filename without the extension in POSIX format. */ export const stripExtPosix = (filepath: string) => { - const { dir, file } = splitDirAndFile(filepath) + const { dir, file } = splitDirAndFile(filepath) - return path.posix.join(dir, path.basename(file, path.extname(file))) + return path.posix.join(dir, path.basename(file, path.extname(file))) } /** @@ -47,13 +47,13 @@ export const stripExtPosix = (filepath: string) => { * @returns The extracted title, or `undefined` if no title is found. */ export function extractTitle(file: GrayMatterFile): string { - const titleFromFrontmatter = file.data?.title || file.data?.titleTemplate - let titleFromMarkdown: string | undefined + const titleFromFrontmatter = file.data?.title || file.data?.titleTemplate + let titleFromMarkdown: string | undefined - if (!titleFromFrontmatter) { - titleFromMarkdown = markdownTitle(file.content) - } - return titleFromFrontmatter || titleFromMarkdown + if (!titleFromFrontmatter) { + titleFromMarkdown = markdownTitle(file.content) + } + return titleFromFrontmatter || titleFromMarkdown } /** @@ -69,7 +69,7 @@ export function extractTitle(file: GrayMatterFile): string { * ``` */ const templateVariable = (key: string) => - new RegExp(`(\\n\\s*\\n)?\\{${key}\\}`, 'gi') + new RegExp(`(\\n\\s*\\n)?\\{${key}\\}`, 'gi') /** * Replaces occurrences of a template variable `{variable}` in a given content string with a provided value. @@ -89,15 +89,15 @@ const templateVariable = (key: string) => * ``` */ export function replaceTemplateVariable( - content: string, - variable: string, - value: string | undefined, - fallback?: string, + content: string, + variable: string, + value: string | undefined, + fallback?: string ) { - return content.replace(templateVariable(variable), (_, prefix) => { - const val = value?.length ? value : fallback?.length ? fallback : '' - return val ? `${prefix ? '\n\n' : ''}${val}` : '' - }) + return content.replace(templateVariable(variable), (_, prefix) => { + const val = value?.length ? value : fallback?.length ? fallback : '' + return val ? `${prefix ? '\n\n' : ''}${val}` : '' + }) } /** @@ -116,13 +116,13 @@ export function replaceTemplateVariable( * ``` */ export const expandTemplate = ( - template: string, - variables: Record, + template: string, + variables: Record ) => { - return Object.entries(variables).reduce( - (result, [key, value]) => replaceTemplateVariable(result, key, value), - template, - ) + return Object.entries(variables).reduce( + (result, [key, value]) => replaceTemplateVariable(result, key, value), + template + ) } /** @@ -134,32 +134,32 @@ export const expandTemplate = ( * @returns The generated link */ export const generateLink = ( - path: string, - domain?: string, - extension?: LinksExtension, - cleanUrls?: VitePressConfig['cleanUrls'], + path: string, + domain?: string, + extension?: LinksExtension, + cleanUrls?: VitePressConfig['cleanUrls'] ) => - expandTemplate('{domain}/{path}{extension}', { - domain: domain || '', - path, - extension: cleanUrls ? '' : extension, - }) + expandTemplate('{domain}/{path}{extension}', { + domain: domain || '', + path, + extension: cleanUrls ? '' : extension + }) /** * Options for generating metadata for markdown files. */ export interface GenerateMetadataOptions { - /** Optional domain name to prepend to the URL. */ - domain?: LlmstxtSettings['domain'] + /** Optional domain name to prepend to the URL. */ + domain?: LlmstxtSettings['domain'] - /** Path to the file relative to the content root. */ - filePath: string + /** Path to the file relative to the content root. */ + filePath: string - /** The link extension for generated links. */ - linksExtension?: LinksExtension + /** The link extension for generated links. */ + linksExtension?: LinksExtension - /** Whether to use clean URLs (without the extension). */ - cleanUrls?: VitePressConfig['cleanUrls'] + /** Whether to use clean URLs (without the extension). */ + cleanUrls?: VitePressConfig['cleanUrls'] } /** @@ -174,24 +174,24 @@ export interface GenerateMetadataOptions { * // Returns { url: 'https://example.com/docs/guide.md', description: 'A guide' } */ export function generateMetadata>( - sourceFile: GrayMatter, - options: GenerateMetadataOptions, + sourceFile: GrayMatter, + options: GenerateMetadataOptions ) { - const { domain, filePath, linksExtension, cleanUrls } = options - const frontmatterMetadata: Record = {} + const { domain, filePath, linksExtension, cleanUrls } = options + const frontmatterMetadata: Record = {} - frontmatterMetadata.url = generateLink( - stripExtPosix(filePath), - domain, - linksExtension ?? '.md', - cleanUrls, - ) + frontmatterMetadata.url = generateLink( + stripExtPosix(filePath), + domain, + linksExtension ?? '.md', + cleanUrls + ) - if (sourceFile.data?.description?.length) { - frontmatterMetadata.description = sourceFile.data?.description - } + if (sourceFile.data?.description?.length) { + frontmatterMetadata.description = sourceFile.data?.description + } - return frontmatterMetadata + return frontmatterMetadata } /** @@ -204,4 +204,4 @@ export function generateMetadata>( * @returns A human-readable size string (e.g., "1.2 KB", "500 B"). */ export const getHumanReadableSizeOf = (string: string) => - byteSize(new Blob([string]).size).toString() + byteSize(new Blob([string]).size).toString() diff --git a/src/node/plugins/llmstxt/index.ts b/src/node/plugins/llmstxt/index.ts index 0e196b94..a0d5c983 100644 --- a/src/node/plugins/llmstxt/index.ts +++ b/src/node/plugins/llmstxt/index.ts @@ -17,17 +17,17 @@ import { defaultLLMsTxtTemplate } from './constants' import { generateLLMsFullTxt, generateLLMsTxt } from './helpers/index' import log from './helpers/logger' import { - expandTemplate, - extractTitle, - generateMetadata, - getHumanReadableSizeOf, - stripExt, + expandTemplate, + extractTitle, + generateMetadata, + getHumanReadableSizeOf, + stripExt } from './helpers/utils' import type { - CustomTemplateVariables, - LlmstxtSettings, - PreparedFile, - VitePressConfig, + CustomTemplateVariables, + LlmstxtSettings, + PreparedFile, + VitePressConfig } from './types' const PLUGIN_NAME = 'llmstxt' @@ -42,324 +42,324 @@ const PLUGIN_NAME = 'llmstxt' * @see https://llmstxt.org/ */ export default function llmstxt(userSettings: LlmstxtSettings = {}): Plugin { - // Create a settings object with defaults explicitly merged - const settings: Omit & { workDir: string } = { - generateLLMsTxt: true, - generateLLMsFullTxt: true, - generateLLMFriendlyDocsForEachPage: true, - ignoreFiles: [], - workDir: undefined as unknown as string, - stripHTML: true, - ...userSettings, - } - - // Store the resolved Vite config - let config: VitePressConfig - - // Set to store all markdown file paths - const mdFiles: Set = new Set() - - // Flag to identify which build we're in - let isSsrBuild = false - - return { - name: PLUGIN_NAME, - - /** Resolves the Vite configuration and sets up the working directory. */ - configResolved(resolvedConfig) { - config = resolvedConfig as VitePressConfig - if (settings.workDir) { - settings.workDir = path.resolve( - config.vitepress.srcDir, - settings.workDir as string, - ) - } else { - settings.workDir = config.vitepress.srcDir - } - // Detect if this is the SSR build - isSsrBuild = !!resolvedConfig.build?.ssr - log.info( - `${pc.bold(PLUGIN_NAME)} initialized ${isSsrBuild ? pc.dim('(SSR build)') : pc.dim('(client build)')} with workDir: ${pc.cyan(settings.workDir as string)}`, - ) - }, - - /** Configures the development server to handle `llms.txt` and markdown files for LLMs. */ - async configureServer(server: ViteDevServer) { - log.info('Dev server configured for serving plain text docs for LLMs') - server.middlewares.use(async (req, res, next) => { - if (req.url?.endsWith('.md') || req.url?.endsWith('.txt')) { - try { - // Try to read and serve the markdown file - const filePath = path.resolve( - config.vitepress?.outDir ?? 'dist', - `${stripExt(req.url)}.md`, - ) - const content = await fs.readFile(filePath, 'utf-8') - res.setHeader('Content-Type', 'text/plain; charset=utf-8') - res.end(content) - return - } catch (e) { - // If file doesn't exist or can't be read, continue to next middleware - log.warn(`Failed to return ${pc.cyan(req.url)}: File not found`) - next() - } - } - - // Pass to next middleware if not handled - next() - }) - }, - - /** - * Resets the collection of markdown files when the build starts. - * This ensures we don't include stale data from previous builds. - */ - buildStart() { - mdFiles.clear() - log.info('Build started, file collection cleared') - }, - - /** - * Processes each file that Vite transforms and collects markdown files. - * - * @param _ - The file content (not used). - * @param id - The file identifier (path). - * @returns null if the file is processed, otherwise returns the original content. - */ - async transform(_, id: string) { - if (!id.endsWith('.md')) { - return null - } - - // Skip files outside workDir if it's configured - if (!id.startsWith(settings.workDir as string)) { - return null - } - - if (settings.ignoreFiles?.length) { - const shouldIgnore = await Promise.all( - settings.ignoreFiles.map(async (pattern) => { - if (typeof pattern === 'string') { - return await Promise.resolve( - minimatch( - path.relative(settings.workDir as string, id), - pattern, - ), - ) - } - return false - }), - ) - - if (shouldIgnore.some((result) => result === true)) { - return null - } - } - - // Add markdown file path to our collection - mdFiles.add(id) - // Return null to avoid modifying the file - return null - }, - - /** - * Runs only in the client build (not SSR) after completion. - * This ensures the processing happens exactly once. - */ - async generateBundle() { - // Skip processing during SSR build - if (isSsrBuild) { - log.info('Skipping LLMs docs generation in SSR build') - return - } - - const outDir = config.vitepress?.outDir ?? 'dist' - - // Create output directory if it doesn't exist - try { - await fs.access(outDir) - } catch { - log.info(`Creating output directory: ${pc.cyan(outDir)}`) - await fs.mkdir(outDir, { recursive: true }) - } - - const mdFilesList = Array.from(mdFiles) - const fileCount = mdFilesList.length - - // Skip if no files found - if (fileCount === 0) { - log.warn( - `No markdown files found to process. Check your \`${pc.bold('workDir')}\` and \`${pc.bold('ignoreFiles')}\` settings.`, - ) - return - } - - log.info( - `Processing ${pc.bold(fileCount.toString())} markdown files from ${pc.cyan(settings.workDir)}`, - ) - - const preparedFiles: PreparedFile[] = await Promise.all( - mdFilesList.map(async (file) => { - const content = await fs.readFile(file, 'utf-8') - - let mdFile: matter.GrayMatterFile - - if (settings.stripHTML) { - const cleanedMarkdown = await remark() - .use(remarkFrontmatter) - .use(() => { - // Strip HTML tags - return (tree) => { - remove(tree, { type: 'html' }) - return tree - } - }) - .process(content) - - mdFile = matter(String(cleanedMarkdown)) - } else { - mdFile = matter(content) - } - // Extract title from frontmatter or use the first heading - const title = extractTitle(mdFile)?.trim() || 'Untitled' - - const filePath = - path.basename(file) === 'index.md' && - path.dirname(file) !== settings.workDir - ? `${path.dirname(file)}.md` - : file - - return { path: filePath, title, file: mdFile } - }), - ) - - if (settings.generateLLMFriendlyDocsForEachPage) { - await Promise.all( - preparedFiles.map(async (file) => { - const relativePath = path.relative(settings.workDir, file.path) - try { - const mdFile = file.file - const targetPath = path.resolve(outDir, relativePath) - - // Ensure target directory exists (async version) - await fs.mkdir(path.dirname(targetPath), { - recursive: true, - }) - - // Copy file to output directory (async version) - await fs.writeFile( - targetPath, - matter.stringify( - mdFile.content, - generateMetadata(mdFile, { - domain: settings.domain, - filePath: relativePath, - }), - ), - ) - - log.success(`Processed ${pc.cyan(relativePath)}`) - } catch (error) { - log.error( - // @ts-ignore - `Failed to process ${pc.cyan(relativePath)}: ${error.message}`, - ) - } - }), - ) - } - - // Sort files by title for better organization - preparedFiles.sort((a, b) => a.title.localeCompare(b.title)) - - const tasks: Promise[] = [] - - // Generate llms.txt - table of contents with links - if (settings.generateLLMsTxt) { - const llmsTxtPath = path.resolve(outDir, 'llms.txt') - const templateVariables: CustomTemplateVariables = { - title: settings.title, - description: settings.description, - details: settings.details, - toc: settings.toc, - ...settings.customTemplateVariables, - } - - tasks.push( - (async () => { - log.info(`Generating ${pc.cyan('llms.txt')}...`) - - const llmsTxt = await generateLLMsTxt(preparedFiles, { - indexMd: path.resolve(settings.workDir as string, 'index.md'), - srcDir: settings.workDir as string, - LLMsTxtTemplate: - settings.customLLMsTxtTemplate || defaultLLMsTxtTemplate, - templateVariables, - vitepressConfig: config?.vitepress?.userConfig, - domain: settings.domain, - sidebar: settings.sidebar, - linksExtension: !settings.generateLLMFriendlyDocsForEachPage - ? '.html' - : undefined, - cleanUrls: config.cleanUrls, - }) - - await fs.writeFile(llmsTxtPath, llmsTxt, 'utf-8') - - log.success( - expandTemplate( - 'Generated {file} (~{tokens} tokens, {size}) with {fileCount} documentation links', - { - file: pc.cyan('llms.txt'), - tokens: pc.bold(millify(approximateTokenSize(llmsTxt))), - size: pc.bold(getHumanReadableSizeOf(llmsTxt)), - fileCount: pc.bold(fileCount.toString()), - }, - ), - ) - })(), - ) - } - - // Generate llms-full.txt - all content in one file - if (settings.generateLLMsFullTxt) { - const llmsFullTxtPath = path.resolve(outDir, 'llms-full.txt') - - tasks.push( - (async () => { - log.info( - `Generating full documentation bundle (${pc.cyan('llms-full.txt')})...`, - ) - - const llmsFullTxt = generateLLMsFullTxt(preparedFiles, { - srcDir: settings.workDir as string, - domain: settings.domain, - linksExtension: !settings.generateLLMFriendlyDocsForEachPage - ? '.html' - : undefined, - cleanUrls: config.cleanUrls, - }) - - // Write content to llms-full.txt - await fs.writeFile(llmsFullTxtPath, llmsFullTxt, 'utf-8') - log.success( - expandTemplate( - 'Generated {file} (~{tokens} tokens, {size}) with {fileCount} markdown files', - { - file: pc.cyan('llms-full.txt'), - tokens: pc.bold(millify(approximateTokenSize(llmsFullTxt))), - size: pc.bold(getHumanReadableSizeOf(llmsFullTxt)), - fileCount: pc.bold(fileCount.toString()), - }, - ), - ) - })(), - ) - } - - if (tasks.length) { - await Promise.all(tasks) - } - }, - } + // Create a settings object with defaults explicitly merged + const settings: Omit & { workDir: string } = { + generateLLMsTxt: true, + generateLLMsFullTxt: true, + generateLLMFriendlyDocsForEachPage: true, + ignoreFiles: [], + workDir: undefined as unknown as string, + stripHTML: true, + ...userSettings + } + + // Store the resolved Vite config + let config: VitePressConfig + + // Set to store all markdown file paths + const mdFiles: Set = new Set() + + // Flag to identify which build we're in + let isSsrBuild = false + + return { + name: PLUGIN_NAME, + + /** Resolves the Vite configuration and sets up the working directory. */ + configResolved(resolvedConfig) { + config = resolvedConfig as VitePressConfig + if (settings.workDir) { + settings.workDir = path.resolve( + config.vitepress.srcDir, + settings.workDir as string + ) + } else { + settings.workDir = config.vitepress.srcDir + } + // Detect if this is the SSR build + isSsrBuild = !!resolvedConfig.build?.ssr + log.info( + `${pc.bold(PLUGIN_NAME)} initialized ${isSsrBuild ? pc.dim('(SSR build)') : pc.dim('(client build)')} with workDir: ${pc.cyan(settings.workDir as string)}` + ) + }, + + /** Configures the development server to handle `llms.txt` and markdown files for LLMs. */ + async configureServer(server: ViteDevServer) { + log.info('Dev server configured for serving plain text docs for LLMs') + server.middlewares.use(async (req, res, next) => { + if (req.url?.endsWith('.md') || req.url?.endsWith('.txt')) { + try { + // Try to read and serve the markdown file + const filePath = path.resolve( + config.vitepress?.outDir ?? 'dist', + `${stripExt(req.url)}.md` + ) + const content = await fs.readFile(filePath, 'utf-8') + res.setHeader('Content-Type', 'text/plain; charset=utf-8') + res.end(content) + return + } catch (e) { + // If file doesn't exist or can't be read, continue to next middleware + log.warn(`Failed to return ${pc.cyan(req.url)}: File not found`) + next() + } + } + + // Pass to next middleware if not handled + next() + }) + }, + + /** + * Resets the collection of markdown files when the build starts. + * This ensures we don't include stale data from previous builds. + */ + buildStart() { + mdFiles.clear() + log.info('Build started, file collection cleared') + }, + + /** + * Processes each file that Vite transforms and collects markdown files. + * + * @param _ - The file content (not used). + * @param id - The file identifier (path). + * @returns null if the file is processed, otherwise returns the original content. + */ + async transform(_, id: string) { + if (!id.endsWith('.md')) { + return null + } + + // Skip files outside workDir if it's configured + if (!id.startsWith(settings.workDir as string)) { + return null + } + + if (settings.ignoreFiles?.length) { + const shouldIgnore = await Promise.all( + settings.ignoreFiles.map(async (pattern) => { + if (typeof pattern === 'string') { + return await Promise.resolve( + minimatch( + path.relative(settings.workDir as string, id), + pattern + ) + ) + } + return false + }) + ) + + if (shouldIgnore.some((result) => result === true)) { + return null + } + } + + // Add markdown file path to our collection + mdFiles.add(id) + // Return null to avoid modifying the file + return null + }, + + /** + * Runs only in the client build (not SSR) after completion. + * This ensures the processing happens exactly once. + */ + async generateBundle() { + // Skip processing during SSR build + if (isSsrBuild) { + log.info('Skipping LLMs docs generation in SSR build') + return + } + + const outDir = config.vitepress?.outDir ?? 'dist' + + // Create output directory if it doesn't exist + try { + await fs.access(outDir) + } catch { + log.info(`Creating output directory: ${pc.cyan(outDir)}`) + await fs.mkdir(outDir, { recursive: true }) + } + + const mdFilesList = Array.from(mdFiles) + const fileCount = mdFilesList.length + + // Skip if no files found + if (fileCount === 0) { + log.warn( + `No markdown files found to process. Check your \`${pc.bold('workDir')}\` and \`${pc.bold('ignoreFiles')}\` settings.` + ) + return + } + + log.info( + `Processing ${pc.bold(fileCount.toString())} markdown files from ${pc.cyan(settings.workDir)}` + ) + + const preparedFiles: PreparedFile[] = await Promise.all( + mdFilesList.map(async (file) => { + const content = await fs.readFile(file, 'utf-8') + + let mdFile: matter.GrayMatterFile + + if (settings.stripHTML) { + const cleanedMarkdown = await remark() + .use(remarkFrontmatter) + .use(() => { + // Strip HTML tags + return (tree) => { + remove(tree, { type: 'html' }) + return tree + } + }) + .process(content) + + mdFile = matter(String(cleanedMarkdown)) + } else { + mdFile = matter(content) + } + // Extract title from frontmatter or use the first heading + const title = extractTitle(mdFile)?.trim() || 'Untitled' + + const filePath = + path.basename(file) === 'index.md' && + path.dirname(file) !== settings.workDir + ? `${path.dirname(file)}.md` + : file + + return { path: filePath, title, file: mdFile } + }) + ) + + if (settings.generateLLMFriendlyDocsForEachPage) { + await Promise.all( + preparedFiles.map(async (file) => { + const relativePath = path.relative(settings.workDir, file.path) + try { + const mdFile = file.file + const targetPath = path.resolve(outDir, relativePath) + + // Ensure target directory exists (async version) + await fs.mkdir(path.dirname(targetPath), { + recursive: true + }) + + // Copy file to output directory (async version) + await fs.writeFile( + targetPath, + matter.stringify( + mdFile.content, + generateMetadata(mdFile, { + domain: settings.domain, + filePath: relativePath + }) + ) + ) + + log.success(`Processed ${pc.cyan(relativePath)}`) + } catch (error) { + log.error( + // @ts-ignore + `Failed to process ${pc.cyan(relativePath)}: ${error.message}` + ) + } + }) + ) + } + + // Sort files by title for better organization + preparedFiles.sort((a, b) => a.title.localeCompare(b.title)) + + const tasks: Promise[] = [] + + // Generate llms.txt - table of contents with links + if (settings.generateLLMsTxt) { + const llmsTxtPath = path.resolve(outDir, 'llms.txt') + const templateVariables: CustomTemplateVariables = { + title: settings.title, + description: settings.description, + details: settings.details, + toc: settings.toc, + ...settings.customTemplateVariables + } + + tasks.push( + (async () => { + log.info(`Generating ${pc.cyan('llms.txt')}...`) + + const llmsTxt = await generateLLMsTxt(preparedFiles, { + indexMd: path.resolve(settings.workDir as string, 'index.md'), + srcDir: settings.workDir as string, + LLMsTxtTemplate: + settings.customLLMsTxtTemplate || defaultLLMsTxtTemplate, + templateVariables, + vitepressConfig: config?.vitepress?.userConfig, + domain: settings.domain, + sidebar: settings.sidebar, + linksExtension: !settings.generateLLMFriendlyDocsForEachPage + ? '.html' + : undefined, + cleanUrls: config.cleanUrls + }) + + await fs.writeFile(llmsTxtPath, llmsTxt, 'utf-8') + + log.success( + expandTemplate( + 'Generated {file} (~{tokens} tokens, {size}) with {fileCount} documentation links', + { + file: pc.cyan('llms.txt'), + tokens: pc.bold(millify(approximateTokenSize(llmsTxt))), + size: pc.bold(getHumanReadableSizeOf(llmsTxt)), + fileCount: pc.bold(fileCount.toString()) + } + ) + ) + })() + ) + } + + // Generate llms-full.txt - all content in one file + if (settings.generateLLMsFullTxt) { + const llmsFullTxtPath = path.resolve(outDir, 'llms-full.txt') + + tasks.push( + (async () => { + log.info( + `Generating full documentation bundle (${pc.cyan('llms-full.txt')})...` + ) + + const llmsFullTxt = generateLLMsFullTxt(preparedFiles, { + srcDir: settings.workDir as string, + domain: settings.domain, + linksExtension: !settings.generateLLMFriendlyDocsForEachPage + ? '.html' + : undefined, + cleanUrls: config.cleanUrls + }) + + // Write content to llms-full.txt + await fs.writeFile(llmsFullTxtPath, llmsFullTxt, 'utf-8') + log.success( + expandTemplate( + 'Generated {file} (~{tokens} tokens, {size}) with {fileCount} markdown files', + { + file: pc.cyan('llms-full.txt'), + tokens: pc.bold(millify(approximateTokenSize(llmsFullTxt))), + size: pc.bold(getHumanReadableSizeOf(llmsFullTxt)), + fileCount: pc.bold(fileCount.toString()) + } + ) + ) + })() + ) + } + + if (tasks.length) { + await Promise.all(tasks) + } + } + } } diff --git a/src/node/plugins/llmstxt/types.d.ts b/src/node/plugins/llmstxt/types.d.ts index b307d4fc..bc809c0d 100644 --- a/src/node/plugins/llmstxt/types.d.ts +++ b/src/node/plugins/llmstxt/types.d.ts @@ -3,240 +3,240 @@ import type { ResolvedConfig } from 'vite' import type { DefaultTheme, SiteConfig, UserConfig } from 'vitepress' interface TemplateVariables { - /** - * The title extracted from the frontmatter or the first h1 heading in the main document (`index.md`). - * - * @example 'Awesome tool' - */ - title?: string - - /** - * The description. - * - * @example 'Blazing fast build tool' - */ - description?: string - - /** - * The details. - * - * @example 'A multi-user version of the notebook designed for companies, classrooms and research labs' - */ - details?: string - - /** - * An automatically generated **T**able **O**f **C**ontents. - * - * @example - * ```markdown - * - [Title](/foo.md): Lorem ipsum dolor sit amet, consectetur adipiscing elit. - * - [Title 2](/bar/baz.md): Cras vel nibh id ipsum pharetra efficitur. - * ``` - */ - toc?: string + /** + * The title extracted from the frontmatter or the first h1 heading in the main document (`index.md`). + * + * @example 'Awesome tool' + */ + title?: string + + /** + * The description. + * + * @example 'Blazing fast build tool' + */ + description?: string + + /** + * The details. + * + * @example 'A multi-user version of the notebook designed for companies, classrooms and research labs' + */ + details?: string + + /** + * An automatically generated **T**able **O**f **C**ontents. + * + * @example + * ```markdown + * - [Title](/foo.md): Lorem ipsum dolor sit amet, consectetur adipiscing elit. + * - [Title 2](/bar/baz.md): Cras vel nibh id ipsum pharetra efficitur. + * ``` + */ + toc?: string } interface CustomTemplateVariables extends TemplateVariables { - /** Any custom variable */ - [key: string]: string | undefined + /** Any custom variable */ + [key: string]: string | undefined } export interface LlmstxtSettings extends TemplateVariables { - /** - * The domain that will be appended to the beginning of URLs in `llms.txt` and in the context of other files - * - * Domain attachment is not yet agreed upon (since it depends on the AI ​​whether it can resolve the relative paths that are currently there), but if you want you can add it - * - * ℹ️ **Note**: Domain cannot end with `/`. - * - * Without a {@link LlmstxtSettings.domain | `domain`}: - * ```markdown - * - [Title](/foo/bar.md) - * ``` - * - * With a {@link LlmstxtSettings.domain | `domain`}: - * ```markdown - * - [Title](https://example.com/foo/bar.md) - * ``` - * - * @example - * ```typescript - * llmstxt({ domain: 'https://example.com' }) - * ``` - */ - domain?: string - - /** - * Indicates whether to generate the `llms.txt` file, which contains a list of sections with corresponding links. - * - * @default true - */ - generateLLMsTxt?: boolean - - /** - * Determines whether to generate the `llms-full.txt` which contains all the documentation in one file. - * - * @default true - */ - generateLLMsFullTxt?: boolean - - /** - * Determines whether to generate an LLM-friendly version of the documentation for each page on the website. - * - * @default true - */ - generateLLMFriendlyDocsForEachPage?: boolean - - /** - * Whether to strip HTML tags from Markdown files - * - * @default true - */ - stripHTML?: boolean - - /** - * The directory from which files will be processed. - * - * This is useful for configuring the plugin to generate documentation for LLMs in a specific language. - * - * @example - * ```typescript - * llmstxt({ - * // Generate documentation for LLMs from English documentation only - * workDir: 'en' - * }) - * ``` - * - * @default vitepress.srcDir - */ - workDir?: string - - /** - * An array of file path patterns to be ignored during processing. - * - * This is useful for excluding certain files from LLMs, such as those not related to documentation (e.g., sponsors, team, etc.). - * - * @example - * ```typescript - * llmstxt({ - * ignoreFiles: [ - * 'about/team/*', - * 'sponsor/*' - * // ... - * ] - * }) - * ``` - * - * @default [] - */ - ignoreFiles?: string[] - - /** - * A custom template for the `llms.txt` file, allowing for a personalized order of elements. - * - * Available template elements include: - * - * - `{title}`: The title extracted from the frontmatter or the first h1 heading in the main document (`index.md`). - * - `{description}`: The description. - * - `{details}`: The details. - * - `{toc}`: An automatically generated **T**able **O**f **C**ontents. - * - * You can also add custom variables using the {@link LlmstxtSettings.customTemplateVariables | `customTemplateVariables`} parameter - * - * @default - * ```markdown - * # {title} - * - * > {description} - * - * {details} - * - * ## Table of Contents - * - * {toc} - * ``` - */ - customLLMsTxtTemplate?: string - - /** - * Custom variables for {@link LlmstxtSettings.customLLMsTxtTemplate | `customLLMsTxtTemplate`}. - * - * With this option you can edit or add variables to the template. - * - * You can change the title in `llms.txt` without having to change the template: - * - * @example - * ```typescript - * llmstxt({ - * customTemplateVariables: { - * title: 'Very custom title', - * } - * }) - * ``` - * - * You can also combine this with a custom template: - * - * @example - * ```typescript - * llmstxt({ - * customLLMsTxtTemplate: '# {title}\n\n{foo}', - * customTemplateVariables: { - * foo: 'Very custom title', - * } - * }) - * ``` - */ - customTemplateVariables?: CustomTemplateVariables - - /** - * VitePress {@link DefaultTheme.Sidebar | Sidebar} - * - * Here you can insert your {@link DefaultTheme.Sidebar | `sidebar`} if it is not in the VitePress configuration - * - * Usually this parameter is used in rare cases - */ - sidebar?: DefaultTheme.Sidebar + /** + * The domain that will be appended to the beginning of URLs in `llms.txt` and in the context of other files + * + * Domain attachment is not yet agreed upon (since it depends on the AI ​​whether it can resolve the relative paths that are currently there), but if you want you can add it + * + * ℹ️ **Note**: Domain cannot end with `/`. + * + * Without a {@link LlmstxtSettings.domain | `domain`}: + * ```markdown + * - [Title](/foo/bar.md) + * ``` + * + * With a {@link LlmstxtSettings.domain | `domain`}: + * ```markdown + * - [Title](https://example.com/foo/bar.md) + * ``` + * + * @example + * ```typescript + * llmstxt({ domain: 'https://example.com' }) + * ``` + */ + domain?: string + + /** + * Indicates whether to generate the `llms.txt` file, which contains a list of sections with corresponding links. + * + * @default true + */ + generateLLMsTxt?: boolean + + /** + * Determines whether to generate the `llms-full.txt` which contains all the documentation in one file. + * + * @default true + */ + generateLLMsFullTxt?: boolean + + /** + * Determines whether to generate an LLM-friendly version of the documentation for each page on the website. + * + * @default true + */ + generateLLMFriendlyDocsForEachPage?: boolean + + /** + * Whether to strip HTML tags from Markdown files + * + * @default true + */ + stripHTML?: boolean + + /** + * The directory from which files will be processed. + * + * This is useful for configuring the plugin to generate documentation for LLMs in a specific language. + * + * @example + * ```typescript + * llmstxt({ + * // Generate documentation for LLMs from English documentation only + * workDir: 'en' + * }) + * ``` + * + * @default vitepress.srcDir + */ + workDir?: string + + /** + * An array of file path patterns to be ignored during processing. + * + * This is useful for excluding certain files from LLMs, such as those not related to documentation (e.g., sponsors, team, etc.). + * + * @example + * ```typescript + * llmstxt({ + * ignoreFiles: [ + * 'about/team/*', + * 'sponsor/*' + * // ... + * ] + * }) + * ``` + * + * @default [] + */ + ignoreFiles?: string[] + + /** + * A custom template for the `llms.txt` file, allowing for a personalized order of elements. + * + * Available template elements include: + * + * - `{title}`: The title extracted from the frontmatter or the first h1 heading in the main document (`index.md`). + * - `{description}`: The description. + * - `{details}`: The details. + * - `{toc}`: An automatically generated **T**able **O**f **C**ontents. + * + * You can also add custom variables using the {@link LlmstxtSettings.customTemplateVariables | `customTemplateVariables`} parameter + * + * @default + * ```markdown + * # {title} + * + * > {description} + * + * {details} + * + * ## Table of Contents + * + * {toc} + * ``` + */ + customLLMsTxtTemplate?: string + + /** + * Custom variables for {@link LlmstxtSettings.customLLMsTxtTemplate | `customLLMsTxtTemplate`}. + * + * With this option you can edit or add variables to the template. + * + * You can change the title in `llms.txt` without having to change the template: + * + * @example + * ```typescript + * llmstxt({ + * customTemplateVariables: { + * title: 'Very custom title', + * } + * }) + * ``` + * + * You can also combine this with a custom template: + * + * @example + * ```typescript + * llmstxt({ + * customLLMsTxtTemplate: '# {title}\n\n{foo}', + * customTemplateVariables: { + * foo: 'Very custom title', + * } + * }) + * ``` + */ + customTemplateVariables?: CustomTemplateVariables + + /** + * VitePress {@link DefaultTheme.Sidebar | Sidebar} + * + * Here you can insert your {@link DefaultTheme.Sidebar | `sidebar`} if it is not in the VitePress configuration + * + * Usually this parameter is used in rare cases + */ + sidebar?: DefaultTheme.Sidebar } /** * Represents a prepared file, including its title and path. */ export type PreparedFile = { - /** - * The title of the file. - * - * @example 'Guide' - */ - title: string - - /** - * The absolute path to the file. - * - * @example 'guide/getting-started.md' - */ - path: string - - /** - * The prepared file itself. - * - * @example - * ```typescript - * { - * data: { - * title: 'Guide' - * }, - * content: 'Content goes here' - * orig: '---\ntitle: Guide\n---\n\nContent goes here' - * } - * ``` - */ - file: GrayMatterFile + /** + * The title of the file. + * + * @example 'Guide' + */ + title: string + + /** + * The absolute path to the file. + * + * @example 'guide/getting-started.md' + */ + path: string + + /** + * The prepared file itself. + * + * @example + * ```typescript + * { + * data: { + * title: 'Guide' + * }, + * content: 'Content goes here' + * orig: '---\ntitle: Guide\n---\n\nContent goes here' + * } + * ``` + */ + file: GrayMatterFile } interface VitePressConfig - extends Omit, - ResolvedConfig { - vitepress: SiteConfig + extends Omit, + ResolvedConfig { + vitepress: SiteConfig } /** Represents the link extension options for generated links. */