mirror of https://github.com/vuejs/vitepress
Okinea Dev
5 months ago
parent
1ec84c1504
commit
673db7d4b6
@ -0,0 +1,11 @@
|
||||
/** Default template for the `llms.txt` file. */
|
||||
export const defaultLLMsTxtTemplate = `\
|
||||
# {title}
|
||||
|
||||
{description}
|
||||
|
||||
{details}
|
||||
|
||||
## Table of Contents
|
||||
|
||||
{toc}`
|
||||
@ -0,0 +1,175 @@
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
import matter from 'gray-matter'
|
||||
|
||||
import type { DefaultTheme } from 'vitepress'
|
||||
import { defaultLLMsTxtTemplate } from '../constants'
|
||||
import type {
|
||||
LinksExtension,
|
||||
LlmstxtSettings,
|
||||
PreparedFile,
|
||||
VitePressConfig,
|
||||
} from '../types'
|
||||
import { generateTOC } from './toc'
|
||||
import { expandTemplate, extractTitle, generateMetadata } from './utils'
|
||||
|
||||
/**
|
||||
* Options for generating the `llms.txt` file.
|
||||
*/
|
||||
export interface GenerateLLMsTxtOptions {
|
||||
/** Path to the main documentation file `index.md`.*/
|
||||
indexMd: string
|
||||
|
||||
/** The source directory for the files. */
|
||||
srcDir: VitePressConfig['vitepress']['srcDir']
|
||||
|
||||
/** Template to use for generating `llms.txt`. */
|
||||
LLMsTxtTemplate?: LlmstxtSettings['customLLMsTxtTemplate']
|
||||
|
||||
/** Template variables for `customLLMsTxtTemplate`. */
|
||||
templateVariables?: LlmstxtSettings['customTemplateVariables']
|
||||
|
||||
/** The VitePress configuration. */
|
||||
vitepressConfig?: VitePressConfig['vitepress']['userConfig']
|
||||
|
||||
/** The base domain for the generated links. */
|
||||
domain?: LlmstxtSettings['domain']
|
||||
|
||||
/** The link extension for generated links. */
|
||||
linksExtension?: LinksExtension
|
||||
|
||||
/** Whether to use clean URLs (without the extension). */
|
||||
cleanUrls?: VitePressConfig['cleanUrls']
|
||||
|
||||
/** Optional sidebar configuration for organizing the TOC. */
|
||||
sidebar?: DefaultTheme.Sidebar
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a LLMs.txt file with a table of contents and links to all documentation sections.
|
||||
*
|
||||
* @param preparedFiles - An array of prepared files.
|
||||
* @param options - Options for generating the `llms.txt` file.
|
||||
* @returns A string representing the content of the `llms.txt` file.
|
||||
*
|
||||
* @example
|
||||
* ```markdown
|
||||
* # Shadcn for Vue
|
||||
*
|
||||
* > Beautifully designed components built with Radix Vue and Tailwind CSS.
|
||||
*
|
||||
* ## Table of Contents
|
||||
*
|
||||
* - [Getting started](/docs/getting-started.md)
|
||||
* - [About](/docs/about.md)
|
||||
* - ...
|
||||
* ```
|
||||
*
|
||||
* @see https://llmstxt.org/#format
|
||||
*/
|
||||
export async function generateLLMsTxt(
|
||||
preparedFiles: PreparedFile[],
|
||||
options: GenerateLLMsTxtOptions,
|
||||
): Promise<string> {
|
||||
const {
|
||||
indexMd,
|
||||
srcDir,
|
||||
LLMsTxtTemplate = defaultLLMsTxtTemplate,
|
||||
templateVariables = {},
|
||||
vitepressConfig,
|
||||
domain,
|
||||
sidebar,
|
||||
linksExtension,
|
||||
cleanUrls,
|
||||
} = options
|
||||
|
||||
// @ts-expect-error
|
||||
matter.clearCache()
|
||||
|
||||
const indexMdContent = await fs.readFile(indexMd, 'utf-8')
|
||||
const indexMdFile = matter(indexMdContent as string)
|
||||
|
||||
templateVariables.title ??=
|
||||
indexMdFile.data?.hero?.name ||
|
||||
indexMdFile.data?.title ||
|
||||
vitepressConfig?.title ||
|
||||
vitepressConfig?.titleTemplate ||
|
||||
extractTitle(indexMdFile) ||
|
||||
'LLMs Documentation'
|
||||
|
||||
templateVariables.description ??=
|
||||
indexMdFile.data?.hero?.text ||
|
||||
vitepressConfig?.description ||
|
||||
indexMdFile?.data?.description ||
|
||||
indexMdFile.data?.titleTemplate
|
||||
|
||||
if (templateVariables.description) {
|
||||
templateVariables.description = `> ${templateVariables.description}`
|
||||
}
|
||||
|
||||
templateVariables.details ??=
|
||||
indexMdFile.data?.hero?.tagline ||
|
||||
indexMdFile.data?.tagline ||
|
||||
(!templateVariables.description &&
|
||||
'This file contains links to all documentation sections.')
|
||||
|
||||
templateVariables.toc ??= await generateTOC(preparedFiles, {
|
||||
srcDir,
|
||||
domain,
|
||||
sidebarConfig: sidebar || vitepressConfig?.themeConfig?.sidebar,
|
||||
linksExtension,
|
||||
cleanUrls,
|
||||
})
|
||||
|
||||
return expandTemplate(LLMsTxtTemplate, templateVariables)
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for generating the `llms-full.txt` file.
|
||||
*/
|
||||
export interface GenerateLLMsFullTxtOptions {
|
||||
/** The source directory for the files. */
|
||||
srcDir: VitePressConfig['vitepress']['srcDir']
|
||||
|
||||
/** The base domain for the generated links. */
|
||||
domain?: LlmstxtSettings['domain']
|
||||
|
||||
/** The link extension for generated links. */
|
||||
linksExtension?: LinksExtension
|
||||
|
||||
/** Whether to use clean URLs (without the extension). */
|
||||
cleanUrls?: VitePressConfig['cleanUrls']
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a `llms-full.txt` file content with all documentation in one file.
|
||||
*
|
||||
* @param preparedFiles - An array of prepared files.
|
||||
* @param options - Options for generating the `llms-full.txt` file.
|
||||
* @returns A string representing the full content of the LLMs.txt file.
|
||||
*/
|
||||
export function generateLLMsFullTxt(
|
||||
preparedFiles: PreparedFile[],
|
||||
options: GenerateLLMsFullTxtOptions,
|
||||
) {
|
||||
const { srcDir, domain, linksExtension, cleanUrls } = options
|
||||
|
||||
const llmsFullTxtContent = preparedFiles
|
||||
.map((preparedFile) => {
|
||||
const relativePath = path.relative(srcDir, preparedFile.path)
|
||||
|
||||
return matter.stringify(
|
||||
preparedFile.file.content,
|
||||
generateMetadata(preparedFile.file, {
|
||||
domain,
|
||||
filePath: relativePath,
|
||||
linksExtension,
|
||||
cleanUrls,
|
||||
}),
|
||||
)
|
||||
})
|
||||
.join('\n---\n\n')
|
||||
|
||||
return llmsFullTxtContent
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
import pc from 'picocolors'
|
||||
|
||||
/**
|
||||
* Log prefix styling with the plugin name and separator.
|
||||
* @constant {string}
|
||||
*/
|
||||
const logPrefix = pc.blue('llmstxt') + pc.dim(' » ')
|
||||
|
||||
/** Logger object with standardized logging methods. */
|
||||
const log = {
|
||||
/**
|
||||
* Logs an informational message to the console.
|
||||
*
|
||||
* @param message - The message to log.
|
||||
*/
|
||||
info: (message: string) => console.log(`${logPrefix} ${message}`),
|
||||
|
||||
/**
|
||||
* Logs a success message to the console.
|
||||
*
|
||||
* @param message - The message to log.
|
||||
*/
|
||||
success: (message: string) =>
|
||||
console.log(`${logPrefix}${pc.green('✓')} ${message}`),
|
||||
|
||||
/**
|
||||
* Logs a warning message to the console.
|
||||
*
|
||||
* @param message - The message to log.
|
||||
*/
|
||||
warn: (message: string) =>
|
||||
console.warn(`${logPrefix}${pc.yellow('⚠')} ${pc.yellow(message)}`),
|
||||
|
||||
/**
|
||||
* Logs an error message to the console.
|
||||
*
|
||||
* @param message - The message to log.
|
||||
*/
|
||||
error: (message: string) =>
|
||||
console.error(`${logPrefix}${pc.red('✗')} ${pc.red(message)}`),
|
||||
}
|
||||
|
||||
export default log
|
||||
@ -0,0 +1,309 @@
|
||||
import path from 'node:path'
|
||||
import type { DefaultTheme } from 'vitepress'
|
||||
import type {
|
||||
LinksExtension,
|
||||
LlmstxtSettings,
|
||||
PreparedFile,
|
||||
VitePressConfig,
|
||||
} from '../types'
|
||||
import { generateLink, stripExtPosix } from './utils'
|
||||
|
||||
/**
|
||||
* Generates a Markdown-formatted table of contents (TOC) link for a given file.
|
||||
*
|
||||
* @param file - The prepared file.
|
||||
* @param domain - The base domain for the generated link.
|
||||
* @param relativePath - The relative path of the file, which is converted to a `.md` link.
|
||||
* @param extension - The link extension for the generated link (default is `.md`).
|
||||
* @param cleanUrls - Whether to use clean URLs (without the extension).
|
||||
* @returns The formatted TOC entry as a Markdown list item.
|
||||
*/
|
||||
export const generateTOCLink = (
|
||||
file: PreparedFile,
|
||||
domain: LlmstxtSettings['domain'],
|
||||
relativePath: string,
|
||||
extension?: LinksExtension,
|
||||
cleanUrls: VitePressConfig['cleanUrls'] = false,
|
||||
) => {
|
||||
const description: string = file.file.data.description
|
||||
return `- [${file.title}](${generateLink(stripExtPosix(relativePath), domain, extension ?? '.md', cleanUrls)})${description ? `: ${description.trim()}` : ''}\n`
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively collects all paths from sidebar items.
|
||||
*
|
||||
* @param items - Array of sidebar items to process.
|
||||
* @returns Array of paths collected from the sidebar items.
|
||||
*/
|
||||
function collectPathsFromSidebarItems(
|
||||
items: DefaultTheme.SidebarItem[],
|
||||
): string[] {
|
||||
const paths: string[] = []
|
||||
|
||||
for (const item of items) {
|
||||
// Add the current item's path if it exists
|
||||
if (item.link) {
|
||||
paths.push(item.link)
|
||||
}
|
||||
|
||||
// Recursively add paths from nested items
|
||||
if (item.items && Array.isArray(item.items)) {
|
||||
paths.push(...collectPathsFromSidebarItems(item.items))
|
||||
}
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes link path for comparison, handling both index.md and directory paths.
|
||||
*
|
||||
* @param link - The link path to normalize.
|
||||
* @returns Normalized link path for consistent comparison.
|
||||
*/
|
||||
export function normalizeLinkPath(link: string): string {
|
||||
const normalizedPath = stripExtPosix(link)
|
||||
|
||||
if (path.basename(normalizedPath) === 'index') {
|
||||
return path.dirname(normalizedPath)
|
||||
}
|
||||
|
||||
return normalizedPath
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a file path matches a sidebar path, handling various path formats.
|
||||
*
|
||||
* @param filePath - The file path to check.
|
||||
* @param sidebarPath - The sidebar path to compare against.
|
||||
* @returns True if paths match, false otherwise
|
||||
*/
|
||||
export function isPathMatch(filePath: string, sidebarPath: string): boolean {
|
||||
const normalizedFilePath = normalizeLinkPath(filePath)
|
||||
const normalizedSidebarPath = normalizeLinkPath(sidebarPath)
|
||||
|
||||
return (
|
||||
normalizedFilePath === normalizedSidebarPath ||
|
||||
normalizedFilePath === `${normalizedSidebarPath}.md`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes sidebar items and generates TOC entries in the exact order they appear in sidebar config
|
||||
*
|
||||
* @param section - A sidebar section
|
||||
* @param preparedFiles - An array of prepared files
|
||||
* @param srcDir - The VitePress source directory
|
||||
* @param domain - Optional domain to prefix URLs with
|
||||
* @param linksExtension - The link extension for generated links.
|
||||
* @param depth - Current depth level for headings
|
||||
* @returns A string representing the formatted section of the TOC
|
||||
*/
|
||||
async function processSidebarSection(
|
||||
section: DefaultTheme.SidebarItem,
|
||||
preparedFiles: PreparedFile[],
|
||||
srcDir: VitePressConfig['vitepress']['srcDir'],
|
||||
domain?: LlmstxtSettings['domain'],
|
||||
linksExtension?: LinksExtension,
|
||||
cleanUrls?: VitePressConfig['cleanUrls'],
|
||||
depth = 3,
|
||||
): Promise<string> {
|
||||
let sectionTOC = ''
|
||||
|
||||
// Add section header only if it has text and is not just a link container
|
||||
if (section.text) {
|
||||
sectionTOC += `${'#'.repeat(depth)} ${section.text}\n\n`
|
||||
}
|
||||
|
||||
// Process items in this section
|
||||
if (section.items && Array.isArray(section.items)) {
|
||||
const linkItems: string[] = []
|
||||
const nestedSections: string[] = []
|
||||
|
||||
// First pass: separate link items and nested sections
|
||||
await Promise.all(
|
||||
section.items.map(async (item) => {
|
||||
// Process nested sections
|
||||
if (item.items && item.items.length > 0) {
|
||||
const processedSection = await processSidebarSection(
|
||||
item,
|
||||
preparedFiles,
|
||||
srcDir,
|
||||
domain,
|
||||
linksExtension,
|
||||
cleanUrls,
|
||||
// Increase depth for nested sections to maintain proper heading levels
|
||||
depth + 1,
|
||||
)
|
||||
nestedSections.push(processedSection)
|
||||
}
|
||||
// Process link items
|
||||
else if (item.link) {
|
||||
// Normalize the link for matching
|
||||
const normalizedItemLink = normalizeLinkPath(item.link)
|
||||
|
||||
const matchingFile = preparedFiles.find((file) => {
|
||||
const relativePath = `/${stripExtPosix(path.relative(srcDir, file.path))}`
|
||||
return isPathMatch(relativePath, normalizedItemLink)
|
||||
})
|
||||
|
||||
if (matchingFile) {
|
||||
const relativePath = path.relative(srcDir, matchingFile.path)
|
||||
linkItems.push(
|
||||
generateTOCLink(
|
||||
matchingFile,
|
||||
domain,
|
||||
relativePath,
|
||||
linksExtension,
|
||||
cleanUrls,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
// Add link items if any
|
||||
if (linkItems.length > 0) {
|
||||
sectionTOC += linkItems.join('')
|
||||
}
|
||||
|
||||
// Add a blank line before nested sections if we have link items
|
||||
if (linkItems.length > 0 && nestedSections.length > 0) {
|
||||
sectionTOC += '\n'
|
||||
}
|
||||
|
||||
// Add nested sections with appropriate spacing
|
||||
if (nestedSections.length > 0) {
|
||||
sectionTOC += nestedSections.join('\n')
|
||||
}
|
||||
}
|
||||
|
||||
return sectionTOC
|
||||
}
|
||||
|
||||
/**
|
||||
* Flattens the sidebar configuration when it's an object with path keys.
|
||||
*
|
||||
* @param sidebarConfig - The sidebar configuration from VitePress.
|
||||
* @returns An array of sidebar items.
|
||||
*/
|
||||
function flattenSidebarConfig(
|
||||
sidebarConfig: DefaultTheme.Sidebar,
|
||||
): DefaultTheme.SidebarItem[] {
|
||||
// If it's already an array, return as is
|
||||
if (Array.isArray(sidebarConfig)) {
|
||||
return sidebarConfig
|
||||
}
|
||||
|
||||
// If it's an object with path keys, flatten it
|
||||
if (typeof sidebarConfig === 'object') {
|
||||
return Object.values(sidebarConfig).flat()
|
||||
}
|
||||
|
||||
// If it's neither, return an empty array
|
||||
return []
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for generating a Table of Contents (TOC).
|
||||
*/
|
||||
export interface GenerateTOCOptions {
|
||||
/**
|
||||
* The VitePress source directory.
|
||||
*/
|
||||
srcDir: VitePressConfig['vitepress']['srcDir']
|
||||
|
||||
/**
|
||||
* Optional domain to prefix URLs with.
|
||||
*/
|
||||
domain?: LlmstxtSettings['domain']
|
||||
|
||||
/**
|
||||
* Optional VitePress sidebar configuration.
|
||||
*/
|
||||
sidebarConfig?: DefaultTheme.Sidebar
|
||||
|
||||
/** The link extension for generated links. */
|
||||
linksExtension?: LinksExtension
|
||||
|
||||
/** Whether to use clean URLs (without the extension). */
|
||||
cleanUrls?: VitePressConfig['cleanUrls']
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a Table of Contents (TOC) for the provided prepared files.
|
||||
*
|
||||
* Each entry in the TOC is formatted as a markdown link to the corresponding
|
||||
* text file. If a VitePress sidebar configuration is provided, the TOC will be
|
||||
* organized into sections based on the sidebar structure, with heading levels (#, ##, ###)
|
||||
* reflecting the nesting depth of the sections.
|
||||
*
|
||||
* @param preparedFiles - An array of prepared files.
|
||||
* @param options - Options for generating the TOC.
|
||||
* @returns A string representing the formatted Table of Contents.
|
||||
*/
|
||||
export async function generateTOC(
|
||||
preparedFiles: PreparedFile[],
|
||||
options: GenerateTOCOptions,
|
||||
): Promise<string> {
|
||||
const { srcDir, domain, sidebarConfig, linksExtension, cleanUrls } = options
|
||||
let tableOfContent = ''
|
||||
|
||||
let filesToProcess = preparedFiles
|
||||
|
||||
// If sidebar configuration exists
|
||||
if (sidebarConfig) {
|
||||
// Flatten sidebar config if it's an object with path keys
|
||||
const flattenedSidebarConfig = flattenSidebarConfig(sidebarConfig)
|
||||
|
||||
// Process each top-level section in the flattened sidebar
|
||||
if (flattenedSidebarConfig.length > 0) {
|
||||
for (const section of flattenedSidebarConfig) {
|
||||
tableOfContent += await processSidebarSection(
|
||||
section,
|
||||
filesToProcess,
|
||||
srcDir,
|
||||
domain,
|
||||
linksExtension,
|
||||
cleanUrls,
|
||||
)
|
||||
|
||||
// tableOfContent = `${tableOfContent.trimEnd()}\n\n`
|
||||
tableOfContent += '\n'
|
||||
}
|
||||
|
||||
// Find files that didn't match any section
|
||||
const allSidebarPaths = collectPathsFromSidebarItems(
|
||||
flattenedSidebarConfig,
|
||||
)
|
||||
const unsortedFiles = filesToProcess.filter((file) => {
|
||||
const relativePath = `/${stripExtPosix(path.relative(srcDir, file.path))}`
|
||||
return !allSidebarPaths.some((sidebarPath) =>
|
||||
isPathMatch(relativePath, sidebarPath),
|
||||
)
|
||||
})
|
||||
|
||||
// Add files that didn't match any section
|
||||
if (unsortedFiles.length > 0) {
|
||||
tableOfContent += '### Other\n\n'
|
||||
filesToProcess = unsortedFiles
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const tocEntries: string[] = []
|
||||
|
||||
await Promise.all(
|
||||
filesToProcess.map(async (file) => {
|
||||
const relativePath = path.relative(srcDir, file.path)
|
||||
tocEntries.push(
|
||||
generateTOCLink(file, domain, relativePath, linksExtension, cleanUrls),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
tableOfContent += tocEntries.join('')
|
||||
|
||||
return tableOfContent
|
||||
}
|
||||
@ -0,0 +1,207 @@
|
||||
import path from 'node:path'
|
||||
import byteSize from 'byte-size'
|
||||
import type { GrayMatterFile, Input } from 'gray-matter'
|
||||
// @ts-expect-error
|
||||
import markdownTitle from 'markdown-title'
|
||||
import type { LinksExtension, LlmstxtSettings, VitePressConfig } from '../types'
|
||||
|
||||
/**
|
||||
* Splits a file path into its directory and file components.
|
||||
*
|
||||
* @param filepath - The path to the file.
|
||||
* @returns An object containing the directory and file name.
|
||||
*/
|
||||
export const splitDirAndFile = (filepath: string) => ({
|
||||
dir: path.dirname(filepath),
|
||||
file: path.basename(filepath),
|
||||
})
|
||||
|
||||
/**
|
||||
* Strips the file extension from a given file path.
|
||||
*
|
||||
* @param filepath - The path to the file.
|
||||
* @returns The filename without the extension.
|
||||
*/
|
||||
export const stripExt = (filepath: string) => {
|
||||
const { dir, file } = splitDirAndFile(filepath)
|
||||
|
||||
return path.join(dir, path.basename(file, path.extname(file)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips the file extension from a given file path using POSIX format.
|
||||
*
|
||||
* @param filepath - The path to the file.
|
||||
* @returns The filename without the extension in POSIX format.
|
||||
*/
|
||||
export const stripExtPosix = (filepath: string) => {
|
||||
const { dir, file } = splitDirAndFile(filepath)
|
||||
|
||||
return path.posix.join(dir, path.basename(file, path.extname(file)))
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the title from a markdown file's frontmatter or first heading.
|
||||
*
|
||||
* @param file - The markdown file to extract the title from.
|
||||
* @returns The extracted title, or `undefined` if no title is found.
|
||||
*/
|
||||
export function extractTitle(file: GrayMatterFile<Input>): string {
|
||||
const titleFromFrontmatter = file.data?.title || file.data?.titleTemplate
|
||||
let titleFromMarkdown: string | undefined
|
||||
|
||||
if (!titleFromFrontmatter) {
|
||||
titleFromMarkdown = markdownTitle(file.content)
|
||||
}
|
||||
return titleFromFrontmatter || titleFromMarkdown
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a regular expression to match a specific template variable in the format `{key}`.
|
||||
*
|
||||
* @param key - The name of the template variable to match.
|
||||
* @returns A case-insensitive regular expression that detects `{key}` occurrences in a string.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const regex = templateVariable('name');
|
||||
* console.log(regex.test('Hello {name}')); // true
|
||||
* ```
|
||||
*/
|
||||
const templateVariable = (key: string) =>
|
||||
new RegExp(`(\\n\\s*\\n)?\\{${key}\\}`, 'gi')
|
||||
|
||||
/**
|
||||
* Replaces occurrences of a template variable `{variable}` in a given content string with a provided value.
|
||||
* If the value is empty or undefined, it falls back to a specified fallback value.
|
||||
*
|
||||
* @param content - The template string containing placeholders.
|
||||
* @param variable - The template variable name to replace.
|
||||
* @param value - The value to replace the variable with.
|
||||
* @param fallback - An optional fallback value if `value` is empty.
|
||||
* @returns A new string with the template variable replaced.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const template = 'Hello {name}!';
|
||||
* const result = replaceTemplateVariable(template, 'name', 'Alice', 'User');
|
||||
* console.log(result); // 'Hello Alice!'
|
||||
* ```
|
||||
*/
|
||||
export function replaceTemplateVariable(
|
||||
content: string,
|
||||
variable: string,
|
||||
value: string | undefined,
|
||||
fallback?: string,
|
||||
) {
|
||||
return content.replace(templateVariable(variable), (_, prefix) => {
|
||||
const val = value?.length ? value : fallback?.length ? fallback : ''
|
||||
return val ? `${prefix ? '\n\n' : ''}${val}` : ''
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Expands a template string by replacing multiple template variables with their corresponding values.
|
||||
*
|
||||
* @param template - The template string containing placeholders.
|
||||
* @param values - An object mapping variable names to their respective values.
|
||||
* @returns A string with all template variables replaced.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const template = 'Hello {name}, welcome to {place}!';
|
||||
* const values = { name: 'Alice', place: 'Wonderland' };
|
||||
* const result = expandTemplate(template, values);
|
||||
* console.log(result); // 'Hello Alice, welcome to Wonderland!'
|
||||
* ```
|
||||
*/
|
||||
export const expandTemplate = (
|
||||
template: string,
|
||||
variables: Record<string, string | undefined>,
|
||||
) => {
|
||||
return Object.entries(variables).reduce(
|
||||
(result, [key, value]) => replaceTemplateVariable(result, key, value),
|
||||
template,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a complete link by combining a domain, path, and an optional extension.
|
||||
*
|
||||
* @param domain - The base domain of the link (e.g., "https://example.com").
|
||||
* @param path - The path to append to the domain (e.g., "guide").
|
||||
* @param extension - An optional extension to append to the path (e.g., ".md").
|
||||
* @returns The generated link
|
||||
*/
|
||||
export const generateLink = (
|
||||
path: string,
|
||||
domain?: string,
|
||||
extension?: LinksExtension,
|
||||
cleanUrls?: VitePressConfig['cleanUrls'],
|
||||
) =>
|
||||
expandTemplate('{domain}/{path}{extension}', {
|
||||
domain: domain || '',
|
||||
path,
|
||||
extension: cleanUrls ? '' : extension,
|
||||
})
|
||||
|
||||
/**
|
||||
* Options for generating metadata for markdown files.
|
||||
*/
|
||||
export interface GenerateMetadataOptions {
|
||||
/** Optional domain name to prepend to the URL. */
|
||||
domain?: LlmstxtSettings['domain']
|
||||
|
||||
/** Path to the file relative to the content root. */
|
||||
filePath: string
|
||||
|
||||
/** The link extension for generated links. */
|
||||
linksExtension?: LinksExtension
|
||||
|
||||
/** Whether to use clean URLs (without the extension). */
|
||||
cleanUrls?: VitePressConfig['cleanUrls']
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates metadata for markdown files to provide additional context for LLMs.
|
||||
*
|
||||
* @param sourceFile - Parsed markdown file with frontmatter using gray-matter.
|
||||
* @param options - Options for generating metadata.
|
||||
* @returns Object containing metadata properties for the file.
|
||||
*
|
||||
* @example
|
||||
* generateMetadata(preparedFile, { domain: 'https://example.com', filePath: 'docs/guide' })
|
||||
* // Returns { url: 'https://example.com/docs/guide.md', description: 'A guide' }
|
||||
*/
|
||||
export function generateMetadata<GrayMatter extends GrayMatterFile<Input>>(
|
||||
sourceFile: GrayMatter,
|
||||
options: GenerateMetadataOptions,
|
||||
) {
|
||||
const { domain, filePath, linksExtension, cleanUrls } = options
|
||||
const frontmatterMetadata: Record<string, string> = {}
|
||||
|
||||
frontmatterMetadata.url = generateLink(
|
||||
stripExtPosix(filePath),
|
||||
domain,
|
||||
linksExtension ?? '.md',
|
||||
cleanUrls,
|
||||
)
|
||||
|
||||
if (sourceFile.data?.description?.length) {
|
||||
frontmatterMetadata.description = sourceFile.data?.description
|
||||
}
|
||||
|
||||
return frontmatterMetadata
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a human-readable string representation of the given string's size in bytes.
|
||||
*
|
||||
* This function calculates the byte size of a given string by creating a `Blob`
|
||||
* and then converts it into a human-readable format using `byte-size`.
|
||||
*
|
||||
* @param string - The input string whose size needs to be determined.
|
||||
* @returns A human-readable size string (e.g., "1.2 KB", "500 B").
|
||||
*/
|
||||
export const getHumanReadableSizeOf = (string: string) =>
|
||||
byteSize(new Blob([string]).size).toString()
|
||||
@ -0,0 +1,365 @@
|
||||
import type { Plugin, ViteDevServer } from 'vite'
|
||||
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
import matter, { type Input } from 'gray-matter'
|
||||
import { minimatch } from 'minimatch'
|
||||
import pc from 'picocolors'
|
||||
import { remark } from 'remark'
|
||||
import remarkFrontmatter from 'remark-frontmatter'
|
||||
|
||||
import { remove } from 'unist-util-remove'
|
||||
|
||||
import { millify } from 'millify'
|
||||
import { approximateTokenSize } from 'tokenx'
|
||||
import { defaultLLMsTxtTemplate } from './constants'
|
||||
import { generateLLMsFullTxt, generateLLMsTxt } from './helpers/index'
|
||||
import log from './helpers/logger'
|
||||
import {
|
||||
expandTemplate,
|
||||
extractTitle,
|
||||
generateMetadata,
|
||||
getHumanReadableSizeOf,
|
||||
stripExt,
|
||||
} from './helpers/utils'
|
||||
import type {
|
||||
CustomTemplateVariables,
|
||||
LlmstxtSettings,
|
||||
PreparedFile,
|
||||
VitePressConfig,
|
||||
} from './types'
|
||||
|
||||
const PLUGIN_NAME = 'llmstxt'
|
||||
|
||||
/**
|
||||
* [VitePress](http://vitepress.dev/) plugin for generating raw documentation
|
||||
* for **LLMs** in Markdown format which is much lighter and more efficient for LLMs
|
||||
*
|
||||
* @param [userSettings={}] - Plugin settings.
|
||||
*
|
||||
* @see https://github.com/okineadev/vitepress-plugin-llms
|
||||
* @see https://llmstxt.org/
|
||||
*/
|
||||
export default function llmstxt(userSettings: LlmstxtSettings = {}): Plugin {
|
||||
// Create a settings object with defaults explicitly merged
|
||||
const settings: Omit<LlmstxtSettings, 'workDir'> & { workDir: string } = {
|
||||
generateLLMsTxt: true,
|
||||
generateLLMsFullTxt: true,
|
||||
generateLLMFriendlyDocsForEachPage: true,
|
||||
ignoreFiles: [],
|
||||
workDir: undefined as unknown as string,
|
||||
stripHTML: true,
|
||||
...userSettings,
|
||||
}
|
||||
|
||||
// Store the resolved Vite config
|
||||
let config: VitePressConfig
|
||||
|
||||
// Set to store all markdown file paths
|
||||
const mdFiles: Set<string> = new Set()
|
||||
|
||||
// Flag to identify which build we're in
|
||||
let isSsrBuild = false
|
||||
|
||||
return {
|
||||
name: PLUGIN_NAME,
|
||||
|
||||
/** Resolves the Vite configuration and sets up the working directory. */
|
||||
configResolved(resolvedConfig) {
|
||||
config = resolvedConfig as VitePressConfig
|
||||
if (settings.workDir) {
|
||||
settings.workDir = path.resolve(
|
||||
config.vitepress.srcDir,
|
||||
settings.workDir as string,
|
||||
)
|
||||
} else {
|
||||
settings.workDir = config.vitepress.srcDir
|
||||
}
|
||||
// Detect if this is the SSR build
|
||||
isSsrBuild = !!resolvedConfig.build?.ssr
|
||||
log.info(
|
||||
`${pc.bold(PLUGIN_NAME)} initialized ${isSsrBuild ? pc.dim('(SSR build)') : pc.dim('(client build)')} with workDir: ${pc.cyan(settings.workDir as string)}`,
|
||||
)
|
||||
},
|
||||
|
||||
/** Configures the development server to handle `llms.txt` and markdown files for LLMs. */
|
||||
async configureServer(server: ViteDevServer) {
|
||||
log.info('Dev server configured for serving plain text docs for LLMs')
|
||||
server.middlewares.use(async (req, res, next) => {
|
||||
if (req.url?.endsWith('.md') || req.url?.endsWith('.txt')) {
|
||||
try {
|
||||
// Try to read and serve the markdown file
|
||||
const filePath = path.resolve(
|
||||
config.vitepress?.outDir ?? 'dist',
|
||||
`${stripExt(req.url)}.md`,
|
||||
)
|
||||
const content = await fs.readFile(filePath, 'utf-8')
|
||||
res.setHeader('Content-Type', 'text/plain; charset=utf-8')
|
||||
res.end(content)
|
||||
return
|
||||
} catch (e) {
|
||||
// If file doesn't exist or can't be read, continue to next middleware
|
||||
log.warn(`Failed to return ${pc.cyan(req.url)}: File not found`)
|
||||
next()
|
||||
}
|
||||
}
|
||||
|
||||
// Pass to next middleware if not handled
|
||||
next()
|
||||
})
|
||||
},
|
||||
|
||||
/**
|
||||
* Resets the collection of markdown files when the build starts.
|
||||
* This ensures we don't include stale data from previous builds.
|
||||
*/
|
||||
buildStart() {
|
||||
mdFiles.clear()
|
||||
log.info('Build started, file collection cleared')
|
||||
},
|
||||
|
||||
/**
|
||||
* Processes each file that Vite transforms and collects markdown files.
|
||||
*
|
||||
* @param _ - The file content (not used).
|
||||
* @param id - The file identifier (path).
|
||||
* @returns null if the file is processed, otherwise returns the original content.
|
||||
*/
|
||||
async transform(_, id: string) {
|
||||
if (!id.endsWith('.md')) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Skip files outside workDir if it's configured
|
||||
if (!id.startsWith(settings.workDir as string)) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (settings.ignoreFiles?.length) {
|
||||
const shouldIgnore = await Promise.all(
|
||||
settings.ignoreFiles.map(async (pattern) => {
|
||||
if (typeof pattern === 'string') {
|
||||
return await Promise.resolve(
|
||||
minimatch(
|
||||
path.relative(settings.workDir as string, id),
|
||||
pattern,
|
||||
),
|
||||
)
|
||||
}
|
||||
return false
|
||||
}),
|
||||
)
|
||||
|
||||
if (shouldIgnore.some((result) => result === true)) {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// Add markdown file path to our collection
|
||||
mdFiles.add(id)
|
||||
// Return null to avoid modifying the file
|
||||
return null
|
||||
},
|
||||
|
||||
/**
|
||||
* Runs only in the client build (not SSR) after completion.
|
||||
* This ensures the processing happens exactly once.
|
||||
*/
|
||||
async generateBundle() {
|
||||
// Skip processing during SSR build
|
||||
if (isSsrBuild) {
|
||||
log.info('Skipping LLMs docs generation in SSR build')
|
||||
return
|
||||
}
|
||||
|
||||
const outDir = config.vitepress?.outDir ?? 'dist'
|
||||
|
||||
// Create output directory if it doesn't exist
|
||||
try {
|
||||
await fs.access(outDir)
|
||||
} catch {
|
||||
log.info(`Creating output directory: ${pc.cyan(outDir)}`)
|
||||
await fs.mkdir(outDir, { recursive: true })
|
||||
}
|
||||
|
||||
const mdFilesList = Array.from(mdFiles)
|
||||
const fileCount = mdFilesList.length
|
||||
|
||||
// Skip if no files found
|
||||
if (fileCount === 0) {
|
||||
log.warn(
|
||||
`No markdown files found to process. Check your \`${pc.bold('workDir')}\` and \`${pc.bold('ignoreFiles')}\` settings.`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
log.info(
|
||||
`Processing ${pc.bold(fileCount.toString())} markdown files from ${pc.cyan(settings.workDir)}`,
|
||||
)
|
||||
|
||||
const preparedFiles: PreparedFile[] = await Promise.all(
|
||||
mdFilesList.map(async (file) => {
|
||||
const content = await fs.readFile(file, 'utf-8')
|
||||
|
||||
let mdFile: matter.GrayMatterFile<Input>
|
||||
|
||||
if (settings.stripHTML) {
|
||||
const cleanedMarkdown = await remark()
|
||||
.use(remarkFrontmatter)
|
||||
.use(() => {
|
||||
// Strip HTML tags
|
||||
return (tree) => {
|
||||
remove(tree, { type: 'html' })
|
||||
return tree
|
||||
}
|
||||
})
|
||||
.process(content)
|
||||
|
||||
mdFile = matter(String(cleanedMarkdown))
|
||||
} else {
|
||||
mdFile = matter(content)
|
||||
}
|
||||
// Extract title from frontmatter or use the first heading
|
||||
const title = extractTitle(mdFile)?.trim() || 'Untitled'
|
||||
|
||||
const filePath =
|
||||
path.basename(file) === 'index.md' &&
|
||||
path.dirname(file) !== settings.workDir
|
||||
? `${path.dirname(file)}.md`
|
||||
: file
|
||||
|
||||
return { path: filePath, title, file: mdFile }
|
||||
}),
|
||||
)
|
||||
|
||||
if (settings.generateLLMFriendlyDocsForEachPage) {
|
||||
await Promise.all(
|
||||
preparedFiles.map(async (file) => {
|
||||
const relativePath = path.relative(settings.workDir, file.path)
|
||||
try {
|
||||
const mdFile = file.file
|
||||
const targetPath = path.resolve(outDir, relativePath)
|
||||
|
||||
// Ensure target directory exists (async version)
|
||||
await fs.mkdir(path.dirname(targetPath), {
|
||||
recursive: true,
|
||||
})
|
||||
|
||||
// Copy file to output directory (async version)
|
||||
await fs.writeFile(
|
||||
targetPath,
|
||||
matter.stringify(
|
||||
mdFile.content,
|
||||
generateMetadata(mdFile, {
|
||||
domain: settings.domain,
|
||||
filePath: relativePath,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
log.success(`Processed ${pc.cyan(relativePath)}`)
|
||||
} catch (error) {
|
||||
log.error(
|
||||
// @ts-ignore
|
||||
`Failed to process ${pc.cyan(relativePath)}: ${error.message}`,
|
||||
)
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// Sort files by title for better organization
|
||||
preparedFiles.sort((a, b) => a.title.localeCompare(b.title))
|
||||
|
||||
const tasks: Promise<void>[] = []
|
||||
|
||||
// Generate llms.txt - table of contents with links
|
||||
if (settings.generateLLMsTxt) {
|
||||
const llmsTxtPath = path.resolve(outDir, 'llms.txt')
|
||||
const templateVariables: CustomTemplateVariables = {
|
||||
title: settings.title,
|
||||
description: settings.description,
|
||||
details: settings.details,
|
||||
toc: settings.toc,
|
||||
...settings.customTemplateVariables,
|
||||
}
|
||||
|
||||
tasks.push(
|
||||
(async () => {
|
||||
log.info(`Generating ${pc.cyan('llms.txt')}...`)
|
||||
|
||||
const llmsTxt = await generateLLMsTxt(preparedFiles, {
|
||||
indexMd: path.resolve(settings.workDir as string, 'index.md'),
|
||||
srcDir: settings.workDir as string,
|
||||
LLMsTxtTemplate:
|
||||
settings.customLLMsTxtTemplate || defaultLLMsTxtTemplate,
|
||||
templateVariables,
|
||||
vitepressConfig: config?.vitepress?.userConfig,
|
||||
domain: settings.domain,
|
||||
sidebar: settings.sidebar,
|
||||
linksExtension: !settings.generateLLMFriendlyDocsForEachPage
|
||||
? '.html'
|
||||
: undefined,
|
||||
cleanUrls: config.cleanUrls,
|
||||
})
|
||||
|
||||
await fs.writeFile(llmsTxtPath, llmsTxt, 'utf-8')
|
||||
|
||||
log.success(
|
||||
expandTemplate(
|
||||
'Generated {file} (~{tokens} tokens, {size}) with {fileCount} documentation links',
|
||||
{
|
||||
file: pc.cyan('llms.txt'),
|
||||
tokens: pc.bold(millify(approximateTokenSize(llmsTxt))),
|
||||
size: pc.bold(getHumanReadableSizeOf(llmsTxt)),
|
||||
fileCount: pc.bold(fileCount.toString()),
|
||||
},
|
||||
),
|
||||
)
|
||||
})(),
|
||||
)
|
||||
}
|
||||
|
||||
// Generate llms-full.txt - all content in one file
|
||||
if (settings.generateLLMsFullTxt) {
|
||||
const llmsFullTxtPath = path.resolve(outDir, 'llms-full.txt')
|
||||
|
||||
tasks.push(
|
||||
(async () => {
|
||||
log.info(
|
||||
`Generating full documentation bundle (${pc.cyan('llms-full.txt')})...`,
|
||||
)
|
||||
|
||||
const llmsFullTxt = generateLLMsFullTxt(preparedFiles, {
|
||||
srcDir: settings.workDir as string,
|
||||
domain: settings.domain,
|
||||
linksExtension: !settings.generateLLMFriendlyDocsForEachPage
|
||||
? '.html'
|
||||
: undefined,
|
||||
cleanUrls: config.cleanUrls,
|
||||
})
|
||||
|
||||
// Write content to llms-full.txt
|
||||
await fs.writeFile(llmsFullTxtPath, llmsFullTxt, 'utf-8')
|
||||
log.success(
|
||||
expandTemplate(
|
||||
'Generated {file} (~{tokens} tokens, {size}) with {fileCount} markdown files',
|
||||
{
|
||||
file: pc.cyan('llms-full.txt'),
|
||||
tokens: pc.bold(millify(approximateTokenSize(llmsFullTxt))),
|
||||
size: pc.bold(getHumanReadableSizeOf(llmsFullTxt)),
|
||||
fileCount: pc.bold(fileCount.toString()),
|
||||
},
|
||||
),
|
||||
)
|
||||
})(),
|
||||
)
|
||||
}
|
||||
|
||||
if (tasks.length) {
|
||||
await Promise.all(tasks)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue