diff --git a/__tests__/unit/node/markdown/plugins/include.test.ts b/__tests__/unit/node/markdown/plugins/include.test.ts index 0a8aa757..fadfa216 100644 --- a/__tests__/unit/node/markdown/plugins/include.test.ts +++ b/__tests__/unit/node/markdown/plugins/include.test.ts @@ -533,6 +533,110 @@ describe('node/markdown/plugins/include', () => { expect(html).toContain('href="/abs/target.html"') }) + describe('line map', () => { + async function renderLocs( + src: string, + options: MarkdownOptions = {}, + env: Partial = {} + ) { + const locs: { file?: string; line: number; column?: number }[] = [] + const result = await render( + src, + { + ...options, + config(md) { + md.core.ruler.push('test_capture_locs', (state) => { + for (const token of state.tokens) { + for (const child of token.children ?? []) { + if (child.type === 'link_open' && child.meta?.vpLoc) { + locs.push(child.meta.vpLoc) + } + } + } + }) + } + }, + env + ) + return { ...result, locs } + } + + test('positions resolve into included files and stay exact after them', async () => { + await write( + 'sub/part.md', + '---\nt: 1\n---\nSome text\n[x](./nope)\nMore text\n' + ) + + const { locs } = await renderLocs( + '---\ntitle: x\n---\n\n# Guide\n\n\n\npara [a](./a)\nand [b](./b)\n' + ) + expect(locs).toEqual([ + { file: path.join(root, 'sub/part.md'), line: 5, column: 1 }, + { file: path.join(root, 'index.md'), line: 9, column: 6 }, + { file: path.join(root, 'index.md'), line: 10, column: 5 } + ]) + }) + + test('positions resolve through nested includes', async () => { + await write('a/one.md', 'one\n\n\n') + await write('b/two.md', 'two [t](./deep)\n') + + const { locs } = await renderLocs('\n') + expect(locs).toEqual([ + { file: path.join(root, 'b/two.md'), line: 1, column: 5 } + ]) + }) + + test('region includes point at real editor lines past frontmatter', async () => { + await write( + 'sub/part.md', + '---\nt: 1\n---\nbefore\n\nin [r](./r) region\n\nafter\n' + ) + + const { locs } = await renderLocs( + '\n' + ) + expect(locs).toEqual([ + { file: path.join(root, 'sub/part.md'), line: 6, column: 4 } + ]) + }) + + test('range includes keep frontmatter lines and stay file-accurate', async () => { + await write('sub/part.md', 'one\ntwo [r](./r)\nthree\n') + + const { locs } = await renderLocs( + '\n' + ) + expect(locs).toEqual([ + { file: path.join(root, 'sub/part.md'), line: 2, column: 5 } + ]) + }) + + test('inline includes attribute the splice line to the page', async () => { + await write('sub/part.md', 'spliced [s](./s)\nnext [n](./n)\n') + + const { locs } = await renderLocs( + 'before after\n' + ) + // the first included line merges into the page's line — it keeps the + // page as its file and gets no column, since the stitched line matches + // neither file's authored text; content on the included file's own + // subsequent lines resolves into it exactly + expect(locs).toEqual([ + { file: path.join(root, 'index.md'), line: 1 }, + { file: path.join(root, 'sub/part.md'), line: 2, column: 6 } + ]) + }) + + test('pages without includes get an identity line map', async () => { + const { env } = await render('# Hi\n\n[a](./a)\n') + expect(env.lineMap!.resolve(2)).toEqual({ + file: path.join(root, 'index.md'), + line: 2 + }) + }) + }) + test('does not rebase destinations resolved from frontmatter', async () => { await write( 'guide/shared/note.md', diff --git a/src/node/markdown/lineMap.ts b/src/node/markdown/lineMap.ts new file mode 100644 index 00000000..20ec1c17 --- /dev/null +++ b/src/node/markdown/lineMap.ts @@ -0,0 +1,179 @@ +import type { MarkdownLineMap } from '../shared' + +/** + * One contiguous run of rendered-source lines that all come from the same + * file. A segment covers the lines from `start` up to the next segment's + * `start` (or the end of the document). + */ +export interface LineMapSegment { + /** first rendered-source line this segment covers (0-based) */ + start: number + /** absolute path of the physical source file */ + file: string + /** line in `file` that `start` corresponds to (0-based) */ + line: number +} + +/** + * Maps 0-based lines of the rendered markdown source (`env.src`) back to the + * physical file and 0-based line they came from. Built by the include plugin + * while expanding `` directives; a page without includes + * gets a single identity segment. + */ +export class LineMap implements MarkdownLineMap { + readonly segments: LineMapSegment[] + /** lines stitched together from more than one source (mid-line splices) */ + private readonly splicedLines: ReadonlySet + + constructor(segments: LineMapSegment[], splicedLines?: ReadonlySet) { + this.segments = segments + this.splicedLines = splicedLines ?? new Set() + } + + resolve(line: number): { file: string; line: number; spliced?: boolean } { + const segments = this.segments + let lo = 0 + let hi = segments.length + while (lo < hi) { + const mid = (lo + hi) >>> 1 + if (segments[mid].start <= line) lo = mid + 1 + else hi = mid + } + const segment = segments[Math.max(0, lo - 1)] + return { + file: segment.file, + line: segment.line + Math.max(0, line - segment.start), + ...(this.splicedLines.has(line) && { spliced: true }) + } + } +} + +/** + * Assembles an output string from mapped chunks, tracking which file each + * output line starts in. A chunk appended mid-line contributes no mapping + * for that line — the line is attributed to whoever started it. + */ +export class MappedBuilder { + private out = '' + private outLine = 0 + private midLine = false + private readonly segments: LineMapSegment[] = [] + private readonly splicedLines = new Set() + + /** + * Appends `text`, whose lines are described by `segments` in the text's + * own 0-based line coordinates. + */ + append( + text: string, + segments: LineMapSegment[], + splicedLines?: ReadonlySet + ): void { + if (!text) return + + // a mid-line append stitches this output line together from more than + // one source, so column positions on it are not meaningful + if (this.midLine) this.splicedLines.add(this.outLine) + if (splicedLines) { + for (const line of splicedLines) + this.splicedLines.add(this.outLine + line) + } + + const breaks = countLineBreaks(text) + for (let i = 0; i < segments.length; i++) { + const segment = segments[i] + const segmentEnd = + i + 1 < segments.length ? segments[i + 1].start : Infinity + // the first line of a mid-line append merges into the current output + // line, which already has an owner + const from = this.midLine ? Math.max(segment.start, 1) : segment.start + if (from >= segmentEnd || from > breaks) continue + this.push({ + start: this.outLine + from, + file: segment.file, + line: segment.line + (from - segment.start) + }) + } + + this.out += text + this.outLine += breaks + this.midLine = text[text.length - 1] !== '\n' + } + + private push(segment: LineMapSegment): void { + const prev = this.segments[this.segments.length - 1] + if (prev) { + // exact continuation of the previous segment — nothing new to record + if ( + prev.file === segment.file && + segment.line - prev.line === segment.start - prev.start + ) { + return + } + if (prev.start === segment.start) { + this.segments[this.segments.length - 1] = segment + return + } + } + this.segments.push(segment) + } + + build(): { + src: string + segments: LineMapSegment[] + splicedLines: Set + } { + return { + src: this.out, + segments: this.segments, + splicedLines: this.splicedLines + } + } +} + +/** + * The sub-list of segments covering lines `[from, to)`, rebased so `from` + * becomes line 0. + */ +export function sliceSegments( + segments: LineMapSegment[], + from: number, + to: number +): LineMapSegment[] { + const out: LineMapSegment[] = [] + for (let i = 0; i < segments.length; i++) { + const segment = segments[i] + const segmentEnd = + i + 1 < segments.length ? segments[i + 1].start : Infinity + const start = Math.max(segment.start, from) + if (start >= Math.min(segmentEnd, to)) continue + out.push({ + start: start - from, + file: segment.file, + line: segment.line + (start - segment.start) + }) + } + return out +} + +export function offsetSegments( + segments: LineMapSegment[], + delta: number +): LineMapSegment[] { + return segments.map((s) => ({ ...s, start: s.start + delta })) +} + +/** resolves a line against raw segments, like `LineMap.resolve` */ +export function resolveInSegments( + segments: LineMapSegment[], + line: number +): { file: string; line: number } { + return new LineMap(segments).resolve(line) +} + +// mirrors markdown-it's newline normalization (`\r\n?|\n`) +const lineBreakRE = /\r\n?|\n/g + +export function countLineBreaks(str: string): number { + return str.match(lineBreakRE)?.length ?? 0 +} diff --git a/src/node/markdown/plugins/include.ts b/src/node/markdown/plugins/include.ts index ec258e3b..1a577ab5 100644 --- a/src/node/markdown/plugins/include.ts +++ b/src/node/markdown/plugins/include.ts @@ -1,11 +1,20 @@ import path from 'node:path' import matter from 'gray-matter' -import { replaceAsync, type MarkdownItAsync } from 'markdown-it-async' +import type { MarkdownItAsync } from 'markdown-it-async' import type { Logger } from 'vite' import { slash, type MarkdownEnv } from '../../shared' import { readTextFile } from '../../utils/fs' +import { + countLineBreaks, + LineMap, + MappedBuilder, + offsetSegments, + resolveInSegments, + sliceSegments, + type LineMapSegment +} from '../lineMap' import { findRegions } from '../regions' export interface Options { @@ -29,18 +38,24 @@ const rangeRE = /\{(\d*),(\d*)\}$/ const regionRE = /#([^\s{]+)$/ const separatorRE = /[\\/]/ const fenceRE = /^ {0,3}(`{3,}|~{3,})/ -const rebaseMarkerRE = /^[ \t]*[ \t]*$/gm -// per-render stacks of included-file directories, driven by the rebase -// markers while rendering -const rebaseStacks = new WeakMap() +interface Expanded { + src: string + /** describes `src` in its own 0-based line coordinates */ + segments: LineMapSegment[] + /** lines of `src` stitched together from more than one source */ + splicedLines: ReadonlySet +} + +const noSplices: ReadonlySet = new Set() /** * Expands `` directives before rendering. Wraps * `renderAsync` so every consumer of the renderer (page rendering, local * search indexing, the content loader and `createMarkdownRenderer` users) - * gets the same expansion. Included files are recorded in `env.includes` - * and the expanded source is exposed as `env.src`. + * gets the same expansion. Included files are recorded in `env.includes`, + * the expanded source is exposed as `env.src`, and `env.lineMap` maps its + * lines back to the physical files they came from. */ export function includePlugin( md: MarkdownItAsync, @@ -55,9 +70,18 @@ export function includePlugin( if (file == null) return renderAsync(src, env) mdEnv!.includes ??= [] - src = await processIncludes(md, srcDir, src, file, mdEnv!, options, logger) - mdEnv!.src = src - return renderAsync(src, env) + const expanded = await processIncludes( + md, + srcDir, + src, + file, + mdEnv!, + options, + logger + ) + mdEnv!.src = expanded.src + mdEnv!.lineMap = new LineMap(expanded.segments, expanded.splicedLines) + return renderAsync(expanded.src, env) } if (options.rebaseRelativeUrls !== false) registerRebaseRules(md) @@ -71,17 +95,36 @@ async function processIncludes( env: MarkdownEnv, options: Options, logger: Pick, - ancestors: string[] = [] -): Promise { - return replaceAsync(src, includeRE, async (...args: string[]) => { - const [m, , rawOffset] = args - let [, m1] = args - if (!m1.length) return m - - const fail = (message: string): string => { + ancestors: string[] = [], + base: LineMapSegment[] = [{ start: 0, file, line: 0 }] +): Promise { + const matches = [...src.matchAll(includeRE)] + if (!matches.length) return { src, segments: base, splicedLines: noSplices } + + const out = new MappedBuilder() + let cursor = 0 + let cursorLine = 0 + + const passthrough = (to: number) => { + if (to <= cursor) return + const text = src.slice(cursor, to) + const breaks = countLineBreaks(text) + out.append(text, sliceSegments(base, cursorLine, cursorLine + breaks + 1)) + cursor = to + cursorLine += breaks + } + + const expandInclude = async ( + m: RegExpExecArray, + directiveLine: number + ): Promise => { + const directive = m[0] + let m1 = m[1] + + const fail = (message: string): Expanded => { if (!options.silent) throw new Error(message) logger.warn(`${message} (in ${file})`) - return '' + return { src: '', segments: [], splicedLines: noSplices } } const range = rangeRE.exec(m1) @@ -95,7 +138,9 @@ async function processIncludes( // leave circular includes unexpanded — only repeats along the ancestor // chain are cycles, the same file may still be included by siblings - if (includePath === file || ancestors.includes(includePath)) return m + if (includePath === file || ancestors.includes(includePath)) { + return undefined + } // record the dependency before reading it, so that creating a missing // file is picked up by the watcher @@ -116,19 +161,36 @@ async function processIncludes( } // for markdown files, if a range is used without a region, the line - // numbers must account for the frontmatter, so it is kept; otherwise - // it is stripped before selecting content + // numbers must account for the frontmatter, so it is kept; otherwise it + // is stripped before selecting content, and the removed height is folded + // into the segments so they keep pointing at real editor lines + let fmOffset = 0 if (path.extname(includePath) === '.md' && (region || !range)) { - content = matter(content, {}).content + const stripped = matter(content, {}).content + fmOffset = countLineBreaks(content) - countLineBreaks(stripped) + content = stripped } let lines = content.split('\n') + let childBase: LineMapSegment[] = [ + { start: 0, file: includePath, line: fmOffset } + ] if (region) { const name = region[1] const regions = findRegions(lines, name) if (regions.length > 0) { - lines = regions.flatMap((r) => lines.slice(r.start, r.end)) + const selected: string[] = [] + childBase = [] + for (const r of regions) { + childBase.push({ + start: selected.length, + file: includePath, + line: fmOffset + r.start + }) + selected.push(...lines.slice(r.start, r.end)) + } + lines = selected } else { // no editor-style region matched — try heading anchors const section = findHeadingSection(md, content, includePath, name, { @@ -140,6 +202,9 @@ async function processIncludes( `Include region or heading "${name}" not found in ${includePath}` ) } + childBase = [ + { start: 0, file: includePath, line: fmOffset + section.start } + ] lines = lines.slice(section.start, section.end) } } @@ -152,11 +217,12 @@ async function processIncludes( `Include range ${range[0]} is out of bounds in ${includePath}` ) } + childBase = sliceSegments(childBase, start - 1, end) lines = lines.slice(start - 1, end) } // recursively process includes in the content - const expanded = await processIncludes( + const child = await processIncludes( md, srcDir, lines.join('\n'), @@ -164,22 +230,52 @@ async function processIncludes( env, options, logger, - [...ancestors, file] + [...ancestors, file], + childBase ) - // wrap included markdown in markers driving the url rebasing at render - // time; they are removed from the output by the html_block rule. Blank - // lines keep them out of adjacent html blocks, and directives that are - // not on a line of their own - inline ones and those inside fences - - // are left unwrapped so the markers can't end up in the output. - const offset = rawOffset as unknown as number - return options.rebaseRelativeUrls !== false && + // wrap included markdown in blank lines so its blocks stay isolated from + // adjacent page content; directives that are not on a line of their own - + // inline ones and those inside fences - are left unwrapped so the blanks + // can't end up inside surrounding constructs. The blank lines belong to + // the include directive's own line. + if ( path.extname(includePath) === '.md' && - isOwnLine(src, offset, m.length) && - !isInsideFence(src, offset) - ? `\n\n${expanded}\n\n` - : expanded - }) + isOwnLine(src, m.index, directive.length) && + !isInsideFence(src, m.index) + ) { + const at = resolveInSegments(base, directiveLine) + const childLineCount = countLineBreaks(child.src) + 1 + return { + src: `\n\n${child.src}\n\n`, + segments: [ + { start: 0, ...at }, + ...offsetSegments(child.segments, 2), + { start: 2 + childLineCount, ...at } + ], + splicedLines: new Set([...child.splicedLines].map((l) => l + 2)) + } + } + return child + } + + for (const m of matches) { + passthrough(m.index) + const expanded = m[1].length + ? await expandInclude(m, cursorLine) + : undefined + if (expanded === undefined) { + // left unexpanded — passes through with the surrounding text + passthrough(m.index + m[0].length) + } else { + out.append(expanded.src, expanded.segments, expanded.splicedLines) + cursor = m.index + m[0].length + cursorLine += countLineBreaks(m[0]) + } + } + passthrough(src.length) + + return out.build() } function findHeadingSection( @@ -237,43 +333,29 @@ function isInsideFence(src: string, offset: number) { } function registerRebaseRules(md: MarkdownItAsync) { - const htmlBlock = md.renderer.rules.html_block! - md.renderer.rules.html_block = (tokens, idx, opts, env, self) => { - const token = tokens[idx] - if (!token.content.includes('