feat(markdown)!: map rendered source lines back through includes

Include expansion now emits a line map alongside the expanded source
(env.lineMap): one segment per contiguous run of lines, composed through
nested includes, region/heading/range slices (with the included file's
stripped frontmatter height folded in) and the isolation blank lines. Every
resolved source position now names the physical file a construct was
authored in.

This replaces the rebase-marker mechanism outright: relative URLs in
included markdown are rebased by resolving the token's position instead of
maintaining a marker stack scrubbed out of html_block tokens, which also
extends rebasing to lines following an inline include and makes block
isolation of own-line markdown includes unconditional (previously both were
skipped when markers could not be emitted).

BREAKING CHANGE: the `<!-- @include-start/end -->` markers no longer exist
in the expanded source (`env.src`).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

__tests__/unit/node/markdown/plugins/include.test.ts

@@ -533,6 +533,110 @@ describe('node/markdown/plugins/include', () => {
     expect(html).toContain('href="/abs/target.html"')
   })
 
+  describe('line map', () => {
+    async function renderLocs(
+      src: string,
+      options: MarkdownOptions = {},
+      env: Partial<MarkdownEnv> = {}
+    ) {
+      const locs: { file?: string; line: number; column?: number }[] = []
+      const result = await render(
+        src,
+        {
+          ...options,
+          config(md) {
+            md.core.ruler.push('test_capture_locs', (state) => {
+              for (const token of state.tokens) {
+                for (const child of token.children ?? []) {
+                  if (child.type === 'link_open' && child.meta?.vpLoc) {
+                    locs.push(child.meta.vpLoc)
+                  }
+                }
+              }
+            })
+          }
+        },
+        env
+      )
+      return { ...result, locs }
+    }
+
+    test('positions resolve into included files and stay exact after them', async () => {
+      await write(
+        'sub/part.md',
+        '---\nt: 1\n---\nSome text\n[x](./nope)\nMore text\n'
+      )
+
+      const { locs } = await renderLocs(
+        '---\ntitle: x\n---\n\n# Guide\n\n<!-- @include: ./sub/part.md -->\n\npara [a](./a)\nand [b](./b)\n'
+      )
+      expect(locs).toEqual([
+        { file: path.join(root, 'sub/part.md'), line: 5, column: 1 },
+        { file: path.join(root, 'index.md'), line: 9, column: 6 },
+        { file: path.join(root, 'index.md'), line: 10, column: 5 }
+      ])
+    })
+
+    test('positions resolve through nested includes', async () => {
+      await write('a/one.md', 'one\n\n<!-- @include: ../b/two.md -->\n')
+      await write('b/two.md', 'two [t](./deep)\n')
+
+      const { locs } = await renderLocs('<!-- @include: ./a/one.md -->\n')
+      expect(locs).toEqual([
+        { file: path.join(root, 'b/two.md'), line: 1, column: 5 }
+      ])
+    })
+
+    test('region includes point at real editor lines past frontmatter', async () => {
+      await write(
+        'sub/part.md',
+        '---\nt: 1\n---\nbefore\n<!-- #region sec -->\nin [r](./r) region\n<!-- #endregion sec -->\nafter\n'
+      )
+
+      const { locs } = await renderLocs(
+        '<!-- @include: ./sub/part.md#sec -->\n'
+      )
+      expect(locs).toEqual([
+        { file: path.join(root, 'sub/part.md'), line: 6, column: 4 }
+      ])
+    })
+
+    test('range includes keep frontmatter lines and stay file-accurate', async () => {
+      await write('sub/part.md', 'one\ntwo [r](./r)\nthree\n')
+
+      const { locs } = await renderLocs(
+        '<!-- @include: ./sub/part.md{2,2} -->\n'
+      )
+      expect(locs).toEqual([
+        { file: path.join(root, 'sub/part.md'), line: 2, column: 5 }
+      ])
+    })
+
+    test('inline includes attribute the splice line to the page', async () => {
+      await write('sub/part.md', 'spliced [s](./s)\nnext [n](./n)\n')
+
+      const { locs } = await renderLocs(
+        'before <!-- @include: ./sub/part.md --> after\n'
+      )
+      // the first included line merges into the page's line — it keeps the
+      // page as its file and gets no column, since the stitched line matches
+      // neither file's authored text; content on the included file's own
+      // subsequent lines resolves into it exactly
+      expect(locs).toEqual([
+        { file: path.join(root, 'index.md'), line: 1 },
+        { file: path.join(root, 'sub/part.md'), line: 2, column: 6 }
+      ])
+    })
+
+    test('pages without includes get an identity line map', async () => {
+      const { env } = await render('# Hi\n\n[a](./a)\n')
+      expect(env.lineMap!.resolve(2)).toEqual({
+        file: path.join(root, 'index.md'),
+        line: 2
+      })
+    })
+  })
+
   test('does not rebase destinations resolved from frontmatter', async () => {
     await write(
       'guide/shared/note.md',

src/node/markdown/lineMap.ts

@@ -0,0 +1,179 @@
+import type { MarkdownLineMap } from '../shared'
+
+/**
+ * One contiguous run of rendered-source lines that all come from the same
+ * file. A segment covers the lines from `start` up to the next segment's
+ * `start` (or the end of the document).
+ */
+export interface LineMapSegment {
+  /** first rendered-source line this segment covers (0-based) */
+  start: number
+  /** absolute path of the physical source file */
+  file: string
+  /** line in `file` that `start` corresponds to (0-based) */
+  line: number
+}
+
+/**
+ * Maps 0-based lines of the rendered markdown source (`env.src`) back to the
+ * physical file and 0-based line they came from. Built by the include plugin
+ * while expanding `<!-- @include -->` directives; a page without includes
+ * gets a single identity segment.
+ */
+export class LineMap implements MarkdownLineMap {
+  readonly segments: LineMapSegment[]
+  /** lines stitched together from more than one source (mid-line splices) */
+  private readonly splicedLines: ReadonlySet<number>
+
+  constructor(segments: LineMapSegment[], splicedLines?: ReadonlySet<number>) {
+    this.segments = segments
+    this.splicedLines = splicedLines ?? new Set()
+  }
+
+  resolve(line: number): { file: string; line: number; spliced?: boolean } {
+    const segments = this.segments
+    let lo = 0
+    let hi = segments.length
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1
+      if (segments[mid].start <= line) lo = mid + 1
+      else hi = mid
+    }
+    const segment = segments[Math.max(0, lo - 1)]
+    return {
+      file: segment.file,
+      line: segment.line + Math.max(0, line - segment.start),
+      ...(this.splicedLines.has(line) && { spliced: true })
+    }
+  }
+}
+
+/**
+ * Assembles an output string from mapped chunks, tracking which file each
+ * output line starts in. A chunk appended mid-line contributes no mapping
+ * for that line — the line is attributed to whoever started it.
+ */
+export class MappedBuilder {
+  private out = ''
+  private outLine = 0
+  private midLine = false
+  private readonly segments: LineMapSegment[] = []
+  private readonly splicedLines = new Set<number>()
+
+  /**
+   * Appends `text`, whose lines are described by `segments` in the text's
+   * own 0-based line coordinates.
+   */
+  append(
+    text: string,
+    segments: LineMapSegment[],
+    splicedLines?: ReadonlySet<number>
+  ): void {
+    if (!text) return
+
+    // a mid-line append stitches this output line together from more than
+    // one source, so column positions on it are not meaningful
+    if (this.midLine) this.splicedLines.add(this.outLine)
+    if (splicedLines) {
+      for (const line of splicedLines)
+        this.splicedLines.add(this.outLine + line)
+    }
+
+    const breaks = countLineBreaks(text)
+    for (let i = 0; i < segments.length; i++) {
+      const segment = segments[i]
+      const segmentEnd =
+        i + 1 < segments.length ? segments[i + 1].start : Infinity
+      // the first line of a mid-line append merges into the current output
+      // line, which already has an owner
+      const from = this.midLine ? Math.max(segment.start, 1) : segment.start
+      if (from >= segmentEnd || from > breaks) continue
+      this.push({
+        start: this.outLine + from,
+        file: segment.file,
+        line: segment.line + (from - segment.start)
+      })
+    }
+
+    this.out += text
+    this.outLine += breaks
+    this.midLine = text[text.length - 1] !== '\n'
+  }
+
+  private push(segment: LineMapSegment): void {
+    const prev = this.segments[this.segments.length - 1]
+    if (prev) {
+      // exact continuation of the previous segment — nothing new to record
+      if (
+        prev.file === segment.file &&
+        segment.line - prev.line === segment.start - prev.start
+      ) {
+        return
+      }
+      if (prev.start === segment.start) {
+        this.segments[this.segments.length - 1] = segment
+        return
+      }
+    }
+    this.segments.push(segment)
+  }
+
+  build(): {
+    src: string
+    segments: LineMapSegment[]
+    splicedLines: Set<number>
+  } {
+    return {
+      src: this.out,
+      segments: this.segments,
+      splicedLines: this.splicedLines
+    }
+  }
+}
+
+/**
+ * The sub-list of segments covering lines `[from, to)`, rebased so `from`
+ * becomes line 0.
+ */
+export function sliceSegments(
+  segments: LineMapSegment[],
+  from: number,
+  to: number
+): LineMapSegment[] {
+  const out: LineMapSegment[] = []
+  for (let i = 0; i < segments.length; i++) {
+    const segment = segments[i]
+    const segmentEnd =
+      i + 1 < segments.length ? segments[i + 1].start : Infinity
+    const start = Math.max(segment.start, from)
+    if (start >= Math.min(segmentEnd, to)) continue
+    out.push({
+      start: start - from,
+      file: segment.file,
+      line: segment.line + (start - segment.start)
+    })
+  }
+  return out
+}
+
+export function offsetSegments(
+  segments: LineMapSegment[],
+  delta: number
+): LineMapSegment[] {
+  return segments.map((s) => ({ ...s, start: s.start + delta }))
+}
+
+/** resolves a line against raw segments, like `LineMap.resolve` */
+export function resolveInSegments(
+  segments: LineMapSegment[],
+  line: number
+): { file: string; line: number } {
+  return new LineMap(segments).resolve(line)
+}
+
+// mirrors markdown-it's newline normalization (`\r\n?|\n`)
+const lineBreakRE = /\r\n?|\n/g
+
+export function countLineBreaks(str: string): number {
+  return str.match(lineBreakRE)?.length ?? 0
+}

src/node/markdown/plugins/include.ts

@@ -1,11 +1,20 @@
 import path from 'node:path'
 
 import matter from 'gray-matter'
-import { replaceAsync, type MarkdownItAsync } from 'markdown-it-async'
+import type { MarkdownItAsync } from 'markdown-it-async'
 import type { Logger } from 'vite'
 
 import { slash, type MarkdownEnv } from '../../shared'
 import { readTextFile } from '../../utils/fs'
+import {
+  countLineBreaks,
+  LineMap,
+  MappedBuilder,
+  offsetSegments,
+  resolveInSegments,
+  sliceSegments,
+  type LineMapSegment
+} from '../lineMap'
 import { findRegions } from '../regions'
 
 export interface Options {
@@ -29,18 +38,24 @@ const rangeRE = /\{(\d*),(\d*)\}$/
 const regionRE = /#([^\s{]+)$/
 const separatorRE = /[\\/]/
 const fenceRE = /^ {0,3}(`{3,}|~{3,})/
-const rebaseMarkerRE = /^[ \t]*<!-- @include-(?:start: (.*)|end) -->[ \t]*$/gm
 
-// per-render stacks of included-file directories, driven by the rebase
+interface Expanded {
-// markers while rendering
+  src: string
-const rebaseStacks = new WeakMap<object, string[]>()
+  /** describes `src` in its own 0-based line coordinates */
+  segments: LineMapSegment[]
+  /** lines of `src` stitched together from more than one source */
+  splicedLines: ReadonlySet<number>
+}
+
+const noSplices: ReadonlySet<number> = new Set()
 
 /**
  * Expands `<!-- @include: path -->` directives before rendering. Wraps
  * `renderAsync` so every consumer of the renderer (page rendering, local
  * search indexing, the content loader and `createMarkdownRenderer` users)
- * gets the same expansion. Included files are recorded in `env.includes`
+ * gets the same expansion. Included files are recorded in `env.includes`,
- * and the expanded source is exposed as `env.src`.
+ * the expanded source is exposed as `env.src`, and `env.lineMap` maps its
+ * lines back to the physical files they came from.
  */
 export function includePlugin(
   md: MarkdownItAsync,
@@ -55,9 +70,18 @@ export function includePlugin(
     if (file == null) return renderAsync(src, env)
 
     mdEnv!.includes ??= []
-    src = await processIncludes(md, srcDir, src, file, mdEnv!, options, logger)
+    const expanded = await processIncludes(
-    mdEnv!.src = src
+      md,
-    return renderAsync(src, env)
+      srcDir,
+      src,
+      file,
+      mdEnv!,
+      options,
+      logger
+    )
+    mdEnv!.src = expanded.src
+    mdEnv!.lineMap = new LineMap(expanded.segments, expanded.splicedLines)
+    return renderAsync(expanded.src, env)
   }
 
   if (options.rebaseRelativeUrls !== false) registerRebaseRules(md)
@@ -71,17 +95,36 @@ async function processIncludes(
   env: MarkdownEnv,
   options: Options,
   logger: Pick<Logger, 'warn'>,
-  ancestors: string[] = []
+  ancestors: string[] = [],
-): Promise<string> {
+  base: LineMapSegment[] = [{ start: 0, file, line: 0 }]
-  return replaceAsync(src, includeRE, async (...args: string[]) => {
+): Promise<Expanded> {
-    const [m, , rawOffset] = args
+  const matches = [...src.matchAll(includeRE)]
-    let [, m1] = args
+  if (!matches.length) return { src, segments: base, splicedLines: noSplices }
-    if (!m1.length) return m
+
-
+  const out = new MappedBuilder()
-    const fail = (message: string): string => {
+  let cursor = 0
+  let cursorLine = 0
+
+  const passthrough = (to: number) => {
+    if (to <= cursor) return
+    const text = src.slice(cursor, to)
+    const breaks = countLineBreaks(text)
+    out.append(text, sliceSegments(base, cursorLine, cursorLine + breaks + 1))
+    cursor = to
+    cursorLine += breaks
+  }
+
+  const expandInclude = async (
+    m: RegExpExecArray,
+    directiveLine: number
+  ): Promise<Expanded | undefined> => {
+    const directive = m[0]
+    let m1 = m[1]
+
+    const fail = (message: string): Expanded => {
       if (!options.silent) throw new Error(message)
       logger.warn(`${message} (in ${file})`)
-      return ''
+      return { src: '', segments: [], splicedLines: noSplices }
     }
 
     const range = rangeRE.exec(m1)
@@ -95,7 +138,9 @@ async function processIncludes(
 
     // leave circular includes unexpanded — only repeats along the ancestor
     // chain are cycles, the same file may still be included by siblings
-    if (includePath === file || ancestors.includes(includePath)) return m
+    if (includePath === file || ancestors.includes(includePath)) {
+      return undefined
+    }
 
     // record the dependency before reading it, so that creating a missing
     // file is picked up by the watcher
@@ -116,19 +161,36 @@ async function processIncludes(
     }
 
     // for markdown files, if a range is used without a region, the line
-    // numbers must account for the frontmatter, so it is kept; otherwise
+    // numbers must account for the frontmatter, so it is kept; otherwise it
-    // it is stripped before selecting content
+    // is stripped before selecting content, and the removed height is folded
+    // into the segments so they keep pointing at real editor lines
+    let fmOffset = 0
     if (path.extname(includePath) === '.md' && (region || !range)) {
-      content = matter(content, {}).content
+      const stripped = matter(content, {}).content
+      fmOffset = countLineBreaks(content) - countLineBreaks(stripped)
+      content = stripped
     }
 
     let lines = content.split('\n')
+    let childBase: LineMapSegment[] = [
+      { start: 0, file: includePath, line: fmOffset }
+    ]
 
     if (region) {
       const name = region[1]
       const regions = findRegions(lines, name)
       if (regions.length > 0) {
-        lines = regions.flatMap((r) => lines.slice(r.start, r.end))
+        const selected: string[] = []
+        childBase = []
+        for (const r of regions) {
+          childBase.push({
+            start: selected.length,
+            file: includePath,
+            line: fmOffset + r.start
+          })
+          selected.push(...lines.slice(r.start, r.end))
+        }
+        lines = selected
       } else {
         // no editor-style region matched — try heading anchors
         const section = findHeadingSection(md, content, includePath, name, {
@@ -140,6 +202,9 @@ async function processIncludes(
             `Include region or heading "${name}" not found in ${includePath}`
           )
         }
+        childBase = [
+          { start: 0, file: includePath, line: fmOffset + section.start }
+        ]
         lines = lines.slice(section.start, section.end)
       }
     }
@@ -152,11 +217,12 @@ async function processIncludes(
           `Include range ${range[0]} is out of bounds in ${includePath}`
         )
       }
+      childBase = sliceSegments(childBase, start - 1, end)
       lines = lines.slice(start - 1, end)
     }
 
     // recursively process includes in the content
-    const expanded = await processIncludes(
+    const child = await processIncludes(
       md,
       srcDir,
       lines.join('\n'),
@@ -164,22 +230,52 @@ async function processIncludes(
       env,
       options,
       logger,
-      [...ancestors, file]
+      [...ancestors, file],
+      childBase
     )
 
-    // wrap included markdown in markers driving the url rebasing at render
+    // wrap included markdown in blank lines so its blocks stay isolated from
-    // time; they are removed from the output by the html_block rule. Blank
+    // adjacent page content; directives that are not on a line of their own -
-    // lines keep them out of adjacent html blocks, and directives that are
+    // inline ones and those inside fences - are left unwrapped so the blanks
-    // not on a line of their own - inline ones and those inside fences -
+    // can't end up inside surrounding constructs. The blank lines belong to
-    // are left unwrapped so the markers can't end up in the output.
+    // the include directive's own line.
-    const offset = rawOffset as unknown as number
+    if (
-    return options.rebaseRelativeUrls !== false &&
       path.extname(includePath) === '.md' &&
-      isOwnLine(src, offset, m.length) &&
+      isOwnLine(src, m.index, directive.length) &&
-      !isInsideFence(src, offset)
+      !isInsideFence(src, m.index)
-      ? `<!-- @include-start: ${path.dirname(includePath)} -->\n\n${expanded}\n\n<!-- @include-end -->`
+    ) {
-      : expanded
+      const at = resolveInSegments(base, directiveLine)
-  })
+      const childLineCount = countLineBreaks(child.src) + 1
+      return {
+        src: `\n\n${child.src}\n\n`,
+        segments: [
+          { start: 0, ...at },
+          ...offsetSegments(child.segments, 2),
+          { start: 2 + childLineCount, ...at }
+        ],
+        splicedLines: new Set([...child.splicedLines].map((l) => l + 2))
+      }
+    }
+    return child
+  }
+
+  for (const m of matches) {
+    passthrough(m.index)
+    const expanded = m[1].length
+      ? await expandInclude(m, cursorLine)
+      : undefined
+    if (expanded === undefined) {
+      // left unexpanded — passes through with the surrounding text
+      passthrough(m.index + m[0].length)
+    } else {
+      out.append(expanded.src, expanded.segments, expanded.splicedLines)
+      cursor = m.index + m[0].length
+      cursorLine += countLineBreaks(m[0])
+    }
+  }
+  passthrough(src.length)
+
+  return out.build()
 }
 
 function findHeadingSection(
@@ -237,43 +333,29 @@ function isInsideFence(src: string, offset: number) {
 }
 
 function registerRebaseRules(md: MarkdownItAsync) {
-  const htmlBlock = md.renderer.rules.html_block!
-  md.renderer.rules.html_block = (tokens, idx, opts, env, self) => {
-    const token = tokens[idx]
-    if (!token.content.includes('<!-- @include-')) {
-      return htmlBlock(tokens, idx, opts, env, self)
-    }
-
-    // markers are emitted on their own lines, but an adjacent html block can
-    // still absorb them into its token, so they are matched anywhere in the
-    // content and removed from it
-    token.content = token.content.replace(rebaseMarkerRE, (_, dir?: string) => {
-      let stack = rebaseStacks.get(env)
-      if (!stack) rebaseStacks.set(env, (stack = []))
-      if (dir == null) stack.pop()
-      else stack.push(dir)
-      return ''
-    })
-
-    return token.content.trim() ? htmlBlock(tokens, idx, opts, env, self) : ''
-  }
-
   for (const rule of ['image', 'link_open'] as const) {
     const render =
       md.renderer.rules[rule] ??
       ((tokens, idx, opts, _env, self) => self.renderToken(tokens, idx, opts))
     md.renderer.rules[rule] = (tokens, idx, opts, env, self) => {
-      const dir = rebaseStacks.get(env)?.at(-1)
+      const token = tokens[idx]
-      const file = (env as MarkdownEnv).path
+      const mdEnv = env as MarkdownEnv
-      if (dir && file) {
+      const page = mdEnv.path
-        const token = tokens[idx]
+      const origin = mdEnv.realPath ?? mdEnv.path
+      // the physical file the construct was authored in, resolved through
+      // the line map — different from the page means it came from an include
+      const sourceFile: string | undefined = token.meta?.vpLoc?.file
+      if (page && sourceFile && sourceFile !== origin) {
         const attr = rule === 'image' ? 'src' : 'href'
         const url = token.attrGet(attr)
         // a destination resolved from `$frontmatter` belongs to the page the
         // frontmatter came from, not to the included file
         if (url?.[0] === '.' && !token.meta?.frontmatterDest) {
           const rebased = slash(
-            path.join(path.relative(path.dirname(file), dir), url)
+            path.join(
+              path.relative(path.dirname(page), path.dirname(sourceFile)),
+              url
+            )
           )
           token.attrSet(attr, rebased[0] === '.' ? rebased : `./${rebased}`)
         }

src/node/markdown/plugins/sourcePositions.ts

@@ -207,11 +207,16 @@ function sourceLocs(state: StateCore): void {
       if (!pos) continue
 
       const line = token.map[0] + pos.dLine
-      const loc = resolveLoc(env, line)
+      const resolved = env.lineMap?.resolve(line)
+      const loc: MarkdownSourceLoc = resolved
+        ? { file: resolved.file, line: resolved.line + 1 }
+        : { file: env.realPath ?? env.path, line: line + 1 }
 
       // block parsing only ever strips a prefix per line, so the inline line
-      // is a suffix of the raw source line; re-align to get the true column
+      // is a suffix of the raw source line; re-align to get the true column.
-      if (pos.startInLine >= 0) {
+      // A spliced line is stitched from more than one source, so no column
+      // on it is meaningful in any single file.
+      if (pos.startInLine >= 0 && !resolved?.spliced) {
         srcLineStarts ??= makeLineStarts(state.src)
         const raw = getLine(state.src, srcLineStarts, line)
         if (raw !== undefined) {
@@ -236,12 +241,6 @@ function sourceLocs(state: StateCore): void {
   }
 }
 
-function resolveLoc(env: MarkdownEnv, line: number): MarkdownSourceLoc {
-  const resolved = env.lineMap?.resolve(line)
-  if (resolved) return { file: resolved.file, line: resolved.line + 1 }
-  return { file: env.realPath ?? env.path, line: line + 1 }
-}
-
 function safeDecodeURI(str: string): string {
   try {
     return decodeURI(str)

types/shared.d.ts

@@ -657,5 +657,14 @@ export interface MarkdownSourceLoc {
  * the physical file and 0-based line they came from.
  */
 export interface MarkdownLineMap {
-  resolve(line: number): { file: string; line: number }
+  resolve(line: number): {
+    file: string
+    line: number
+    /**
+     * Set when the line was stitched together from more than one source
+     * (a mid-line include splice) — column positions on it are not
+     * meaningful in any single file.
+     */
+    spliced?: boolean
+  }
 }