feat(markdown): add region marker engine

Shared engine for VS Code-style region extraction, used by the snippet
and include rewrites. Compared to findRegion in the snippet plugin, it
returns all same-named regions in document order and matches marker
styles per line instead of locking onto the first style found, so
same-named regions in mixed-comment files (e.g. Vue SFCs) merge. All
open regions are tracked, and an end marker without a name closes the
innermost region opened in its own comment style, so it neither gets
captured by a region of another language nested inside it nor closes
one that was left open there.

The marker regexes follow the folding.markers definitions VS Code ships
per language, which corrects two of the previous ones: REM is
case-insensitive in bat, and #pragma allows a space after the hash.
Those definitions disagree about whether the hash is required - the
marker makes it optional for js and markdown, while html, css, sql, bat
and f# require it - so related comment styles are merged into one regex
each, keeping the hash optional only where at least one of the merged
languages makes it optional. Since a markdown renderer cannot know the
language of an imported file, all styles are tried on every file, which
makes this a superset of what an editor folds. Note that a language
service can disagree with the marker its language ships: TypeScript
requires the hash the ts/js marker makes optional, while the Lua one
accepts it hash-less where we require it. Fixtures covering the marker
forms per language live in brc-dd/region-marker-fixtures.

Quoted Visual Basic region names are matched without their quotes, and
region markers declared as JSON keys are recognized as an extension,
plain JSON having no comments to put a marker in.

Co-authored-by: Miroma <its.miroma@proton.me>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
pull/5307/merge
Divyansh Singh 2 days ago
parent 5a3830c4b7
commit b303dd341d

@ -0,0 +1,529 @@
import {
dedent,
findRegions,
markers,
stripRegionMarkers
} from 'node/markdown/regions'
const extract = (lines: string[], name: string) =>
findRegions(lines, name)
.flatMap((r) => lines.slice(r.start, r.end))
.join('\n')
describe('node/markdown/regions', () => {
describe('dedent', () => {
test('keeps lines when 0-level is minimal', () => {
expect(dedent(['fn main() {', ' println!("Hello");', '}'])).toEqual([
'fn main() {',
' println!("Hello");',
'}'
])
})
test('removes the common minimal indent', () => {
expect(dedent([' let a = {', ' value: 42', ' };'])).toEqual([
'let a = {',
' value: 42',
'};'
])
})
test('dedents a single line', () => {
expect(dedent([' let a = 42;'])).toEqual(['let a = 42;'])
})
test('handles tabs', () => {
expect(dedent(['\tlet a = {', '\t\tvalue: 42', '\t};'])).toEqual([
'let a = {',
'\tvalue: 42',
'};'
])
})
test('ignores blank lines when computing the minimal indent', () => {
expect(dedent([' a', '', ' b'])).toEqual(['a', '', 'b'])
})
test('keeps whitespace-only input as is', () => {
expect(dedent(['', ' '])).toEqual(['', ' '])
})
})
describe('findRegions', () => {
it('returns no regions without markers', () => {
const lines = ['function foo() {', ' console.log("hello");', '}']
expect(findRegions(lines, 'foo')).toHaveLength(0)
})
it('ignores non-matching and prefix-matching region names', () => {
const lines = [
'// #region regionA',
'some code here',
'// #endregion regionA'
]
expect(findRegions(lines, 'regionC')).toHaveLength(0)
expect(findRegions(lines, 'region')).toHaveLength(0)
})
it('returns no regions for a start marker without a matching end', () => {
const lines = [
'// #region missingEnd',
'console.log("inside region");',
'console.log("still inside");'
]
expect(findRegions(lines, 'missingEnd')).toHaveLength(0)
})
it('returns no regions for an end marker without a preceding start', () => {
const lines = [
'// #endregion ghostRegion',
'console.log("stray end marker");'
]
expect(findRegions(lines, 'ghostRegion')).toHaveLength(0)
})
it('ignores non-marker lines containing the word region', () => {
const lines = [
'const region = "region"',
'// #region hello',
'const x = 1',
'// endregion hello is mentioned here without a comment prefix'
]
expect(findRegions(lines, 'hello')).toHaveLength(0)
})
it('detects C#-style markers', () => {
const lines = [
'Console.WriteLine("Before region");',
'#region hello',
'Console.WriteLine("Hello, World!");',
'#endregion hello',
'Console.WriteLine("After region");'
]
expect(extract(lines, 'hello')).toBe(
'Console.WriteLine("Hello, World!");'
)
})
it('closes a named region with an anonymous end marker', () => {
const lines = [
'#region hello',
'Console.WriteLine("Hello, World!");',
'#endregion',
'Console.WriteLine("After region");'
]
expect(extract(lines, 'hello')).toBe(
'Console.WriteLine("Hello, World!");'
)
})
it('does not close a region with a differently named end marker', () => {
const lines = [
'#region hello',
'Console.WriteLine("Hello, World!");',
'#endregion world'
]
expect(findRegions(lines, 'hello')).toHaveLength(0)
})
it('keeps indentation of indented markers and content', () => {
const lines = [
' #region hello',
' Console.WriteLine("Hello, World!");',
' #endregion hello'
]
expect(extract(lines, 'hello')).toBe(
' Console.WriteLine("Hello, World!");'
)
})
it('detects double-slash markers with and without spacing', () => {
const lines = [
'let regexp: RegExp[] = [];',
'// #region foo',
'let start = -1;',
'//#endregion foo'
]
expect(extract(lines, 'foo')).toBe('let start = -1;')
})
it('detects hash-less double-slash markers like VS Code', () => {
const lines = ['// region foo', 'let start = -1;', '// endregion foo']
expect(extract(lines, 'foo')).toBe('let start = -1;')
})
it('detects CSS-style markers', () => {
const lines = [
'/* #region foo */',
' padding-left: 15px;',
'/*#endregion foo*/'
]
expect(extract(lines, 'foo')).toBe(' padding-left: 15px;')
})
it('detects HTML-style markers, with the hash optional', () => {
const lines = [
'<!-- #region foo -->',
' <h1>Hello world</h1>',
'<!--#endregion foo-->',
'<!-- region bar -->',
' <h2>Other</h2>',
'<!-- endregion bar -->'
]
expect(extract(lines, 'foo')).toBe(' <h1>Hello world</h1>')
expect(extract(lines, 'bar')).toBe(' <h2>Other</h2>')
})
it('detects Visual Basic-style markers', () => {
const lines = [
'#Region VBRegion',
' Console.WriteLine("Inside region")',
'#End Region VBRegion'
]
expect(extract(lines, 'VBRegion')).toBe(
' Console.WriteLine("Inside region")'
)
})
it('ignores the quotes around a Visual Basic region name', () => {
const lines = [
'#Region "Quoted Name"',
' Console.WriteLine("Inside region")',
'#End Region',
'#Region "Other"',
' Console.WriteLine("Other region")',
'#End Region "Other"'
]
expect(extract(lines, 'Quoted Name')).toBe(
' Console.WriteLine("Inside region")'
)
expect(extract(lines, 'Other')).toBe(
' Console.WriteLine("Other region")'
)
})
it('detects bat-style markers with case-insensitive REM', () => {
const lines = [
'@REM #region hello',
'@ECHO OFF',
'::#endregion hello',
'echo out',
'rem #region hello',
'exit 0',
'Rem #endregion hello'
]
expect(extract(lines, 'hello')).toBe('@ECHO OFF\nexit 0')
})
it('detects dash-dash markers, which require the hash', () => {
const lines = [
'-- #region foo',
'select 1;',
'-- #endregion foo',
'--#region bar',
'select 2;',
'--#endregion bar'
]
expect(extract(lines, 'foo')).toBe('select 1;')
expect(extract(lines, 'bar')).toBe('select 2;')
// prose comments are not markers
expect(findRegions(['-- region of interest', 'select 1;'], '')).toEqual(
[]
)
})
it('detects pragma markers, allowing space after the hash', () => {
const lines = [
'#pragma region foo',
'int a = 1;',
'#pragma endregion foo',
'# pragma region bar',
'int b = 2;',
'# pragma endregion bar'
]
expect(extract(lines, 'foo')).toBe('int a = 1;')
expect(extract(lines, 'bar')).toBe('int b = 2;')
})
it('detects paren-star markers', () => {
const lines = ['(* #region foo *)', 'let a = 1', '(* #endregion foo *)']
expect(extract(lines, 'foo')).toBe('let a = 1')
})
it('detects shell and python style hash markers', () => {
const lines = [
'# region hello',
'echo "inside"',
'#\tendregion hello',
'# #region hello',
'exit 0',
'# #endregion'
]
expect(extract(lines, 'hello')).toBe('echo "inside"\nexit 0')
})
it('detects JSON key-style markers with two or more slashes', () => {
const lines = [
'{',
' "// #region hello": "",',
' "one": true,',
' "//#endregion hello": "",',
' "two": false,',
' "/// #region hello": "",',
' "three": true,',
' "//// #endregion hello": ""',
'}'
]
expect(extract(lines, 'hello')).toBe(' "one": true,\n "three": true,')
})
it('concatenates multiple same-named regions in document order', () => {
const lines = [
'// #region hello',
'first region content',
'// #endregion hello',
'other content',
'// #region hello',
'second region content',
'// #endregion',
'// #region hello',
'third region content',
'// #endregion hello'
]
expect(extract(lines, 'hello')).toBe(
'first region content\nsecond region content\nthird region content'
)
})
it('merges same-named regions across different comment styles', () => {
const lines = [
'<template>',
' <!-- #region shared -->',
' <div>template part</div>',
' <!-- #endregion shared -->',
'</template>',
'<script>',
'// #region shared',
'const scriptPart = true',
'// #endregion shared',
'/* #region shared */',
'console.log(scriptPart)',
'/* #endregion shared */',
'</script>',
'<style>',
'/* #region shared */',
'.style-part {}',
'/* #endregion shared */',
'</style>'
]
const regions = findRegions(lines, 'shared')
expect(regions).toHaveLength(4)
expect(extract(lines, 'shared')).toBe(
[
' <div>template part</div>',
'const scriptPart = true',
'console.log(scriptPart)',
'.style-part {}'
].join('\n')
)
})
it('tracks nesting of same-named regions across styles', () => {
const lines = [
'// #region foo',
"console.log('double-slash only');",
'/* #region foo */',
"console.log('nested in both');",
'// #endregion foo',
"console.log('still in outer');",
'/* #endregion foo */',
"console.log('outside');"
]
const regions = findRegions(lines, 'foo')
expect(regions).toHaveLength(1)
expect(regions[0]).toMatchObject({ start: 1, end: 6 })
})
it('closes the innermost open region with an anonymous end marker', () => {
const lines = [
'<!-- #region demo -->',
'<template><div /></template>',
'<script setup>',
'// #region state',
'const count = ref(0)',
'// #endregion',
'function inc() {}',
'</script>',
'<!-- #endregion demo -->'
]
expect(extract(lines, 'demo')).toBe(
[
'<template><div /></template>',
'<script setup>',
'// #region state',
'const count = ref(0)',
'// #endregion',
'function inc() {}',
'</script>'
].join('\n')
)
expect(extract(lines, 'state')).toBe('const count = ref(0)')
})
it('is not closed by an anonymous end marker of a nested region in a fence', () => {
const lines = [
'<!-- #region sample -->',
'Use markers like this:',
'',
'```js',
'// #region foo',
'const a = 1',
'// #endregion',
'```',
'<!-- #endregion sample -->'
]
expect(extract(lines, 'sample')).toBe(
[
'Use markers like this:',
'',
'```js',
'// #region foo',
'const a = 1',
'// #endregion',
'```'
].join('\n')
)
})
it('is closed by an anonymous end marker of its own style', () => {
// a region documenting region syntax: the marker inside the fence is
// never closed, and the anonymous end marker belongs to the region
const lines = [
'<!-- #region real -->',
'How regions work:',
'',
'```js',
'// #region example',
'const a = 1',
'```',
'<!-- #endregion -->'
]
expect(extract(lines, 'real')).toBe(
[
'How regions work:',
'',
'```js',
'// #region example',
'const a = 1',
'```'
].join('\n')
)
})
it('ignores an anonymous end marker with nothing open in its style', () => {
const lines = [
'<!-- #region real -->',
'// #endregion',
'body',
'<!-- #endregion -->'
]
expect(extract(lines, 'real')).toBe('// #endregion\nbody')
})
it('tolerates an unclosed region nested inside the requested one', () => {
const lines = [
'// #region outer',
'const a = 1',
'// #region inner',
'const b = 2',
'// #endregion outer'
]
expect(extract(lines, 'outer')).toBe(
['const a = 1', '// #region inner', 'const b = 2'].join('\n')
)
})
it('keeps differently named nested regions verbatim', () => {
const lines = [
'// #region foo',
"console.log('line before nested');",
'// #region bar',
"console.log('nested content');",
'// #endregion bar',
'// #endregion foo'
]
expect(extract(lines, 'foo')).toBe(
[
"console.log('line before nested');",
'// #region bar',
"console.log('nested content');",
'// #endregion bar'
].join('\n')
)
})
it('supports special characters in region names', () => {
const lines = [
'// #region complex-name_123',
'const x = 1;',
'// #endregion complex-name_123',
'// #region my.region',
'const y = 2;',
'// #endregion my.region'
]
expect(extract(lines, 'complex-name_123')).toBe('const x = 1;')
expect(extract(lines, 'my.region')).toBe('const y = 2;')
})
it('reports the marker style that opened each region', () => {
const lines = ['// #region foo', 'const x = 1;', '// #endregion foo']
const [region] = findRegions(lines, 'foo')
expect(markers).toContain(region.marker)
expect(region.marker.start.test('// #region other')).toBe(true)
expect(region.marker.start.test('# region other')).toBe(false)
})
})
describe('stripRegionMarkers', () => {
it('strips marker lines of every style and name by default', () => {
const lines = [
'// #region name',
'const a = 0;',
'/* #region HELLO */',
'const b = 0;',
'//\t#endregion complex_name-123',
'const c = 0;',
'/*#endregion*/'
]
expect(stripRegionMarkers(lines)).toEqual([
'const a = 0;',
'const b = 0;',
'const c = 0;'
])
})
it('keeps non-marker lines mentioning regions', () => {
const lines = ['const region = "region"', 'let a = 1']
expect(stripRegionMarkers(lines)).toEqual(lines)
})
it('strips only the given marker styles when provided', () => {
const lines = [
'// #region a',
'const x = 1',
'// #endregion a',
'# region b',
'const y = 2',
'# endregion b'
]
const [region] = findRegions(lines, 'a')
expect(stripRegionMarkers(lines, [region.marker])).toEqual([
'const x = 1',
'# region b',
'const y = 2',
'# endregion b'
])
})
})
})

@ -0,0 +1,159 @@
export interface RegionMarker {
start: RegExp
end: RegExp
}
export interface Region {
start: number
end: number
marker: RegionMarker
}
// cheap pre-filter so the marker regexes only run on candidate lines
const maybeMarkerRE = /region/i
const quotedRE = /^"(.*)"$/
// visual basic names its regions `#Region "Name"`, so the quotes are part of
// the captured name and have to be dropped to make it referenceable
function unquote(name: string) {
return quotedRE.exec(name)?.[1] ?? name
}
/**
* Region marker styles, derived from the `folding.markers` definitions VS Code
* ships per language and merged per comment syntax, keeping the hash optional
* where at least one of the merged languages makes it optional. Since a
* markdown renderer cannot know the language of an imported file, every style
* is tried on every file, which makes this a superset: everything an editor
* folds is extracted, but not the reverse. Note that a language service can
* be stricter than the marker it ships - TypeScript, for one, requires the
* hash that the ts/js marker makes optional.
*/
export const markers: RegionMarker[] = [
// line comments: js, ts, go, rust, java and json with comments, whose
// markers make the hash optional, plus sql and bat, which require it
{
start: /^\s*(?:\/\/\s*#?|(?:--|::|@?[rR][eE][mM])\s*#)region\b\s*(.*?)\s*$/,
end: /^\s*(?:\/\/\s*#?|(?:--|::|@?[rR][eE][mM])\s*#)endregion\b\s*(.*?)\s*$/
},
// hash comments: c# and coffeescript (`#region`), python, yaml and shell
// (`# region`, and `# #region` in shell), visual basic (`#Region` closed by
// `#End Region`), powershell (`#EndRegion`) and c/c++ (`#pragma region`)
{
start: /^\s*#\s*(?:#\s*|pragma\s+)?[rR]egion\b\s*(.*?)\s*$/,
end: /^\s*#\s*(?:#\s*|pragma\s+)?[eE]nd ?[rR]egion\b\s*(.*?)\s*$/
},
// markdown (hash optional) and html (hash required), vue templates
{
start: /^\s*<!--\s*#?region\b\s*(.*?)\s*-->/,
end: /^\s*<!--\s*#?endregion\b\s*(.*?)\s*-->/
},
// css, less and scss
{
start: /^\s*\/\*\s*#region\b\s*(.*?)\s*\*\//,
end: /^\s*\/\*\s*#endregion\b\s*(.*?)\s*\*\//
},
// f# block comments; its `// #region` form is covered by the line comments
// above
{
start: /^\s*\(\*\s*#region\b\s*(.*?)\s*\*\)/,
end: /^\s*\(\*\s*#endregion\b\s*(.*?)\s*\*\)/
},
// json keys, e.g. `"// #region name": "",` - VS Code cannot fold these,
// since plain json has no comments to put a marker in
{
start: /^\s*"\/{2,}\s*#region\b\s*(.*?)":\s*"",?\s*$/,
end: /^\s*"\/{2,}\s*#endregion\b\s*(.*?)":\s*"",?\s*$/
}
]
/**
* Finds all regions with the given name, in document order. Matching is
* name-based across all marker styles, so a region opened by one comment
* style may be closed by another - useful in files mixing languages, like
* Vue SFCs. All regions are tracked, not just the requested ones, so that an
* end marker without a name closes the innermost open region rather than the
* requested one. Nested regions with the requested name yield the outermost
* span.
*/
export function findRegions(lines: string[], name: string): Region[] {
const regions: Region[] = []
const open: { name: string; start: number; marker: RegionMarker }[] = []
for (let i = 0; i < lines.length; i++) {
const line = lines[i]
if (!maybeMarkerRE.test(line)) continue
let isStart = false
for (const marker of markers) {
const startName = marker.start.exec(line)?.[1]
if (startName != null) {
open.push({ name: unquote(startName), start: i + 1, marker })
isStart = true
break
}
}
if (isStart || open.length === 0) continue
for (const marker of markers) {
const rawEndName = marker.end.exec(line)?.[1]
if (rawEndName == null) continue
const endName = unquote(rawEndName)
// a named end marker closes the innermost region it names, whichever
// comment style opened it, while an unnamed one closes the innermost
// region opened in its own comment style - so it can neither be
// captured by a region of another language nested inside, nor close
// one that was left open there
const index = endName
? open.findLastIndex((r) => r.name === endName)
: open.findLastIndex((r) => r.marker === marker)
if (index === -1) continue
const [closed] = open.splice(index, open.length - index)
if (closed.name === name && !open.some((r) => r.name === name)) {
regions.push({ start: closed.start, end: i, marker: closed.marker })
}
break
}
}
return regions
}
/**
* Removes region marker lines of the given styles (all styles by default),
* regardless of region name.
*/
export function stripRegionMarkers(
lines: string[],
styles: RegionMarker[] = markers
): string[] {
return lines.filter(
(line) =>
!maybeMarkerRE.test(line) ||
!styles.some((m) => m.start.test(line) || m.end.test(line))
)
}
/**
* Removes the common minimal indentation (spaces and tabs counted per
* character) from the given lines. Whitespace-only lines don't constrain
* the minimum.
*/
export function dedent(lines: string[]): string[] {
let minIndent = Infinity
for (const line of lines) {
for (let i = 0; i < line.length; i++) {
if (line[i] !== ' ' && line[i] !== '\t') {
minIndent = Math.min(i, minIndent)
break
}
}
if (minIndent === 0) break
}
if (minIndent === Infinity || minIndent === 0) return lines
return lines.map((line) => line.slice(minIndent))
}
Loading…
Cancel
Save