add source map support for preprocessors

pull/5428/head
halfnelson 5 years ago committed by Milan Hauth
parent b5b02f8561
commit e223c35f1a

28
package-lock.json generated

@ -4,6 +4,16 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@ampproject/remapping": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-0.3.0.tgz",
"integrity": "sha512-dqmASpaTCavldZqwdEpokgG4yOXmEiEGPP3ATTsBbdXXSKf6kx8jt2fPcKhodABdZlYe82OehR2oFK1y9gwZxw==",
"dev": true,
"requires": {
"@jridgewell/resolve-uri": "1.0.0",
"sourcemap-codec": "1.4.8"
}
},
"@babel/code-frame": {
"version": "7.10.1",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.1.tgz",
@ -36,6 +46,12 @@
"integrity": "sha512-KioOCsSvSvXx6xUNLiJz+P+VMb7NRcePjoefOr74Y5P6lEKsiOn35eZyZzgpK4XCNJdXTDR7+zykj0lwxRvZ2g==",
"dev": true
},
"@jridgewell/resolve-uri": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-1.0.0.tgz",
"integrity": "sha512-9oLAnygRMi8Q5QkYEU4XWK04B+nuoXoxjRvRxgjuChkLZFBja0YPSgdZ7dZtwhncLBcQe/I/E+fLuk5qxcYVJA==",
"dev": true
},
"@rollup/plugin-commonjs": {
"version": "11.0.0",
"resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-11.0.0.tgz",
@ -2645,9 +2661,9 @@
}
},
"magic-string": {
"version": "0.25.3",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.3.tgz",
"integrity": "sha512-6QK0OpF/phMz0Q2AxILkX2mFhi7m+WMwTRg0LQKq/WBB0cDP4rYH3Wp4/d3OTXlrPLVJT/RFqj8tFeAR4nk8AA==",
"version": "0.25.7",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz",
"integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==",
"dev": true,
"requires": {
"sourcemap-codec": "^1.4.4"
@ -3737,9 +3753,9 @@
}
},
"sourcemap-codec": {
"version": "1.4.6",
"resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.6.tgz",
"integrity": "sha512-1ZooVLYFxC448piVLBbtOxFcXwnymH9oUF8nRd3CuYDVvkRBxRl6pB4Mtas5a4drtL+E8LDgFkQNcgIw6tc8Hg==",
"version": "1.4.8",
"resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz",
"integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==",
"dev": true
},
"spdx-correct": {

@ -56,6 +56,7 @@
},
"homepage": "https://github.com/sveltejs/svelte#README",
"devDependencies": {
"@ampproject/remapping": "^0.3.0",
"@rollup/plugin-commonjs": "^11.0.0",
"@rollup/plugin-json": "^4.0.1",
"@rollup/plugin-node-resolve": "^6.0.0",
@ -91,7 +92,8 @@
"source-map-support": "^0.5.13",
"tiny-glob": "^0.2.6",
"tslib": "^1.10.0",
"typescript": "^3.5.3"
"typescript": "^3.5.3",
"sourcemap-codec": "^1.4.8"
},
"nyc": {
"include": [

@ -29,6 +29,7 @@ import add_to_set from './utils/add_to_set';
import check_graph_for_cycles from './utils/check_graph_for_cycles';
import { print, x, b } from 'code-red';
import { is_reserved_keyword } from './utils/reserved_keywords';
import remapping from '@ampproject/remapping';
interface ComponentOptions {
namespace?: string;
@ -324,6 +325,35 @@ export default class Component {
js.map.sourcesContent = [
this.source
];
if (compile_options.sourceMap) {
if (js.map) {
const pre_remap_sources = js.map.sources;
js.map = remapping([js.map, compile_options.sourceMap], () => null);
// remapper can remove our source if it isn't used (no segments map back to it). It is still handy to have a source
// so we add it back
if (js.map.sources && js.map.sources.length == 0) {
js.map.sources = pre_remap_sources;
}
Object.defineProperties(js.map, {
toString: {
enumerable: false,
value: function toString() {
return JSON.stringify(this);
}
},
toUrl: {
enumerable: false,
value: function toUrl() {
return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString());
}
}
});
}
if (css.map) {
css.map = remapping([css.map, compile_options.sourceMap], () => null);
}
}
}
return {

@ -12,6 +12,7 @@ const valid_options = [
'format',
'name',
'filename',
'sourceMap',
'generate',
'outputFilename',
'cssOutputFilename',

@ -110,6 +110,7 @@ export interface CompileOptions {
filename?: string;
generate?: 'dom' | 'ssr' | false;
sourceMap?: object | string;
outputFilename?: string;
cssOutputFilename?: string;
sveltePath?: string;

@ -1,3 +1,9 @@
import remapper from '@ampproject/remapping';
import { decode } from 'sourcemap-codec';
import { getLocator } from 'locate-character';
import { GeneratedStringWithMap, offset_source_location } from '../utils/string_with_map';
export interface Processed {
code: string;
map?: object | string;
@ -37,34 +43,7 @@ function parse_attributes(str: string) {
interface Replacement {
offset: number;
length: number;
replacement: string;
}
async function replace_async(str: string, re: RegExp, func: (...any) => Promise<string>) {
const replacements: Array<Promise<Replacement>> = [];
str.replace(re, (...args) => {
replacements.push(
func(...args).then(
res =>
({
offset: args[args.length - 2],
length: args[0].length,
replacement: res
}) as Replacement
)
);
return '';
});
let out = '';
let last_end = 0;
for (const { offset, length, replacement } of await Promise.all(
replacements
)) {
out += str.slice(last_end, offset) + replacement;
last_end = offset + length;
}
out += str.slice(last_end);
return out;
replacement: GeneratedStringWithMap;
}
export default async function preprocess(
@ -81,7 +60,58 @@ export default async function preprocess(
const markup = preprocessors.map(p => p.markup).filter(Boolean);
const script = preprocessors.map(p => p.script).filter(Boolean);
const style = preprocessors.map(p => p.style).filter(Boolean);
const source_maps: Array<Processed['map']> = [];
let source_locator: ReturnType<typeof getLocator>;
function get_replacement(offset: number, original: string, processed: Processed, prefix: string, suffix: string): GeneratedStringWithMap {
const generated_prefix = GeneratedStringWithMap.from_source(filename, prefix, source_locator(offset));
const generated_suffix = GeneratedStringWithMap.from_source(filename, suffix, source_locator(offset + prefix.length + original.length));
let generated;
if (processed.map) {
const full_map = typeof processed.map === "string" ? JSON.parse(processed.map) : processed.map;
const decoded_map = { ...full_map, mappings: decode(full_map.mappings) };
const processed_offset = source_locator(offset + prefix.length);
generated = GeneratedStringWithMap.from_generated(processed.code, offset_source_location(processed_offset, decoded_map));
} else {
generated = GeneratedStringWithMap.from_generated(processed.code);
}
const map = generated_prefix.concat(generated).concat(generated_suffix);
return map;
}
async function replace_async(str: string, re: RegExp, func: (...any) => Promise<GeneratedStringWithMap>): Promise<GeneratedStringWithMap> {
const replacement_promises: Array<Promise<Replacement>> = [];
str.replace(re, (...args) => {
replacement_promises.push(
func(...args).then(
(replacement) =>
({
offset: args[args.length - 2],
length: args[0].length,
replacement
}) as Replacement
)
);
return '';
});
const replacements = await Promise.all(replacement_promises);
let out: GeneratedStringWithMap;
let last_end = 0;
for (const { offset, length, replacement } of replacements)
{
const content = GeneratedStringWithMap.from_source(filename, str.slice(last_end, offset), source_locator(last_end));
out = out ? out.concat(content) : content;
out = out.concat(replacement);
last_end = offset + length;
}
const final_content = GeneratedStringWithMap.from_source(filename, str.slice(last_end), source_locator(last_end));
out = out.concat(final_content);
return out;
}
for (const fn of markup) {
const processed = await fn({
content: source,
@ -89,47 +119,65 @@ export default async function preprocess(
});
if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
source = processed ? processed.code : source;
if (processed && processed.map) source_maps.unshift(processed.map);
}
for (const fn of script) {
source = await replace_async(
source_locator = getLocator(source);
const res = await replace_async(
source,
/<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi,
async (match, attributes = '', content = '') => {
async (match, attributes = '', content, offset) => {
const no_change = () => GeneratedStringWithMap.from_source(filename, match, source_locator(offset));
if (!attributes && !content) {
return match;
return no_change();
}
attributes = attributes || '';
const processed = await fn({
content,
attributes: parse_attributes(attributes),
filename
});
if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
return processed ? `<script${attributes}>${processed.code}</script>` : match;
if (!processed) return no_change();
if (processed.dependencies) dependencies.push(...processed.dependencies);
return get_replacement(offset, content, processed, `<script${attributes}>`, `</script>`);
}
);
source = res.generated;
source_maps.unshift(res.as_sourcemap());
}
for (const fn of style) {
source = await replace_async(
source_locator = getLocator(source);
const res = await replace_async(
source,
/<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi,
async (match, attributes = '', content = '') => {
/<!--[^]*?-->|<style(\s[^]*?)?>([^]*?)<\/style>/gi,
async (match, attributes = '', content, offset) => {
const no_change = () => GeneratedStringWithMap.from_source(filename, match, source_locator(offset));
if (!attributes && !content) {
return match;
return no_change();
}
const processed: Processed = await fn({
content,
attributes: parse_attributes(attributes),
filename
});
if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
return processed ? `<style${attributes}>${processed.code}</style>` : match;
if (!processed) return no_change();
if (processed.dependencies) dependencies.push(...processed.dependencies);
return get_replacement(offset, content, processed, `<style${attributes}>`, `</style>`);
}
);
source = res.generated;
source_maps.unshift(res.as_sourcemap());
}
const map: ReturnType<typeof remapper> = source_maps.length == 0 ? null : remapper(source_maps as any, () => null);
return {
// TODO return separated output, in future version where svelte.compile supports it:
// style: { code: styleCode, map: styleMap },
@ -138,7 +186,7 @@ export default async function preprocess(
code: source,
dependencies: [...new Set(dependencies)],
map,
toString() {
return source;
}

@ -0,0 +1,189 @@
import { encode } from "sourcemap-codec";
type MappingSegment = [ number ] | [ number, number, number, number ] | [ number, number, number, number, number ]
type SourceMappings = {
sources: string[];
names: string[];
mappings: MappingSegment[][];
}
type SourceLocation = {
line: number;
column: number;
}
function get_end_location(s: string): SourceLocation {
const parts = s.split('\n');
return {
line: parts.length - 1,
column: parts[parts.length - 1].length - 1
};
}
export function offset_source_location(offset: SourceLocation, map: SourceMappings): SourceMappings {
const new_mappings = map.mappings.map(line => line.map(seg => {
if (seg.length < 3) return seg;
const new_seg = seg.slice() as MappingSegment;
new_seg[2] = new_seg[2] + offset.line;
return new_seg;
}));
// first line has column altered
if (new_mappings.length > 0) {
new_mappings[0] = new_mappings[0].map(seg => {
if (seg.length < 4) return seg;
const newSeg = seg.slice() as MappingSegment;
newSeg[3] = newSeg[3] + offset.column;
return newSeg;
});
}
return {
sources: map.sources,
mappings: new_mappings
} as SourceMappings;
}
function merge_tables<T>( original: T[], extended: T[]): { table: T[]; new_idx: number[] } {
const table = original.slice();
const new_idx = [];
for (let j = 0; j < original.length; j++) {
const current = extended[j];
const existing = table.indexOf(current);
if (existing < 0) {
table.push(current);
new_idx[j] = table.length - 1;
} else {
new_idx[j] = existing;
}
}
return { table, new_idx };
}
export class GeneratedStringWithMap {
readonly generated: string;
readonly map: SourceMappings;
constructor(generated: string , map: SourceMappings) {
this.generated = generated;
this.map = map;
}
as_sourcemap() {
return {
version: 3,
sources: this.map.sources,
names: [],
mappings: encode(this.map.mappings as any)
};
}
concat(other: GeneratedStringWithMap): GeneratedStringWithMap {
// if one is empty, return the other
if (this.generated.length == 0) return other;
if (other.generated.length == 0) return this;
//combine sources
const { table: new_sources, new_idx: other_source_idx } = merge_tables(this.map.sources, other.map.sources);
const { table: new_names, new_idx: other_name_idx } = merge_tables(this.map.names, other.map.names);
//update source and name references in segments
const other_mappings = other.map.mappings.map(line => line.map(seg => {
//to reduce allocations, we only return a new segment if a value has changed
if (
(seg.length > 1 && other_source_idx[seg[1]] != seg[1]) // has source idx that has been updated
|| (seg.length == 5 && other_name_idx[seg[4]] != seg[4])) // has name idx that has been updated
{
const new_seg = seg.slice() as MappingSegment;
new_seg[1] = other_source_idx[seg[1]];
if (seg.length == 5) {
new_seg[4] = other_name_idx[seg[4]];
}
return new_seg;
} else {
return seg;
}
}));
//combine the mappings
let new_mappings = this.map.mappings.slice();
//shift the first line of the second mapping by the number of columns in the last line of the first
const end = get_end_location(this.generated);
const col_offset = end.column + 1;
const first_line = other_mappings.length == 0 ? [] : other_mappings[0].map(seg => {
const new_seg = seg.slice() as MappingSegment;
new_seg[0] = seg[0] + col_offset;
return new_seg;
});
new_mappings[new_mappings.length - 1] = new_mappings[new_mappings.length - 1].concat(first_line);
//the rest don't need modification and can just be appended
new_mappings = new_mappings.concat(other_mappings.slice(1) as MappingSegment[][]);
return new GeneratedStringWithMap(this.generated + other.generated, {
sources: new_sources,
names: new_names,
mappings: new_mappings
});
}
static from_generated(generated: string, map?: SourceMappings): GeneratedStringWithMap {
if (map) return new GeneratedStringWithMap(generated, map);
const replacement_map: SourceMappings = {
names: [],
sources: [],
mappings: []
};
if (generated.length == 0) return new GeneratedStringWithMap(generated, replacement_map);
// we generate a mapping where the source was overwritten by the generated
const end = get_end_location(generated);
for (let i = 0; i <= end.line; i++) {
replacement_map.mappings.push([]); // unmapped line
}
return new GeneratedStringWithMap(generated, replacement_map);
}
static from_source(source_file: string, source: string, offset_in_source?: SourceLocation): GeneratedStringWithMap {
const offset = offset_in_source || { line: 0, column: 0 };
const map: SourceMappings = {
names: [],
sources: [ source_file ],
mappings: []
};
if (source.length == 0) return new GeneratedStringWithMap(source, map);
// we create a high resolution identity map here, we know that it will eventually be
// merged with svelte's map, at which stage the resolution will decrease.
const lines = source.split('\n');
let pos = 0;
const identity_map = lines.map((line, line_idx) => {
const segs = line.split(/([^\d\w\s]|\s+)/g).filter(x => x !== "").map(s => {
const seg: MappingSegment = [pos, 0, offset.line + line_idx, pos + (line_idx == 0 ? offset.column : 0)];
pos = pos + s.length;
return seg;
});
pos = 0;
return segs;
});
map.mappings = identity_map;
return new GeneratedStringWithMap(source, map);
}
}

@ -19,6 +19,9 @@ describe('preprocess', () => {
const result = await svelte.preprocess(input, config.preprocess);
fs.writeFileSync(`${__dirname}/samples/${dir}/_actual.html`, result.code);
if (result.map) {
fs.writeFileSync(`${__dirname}/samples/${dir}/_actual.html.map`, JSON.stringify(result.map, null, 2));
}
assert.equal(result.code, expected);

@ -25,9 +25,24 @@ describe("sourcemaps", () => {
`${__dirname}/samples/${dir}/output`
);
const preprocessorFilename = path.resolve(
`${__dirname}/samples/${dir}/_preprocessor.js`
)
const input = fs.readFileSync(filename, "utf-8").replace(/\s+$/, "");
const { js, css } = svelte.compile(input, {
let processed_input = input;
let processed_map = null;
if (fs.existsSync(preprocessorFilename)) {
let { preprocessors } = require(preprocessorFilename);
if (preprocessors.length > 0) {
({ code: processed_input, map: processed_map } = await svelte.preprocess(input, preprocessors, { filename: 'input.svelte' }));
}
}
const { js, css } = svelte.compile(processed_input, {
filename,
sourceMap: processed_map,
outputFilename: `${outputFilename}.js`,
cssOutputFilename: `${outputFilename}.css`
});
@ -55,6 +70,7 @@ describe("sourcemaps", () => {
}
assert.deepEqual(js.map.sources, ["input.svelte"]);
if (css.map) assert.deepEqual(css.map.sources, ["input.svelte"]);
const { test } = require(`./samples/${dir}/test.js`);

@ -0,0 +1,16 @@
import MagicString from 'magic-string';
export const preprocessors = [{
markup: ({content, filename}) => {
const src = new MagicString(content);
const idx = content.indexOf("baritone");
src.overwrite(idx, idx+"baritone".length, "bar");
return {
code: src.toString(),
map: src.generateMap({
source: filename,
includeContent: false
})
};
}
}];

@ -0,0 +1,5 @@
<script>
export let foo;
</script>
{foo.baritone.baz}

@ -0,0 +1,32 @@
export function test({ assert, smc, locateInSource, locateInGenerated }) {
const expectedBar = locateInSource('baritone.baz');
const expectedBaz = locateInSource('.baz');
let start = locateInGenerated('bar.baz');
const actualbar = smc.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbar, {
source: 'input.svelte',
name: null,
line: expectedBar.line + 1,
column: expectedBar.column
});
start = locateInGenerated('.baz');
const actualbaz = smc.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbaz, {
source: 'input.svelte',
name: null,
line: expectedBaz.line + 1,
column: expectedBaz.column
});
}

@ -0,0 +1,51 @@
import MagicString from 'magic-string';
export const preprocessors = [{
markup: ({ content, filename }) => {
const src = new MagicString(content);
const idx = content.indexOf("baritone");
src.overwrite(idx, idx + "baritone".length, "bar");
const css_idx = content.indexOf("--bazitone");
src.overwrite(css_idx, css_idx + "--bazitone".length, "--baz");
return {
code: src.toString(),
map: src.generateMap({
source: filename,
hires: true,
includeContent: false
})
};
}
},
{
script: ({ content, filename }) => {
const src = new MagicString(content);
const idx = content.indexOf("bar");
src.prependLeft(idx, " ");
return {
code: src.toString(),
map: src.generateMap({
source: filename,
hires: true,
includeContent: false
})
};
}
},
{
style: ({ content, filename }) => {
const src = new MagicString(content);
const idx = content.indexOf("--baz");
src.prependLeft(idx, " ");
return {
code: src.toString(),
map: src.generateMap({
source: filename,
hires: true,
includeContent: false
})
};
}
}
];

@ -0,0 +1,9 @@
<script>
export let foo = { baritone: 5 }
</script>
<style>
h1 {
background-color: var(--bazitone);
}
</style>
<h1>multiple {foo}</h1>

@ -0,0 +1,32 @@
export function test({ assert, smc, smcCss, locateInSource, locateInGenerated, locateInGeneratedCss }) {
const expectedBar = locateInSource('baritone');
const expectedBaz = locateInSource('--bazitone');
let start = locateInGenerated('bar');
const actualbar = smc.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbar, {
source: 'input.svelte',
name: null,
line: expectedBar.line + 1,
column: expectedBar.column
});
start = locateInGeneratedCss('--baz');
const actualbaz = smcCss.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbaz, {
source: 'input.svelte',
name: null,
line: expectedBaz.line + 1,
column: expectedBaz.column
}, `couldn't find baz in css,\n gen:${JSON.stringify(start)}\n actual:${JSON.stringify(actualbaz)}\n expected:${JSON.stringify(expectedBaz)}`);
}

@ -0,0 +1,17 @@
import MagicString from 'magic-string';
export const preprocessors = [{
script: ({content, filename}) => {
const src = new MagicString(content);
const idx = content.indexOf("baritone");
src.overwrite(idx, idx+"baritone".length, "bar");
return {
code: src.toString(),
map: src.generateMap({
source: filename,
hires: true,
includeContent: false
})
};
}
}];

@ -0,0 +1,9 @@
<style>
h1 {
color: red;
}
</style>
<script>
export let foo = { baritone: { baz: 5 } }
</script>
<h1>{foo.bar.baz}</h1>

@ -0,0 +1,32 @@
export function test({ assert, smc, locateInSource, locateInGenerated }) {
const expectedBar = locateInSource('baritone:');
const expectedBaz = locateInSource('baz:');
let start = locateInGenerated('bar:');
const actualbar = smc.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbar, {
source: 'input.svelte',
name: null,
line: expectedBar.line + 1,
column: expectedBar.column
}, `couldn't find bar: in source` );
start = locateInGenerated('baz:');
const actualbaz = smc.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbaz, {
source: 'input.svelte',
name: null,
line: expectedBaz.line + 1,
column: expectedBaz.column
}, `couldn't find baz: in source` );
}

@ -0,0 +1,17 @@
import MagicString from 'magic-string';
export const preprocessors = [{
style: ({content, filename}) => {
const src = new MagicString(content);
const idx = content.indexOf("baritone");
src.overwrite(idx, idx+"baritone".length, "bar");
return {
code: src.toString(),
map: src.generateMap({
source: filename,
hires: true,
includeContent: false
})
};
}
}];

@ -0,0 +1,13 @@
<h1>Testing Styles</h1>
<h2>Testing Styles 2</h2>
<script>export const b = 2;</script>
<style>
h1 {
--baritone: red;
}
h2 {
--baz: blue;
}
</style>

@ -0,0 +1,32 @@
export function test({ assert, smcCss, locateInSource, locateInGeneratedCss }) {
const expectedBar = locateInSource('--baritone');
const expectedBaz = locateInSource('--baz');
let start = locateInGeneratedCss('--bar');
const actualbar = smcCss.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbar, {
source: 'input.svelte',
name: null,
line: expectedBar.line + 1,
column: expectedBar.column
}, `couldn't find bar in source` );
start = locateInGeneratedCss('--baz');
const actualbaz = smcCss.originalPositionFor({
line: start.line + 1,
column: start.column
});
assert.deepEqual(actualbaz, {
source: 'input.svelte',
name: null,
line: expectedBaz.line + 1,
column: expectedBaz.column
}, `couldn't find baz in source` );
}
Loading…
Cancel
Save