Fix various issues with preprocess source maps (#5754)

pull/5801/head
dmitrage 5 years ago committed by GitHub
parent 91376d285e
commit 68538c61eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -39,6 +39,10 @@ function parse_attributes(str: string) {
return attrs;
}
function get_file_basename(filename: string) {
return filename.split(/[/\\]/).pop();
}
interface Replacement {
offset: number;
length: number;
@ -46,7 +50,7 @@ interface Replacement {
}
async function replace_async(
filename: string,
file_basename: string,
source: string,
get_location: ReturnType<typeof getLocator>,
re: RegExp,
@ -73,13 +77,13 @@ async function replace_async(
)) {
// content = unchanged source characters before the replaced segment
const content = StringWithSourcemap.from_source(
filename, source.slice(last_end, offset), get_location(last_end));
file_basename, source.slice(last_end, offset), get_location(last_end));
out.concat(content).concat(replacement);
last_end = offset + length;
}
// final_content = unchanged source characters after last replaced segment
const final_content = StringWithSourcemap.from_source(
filename, source.slice(last_end), get_location(last_end));
file_basename, source.slice(last_end), get_location(last_end));
return out.concat(final_content);
}
@ -160,7 +164,7 @@ function decoded_sourcemap_from_generator(generator: any) {
* Convert a preprocessor output and its leading prefix and trailing suffix into StringWithSourceMap
*/
function get_replacement(
filename: string,
file_basename: string,
offset: number,
get_location: ReturnType<typeof getLocator>,
original: string,
@ -171,9 +175,9 @@ function get_replacement(
// Convert the unchanged prefix and suffix to StringWithSourcemap
const prefix_with_map = StringWithSourcemap.from_source(
filename, prefix, get_location(offset));
file_basename, prefix, get_location(offset));
const suffix_with_map = StringWithSourcemap.from_source(
filename, suffix, get_location(offset + prefix.length + original.length));
file_basename, suffix, get_location(offset + prefix.length + original.length));
// Convert the preprocessed code and its sourcemap to a StringWithSourcemap
let decoded_map: DecodedSourceMap;
@ -186,7 +190,11 @@ function get_replacement(
// import decoded sourcemap from mozilla/source-map/SourceMapGenerator
decoded_map = decoded_sourcemap_from_generator(decoded_map);
}
sourcemap_add_offset(decoded_map, get_location(offset + prefix.length));
// offset only segments pointing at original component source
const source_index = decoded_map.sources.indexOf(file_basename);
if (source_index !== -1) {
sourcemap_add_offset(decoded_map, get_location(offset + prefix.length), source_index);
}
}
const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map);
@ -203,6 +211,9 @@ export default async function preprocess(
const filename = (options && options.filename) || preprocessor.filename; // legacy
const dependencies = [];
// preprocess source must be relative to itself or equal null
const file_basename = filename == null ? null : get_file_basename(filename);
const preprocessors = preprocessor
? Array.isArray(preprocessor) ? preprocessor : [preprocessor]
: [];
@ -246,13 +257,13 @@ export default async function preprocess(
: /<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi;
const res = await replace_async(
filename,
file_basename,
source,
get_location,
tag_regex,
async (match, attributes = '', content = '', offset) => {
const no_change = () => StringWithSourcemap.from_source(
filename, match, get_location(offset));
file_basename, match, get_location(offset));
if (!attributes && !content) {
return no_change();
}
@ -265,10 +276,13 @@ export default async function preprocess(
attributes: parse_attributes(attributes),
filename
});
if (!processed) return no_change();
if (processed.dependencies) dependencies.push(...processed.dependencies);
return get_replacement(filename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`);
if (processed && processed.dependencies) {
dependencies.push(...processed.dependencies);
}
if (!processed || !processed.map && processed.code === content) {
return no_change();
}
return get_replacement(file_basename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`);
}
);
source = res.string;
@ -285,7 +299,7 @@ export default async function preprocess(
// Combine all the source maps for each preprocessor function into one
const map: RawSourceMap = combine_sourcemaps(
filename,
file_basename,
sourcemap_list
);

@ -13,21 +13,22 @@ function last_line_length(s: string) {
// mutate map in-place
export function sourcemap_add_offset(
map: DecodedSourceMap, offset: SourceLocation
map: DecodedSourceMap, offset: SourceLocation, source_index: number
) {
if (map.mappings.length == 0) return map;
// shift columns in first line
const segment_list = map.mappings[0];
for (let segment = 0; segment < segment_list.length; segment++) {
const seg = segment_list[segment];
if (seg[3]) seg[3] += offset.column;
}
// shift lines
if (map.mappings.length == 0) return;
for (let line = 0; line < map.mappings.length; line++) {
const segment_list = map.mappings[line];
for (let segment = 0; segment < segment_list.length; segment++) {
const seg = segment_list[segment];
if (seg[2]) seg[2] += offset.line;
// shift only segments that belong to component source file
if (seg[1] === source_index) { // also ensures that seg.length >= 4
// shift column if it points at the first line
if (seg[2] === 0) {
seg[3] += offset.column;
}
// shift line
seg[2] += offset.line;
}
}
}
}
@ -97,6 +98,9 @@ export class StringWithSourcemap {
return this;
}
// compute last line length before mutating
const column_offset = last_line_length(this.string);
this.string += other.string;
const m1 = this.map;
@ -117,8 +121,8 @@ export class StringWithSourcemap {
const segment_list = m2.mappings[line];
for (let segment = 0; segment < segment_list.length; segment++) {
const seg = segment_list[segment];
if (seg[1]) seg[1] = new_source_idx[seg[1]];
if (seg[4]) seg[4] = new_name_idx[seg[4]];
if (seg[1] >= 0) seg[1] = new_source_idx[seg[1]];
if (seg[4] >= 0) seg[4] = new_name_idx[seg[4]];
}
}
} else if (sources_idx_changed) {
@ -126,7 +130,7 @@ export class StringWithSourcemap {
const segment_list = m2.mappings[line];
for (let segment = 0; segment < segment_list.length; segment++) {
const seg = segment_list[segment];
if (seg[1]) seg[1] = new_source_idx[seg[1]];
if (seg[1] >= 0) seg[1] = new_source_idx[seg[1]];
}
}
} else if (names_idx_changed) {
@ -134,7 +138,7 @@ export class StringWithSourcemap {
const segment_list = m2.mappings[line];
for (let segment = 0; segment < segment_list.length; segment++) {
const seg = segment_list[segment];
if (seg[4]) seg[4] = new_name_idx[seg[4]];
if (seg[4] >= 0) seg[4] = new_name_idx[seg[4]];
}
}
}
@ -146,7 +150,6 @@ export class StringWithSourcemap {
// 2. first line of second map
// columns of 2 must be shifted
const column_offset = last_line_length(this.string);
if (m2.mappings.length > 0 && column_offset > 0) {
const first_line = m2.mappings[0];
for (let i = 0; i < first_line.length; i++) {
@ -164,12 +167,23 @@ export class StringWithSourcemap {
}
static from_processed(string: string, map?: DecodedSourceMap): StringWithSourcemap {
if (map) return new StringWithSourcemap(string, map);
const line_count = string.split('\n').length;
if (map) {
// ensure that count of source map mappings lines
// is equal to count of generated code lines
// (some tools may produce less)
const missing_lines = line_count - map.mappings.length;
for (let i = 0; i < missing_lines; i++) {
map.mappings.push([]);
}
return new StringWithSourcemap(string, map);
}
if (string == '') return new StringWithSourcemap();
map = { version: 3, names: [], sources: [], mappings: [] };
// add empty SourceMapSegment[] for every line
const line_count = (string.match(/\n/g) || '').length;
for (let i = 0; i < line_count; i++) map.mappings.push([]);
return new StringWithSourcemap(string, map);
}

@ -1,4 +1,111 @@
import MagicString from 'magic-string';
import * as assert from 'assert';
import { getLocator } from 'locate-character';
import MagicString, { Bundle } from 'magic-string';
type AssertMappedParameters = {
code: string;
filename?: string;
input: string | ReturnType<typeof getLocator>;
input_code?: string;
preprocessed: any;
};
export function assert_mapped(
{ code, filename, input, input_code, preprocessed }: AssertMappedParameters
) {
const locate_input = typeof input === 'function' ? input : getLocator(input);
if (filename === undefined) filename = 'input.svelte';
if (input_code === undefined) input_code = code;
const source_loc = locate_input(input_code);
assert.notEqual(
source_loc,
undefined,
`failed to locate "${input_code}" in "${filename}"`
);
const transformed_loc = preprocessed.locate_1(code);
assert.notEqual(
transformed_loc,
undefined,
`failed to locate "${code}" in transformed "${filename}"`
);
assert.deepEqual(
preprocessed.mapConsumer.originalPositionFor(transformed_loc),
{
source: filename,
name: null,
line: source_loc.line + 1,
column: source_loc.column
},
`incorrect mappings for "${input_code}" in "${filename}"`
);
}
type AssertNotMappedParameters = {
code: string;
filename?: string;
preprocessed: any;
};
export function assert_not_mapped(
{ code, filename, preprocessed }: AssertNotMappedParameters
) {
if (filename === undefined) filename = 'input.svelte';
const transformed_loc = preprocessed.locate_1(code);
assert.notEqual(
transformed_loc,
undefined,
`failed to locate "${code}" in transformed "${filename}"`
);
assert.deepEqual(
preprocessed.mapConsumer.originalPositionFor(transformed_loc),
{
source: null,
name: null,
line: null,
column: null
},
`incorrect mappings for "${code}" in "${filename}"`
);
}
export function assert_not_located(
code: string,
locate: ReturnType<typeof getLocator>,
filename = 'input.svelte'
) {
assert.equal(
locate(code),
undefined,
`located "${code}" that should be removed from ${filename}`
);
}
export function magic_string_bundle(
inputs: Array<{ code: string | MagicString, filename: string }>,
filename = 'bundle.js',
separator = '\n'
) {
const bundle = new Bundle({ separator });
inputs.forEach(({ code, filename }) => {
bundle.addSource({
filename,
content: typeof code === 'string' ? new MagicString(code) : code
});
});
return {
code: bundle.toString(),
map: bundle.generateMap({
source: filename,
hires: true,
includeContent: false
})
};
}
export function magic_string_preprocessor_result(filename: string, src: MagicString) {
return {

@ -0,0 +1,24 @@
import { magic_string_bundle } from '../../helpers';
export const COMMON = ':global(html) { height: 100%; }\n';
// TODO: removing '\n' breaks test
// - _actual.svelte.map looks correct
// - _actual.css.map adds reference to </style> on input.svelte
// - Most probably caused by bug in current magic-string version (fixed in 0.25.7)
export const STYLES = '.awesome { color: orange; }\n';
export default {
css_map_sources: ['common.scss', 'styles.scss'],
js_map_sources: [],
preprocess: [
{
style: () => {
return magic_string_bundle([
{ filename: 'common.scss', code: COMMON },
{ filename: 'styles.scss', code: STYLES }
]);
}
}
]
};

@ -0,0 +1,3 @@
<style lang="scss" src="./styles.scss"></style>
<div class="awesome">Divs ftw!</div>

@ -0,0 +1,26 @@
import { assert_mapped } from '../../helpers';
import { COMMON, STYLES } from './_config';
export function test({ input, preprocessed }) {
// Transformed script, main file
assert_mapped({
filename: 'input.svelte',
code: 'Divs ftw!',
input: input.locate,
preprocessed
});
// External files
assert_mapped({
filename: 'common.scss',
code: 'height: 100%;',
input: COMMON,
preprocessed
});
assert_mapped({
filename: 'styles.scss',
code: 'color: orange;',
input: STYLES,
preprocessed
});
}

@ -0,0 +1,15 @@
export default {
css_map_sources: [],
preprocess: [
{
style: ({ content }) => {
// Modified without source map
return { code: content + ' ' };
},
script: ({ content }) => {
// Not modified
return { code: content };
}
}
]
};

@ -0,0 +1,13 @@
<script>
export let name;
console.log(name);
</script>
<main>
<div>{name}</div>
</main>
<style>
main { font-weight: bold; }
</style>

@ -0,0 +1,37 @@
import { assert_mapped, assert_not_mapped } from '../../helpers';
export function test({ input, preprocessed }) {
// markup (start)
assert_mapped({
code: '<script>',
input: input.locate,
preprocessed
});
// script content (preprocessed without map, content not changed)
assert_mapped({
code: 'console.log(name);',
input: input.locate,
preprocessed
});
// markup (middle)
assert_mapped({
code: '<div>{name}</div>',
input: input.locate,
preprocessed
});
// style content (preprocessed without map, content changed)
assert_not_mapped({
code: 'font-weight: bold;',
preprocessed
});
// markup (end)
assert_mapped({
code: '</style>',
input: input.locate,
preprocessed
});
}

@ -0,0 +1,33 @@
import { magic_string_bundle } from '../../helpers';
export const component_filepath = 'src/input.svelte';
export const component_file_basename = 'input.svelte';
// as output by external tool for src/external_code.css (relative to src/input.svelte)
export const external_relative_filename = 'external_code.css';
const external_code = `
span {
--external_code-var: 1px;
}
`;
export default {
css_map_sources: [external_relative_filename],
js_map_sources: [],
preprocess: [
{
style: ({ content, filename }) => {
const external =`/* Filename from preprocess: ${filename} */` + external_code;
return magic_string_bundle([
{ code: external, filename: external_relative_filename },
{ code: content, filename }
]);
}
}
],
options: {
filename: component_filepath
}
};

@ -0,0 +1,3 @@
<style src="external_code.css"></style>
<span>Hello world</span>

@ -0,0 +1,15 @@
import { component_filepath, component_file_basename, external_relative_filename } from './_config';
export function test({ assert, preprocessed }) {
assert.notEqual(
preprocessed.locate(`/* Filename from preprocess: ${component_filepath} */`),
undefined,
'Preprocessor should receive same value for filename as passed to preprocess function'
);
assert.deepEqual(
preprocessed.map.sources.slice().sort(),
[external_relative_filename, component_file_basename].sort(),
'Preprocessed map should contain sources relative to filepath'
);
}

@ -0,0 +1,15 @@
import MagicString from 'magic-string';
import { magic_string_preprocessor_result } from '../../helpers';
export default {
js_map_sources: ['input.svelte'],
preprocess: [
{
script: ({ content }) => {
const src = new MagicString(content);
src.prepend('console.log("Injected first line");\n');
return magic_string_preprocessor_result('input.svelte', src);
}
}
]
};

@ -0,0 +1,3 @@
<script>console.log('Target')</script>
<h1>Hello</h1>

@ -0,0 +1,9 @@
import { assert_mapped } from '../../helpers';
export function test({ input, preprocessed }) {
assert_mapped({
code: 'Target',
input: input.locate,
preprocessed
});
}

@ -0,0 +1,18 @@
import { magic_string_bundle } from '../../helpers';
export const EXTERNAL = 'span { --external-var: 1px; }';
export default {
js_map_sources: [],
css_map_sources: ['input.svelte', 'external.css'],
preprocess: [
{
style: ({ content, filename }) => {
return magic_string_bundle([
{ code: EXTERNAL, filename: 'external.css' },
{ code: content, filename }
]);
}
}
]
};

@ -0,0 +1,9 @@
<script>
// This block is here to offset style block
</script>
<style>
div { --component-var: 2px; }
</style>
<div><span>Text</span></div>

@ -0,0 +1,19 @@
import { assert_mapped } from '../../helpers';
import { EXTERNAL } from './_config';
export function test({ input, preprocessed }) {
// Part from component, should be with offset
assert_mapped({
code: '--component-var',
input: input.locate,
preprocessed
});
// Part from external file, should be without offset
assert_mapped({
filename: 'external.css',
code: '--external-var',
input: EXTERNAL,
preprocessed
});
}

@ -0,0 +1,26 @@
import * as ts from 'typescript';
export default {
js_map_sources: [
'input.svelte'
],
preprocess: [
{
script: ({ content, filename }) => {
const { outputText, sourceMapText } = ts.transpileModule(content, {
fileName: filename,
compilerOptions: {
target: ts.ScriptTarget.ES2015,
module: ts.ModuleKind.ES2015,
sourceMap: true
}
});
return {
code: outputText,
map: sourceMapText
};
}
}
]
};

@ -0,0 +1,18 @@
<script lang="ts">
import { onMount } from 'svelte';
let count: number = 0;
interface ITimeoutDestroyer {
(): void; // send timeout to the void!
}
onMount(() => {
const id = setInterval(() => count++, 1000);
const clear: ITimeoutDestroyer = () => clearInterval(id);
return clear;
});
</script>
<h1>Hello world!</h1>
<div>Counter value: {count}</div>

@ -0,0 +1,21 @@
import { assert_mapped, assert_not_located } from '../../helpers';
export function test({ input, preprocessed }) {
// TS => JS code
assert_mapped({
code: 'let count = 0;',
input_code: 'let count: number = 0;',
input: input.locate,
preprocessed
});
// Markup, not touched
assert_mapped({
code: '<h1>Hello world!</h1>',
input: input.locate,
preprocessed
});
// TS types, removed
assert_not_located('ITimeoutDestroyer', preprocessed.locate_1);
}
Loading…
Cancel
Save