From dee3aabf9c6f2d0b2c4b8f217be0f0eaaf3b7351 Mon Sep 17 00:00:00 2001 From: Milan Hauth Date: Sat, 19 Sep 2020 21:17:30 +0200 Subject: [PATCH] move fn replace_async, etc move fn replace_async back to old place to reduce line diffs fn preprocess: allow empty argument `preprocessor` (noop) make fns replace_async, get_replacement pure fns pass filename, get_location as argument restore old format to reduce line diffs remove empty fields of sourcemap: file, sourcesContent --- package-lock.json | 6 +- src/compiler/preprocess/index.ts | 157 ++++++++++-------- src/compiler/utils/string_with_sourcemap.ts | 16 +- .../{_preprocessor.js => _config.js} | 0 .../{_preprocessor.js => _config.js} | 0 .../{_preprocessor.js => _config.js} | 0 .../{_preprocessor.js => _config.js} | 0 7 files changed, 97 insertions(+), 82 deletions(-) rename test/sourcemaps/samples/preprocessed-markup/{_preprocessor.js => _config.js} (100%) rename test/sourcemaps/samples/preprocessed-multiple/{_preprocessor.js => _config.js} (100%) rename test/sourcemaps/samples/preprocessed-script/{_preprocessor.js => _config.js} (100%) rename test/sourcemaps/samples/preprocessed-styles/{_preprocessor.js => _config.js} (100%) diff --git a/package-lock.json b/package-lock.json index d6d72944b8..d8c8b2c9f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2661,9 +2661,9 @@ } }, "magic-string": { - "version": "0.25.7", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz", - "integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==", + "version": "0.25.3", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.3.tgz", + "integrity": "sha512-6QK0OpF/phMz0Q2AxILkX2mFhi7m+WMwTRg0LQKq/WBB0cDP4rYH3Wp4/d3OTXlrPLVJT/RFqj8tFeAR4nk8AA==", "dev": true, "requires": { "sourcemap-codec": "^1.4.4" diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index c8049ecb1e..e804aa4919 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -46,6 +46,46 @@ interface Replacement { replacement: StringWithSourcemap; } +async function replace_async( + filename: string, + source: string, + get_location: ReturnType, + re: RegExp, + func: (...any) => Promise +): Promise { + const replacements: Array> = []; + source.replace(re, (...args) => { + replacements.push( + func(...args).then( + res => + ({ + offset: args[args.length - 2], + length: args[0].length, + replacement: res + }) as Replacement + ) + ); + return ''; + }); + let out: StringWithSourcemap; + let last_end = 0; + for (const { offset, length, replacement } of await Promise.all( + replacements + )) { + // content = source before replacement + const content = StringWithSourcemap.from_source( + filename, source.slice(last_end, offset), get_location(last_end)); + out = out ? out.concat(content) : content; + out = out.concat(replacement); + last_end = offset + length; + } + // final_content = source after last replacement + const final_content = StringWithSourcemap.from_source( + filename, source.slice(last_end), get_location(last_end)); + out = out.concat(final_content); + return out; +} + export default async function preprocess( source: string, preprocessor: PreprocessorGroup | PreprocessorGroup[], @@ -55,23 +95,29 @@ export default async function preprocess( const filename = (options && options.filename) || preprocessor.filename; // legacy const dependencies = []; - const preprocessors = Array.isArray(preprocessor) ? preprocessor : [preprocessor]; + const preprocessors = preprocessor + ? Array.isArray(preprocessor) ? preprocessor : [preprocessor] + : []; // noop const markup = preprocessors.map(p => p.markup).filter(Boolean); const script = preprocessors.map(p => p.script).filter(Boolean); const style = preprocessors.map(p => p.style).filter(Boolean); + // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) + // so we use sourcemap_list.unshift() to add new maps + // https://github.com/ampproject/remapping#multiple-transformations-of-a-file const sourcemap_list: Array = []; - let get_location: ReturnType; - function get_replacement( + filename: string, offset: number, + get_location: ReturnType, original: string, processed: Processed, prefix: string, suffix: string ): StringWithSourcemap { + const generated_prefix = StringWithSourcemap.from_source( filename, prefix, get_location(offset)); const generated_suffix = StringWithSourcemap.from_source( @@ -90,44 +136,6 @@ export default async function preprocess( return map; } - async function replace_async( - str: string, - re: RegExp, - func: (...any) => Promise - ): Promise { - const replacement_promises: Array> = []; - str.replace(re, (...args) => { - replacement_promises.push( - func(...args).then( - (replacement) => - ({ - offset: args[args.length - 2], - length: args[0].length, - replacement - }) as Replacement - ) - ); - return ''; - }); - const replacements = await Promise.all(replacement_promises); - - let out: StringWithSourcemap; - let last_end = 0; - for (const { offset, length, replacement } of replacements) { - // content = source before replacement - const content = StringWithSourcemap.from_source( - filename, str.slice(last_end, offset), get_location(last_end)); - out = out ? out.concat(content) : content; - out = out.concat(replacement); - last_end = offset + length; - } - // final_content = source after last replacement - const final_content = StringWithSourcemap.from_source( - filename, str.slice(last_end), get_location(last_end)); - out = out.concat(final_content); - return out; - } - for (const fn of markup) { // run markup preprocessor @@ -136,21 +144,19 @@ export default async function preprocess( filename }); - if (processed && processed.dependencies) { - dependencies.push(...processed.dependencies); - } + if (processed && processed.dependencies) dependencies.push(...processed.dependencies); source = processed ? processed.code : source; - if (processed && processed.map) { - sourcemap_list.unshift(processed.map); - } + if (processed && processed.map) sourcemap_list.unshift(processed.map); } for (const fn of script) { - get_location = getLocator(source); + const get_location = getLocator(source); const res = await replace_async( + filename, source, + get_location, /|([^]*?)<\/script>|\/>)/gi, - async (match, attributes = '', content, offset) => { + async (match, attributes = '', content = '', offset) => { const no_change = () => StringWithSourcemap.from_source( filename, match, get_location(offset)); if (!attributes && !content) { @@ -166,16 +172,10 @@ export default async function preprocess( filename }); - if (!processed) { - return no_change(); - } - if (processed.dependencies) { - dependencies.push(...processed.dependencies); - } - return get_replacement( - offset, content, processed, - ``, `` - ); + if (processed && processed.dependencies) dependencies.push(...processed.dependencies); + return processed + ? get_replacement(filename, offset, get_location, content, processed, ``, ``) + : no_change(); } ); source = res.generated; @@ -183,11 +183,13 @@ export default async function preprocess( } for (const fn of style) { - get_location = getLocator(source); + const get_location = getLocator(source); const res = await replace_async( + filename, source, + get_location, /|([^]*?)<\/style>|\/>)/gi, - async (match, attributes = '', content, offset) => { + async (match, attributes = '', content = '', offset) => { const no_change = () => StringWithSourcemap.from_source( filename, match, get_location(offset)); if (!attributes && !content) { @@ -203,27 +205,35 @@ export default async function preprocess( filename }); - if (!processed) { - return no_change(); - } - if (processed.dependencies) { - dependencies.push(...processed.dependencies); - } - return get_replacement( - offset, content, processed, - ``, `` - ); + if (processed && processed.dependencies) dependencies.push(...processed.dependencies); + return processed + ? get_replacement(filename, offset, get_location, content, processed, ``, ``) + : no_change(); } ); - source = res.generated; sourcemap_list.unshift(res.get_sourcemap()); } + // HACK + // remove `undefined` sources in first sourcemap + // otherwise remapper throws error: + // Error: Transformation map 0 must have exactly one source file. + // Did you specify these with the most recent transformation maps first? + // test: preprocess comments + (firstMap => { + if (firstMap && firstMap.sources) + firstMap.sources = firstMap.sources.filter(Boolean); + })(sourcemap_list[0] as any); + + // https://github.com/ampproject/remapping#usage + // https://github.com/mozilla/source-map#new-sourcemapconsumerrawsourcemap const map: ReturnType = sourcemap_list.length == 0 ? null - : remapper(sourcemap_list as any, () => null); + : remapper(sourcemap_list as any, () => null, true); // true: skip optional field `sourcesContent` + + if (map) delete map.file; // skip optional field `file` return { // TODO return separated output, in future version where svelte.compile supports it: @@ -234,6 +244,7 @@ export default async function preprocess( code: source, dependencies: [...new Set(dependencies)], map, + toString() { return source; } diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index f899b03927..e9340a2080 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -138,7 +138,7 @@ export class StringWithSourcemap { // combine the mappings - // this.map is readonly, so we copy + // this.map is read-only, so we copy let new_mappings = this.map.mappings.slice(); // combine: @@ -147,14 +147,18 @@ export class StringWithSourcemap { // columns of 2 must be shifted const end = get_end_location(this.generated); const col_offset = end.column + 1; + const first_line = other_mappings.length == 0 ? [] - : other_mappings[0].map((seg) => { - const new_seg = seg.slice() as MappingSegment; - new_seg[0] = seg[0] + col_offset; - return new_seg; - }); + : col_offset == 0 + ? other_mappings[0] + : other_mappings[0].map((seg) => { + // shift columns + const new_seg = seg.slice() as MappingSegment; + new_seg[0] = seg[0] + col_offset; + return new_seg; + }); // append segments to last line of first map new_mappings[new_mappings.length - 1] = diff --git a/test/sourcemaps/samples/preprocessed-markup/_preprocessor.js b/test/sourcemaps/samples/preprocessed-markup/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-markup/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-markup/_config.js diff --git a/test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js b/test/sourcemaps/samples/preprocessed-multiple/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-multiple/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-multiple/_config.js diff --git a/test/sourcemaps/samples/preprocessed-script/_preprocessor.js b/test/sourcemaps/samples/preprocessed-script/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-script/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-script/_config.js diff --git a/test/sourcemaps/samples/preprocessed-styles/_preprocessor.js b/test/sourcemaps/samples/preprocessed-styles/_config.js similarity index 100% rename from test/sourcemaps/samples/preprocessed-styles/_preprocessor.js rename to test/sourcemaps/samples/preprocessed-styles/_config.js