From f0b9a292063e3514bf51002c8c83849ef7c1c4a6 Mon Sep 17 00:00:00 2001 From: halfnelson Date: Mon, 26 Oct 2020 14:56:51 +1000 Subject: [PATCH] Performance improvements, mild refactoring, and better css map support Co-authored-by: Milan Hauth --- src/compiler/compile/Component.ts | 26 +-- src/compiler/compile/render_dom/index.ts | 5 + src/compiler/utils/string_with_sourcemap.ts | 160 +++++++++++------- test/setup.js | 2 +- test/sourcemaps/index.ts | 5 +- .../samples/compile-option-dev/_config.js | 41 +++++ .../samples/compile-option-dev/input.svelte | 15 ++ .../samples/compile-option-dev/test.js | 40 +++++ 8 files changed, 205 insertions(+), 89 deletions(-) create mode 100644 test/sourcemaps/samples/compile-option-dev/_config.js create mode 100644 test/sourcemaps/samples/compile-option-dev/input.svelte create mode 100644 test/sourcemaps/samples/compile-option-dev/test.js diff --git a/src/compiler/compile/Component.ts b/src/compiler/compile/Component.ts index 340c1cacb7..6a70190e68 100644 --- a/src/compiler/compile/Component.ts +++ b/src/compiler/compile/Component.ts @@ -29,8 +29,9 @@ import add_to_set from './utils/add_to_set'; import check_graph_for_cycles from './utils/check_graph_for_cycles'; import { print, x, b } from 'code-red'; import { is_reserved_keyword } from './utils/reserved_keywords'; -import { combine_sourcemaps, sourcemap_define_tostring_tourl } from '../utils/string_with_sourcemap'; +import { apply_preprocessor_sourcemap } from '../utils/string_with_sourcemap'; import Element from './nodes/Element'; +import { DecodedSourceMap, RawSourceMap } from '@ampproject/remapping/dist/types/types'; interface ComponentOptions { namespace?: string; @@ -332,28 +333,7 @@ export default class Component { this.source ]; - if (compile_options.sourcemap) { - if (js.map) { - js.map = combine_sourcemaps( - this.file, - [ - js.map, // idx 1: internal - compile_options.sourcemap // idx 0: external: svelte.preprocess, etc - ] - ); - sourcemap_define_tostring_tourl(js.map); - } - if (css.map) { - css.map = combine_sourcemaps( - this.file, - [ - css.map, // idx 1: internal - compile_options.sourcemap // idx 0: external: svelte.preprocess, etc - ] - ); - sourcemap_define_tostring_tourl(css.map); - } - } + js.map = apply_preprocessor_sourcemap(this.file, js.map, compile_options.sourcemap as (string | RawSourceMap | DecodedSourceMap)); } return { diff --git a/src/compiler/compile/render_dom/index.ts b/src/compiler/compile/render_dom/index.ts index 024aafde14..4a767dfed1 100644 --- a/src/compiler/compile/render_dom/index.ts +++ b/src/compiler/compile/render_dom/index.ts @@ -7,6 +7,8 @@ import { extract_names, Scope } from '../utils/scope'; import { invalidate } from './invalidate'; import Block from './Block'; import { ClassDeclaration, FunctionExpression, Node, Statement, ObjectExpression, Expression } from 'estree'; +import { apply_preprocessor_sourcemap } from '../../utils/string_with_sourcemap'; +import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types'; export default function dom( component: Component, @@ -30,6 +32,9 @@ export default function dom( } const css = component.stylesheet.render(options.filename, !options.customElement); + + css.map = apply_preprocessor_sourcemap(options.filename, css.map, options.sourcemap as string | RawSourceMap | DecodedSourceMap); + const styles = component.stylesheet.has_styles && options.dev ? `${css.code}\n/*# sourceMappingURL=${css.map.toUrl()} */` : css.code; diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index 470364e845..845d46ec7d 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -1,5 +1,6 @@ -import { DecodedSourceMap, RawSourceMap, SourceMapSegment, SourceMapLoader } from '@ampproject/remapping/dist/types/types'; +import { DecodedSourceMap, RawSourceMap, SourceMapLoader } from '@ampproject/remapping/dist/types/types'; import remapping from '@ampproject/remapping'; +import { SourceMap } from 'magic-string'; type SourceLocation = { line: number; @@ -14,17 +15,21 @@ function last_line_length(s: string) { export function sourcemap_add_offset( map: DecodedSourceMap, offset: SourceLocation ) { + if (map.mappings.length == 0) return map; // shift columns in first line - const m = map.mappings; - m[0].forEach(seg => { + const segment_list = map.mappings[0]; + for (let segment = 0; segment < segment_list.length; segment++) { + const seg = segment_list[segment]; if (seg[3]) seg[3] += offset.column; - }); + } // shift lines - m.forEach(line => { - line.forEach(seg => { + for (let line = 0; line < map.mappings.length; line++) { + const segment_list = map.mappings[line]; + for (let segment = 0; segment < segment_list.length; segment++) { + const seg = segment_list[segment]; if (seg[2]) seg[2] += offset.line; - }); - }); + } + } } function merge_tables(this_table: T[], other_table): [T[], number[], boolean, boolean] { @@ -91,6 +96,8 @@ export class StringWithSourcemap { const m1 = this.map; const m2 = other.map; + if (m2.mappings.length == 0) return this; + // combine sources and names const [sources, new_source_idx, sources_changed, sources_idx_changed] = merge_tables(m1.sources, m2.sources); const [names, new_name_idx, names_changed, names_idx_changed] = merge_tables(m1.names, m2.names); @@ -100,24 +107,30 @@ export class StringWithSourcemap { // unswitched loops are faster if (sources_idx_changed && names_idx_changed) { - m2.mappings.forEach(line => { - line.forEach(seg => { + for (let line = 0; line < m2.mappings.length; line++) { + const segment_list = m2.mappings[line]; + for (let segment = 0; segment < segment_list.length; segment++) { + const seg = segment_list[segment]; if (seg[1]) seg[1] = new_source_idx[seg[1]]; if (seg[4]) seg[4] = new_name_idx[seg[4]]; - }); - }); + } + } } else if (sources_idx_changed) { - m2.mappings.forEach(line => { - line.forEach(seg => { + for (let line = 0; line < m2.mappings.length; line++) { + const segment_list = m2.mappings[line]; + for (let segment = 0; segment < segment_list.length; segment++) { + const seg = segment_list[segment]; if (seg[1]) seg[1] = new_source_idx[seg[1]]; - }); - }); + } + } } else if (names_idx_changed) { - m2.mappings.forEach(line => { - line.forEach(seg => { + for (let line = 0; line < m2.mappings.length; line++) { + const segment_list = m2.mappings[line]; + for (let segment = 0; segment < segment_list.length; segment++) { + const seg = segment_list[segment]; if (seg[4]) seg[4] = new_name_idx[seg[4]]; - }); - }); + } + } } // combine the mappings @@ -129,10 +142,10 @@ export class StringWithSourcemap { const column_offset = last_line_length(this.string); if (m2.mappings.length > 0 && column_offset > 0) { - // shift columns in first line - m2.mappings[0].forEach(seg => { - seg[0] += column_offset; - }); + const first_line = m2.mappings[0]; + for (let i = 0; i < first_line.length; i++) { + first_line[i][0] += column_offset; + } } // combine last line + first line @@ -146,38 +159,40 @@ export class StringWithSourcemap { static from_processed(string: string, map?: DecodedSourceMap): StringWithSourcemap { if (map) return new StringWithSourcemap(string, map); + if (string == '') return new StringWithSourcemap(); map = { version: 3, names: [], sources: [], mappings: [] }; - if (string == '') return new StringWithSourcemap(string, map); + // add empty SourceMapSegment[] for every line - const lineCount = string.split('\n').length; - map.mappings = Array.from({length: lineCount}).map(_ => []); + const line_count = (string.match(/\n/g) || '').length; + for (let i = 0; i < line_count; i++) map.mappings.push([]); return new StringWithSourcemap(string, map); } static from_source( - source_file: string, source: string, offset_in_source?: SourceLocation + source_file: string, source: string, offset?: SourceLocation ): StringWithSourcemap { - const offset = offset_in_source || { line: 0, column: 0 }; + if (!offset) offset = { line: 0, column: 0 }; const map: DecodedSourceMap = { version: 3, names: [], sources: [source_file], mappings: [] }; - if (source.length == 0) return new StringWithSourcemap(source, map); + if (source == '') return new StringWithSourcemap(source, map); // we create a high resolution identity map here, // we know that it will eventually be merged with svelte's map, // at which stage the resolution will decrease. - map.mappings = source.split('\n').map((line, line_idx) => { - let pos = 0; - const segs = line.split(/([^\d\w\s]|\s+)/g) - .filter(s => s !== '').map(s => { - const seg: SourceMapSegment = [ - pos, 0, - line_idx + offset.line, - pos + (line_idx == 0 ? offset.column : 0) // shift first line - ]; - pos = pos + s.length; - return seg; - }); - return segs; - }); + const line_list = source.split('\n'); + for (let line = 0; line < line_list.length; line++) { + map.mappings.push([]); + const token_list = line_list[line].split(/([^\d\w\s]|\s+)/g); + for (let token = 0, column = 0; token < token_list.length; token++) { + if (token_list[token] == '') continue; + map.mappings[line].push([column, 0, offset.line + line, column]); + column += token_list[token].length; + } + } + // shift columns in first line + const segment_list = map.mappings[0]; + for (let segment = 0; segment < segment_list.length; segment++) { + segment_list[segment][3] += offset.column; + } return new StringWithSourcemap(source, map); } } @@ -191,34 +206,51 @@ export function combine_sourcemaps( let map_idx = 1; const map: RawSourceMap = sourcemap_list.slice(0, -1) - .find(m => m.sources.length !== 1) === undefined + .find(m => m.sources.length !== 1) === undefined ? remapping( // use array interface - // only the oldest sourcemap can have multiple sources - sourcemap_list, - () => null, - true // skip optional field `sourcesContent` - ) + // only the oldest sourcemap can have multiple sources + sourcemap_list, + () => null, + true // skip optional field `sourcesContent` + ) : remapping( // use loader interface - sourcemap_list[0], // last map - function loader(sourcefile) { - if (sourcefile === filename && sourcemap_list[map_idx]) { - return sourcemap_list[map_idx++]; // idx 1, 2, ... - // bundle file = branch node - } - else return null; // source file = leaf node - } as SourceMapLoader, - true - ); + sourcemap_list[0], // last map + function loader(sourcefile) { + if (sourcefile === filename && sourcemap_list[map_idx]) { + return sourcemap_list[map_idx++]; // idx 1, 2, ... + // bundle file = branch node + } + else return null; // source file = leaf node + } as SourceMapLoader, + true + ); if (!map.file) delete map.file; // skip optional field `file` return map; } -export function sourcemap_define_tostring_tourl(map) { - Object.defineProperties(map, { +// browser vs node.js +const b64enc = typeof btoa == 'function' ? btoa : b => Buffer.from(b).toString('base64'); + +export function apply_preprocessor_sourcemap(filename: string, svelte_map: SourceMap, preprocessor_map_input: string | DecodedSourceMap | RawSourceMap): SourceMap { + if (!svelte_map || !preprocessor_map_input) return svelte_map; + + const preprocessor_map = typeof preprocessor_map_input === 'string' ? JSON.parse(preprocessor_map_input) : preprocessor_map_input; + + const result_map = combine_sourcemaps( + filename, + [ + svelte_map as RawSourceMap, + preprocessor_map + ] + ) as RawSourceMap; + + //Svelte expects a SourceMap which includes toUrl and toString. Instead of using the magic-string constructor that takes a decoded map + //we just tack on the extra properties. + Object.defineProperties(result_map, { toString: { enumerable: false, value: function toString() { @@ -228,8 +260,10 @@ export function sourcemap_define_tostring_tourl(map) { toUrl: { enumerable: false, value: function toUrl() { - return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString()); + return 'data:application/json;charset=utf-8;base64,' + b64enc(this.toString()); } } }); + + return result_map as SourceMap; } diff --git a/test/setup.js b/test/setup.js index 7406a07dd9..74250c10eb 100644 --- a/test/setup.js +++ b/test/setup.js @@ -12,7 +12,7 @@ require.extensions['.js'] = function(module, filename) { .replace(/^import (\w+) from ['"]([^'"]+)['"];?/gm, 'var {default: $1} = require("$2");') .replace(/^import {([^}]+)} from ['"](.+)['"];?/gm, 'var {$1} = require("$2");') .replace(/^export default /gm, 'exports.default = ') - .replace(/^export (const|let|var|class|function) (\w+)/gm, (match, type, name) => { + .replace(/^export (const|let|var|class|function|async\s+function) (\w+)/gm, (match, type, name) => { exports.push(name); return `${type} ${name}`; }) diff --git a/test/sourcemaps/index.ts b/test/sourcemaps/index.ts index c657ab8f47..4122c3a419 100644 --- a/test/sourcemaps/index.ts +++ b/test/sourcemaps/index.ts @@ -48,7 +48,8 @@ describe('sourcemaps', () => { // filenames for sourcemaps sourcemap: preprocessed.map, outputFilename: `${outputName}.js`, - cssOutputFilename: `${outputName}.css` + cssOutputFilename: `${outputName}.css`, + ...(config.compile_options || {}) }); js.code = js.code.replace( @@ -108,7 +109,7 @@ describe('sourcemaps', () => { css.mapConsumer = css.map && await new SourceMapConsumer(css.map); css.locate = getLocator(css.code || ''); css.locate_1 = getLocator(css.code || '', { offsetLine: 1 }); - test({ assert, input, preprocessed, js, css }); + await test({ assert, input, preprocessed, js, css }); }); }); }); diff --git a/test/sourcemaps/samples/compile-option-dev/_config.js b/test/sourcemaps/samples/compile-option-dev/_config.js new file mode 100644 index 0000000000..5c638f5a10 --- /dev/null +++ b/test/sourcemaps/samples/compile-option-dev/_config.js @@ -0,0 +1,41 @@ +import MagicString from 'magic-string'; + +// TODO move util fns to test index.js + +function result(filename, src) { + return { + code: src.toString(), + map: src.generateMap({ + source: filename, + hires: true, + includeContent: false + }) + }; +} + +function replace_all(src, search, replace) { + let idx = src.original.indexOf(search); + if (idx == -1) throw new Error('search not found in src'); + do { + src.overwrite(idx, idx + search.length, replace); + } while ((idx = src.original.indexOf(search, idx + 1)) != -1); +} + +export default { + compile_options: { + dev: true + }, + preprocess: [ + { style: ({ content, filename }) => { + const src = new MagicString(content); + replace_all(src, '--replace-me-once', '\n --done-replace-once'); + replace_all(src, '--replace-me-twice', '\n--almost-done-replace-twice'); + return result(filename, src); + } }, + { style: ({ content, filename }) => { + const src = new MagicString(content); + replace_all(src, '--almost-done-replace-twice', '\n --done-replace-twice'); + return result(filename, src); + } } + ] +}; diff --git a/test/sourcemaps/samples/compile-option-dev/input.svelte b/test/sourcemaps/samples/compile-option-dev/input.svelte new file mode 100644 index 0000000000..f16bf0d5d8 --- /dev/null +++ b/test/sourcemaps/samples/compile-option-dev/input.svelte @@ -0,0 +1,15 @@ +

Testing Styles

+

Testing Styles 2

+
Testing Styles 3
+ + \ No newline at end of file diff --git a/test/sourcemaps/samples/compile-option-dev/test.js b/test/sourcemaps/samples/compile-option-dev/test.js new file mode 100644 index 0000000000..97a4931597 --- /dev/null +++ b/test/sourcemaps/samples/compile-option-dev/test.js @@ -0,0 +1,40 @@ +import { SourceMapConsumer } from 'source-map'; + +const b64dec = s => Buffer.from(s, 'base64').toString(); + +export async function test({ assert, css, js }) { + + //We check that the css source map embedded in the js is accurate + const match = js.code.match(/\tstyle\.textContent = "(.*?)(?:\\n\/\*# sourceMappingURL=data:(.*?);charset=(.*?);base64,(.*?) \*\/)?";\n/); + assert.notEqual(match, null); + + const [mimeType, encoding, cssMapBase64] = match.slice(2); + assert.equal(mimeType, 'application/json'); + assert.equal(encoding, 'utf-8'); + + const cssMapJson = b64dec(cssMapBase64); + css.mapConsumer = await new SourceMapConsumer(cssMapJson); + + // TODO make util fn + move to test index.js + const sourcefile = 'input.svelte'; + [ + // TODO how to get line + column numbers? + [css, '--keep-me', 13, 2], + [css, '--done-replace-once', 6, 5], + [css, '--done-replace-twice', 9, 5] + ] + .forEach(([where, content, line, column]) => { + assert.deepEqual( + where.mapConsumer.originalPositionFor( + where.locate_1(content) + ), + { + source: sourcefile, + name: null, + line, + column + }, + `failed to locate "${content}" from "${sourcefile}"` + ); + }); +}