Performance improvements, mild refactoring, and better css map support

Co-authored-by: Milan Hauth <milahu@gmail.com>

src/compiler/compile/Component.ts

@@ -29,8 +29,9 @@ import add_to_set from './utils/add_to_set';
 import check_graph_for_cycles from './utils/check_graph_for_cycles';
 import { print, x, b } from 'code-red';
 import { is_reserved_keyword } from './utils/reserved_keywords';
-import { combine_sourcemaps, sourcemap_define_tostring_tourl } from '../utils/string_with_sourcemap';
+import { apply_preprocessor_sourcemap } from '../utils/string_with_sourcemap';
 import Element from './nodes/Element';
+import { DecodedSourceMap, RawSourceMap } from '@ampproject/remapping/dist/types/types';
 
 interface ComponentOptions {
 	namespace?: string;
@@ -332,28 +333,7 @@ export default class Component {
 				this.source
 			];
 
-			if (compile_options.sourcemap) {
+			js.map = apply_preprocessor_sourcemap(this.file, js.map, compile_options.sourcemap as (string | RawSourceMap | DecodedSourceMap));
-				if (js.map) {
-					js.map = combine_sourcemaps(
-						this.file,
-						[
-							js.map, // idx 1: internal
-							compile_options.sourcemap // idx 0: external: svelte.preprocess, etc
-						]
-					);
-					sourcemap_define_tostring_tourl(js.map);
-				}
-				if (css.map) {
-					css.map = combine_sourcemaps(
-						this.file,
-						[
-							css.map, // idx 1: internal
-							compile_options.sourcemap // idx 0: external: svelte.preprocess, etc
-						]
-					);
-					sourcemap_define_tostring_tourl(css.map);
-				}
-			}
 		}
 
 		return {

src/compiler/compile/render_dom/index.ts

@@ -7,6 +7,8 @@ import { extract_names, Scope } from '../utils/scope';
 import { invalidate } from './invalidate';
 import Block from './Block';
 import { ClassDeclaration, FunctionExpression, Node, Statement, ObjectExpression, Expression } from 'estree';
+import { apply_preprocessor_sourcemap } from '../../utils/string_with_sourcemap';
+import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
 
 export default function dom(
 	component: Component,
@@ -30,6 +32,9 @@ export default function dom(
 	}
 
 	const css = component.stylesheet.render(options.filename, !options.customElement);
+
+	css.map = apply_preprocessor_sourcemap(options.filename, css.map, options.sourcemap as string | RawSourceMap | DecodedSourceMap);
+
 	const styles = component.stylesheet.has_styles && options.dev
 		? `${css.code}\n/*# sourceMappingURL=${css.map.toUrl()} */`
 		: css.code;

src/compiler/utils/string_with_sourcemap.ts

@@ -1,5 +1,6 @@
-import { DecodedSourceMap, RawSourceMap, SourceMapSegment, SourceMapLoader } from '@ampproject/remapping/dist/types/types';
+import { DecodedSourceMap, RawSourceMap, SourceMapLoader } from '@ampproject/remapping/dist/types/types';
 import remapping from '@ampproject/remapping';
+import { SourceMap } from 'magic-string';
 
 type SourceLocation = {
 	line: number;
@@ -14,17 +15,21 @@ function last_line_length(s: string) {
 export function sourcemap_add_offset(
 	map: DecodedSourceMap, offset: SourceLocation
 ) {
+	if (map.mappings.length == 0) return map;
 	// shift columns in first line
-	const m = map.mappings;
+	const segment_list = map.mappings[0];
-	m[0].forEach(seg => {
+	for (let segment = 0; segment < segment_list.length; segment++) {
+		const seg = segment_list[segment];
 		if (seg[3]) seg[3] += offset.column;
-	});
+	}
 	// shift lines
-	m.forEach(line => {
+	for (let line = 0; line < map.mappings.length; line++) {
-		line.forEach(seg => {
+		const segment_list = map.mappings[line];
+		for (let segment = 0; segment < segment_list.length; segment++) {
+			const seg = segment_list[segment];
 			if (seg[2]) seg[2] += offset.line;
-		});
+		}
-	});
+	}
 }
 
 function merge_tables<T>(this_table: T[], other_table): [T[], number[], boolean, boolean] {
@@ -91,6 +96,8 @@ export class StringWithSourcemap {
 		const m1 = this.map;
 		const m2 = other.map;
 
+		if (m2.mappings.length == 0) return this;
+
 		// combine sources and names
 		const [sources, new_source_idx, sources_changed, sources_idx_changed] = merge_tables(m1.sources, m2.sources);
 		const [names, new_name_idx, names_changed, names_idx_changed] = merge_tables(m1.names, m2.names);
@@ -100,24 +107,30 @@ export class StringWithSourcemap {
 
 		// unswitched loops are faster
 		if (sources_idx_changed && names_idx_changed) {
-			m2.mappings.forEach(line => {
+			for (let line = 0; line < m2.mappings.length; line++) {
-				line.forEach(seg => {
+				const segment_list = m2.mappings[line];
+				for (let segment = 0; segment < segment_list.length; segment++) {
+					const seg = segment_list[segment];
 					if (seg[1]) seg[1] = new_source_idx[seg[1]];
 					if (seg[4]) seg[4] = new_name_idx[seg[4]];
-				});
+				}
-			});
+			}
 		} else if (sources_idx_changed) {
-			m2.mappings.forEach(line => {
+			for (let line = 0; line < m2.mappings.length; line++) {
-				line.forEach(seg => {
+				const segment_list = m2.mappings[line];
+				for (let segment = 0; segment < segment_list.length; segment++) {
+					const seg = segment_list[segment];
 					if (seg[1]) seg[1] = new_source_idx[seg[1]];
-				});
+				}
-			});
+			}
 		} else if (names_idx_changed) {
-			m2.mappings.forEach(line => {
+			for (let line = 0; line < m2.mappings.length; line++) {
-				line.forEach(seg => {
+				const segment_list = m2.mappings[line];
+				for (let segment = 0; segment < segment_list.length; segment++) {
+					const seg = segment_list[segment];
 					if (seg[4]) seg[4] = new_name_idx[seg[4]];
-				});
+				}
-			});
+			}
 		}
 
 		// combine the mappings
@@ -129,10 +142,10 @@ export class StringWithSourcemap {
 
 		const column_offset = last_line_length(this.string);
 		if (m2.mappings.length > 0 && column_offset > 0) {
-			// shift columns in first line
+			const first_line = m2.mappings[0];
-			m2.mappings[0].forEach(seg => {
+			for (let i = 0; i < first_line.length; i++) {
-				seg[0] += column_offset;
+				first_line[i][0] += column_offset;
-			});
+			}
 		}
 
 		// combine last line + first line
@@ -146,38 +159,40 @@ export class StringWithSourcemap {
 
 	static from_processed(string: string, map?: DecodedSourceMap): StringWithSourcemap {
 		if (map) return new StringWithSourcemap(string, map);
+		if (string == '') return new StringWithSourcemap();
 		map = { version: 3, names: [], sources: [], mappings: [] };
-		if (string == '') return new StringWithSourcemap(string, map);
+
 		// add empty SourceMapSegment[] for every line
-		const lineCount = string.split('\n').length;
+		const line_count = (string.match(/\n/g) || '').length;
-		map.mappings = Array.from({length: lineCount}).map(_ => []);
+		for (let i = 0; i < line_count; i++) map.mappings.push([]);
 		return new StringWithSourcemap(string, map);
 	}
 
 	static from_source(
-		source_file: string, source: string, offset_in_source?: SourceLocation
+		source_file: string, source: string, offset?: SourceLocation
 	): StringWithSourcemap {
-		const offset = offset_in_source || { line: 0, column: 0 };
+		if (!offset) offset = { line: 0, column: 0 };
 		const map: DecodedSourceMap = { version: 3, names: [], sources: [source_file], mappings: [] };
-		if (source.length == 0) return new StringWithSourcemap(source, map);
+		if (source == '') return new StringWithSourcemap(source, map);
 
 		// we create a high resolution identity map here,
 		// we know that it will eventually be merged with svelte's map,
 		// at which stage the resolution will decrease.
-		map.mappings = source.split('\n').map((line, line_idx) => {
+		const line_list = source.split('\n');
-			let pos = 0;
+		for (let line = 0; line < line_list.length; line++) {
-			const segs = line.split(/([^\d\w\s]|\s+)/g)
+			map.mappings.push([]);
-				.filter(s => s !== '').map(s => {
+			const token_list = line_list[line].split(/([^\d\w\s]|\s+)/g);
-					const seg: SourceMapSegment = [
+			for (let token = 0, column = 0; token < token_list.length; token++) {
-						pos, 0,
+				if (token_list[token] == '') continue;
-						line_idx + offset.line,
+				map.mappings[line].push([column, 0, offset.line + line, column]);
-						pos + (line_idx == 0 ? offset.column : 0) // shift first line
+				column += token_list[token].length;
-					];
+			}
-					pos = pos + s.length;
+		}
-					return seg;
+		// shift columns in first line
-				});
+		const segment_list = map.mappings[0];
-			return segs;
+		for (let segment = 0; segment < segment_list.length; segment++) {
-		});
+			segment_list[segment][3] += offset.column;
+		}
 		return new StringWithSourcemap(source, map);
 	}
 }
@@ -191,34 +206,51 @@ export function combine_sourcemaps(
 	let map_idx = 1;
 	const map: RawSourceMap =
 		sourcemap_list.slice(0, -1)
-		.find(m => m.sources.length !== 1) === undefined
+			.find(m => m.sources.length !== 1) === undefined
 
 			? remapping( // use array interface
-					// only the oldest sourcemap can have multiple sources
+				// only the oldest sourcemap can have multiple sources
-					sourcemap_list,
+				sourcemap_list,
-					() => null,
+				() => null,
-					true // skip optional field `sourcesContent`
+				true // skip optional field `sourcesContent`
-				)
+			)
 
 			: remapping( // use loader interface
-					sourcemap_list[0], // last map
+				sourcemap_list[0], // last map
-					function loader(sourcefile) {
+				function loader(sourcefile) {
-						if (sourcefile === filename && sourcemap_list[map_idx]) {
+					if (sourcefile === filename && sourcemap_list[map_idx]) {
-							return sourcemap_list[map_idx++]; // idx 1, 2, ...
+						return sourcemap_list[map_idx++]; // idx 1, 2, ...
-							// bundle file = branch node
+						// bundle file = branch node
-						}
+					}
-						else return null; // source file = leaf node
+					else return null; // source file = leaf node
-					} as SourceMapLoader,
+				} as SourceMapLoader,
-					true
+				true
-				);
+			);
 
 	if (!map.file) delete map.file; // skip optional field `file`
 
 	return map;
 }
 
-export function sourcemap_define_tostring_tourl(map) {
+// browser vs node.js
-	Object.defineProperties(map, {
+const b64enc = typeof btoa == 'function' ? btoa : b => Buffer.from(b).toString('base64');
+
+export function apply_preprocessor_sourcemap(filename: string, svelte_map: SourceMap, preprocessor_map_input: string | DecodedSourceMap | RawSourceMap): SourceMap {
+	if (!svelte_map || !preprocessor_map_input) return svelte_map;
+
+	const preprocessor_map = typeof preprocessor_map_input === 'string' ? JSON.parse(preprocessor_map_input) : preprocessor_map_input;
+
+	const result_map = combine_sourcemaps(
+		filename,
+		[
+			svelte_map as RawSourceMap,
+			preprocessor_map
+		]
+	) as RawSourceMap;
+
+	//Svelte expects a SourceMap which includes toUrl and toString. Instead of using the magic-string constructor that takes a decoded map
+	//we just tack on the extra properties.
+	Object.defineProperties(result_map, {
 		toString: {
 			enumerable: false,
 			value: function toString() {
@@ -228,8 +260,10 @@ export function sourcemap_define_tostring_tourl(map) {
 		toUrl: {
 			enumerable: false,
 			value: function toUrl() {
-				return 'data:application/json;charset=utf-8;base64,' + btoa(this.toString());
+				return 'data:application/json;charset=utf-8;base64,' + b64enc(this.toString());
 			}
 		}
 	});
+
+	return result_map as SourceMap;
 }

test/setup.js

@@ -12,7 +12,7 @@ require.extensions['.js'] = function(module, filename) {
 		.replace(/^import (\w+) from ['"]([^'"]+)['"];?/gm, 'var {default: $1} = require("$2");')
 		.replace(/^import {([^}]+)} from ['"](.+)['"];?/gm, 'var {$1} = require("$2");')
 		.replace(/^export default /gm, 'exports.default = ')
-		.replace(/^export (const|let|var|class|function) (\w+)/gm, (match, type, name) => {
+		.replace(/^export (const|let|var|class|function|async\s+function) (\w+)/gm, (match, type, name) => {
 			exports.push(name);
 			return `${type} ${name}`;
 		})

test/sourcemaps/index.ts

@@ -48,7 +48,8 @@ describe('sourcemaps', () => {
 				// filenames for sourcemaps
 				sourcemap: preprocessed.map,
 				outputFilename: `${outputName}.js`,
-				cssOutputFilename: `${outputName}.css`
+				cssOutputFilename: `${outputName}.css`,
+				...(config.compile_options || {})
 			});
 
 			js.code = js.code.replace(
@@ -108,7 +109,7 @@ describe('sourcemaps', () => {
 			css.mapConsumer = css.map && await new SourceMapConsumer(css.map);
 			css.locate = getLocator(css.code || '');
 			css.locate_1 = getLocator(css.code || '', { offsetLine: 1 });
-			test({ assert, input, preprocessed, js, css });
+			await test({ assert, input, preprocessed, js, css });
 		});
 	});
 });

test/sourcemaps/samples/compile-option-dev/_config.js

@@ -0,0 +1,41 @@
+import MagicString from 'magic-string';
+
+// TODO move util fns to test index.js
+
+function result(filename, src) {
+	return {
+		code: src.toString(),
+		map: src.generateMap({
+			source: filename,
+			hires: true,
+			includeContent: false
+		})
+	};
+}
+
+function replace_all(src, search, replace) {
+	let idx = src.original.indexOf(search);
+	if (idx == -1) throw new Error('search not found in src');
+	do {
+		src.overwrite(idx, idx + search.length, replace);
+	} while ((idx = src.original.indexOf(search, idx + 1)) != -1);
+}
+
+export default {
+	compile_options: {
+		dev: true
+	},
+	preprocess: [
+		{ style: ({ content, filename }) =>  {
+				const src = new MagicString(content);
+				replace_all(src, '--replace-me-once', '\n --done-replace-once');
+				replace_all(src, '--replace-me-twice', '\n--almost-done-replace-twice');
+				return result(filename, src);
+		} },
+		{ style: ({ content, filename }) =>  {
+				const src = new MagicString(content);
+				replace_all(src, '--almost-done-replace-twice', '\n  --done-replace-twice');
+				return result(filename, src);
+		} }
+	]
+};

test/sourcemaps/samples/compile-option-dev/input.svelte

@@ -0,0 +1,15 @@
+<h1>Testing Styles</h1>
+<h2>Testing Styles 2</h2>
+<div>Testing Styles 3</div>
+<script>export const b = 2;</script>
+<style>
+	h1 {
+		--replace-me-once: red;
+	}
+	h2 {
+		--replace-me-twice: green;
+	}
+	div {
+		--keep-me: blue;
+	}
+</style>

test/sourcemaps/samples/compile-option-dev/test.js

@@ -0,0 +1,40 @@
+import { SourceMapConsumer } from 'source-map';
+
+const b64dec = s => Buffer.from(s, 'base64').toString();
+
+export async function test({ assert, css,  js }) {
+
+    //We check that the css source map embedded in the js is accurate
+    const match = js.code.match(/\tstyle\.textContent = "(.*?)(?:\\n\/\*# sourceMappingURL=data:(.*?);charset=(.*?);base64,(.*?) \*\/)?";\n/);
+	assert.notEqual(match, null);
+
+	const [mimeType, encoding, cssMapBase64] = match.slice(2);
+	assert.equal(mimeType, 'application/json');
+	assert.equal(encoding, 'utf-8');
+
+	const cssMapJson = b64dec(cssMapBase64);
+	css.mapConsumer = await new SourceMapConsumer(cssMapJson);
+
+	// TODO make util fn + move to test index.js
+	const sourcefile = 'input.svelte';
+	[
+		// TODO how to get line + column numbers?
+		[css, '--keep-me', 13, 2],
+		[css, '--done-replace-once', 6, 5],
+		[css, '--done-replace-twice', 9, 5]
+	]
+	.forEach(([where, content, line, column]) => {
+		assert.deepEqual(
+			where.mapConsumer.originalPositionFor(
+				where.locate_1(content)
+			),
+			{
+				source: sourcefile,
+				name: null,
+				line,
+				column
+			},
+			`failed to locate "${content}" from "${sourcefile}"`
+		);
+	});
+}