Refactored preprocess for readability

pull/5770/head
Andreas Ehrencrona 5 years ago
parent 391455c58e
commit 90274b9739

@ -0,0 +1,88 @@
import { decode as decode_mappings } from 'sourcemap-codec';
import { Processed } from './types';
/**
* Import decoded sourcemap from mozilla/source-map/SourceMapGenerator
* Forked from source-map/lib/source-map-generator.js
* from methods _serializeMappings and toJSON.
* We cannot use source-map.d.ts types, because we access hidden properties.
*/
function decoded_sourcemap_from_generator(generator: any) {
let previous_generated_line = 1;
const converted_mappings = [[]];
let result_line;
let result_segment;
let mapping;
const source_idx = generator._sources.toArray()
.reduce((acc, val, idx) => (acc[val] = idx, acc), {});
const name_idx = generator._names.toArray()
.reduce((acc, val, idx) => (acc[val] = idx, acc), {});
const mappings = generator._mappings.toArray();
result_line = converted_mappings[0];
for (let i = 0, len = mappings.length; i < len; i++) {
mapping = mappings[i];
if (mapping.generatedLine > previous_generated_line) {
while (mapping.generatedLine > previous_generated_line) {
converted_mappings.push([]);
previous_generated_line++;
}
result_line = converted_mappings[mapping.generatedLine - 1]; // line is one-based
} else if (i > 0) {
const previous_mapping = mappings[i - 1];
if (
// sorted by selectivity
mapping.generatedColumn === previous_mapping.generatedColumn &&
mapping.originalColumn === previous_mapping.originalColumn &&
mapping.name === previous_mapping.name &&
mapping.generatedLine === previous_mapping.generatedLine &&
mapping.originalLine === previous_mapping.originalLine &&
mapping.source === previous_mapping.source
) {
continue;
}
}
result_line.push([mapping.generatedColumn]);
result_segment = result_line[result_line.length - 1];
if (mapping.source != null) {
result_segment.push(...[
source_idx[mapping.source],
mapping.originalLine - 1, // line is one-based
mapping.originalColumn
]);
if (mapping.name != null) {
result_segment.push(name_idx[mapping.name]);
}
}
}
const map = {
version: generator._version,
sources: generator._sources.toArray(),
names: generator._names.toArray(),
mappings: converted_mappings
};
if (generator._file != null) {
(map as any).file = generator._file;
}
// not needed: map.sourcesContent and map.sourceRoot
return map;
}
export function decode_map(processed: Processed) {
let decoded_map = typeof processed.map === 'string' ? JSON.parse(processed.map) : processed.map;
if (typeof(decoded_map.mappings) === 'string') {
decoded_map.mappings = decode_mappings(decoded_map.mappings);
}
if ((decoded_map as any)._mappings && decoded_map.constructor.name === 'SourceMapGenerator') {
// import decoded sourcemap from mozilla/source-map/SourceMapGenerator
decoded_map = decoded_sourcemap_from_generator(decoded_map);
}
return decoded_map;
}

@ -1,207 +1,153 @@
import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types';
import { decode as decode_mappings } from 'sourcemap-codec';
import { getLocator } from 'locate-character';
import { StringWithSourcemap, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap';
export interface Processed {
code: string;
map?: string | object; // we are opaque with the type here to avoid dependency on the remapping module for our public types.
import { StringWithSourcemap, SourceLocation, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap';
import { decode_map } from './decode_sourcemap';
import parse_tag_attributes from './parse_tag_attributes';
import { replace_in_code, Source } from './replace_in_code';
import { Preprocessor, PreprocessorGroup, Processed } from './types';
interface SourceUpdate {
string: string;
map?: DecodedSourceMap;
dependencies?: string[];
}
export interface PreprocessorGroup {
markup?: (options: {
content: string;
filename: string;
}) => Processed | Promise<Processed>;
style?: Preprocessor;
script?: Preprocessor;
}
class PreprocessResult {
// sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1)
// so we use sourcemap_list.unshift() to add new maps
// https://github.com/ampproject/remapping#multiple-transformations-of-a-file
sourcemap_list: Array<DecodedSourceMap | RawSourceMap> = [];
dependencies: string[] = [];
get_location: ReturnType<typeof getLocator>;
constructor(public source: string, public filename: string) {
this.update_source({string: source});
}
export type Preprocessor = (options: {
content: string;
attributes: Record<string, string | boolean>;
filename?: string;
}) => Processed | Promise<Processed>;
function parse_attributes(str: string) {
const attrs = {};
str.split(/\s+/).filter(Boolean).forEach(attr => {
const p = attr.indexOf('=');
if (p === -1) {
attrs[attr] = true;
} else {
attrs[attr.slice(0, p)] = '\'"'.includes(attr[p + 1]) ?
attr.slice(p + 2, -1) :
attr.slice(p + 1);
update_source({string: source, map, dependencies}: SourceUpdate) {
this.source = source;
this.get_location = getLocator(source);
if (map) {
this.sourcemap_list.unshift(map);
}
});
return attrs;
}
interface Replacement {
offset: number;
length: number;
replacement: StringWithSourcemap;
}
if (dependencies) {
this.dependencies.push(...dependencies);
}
}
async function replace_async(
filename: string,
source: string,
get_location: ReturnType<typeof getLocator>,
re: RegExp,
func: (...any) => Promise<StringWithSourcemap>
): Promise<StringWithSourcemap> {
const replacements: Array<Promise<Replacement>> = [];
source.replace(re, (...args) => {
replacements.push(
func(...args).then(
res =>
({
offset: args[args.length - 2],
length: args[0].length,
replacement: res
}) as Replacement
)
processed(): Processed {
// Combine all the source maps for each preprocessor function into one
const map: RawSourceMap = combine_sourcemaps(
this.filename,
this.sourcemap_list
);
return '';
});
const out = new StringWithSourcemap();
let last_end = 0;
for (const { offset, length, replacement } of await Promise.all(
replacements
)) {
// content = unchanged source characters before the replaced segment
const content = StringWithSourcemap.from_source(
filename, source.slice(last_end, offset), get_location(last_end));
out.concat(content).concat(replacement);
last_end = offset + length;
return {
// TODO return separated output, in future version where svelte.compile supports it:
// style: { code: styleCode, map: styleMap },
// script { code: scriptCode, map: scriptMap },
// markup { code: markupCode, map: markupMap },
code: this.source,
dependencies: [...new Set(this.dependencies)],
map: (map as object),
toString: () => this.source
};
}
// final_content = unchanged source characters after last replaced segment
const final_content = StringWithSourcemap.from_source(
filename, source.slice(last_end), get_location(last_end));
return out.concat(final_content);
}
/**
* Import decoded sourcemap from mozilla/source-map/SourceMapGenerator
* Forked from source-map/lib/source-map-generator.js
* from methods _serializeMappings and toJSON.
* We cannot use source-map.d.ts types, because we access hidden properties.
* Convert preprocessor output for the tag content into StringWithSourceMap
*/
function decoded_sourcemap_from_generator(generator: any) {
let previous_generated_line = 1;
const converted_mappings = [[]];
let result_line;
let result_segment;
let mapping;
const source_idx = generator._sources.toArray()
.reduce((acc, val, idx) => (acc[val] = idx, acc), {});
const name_idx = generator._names.toArray()
.reduce((acc, val, idx) => (acc[val] = idx, acc), {});
const mappings = generator._mappings.toArray();
result_line = converted_mappings[0];
for (let i = 0, len = mappings.length; i < len; i++) {
mapping = mappings[i];
if (mapping.generatedLine > previous_generated_line) {
while (mapping.generatedLine > previous_generated_line) {
converted_mappings.push([]);
previous_generated_line++;
}
result_line = converted_mappings[mapping.generatedLine - 1]; // line is one-based
} else if (i > 0) {
const previous_mapping = mappings[i - 1];
if (
// sorted by selectivity
mapping.generatedColumn === previous_mapping.generatedColumn &&
mapping.originalColumn === previous_mapping.originalColumn &&
mapping.name === previous_mapping.name &&
mapping.generatedLine === previous_mapping.generatedLine &&
mapping.originalLine === previous_mapping.originalLine &&
mapping.source === previous_mapping.source
) {
continue;
}
}
result_line.push([mapping.generatedColumn]);
result_segment = result_line[result_line.length - 1];
if (mapping.source != null) {
result_segment.push(...[
source_idx[mapping.source],
mapping.originalLine - 1, // line is one-based
mapping.originalColumn
]);
if (mapping.name != null) {
result_segment.push(name_idx[mapping.name]);
}
}
function processed_content_to_sws(
processed: Processed,
location: SourceLocation
): StringWithSourcemap {
// Convert the preprocessed code and its sourcemap to a StringWithSourcemap
let decoded_map: DecodedSourceMap;
if (processed.map) {
decoded_map = decode_map(processed);
sourcemap_add_offset(decoded_map, location);
}
const map = {
version: generator._version,
sources: generator._sources.toArray(),
names: generator._names.toArray(),
mappings: converted_mappings
};
if (generator._file != null) {
(map as any).file = generator._file;
}
// not needed: map.sourcesContent and map.sourceRoot
return map;
return StringWithSourcemap.from_processed(processed.code, decoded_map);
}
/**
* Convert a preprocessor output and its leading prefix and trailing suffix into StringWithSourceMap
* Convert the whole tag including content (replacing it with `processed`)
* into a `StringWithSourcemap` representing the transformed code.
*/
function get_replacement(
filename: string,
offset: number,
get_location: ReturnType<typeof getLocator>,
original: string,
function processed_tag_to_sws(
processed: Processed,
prefix: string,
suffix: string
): StringWithSourcemap {
tag_name: 'style' | 'script',
attributes: string,
content: string,
{ filename, get_location }: Source): StringWithSourcemap {
const build_sws = (content: string, offset: number) =>
StringWithSourcemap.from_source(filename, content, get_location(offset));
// Convert the unchanged prefix and suffix to StringWithSourcemap
const prefix_with_map = StringWithSourcemap.from_source(
filename, prefix, get_location(offset));
const suffix_with_map = StringWithSourcemap.from_source(
filename, suffix, get_location(offset + prefix.length + original.length));
const tag_open = `<${tag_name}${attributes || ''}>`;
const tag_close = `</${tag_name}>`;
// Convert the preprocessed code and its sourcemap to a StringWithSourcemap
let decoded_map: DecodedSourceMap;
if (processed.map) {
decoded_map = typeof processed.map === 'string' ? JSON.parse(processed.map) : processed.map;
if (typeof(decoded_map.mappings) === 'string') {
decoded_map.mappings = decode_mappings(decoded_map.mappings);
}
if ((decoded_map as any)._mappings && decoded_map.constructor.name === 'SourceMapGenerator') {
// import decoded sourcemap from mozilla/source-map/SourceMapGenerator
decoded_map = decoded_sourcemap_from_generator(decoded_map);
}
sourcemap_add_offset(decoded_map, get_location(offset + prefix.length));
const tag_open_sws = build_sws(tag_open, 0);
const tag_close_sws = build_sws(tag_close, tag_open.length + content.length);
const content_sws = processed_content_to_sws(processed, get_location(tag_open.length));
return tag_open_sws.concat(content_sws).concat(tag_close_sws);
}
async function process_tag(
tag_name: 'style' | 'script',
preprocessor: Preprocessor,
source: Source
): Promise<SourceUpdate> {
const { filename, get_location } = source;
const tag_regex =
tag_name === 'style'
? /<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi
: /<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi;
const dependencies: string[] = [];
async function process_single_tag(
tag_with_content: string,
attributes = '',
content = '',
tag_offset: number
): Promise<StringWithSourcemap> {
const no_change = () =>
StringWithSourcemap.from_source(filename, tag_with_content, get_location(tag_offset));
if (!attributes && !content) return no_change();
const processed = await preprocessor({
content: content || '',
attributes: parse_tag_attributes(attributes || ''),
filename
});
if (!processed) return no_change();
if (processed.dependencies) dependencies.push(...processed.dependencies);
return processed_tag_to_sws(processed, tag_name, attributes, content,
{...source, get_location: offset => source.get_location(offset + tag_offset)});
}
const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map);
// Surround the processed code with the prefix and suffix, retaining valid sourcemappings
return prefix_with_map.concat(processed_with_map).concat(suffix_with_map);
return {...await replace_in_code(tag_regex, process_single_tag, source), dependencies};
}
export default async function preprocess(
source: string,
preprocessor: PreprocessorGroup | PreprocessorGroup[],
options?: { filename?: string }
) {
): Promise<Processed> {
// @ts-ignore todo: doublecheck
const filename = (options && options.filename) || preprocessor.filename; // legacy
const dependencies = [];
const preprocessors = preprocessor
? Array.isArray(preprocessor) ? preprocessor : [preprocessor]
@ -211,95 +157,38 @@ export default async function preprocess(
const script = preprocessors.map(p => p.script).filter(Boolean);
const style = preprocessors.map(p => p.style).filter(Boolean);
// sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1)
// so we use sourcemap_list.unshift() to add new maps
// https://github.com/ampproject/remapping#multiple-transformations-of-a-file
const sourcemap_list: Array<DecodedSourceMap | RawSourceMap> = [];
// TODO keep track: what preprocessor generated what sourcemap? to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings
const result = new PreprocessResult(source, filename);
for (const fn of markup) {
// TODO keep track: what preprocessor generated what sourcemap?
// to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings
// run markup preprocessor
const processed = await fn({
content: source,
for (const process of markup) {
const processed = await process({
content: result.source,
filename
});
if (!processed) continue;
if (processed.dependencies) dependencies.push(...processed.dependencies);
source = processed.code;
if (processed.map) {
sourcemap_list.unshift(
typeof(processed.map) === 'string'
result.update_source({
string: processed.code,
map: processed.map
? // TODO: can we use decode_sourcemap?
typeof processed.map === 'string'
? JSON.parse(processed.map)
: processed.map
);
}
}
async function preprocess_tag_content(tag_name: 'style' | 'script', preprocessor: Preprocessor) {
const get_location = getLocator(source);
const tag_regex = tag_name === 'style'
? /<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi
: /<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi;
const res = await replace_async(
filename,
source,
get_location,
tag_regex,
async (match, attributes = '', content = '', offset) => {
const no_change = () => StringWithSourcemap.from_source(
filename, match, get_location(offset));
if (!attributes && !content) {
return no_change();
}
attributes = attributes || '';
content = content || '';
// run script preprocessor
const processed = await preprocessor({
content,
attributes: parse_attributes(attributes),
filename
});
if (!processed) return no_change();
if (processed.dependencies) dependencies.push(...processed.dependencies);
return get_replacement(filename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, `</${tag_name}>`);
}
);
source = res.string;
sourcemap_list.unshift(res.map);
: undefined,
dependencies: processed.dependencies
});
}
for (const fn of script) {
await preprocess_tag_content('script', fn);
for (const process of script) {
result.update_source(await process_tag('script', process, result));
}
for (const fn of style) {
await preprocess_tag_content('style', fn);
for (const preprocess of style) {
result.update_source(await process_tag('style', preprocess, result));
}
// Combine all the source maps for each preprocessor function into one
const map: RawSourceMap = combine_sourcemaps(
filename,
sourcemap_list
);
return {
// TODO return separated output, in future version where svelte.compile supports it:
// style: { code: styleCode, map: styleMap },
// script { code: scriptCode, map: scriptMap },
// markup { code: markupCode, map: markupMap },
code: source,
dependencies: [...new Set(dependencies)],
map: (map as object),
toString() {
return source;
}
};
return result.processed();
}

@ -0,0 +1,14 @@
export default function parse_tag_attributes(str: string) {
const attrs = {};
str.split(/\s+/).filter(Boolean).forEach(attr => {
const p = attr.indexOf('=');
if (p === -1) {
attrs[attr] = true;
} else {
attrs[attr.slice(0, p)] = '\'"'.includes(attr[p + 1]) ?
attr.slice(p + 2, -1) :
attr.slice(p + 1);
}
});
return attrs;
}

@ -0,0 +1,68 @@
import { getLocator } from 'locate-character';
import { StringWithSourcemap } from '../utils/string_with_sourcemap';
export interface Source {
source: string;
get_location: ReturnType<typeof getLocator>;
filename: string;
}
interface Replacement {
offset: number;
length: number;
replacement: StringWithSourcemap;
}
function calculate_replacements(
re: RegExp,
get_replacement: (...match: any[]) => Promise<StringWithSourcemap>,
source: string
) {
const replacements: Array<Promise<Replacement>> = [];
source.replace(re, (...match) => {
replacements.push(
get_replacement(...match).then(
replacement => {
const matched_string = match[0];
const offset = match[match.length-2];
return ({ offset, length: matched_string.length, replacement });
}
)
);
return '';
});
return Promise.all(replacements);
}
function perform_replacements(
replacements: Replacement[],
{ filename, source, get_location }: Source
): StringWithSourcemap {
const out = new StringWithSourcemap();
let last_end = 0;
for (const { offset, length, replacement } of replacements) {
const unchanged_prefix = StringWithSourcemap.from_source(
filename, source.slice(last_end, offset), get_location(last_end));
out.concat(unchanged_prefix).concat(replacement);
last_end = offset + length;
}
const unchanged_suffix = StringWithSourcemap.from_source(
filename, source.slice(last_end), get_location(last_end));
return out.concat(unchanged_suffix);
}
export async function replace_in_code(
regex: RegExp,
get_replacement: (...match: any[]) => Promise<StringWithSourcemap>,
location: Source
): Promise<StringWithSourcemap> {
const replacements = await calculate_replacements(regex, get_replacement, location.source);
return perform_replacements(replacements, location);
}

@ -0,0 +1,18 @@
export interface Processed {
code: string;
map?: string | object; // we are opaque with the type here to avoid dependency on the remapping module for our public types.
dependencies?: string[];
toString?: () => string;
}
export interface PreprocessorGroup {
markup?: (options: { content: string; filename: string }) => Processed | Promise<Processed>;
style?: Preprocessor;
script?: Preprocessor;
}
export type Preprocessor = (options: {
content: string;
attributes: Record<string, string | boolean>;
filename?: string;
}) => Processed | Promise<Processed>;

@ -2,7 +2,7 @@ import { DecodedSourceMap, RawSourceMap, SourceMapLoader } from '@ampproject/rem
import remapping from '@ampproject/remapping';
import { SourceMap } from 'magic-string';
type SourceLocation = {
export type SourceLocation = {
line: number;
column: number;
};

Loading…
Cancel
Save