Convert src/compiler/preprocess/index.ts to JavaScript

pull/8569/head
Simon Holthausen 3 years ago
parent 1069a2be21
commit a172a75427

@ -1,97 +1,106 @@
import type { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping';
import { getLocator } from 'locate-character'; import { getLocator } from 'locate-character';
import { import { MappedCode, parse_attached_sourcemap, sourcemap_add_offset, combine_sourcemaps } from '../utils/mapped_code';
MappedCode, import { decode_map } from './decode_sourcemap.js';
SourceLocation,
parse_attached_sourcemap,
sourcemap_add_offset,
combine_sourcemaps
} from '../utils/mapped_code';
import { decode_map } from './decode_sourcemap';
import { replace_in_code, slice_source } from './replace_in_code'; import { replace_in_code, slice_source } from './replace_in_code';
import { MarkupPreprocessor, Source, Preprocessor, PreprocessorGroup, Processed } from './types';
import { regex_whitespaces } from '../utils/patterns'; import { regex_whitespaces } from '../utils/patterns';
export * from './types'; export * from './types';
interface SourceUpdate {
string?: string;
map?: DecodedSourceMap;
dependencies?: string[];
}
const regex_filepath_separator = /[/\\]/; const regex_filepath_separator = /[/\\]/;
function get_file_basename(filename: string) { /**
* @param {string} filename
*/
function get_file_basename(filename) {
return filename.split(regex_filepath_separator).pop(); return filename.split(regex_filepath_separator).pop();
} }
/** /**
* Represents intermediate states of the preprocessing. * Represents intermediate states of the preprocessing.
*/ */
class PreprocessResult implements Source { class PreprocessResult {
source;
filename;
// sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1)
// so we use sourcemap_list.unshift() to add new maps // so we use sourcemap_list.unshift() to add new maps
// https://github.com/ampproject/remapping#multiple-transformations-of-a-file // https://github.com/ampproject/remapping#multiple-transformations-of-a-file
sourcemap_list: Array<DecodedSourceMap | RawSourceMap> = [];
dependencies: string[] = [];
file_basename: string;
get_location: ReturnType<typeof getLocator>; /**
* @default []
* @type {Array<DecodedSourceMap | RawSourceMap>}
*/
sourcemap_list = [];
constructor(public source: string, public filename?: string) { /**
this.update_source({ string: source }); * @default []
* @type {string[]}
*/
dependencies = [];
/**
* @type {string}
*/
file_basename = undefined;
/**
* @type {Class<getLocator>>}
*/
get_location = undefined;
constructor(source, filename) {
this.source = source;
this.filename = filename;
this.update_source({ string: source });
// preprocess source must be relative to itself or equal null // preprocess source must be relative to itself or equal null
this.file_basename = filename == null ? null : get_file_basename(filename); this.file_basename = filename == null ? null : get_file_basename(filename);
} }
update_source({ string: source, map, dependencies }: SourceUpdate) { /**
* @param {SourceUpdate}
*/
update_source({ string: source, map, dependencies }) {
if (source != null) { if (source != null) {
this.source = source; this.source = source;
this.get_location = getLocator(source); this.get_location = getLocator(source);
} }
if (map) { if (map) {
this.sourcemap_list.unshift(map); this.sourcemap_list.unshift(map);
} }
if (dependencies) { if (dependencies) {
this.dependencies.push(...dependencies); this.dependencies.push(...dependencies);
} }
} }
to_processed(): Processed { /**
* @returns {Processed}
*/
to_processed() {
// Combine all the source maps for each preprocessor function into one // Combine all the source maps for each preprocessor function into one
const map = combine_sourcemaps(this.file_basename, this.sourcemap_list); const map = combine_sourcemaps(this.file_basename, this.sourcemap_list);
return { return {
// TODO return separated output, in future version where svelte.compile supports it: // TODO return separated output, in future version where svelte.compile supports it:
// style: { code: styleCode, map: styleMap }, // style: { code: styleCode, map: styleMap },
// script { code: scriptCode, map: scriptMap }, // script { code: scriptCode, map: scriptMap },
// markup { code: markupCode, map: markupMap }, // markup { code: markupCode, map: markupMap },
code: this.source, code: this.source,
dependencies: [...new Set(this.dependencies)], dependencies: [...new Set(this.dependencies)],
map: map as object, map: map,
toString: () => this.source toString: () => this.source
}; };
} }
} }
/** /**
* Convert preprocessor output for the tag content into MappedCode * Convert preprocessor output for the tag content into MappedCode
* @param {Processed} processed
* @param {SourceLocation} location
* @param {string} file_basename
* @returns {MappedCode}
*/ */
function processed_content_to_code( function processed_content_to_code(processed, location, file_basename) {
processed: Processed,
location: SourceLocation,
file_basename: string
): MappedCode {
// Convert the preprocessed code and its sourcemap to a MappedCode // Convert the preprocessed code and its sourcemap to a MappedCode
let decoded_map: DecodedSourceMap;
/**
* @type {DecodedSourceMap}
*/
let decoded_map;
if (processed.map) { if (processed.map) {
decoded_map = decode_map(processed); decoded_map = decode_map(processed);
// decoded map may not have sources for empty maps like `{ mappings: '' }` // decoded map may not have sources for empty maps like `{ mappings: '' }`
if (decoded_map.sources) { if (decoded_map.sources) {
// offset only segments pointing at original component source // offset only segments pointing at original component source
@ -101,45 +110,39 @@ function processed_content_to_code(
} }
} }
} }
return MappedCode.from_processed(processed.code, decoded_map); return MappedCode.from_processed(processed.code, decoded_map);
} }
/** /**
* Given the whole tag including content, return a `MappedCode` * Given the whole tag including content, return a `MappedCode`
* representing the tag content replaced with `processed`. * representing the tag content replaced with `processed`.
* @param {Processed} processed
* @param {'style' | 'script'} tag_name
* @param {string} attributes
* @param {Source} source
* @returns {MappedCode}
*/ */
function processed_tag_to_code( function processed_tag_to_code(processed, tag_name, attributes, source) {
processed: Processed,
tag_name: 'style' | 'script',
attributes: string,
source: Source
): MappedCode {
const { file_basename, get_location } = source; const { file_basename, get_location } = source;
const build_mapped_code = (code: string, offset: number) => /**
MappedCode.from_source(slice_source(code, offset, source)); * @param {string} code
* @param {number} offset
*/
const build_mapped_code = (code, offset) => MappedCode.from_source(slice_source(code, offset, source));
const tag_open = `<${tag_name}${attributes || ''}>`; const tag_open = `<${tag_name}${attributes || ''}>`;
const tag_close = `</${tag_name}>`; const tag_close = `</${tag_name}>`;
const tag_open_code = build_mapped_code(tag_open, 0); const tag_open_code = build_mapped_code(tag_open, 0);
const tag_close_code = build_mapped_code(tag_close, tag_open.length + source.source.length); const tag_close_code = build_mapped_code(tag_close, tag_open.length + source.source.length);
parse_attached_sourcemap(processed, tag_name); parse_attached_sourcemap(processed, tag_name);
const content_code = processed_content_to_code(processed, get_location(tag_open.length), file_basename);
const content_code = processed_content_to_code(
processed,
get_location(tag_open.length),
file_basename
);
return tag_open_code.concat(content_code).concat(tag_close_code); return tag_open_code.concat(content_code).concat(tag_close_code);
} }
const regex_quoted_value = /^['"](.*)['"]$/; const regex_quoted_value = /^['"](.*)['"]$/;
function parse_tag_attributes(str: string) { /**
* @param {string} str
*/
function parse_tag_attributes(str) {
// note: won't work with attribute values containing spaces. // note: won't work with attribute values containing spaces.
return str return str
.split(regex_whitespaces) .split(regex_whitespaces)
@ -148,68 +151,63 @@ function parse_tag_attributes(str: string) {
const i = attr.indexOf('='); const i = attr.indexOf('=');
const [key, value] = i > 0 ? [attr.slice(0, i), attr.slice(i + 1)] : [attr]; const [key, value] = i > 0 ? [attr.slice(0, i), attr.slice(i + 1)] : [attr];
const [, unquoted] = (value && value.match(regex_quoted_value)) || []; const [, unquoted] = (value && value.match(regex_quoted_value)) || [];
return { ...attrs, [key]: unquoted ?? value ?? true }; return { ...attrs, [key]: unquoted ?? value ?? true };
}, {}); }, {});
} }
const regex_style_tags = /<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi; const regex_style_tags = /<!--[^]*?-->|<style(\s[^]*?)?(?:>([^]*?)<\/style>|\/>)/gi;
const regex_script_tags = /<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi; const regex_script_tags = /<!--[^]*?-->|<script(\s[^]*?)?(?:>([^]*?)<\/script>|\/>)/gi;
/** /**
* Calculate the updates required to process all instances of the specified tag. * Calculate the updates required to process all instances of the specified tag.
* @param {'style' | 'script'} tag_name
* @param {Preprocessor} preprocessor
* @param {Source} source
* @returns {Promise<SourceUpdate>}
*/ */
async function process_tag( async function process_tag(tag_name, preprocessor, source) {
tag_name: 'style' | 'script',
preprocessor: Preprocessor,
source: Source
): Promise<SourceUpdate> {
const { filename, source: markup } = source; const { filename, source: markup } = source;
const tag_regex = tag_name === 'style' ? regex_style_tags : regex_script_tags; const tag_regex = tag_name === 'style' ? regex_style_tags : regex_script_tags;
const dependencies: string[] = []; /**
* @type {string[]}
async function process_single_tag( */
tag_with_content: string, const dependencies = [];
attributes = '',
content = '',
tag_offset: number
): Promise<MappedCode> {
const no_change = () =>
MappedCode.from_source(slice_source(tag_with_content, tag_offset, source));
if (!attributes && !content) return no_change();
/**
* @param {string} tag_with_content
* @param {number} tag_offset
* @returns {Promise<MappedCode>}
*/
async function process_single_tag(tag_with_content, attributes = '', content = '', tag_offset) {
const no_change = () => MappedCode.from_source(slice_source(tag_with_content, tag_offset, source));
if (!attributes && !content)
return no_change();
const processed = await preprocessor({ const processed = await preprocessor({
content: content || '', content: content || '',
attributes: parse_tag_attributes(attributes || ''), attributes: parse_tag_attributes(attributes || ''),
markup, markup,
filename filename
}); });
if (!processed)
if (!processed) return no_change(); return no_change();
if (processed.dependencies) dependencies.push(...processed.dependencies); if (processed.dependencies)
if (!processed.map && processed.code === content) return no_change(); dependencies.push(...processed.dependencies);
if (!processed.map && processed.code === content)
return processed_tag_to_code( return no_change();
processed, return processed_tag_to_code(processed, tag_name, attributes, slice_source(content, tag_offset, source));
tag_name,
attributes,
slice_source(content, tag_offset, source)
);
} }
const { string, map } = await replace_in_code(tag_regex, process_single_tag, source); const { string, map } = await replace_in_code(tag_regex, process_single_tag, source);
return { string, map, dependencies }; return { string, map, dependencies };
} }
async function process_markup(process: MarkupPreprocessor, source: Source) { /**
* @param {MarkupPreprocessor} process
* @param {Source} source
*/
async function process_markup(process, source) {
const processed = await process({ const processed = await process({
content: source.source, content: source.source,
filename: source.filename filename: source.filename
}); });
if (processed) { if (processed) {
return { return {
string: processed.code, string: processed.code,
@ -221,45 +219,52 @@ async function process_markup(process: MarkupPreprocessor, source: Source) {
: undefined, : undefined,
dependencies: processed.dependencies dependencies: processed.dependencies
}; };
} else { }
else {
return {}; return {};
} }
} }
export default async function preprocess( /**
source: string, * @param {string} source
preprocessor: PreprocessorGroup | PreprocessorGroup[], * @param {PreprocessorGroup | PreprocessorGroup[]} preprocessor
options?: { filename?: string } * @param {{ filename?: string }} options
): Promise<Processed> { * @returns {Promise<Processed>}
const filename: string | undefined = */
(options && options.filename) || (preprocessor as any).filename; // legacy export default async function preprocess(source, preprocessor, options) {
/**
* @type {string | undefined}
*/
const filename = (options && options.filename) || preprocessor.filename; // legacy
const preprocessors = preprocessor const preprocessors = preprocessor
? Array.isArray(preprocessor) ? Array.isArray(preprocessor)
? preprocessor ? preprocessor
: [preprocessor] : [preprocessor]
: []; : [];
const markup = preprocessors.map((p) => p.markup).filter(Boolean); const markup = preprocessors.map((p) => p.markup).filter(Boolean);
const script = preprocessors.map((p) => p.script).filter(Boolean); const script = preprocessors.map((p) => p.script).filter(Boolean);
const style = preprocessors.map((p) => p.style).filter(Boolean); const style = preprocessors.map((p) => p.style).filter(Boolean);
const result = new PreprocessResult(source, filename); const result = new PreprocessResult(source, filename);
// TODO keep track: what preprocessor generated what sourcemap? // TODO keep track: what preprocessor generated what sourcemap?
// to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings // to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings
for (const process of markup) { for (const process of markup) {
result.update_source(await process_markup(process, result)); result.update_source(await process_markup(process, result));
} }
for (const process of script) { for (const process of script) {
result.update_source(await process_tag('script', process, result)); result.update_source(await process_tag('script', process, result));
} }
for (const preprocess of style) { for (const preprocess of style) {
result.update_source(await process_tag('style', preprocess, result)); result.update_source(await process_tag('style', preprocess, result));
} }
return result.to_processed(); return result.to_processed();
} }
/** @typedef {Object} SourceUpdate
* @property {string} [string]
* @property {DecodedSourceMap} [map]
* @property {string[]} [dependencies]
*/
Loading…
Cancel
Save