[chore] fix test flakiness (#7076)

package.json

@@ -83,8 +83,8 @@
   },
   "types": "types/runtime/index.d.ts",
   "scripts": {
-    "test": "mocha",
-    "test:unit": "mocha --require sucrase/register --recursive src/**/__test__.ts",
+    "test": "mocha --exit",
+    "test:unit": "mocha --require sucrase/register --recursive src/**/__test__.ts --exit",
     "quicktest": "mocha",
     "precoverage": "c8 mocha",
     "coverage": "c8 report --reporter=text-lcov > coverage.lcov && c8 report --reporter=html",

test/custom-elements/index.ts

@@ -4,7 +4,7 @@ import * as http from 'http';
 import { rollup } from 'rollup';
 import virtual from '@rollup/plugin-virtual';
 import puppeteer from 'puppeteer';
-import { addLineNumbers, loadConfig, loadSvelte } from '../helpers';
+import { addLineNumbers, loadConfig, loadSvelte, retryAsync, executeBrowserTest } from '../helpers';
 import { deepEqual } from 'assert';
 
 const page = `
@@ -17,8 +17,8 @@ const page = `
 const assert = fs.readFileSync(`${__dirname}/assert.js`, 'utf-8');
 
 describe('custom-elements', function() {
+	// Note: Increase the timeout in preparation for restarting Chromium due to SIGSEGV.
 	this.timeout(10000);
-
 	let svelte;
 	let server;
 	let browser;
@@ -44,12 +44,16 @@ describe('custom-elements', function() {
 		});
 	}
 
+	async function launchPuppeteer() {
+		return await retryAsync(() => puppeteer.launch());
+	}
+
 	before(async () => {
 		svelte = loadSvelte();
 		console.log('[custom-element] Loaded Svelte');
 		server = await create_server();
 		console.log('[custom-element] Started server');
-		browser = await puppeteer.launch();
+		browser = await launchPuppeteer();
 		console.log('[custom-element] Launched puppeteer browser');
 	});
 
@@ -108,26 +112,7 @@ describe('custom-elements', function() {
 			const result = await bundle.generate({ format: 'iife', name: 'test' });
 			code = result.output[0].code;
 
-			const page = await browser.newPage();
-
-			page.on('console', (type) => {
-				console[type._type](type._text);
-			});
-
-			page.on('error', error => {
-				console.log('>>> an error happened');
-				console.error(error);
-			});
-
-			try {
-				await page.goto('http://localhost:6789');
-
-				const result = await page.evaluate(() => test(document.querySelector('main')));
-				if (result) console.log(result);
-			} catch (err) {
-				console.log(addLineNumbers(code));
-				throw err;
-			} finally {
+			function assertWarnings() {
 				if (expected_warnings) {
 					deepEqual(warnings.map(w => ({
 						code: w.code,
@@ -138,6 +123,15 @@ describe('custom-elements', function() {
 					})), expected_warnings);
 				}
 			}
+
+			browser = await executeBrowserTest(
+				browser,
+				launchPuppeteer,
+				assertWarnings,
+				() => {
+					console.log(addLineNumbers(code));
+					assertWarnings();
+				});
 		});
 	});
 });

test/helpers.ts

@@ -8,7 +8,7 @@ export const assert = (assert$1 as unknown) as typeof assert$1 & { htmlEqual: (a
 
 // for coverage purposes, we need to test source files,
 // but for sanity purposes, we need to test dist files
-export function loadSvelte(test) {
+export function loadSvelte(test: boolean = false) {
 	process.env.TEST = test ? 'true' : '';
 
 	const resolved = require.resolve('../compiler.js');
@@ -55,7 +55,7 @@ export function cleanRequireCache() {
 const virtualConsole = new jsdom.VirtualConsole();
 virtualConsole.sendTo(console);
 
-const window = new jsdom.JSDOM('<main></main>', {virtualConsole}).window;
+const window = new jsdom.JSDOM('<main></main>', { virtualConsole }).window;
 global.document = window.document;
 global.navigator = window.navigator;
 global.getComputedStyle = window.getComputedStyle;
@@ -279,3 +279,50 @@ export function prettyPrintPuppeteerAssertionError(message) {
 		assert.equal(match[1], match[2]);
 	}
 }
+
+export async function retryAsync<T>(fn: () => Promise<T>, maxAttempts: number = 3, interval: number = 1000): Promise<T> {
+	let attempts = 0;
+	while (attempts <= maxAttempts) {
+		try {
+			return await fn();
+		} catch (err) {
+			if (++attempts >= maxAttempts) throw err;
+			await new Promise(resolve => setTimeout(resolve, interval));
+		}
+	}
+}
+
+// NOTE: Chromium may exit with SIGSEGV, so retry in that case
+export async function executeBrowserTest<T>(browser, launchPuppeteer: () => Promise<T>, additionalAssertion: () => void, onError: (err: Error) => void) {
+	let count = 0;
+	do {
+		count++;
+		try {
+			const page = await browser.newPage();
+
+			page.on('console', (type) => {
+				console[type._type](type._text);
+			});
+
+			page.on('error', error => {
+				console.log('>>> an error happened');
+				console.error(error);
+			});
+			await page.goto('http://localhost:6789');
+			const result = await page.evaluate(() => test(document.querySelector('main')));
+			if (result) console.log(result);
+			additionalAssertion();
+			await page.close();
+			break;
+		} catch (err) {
+			if (count === 5 || browser.isConnected()) {
+				onError(err);
+				throw err;
+			}
+			console.debug(err.stack || err);
+			console.log('RESTARTING Chromium...');
+			browser = await launchPuppeteer();
+		}
+	} while (count <= 5);
+	return browser;
+}

test/runtime-puppeteer/index.ts

@@ -9,7 +9,9 @@ import {
 	loadConfig,
 	loadSvelte,
 	mkdirp,
-	prettyPrintPuppeteerAssertionError
+	prettyPrintPuppeteerAssertionError,
+	retryAsync,
+	executeBrowserTest
 } from '../helpers';
 import { deepEqual } from 'assert';
 
@@ -48,15 +50,21 @@ function create_server() {
 	});
 }
 
+async function launchPuppeteer() {
+	return await retryAsync(() => puppeteer.launch());
+}
+
 const assert = fs.readFileSync(`${__dirname}/assert.js`, 'utf-8');
 
-describe('runtime (puppeteer)', () => {
+describe('runtime (puppeteer)', function() {
+	// Note: Increase the timeout in preparation for restarting Chromium due to SIGSEGV.
+	this.timeout(10000);
 	before(async () => {
 		svelte = loadSvelte(false);
 		console.log('[runtime-puppeteer] Loaded Svelte');
 		server = await create_server();
 		console.log('[runtime-puppeteer] Started server');
-		browser = await puppeteer.launch();
+		browser = await launchPuppeteer();
 		console.log('[runtime-puppeteer] Launched puppeteer browser');
 	});
 
@@ -205,27 +213,7 @@ describe('runtime (puppeteer)', () => {
 			const result = await bundle.generate({ format: 'iife', name: 'test' });
 			code = result.output[0].code;
 
-			const page = await browser.newPage();
-
-			page.on('console', (type) => {
-				console[type._type](type._text);
-			});
-
-			page.on('error', error => {
-				console.log('>>> an error happened');
-				console.error(error);
-			});
-
-			try {
-				await page.goto('http://localhost:6789');
-
-				const result = await page.evaluate(() => test(document.querySelector('main')));
-				if (result) console.log(result);
-			} catch (err) {
-				failed.add(dir);
-				prettyPrintPuppeteerAssertionError(err.message);
-				throw err;
-			} finally {
+			function assertWarnings() {
 				if (config.warnings) {
 					deepEqual(warnings.map(w => ({
 						code: w.code,
@@ -240,6 +228,16 @@ describe('runtime (puppeteer)', () => {
 					throw new Error('Received unexpected warnings');
 				}
 			}
+
+			browser = await executeBrowserTest(
+				browser,
+				launchPuppeteer,
+				assertWarnings,
+				(err) => {
+					failed.add(dir);
+					prettyPrintPuppeteerAssertionError(err.message);
+					assertWarnings();
+				});
 		});
 	}
 

test/runtime/index.ts

@@ -65,7 +65,7 @@ describe('runtime', () => {
 		}
 
 		const testName = `${dir} ${hydrate ? `(with hydration${from_ssr_html ? ' from ssr rendered html' : ''})` : ''}`;
-		(config.skip ? it.skip : solo ? it.only : it)(testName, () => {
+		(config.skip ? it.skip : solo ? it.only : it)(testName, (done) => {
 			if (failed.has(dir)) {
 				// this makes debugging easier, by only printing compiled output once
 				throw new Error('skipping test, already failed');
@@ -96,7 +96,7 @@ describe('runtime', () => {
 			glob('**/*.svelte', { cwd }).forEach(file => {
 				if (file[0] === '_') return;
 
-				const dir  = `${cwd}/_output/${hydrate ? 'hydratable' : 'normal'}`;
+				const dir = `${cwd}/_output/${hydrate ? 'hydratable' : 'normal'}`;
 				const out = `${dir}/${file.replace(/\.svelte$/, '.js')}`;
 
 				if (fs.existsSync(out)) {
@@ -120,7 +120,7 @@ describe('runtime', () => {
 				}
 			});
 
-			return Promise.resolve()
+			Promise.resolve()
 				.then(() => {
 					// hack to support transition tests
 					clear_loops();
@@ -245,6 +245,7 @@ describe('runtime', () => {
 				.catch(err => {
 					// print a clickable link to open the directory
 					err.stack += `\n\ncmd-click: ${path.relative(process.cwd(), cwd)}/main.svelte`;
+					done();
 					throw err;
 				})
 				.then(() => {
@@ -255,6 +256,7 @@ describe('runtime', () => {
 					flush();
 
 					if (config.after_test) config.after_test();
+					done();
 				});
 		});
 	}

test/server-side-rendering/index.ts

@@ -52,21 +52,23 @@ describe('ssr', () => {
 			throw new Error('Forgot to remove `solo: true` from test');
 		}
 
-		(solo ? it.only : it)(dir, () => {
-			dir = path.resolve(`${__dirname}/samples`, dir);
+		(solo ? it.only : it)(dir, (done) => {
 
-			cleanRequireCache();
+			try {
 
-			const compileOptions = {
-				sveltePath,
-				...config.compileOptions,
-				generate: 'ssr',
-				format: 'cjs'
-			};
+				dir = path.resolve(`${__dirname}/samples`, dir);
 
-			require('../../register')(compileOptions);
+				cleanRequireCache();
+
+				const compileOptions = {
+					sveltePath,
+					...config.compileOptions,
+					generate: 'ssr',
+					format: 'cjs'
+				};
+
+				require('../../register')(compileOptions);
 
-			try {
 				const Component = require(`${dir}/main.svelte`).default;
 
 				const expectedHtml = tryToReadFile(`${dir}/_expected.html`);
@@ -132,6 +134,7 @@ describe('ssr', () => {
 				throw err;
 			} finally {
 				set_current_component(null);
+				done();
 			}
 		});
 	});