diff --git a/test/runtime/index.js b/test/runtime/index.js
index 880e316e0c..f23379cd88 100644
--- a/test/runtime/index.js
+++ b/test/runtime/index.js
@@ -192,10 +192,12 @@ describe("runtime", () => {
 							assert.equal(config.error, err.message);
 						}
 					} else {
-						failed.add(dir);
-						showOutput(cwd, compileOptions, compile); // eslint-disable-line no-console
 						throw err;
 					}
+				 }).catch(err => {
+					failed.add(dir);
+					showOutput(cwd, compileOptions, compile); // eslint-disable-line no-console
+					throw err;
 				})
 				.then(() => {
 					if (config.show) {