diff --git a/test/preprocess/index.js b/test/preprocess/index.js
index 10661efd39..9d1e8b01c0 100644
--- a/test/preprocess/index.js
+++ b/test/preprocess/index.js
@@ -1,226 +1,25 @@
+import * as fs from 'fs';
 import * as assert from 'assert';
-import { svelte } from '../helpers.js';
+import { loadConfig, svelte } from '../helpers.js';
 
-describe('preprocess', () => {
-	it('preprocesses entire component', () => {
-		const source = `
-			<h1>Hello __NAME__!</h1>
-		`;
+describe.only('preprocess', () => {
+	fs.readdirSync('test/preprocess/samples').forEach(dir => {
+		if (dir[0] === '.') return;
 
-		const expected = `
-			<h1>Hello world!</h1>
-		`;
+		const config = loadConfig(`./preprocess/samples/${dir}/_config.js`);
 
-		return svelte.preprocess(source, {
-			markup: ({ content }) => {
-				return {
-					code: content.replace('__NAME__', 'world')
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses style', () => {
-		const source = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: $brand;
-				}
-			</style>
-		`;
-
-		const expected = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: purple;
-				}
-			</style>
-		`;
-
-		return svelte.preprocess(source, {
-			style: ({ content }) => {
-				return {
-					code: content.replace('$brand', 'purple')
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses style asynchronously', () => {
-		const source = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: $brand;
-				}
-			</style>
-		`;
-
-		const expected = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: purple;
-				}
-			</style>
-		`;
-
-		return svelte.preprocess(source, {
-			style: ({ content }) => {
-				return Promise.resolve({
-					code: content.replace('$brand', 'purple')
-				});
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses script', () => {
-		const source = `
-			<script>
-				console.log(__THE_ANSWER__);
-			</script>
-		`;
-
-		const expected = `
-			<script>
-				console.log(42);
-			</script>
-		`;
-
-		return svelte.preprocess(source, {
-			script: ({ content }) => {
-				return {
-					code: content.replace('__THE_ANSWER__', '42')
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses multiple matching tags', () => {
-		const source = `
-			<script>
-				REPLACEME
-			</script>
-			<style>
-				SHOULD NOT BE REPLACED
-			</style>
-			<script>
-				REPLACEMETOO
-			</script>
-		`;
-
-		const expected = `
-			<script>
-				replaceme
-			</script>
-			<style>
-				SHOULD NOT BE REPLACED
-			</style>
-			<script>
-				replacemetoo
-			</script>
-		`;
-
-		return svelte.preprocess(source, {
-			script: ({ content }) => {
-				return {
-					code: content.toLowerCase()
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('parses attributes', () => {
-		const source = `
-			<style type='text/scss' data-foo="bar" bool></style>
-		`;
-
-		const expected = `
-			<style type='text/scss' data-foo="bar" bool>PROCESSED</style>
-		`;
-
-		return svelte.preprocess(source, {
-			style: ({ attributes }) => {
-				assert.deepEqual(attributes, {
-					type: 'text/scss',
-					'data-foo': 'bar',
-					bool: true
-				});
-				return { code: 'PROCESSED' };
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('provides filename to processing hooks', () => {
-		const source = `
-			<h1>Hello __MARKUP_FILENAME__!</h1>
-			<style>.red { color: __STYLE_FILENAME__; }</style>
-			<script>console.log('__SCRIPT_FILENAME__');</script>
-		`;
-
-		const expected = `
-			<h1>Hello file.html!</h1>
-			<style>.red { color: file.html; }</style>
-			<script>console.log('file.html');</script>
-		`;
-
-		return svelte.preprocess(source, {
-			filename: 'file.html',
-			markup: ({ content, filename }) => {
-				return {
-					code: content.replace('__MARKUP_FILENAME__', filename)
-				};
-			},
-			style: ({ content, filename }) => {
-				return {
-					code: content.replace('__STYLE_FILENAME__', filename)
-				};
-			},
-			script: ({ content, filename }) => {
-				return {
-					code: content.replace('__SCRIPT_FILENAME__', filename)
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
+		if (config.solo && process.env.CI) {
+			throw new Error('Forgot to remove `solo: true` from test');
+		}
 
-	it('ignores null/undefined returned from preprocessor', () => {
-		const source = `
-			<script>
-				console.log('ignore me');
-			</script>
-		`;
+		(config.skip ? it.skip : config.solo ? it.only : it)(dir, async () => {
+			const input = fs.readFileSync(`test/preprocess/samples/${dir}/input.html`, 'utf-8');
+			const expected = fs.readFileSync(`test/preprocess/samples/${dir}/output.html`, 'utf-8');
 
-		const expected = `
-			<script>
-				console.log('ignore me');
-			</script>
-		`;
+			const actual = await svelte.preprocess(input, config.preprocess);
+			fs.writeFileSync(`test/preprocess/samples/${dir}/_actual.html`, actual);
 
-		return svelte.preprocess(source, {
-			script: () => null
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
+			assert.equal(actual, expected);
 		});
 	});
 });
diff --git a/test/preprocess/samples/filename/_config.js b/test/preprocess/samples/filename/_config.js
new file mode 100644
index 0000000000..d7a3c19770
--- /dev/null
+++ b/test/preprocess/samples/filename/_config.js
@@ -0,0 +1,20 @@
+export default {
+	preprocess: {
+		filename: 'file.html',
+		markup: ({ content, filename }) => {
+			return {
+				code: content.replace('__MARKUP_FILENAME__', filename)
+			};
+		},
+		style: ({ content, filename }) => {
+			return {
+				code: content.replace('__STYLE_FILENAME__', filename)
+			};
+		},
+		script: ({ content, filename }) => {
+			return {
+				code: content.replace('__SCRIPT_FILENAME__', filename)
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/filename/input.html b/test/preprocess/samples/filename/input.html
new file mode 100644
index 0000000000..f03a890f6e
--- /dev/null
+++ b/test/preprocess/samples/filename/input.html
@@ -0,0 +1,3 @@
+<h1>Hello __MARKUP_FILENAME__!</h1>
+<style>.red { color: __STYLE_FILENAME__; }</style>
+<script>console.log('__SCRIPT_FILENAME__');</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/filename/output.html b/test/preprocess/samples/filename/output.html
new file mode 100644
index 0000000000..90fa180704
--- /dev/null
+++ b/test/preprocess/samples/filename/output.html
@@ -0,0 +1,3 @@
+<h1>Hello file.html!</h1>
+<style>.red { color: file.html; }</style>
+<script>console.log('file.html');</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/ignores-null/_config.js b/test/preprocess/samples/ignores-null/_config.js
new file mode 100644
index 0000000000..042ee89aed
--- /dev/null
+++ b/test/preprocess/samples/ignores-null/_config.js
@@ -0,0 +1,5 @@
+export default {
+	preprocess: {
+		script: () => null
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/ignores-null/input.html b/test/preprocess/samples/ignores-null/input.html
new file mode 100644
index 0000000000..b3fc25b643
--- /dev/null
+++ b/test/preprocess/samples/ignores-null/input.html
@@ -0,0 +1,3 @@
+<script>
+	console.log('ignore me');
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/ignores-null/output.html b/test/preprocess/samples/ignores-null/output.html
new file mode 100644
index 0000000000..b3fc25b643
--- /dev/null
+++ b/test/preprocess/samples/ignores-null/output.html
@@ -0,0 +1,3 @@
+<script>
+	console.log('ignore me');
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/markup/_config.js b/test/preprocess/samples/markup/_config.js
new file mode 100644
index 0000000000..857da608af
--- /dev/null
+++ b/test/preprocess/samples/markup/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		markup: ({ content }) => {
+			return {
+				code: content.replace('__NAME__', 'world')
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/markup/input.html b/test/preprocess/samples/markup/input.html
new file mode 100644
index 0000000000..d65a1a50a6
--- /dev/null
+++ b/test/preprocess/samples/markup/input.html
@@ -0,0 +1 @@
+<h1>Hello __NAME__!</h1>
\ No newline at end of file
diff --git a/test/preprocess/samples/markup/output.html b/test/preprocess/samples/markup/output.html
new file mode 100644
index 0000000000..efe5048cbc
--- /dev/null
+++ b/test/preprocess/samples/markup/output.html
@@ -0,0 +1 @@
+<h1>Hello world!</h1>
\ No newline at end of file
diff --git a/test/preprocess/samples/script-multiple/_config.js b/test/preprocess/samples/script-multiple/_config.js
new file mode 100644
index 0000000000..d6656ff48e
--- /dev/null
+++ b/test/preprocess/samples/script-multiple/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		script: ({ content }) => {
+			return {
+				code: content.toLowerCase()
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/script-multiple/input.html b/test/preprocess/samples/script-multiple/input.html
new file mode 100644
index 0000000000..5a5221dc84
--- /dev/null
+++ b/test/preprocess/samples/script-multiple/input.html
@@ -0,0 +1,9 @@
+<script>
+	REPLACEME
+</script>
+<style>
+	SHOULD NOT BE REPLACED
+</style>
+<script>
+	REPLACEMETOO
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/script-multiple/output.html b/test/preprocess/samples/script-multiple/output.html
new file mode 100644
index 0000000000..64fe4c1f25
--- /dev/null
+++ b/test/preprocess/samples/script-multiple/output.html
@@ -0,0 +1,9 @@
+<script>
+	replaceme
+</script>
+<style>
+	SHOULD NOT BE REPLACED
+</style>
+<script>
+	replacemetoo
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/script/_config.js b/test/preprocess/samples/script/_config.js
new file mode 100644
index 0000000000..8e8ac1d2f9
--- /dev/null
+++ b/test/preprocess/samples/script/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		script: ({ content }) => {
+			return {
+				code: content.replace('__THE_ANSWER__', '42')
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/script/input.html b/test/preprocess/samples/script/input.html
new file mode 100644
index 0000000000..8236f6d9d2
--- /dev/null
+++ b/test/preprocess/samples/script/input.html
@@ -0,0 +1,3 @@
+<script>
+	console.log(__THE_ANSWER__);
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/script/output.html b/test/preprocess/samples/script/output.html
new file mode 100644
index 0000000000..5bef15aab0
--- /dev/null
+++ b/test/preprocess/samples/script/output.html
@@ -0,0 +1,3 @@
+<script>
+	console.log(42);
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-async/_config.js b/test/preprocess/samples/style-async/_config.js
new file mode 100644
index 0000000000..a91f8f4307
--- /dev/null
+++ b/test/preprocess/samples/style-async/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		style: ({ content }) => {
+			return Promise.resolve({
+				code: content.replace('$brand', 'purple')
+			});
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/style-async/input.html b/test/preprocess/samples/style-async/input.html
new file mode 100644
index 0000000000..303ac6366a
--- /dev/null
+++ b/test/preprocess/samples/style-async/input.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: $brand;
+	}
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-async/output.html b/test/preprocess/samples/style-async/output.html
new file mode 100644
index 0000000000..2847cc525a
--- /dev/null
+++ b/test/preprocess/samples/style-async/output.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: purple;
+	}
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-attributes/_config.js b/test/preprocess/samples/style-attributes/_config.js
new file mode 100644
index 0000000000..7a619054bd
--- /dev/null
+++ b/test/preprocess/samples/style-attributes/_config.js
@@ -0,0 +1,14 @@
+import * as assert from 'assert';
+
+export default {
+	preprocess: {
+		style: ({ attributes }) => {
+			assert.deepEqual(attributes, {
+				type: 'text/scss',
+				'data-foo': 'bar',
+				bool: true
+			});
+			return { code: 'PROCESSED' };
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/style-attributes/input.html b/test/preprocess/samples/style-attributes/input.html
new file mode 100644
index 0000000000..3a55a5a3f6
--- /dev/null
+++ b/test/preprocess/samples/style-attributes/input.html
@@ -0,0 +1 @@
+<style type='text/scss' data-foo="bar" bool></style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-attributes/output.html b/test/preprocess/samples/style-attributes/output.html
new file mode 100644
index 0000000000..c83defeebb
--- /dev/null
+++ b/test/preprocess/samples/style-attributes/output.html
@@ -0,0 +1 @@
+<style type='text/scss' data-foo="bar" bool>PROCESSED</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style/_config.js b/test/preprocess/samples/style/_config.js
new file mode 100644
index 0000000000..59040eccdd
--- /dev/null
+++ b/test/preprocess/samples/style/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		style: ({ content }) => {
+			return {
+				code: content.replace('$brand', 'purple')
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/style/input.html b/test/preprocess/samples/style/input.html
new file mode 100644
index 0000000000..303ac6366a
--- /dev/null
+++ b/test/preprocess/samples/style/input.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: $brand;
+	}
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style/output.html b/test/preprocess/samples/style/output.html
new file mode 100644
index 0000000000..2847cc525a
--- /dev/null
+++ b/test/preprocess/samples/style/output.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: purple;
+	}
+</style>
\ No newline at end of file