diff --git a/GNUmakefile b/GNUmakefile
index f08bad676..adc90ac50 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -7,9 +7,9 @@ check: expected output
 
 output: $(OUTPUTS)
 
-output/%:
+output/%.txt: demos/%.lmp
 	@mkdir -p $(dir $@)
-	touch $@ # TODO
+	./testrunner $< $@
 
 extract/%:
 	unzip $(UNZIPOPTS) -d extract $< $(notdir $@)
diff --git a/demos/pwads/av/.democonfig b/demos/pwads/av/.democonfig
new file mode 100644
index 000000000..e4744cfba
--- /dev/null
+++ b/demos/pwads/av/.democonfig
@@ -0,0 +1,2 @@
+iwad: doom2.wad
+pwad: av.wad
diff --git a/demos/pwads/class_ep/.democonfig b/demos/pwads/class_ep/.democonfig
new file mode 100644
index 000000000..4874e4bb4
--- /dev/null
+++ b/demos/pwads/class_ep/.democonfig
@@ -0,0 +1,2 @@
+iwad: doom.wad
+pwad: class_ep.wad
diff --git a/demos/pwads/hr/.democonfig b/demos/pwads/hr/.democonfig
new file mode 100644
index 000000000..11370a748
--- /dev/null
+++ b/demos/pwads/hr/.democonfig
@@ -0,0 +1,2 @@
+iwad: doom2.wad
+pwad: hr.wad
diff --git a/demos/pwads/mm/.democonfig b/demos/pwads/mm/.democonfig
new file mode 100644
index 000000000..34f59e5f7
--- /dev/null
+++ b/demos/pwads/mm/.democonfig
@@ -0,0 +1,2 @@
+iwad: doom2.wad
+pwad: mm.wad
diff --git a/demos/pwads/mm2/.democonfig b/demos/pwads/mm2/.democonfig
new file mode 100644
index 000000000..f131fcab8
--- /dev/null
+++ b/demos/pwads/mm2/.democonfig
@@ -0,0 +1,2 @@
+iwad: doom2.wad
+pwad: mm2.wad
diff --git a/demos/pwads/requiem/.democonfig b/demos/pwads/requiem/.democonfig
new file mode 100644
index 000000000..2da4f18ec
--- /dev/null
+++ b/demos/pwads/requiem/.democonfig
@@ -0,0 +1,2 @@
+iwad: doom2.wad
+pwad: requiem.wad
diff --git a/testrunner b/testrunner
new file mode 100755
index 000000000..444d77b4f
--- /dev/null
+++ b/testrunner
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import os
+from os.path import dirname, basename, join, exists
+import shlex
+import sys
+import yaml
+
+def read_config(path):
+    if path == '':
+        return {}
+
+    # Config deeper in the hierarchy can override config files from
+    # higher up in the hierarchy:
+    result = read_config(dirname(path))
+    config_file = join(path, ".democonfig")
+    if exists(config_file):
+        with open(config_file) as f:
+            result.update(yaml.safe_load(f))
+
+    return result
+
+lmp_file, out_file = sys.argv[1:]
+cfg = read_config(lmp_file)
+
+os.environ["SDL_VIDEODRIVER"] = "dummy"
+
+args = [
+    "chocolate-doom", "-iwad", cfg["iwad"],
+    "-statdump", out_file,
+    "-timedemo", lmp_file,
+] + shlex.split("""
+     -mb 16 -nodraw -noblit -nosound
+     -noautoload -nogui -nograbmouse
+""")
+
+if "pwad" in cfg:
+    args.extend(("-file", join("extract", cfg["pwad"])))
+
+print(shlex.join(args))
+sys.stdout.close()
+os.spawnvp(os.P_WAIT, args[0], args)