statcheck/regression-test

#!/usr/bin/env python
#
# Copyright(C) 2011 Simon Howard
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#

# statchecker regression testing script.
#
# This script examines the zip files in the Compet-N tree, extracts
# the demos and plays them back using the configured source port.
# Statistics information is logged to a file using the -statdump
# parameter and the output is compared with the reference version
# for Vanilla Doom. A summary is outputted of the demos that pass
# or fail this test.

from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals

import os
import re
import sys
import tempfile

from common import *
from config import *

EXCEPTIONS_PARSE_RE = re.compile(r'(\S+)\s+(\S+)')

GREEN = '\033[92m'
RED = '\033[91m'
COLOR_END = '\033[0m'

def read_exceptions(filename):
	"""Read exceptions config file."""
	with open(filename) as f:
		result = {}
		for line in f:
			if line.strip().startswith("#"):
				continue
			match = EXCEPTIONS_PARSE_RE.match(line)
			if match:
				result[match.group(1)] = match.group(2)

		return result

def read_file(filename):
	result = []

	stream = open(filename)
	lines = stream.readlines()
	stream.close()

	lines = map(lambda x: x.strip(), lines)

	return "\n".join(lines)

def check_output(output_filename, zipfile_path, lmpname):
	expected_file = os.path.join("output", zipfile_path,
	                             lmpname.lower().replace(".lmp", ".txt"))

	output = read_file(output_filename)
	expected = read_file(expected_file)

	if output != expected:
		os.system("diff --strip-trailing-cr -u %s %s" % (
		              expected_file, output_filename))

	return output == expected

def print_result(zipfile_path, lmpname, success):

	if success:
		success_string = GREEN + " OK " + COLOR_END
	else:
		success_string = RED + "FAIL" + COLOR_END

	print("%-50s%-15s%s" % (zipfile_path, lmpname, success_string))
	sys.stdout.flush()

# Run the specified demo and check the output is correct.

def process_lmp(gametype, zipfile_path, zf, lmpname, pwad):

	global output_counter

	gamepath, exe = GAME_PATHS[gametype]
	iwad = os.path.join(gamepath, "%s.wad" % gametype)

	# Extract .lmp file. To avoid conflicts between concurrent
	# tests, rename to a generated name.
	lmpname2 = os.path.join(tempdir, "tst%i.lmp" % output_counter)
	zf.extract(lmpname, path=tempdir)
	os.rename(os.path.join(tempdir, lmpname), lmpname2)

	output_filename = (
		os.path.join(tempdir, "output-%i.txt" % output_counter))
	output_counter += 1

	cmd = "%s %s" % (PORT_EXE, PORT_OPTIONS)
	cmd += " -iwad %s" % iwad
	cmd += " -timedemo %s" % lmpname2
	cmd += " -statdump %s" % output_filename

	if pwad is not None:
		cmd += " -file %s" % os.path.join(gamepath, pwad)

	zipfile_relpath = os.path.relpath(zipfile_path, COMPETN_PATH)
	if zipfile_relpath in exceptions:
		cmd += " -gameversion %s" % exceptions[zipfile_relpath]

	def process_complete(exit_code, stdout, stderr):
		global passed, failed

		relpath = os.path.relpath(zipfile_path, COMPETN_PATH)

		success = check_output(output_filename, relpath, lmpname)

		print_result(relpath, lmpname, success)

		if success:
			passed += 1
		else:
			failed += 1

		os.remove(lmpname2)
		os.remove(output_filename)

	pipeline.call(cmd, process_complete)

# Use SDL's dummy video driver that does not open a window:
os.putenv("SDL_VIDEODRIVER", "dummy")

tempdir = tempfile.mkdtemp()
output_counter = 0
passed, failed = 0, 0
exceptions = read_exceptions("exceptions.cfg")
pipeline = CommandPipeline(CONCURRENT_PROCESSES)

process_all_zips(COMPETN_PATH, process_lmp)

pipeline.finish()

print("%i demo(s) passed, %i demo(s) failed." % (passed, failed))