diff --git a/buildutils.py b/buildutils.py index bba396b92..f1ec73fd3 100644 --- a/buildutils.py +++ b/buildutils.py @@ -48,6 +48,31 @@ class ConfigBuilder(object): print " %-35s %s" % (key, '*undefined*') +class TestResults(object): + def __init__(self): + self.tests = {} + self.passed = {} + self.failed = {} + + def printReport(self, target, source, env): + total = len(self.passed) + len(self.failed) + print """ +********************************** +*** Regression Testing Summary *** +********************************** + +Tests passed: %(passed)s +Tests failed: %(failed)s +Up-to-date tests skipped: %(skipped)s + +**********************************""" % dict( + passed=len(self.passed), + failed=len(self.failed), + skipped=len(self.tests)) + +testResults = TestResults() + + def regression_test(target, source, env): # unpack: program = source[0] @@ -83,15 +108,19 @@ def regression_test(target, source, env): print """Comparing '%s' with '%s'""" % (blessed, output) diff |= compareFiles(env, pjoin(dir, blessed), pjoin(dir, output)) + del testResults.tests[env['active_test_name']] + if diff or code: print 'FAILED' if os.path.exists(target[0].abspath): os.path.unlink(target[0].abspath) - return -1 + + testResults.failed[env['active_test_name']] = 1 else: print 'PASSED' open(target[0].path, 'w').write(time.asctime()+'\n') + testResults.passed[env['active_test_name']] = 1 def compareFiles(env, file1, file2): @@ -117,6 +146,7 @@ def compareTextFiles(env, file1, file2): return 0 + def compareCsvFiles(env, file1, file2): try: import numpy as np @@ -137,7 +167,7 @@ def compareCsvFiles(env, file1, file2): try: data1 = np.genfromtxt(file1, skiprows=headerRows, delimiter=',') data2 = np.genfromtxt(file2, skiprows=headerRows, delimiter=',') - except IOError as e: + except (IOError, StopIteration) as e: print e return 1 diff --git a/test_problems/SConscript b/test_problems/SConscript index f11502bbe..f68f8e8a9 100644 --- a/test_problems/SConscript +++ b/test_problems/SConscript @@ -29,7 +29,8 @@ class Test(object): self.testName = testName self.passedFile = '.passed-%s' % testName - localenv.Alias('test', self.run(localenv)) + testResults.tests[self.testName] = self + localenv.Alias('test-run', self.run(localenv)) localenv.Alias('test-clean', self.clean(localenv)) def run(self, env, *args): @@ -64,7 +65,7 @@ class Test(object): files += [comp[1] for comp in self.comparisons] files = [pjoin(os.getcwd(), self.subdir, name) for name in files] - uniqueName = 'clean-%s-%s' % ('-'.join(psplit(self.subdir)), self.testName) + uniqueName = 'clean-%s-' % self.testName target = env.Command(uniqueName, [], [Delete(f) for f in files if os.path.exists(f)]) @@ -72,7 +73,7 @@ class Test(object): class CompileAndTest(Test): def __init__(self, subdir, programName, blessedName, **kwargs): - testName = '%s-%s' % (programName, blessedName) + testName = '%s-%s-%s' % ('-'.join(psplit(subdir)), programName, blessedName) self.extensions = kwargs.get('extensions') or ('cpp',) Test.__init__(self, subdir, testName, programName, blessedName, **kwargs) @@ -221,7 +222,6 @@ CompileAndTest(pjoin('VCSnonideal', 'NaCl_equil'), CompileAndTest('VPsilane_test', 'VPsilane_test', 'output_blessed.txt') # Python Tests - if localenv['python_package'] == 'full': Test('python', 'python-diamond', '$python_cmd', None, options='../../Cantera/python/examples/surface_chemistry/diamond_cvd/diamond.py', @@ -237,3 +237,7 @@ if localenv['python_package'] == 'full': # Skipping Python Tutorial 3 (documentation only) Test(pjoin('python','tut4'), 'python-tut4', '$python_cmd', 'output_blessed.txt', arguments='tut4.py', artifacts=['gri30.xml']) + +finish_tests = localenv.Command('finish_tests', [], testResults.printReport) +localenv.Depends(finish_tests, 'test-run') +Alias('test', finish_tests)