SCons now continues over failed tests and prints a summary at the end

buildutils.py

@@ -48,6 +48,31 @@ class ConfigBuilder(object):
             print "    %-35s %s" % (key, '*undefined*')
 
 
+class TestResults(object):
+    def __init__(self):
+        self.tests = {}
+        self.passed = {}
+        self.failed = {}
+
+    def printReport(self, target, source, env):
+        total = len(self.passed) + len(self.failed)
+        print """
+**********************************
+*** Regression Testing Summary ***
+**********************************
+
+Tests passed: %(passed)s
+Tests failed: %(failed)s
+Up-to-date tests skipped: %(skipped)s
+
+**********************************""" % dict(
+            passed=len(self.passed),
+            failed=len(self.failed),
+            skipped=len(self.tests))
+
+testResults = TestResults()
+
+
 def regression_test(target, source, env):
     # unpack:
     program = source[0]
@@ -83,15 +108,19 @@ def regression_test(target, source, env):
         print """Comparing '%s' with '%s'""" % (blessed, output)
         diff |= compareFiles(env, pjoin(dir, blessed), pjoin(dir, output))
 
+    del testResults.tests[env['active_test_name']]
+
     if diff or code:
         print 'FAILED'
 
         if os.path.exists(target[0].abspath):
             os.path.unlink(target[0].abspath)
-        return -1
+
+        testResults.failed[env['active_test_name']] = 1
     else:
         print 'PASSED'
         open(target[0].path, 'w').write(time.asctime()+'\n')
+        testResults.passed[env['active_test_name']] = 1
 
 
 def compareFiles(env, file1, file2):
@@ -117,6 +146,7 @@ def compareTextFiles(env, file1, file2):
 
     return 0
 
+
 def compareCsvFiles(env, file1, file2):
     try:
         import numpy as np
@@ -137,7 +167,7 @@ def compareCsvFiles(env, file1, file2):
     try:
         data1 = np.genfromtxt(file1, skiprows=headerRows, delimiter=',')
         data2 = np.genfromtxt(file2, skiprows=headerRows, delimiter=',')
-    except IOError as e:
+    except (IOError, StopIteration) as e:
         print e
         return 1
 

test_problems/SConscript

@@ -29,7 +29,8 @@ class Test(object):
         self.testName = testName
         self.passedFile = '.passed-%s' % testName
 
-        localenv.Alias('test', self.run(localenv))
+        testResults.tests[self.testName] = self
+        localenv.Alias('test-run', self.run(localenv))
         localenv.Alias('test-clean', self.clean(localenv))
 
     def run(self, env, *args):
@@ -64,7 +65,7 @@ class Test(object):
         files += [comp[1] for comp in self.comparisons]
         files = [pjoin(os.getcwd(), self.subdir, name) for name in files]
 
-        uniqueName = 'clean-%s-%s' % ('-'.join(psplit(self.subdir)), self.testName)
+        uniqueName = 'clean-%s-' % self.testName
         target = env.Command(uniqueName, [],
                              [Delete(f) for f in files
                               if os.path.exists(f)])
@@ -72,7 +73,7 @@ class Test(object):
 
 class CompileAndTest(Test):
     def __init__(self, subdir, programName, blessedName, **kwargs):
-        testName = '%s-%s' % (programName, blessedName)
+        testName = '%s-%s-%s' % ('-'.join(psplit(subdir)), programName, blessedName)
         self.extensions = kwargs.get('extensions') or ('cpp',)
         Test.__init__(self, subdir, testName, programName, blessedName, **kwargs)
 
@@ -221,7 +222,6 @@ CompileAndTest(pjoin('VCSnonideal', 'NaCl_equil'),
 CompileAndTest('VPsilane_test', 'VPsilane_test', 'output_blessed.txt')
 
 # Python Tests
-
 if localenv['python_package'] == 'full':
     Test('python', 'python-diamond', '$python_cmd', None,
          options='../../Cantera/python/examples/surface_chemistry/diamond_cvd/diamond.py',
@@ -237,3 +237,7 @@ if localenv['python_package'] == 'full':
     # Skipping Python Tutorial 3 (documentation only)
     Test(pjoin('python','tut4'), 'python-tut4', '$python_cmd',
          'output_blessed.txt', arguments='tut4.py', artifacts=['gri30.xml'])
+
+finish_tests = localenv.Command('finish_tests', [], testResults.printReport)
+localenv.Depends(finish_tests, 'test-run')
+Alias('test', finish_tests)