From 6d4271fdfed5e8a4d9fa60fa78bef1ab62ab0c7c Mon Sep 17 00:00:00 2001
From: Ray Speth <speth@mit.edu>
Date: Fri, 3 Oct 2014 22:15:06 +0000
Subject: [PATCH] [Test] Improve visibility of total test suite failures

Treat crashes when running the GTest test programs and the Python test suite as
being more severe than individual test failures.
---
 site_scons/buildutils.py |  4 ++--
 test/SConscript          | 18 +++++++++++-------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/site_scons/buildutils.py b/site_scons/buildutils.py
index f2391e5bf..168c67ad3 100644
--- a/site_scons/buildutils.py
+++ b/site_scons/buildutils.py
@@ -93,8 +93,8 @@ Up-to-date tests skipped: %(skipped)s
 Tests failed: %(failed)s
 %(failures)s
 *****************************""" % dict(
-            passed=len(self.passed),
-            failed=len(self.failed),
+            passed=sum(self.passed.values()),
+            failed=sum(self.failed.values()),
             skipped=len(self.tests),
             failures=failures)
 
diff --git a/test/SConscript b/test/SConscript
index 629a00e06..fd2dac8e6 100644
--- a/test/SConscript
+++ b/test/SConscript
@@ -70,9 +70,11 @@ def addTestProgram(subdir, progName, env_vars={}):
                 else:
                     testResults.passed[testName] = 1
         else:
-            # Total failure of the test program - unable to determine status
-            # of individual tests
-            testResults.failed[passedFile.name] = program
+            # Total failure of the test program - unable to determine status of
+            # individual tests. This is potentially very bad, so it counts as
+            # more than one failure.
+            testResults.failed[passedFile.name +
+                ' ***no results for entire test suite***'] = 100
 
     testenv = localenv.Clone()
     testenv['ENV'].update(env_vars)
@@ -127,8 +129,10 @@ def addPythonTest(testname, subdir, script, interpreter, outfile,
                     failures += 1
 
         if code and failures == 0:
-            # Failure, but unable to determine status of individual tests
-            testResults.failed[testname] = True
+            # Failure, but unable to determine status of individual tests. This
+            # gets counted as many failures.
+            testResults.failed[testname +
+                ' ***no results for entire test suite***'] = 100
 
     testenv = localenv.Clone()
     passedFile = File(pjoin(subdir, '%s.passed' % testname))
@@ -176,9 +180,9 @@ def addMatlabTest(script, testName, dependencies=None, env_vars=()):
         print results
         print '------ end Matlab test results ------'
         if 'FAILED' in results:
-            testResults.failed[passedFile.name] = True
+            testResults.failed[passedFile.name] = 1
         else:
-            testResults.passed[passedFile.name] = True
+            testResults.passed[passedFile.name] = 1
             open(target[0].path, 'w').write(time.asctime()+'\n')
 
     testenv = localenv.Clone()