Extended SCons testing to be able to handle multiple output files

Extra files to be compared are passed as a list of tuples of
(blessed_output, test_output) using the 'comparisons' keyword
argument.

buildutils.py

@@ -62,15 +62,20 @@ def regression_test(target, source, env):
     else:
         output = pjoin(blessedFile.dir.abspath, 'test_output.txt')
 
-    dir = str(target[0].dir)
-    blessed = [line.rstrip() for line in open(blessedFile.abspath).readlines()]
-
+    dir = str(target[0].dir.abspath)
     with open(output, 'w') as outfile:
         code = subprocess.call([program.abspath] + clargs,
-                               stdout=outfile, stderr=outfile, cwd=dir)
+                               stdout=outfile, stderr=outfile,
+                               cwd=dir)
 
-    outputText = [line.rstrip() for line in open(output).readlines()]
-    diff = list(difflib.unified_diff(blessed, outputText))
+    diff = compareFiles(blessedFile.abspath, output)
+
+    # Compare other output files
+    for blessed,output in env['test_comparisons']:
+        print blessed, output, diff,
+        print pjoin(dir, blessed), ',', pjoin(dir, output)
+        diff |= compareFiles(pjoin(dir, blessed), pjoin(dir, output))
+        print diff
 
     if diff or code:
         print 'FAILED'
@@ -88,6 +93,19 @@ def regression_test(target, source, env):
         print 'PASSED'
         open(target[0].path, 'w').write(time.asctime()+'\n')
 
+def compareFiles(file1, file2):
+    text1 = [line.rstrip() for line in open(file1).readlines()]
+    text2 = [line.rstrip() for line in open(file2).readlines()]
+
+    diff = list(difflib.unified_diff(text1, text2))
+    if diff:
+        'Found differences between %s and %s:' % (file1, file2)
+        print '>>>'
+        print '\n'.join(diff)
+        print '<<<'
+        return 1
+
+    return 0
 
 def regression_test_message(target, source, env):
     print """* Running test '%s'...""" % source[0].name,

test_problems/SConscript

@@ -8,7 +8,8 @@ os.environ['PYTHONPATH'] = pjoin(os.getcwd(), '..','Cantera','python')
 class Test(object):
     def __init__(self, subdir, programName,
                  blessedName, arguments=(),
-                 extensions=('cpp',), artifacts=()):
+                 extensions=('cpp',), artifacts=(),
+                 comparisons=()):
         self.subdir = subdir
         self.programName = programName
         if isinstance(arguments, str):
@@ -18,6 +19,7 @@ class Test(object):
         self.extensions = extensions
         self.artifacts = artifacts
         self.passedFile = '.passed-%s-%s' % (programName, blessedName)
+        self.comparisons = comparisons
 
     def run(self, env):
         prog = env.Program(pjoin(self.subdir, self.programName),
@@ -25,7 +27,8 @@ class Test(object):
                                 LIBS=env['cantera_libs'])
         arguments = [pjoin(self.subdir, arg) for arg in self.arguments]
         source = [prog, pjoin(self.subdir, self.blessedName)] + arguments
-        test = env.RegressionTest(pjoin(self.subdir, self.passedFile), source)
+        test = env.RegressionTest(pjoin(self.subdir, self.passedFile), source,
+                                  test_comparisons=self.comparisons)
 
         return test
 
@@ -140,7 +143,8 @@ tests = [Test(pjoin('cathermo', 'DH_graph_1'),
          Test('ChemEquil_gri_pairs', 'gri_pairs', 'output_blessed.txt'),
          Test('ChemEquil_ionizedGas', 'ionizedGasEquil',
               'output_blessed.txt',
-              artifacts=['table.csv']), # needs .csv comparison
+              artifacts=['table.csv'],
+              comparisons=[('table_blessed.csv', 'table.csv')]),
          Test('ChemEquil_red1', 'basopt_red1', 'output_blessed.txt'),
          # Skipping ck2cti_test because of automatically generated file
          Test('CpJump', 'CpJump', 'output_blessed.txt'),