From c3f4972f52934043c0fdac21a33d5d624a42ba63 Mon Sep 17 00:00:00 2001 From: Ray Speth Date: Wed, 14 Dec 2011 19:36:03 +0000 Subject: [PATCH] Improvements to the SCons regression test handler Refactored SCons testing code to handle tests using prebuilt binaries. Now handles regression tests where the standard output is ignored. Also, allows regression tests to ignore lines starting with specified strings. --- buildutils.py | 22 ++- test_problems/SConscript | 366 +++++++++++++++++++++------------------ 2 files changed, 207 insertions(+), 181 deletions(-) diff --git a/buildutils.py b/buildutils.py index 228c02957..bba396b92 100644 --- a/buildutils.py +++ b/buildutils.py @@ -51,14 +51,14 @@ class ConfigBuilder(object): def regression_test(target, source, env): # unpack: program = source[0] - blessedName = source[1].name - if len(source) > 2: - clargs = [s.name for s in source[2:]] + if len(source) > 1: + clargs = [s.name for s in source[1:]] else: clargs = [] # Name to use for the output file - if 'blessed' in blessedName: + blessedName = env['test_blessed_file'] + if blessedName is not None and 'blessed' in blessedName: outputName = blessedName.replace('blessed', 'output') else: outputName = 'test_output.txt' @@ -75,7 +75,11 @@ def regression_test(target, source, env): diff = 0 # Compare output files - for blessed,output in [(blessedName,outputName)] + env['test_comparisons']: + comparisons = env['test_comparisons'] + if blessedName is not None: + comparisons.append((blessedName,outputName)) + + for blessed,output in comparisons: print """Comparing '%s' with '%s'""" % (blessed, output) diff |= compareFiles(env, pjoin(dir, blessed), pjoin(dir, output)) @@ -98,8 +102,10 @@ def compareFiles(env, file1, file2): def compareTextFiles(env, file1, file2): - text1 = [line.rstrip() for line in open(file1).readlines()] - text2 = [line.rstrip() for line in open(file2).readlines()] + text1 = [line.rstrip() for line in open(file1).readlines() + if not line.startswith(tuple(env['test_ignoreLines']))] + text2 = [line.rstrip() for line in open(file2).readlines() + if not line.startswith(tuple(env['test_ignoreLines']))] diff = list(difflib.unified_diff(text1, text2)) if diff: @@ -146,7 +152,7 @@ def compareCsvFiles(env, file1, file2): def regression_test_message(target, source, env): - return """* Running test '%s'...""" % source[0].name + return """* Running test '%s'...""" % env['active_test_name'] def add_RegressionTest(env): diff --git a/test_problems/SConscript b/test_problems/SConscript index ef4b5ff5f..c3961972b 100644 --- a/test_problems/SConscript +++ b/test_problems/SConscript @@ -6,192 +6,212 @@ localenv = env.Clone() os.environ['PYTHONPATH'] = pjoin(os.getcwd(), '..','Cantera','python') class Test(object): - def __init__(self, subdir, programName, - blessedName, arguments=(), options='', - extensions=('cpp',), artifacts=(), - comparisons=(), tolerance=1e-5, threshold=1e-14): + def __init__(self, subdir, testName, programName, blessedName, **kwargs): + assert set(kwargs.keys()) <= set(['arguments', 'options', 'artifacts', + 'comparisons', 'tolerance', 'threshold', + 'ignoreLines', 'extensions']), kwargs.keys() self.subdir = subdir self.programName = programName + arguments = kwargs.get('arguments') or [] if isinstance(arguments, str): arguments = [arguments] self.arguments = arguments # file arguments - self.options = options + self.options = kwargs.get('options') or '' self.blessedName = blessedName - self.extensions = extensions - self.artifacts = artifacts - self.passedFile = '.passed-%s-%s' % (programName, blessedName) - self.comparisons = comparisons - self.tolerance = tolerance # error tolerance for CSV comparison - self.threshold = threshold # error threshold for CSV comparison + self.artifacts = kwargs.get('artifacts') or () + self.comparisons = kwargs.get('comparisons') or () + self.tolerance = kwargs.get('tolerance') or 1e-5 # error tolerance for CSV comparison + self.threshold = kwargs.get('threshold') or 1e-14 # error threshold for CSV comparison + + # ignore lines starting with specified strings when comparing output files + self.ignoreLines = kwargs.get('ignoreLines') or [] + + self.testName = testName + self.passedFile = '.passed-%s' % testName + + localenv.Alias('test', self.run(localenv)) + localenv.Alias('test-clean', self.clean(localenv)) + + def run(self, env, *args): + source = list(args) + if not source: + source.append(self.programName) + + source.extend(pjoin(self.subdir, arg) for arg in self.arguments) + + test = env.RegressionTest(pjoin(self.subdir, self.passedFile), source, + active_test_name=self.testName, + test_blessed_file=self.blessedName, + test_command_options=self.options, + test_comparisons=self.comparisons, + test_csv_threshold=self.threshold, + test_csv_tolerance=self.tolerance, + test_ignoreLines=self.ignoreLines) + return test + + def clean(self, env, **kwargs): + # Name used for the output file + if self.blessedName is not None and 'blessed' in self.blessedName: + outName = self.blessedName.replace('blessed', 'output') + else: + outName = 'test_output.txt' + + files = kwargs.get('files') or [] + files += [self.passedFile, + 'ct2ctml.log', + outName] + files += list(self.artifacts) + files = [pjoin(os.getcwd(), self.subdir, name) for name in files] + + target = env.Command('clean-'+self.testName, [], + [Delete(f) for f in files + if os.path.exists(f)]) + return target + +class CompileAndTest(Test): + def __init__(self, subdir, programName, blessedName, **kwargs): + testName = '%s-%s' % (programName, blessedName) + self.extensions = kwargs.get('extensions') or ('cpp',) + Test.__init__(self, subdir, testName, programName, blessedName, **kwargs) def run(self, env): prog = env.Program(pjoin(self.subdir, self.programName), mglob(env, self.subdir, *self.extensions), LIBS=env['cantera_libs']) - arguments = [pjoin(self.subdir, arg) for arg in self.arguments] - source = [prog, pjoin(self.subdir, self.blessedName)] + arguments - test = env.RegressionTest(pjoin(self.subdir, self.passedFile), source, - test_command_options=self.options, - test_comparisons=self.comparisons, - test_csv_threshold=self.threshold, - test_csv_tolerance=self.tolerance) - - return test + source = [prog] + return Test.run(self, env, *source) def clean(self, env): - # Name used for the output file - if 'blessed' in self.blessedName: - outName = self.blessedName.replace('blessed', 'output') - else: - outName = 'test_output.txt' - files = [self.programName, - self.programName + '.o', - self.passedFile, - 'ct2ctml.log', - outName] - files += list(self.artifacts) - files = [pjoin(os.getcwd(), self.subdir, name) for name in files] - files = [f for f in files if os.path.exists(f)] + self.programName + '.o'] + return Test.clean(self, env, files=files) - target = env.Command('clean-'+self.programName, [], - [Delete(f) for f in files]) - return target -tests = [Test(pjoin('cathermo', 'DH_graph_1'), - 'DH_graph_1', - 'DH_NaCl_dilute_blessed.csv', - artifacts=['DH_graph_1.log'], - arguments='DH_NaCl_dilute.xml'), - Test(pjoin('cathermo', 'DH_graph_acommon'), - 'DH_graph_acommon', - 'DH_NaCl_acommon_blessed.csv', - artifacts=['DH_graph_1.log'], - arguments='DH_NaCl_acommon.xml'), - Test(pjoin('cathermo', 'DH_graph_bdotak'), - 'DH_graph_bdotak', - 'DH_NaCl_bdotak_blessed.csv', - artifacts=['DH_graph_1.log'], - arguments='DH_NaCl_bdotak.xml'), - Test(pjoin('cathermo', 'DH_graph_NM'), - 'DH_graph_NM', - 'DH_NaCl_NM_blessed.csv', - artifacts=['DH_graph_1.log'], - arguments='DH_NaCl_NM.xml'), - Test(pjoin('cathermo', 'DH_graph_Pitzer'), - 'DH_graph_Pitzer', - 'DH_NaCl_Pitzer_blessed.csv', - artifacts=['DH_graph_1.log'], - arguments='DH_NaCl_Pitzer.xml'), - Test(pjoin('cathermo', 'HMW_dupl_test'), - 'HMW_dupl_test', - 'output_blessed.txt', - artifacts=['DH_graph_1.log'], - arguments='HMW_NaCl_sp1977_alt.xml'), - Test(pjoin('cathermo', 'HMW_graph_CpvT'), - 'HMW_graph_CpvT', - 'output_blessed.txt', - extensions=['^HMW_graph_CpvT.cpp', '^sortAlgorithms.cpp'], - arguments='HMW_NaCl_sp1977_alt.xml'), - # Skipping cathermo/HMW_graph_GvI because of the way it generates output files. - Test(pjoin('cathermo', 'HMW_graph_GvT'), - 'HMW_graph_GvT', - 'output_blessed.txt', - extensions=['^HMW_graph_GvT.cpp', '^sortAlgorithms.cpp'], - arguments='HMW_NaCl_sp1977_alt.xml'), - Test(pjoin('cathermo', 'HMW_graph_HvT'), - 'HMW_graph_HvT', - 'output_blessed.txt', - extensions=['^HMW_graph_HvT.cpp', '^sortAlgorithms.cpp'], - arguments='HMW_NaCl_sp1977_alt.xml'), - Test(pjoin('cathermo', 'HMW_graph_VvT'), - 'HMW_graph_VvT', - 'output_blessed.txt', - extensions=['^HMW_graph_VvT.cpp', '^sortAlgorithms.cpp'], - arguments='HMW_NaCl_sp1977_alt.xml'), - Test(pjoin('cathermo', 'HMW_test_1'), - 'HMW_test_1', - 'output_noD_blessed.txt'), - Test(pjoin('cathermo', 'HMW_test_3'), - 'HMW_test_3', - 'output_noD_blessed.txt'), - Test(pjoin('cathermo', 'ims'), - 'IMSTester', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'issp'), - 'ISSPTester', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'stoichSubSSTP'), - 'stoichSubSSTP', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'testIAPWS'), - 'testIAPWSphi', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'testIAPWSPres'), - 'testIAPWSPres', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'testIAPWSTripP'), - 'testIAPWSTripP', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'testWaterPDSS'), - 'testWaterPDSS', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'testWaterTP'), - 'testWaterSSTP', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'VPissp'), - 'ISSPTester2', - 'output_blessed.txt'), - Test(pjoin('cathermo', 'wtWater'), - 'wtWater', - 'output_blessed.txt'), - Test('ChemEquil_gri_matrix', 'gri_matrix', 'output_blessed.txt'), - Test('ChemEquil_gri_pairs', 'gri_pairs', 'output_blessed.txt'), - Test('ChemEquil_ionizedGas', 'ionizedGasEquil', - 'output_blessed.txt', - artifacts=['table.csv'], - comparisons=[('table_blessed.csv', 'table.csv')]), - Test('ChemEquil_red1', 'basopt_red1', 'output_blessed.txt'), - # Skipping ck2cti_test because of automatically generated file - Test('CpJump', 'CpJump', 'output_blessed.txt'), - Test('cxx_ex', 'cxx_examples', 'output_blessed.txt', - comparisons=[('eq1_blessed.csv', 'eq1.csv'), - ('kin1_blessed.csv', 'kin1.csv'), - ('kin2_blessed.csv', 'kin2.csv'), - ('tr1_blessed.csv', 'tr1.csv'), - ('tr2_blessed.csv', 'tr2.csv')], - tolerance=2e-3, - threshold=1e-7, - artifacts=['eq1.csv', 'eq1.dat', 'kin1.csv', 'kin1.dat', - 'kin2.csv', 'kin2.dat', 'kin3.csv', 'kin3.dat', - 'tr1.csv', 'tr1.dat', 'tr2.csv', 'tr2.dat']), - Test('diamondSurf', 'runDiamond', 'runDiamond_blessed.out'), - Test('fracCoeff', 'fracCoeff', 'frac_blessed.out'), - # skipping min_python - Test('mixGasTransport', 'mixGasTransport', 'output_blessed.txt'), - Test('multiGasTransport', 'multiGasTransport', 'output_blessed.txt'), - Test('NASA9poly_test', 'NASA9poly_test', 'output_blessed.txt'), - # skipping nasa9_reader because of automatically generated file - Test('negATest', 'negATest', 'negATest_blessed.out'), - Test('printUtilUnitTest', 'pUtest', 'output_blessed.txt'), - Test('pureFluidTest', 'testPureWater', 'output_blessed.txt'), - # skipping python - Test('rankine_democxx', 'rankine', 'output_blessed.txt', - artifacts=['liquidvapor.xml']), - Test('silane_equil', 'silane_equi', 'output_blessed.txt'), - # spectroscopy is incomplete - Test('surfkin', 'surfdemo', 'output_blessed.txt'), - Test('surfSolverTest', 'surfaceSolver', - 'surfaceSolver_blessed.out', - arguments='haca2.xml', - artifacts=['results.txt', 'diamond.xml'], - extensions=['^surfaceSolver.cpp']), # needs .csv, extra tests - Test(pjoin('VCSnonideal', 'NaCl_equil'), - 'nacl_equil', 'good_out.txt', - options='-d 3', - artifacts=['vcs_equilibrate_res.csv']), # not testing this file because it's not really csv - Test('VPsilane_test', 'VPsilane_test', 'output_blessed.txt') - ] - -env.Alias('test', [test.run(localenv) for test in tests]) -env.Alias('test-clean', sum([test.clean(localenv) for test in tests], [])) +CompileAndTest(pjoin('cathermo', 'DH_graph_1'), + 'DH_graph_1', 'DH_NaCl_dilute_blessed.csv', + artifacts=['DH_graph_1.log'], + arguments='DH_NaCl_dilute.xml') +CompileAndTest(pjoin('cathermo', 'DH_graph_acommon'), + 'DH_graph_acommon', 'DH_NaCl_acommon_blessed.csv', + artifacts=['DH_graph_1.log'], + arguments='DH_NaCl_acommon.xml') +CompileAndTest(pjoin('cathermo', 'DH_graph_bdotak'), + 'DH_graph_bdotak', 'DH_NaCl_bdotak_blessed.csv', + artifacts=['DH_graph_1.log'], + arguments='DH_NaCl_bdotak.xml') +CompileAndTest(pjoin('cathermo', 'DH_graph_NM'), + 'DH_graph_NM', 'DH_NaCl_NM_blessed.csv', + artifacts=['DH_graph_1.log'], + arguments='DH_NaCl_NM.xml') +CompileAndTest(pjoin('cathermo', 'DH_graph_Pitzer'), + 'DH_graph_Pitzer', 'DH_NaCl_Pitzer_blessed.csv', + artifacts=['DH_graph_1.log'], + arguments='DH_NaCl_Pitzer.xml') +CompileAndTest(pjoin('cathermo', 'HMW_dupl_test'), + 'HMW_dupl_test', 'output_blessed.txt', + artifacts=['DH_graph_1.log'], + arguments='HMW_NaCl_sp1977_alt.xml') +CompileAndTest(pjoin('cathermo', 'HMW_graph_CpvT'), + 'HMW_graph_CpvT', 'output_blessed.txt', + extensions=['^HMW_graph_CpvT.cpp', '^sortAlgorithms.cpp'], + arguments='HMW_NaCl_sp1977_alt.xml') +CompileAndTest(pjoin('cathermo', 'HMW_graph_GvI'), + 'HMW_graph_GvI', None, + comparisons=[('T298_blessed.csv', 'T298.csv'), + ('T523_blessed.csv', 'T523.csv')], + artifacts=['T298.csv','T373.csv','T423.csv','T473.csv', + 'T548.csv','T523.csv','T573.csv']) +CompileAndTest(pjoin('cathermo', 'HMW_graph_GvT'), + 'HMW_graph_GvT', 'output_blessed.txt', + extensions=['^HMW_graph_GvT.cpp', '^sortAlgorithms.cpp'], + arguments='HMW_NaCl_sp1977_alt.xml') +CompileAndTest(pjoin('cathermo', 'HMW_graph_HvT'), + 'HMW_graph_HvT', 'output_blessed.txt', + extensions=['^HMW_graph_HvT.cpp', '^sortAlgorithms.cpp'], + arguments='HMW_NaCl_sp1977_alt.xml') +CompileAndTest(pjoin('cathermo', 'HMW_graph_VvT'), + 'HMW_graph_VvT', 'output_blessed.txt', + extensions=['^HMW_graph_VvT.cpp', '^sortAlgorithms.cpp'], + arguments='HMW_NaCl_sp1977_alt.xml') +CompileAndTest(pjoin('cathermo', 'HMW_test_1'), + 'HMW_test_1', 'output_noD_blessed.txt') +CompileAndTest(pjoin('cathermo', 'HMW_test_3'), + 'HMW_test_3', 'output_noD_blessed.txt') +CompileAndTest(pjoin('cathermo', 'ims'), + 'IMSTester', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'issp'), + 'ISSPTester', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'stoichSubSSTP'), + 'stoichSubSSTP', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'testIAPWS'), + 'testIAPWSphi', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'testIAPWSPres'), + 'testIAPWSPres', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'testIAPWSTripP'), + 'testIAPWSTripP', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'testWaterPDSS'), + 'testWaterPDSS', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'testWaterTP'), + 'testWaterSSTP', 'output_blessed.txt') +CompileAndTest( pjoin('cathermo', 'VPissp'), + 'ISSPTester2', 'output_blessed.txt') +CompileAndTest(pjoin('cathermo', 'wtWater'), + 'wtWater', 'output_blessed.txt') +CompileAndTest('ChemEquil_gri_matrix', 'gri_matrix', 'output_blessed.txt') +CompileAndTest('ChemEquil_gri_pairs', 'gri_pairs', 'output_blessed.txt') +CompileAndTest('ChemEquil_ionizedGas', 'ionizedGasEquil', + 'output_blessed.txt', + artifacts=['table.csv'], + comparisons=[('table_blessed.csv', 'table.csv')]) +CompileAndTest('ChemEquil_red1', 'basopt_red1', 'output_blessed.txt') +Test('ck2cti_test', 'ck2cti-gri30', '#build/bin/ck2cti', None, + options='-i gri30.inp -id gri30 -tr gri30_tran.dat', + comparisons=[('gri30a_blessed.cti','gri30.cti')], + ignoreLines=['#'], + artifacts=['ck2cti.log', 'gri30.cti']) +Test('ck2cti_test', 'ck2cti-soot', '#build/bin/ck2cti', None, + options='-i soot.inp -id soot -t therm_soot.dat', + comparisons=[('soot_blessed.cti', 'soot.cti')], + ignoreLines=['#'], + artifacts=['ck2cti.log', 'soot.cti']) +CompileAndTest('CpJump', 'CpJump', 'output_blessed.txt') +CompileAndTest('cxx_ex', 'cxx_examples', 'output_blessed.txt', + comparisons=[('eq1_blessed.csv', 'eq1.csv'), + ('kin1_blessed.csv', 'kin1.csv'), + ('kin2_blessed.csv', 'kin2.csv'), + ('tr1_blessed.csv', 'tr1.csv'), + ('tr2_blessed.csv', 'tr2.csv')], + tolerance=2e-3, + threshold=1e-7, + artifacts=['eq1.csv', 'eq1.dat', 'kin1.csv', 'kin1.dat', + 'kin2.csv', 'kin2.dat', 'kin3.csv', 'kin3.dat', + 'tr1.csv', 'tr1.dat', 'tr2.csv', 'tr2.dat']) +CompileAndTest('diamondSurf', 'runDiamond', 'runDiamond_blessed.out') +CompileAndTest('fracCoeff', 'fracCoeff', 'frac_blessed.out') +# skipping min_python +CompileAndTest('mixGasTransport', 'mixGasTransport', 'output_blessed.txt') +CompileAndTest('multiGasTransport', 'multiGasTransport', 'output_blessed.txt') +CompileAndTest('NASA9poly_test', 'NASA9poly_test', 'output_blessed.txt') +Test('nasa9_reader', 'nasa9_reader', '#build/bin/ck2cti', None, + options='-i sample.inp -id sample -t sampleData.inp', + comparisons=[('sample_blessed.cti', 'sample.cti')], + ignoreLines=['#'], + artifacts=['ck2cti.log', 'sample.cti']) +CompileAndTest('negATest', 'negATest', 'negATest_blessed.out') +CompileAndTest('printUtilUnitTest', 'pUtest', 'output_blessed.txt') +CompileAndTest('pureFluidTest', 'testPureWater', 'output_blessed.txt') +# skipping python +CompileAndTest('rankine_democxx', 'rankine', 'output_blessed.txt', + artifacts=['liquidvapor.xml']) +CompileAndTest('silane_equil', 'silane_equi', 'output_blessed.txt') +# spectroscopy is incomplete +CompileAndTest('surfkin', 'surfdemo', 'output_blessed.txt') +CompileAndTest('surfSolverTest', 'surfaceSolver', 'surfaceSolver_blessed.out', + arguments='haca2.xml', + artifacts=['results.txt', 'diamond.xml'], + extensions=['^surfaceSolver.cpp']) # needs .csv, extra tests +CompileAndTest(pjoin('VCSnonideal', 'NaCl_equil'), + 'nacl_equil', 'good_out.txt', + options='-d 3', + artifacts=['vcs_equilibrate_res.csv']), # not testing this file because it's not really csv +CompileAndTest('VPsilane_test', 'VPsilane_test', 'output_blessed.txt')