diff --git a/buildutils.py b/buildutils.py index 788563782..53f44c8e4 100644 --- a/buildutils.py +++ b/buildutils.py @@ -50,42 +50,34 @@ class ConfigBuilder(object): def regression_test(target, source, env): # unpack: program = source[0] - blessedFile = source[1] + blessedName = source[1].name if len(source) > 2: clargs = [s.name for s in source[2:]] else: clargs = [] + # Name to use for the output file - if 'blessed' in blessedFile.name: - output = blessedFile.abspath.replace('blessed', 'output') + if 'blessed' in blessedName: + outputName = blessedName.replace('blessed', 'output') else: - output = pjoin(blessedFile.dir.abspath, 'test_output.txt') + outputName = 'test_output.txt' dir = str(target[0].dir.abspath) - with open(output, 'w') as outfile: + with open(pjoin(dir,outputName), 'w') as outfile: code = subprocess.call([program.abspath] + clargs, stdout=outfile, stderr=outfile, cwd=dir) - diff = compareFiles(blessedFile.abspath, output) - - # Compare other output files - for blessed,output in env['test_comparisons']: - print blessed, output, diff, - print pjoin(dir, blessed), ',', pjoin(dir, output) - diff |= compareFiles(pjoin(dir, blessed), pjoin(dir, output)) - print diff + diff = 0 + # Compare output files + for blessed,output in [(blessedName,outputName)] + env['test_comparisons']: + print """Comparing '%s' with '%s'""" % (blessed, output) + diff |= compareFiles(env, pjoin(dir, blessed), pjoin(dir, output)) if diff or code: print 'FAILED' - if diff: - print 'Difference between blessed output and current output:' - print '>>>' - print '\n'.join(diff) - print '<<<' - if os.path.exists(target[0].abspath): os.path.unlink(target[0].abspath) return -1 @@ -93,7 +85,15 @@ def regression_test(target, source, env): print 'PASSED' open(target[0].path, 'w').write(time.asctime()+'\n') -def compareFiles(file1, file2): + +def compareFiles(env, file1, file2): + if file1.endswith('csv') and file2.endswith('csv'): + return compareCsvFiles(env, file1, file2) + else: + return compareTextFiles(env, file1, file2) + + +def compareTextFiles(env, file1, file2): text1 = [line.rstrip() for line in open(file1).readlines()] text2 = [line.rstrip() for line in open(file2).readlines()] @@ -107,8 +107,42 @@ def compareFiles(file1, file2): return 0 +def compareCsvFiles(env, file1, file2): + try: + import numpy as np + except ImportError: + print 'WARNING: skipping .csv diff because numpy is not installed' + return 0 + + # decide how many header lines to skip + f = open(file1) + headerRows = 0 + goodChars = set('0123456789.+-eE, ') + for line in f: + if not set(line[:-1]).issubset(goodChars): + headerRows += 1 + else: + break + + try: + data1 = np.genfromtxt(file1, skiprows=headerRows, delimiter=',') + data2 = np.genfromtxt(file2, skiprows=headerRows, delimiter=',') + except IOError as e: + print e + return 1 + + relerror = np.abs(data2-data1) / (np.maximum(np.abs(data2), np.abs(data1)) + env['test_csv_threshold']) + maxerror = np.nanmax(relerror.flat) + tol = env['test_csv_tolerance'] + if maxerror > tol: # Threshold based on printing 6 digits in the CSV file + print "Files differ. %i / %i elements above specified tolerance" % (np.sum(relerror > tol), relerror.size) + return 1 + else: + return 0 + + def regression_test_message(target, source, env): - print """* Running test '%s'...""" % source[0].name, + return """* Running test '%s'...""" % source[0].name def add_RegressionTest(env): diff --git a/test_problems/SConscript b/test_problems/SConscript index 30e65df3a..cc691f0e2 100644 --- a/test_problems/SConscript +++ b/test_problems/SConscript @@ -9,7 +9,7 @@ class Test(object): def __init__(self, subdir, programName, blessedName, arguments=(), extensions=('cpp',), artifacts=(), - comparisons=()): + comparisons=(), tolerance=1e-5, threshold=1e-14): self.subdir = subdir self.programName = programName if isinstance(arguments, str): @@ -20,6 +20,8 @@ class Test(object): self.artifacts = artifacts self.passedFile = '.passed-%s-%s' % (programName, blessedName) self.comparisons = comparisons + self.tolerance = tolerance # error tolerance for CSV comparison + self.threshold = threshold # error threshold for CSV comparison def run(self, env): prog = env.Program(pjoin(self.subdir, self.programName), @@ -28,7 +30,9 @@ class Test(object): arguments = [pjoin(self.subdir, arg) for arg in self.arguments] source = [prog, pjoin(self.subdir, self.blessedName)] + arguments test = env.RegressionTest(pjoin(self.subdir, self.passedFile), source, - test_comparisons=self.comparisons) + test_comparisons=self.comparisons, + test_csv_threshold=self.threshold, + test_csv_tolerance=self.tolerance) return test @@ -148,7 +152,17 @@ tests = [Test(pjoin('cathermo', 'DH_graph_1'), Test('ChemEquil_red1', 'basopt_red1', 'output_blessed.txt'), # Skipping ck2cti_test because of automatically generated file Test('CpJump', 'CpJump', 'output_blessed.txt'), - # Skipping cxx_ex because of complicated behavior + Test('cxx_ex', 'cxx_examples', 'output_blessed.txt', + comparisons=[('eq1_blessed.csv', 'eq1.csv'), + ('kin1_blessed.csv', 'kin1.csv'), + ('kin2_blessed.csv', 'kin2.csv'), + ('tr1_blessed.csv', 'tr1.csv'), + ('tr2_blessed.csv', 'tr2.csv')], + tolerance=2e-3, + threshold=1e-7, + artifacts=['eq1.csv', 'eq1.dat', 'kin1.csv', 'kin1.dat', + 'kin2.csv', 'kin2.dat', 'kin3.csv', 'kin3.dat', + 'tr1.csv', 'tr1.dat', 'tr2.csv', 'tr2.dat']), Test('diamondSurf', 'runDiamond', 'runDiamond_blessed.out'), Test('fracCoeff', 'fracCoeff', 'frac_blessed.out'), # skipping min_python diff --git a/test_problems/cxx_ex/eq1_blessed.dat b/test_problems/cxx_ex/eq1_blessed.dat deleted file mode 100644 index 07df9e839..000000000 --- a/test_problems/cxx_ex/eq1_blessed.dat +++ /dev/null @@ -1,71 +0,0 @@ -TITLE = "equilibrium example 1: chemical equilibrium" -VARIABLES = -"Temperature (K)" -"Pressure (Pa)" -"H2" -"H" -"HE" -"SIH4" -"SI" -"SIH" -"SIH2" -"SIH3" -"H3SISIH" -"SI2H6" -"H2SISIH2" -"SI3H8" -"SI2" -"SI3" -ZONE T="zone1" - I=50,J=1,K=1,F=POINT -DT=( SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE SINGLE ) -500 1013.25 0.990006 6.92216e-20 0 0.00998878 2.87904e-30 7.66715e-27 2.18962e-18 9.60542e-15 1.21806e-20 5.58337e-06 7.35451e-16 1.73503e-08 7.21165e-35 6.24409e-31 -530 1013.25 0.990007 1.37159e-18 0 0.00998608 9.04542e-28 9.49565e-25 5.73955e-17 9.19533e-14 3.75067e-19 6.92449e-06 1.15054e-14 2.51179e-08 1.00286e-31 1.10766e-27 -560 1013.25 0.990009 1.97797e-17 0 0.009983 1.54484e-25 7.04873e-23 1.06312e-15 6.92186e-13 8.06148e-18 8.44666e-06 1.3529e-13 3.54127e-08 6.49105e-29 8.97582e-25 -590 1013.25 0.99001 2.17885e-16 0 0.00997955 1.57234e-23 3.38796e-21 1.46631e-14 4.2488e-12 1.27619e-16 1.01534e-05 1.24753e-12 4.87571e-08 2.19336e-26 3.73357e-22 -620 1013.25 0.990012 1.90628e-15 0 0.00997571 1.02724e-21 1.1223e-19 1.57106e-13 2.19e-11 1.55412e-15 1.20466e-05 9.33594e-12 6.57122e-08 4.24547e-24 8.76109e-20 -650 1013.25 0.990014 1.36746e-14 0 0.00997149 4.57739e-20 2.69625e-18 1.35363e-12 9.70884e-11 1.50853e-14 1.41262e-05 5.83154e-11 8.68702e-08 5.0782e-22 1.25296e-17 -680 1013.25 0.990017 8.25642e-14 0 0.00996688 1.46259e-18 4.89957e-17 9.65025e-12 3.77592e-10 1.20195e-13 1.63908e-05 3.11184e-10 1.12845e-07 3.99637e-20 1.16414e-15 -710 1013.25 0.990019 4.28821e-13 0 0.00996189 3.4937e-17 6.97437e-16 5.82939e-11 1.30966e-09 8.05572e-13 1.88378e-05 1.44636e-09 1.44265e-07 2.18001e-18 7.41155e-14 -740 1013.25 0.990022 1.95117e-12 0 0.00995651 6.46101e-16 8.00874e-15 3.04378e-10 4.10739e-09 4.63609e-12 2.14632e-05 5.95179e-09 1.81763e-07 8.61251e-17 3.38159e-12 -770 1013.25 0.990025 7.89841e-12 0 0.00995074 9.52823e-15 7.60545e-14 1.39712e-09 1.17846e-08 2.33129e-11 2.42622e-05 2.19851e-08 2.25966e-07 2.55737e-15 1.14852e-10 -800 1013.25 0.990028 2.88205e-11 0 0.00994452 1.1491e-13 6.1006e-13 5.71898e-09 3.12405e-08 1.03964e-10 2.72288e-05 7.37633e-08 2.7749e-07 5.89085e-14 2.9994e-09 -830 1013.25 0.990031 9.58626e-11 0 0.00993754 1.15797e-12 4.20888e-12 2.11347e-08 7.71711e-08 4.16419e-10 3.03547e-05 2.27067e-07 3.36901e-07 1.08152e-12 6.19231e-08 -860 1013.25 0.990037 2.93476e-10 0 0.00992686 9.93064e-12 2.53607e-11 7.12425e-08 1.78887e-07 1.51375e-09 3.36123e-05 6.4656e-07 4.04392e-07 1.61904e-11 1.03429e-06 -890 1013.25 0.990068 8.33879e-10 0 0.00987813 7.33954e-11 1.34806e-10 2.20303e-07 3.90238e-07 5.00682e-09 3.67305e-05 1.70399e-06 4.75273e-07 2.0026e-10 1.41212e-05 -920 1013.25 0.990338 2.21535e-09 0 0.00947376 4.58492e-10 6.18642e-10 6.09356e-07 7.79318e-07 1.42165e-08 3.70865e-05 3.91495e-06 4.95463e-07 1.9483e-09 0.000145159 -950 1013.25 0.991572 5.53996e-09 0 0.0076386 2.13463e-09 2.15782e-09 1.32334e-06 1.24834e-06 2.65215e-08 2.63168e-05 5.99229e-06 3.03827e-07 1.14869e-08 0.00075432 -980 1013.25 0.993434 1.31112e-08 0 0.00486966 7.04637e-09 5.43168e-09 2.13589e-06 1.5151e-06 2.89567e-08 1.16163e-05 5.43595e-06 9.12796e-08 3.68486e-08 0.00167503 -1010 1013.25 0.994832 2.94949e-08 0 0.00278981 1.89282e-08 1.13016e-08 2.92542e-06 1.58733e-06 2.39987e-08 4.54277e-06 3.78628e-06 2.353e-08 8.76231e-08 0.00236468 -1040 1013.25 0.99566 6.33409e-08 0 0.00155831 4.54471e-08 2.13198e-08 3.72071e-06 1.56845e-06 1.79964e-08 1.53159e-06 2.41126e-06 4.7146e-09 1.70846e-07 0.00277236 -1070 1013.25 0.996116 1.30369e-07 0 0.000877797 1.01424e-07 3.7876e-08 4.55579e-06 1.51352e-06 1.30714e-08 5.23457e-07 1.50161e-06 9.63426e-10 3.05675e-07 0.00299711 -1100 1013.25 0.996366 2.58077e-07 0 0.000504596 2.14253e-07 6.44712e-08 5.45418e-06 1.44703e-06 9.4513e-09 1.85725e-07 9.39066e-07 2.0808e-10 5.17933e-07 0.00312001 -1130 1013.25 0.996505 4.92903e-07 0 0.000297179 4.32608e-07 1.06077e-07 6.43098e-06 1.37928e-06 6.88008e-09 6.89596e-08 5.96015e-07 4.8074e-11 8.43712e-07 0.00318794 -1160 1013.25 0.996582 9.10799e-07 0 0.00017945 8.39959e-07 1.6958e-07 7.49609e-06 1.31443e-06 5.06558e-09 2.68393e-08 3.85447e-07 1.19107e-11 1.33209e-06 0.00322601 -1190 1013.25 0.996626 1.63236e-06 0 0.000111035 1.57484e-06 2.64315e-07 8.65628e-06 1.25401e-06 3.77851e-09 1.09385e-08 2.5427e-07 3.15985e-12 2.04832e-06 0.00324749 -1220 1013.25 0.99665 2.84389e-06 0 7.03201e-05 2.86038e-06 4.02666e-07 9.91621e-06 1.19836e-06 2.85633e-09 4.65842e-09 1.71067e-07 8.94816e-13 3.07731e-06 0.00325938 -1250 1013.25 0.996662 4.82612e-06 0 4.55221e-05 5.046e-06 6.00764e-07 1.12791e-05 1.14734e-06 2.18755e-09 2.06787e-09 1.17286e-07 2.69476e-13 4.52754e-06 0.00326537 -1280 1013.25 0.996665 7.99222e-06 0 3.00811e-05 8.66492e-06 8.79231e-07 1.27469e-05 1.10061e-06 1.69633e-09 9.54299e-10 8.18634e-08 8.59699e-14 6.53497e-06 0.00326735 -1310 1013.25 0.99666 1.29373e-05 0 2.02633e-05 1.45113e-05 1.26399e-06 1.43205e-05 1.05775e-06 1.3309e-09 4.5668e-10 5.81058e-08 2.89443e-14 9.26704e-06 0.00326616 -1340 1013.25 0.996648 2.05016e-05 0 1.38967e-05 2.37411e-05 1.78711e-06 1.59995e-05 1.01832e-06 1.05565e-09 2.26069e-10 4.18929e-08 1.02468e-14 1.29261e-05 0.00326199 -1370 1013.25 0.996627 3.18495e-05 0 9.69089e-06 3.80007e-05 2.48757e-06 1.77824e-05 9.81904e-07 8.45816e-10 1.15492e-10 3.0645e-08 3.80107e-15 1.77524e-05 0.00325454 -1400 1013.25 0.996594 4.85662e-05 0 6.8637e-06 5.95866e-05 3.41202e-06 1.96659e-05 9.48089e-07 6.83976e-10 6.07545e-11 2.27186e-08 1.47255e-15 2.40244e-05 0.0032431 -1430 1013.25 0.996545 7.27761e-05 0 4.93176e-06 9.16368e-05 4.61528e-06 2.16444e-05 9.1648e-07 5.57738e-10 3.2838e-11 1.70494e-08 5.93866e-16 3.20578e-05 0.00322653 -1460 1013.25 0.996475 0.000107282 0 3.59101e-06 0.000138353 6.16054e-06 2.37097e-05 8.86682e-07 4.58178e-10 1.81984e-11 1.29369e-08 2.48539e-16 4.21992e-05 0.00320328 -1490 1013.25 0.996375 0.000155731 0 2.64683e-06 0.000205241 8.11898e-06 2.58498e-05 8.58297e-07 3.78801e-10 1.03189e-11 9.91302e-09 1.07606e-16 5.48129e-05 0.00317129 -1520 1013.25 0.996238 0.000222803 0 1.97265e-06 0.000299364 1.05686e-05 2.8048e-05 8.30913e-07 3.14819e-10 5.97392e-12 7.66057e-09 4.80437e-17 7.0259e-05 0.00312797 -1550 1013.25 0.996051 0.000314431 0 1.48488e-06 0.00042955 1.35916e-05 3.0281e-05 8.04093e-07 2.62691e-10 3.52333e-12 5.96155e-09 2.20485e-17 8.88566e-05 0.00307014 -1580 1013.25 0.995799 0.000438047 0 1.12752e-06 0.000606528 1.72701e-05 3.25176e-05 7.77367e-07 2.19743e-10 2.11204e-12 4.66428e-09 1.03646e-17 0.00011083 0.00299404 -1610 1013.25 0.995465 0.000602855 0 8.6255e-07 0.00084288 2.16792e-05 3.4716e-05 7.50219e-07 1.83956e-10 1.28353e-12 3.66195e-09 4.97186e-18 0.000136228 0.0028954 -1640 1013.25 0.995028 0.000820132 0 6.63801e-07 0.00115268 2.68764e-05 3.68218e-05 7.22084e-07 1.53794e-10 7.88569e-13 2.87857e-09 2.42354e-18 0.00016482 0.00276964 -1670 1013.25 0.994465 0.00110356 0 5.13064e-07 0.00155063 3.28864e-05 3.87646e-05 6.92342e-07 1.28085e-10 4.88212e-13 2.25957e-09 1.19473e-18 0.00019596 0.00261227 -1700 1013.25 0.993751 0.00146958 0 3.9752e-07 0.00205043 3.96806e-05 4.04562e-05 6.60325e-07 1.0594e-10 3.03457e-13 1.7655e-09 5.92334e-19 0.000228426 0.00241951 -1730 1013.25 0.992861 0.00193775 0 3.08058e-07 0.00266212 4.715e-05 4.17881e-05 6.25335e-07 8.66911e-11 1.88538e-13 1.36776e-09 2.9342e-19 0.000260273 0.00218934 -1760 1013.25 0.991771 0.00253118 0 2.38146e-07 0.00338805 5.50742e-05 4.26323e-05 5.86689e-07 6.98599e-11 1.1648e-13 1.04564e-09 1.44097e-19 0.000288734 0.0019229 -1790 1013.25 0.990462 0.00327691 0 1.8308e-07 0.00421763 6.3091e-05 4.28475e-05 5.43813e-07 5.51278e-11 7.1117e-14 7.84319e-10 6.95117e-20 0.000310301 0.00162621 -1820 1013.25 0.988925 0.00420636 0 1.39469e-07 0.00512174 7.06825e-05 4.22942e-05 4.96409e-07 4.232e-11 4.26092e-14 5.73378e-10 3.25917e-20 0.000321132 0.00131177 -1850 1013.25 0.987159 0.00535582 0 1.04889e-07 0.00604945 7.72045e-05 4.08672e-05 4.4472e-07 3.13842e-11 2.48659e-14 4.05645e-10 1.46877e-20 0.000317969 0.000999051 -1880 1013.25 0.985168 0.00676687 0 7.76215e-08 0.00693209 8.19993e-05 3.85453e-05 3.89851e-07 2.23473e-11 1.40429e-14 2.76011e-10 6.30047e-21 0.000299542 0.000712083 -1910 1013.25 0.982952 0.00848683 0 5.64319e-08 0.00769948 8.45987e-05 3.54437e-05 3.33955e-07 1.52357e-11 7.65012e-15 1.80104e-10 2.56023e-21 0.000267841 0.000473131 -1940 1013.25 0.980487 0.0105691 0 4.03518e-08 0.00830454 8.49298e-05 3.18237e-05 2.79947e-07 9.97286e-12 4.02965e-15 1.13006e-10 9.89071e-22 0.000228063 0.000294369 -1970 1013.25 0.977715 0.0130734 0 2.84932e-08 0.00873957 8.33499e-05 2.8024e-05 2.30633e-07 6.32127e-12 2.06911e-15 6.87602e-11 3.6773e-22 0.000186605 0.000173784