[Test] Fixed calculation of precision for string representations

This corrects the calculation of the least significant digit for all string
representations of for numbers where no decimal point was printed, e.g. "123"
and "45e+06".
This commit is contained in:
Ray Speth 2013-04-24 21:47:16 +00:00
parent aade0ddb67
commit f90d9b80d1

View file

@ -247,15 +247,22 @@ def getPrecision(x):
the number represented by the string 'x'.
"""
x = x.lower()
# Patterns to consider:
# 123
# 123.45
# 123.45e6
# 123e4
if 'e' in x:
x, exponent = x.split('e')
exponent = int(exponent)
else:
exponent = 0
decimalPt = x.find('.')
if decimalPt == -1:
decimalPt = 0
decimalPt = len(x) - 1
if x.find('e') != -1:
precision = decimalPt - x.find('e') + 1
precision += int(x[x.find('e')+1:])
else:
precision = decimalPt - len(x) + 1
precision = decimalPt + exponent - len(x) + 1
return 10**precision