[Test] Fixed calculation of precision for string representations
This corrects the calculation of the least significant digit for all string representations of for numbers where no decimal point was printed, e.g. "123" and "45e+06".
This commit is contained in:
parent
aade0ddb67
commit
f90d9b80d1
1 changed files with 13 additions and 6 deletions
|
|
@ -247,15 +247,22 @@ def getPrecision(x):
|
|||
the number represented by the string 'x'.
|
||||
"""
|
||||
x = x.lower()
|
||||
# Patterns to consider:
|
||||
# 123
|
||||
# 123.45
|
||||
# 123.45e6
|
||||
# 123e4
|
||||
if 'e' in x:
|
||||
x, exponent = x.split('e')
|
||||
exponent = int(exponent)
|
||||
else:
|
||||
exponent = 0
|
||||
|
||||
decimalPt = x.find('.')
|
||||
if decimalPt == -1:
|
||||
decimalPt = 0
|
||||
decimalPt = len(x) - 1
|
||||
|
||||
if x.find('e') != -1:
|
||||
precision = decimalPt - x.find('e') + 1
|
||||
precision += int(x[x.find('e')+1:])
|
||||
else:
|
||||
precision = decimalPt - len(x) + 1
|
||||
precision = decimalPt + exponent - len(x) + 1
|
||||
|
||||
return 10**precision
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue