Commit 9e7cb9fc authored by Timo Koch's avatar Timo Koch
Browse files

[tests] Improvements in fuzzycomparevtu

- Parameters with more than one component are split into components for
  comparison
- The absoulte criteria gets scaled with the maximum absolute value in
  the parameter set
- It is possible to pass a zero threshold, i.e. the threshold under which
  values get treated as zero, for each parameter or
  parameter component via runtest as follows
  --zeroThreshold = {"vel":1e-9,"bla":10}
  The format is a python dictionary except single quotes are not allowed
  (because it uses json)
- More verbose output


git-svn-id: svn://svn.iws.uni-stuttgart.de/DUMUX/dumux/trunk@15254 2fb0f335-1f38-0410-981e-8018bf24f1b0
parent f621f8fe
......@@ -15,7 +15,7 @@ from six.moves import zip
# fuzzy compare VTK tree from VTK strings
def compare_vtk(vtk1, vtk2, absolute=1e-9, relative=1e-2, verbose=True):
def compare_vtk(vtk1, vtk2, absolute=1.5e-7, relative=1e-2, zeroValueThreshold={}, verbose=True):
""" take two vtk files and compare them. Returns an exit key as returnvalue.
Arguments:
......@@ -29,6 +29,13 @@ def compare_vtk(vtk1, vtk2, absolute=1e-9, relative=1e-2, verbose=True):
The epsilon used for comparing numbers with an absolute criterion
relative: float
The epsilon used for comparing numbers with an relative criterion
zeroValueThreshold: dict
A dictionary of parameter value pairs that set the threshold under
which a number is treated as zero for a certain parameter. Use this parameter if
you have to avoid comparisons of very small numbers for a certain parameter.
verbose : bool
If the script should produce informative output. Enabled by default as the details
give the tester a lot more information on why tests fail.
"""
# construct element tree from vtk file
......@@ -42,20 +49,21 @@ def compare_vtk(vtk1, vtk2, absolute=1e-9, relative=1e-2, verbose=True):
if verbose:
print("Comparing {} and {}".format(vtk1, vtk2))
print("...with a maximum relative error of {} and a maximum absolute error of {}*p_max, where p_max is highest absolute parameter value.".format(relative, absolute))
# sort the vtk file so that the comparison is independent of the
# index numbering (coming e.g. from different grid managers)
sortedroot1, sortedroot2 = sort_vtk_by_coordinates(sortedroot1, sortedroot2, verbose)
# do the fuzzy compare
if is_fuzzy_equal_node(sortedroot1, sortedroot2, absolute, relative, verbose):
if is_fuzzy_equal_node(sortedroot1, sortedroot2, absolute, relative, zeroValueThreshold, verbose):
return 0
else:
return 1
# fuzzy compare of VTK nodes
def is_fuzzy_equal_node(node1, node2, absolute, relative, verbose=True):
def is_fuzzy_equal_node(node1, node2, absolute, relative, zeroValueThreshold, verbose):
is_equal = True
for node1child, node2child in zip(node1.iter(), node2.iter()):
......@@ -78,7 +86,10 @@ def is_fuzzy_equal_node(node1, node2, absolute, relative, verbose=True):
else:
return False
if node1child.text or node2child.text:
if not is_fuzzy_equal_text(node1child.text, node2child.text, node1child.attrib["Name"], absolute, relative, verbose):
if not is_fuzzy_equal_text(node1child.text, node2child.text,
node1child.attrib["Name"],
int(node1child.attrib["NumberOfComponents"]),
absolute, relative, zeroValueThreshold, verbose):
if node1child.attrib["Name"] == node2child.attrib["Name"]:
if verbose:
print('Data differs in parameter: {}'.format(node1child.attrib["Name"]))
......@@ -95,7 +106,7 @@ def is_fuzzy_equal_node(node1, node2, absolute, relative, verbose=True):
# fuzzy compare of text (in the xml sense) consisting of whitespace separated numbers
def is_fuzzy_equal_text(text1, text2, parameter, absolute, relative, verbose=True):
def is_fuzzy_equal_text(text1, text2, parameter, numComp, absolute, relative, zeroValueThreshold, verbose):
list1 = text1.split()
list2 = text2.split()
# difference only in whitespace?
......@@ -103,34 +114,75 @@ def is_fuzzy_equal_text(text1, text2, parameter, absolute, relative, verbose=Tru
return True
# compare number by number
is_equal = True
max_relative_difference = 0.0
max_absolute_difference = 0.0
for number1, number2 in zip(list1, list2):
number1 = float(number1)
number2 = float(number2)
if not number2 == 0.0:
# check for the relative difference
if abs(abs(number1 / number2) - 1.0) > relative and not abs(number1 - number2) < absolute:
if verbose:
#print('Relative difference is too large between: {} and {}'.format(number1, number2))
max_relative_difference = max(max_relative_difference, abs(abs(number1 / number2) - 1.0))
is_equal = False
else:
return False
# first split the list into compononents
lists1 = []
lists2 = []
parameters = []
for i in range(0, numComp):
lists1.append(list1[i::numComp])
lists2.append(list2[i::numComp])
if numComp > 1:
parameters.append("{}_{}".format(parameter, i))
# if zero threshold was set for all components one component inherits it from the parameter
if parameter in zeroValueThreshold:
zeroValueThreshold["{}_{}".format(parameter, i)] = zeroValueThreshold[parameter]
else:
# check for the absolute difference
if abs(number1 - number2) > absolute:
parameters.append(parameter)
for list1, list2, parameter in zip(lists1, lists2, parameters):
# for verbose output
max_relative_difference = 0.0
message = ''
# see inspiration, explanations in
# https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/
# get the order of magnitude of the parameter by calculating the max
floatList1 = [float(i) for i in list1]
floatList2 = [float(i) for i in list2]
# manipulate the data set for the sake of sensible comparison
# if the parameter is listed in the zeroThreshold dictionary replace all float under threshold with zero.
if parameter in zeroValueThreshold:
floatList1 = [0.0 if abs(i) < float(zeroValueThreshold[parameter]) else i for i in floatList1]
floatList2 = [0.0 if abs(i) < float(zeroValueThreshold[parameter]) else i for i in floatList2]
absFloatList1 = [abs(i) for i in floatList1]
absFloatList2 = [abs(i) for i in floatList2]
magnitude = max(max(absFloatList1), max(absFloatList2))
minimal = min(min(absFloatList1), min(absFloatList2))
for number1, number2 in zip(floatList1, floatList2):
diff = abs(number1 - number2)
largernumber = max(abs(number1), abs(number2))
# If the absolute criterion is satisfied we consider the numbers equal...
# scale the absolute tolerance with the magnitude of the parameter
if diff <= absolute * magnitude:
continue
# ...if not check the relative criterion
if diff <= largernumber * relative:
continue
else:
# the numbers are not equal
if verbose:
#print('Absolute difference is too large between: {} and {}'.format(number1, number2))
max_absolute_difference = max(max_absolute_difference, abs(number1 - number2))
is_equal = False
if largernumber != 0.0:
if diff / largernumber > max_relative_difference:
message = 'Difference is too large between: {} and {}'.format(number1, number2)
max_relative_difference = diff / largernumber
else:
return False
if verbose:
if max_absolute_difference != 0.0:
print('Maximum absolute difference for parameter {}: {}'.format(parameter, max_absolute_difference))
if max_relative_difference != 0.0:
if verbose and max_relative_difference != 0.0:
print(message)
print('Maximum relative difference for parameter {}: {:.2%}'.format(parameter, max_relative_difference))
print('Info: The highest absolute value of {} is {} and the smallest {}.'.format(parameter, magnitude, minimal))
if parameter in zeroValueThreshold:
print('For parameter {} a zero value threshold of {} was given.'.format(parameter, zeroValueThreshold[parameter]))
return is_equal
......@@ -188,8 +240,8 @@ def sort_vtk(root):
# sorts the data by point coordinates so that it is independent of index numbering
def sort_vtk_by_coordinates(root1, root2, verbose=True):
if not is_fuzzy_equal_node(root1.find(".//Points/DataArray"), root2.find(".//Points/DataArray"), 1e-2, 1e-9, False):
def sort_vtk_by_coordinates(root1, root2, verbose):
if not is_fuzzy_equal_node(root1.find(".//Points/DataArray"), root2.find(".//Points/DataArray"), absolute=1e-2, relative=1.5e-7, zeroValueThreshold=dict(), verbose=False):
if verbose:
print("Sorting vtu by coordinates...")
for root in [root1, root2]:
......@@ -210,8 +262,8 @@ def sort_vtk_by_coordinates(root1, root2, verbose=True):
coords = dataArrays["Coordinates"].split()
# group the coordinates into coordinate tuples
dim = int(numberOfComponents["Coordinates"])
for i in range(len(coords)//dim):
vertexArray.append([float(c) for c in coords[i*dim:i*dim+dim]])
for i in range(len(coords) // dim):
vertexArray.append([float(c) for c in coords[i * dim: i * dim + dim]])
# obtain a vertex index map
vMap = []
......@@ -245,8 +297,8 @@ def sort_vtk_by_coordinates(root1, root2, verbose=True):
# convert if vector
num = int(numberOfComponents[name])
newitems = []
for i in range(len(items)//num):
newitems.append([i for i in items[i*num:i*num+num]])
for i in range(len(items) // num):
newitems.append([i for i in items[i * num: i * num + num]])
items = newitems
# sort the items: we have either vertex or cell data
if name in pointDataArrays:
......@@ -287,7 +339,7 @@ if __name__ == "__main__":
parser.add_argument('vtk_file_1', type=str, help='first file to compare')
parser.add_argument('vtk_file_2', type=str, help='second file to compare')
parser.add_argument('-r', '--relative', type=float, default=1e-2, help='maximum relative error (default=1e-2)')
parser.add_argument('-a', '--absolute', type=float, default=1e-9, help='maximum absolute error (default=1e-9)')
parser.add_argument('-a', '--absolute', type=float, default=1.5e-7, help='maximum absolute error (default=1.5e-7)')
parser.add_argument('-v', '--verbose', type=bool, default=True, help='verbosity of the script')
args = vars(parser.parse_args())
......
import argparse
import os, sys
import subprocess
import json
from fuzzycomparevtu import compare_vtk
# parse arguments
......@@ -9,7 +10,8 @@ parser.add_argument('-c', '--command', nargs=1, help='The executable and optiona
parser.add_argument('-s', '--script', nargs=1, help="The comparison script. [fuzzy, exact, <path_to_script>] where the script takes two vtu files as arguments.")
parser.add_argument('-f', '--files', nargs='+', help="Pairs of reference and vtu file names. Usage: '[-f ref1 vtu1 [[ref2] [vtu2] ...]]'")
parser.add_argument('-r', '--relative', type=float, default=1e-2, help='maximum relative error (default=1e-2) when using fuzzy comparison')
parser.add_argument('-a', '--absolute', type=float, default=1e-9, help='maximum absolute error (default=1e-9) when using fuzzy comparison')
parser.add_argument('-a', '--absolute', type=float, default=1.5e-7, help='maximum absolute error (default=1.5e-7) when using fuzzy comparison')
parser.add_argument('-z', '--zeroThreshold', type=json.loads, default='{}', help='Thresholds for treating numbers as zero for a parameter as a python dict e.g. {"vel":1e-7,"delP":1.0}')
args = vars(parser.parse_args())
# check parameters
......@@ -54,14 +56,7 @@ if args['script']:
return_code = 0
for i in range(0, len(args['files'])//2):
print("\nFuzzy comparison...")
if args['relative'] and args['absolute']:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], relative=args['relative'], absolute=args['absolute'])
elif args['relative'] and not args['absolute']:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], relative=args['relative'])
elif args['absolute'] and not args['relative']:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], absolute=args['absolute'])
else:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1])
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], relative=args['relative'], absolute=args['absolute'], zeroValueThreshold=args['zeroThreshold'])
if result:
return_code = 1
sys.exit(return_code)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment