Commit 7a01fc4a authored by Timo Koch's avatar Timo Koch
Browse files

New runtest script and fuzzy compare.

Adapt all Ctests to the new, architecture.

Reviewd by fetzer


git-svn-id: svn://svn.iws.uni-stuttgart.de/DUMUX/dumux/trunk@15129 2fb0f335-1f38-0410-981e-8018bf24f1b0
parent cd6d4ced
""" A module for fuzzy comparing VTK files.
This module provides methods to compare two VTK files. Applicable
for all VTK style formats like VTK files. Fuzzy compares numbers by
using absolute and/or relative difference comparison.
"""
from __future__ import absolute_import
import argparse
from xml.dom import minidom
# fuzzy compare XML tree from XML strings
def isFuzzyEqualXml(xml1, xml2, absolute, relative):
dom1 = minidom.parseString(xml1)
dom2 = minidom.parseString(xml2)
return isFuzzyEqualNode(dom1.documentElement, dom2.documentElement, absolute, relative)
# fuzzy compare of XML nodes
def isFuzzyEqualNode(node1, node2, absolute, relative):
noError = True
if node1.tagName != node2.tagName:
print 'The name of the node differs in ', node1.tagName, ' and ', node2.tagName
return False
if sorted(node1.attributes.items()) != sorted(node2.attributes.items()):
print 'Attributes differ in node ', node1.tagName
return False
if len(node1.childNodes) != len(node2.childNodes):
print 'Number of children differs in node ', node1.tagName
return False
for node1child, node2child in zip(node1.childNodes, node2.childNodes):
if node1child.nodeType != node2child.nodeType:
print 'Node type differs in ', node1.tagName
return False
if node1child.nodeType == node1child.TEXT_NODE and not isFuzzyEqualText(node1child.data, node2child.data, absolute, relative):
print 'Data differs in parameter ', node2.attributes.items(), ' at node ', node2child.nodeType
noError = False
continue
if node1child.nodeType == node1child.ELEMENT_NODE and not isFuzzyEqualNode(node1child, node2child, absolute, relative):
noError = False
continue
return noError
# fuzzy compare of text consisting of whitespace separated numbers
def isFuzzyEqualText(text1, text2, absolute, relative):
import xml.etree.ElementTree as ET
from operator import attrgetter, itemgetter
import sys
from six.moves import range
from six.moves import zip
# fuzzy compare VTK tree from VTK strings
def compare_vtk(vtk1, vtk2, absolute=1e-9, relative=1e-2, verbose=True):
""" take two vtk files and compare them. Returns an exit key as returnvalue.
Arguments:
----------
vtk1, vtk2 : string
The filenames of the vtk files to compare
Keyword Arguments:
------------------
absolute : float
The epsilon used for comparing numbers with an absolute criterion
relative: float
The epsilon used for comparing numbers with an relative criterion
"""
# construct element tree from vtk file
root1 = ET.fromstring(open(vtk1).read())
root2 = ET.fromstring(open(vtk2).read())
# sort the vtk file in case nodes appear in different positions
# e.g. because of minor changes in the output code
sortedroot1 = sort_vtk(root1)
sortedroot2 = sort_vtk(root2)
if verbose:
print("Comparing {} and {}".format(vtk1, vtk2))
# sort the vtk file so that the comparison is independent of the
# index numbering (coming e.g. from different grid managers)
sortedroot1, sortedroot2 = sort_vtk_by_coordinates(sortedroot1, sortedroot2, verbose)
# do the fuzzy compare
if is_fuzzy_equal_node(sortedroot1, sortedroot2, absolute, relative, verbose):
return 0
else:
return 1
# fuzzy compare of VTK nodes
def is_fuzzy_equal_node(node1, node2, absolute, relative, verbose=True):
is_equal = True
for node1child, node2child in zip(node1.iter(), node2.iter()):
if node1.tag != node2.tag:
if verbose:
print('The name of the node differs in: {} and {}'.format(node1.tag, node2.tag))
is_equal = False
else:
return False
if list(node1.attrib.items()) != list(node2.attrib.items()):
if verbose:
print('Attributes differ in node: {}'.format(node1.tag))
is_equal = False
else:
return False
if len(list(node1.iter())) != len(list(node2.iter())):
if verbose:
print('Number of children differs in node: {}'.format(node1.tag))
is_equal = False
else:
return False
if node1child.text or node2child.text:
if not is_fuzzy_equal_text(node1child.text, node2child.text, node1child.attrib["Name"], absolute, relative, verbose):
if node1child.attrib["Name"] == node2child.attrib["Name"]:
if verbose:
print('Data differs in parameter: {}'.format(node1child.attrib["Name"]))
is_equal = False
else:
return False
else:
if verbose:
print('Comparing different parameters: {} and {}'.format(node1child.attrib["Name"], node2child.attrib["Name"]))
is_equal = False
else:
return False
return is_equal
# fuzzy compare of text (in the xml sense) consisting of whitespace separated numbers
def is_fuzzy_equal_text(text1, text2, parameter, absolute, relative, verbose=True):
list1 = text1.split()
list2 = text2.split()
# difference only in whitespace?
if (list1 == list2):
return True
# compare number by number
maximum1 = 0.0
maximum2 = 0.0
is_equal = True
max_relative_difference = 0.0
max_absolute_difference = 0.0
for number1, number2 in zip(list1, list2):
number1 = float(number1)
number2 = float(number2)
if (abs(number1 - number2) > absolute
and (number2 == 0.0 or abs(abs(number1 / number2) - 1.0) > relative)):
if (abs(number1 - number2) > abs(maximum1 - maximum2)):
maximum1 = float(number1)
maximum2 = float(number2)
if (abs(maximum1 - maximum2) > 0.0):
print 'Difference is too large between', maximum1, ' and ', maximum2
return False
return True
# main program
# handle arguments and print help message
parser = argparse.ArgumentParser(description='Fuzzy compare of two VTK\
(Visualization Toolkit) files. The files are accepted if for every\
value the difference is below the absolute error or below the\
relative error or below both.')
parser.add_argument('vtu_file_1', type=open,
help='first file to compare')
parser.add_argument('vtu_file_2', type=open,
help='second file to compare')
parser.add_argument('-r', '--relative', type=float, default=1e-2,
help='maximum relative error (default=1e-2)')
parser.add_argument('-a', '--absolute', type=float, default=1e-9,
help='maximum absolute error (default=1e-9)')
args = parser.parse_args()
# fuzzy compare
if (isFuzzyEqualXml(args.vtu_file_1.read(), args.vtu_file_2.read(), args.absolute, args.relative)):
exit
else:
exit(1)
if not number2 == 0.0:
# check for the relative difference
if abs(abs(number1 / number2) - 1.0) > relative and not abs(number1 - number2) < absolute:
if verbose:
#print('Relative difference is too large between: {} and {}'.format(number1, number2))
max_relative_difference = max(max_relative_difference, abs(abs(number1 / number2) - 1.0))
is_equal = False
else:
return False
else:
# check for the absolute difference
if abs(number1 - number2) > absolute:
if verbose:
#print('Absolute difference is too large between: {} and {}'.format(number1, number2))
max_absolute_difference = max(max_absolute_difference, abs(number1 - number2))
is_equal = False
else:
return False
if verbose:
if max_absolute_difference != 0.0:
print('Maximum absolute difference for parameter {}: {}'.format(parameter, max_absolute_difference))
if max_relative_difference != 0.0:
print('Maximum relative difference for parameter {}: {:.2%}'.format(parameter, max_relative_difference))
return is_equal
def sort_by_name(elem):
name = elem.get('Name')
if name:
try:
return str(name)
except ValueError:
return ''
return ''
# sorts attributes of an item and returns a sorted item
def sort_attributes(item, sorteditem):
attrkeys = sorted(item.keys())
for key in attrkeys:
sorteditem.set(key, item.get(key))
def sort_elements(items, newroot):
items = sorted(items, key=sort_by_name)
items = sorted(items, key=attrgetter('tag'))
# Once sorted, we sort each of the items
for item in items:
# Create a new item to represent the sorted version
# of the next item, and copy the tag name and contents
newitem = ET.Element(item.tag)
if item.text and item.text.isspace() == False:
newitem.text = item.text
# Copy the attributes (sorted by key) to the new item
sort_attributes(item, newitem)
# Copy the children of item (sorted) to the new item
sort_elements(list(item), newitem)
# Append this sorted item to the sorted root
newroot.append(newitem)
# has to sort all Cell and Point Data after the attribute "Name"!
def sort_vtk(root):
if(root.tag != "VTKFile"):
print('Format is not a VTKFile. Sorting will most likely fail!')
# create a new root for the sorted tree
newroot = ET.Element(root.tag)
# create the sorted copy
# (after the idea of Dale Lane's xmldiff.py)
sort_attributes(root, newroot)
sort_elements(list(root), newroot)
# return the sorted element tree
return newroot
# sorts the data by point coordinates so that it is independent of index numbering
def sort_vtk_by_coordinates(root1, root2, verbose=True):
if not is_fuzzy_equal_node(root1.find(".//Points/DataArray"), root2.find(".//Points/DataArray"), 1e-2, 1e-9, False):
if verbose:
print("Sorting vtu by coordinates...")
for root in [root1, root2]:
# parse all DataArrays in a dictionary
pointDataArrays = []
cellDataArrays = []
dataArrays = {}
numberOfComponents = {}
for dataArray in root.findall(".//PointData/DataArray"):
pointDataArrays.append(dataArray.attrib["Name"])
for dataArray in root.findall(".//CellData/DataArray"):
cellDataArrays.append(dataArray.attrib["Name"])
for dataArray in root.findall(".//DataArray"):
dataArrays[dataArray.attrib["Name"]] = dataArray.text
numberOfComponents[dataArray.attrib["Name"]] = dataArray.attrib["NumberOfComponents"]
vertexArray = []
coords = dataArrays["Coordinates"].split()
# group the coordinates into coordinate tuples
dim = int(numberOfComponents["Coordinates"])
for i in range(len(coords)//dim):
vertexArray.append([float(c) for c in coords[i*dim:i*dim+dim]])
# obtain a vertex index map
vMap = []
for idx, coords in enumerate(vertexArray):
vMap.append((idx, coords))
sortedVMap = sorted(vMap, key=itemgetter(1))
vertexIndexMap = {}
for idxNew, idxOld in enumerate(sortedVMap):
vertexIndexMap[idxOld[0]] = idxNew
# group the cells into vertex index tuples
cellArray = []
offsets = dataArrays["offsets"].split()
connectivity = dataArrays["connectivity"].split()
vertex = 0
for cellIdx, offset in enumerate(offsets):
cellArray.append([])
for v in range(vertex, int(offset)):
cellArray[cellIdx].append(int(connectivity[v]))
vertex += 1
# replace all vertex indices in the cellArray by the new indices
for cell in cellArray:
for idx, vertexIndex in enumerate(cell):
cell[idx] = vertexIndexMap[vertexIndex]
# sort all data arrays
for name, text in list(dataArrays.items()):
# split the text
items = text.split()
# convert if vector
num = int(numberOfComponents[name])
newitems = []
for i in range(len(items)//num):
newitems.append([i for i in items[i*num:i*num+num]])
items = newitems
# sort the items: we have either vertex or cell data
if name in pointDataArrays:
sortedItems = [j for (i, j) in sorted(zip(vertexArray, items), key=itemgetter(0))]
elif name in cellDataArrays or name == "types":
sortedItems = [j for (i, j) in sorted(zip(cellArray, items), key=itemgetter(0))]
elif name == "offsets":
sortedItems = []
counter = 0
for cell in sorted(cellArray):
counter += len(cell)
sortedItems.append([str(counter)])
elif name == "Coordinates":
sortedItems = sorted(vertexArray)
elif name == "connectivity":
sortedItems = sorted(cellArray)
# convert the sorted arrays to a xml text
dataArrays[name] = ""
for i in sortedItems:
for j in i:
dataArrays[name] += str(j) + " "
# do the replacement in the actual elements
for dataArray in root.findall(".//DataArray"):
dataArray.text = dataArrays[dataArray.attrib["Name"]]
return (root1, root2)
# main program if called as script return appropriate error codes
if __name__ == "__main__":
# handle arguments and print help message
parser = argparse.ArgumentParser(description='Fuzzy compare of two VTK\
(Visualization Toolkit) files. The files are accepted if for every\
value the difference is below the absolute error or below the\
relative error or below both.')
parser.add_argument('vtk_file_1', type=str, help='first file to compare')
parser.add_argument('vtk_file_2', type=str, help='second file to compare')
parser.add_argument('-r', '--relative', type=float, default=1e-2, help='maximum relative error (default=1e-2)')
parser.add_argument('-a', '--absolute', type=float, default=1e-9, help='maximum absolute error (default=1e-9)')
parser.add_argument('-v', '--verbose', type=bool, default=True, help='verbosity of the script')
args = vars(parser.parse_args())
sys.exit(compare_vtk(args["vtk_file_1"], args["vtk_file_2"], args["absolute"], args["relative"], args["verbose"]))
#! /bin/bash
#
# Runs a test from the test directory and compare the resulting VTU files.
#
# Usage:
#
# runTest.sh COMPAREFLAG/SCRIPT REFERENCE_RESULT_FILE TEST_RESULT_FILE TEST_BINARY TEST_ARGS
#
function usage() {
echo "Usage:"
echo
echo "runTest.sh COMPAREFLAG/SCRIPT REFERENCE_RESULT_FILE TEST_RESULT_FILE TEST_BINARY [TEST_ARGS]"
echo "COMPAREFLAG: fuzzyvtu - uses the fuzzycomparevtu.py script"
echo " exact - uses diff for exact compare of two files"
echo " PATH_TO_SCRIPT - tries to execute the custom script"
};
CMAKE_SOURCE_DIR=$(dirname "$0")
COMPARE_FLAG="$1"
REFERENCE_RESULT="$2"
TEST_RESULT="$3"
TEST_BINARY="$4"
TEST_ARGS="${@:5:100}"
rm -fv $TEST_RESULT
# make sure we have at least 3 parameters
if test "$#" -lt 4; then
echo "Wrong number of parameters"
echo
usage
exit 1
fi
# make sure the reference result exists
if ! test -r "$REFERENCE_RESULT"; then
echo "File $REFERENCE_RESULT does not exist or is not readable"
echo
usage
exit 1
fi
#run the test
echo "######################"
echo "# Running test"
echo "######################"
$TEST_BINARY $TEST_ARGS
RETURN_CODE_TEST_BINARY=$?
if test "$RETURN_CODE_TEST_BINARY" != "0"; then
echo "Return code: $RETURN_CODE_TEST_BINARY"
exit $RETURN_CODE_TEST_BINARY
fi
# compare the results
echo "######################"
echo "# Comparing results"
echo "######################"
if ! test -r "$TEST_RESULT"; then
echo "File $TEST_RESULT does not exist or is not readable"
exit 1
fi
# running the compare script
NOT_EQUAL=false
if [ "$COMPARE_FLAG" = "fuzzyvtu" -o \
"$COMPARE_FLAG" = "$CMAKE_SOURCE_DIR/fuzzycomparevtu.py" ]; then
if ! python $CMAKE_SOURCE_DIR/fuzzycomparevtu.py "$REFERENCE_RESULT" "$TEST_RESULT"; then
NOT_EQUAL=true
fi
elif [ "$COMPARE_FLAG" = "exact" ]; then
if ! diff "$REFERENCE_RESULT" "$TEST_RESULT"; then
NOT_EQUAL=true
fi
elif [ -e "$COMPARE_FLAG" ]; then
if ! $COMPARE_FLAG "$REFERENCE_RESULT" "$TEST_RESULT"; then
NOT_EQUAL=true
fi
else
echo
echo "ERROR: $0 was not able to run the compare script:"
echo " $COMPARE_FLAG"
echo
exit 2
fi
# printing error message in case of failure
if [ "$NOT_EQUAL" = "true" ]; then
echo "The files \"$TEST_RESULT\" and \"$REFERENCE_RESULT\" are different."
echo "Make sure the contents of \"$TEST_RESULT\" are still valid and "
echo "make it the reference result if necessary."
exit 1
fi
# SUCCESS!!!!!!
echo "Result and reference result are identical"
exit 0
import argparse
import os, sys
import subprocess
from fuzzycomparevtu import compare_vtk
# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--command', nargs=1, help='The executable and optional arguments as a single string', required=True)
parser.add_argument('-s', '--script', nargs=1, help="The comparison script. [fuzzy, exact, <path_to_script>] where the script takes two vtu files as arguments.")
parser.add_argument('-f', '--files', nargs='+', help="Pairs of reference and vtu file names. Usage: '[-f ref1 vtu1 [[ref2] [vtu2] ...]]'")
parser.add_argument('-r', '--relative', type=float, default=1e-2, help='maximum relative error (default=1e-2) when using fuzzy comparison')
parser.add_argument('-a', '--absolute', type=float, default=1e-9, help='maximum absolute error (default=1e-9) when using fuzzy comparison')
args = vars(parser.parse_args())
# check parameters
if args['script']:
if len(args['files'])%2 != 0 or not args['files']:
sys.stderr.write("The files have to be pairs of vtu and reference files. Usage '-f [ref1] [vtu1] [[ref2] [vtu2] ...]'")
parser.print_help()
sys.exit(1)
for i in range(0, len(args['files'])//2):
# delete the vtu files to compare
ref_dir = os.path.dirname(os.path.abspath(__file__)).rstrip("bin") + "test/references"
if os.path.dirname(args['files'][(i*2)+1]) == ref_dir:
sys.stderr.write("Tried to delete a reference solution. Specify reference file first, then the VTU file. Usage: '[-f ref1 vtu1 [[ref2] [vtu2] ...]]'")
sys.exit(1)
subprocess.call(['rm', '-fv', args['files'][(i*2)+1]])
# run the test
res = 1
try:
res = subprocess.call(args['command'][0].split())
except OSError as e:
print(args['command'][0].split())
print("OSError: Command not found. Most likely the executable specified doesn't exist.")
sys.exit(1)
if res:
sys.exit(1)
# run the comparison
if args['script']:
# exact comparison?
if args['script'] == ['exact']:
return_code = 0
for i in range(0, len(args['files'])//2):
print("\nExact comparison...")
result = subprocess.call(['diff', args['files'][i*2], args['files'][(i*2)+1]])
if result:
return_code = 1
sys.exit(return_code)
# fuzzy comparison?
elif args['script'] == ["fuzzy"] or args['script'] == [os.path.dirname(os.path.abspath(__file__)) + "/fuzzycomparevtu.py"]:
return_code = 0
for i in range(0, len(args['files'])//2):
print("\nFuzzy comparison...")
if args['relative'] and args['absolute']:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], relative=args['relative'], absolute=args['absolute'])
elif args['relative'] and not args['absolute']:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], relative=args['relative'])
elif args['absolute'] and not args['relative']:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], absolute=args['absolute'])
else:
result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1])
if result:
return_code = 1
sys.exit(return_code)
# other script?
else:
return_code = 0
for i in range(0, len(args['files'])//2):
print("\n{} comparison...".format(args['script']))
result = subprocess.call(args['script'], args['files'][i*2], args['files'][(i*2)+1])
if result:
return_code = 1
sys.exit(return_code)
# everything is fine
sys.exit(0)
add_dumux_test(test_general_box test_general_box test_generalproblem2p.cc
${CMAKE_SOURCE_DIR}/bin/runTest.sh
${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
${CMAKE_SOURCE_DIR}/test/references/generallens_box-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/generallens_box-00003.vtu
${CMAKE_CURRENT_BINARY_DIR}/test_general_box
-ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
-ModelType Box)
python ${CMAKE_SOURCE_DIR}/bin/runtest.py
--script fuzzy
--files ${CMAKE_SOURCE_DIR}/test/references/generallens_box-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/generallens_box-00003.vtu
--command "${CMAKE_CURRENT_BINARY_DIR}/test_general_box
-ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
-ModelType Box")
add_dumux_test(test_general_dec test_general_dec test_generalproblem2p.cc
${CMAKE_SOURCE_DIR}/bin/runTest.sh
${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
${CMAKE_SOURCE_DIR}/test/references/generallens_decoupled-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/generallens_decoupled-00003.vtu
${CMAKE_CURRENT_BINARY_DIR}/test_general_dec
-ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
-ModelType Decoupled)
python ${CMAKE_SOURCE_DIR}/bin/runtest.py
--script fuzzy
--files ${CMAKE_SOURCE_DIR}/test/references/generallens_decoupled-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/generallens_decoupled-00003.vtu
--command "${CMAKE_CURRENT_BINARY_DIR}/test_general_dec
-ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
-ModelType Decoupled")
add_dumux_test(test_dec1p test_dec1p test_1p.cc
${CMAKE_SOURCE_DIR}/bin/runTest.sh
${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
${CMAKE_SOURCE_DIR}/test/references/test_1p-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/test_1p-00001.vtu
${CMAKE_CURRENT_BINARY_DIR}/test_dec1p
-ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_1p.input)
python ${CMAKE_SOURCE_DIR}/bin/runtest.py
--script fuzzy
--files ${CMAKE_SOURCE_DIR}/test/references/test_1p-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/test_1p-00001.vtu
--command "${CMAKE_CURRENT_BINARY_DIR}/test_dec1p -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_1p.input")
add_dumux_test(test_diffusion test_diffusion test_diffusion.cc
${CMAKE_SOURCE_DIR}/bin/runTest.sh
${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
${CMAKE_SOURCE_DIR}/test/references/diffusion-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/mimeticdiffusion-00001.vtu
${CMAKE_CURRENT_BINARY_DIR}/test_diffusion
3)
python ${CMAKE_SOURCE_DIR}/bin/runtest.py
--script fuzzy
--files ${CMAKE_SOURCE_DIR}/test/references/diffusion-reference.vtu
${CMAKE_CURRENT_BINARY_DIR}/mimeticdiffusion-00001.vtu
--command "${CMAKE_CURRENT_BINARY_DIR}/test_diffusion 3")