New runtest script and fuzzy compare.

Adapt all Ctests to the new, architecture.

Reviewd by fetzer


git-svn-id: svn://svn.iws.uni-stuttgart.de/DUMUX/dumux/trunk@15129 2fb0f335-1f38-0410-981e-8018bf24f1b0

bin/fuzzycomparevtu.py

+""" A module for fuzzy comparing VTK files.
+
+This module provides methods to compare two VTK files. Applicable
+for all VTK style formats like VTK files. Fuzzy compares numbers by
+using absolute and/or relative difference comparison.
+
+"""
+from __future__ import absolute_import
 import argparse
-from xml.dom import minidom
-
-# fuzzy compare XML tree from XML strings
-def isFuzzyEqualXml(xml1, xml2, absolute, relative):
-    dom1 = minidom.parseString(xml1)
-    dom2 = minidom.parseString(xml2)
-    return isFuzzyEqualNode(dom1.documentElement, dom2.documentElement, absolute, relative)
-
-# fuzzy compare of XML nodes
-def isFuzzyEqualNode(node1, node2, absolute, relative):
-    noError = True
-    if node1.tagName != node2.tagName:
-        print 'The name of the node differs in ', node1.tagName, ' and ', node2.tagName
-        return False
-    if sorted(node1.attributes.items()) != sorted(node2.attributes.items()):
-        print 'Attributes differ in node ', node1.tagName
-        return False
-    if len(node1.childNodes) != len(node2.childNodes):
-        print 'Number of children differs in node ', node1.tagName
-        return False
-    for node1child, node2child in zip(node1.childNodes, node2.childNodes):
-        if node1child.nodeType != node2child.nodeType:
-            print 'Node type differs in ', node1.tagName
-            return False
-        if node1child.nodeType == node1child.TEXT_NODE and not isFuzzyEqualText(node1child.data, node2child.data, absolute, relative):
-            print 'Data differs in parameter ', node2.attributes.items(), ' at node ', node2child.nodeType
-            noError = False
-            continue
-        if node1child.nodeType == node1child.ELEMENT_NODE and not isFuzzyEqualNode(node1child, node2child, absolute, relative):
-            noError = False
-            continue
-    return noError
-
-# fuzzy compare of text consisting of whitespace separated numbers
-def isFuzzyEqualText(text1, text2, absolute, relative):
+import xml.etree.ElementTree as ET
+from operator import attrgetter, itemgetter
+import sys
+from six.moves import range
+from six.moves import zip
+
+
+# fuzzy compare VTK tree from VTK strings
+def compare_vtk(vtk1, vtk2, absolute=1e-9, relative=1e-2, verbose=True):
+    """ take two vtk files and compare them. Returns an exit key as returnvalue.
+
+    Arguments:
+    ----------
+    vtk1, vtk2 : string
+        The filenames of the vtk files to compare
+
+    Keyword Arguments:
+    ------------------
+    absolute : float
+        The epsilon used for comparing numbers with an absolute criterion
+    relative: float
+        The epsilon used for comparing numbers with an relative criterion
+    """
+
+    # construct element tree from vtk file
+    root1 = ET.fromstring(open(vtk1).read())
+    root2 = ET.fromstring(open(vtk2).read())
+
+    # sort the vtk file in case nodes appear in different positions
+    # e.g. because of minor changes in the output code
+    sortedroot1 = sort_vtk(root1)
+    sortedroot2 = sort_vtk(root2)
+
+    if verbose:
+        print("Comparing {} and {}".format(vtk1, vtk2))
+
+    # sort the vtk file so that the comparison is independent of the
+    # index numbering (coming e.g. from different grid managers)
+    sortedroot1, sortedroot2 = sort_vtk_by_coordinates(sortedroot1, sortedroot2, verbose)
+
+    # do the fuzzy compare
+    if is_fuzzy_equal_node(sortedroot1, sortedroot2, absolute, relative, verbose):
+        return 0
+    else:
+        return 1
+
+
+# fuzzy compare of VTK nodes
+def is_fuzzy_equal_node(node1, node2, absolute, relative, verbose=True):
+
+    is_equal = True
+    for node1child, node2child in zip(node1.iter(), node2.iter()):
+        if node1.tag != node2.tag:
+            if verbose:
+                print('The name of the node differs in: {} and {}'.format(node1.tag, node2.tag))
+                is_equal = False
+            else:
+                return False
+        if list(node1.attrib.items()) != list(node2.attrib.items()):
+            if verbose:
+                print('Attributes differ in node: {}'.format(node1.tag))
+                is_equal = False
+            else:
+                return False
+        if len(list(node1.iter())) != len(list(node2.iter())):
+            if verbose:
+                print('Number of children differs in node: {}'.format(node1.tag))
+                is_equal = False
+            else:
+                return False
+        if node1child.text or node2child.text:
+            if not is_fuzzy_equal_text(node1child.text, node2child.text, node1child.attrib["Name"], absolute, relative, verbose):
+                if node1child.attrib["Name"] == node2child.attrib["Name"]:
+                    if verbose:
+                        print('Data differs in parameter: {}'.format(node1child.attrib["Name"]))
+                        is_equal = False
+                    else:
+                        return False
+                else:
+                    if verbose:
+                        print('Comparing different parameters: {} and {}'.format(node1child.attrib["Name"], node2child.attrib["Name"]))
+                        is_equal = False
+                    else:
+                        return False
+    return is_equal
+
+
+# fuzzy compare of text (in the xml sense) consisting of whitespace separated numbers
+def is_fuzzy_equal_text(text1, text2, parameter, absolute, relative, verbose=True):
     list1 = text1.split()
     list2 = text2.split()
     # difference only in whitespace?
     if (list1 == list2):
         return True
     # compare number by number
-    maximum1 = 0.0
-    maximum2 = 0.0
+    is_equal = True
+    max_relative_difference = 0.0
+    max_absolute_difference = 0.0
     for number1, number2 in zip(list1, list2):
         number1 = float(number1)
         number2 = float(number2)
-        if (abs(number1 - number2) > absolute
-            and (number2 == 0.0 or abs(abs(number1 / number2) - 1.0) > relative)):
-            if (abs(number1 - number2) > abs(maximum1 - maximum2)):
-                maximum1 = float(number1)
-                maximum2 = float(number2)
-    if (abs(maximum1 - maximum2) > 0.0):
-        print 'Difference is too large between', maximum1, ' and ', maximum2
-        return False
-    return True
-
-# main program
-# handle arguments and print help message
-parser = argparse.ArgumentParser(description='Fuzzy compare of two VTK\
-    (Visualization Toolkit) files. The files are accepted if for every\
-    value the difference is below the absolute error or below the\
-    relative error or below both.')
-parser.add_argument('vtu_file_1', type=open,
-    help='first file to compare')
-parser.add_argument('vtu_file_2', type=open,
-    help='second file to compare')
-parser.add_argument('-r', '--relative', type=float, default=1e-2,
-    help='maximum relative error (default=1e-2)')
-parser.add_argument('-a', '--absolute', type=float, default=1e-9,
-    help='maximum absolute error (default=1e-9)')
-args = parser.parse_args()
-
-# fuzzy compare
-if (isFuzzyEqualXml(args.vtu_file_1.read(), args.vtu_file_2.read(), args.absolute, args.relative)):
-    exit
-else:
-    exit(1)
+        if not number2 == 0.0:
+            # check for the relative difference
+            if abs(abs(number1 / number2) - 1.0) > relative and not abs(number1 - number2) < absolute:
+                if verbose:
+                    #print('Relative difference is too large between: {} and {}'.format(number1, number2))
+                    max_relative_difference = max(max_relative_difference, abs(abs(number1 / number2) - 1.0))
+                    is_equal = False
+                else:
+                    return False
+        else:
+            # check for the absolute difference
+            if abs(number1 - number2) > absolute:
+                if verbose:
+                    #print('Absolute difference is too large between: {} and {}'.format(number1, number2))
+                    max_absolute_difference = max(max_absolute_difference, abs(number1 - number2))
+                    is_equal = False
+                else:
+                    return False
+    if verbose:
+        if max_absolute_difference != 0.0:
+            print('Maximum absolute difference for parameter {}: {}'.format(parameter, max_absolute_difference))
+        if max_relative_difference != 0.0:
+            print('Maximum relative difference for parameter {}: {:.2%}'.format(parameter, max_relative_difference))
+    return is_equal
+
+
+def sort_by_name(elem):
+    name = elem.get('Name')
+    if name:
+        try:
+            return str(name)
+        except ValueError:
+            return ''
+    return ''
+
+
+# sorts attributes of an item and returns a sorted item
+def sort_attributes(item, sorteditem):
+    attrkeys = sorted(item.keys())
+    for key in attrkeys:
+        sorteditem.set(key, item.get(key))
+
+
+def sort_elements(items, newroot):
+    items = sorted(items, key=sort_by_name)
+    items = sorted(items, key=attrgetter('tag'))
+
+    # Once sorted, we sort each of the items
+    for item in items:
+        # Create a new item to represent the sorted version
+        # of the next item, and copy the tag name and contents
+        newitem = ET.Element(item.tag)
+        if item.text and item.text.isspace() == False:
+            newitem.text = item.text
+
+        # Copy the attributes (sorted by key) to the new item
+        sort_attributes(item, newitem)
+
+        # Copy the children of item (sorted) to the new item
+        sort_elements(list(item), newitem)
+
+        # Append this sorted item to the sorted root
+        newroot.append(newitem)
+
+
+# has to sort all Cell and Point Data after the attribute "Name"!
+def sort_vtk(root):
+    if(root.tag != "VTKFile"):
+        print('Format is not a VTKFile. Sorting will most likely fail!')
+    # create a new root for the sorted tree
+    newroot = ET.Element(root.tag)
+    # create the sorted copy
+    # (after the idea of Dale Lane's xmldiff.py)
+    sort_attributes(root, newroot)
+    sort_elements(list(root), newroot)
+    # return the sorted element tree
+    return newroot
+
+
+# sorts the data by point coordinates so that it is independent of index numbering
+def sort_vtk_by_coordinates(root1, root2, verbose=True):
+    if not is_fuzzy_equal_node(root1.find(".//Points/DataArray"), root2.find(".//Points/DataArray"), 1e-2, 1e-9, False):
+        if verbose:
+            print("Sorting vtu by coordinates...")
+        for root in [root1, root2]:
+            # parse all DataArrays in a dictionary
+            pointDataArrays = []
+            cellDataArrays = []
+            dataArrays = {}
+            numberOfComponents = {}
+            for dataArray in root.findall(".//PointData/DataArray"):
+                pointDataArrays.append(dataArray.attrib["Name"])
+            for dataArray in root.findall(".//CellData/DataArray"):
+                cellDataArrays.append(dataArray.attrib["Name"])
+            for dataArray in root.findall(".//DataArray"):
+                dataArrays[dataArray.attrib["Name"]] = dataArray.text
+                numberOfComponents[dataArray.attrib["Name"]] = dataArray.attrib["NumberOfComponents"]
+
+            vertexArray = []
+            coords = dataArrays["Coordinates"].split()
+            # group the coordinates into coordinate tuples
+            dim = int(numberOfComponents["Coordinates"])
+            for i in range(len(coords)//dim):
+                vertexArray.append([float(c) for c in coords[i*dim:i*dim+dim]])
+
+            # obtain a vertex index map
+            vMap = []
+            for idx, coords in enumerate(vertexArray):
+                vMap.append((idx, coords))
+            sortedVMap = sorted(vMap, key=itemgetter(1))
+            vertexIndexMap = {}
+            for idxNew, idxOld in enumerate(sortedVMap):
+                vertexIndexMap[idxOld[0]] = idxNew
+
+            # group the cells into vertex index tuples
+            cellArray = []
+            offsets = dataArrays["offsets"].split()
+            connectivity = dataArrays["connectivity"].split()
+            vertex = 0
+            for cellIdx, offset in enumerate(offsets):
+                cellArray.append([])
+                for v in range(vertex, int(offset)):
+                    cellArray[cellIdx].append(int(connectivity[v]))
+                    vertex += 1
+
+            # replace all vertex indices in the cellArray by the new indices
+            for cell in cellArray:
+                for idx, vertexIndex in enumerate(cell):
+                    cell[idx] = vertexIndexMap[vertexIndex]
+
+            # sort all data arrays
+            for name, text in list(dataArrays.items()):
+                # split the text
+                items = text.split()
+                # convert if vector
+                num = int(numberOfComponents[name])
+                newitems = []
+                for i in range(len(items)//num):
+                    newitems.append([i for i in items[i*num:i*num+num]])
+                items = newitems
+                # sort the items: we have either vertex or cell data
+                if name in pointDataArrays:
+                    sortedItems = [j for (i, j) in sorted(zip(vertexArray, items), key=itemgetter(0))]
+                elif name in cellDataArrays or name == "types":
+                    sortedItems = [j for (i, j) in sorted(zip(cellArray, items), key=itemgetter(0))]
+                elif name == "offsets":
+                    sortedItems = []
+                    counter = 0
+                    for cell in sorted(cellArray):
+                        counter += len(cell)
+                        sortedItems.append([str(counter)])
+                elif name == "Coordinates":
+                    sortedItems = sorted(vertexArray)
+                elif name == "connectivity":
+                    sortedItems = sorted(cellArray)
+
+                # convert the sorted arrays to a xml text
+                dataArrays[name] = ""
+                for i in sortedItems:
+                    for j in i:
+                        dataArrays[name] += str(j) + " "
+
+            # do the replacement in the actual elements
+            for dataArray in root.findall(".//DataArray"):
+                dataArray.text = dataArrays[dataArray.attrib["Name"]]
+
+    return (root1, root2)
+
 
+# main program if called as script return appropriate error codes
+if __name__ == "__main__":
+    # handle arguments and print help message
+    parser = argparse.ArgumentParser(description='Fuzzy compare of two VTK\
+        (Visualization Toolkit) files. The files are accepted if for every\
+        value the difference is below the absolute error or below the\
+        relative error or below both.')
+    parser.add_argument('vtk_file_1', type=str, help='first file to compare')
+    parser.add_argument('vtk_file_2', type=str, help='second file to compare')
+    parser.add_argument('-r', '--relative', type=float, default=1e-2, help='maximum relative error (default=1e-2)')
+    parser.add_argument('-a', '--absolute', type=float, default=1e-9, help='maximum absolute error (default=1e-9)')
+    parser.add_argument('-v', '--verbose', type=bool, default=True, help='verbosity of the script')
+    args = vars(parser.parse_args())
 
+    sys.exit(compare_vtk(args["vtk_file_1"], args["vtk_file_2"], args["absolute"], args["relative"], args["verbose"]))

bin/runTest.sh

-#! /bin/bash
-#
-# Runs a test from the test directory and compare the resulting VTU files.
-#
-# Usage:
-#
-# runTest.sh COMPAREFLAG/SCRIPT REFERENCE_RESULT_FILE TEST_RESULT_FILE TEST_BINARY TEST_ARGS
-#
-
-function usage() {
-    echo "Usage:"
-    echo
-    echo "runTest.sh COMPAREFLAG/SCRIPT REFERENCE_RESULT_FILE TEST_RESULT_FILE TEST_BINARY [TEST_ARGS]"
-    echo "COMPAREFLAG: fuzzyvtu         - uses the fuzzycomparevtu.py script"
-    echo "             exact            - uses diff for exact compare of two files"
-    echo "             PATH_TO_SCRIPT   - tries to execute the custom script"
-};
-
-CMAKE_SOURCE_DIR=$(dirname "$0")
-COMPARE_FLAG="$1"
-REFERENCE_RESULT="$2"
-TEST_RESULT="$3"
-TEST_BINARY="$4"
-TEST_ARGS="${@:5:100}"
-rm -fv $TEST_RESULT
-
-# make sure we have at least 3 parameters
-if test "$#" -lt 4; then
-    echo "Wrong number of parameters"
-    echo
-    usage
-    exit 1
-fi
-
-# make sure the reference result exists
-if ! test -r "$REFERENCE_RESULT"; then
-    echo "File $REFERENCE_RESULT does not exist or is not readable"
-    echo
-    usage
-    exit 1
-fi
-
-#run the test
-echo "######################"
-echo "# Running test"
-echo "######################"
-$TEST_BINARY $TEST_ARGS
-RETURN_CODE_TEST_BINARY=$?
-if test "$RETURN_CODE_TEST_BINARY" != "0"; then
-    echo "Return code: $RETURN_CODE_TEST_BINARY"
-    exit $RETURN_CODE_TEST_BINARY
-fi
-
-# compare the results
-echo "######################"
-echo "# Comparing results"
-echo "######################"
-if ! test -r "$TEST_RESULT"; then
-    echo "File $TEST_RESULT does not exist or is not readable"
-    exit 1
-fi
-
-# running the compare script
-NOT_EQUAL=false
-if [ "$COMPARE_FLAG" = "fuzzyvtu" -o \
-     "$COMPARE_FLAG" = "$CMAKE_SOURCE_DIR/fuzzycomparevtu.py" ]; then
-    if ! python $CMAKE_SOURCE_DIR/fuzzycomparevtu.py "$REFERENCE_RESULT" "$TEST_RESULT"; then
-        NOT_EQUAL=true
-    fi
-elif [ "$COMPARE_FLAG" = "exact" ]; then
-    if ! diff "$REFERENCE_RESULT" "$TEST_RESULT"; then
-        NOT_EQUAL=true
-    fi
-elif [ -e "$COMPARE_FLAG" ]; then
-    if ! $COMPARE_FLAG "$REFERENCE_RESULT" "$TEST_RESULT"; then
-        NOT_EQUAL=true
-    fi
-else
-    echo
-    echo "ERROR: $0 was not able to run the compare script:"
-    echo "       $COMPARE_FLAG"
-    echo
-    exit 2
-fi
-
-# printing error message in case of failure
-if [ "$NOT_EQUAL" = "true" ]; then
-    echo "The files \"$TEST_RESULT\" and \"$REFERENCE_RESULT\" are different."
-    echo "Make sure the contents of \"$TEST_RESULT\" are still valid and "
-    echo "make it the reference result if necessary."
-    exit 1
-fi
-
-# SUCCESS!!!!!!
-echo "Result and reference result are identical"
-exit 0

bin/runtest.py

+import argparse
+import os, sys
+import subprocess
+from fuzzycomparevtu import compare_vtk
+
+# parse arguments
+parser = argparse.ArgumentParser()
+parser.add_argument('-c', '--command', nargs=1, help='The executable and optional arguments as a single string', required=True)
+parser.add_argument('-s', '--script', nargs=1, help="The comparison script. [fuzzy, exact, <path_to_script>] where the script takes two vtu files as arguments.")
+parser.add_argument('-f', '--files', nargs='+', help="Pairs of reference and vtu file names. Usage: '[-f ref1 vtu1 [[ref2] [vtu2] ...]]'")
+parser.add_argument('-r', '--relative', type=float, default=1e-2, help='maximum relative error (default=1e-2) when using fuzzy comparison')
+parser.add_argument('-a', '--absolute', type=float, default=1e-9, help='maximum absolute error (default=1e-9) when using fuzzy comparison')
+args = vars(parser.parse_args())
+
+# check parameters
+if args['script']:
+    if len(args['files'])%2 != 0 or not args['files']:
+        sys.stderr.write("The files have to be pairs of vtu and reference files. Usage '-f [ref1] [vtu1] [[ref2] [vtu2] ...]'")
+        parser.print_help()
+        sys.exit(1)
+    for i in range(0, len(args['files'])//2):
+        # delete the vtu files to compare
+        ref_dir = os.path.dirname(os.path.abspath(__file__)).rstrip("bin") + "test/references"
+        if os.path.dirname(args['files'][(i*2)+1]) == ref_dir:
+            sys.stderr.write("Tried to delete a reference solution. Specify reference file first, then the VTU file. Usage: '[-f ref1 vtu1 [[ref2] [vtu2] ...]]'")
+            sys.exit(1)
+        subprocess.call(['rm', '-fv', args['files'][(i*2)+1]])
+
+# run the test
+res = 1
+try:
+    res = subprocess.call(args['command'][0].split())
+except OSError as e:
+    print(args['command'][0].split())
+    print("OSError: Command not found. Most likely the executable specified doesn't exist.")
+    sys.exit(1)
+if res:
+    sys.exit(1)
+
+# run the comparison
+if args['script']:
+    # exact comparison?
+    if args['script'] == ['exact']:
+        return_code = 0
+        for i in range(0, len(args['files'])//2):
+            print("\nExact comparison...")
+            result = subprocess.call(['diff', args['files'][i*2], args['files'][(i*2)+1]])
+            if result:
+                return_code = 1
+        sys.exit(return_code)
+
+    # fuzzy comparison?
+    elif args['script'] == ["fuzzy"] or args['script'] == [os.path.dirname(os.path.abspath(__file__)) + "/fuzzycomparevtu.py"]:
+        return_code = 0
+        for i in range(0, len(args['files'])//2):
+            print("\nFuzzy comparison...")
+            if args['relative'] and args['absolute']:
+                result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], relative=args['relative'], absolute=args['absolute'])
+            elif args['relative'] and not args['absolute']:
+                result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], relative=args['relative'])
+            elif args['absolute'] and not args['relative']:
+                result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1], absolute=args['absolute'])
+            else:
+                result = compare_vtk(args['files'][i*2], args['files'][(i*2)+1])
+            if result:
+                return_code = 1
+        sys.exit(return_code)
+
+    # other script?
+    else:
+        return_code = 0
+        for i in range(0, len(args['files'])//2):
+            print("\n{} comparison...".format(args['script']))
+            result = subprocess.call(args['script'], args['files'][i*2], args['files'][(i*2)+1])
+            if result:
+                return_code = 1
+        sys.exit(return_code)
+
+# everything is fine
+sys.exit(0)

test/common/generalproblem/CMakeLists.txt

 add_dumux_test(test_general_box test_general_box test_generalproblem2p.cc
-  ${CMAKE_SOURCE_DIR}/bin/runTest.sh
-  ${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
-  ${CMAKE_SOURCE_DIR}/test/references/generallens_box-reference.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/generallens_box-00003.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/test_general_box
-  -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
-  -ModelType Box)
+               python ${CMAKE_SOURCE_DIR}/bin/runtest.py
+                 --script fuzzy
+                 --files ${CMAKE_SOURCE_DIR}/test/references/generallens_box-reference.vtu
+                         ${CMAKE_CURRENT_BINARY_DIR}/generallens_box-00003.vtu
+                 --command "${CMAKE_CURRENT_BINARY_DIR}/test_general_box
+                            -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
+                            -ModelType Box")
 
 add_dumux_test(test_general_dec test_general_dec test_generalproblem2p.cc
-  ${CMAKE_SOURCE_DIR}/bin/runTest.sh
-  ${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
-  ${CMAKE_SOURCE_DIR}/test/references/generallens_decoupled-reference.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/generallens_decoupled-00003.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/test_general_dec
-  -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
-  -ModelType Decoupled)
+               python ${CMAKE_SOURCE_DIR}/bin/runtest.py
+                 --script fuzzy
+                 --files ${CMAKE_SOURCE_DIR}/test/references/generallens_decoupled-reference.vtu
+                         ${CMAKE_CURRENT_BINARY_DIR}/generallens_decoupled-00003.vtu
+                 --command "${CMAKE_CURRENT_BINARY_DIR}/test_general_dec
+                            -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_generalproblem2p_reference.input
+                            -ModelType Decoupled")

test/decoupled/1p/CMakeLists.txt

 add_dumux_test(test_dec1p test_dec1p test_1p.cc
-  ${CMAKE_SOURCE_DIR}/bin/runTest.sh
-  ${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
-  ${CMAKE_SOURCE_DIR}/test/references/test_1p-reference.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/test_1p-00001.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/test_dec1p
-  -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_1p.input)
+               python ${CMAKE_SOURCE_DIR}/bin/runtest.py
+                 --script fuzzy
+                 --files ${CMAKE_SOURCE_DIR}/test/references/test_1p-reference.vtu
+                         ${CMAKE_CURRENT_BINARY_DIR}/test_1p-00001.vtu
+                 --command "${CMAKE_CURRENT_BINARY_DIR}/test_dec1p -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/test_1p.input")
 
 add_dumux_test(test_diffusion test_diffusion test_diffusion.cc
-  ${CMAKE_SOURCE_DIR}/bin/runTest.sh
-  ${CMAKE_SOURCE_DIR}/bin/fuzzycomparevtu.py
-  ${CMAKE_SOURCE_DIR}/test/references/diffusion-reference.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/mimeticdiffusion-00001.vtu
-  ${CMAKE_CURRENT_BINARY_DIR}/test_diffusion
-  3)
+               python ${CMAKE_SOURCE_DIR}/bin/runtest.py
+                 --script fuzzy
+                 --files ${CMAKE_SOURCE_DIR}/test/references/diffusion-reference.vtu
+                         ${CMAKE_CURRENT_BINARY_DIR}/mimeticdiffusion-00001.vtu
+                 --command "${CMAKE_CURRENT_BINARY_DIR}/test_diffusion 3")