From 1c609de6545290577a8a70cf7a8cedec76d33646 Mon Sep 17 00:00:00 2001
From: Timo Koch <timo.koch@iws.uni-stuttgart.de>
Date: Mon, 2 Jan 2023 17:51:42 +0100
Subject: [PATCH] [test] Use the fieldcompare library if available for fuzzy
comparisons
---
bin/testing/runtest.py | 235 ++++++++++++++++++++++++++++++++---------
1 file changed, 187 insertions(+), 48 deletions(-)
diff --git a/bin/testing/runtest.py b/bin/testing/runtest.py
index dc18f84b3f..7d32a0e53f 100755
--- a/bin/testing/runtest.py
+++ b/bin/testing/runtest.py
@@ -11,8 +11,127 @@ import os
import sys
import subprocess
import json
-from fuzzycomparevtu import compareVTK
-from fuzzycomparedata import compareData
+
+
+try:
+ import numpy as np
+ import fieldcompare.mesh as meshcompare
+ import fieldcompare.tabular as tabularcompare
+ from fieldcompare import FieldDataComparator, protocols, DefaultFieldComparisonCallback
+ from fieldcompare.mesh import MeshFieldsComparator
+ from fieldcompare.predicates import FuzzyEquality
+ from fieldcompare.io import CSVFieldReader, read
+
+ protocols.MeshFields = meshcompare.MeshFields
+ protocols.TabularFields = tabularcompare.TabularFields
+
+ def makePredicateSelector(
+ relThreshold,
+ absThreshold,
+ zeroValueThreshold,
+ sourceFieldNameTransform=lambda name: name,
+ ):
+ """Create a predicate selector for fieldcompare emulates the Dumux behaviour"""
+
+ def _selector(
+ sourceField: protocols.Field, referenceField: protocols.Field
+ ) -> protocols.Predicate:
+ sourceFieldName = sourceFieldNameTransform(sourceField.name)
+ magnitude = np.max(np.abs(referenceField.values))
+ _absThreshold = max(
+ float(zeroValueThreshold.get(sourceFieldName, 0.0)), magnitude * absThreshold
+ )
+ return FuzzyEquality(abs_tol=_absThreshold, rel_tol=relThreshold)
+
+ return _selector
+
+ def fieldcompareMeshData(
+ source, ref, absThreshold=0.0, relThreshold=1e-7, zeroValueThreshold=None
+ ):
+ """Compares mesh data with the fieldcompare library"""
+
+ print(f"-- Comparing {source} and {ref}")
+
+ zeroValueThreshold = zeroValueThreshold or {}
+ if zeroValueThreshold:
+ print(f"-- Using the following absolute thresholds: {zeroValueThreshold}")
+
+ # read the files
+ sourceFields = read(source)
+ referenceFields = read(ref)
+
+ # some type checking to be sure we are comparing meshes
+ if not isinstance(sourceFields, protocols.MeshFields):
+ raise IOError("Source file could not been identified as mesh file!")
+ if not isinstance(referenceFields, protocols.MeshFields):
+ raise IOError("Reference file could not been identified as mesh file!")
+
+ # hard-code some values for the mesh comparisons (as for Dumux legacy backend)
+ sourceFields.domain.set_tolerances(abs_tol=1e-2, rel_tol=1.5e-7)
+ referenceFields.domain.set_tolerances(abs_tol=1e-2, rel_tol=1.5e-7)
+
+ compare = MeshFieldsComparator(source=sourceFields, reference=referenceFields)
+ result = compare(
+ predicate_selector=makePredicateSelector(
+ relThreshold, absThreshold, zeroValueThreshold
+ ),
+ fieldcomp_callback=DefaultFieldComparisonCallback(verbosity=1),
+ reordering_callback=lambda msg: print(f"-- {msg}"),
+ )
+
+ print(f"-- Summary: {result.status} ({result.report})\n")
+
+ if not result:
+ return 1
+ return 0
+
+ # pylint: disable=too-many-arguments
+ def fieldcompareCSVData(
+ source, ref, delimiter, absThreshold=0.0, relThreshold=1e-7, zeroValueThreshold=None
+ ):
+ """Compares CSV data with the fieldcompare library"""
+
+ print(f"-- Comparing {source} and {ref}")
+
+ zeroValueThreshold = zeroValueThreshold or {}
+ if zeroValueThreshold:
+ print(f"-- Using the following absolute thresholds: {zeroValueThreshold}")
+
+ sourceFields = CSVFieldReader(delimiter=delimiter, use_names=False).read(source)
+ referenceFields = CSVFieldReader(delimiter=delimiter, use_names=False).read(ref)
+
+ # some type checking to be sure we are comparing CSV data
+ if not isinstance(sourceFields, protocols.TabularFields):
+ raise IOError("Source file could not been identified as CSV-like file!")
+ if not isinstance(referenceFields, protocols.TabularFields):
+ raise IOError("Reference file could not been identified as CSV-like file!")
+
+ compare = FieldDataComparator(source=sourceFields, reference=referenceFields)
+ result = compare(
+ predicate_selector=makePredicateSelector(
+ relThreshold,
+ absThreshold,
+ zeroValueThreshold,
+ lambda name: f"row {float(name.strip('field_'))}",
+ ),
+ fieldcomp_callback=DefaultFieldComparisonCallback(verbosity=1),
+ )
+
+ print(f"-- Summary: {result.status} ({result.report})\n")
+
+ if not result:
+ return 1
+ return 0
+
+ BACKEND = "fieldcompare"
+
+
+# fall back to Dumux legacy backend if we don't have fieldcompare
+except ImportError:
+ from fuzzycomparevtu import compareVTK as fieldcompareMeshData
+ from fuzzycomparedata import compareData as fieldcompareCSVData
+
+ BACKEND = "legacy"
def readCmdParameters():
@@ -98,69 +217,89 @@ def readCmdParameters():
return args
+def _exactComparison(args):
+ """Exact comparison driver"""
+ returnCode = 0
+ for i in range(0, len(args["files"]) // 2):
+ print("\nExact comparison...")
+ result = subprocess.call(["diff", args["files"][i * 2], args["files"][(i * 2) + 1]])
+ if result:
+ returnCode = 1
+ return returnCode
+
+
+def _fuzzyMeshComparison(args):
+ """Fuzzy mesh comparison driver"""
+ numFailed = 0
+ for i in range(0, len(args["files"]) // 2):
+ print(f"\nFuzzy data comparison with {BACKEND} backend")
+ source, ref = args["files"][i * 2], args["files"][(i * 2) + 1]
+ if "reference" in source and "reference" not in ref:
+ source, ref = ref, source
+ relThreshold = args["relative"]
+ absThreshold = args["absolute"]
+ zeroValueThreshold = args["zeroThreshold"]
+ numFailed += fieldcompareMeshData(
+ source, ref, absThreshold, relThreshold, zeroValueThreshold
+ )
+
+ return int(numFailed > 0)
+
+
+def _fuzzyDataComparison(args):
+ """Fuzzy data comparison driver"""
+ numFailed = 0
+ for i in range(0, len(args["files"]) // 2):
+ print(f"\nFuzzy data comparison with {BACKEND} backend")
+ source, ref = args["files"][i * 2], args["files"][(i * 2) + 1]
+ if "reference" in source and "reference" not in ref:
+ source, ref = ref, source
+ delimiter = args["delimiter"]
+ relThreshold = args["relative"]
+ absThreshold = args["absolute"]
+ zeroValueThreshold = args["zeroThreshold"]
+ numFailed += fieldcompareCSVData(
+ source, ref, delimiter, absThreshold, relThreshold, zeroValueThreshold
+ )
+
+ return int(numFailed > 0)
+
+
+def _scriptComparison(args):
+ """Script comparison driver"""
+ returnCode = 0
+ for i in range(0, len(args["files"]) // 2):
+ print(f"\n{args['script']} comparison")
+ result = subprocess.call(args["script"], args["files"][i * 2], args["files"][(i * 2) + 1])
+ if result:
+ returnCode = 1
+ return returnCode
+
+
def runRegressionTest(args):
"""Run regression test scripts against reference data"""
# exact comparison?
if args["script"] == ["exact"]:
- returnCode = 0
- for i in range(0, len(args["files"]) // 2):
- print("\nExact comparison...")
- result = subprocess.call(["diff", args["files"][i * 2], args["files"][(i * 2) + 1]])
- if result:
- returnCode = 1
- sys.exit(returnCode)
+ sys.exit(_exactComparison(args))
- # fuzzy comparison?
+ # fuzzy mesh comparison?
elif args["script"] == ["fuzzy"] or args["script"] == [
os.path.dirname(os.path.abspath(__file__)) + "/fuzzycomparevtu.py"
]:
- returnCode = 0
- for i in range(0, len(args["files"]) // 2):
- print("\nFuzzy comparison...")
- result = compareVTK(
- args["files"][i * 2],
- args["files"][(i * 2) + 1],
- relative=args["relative"],
- absolute=args["absolute"],
- zeroValueThreshold=args["zeroThreshold"],
- )
- if result:
- returnCode = 1
- sys.exit(returnCode)
+ sys.exit(_fuzzyMeshComparison(args))
- # fuzzy comparison of data sets?
+ # fuzzy comparison of CSV-like data sets?
elif args["script"] == ["fuzzyData"]:
- returnCode = 0
- for i in range(0, len(args["files"]) // 2):
- print("\nFuzzy data comparison...")
- result = compareData(
- args["files"][i * 2],
- args["files"][(i * 2) + 1],
- args["delimiter"],
- relative=args["relative"],
- absolute=args["absolute"],
- zeroValueThreshold=args["zeroThreshold"],
- )
- if result:
- returnCode = 1
- sys.exit(returnCode)
+ sys.exit(_fuzzyDataComparison(args))
# other script?
else:
- returnCode = 0
- for i in range(0, len(args["files"]) // 2):
- print(f"\n{args['script']} comparison...")
- result = subprocess.call(
- args["script"], args["files"][i * 2], args["files"][(i * 2) + 1]
- )
- if result:
- returnCode = 1
- sys.exit(returnCode)
+ sys.exit(_scriptComparison(args))
def runTest():
- """Run a DuMux test"""
+ """DuMux test driver"""
args = readCmdParameters()
--
GitLab