Merge branch 'feature/python-with-dune-2.9' into 'master'

[python] Make bindings work with dune 2.9

See merge request !2934

.gitlab-ci/default.yml

@@ -24,8 +24,13 @@ variables:
 configure:
   stage: configure
   script:
-    - dunecontrol --opts=$DUNE_OPTS_FILE --current all
-    - source bin/testing/ci-setup-python-env.sh
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current configure
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current make -j8
+    # cache state of the Dune virtual env for Python if it exists (Dune 2.9)
+    - |
+      if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then
+        cp -r /dune/modules/dune-common/build-cmake/dune-env build-cmake
+      fi
   artifacts:
     paths:
       - build-cmake
@@ -46,9 +51,13 @@ black (python):
 pylint-flake8 (python):
   stage: linting
   script:
-    - source bin/testing/ci-setup-python-env.sh
     - |
       if [ -d build-cmake/python/dumux ] ; then
+        source bin/testing/ci-setup-python-env.sh
+        # if we are in venv (Dune 2.9) install linters
+        if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then
+          python -m pip install pylint flake8
+        fi
         pylint --rcfile=.pylintrc build-cmake/python/dumux
         pylint --rcfile=.pylintrc bin
         flake8 build-cmake/python/dumux
@@ -99,6 +108,11 @@ select tests:
 compile cpp:
   stage: build
   script:
+    # remove cached Python dune-env if existing (not needed for C++) (Dune 2.9)
+    - |
+      if [ -d "build-cmake/dune-env" ]; then
+        rm -r build-cmake/dune-env
+      fi
     - |
       pushd build-cmake
         make clean && make all
@@ -148,7 +162,12 @@ test python:
     OMPI_ALLOW_RUN_AS_ROOT: 1
     OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
   script:
-    - source bin/testing/ci-setup-python-env.sh
+    # restore Python virtual env from cache (job:configure artifacts) (Dune 2.9)
+    - |
+      if [ -d "build-cmake/dune-env" ]; then
+        rm -r /dune/modules/dune-common/build-cmake/dune-env
+        mv build-cmake/dune-env /dune/modules/dune-common/build-cmake/dune-env
+      fi
     - |
       if ([ ! -s changedfiles.txt ] || grep -q python "changedfiles.txt"); then
         if [ ! -s changedfiles.txt ]; then
@@ -158,7 +177,7 @@ test python:
         fi
         source bin/testing/ci-setup-python-env.sh
         pushd build-cmake
-          ctest --output-on-failure -L python
+          DUNE_LOG_LEVEL=DEBUG ctest --output-on-failure -L python
         popd
       else
         echo "No changes in the Python bindings/Python code detected: skipping tests."

CMakeLists.txt

@@ -33,9 +33,12 @@ add_subdirectory(dumux)
 add_subdirectory(test EXCLUDE_FROM_ALL)
 add_subdirectory(examples EXCLUDE_FROM_ALL)
 
-# if Python bindings are enabled, include necessary sub directories.
-if(DUNE_ENABLE_PYTHONBINDINGS)
-  if(${dune-common_VERSION} VERSION_GREATER_EQUAL 2.8)
+# from Dune 2.9 on Python bindings are enabled per default
+if(${dune-common_VERSION} VERSION_GREATER_EQUAL 2.9)
+  add_subdirectory(python)
+else()
+  # with Dune 2.8, only if Python bindings are enabled
+  if(DUNE_ENABLE_PYTHONBINDINGS)
     add_subdirectory(python)
     dune_python_install_package(PATH "python")
   endif()

bin/testing/ci-setup-python-env.sh

 #!/bin/bash
 
-if [ -L /dune/bin/setup-python ] && [ -e /dune/bin/setup-python ] ; then
-    dunecontrol bexec "echo -n :\$(pwd)/python >> $(pwd)/pythonpath.txt"
-    export PYTHONPATH=$PYTHONPATH$(cat pythonpath.txt)
-    rm pythonpath.txt
-    setup-python --opts=$DUNE_OPTS_FILE install
+if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then
+    # Use internal venv of DUNE
+    echo "Activating the Python virtual environment of dune-common"
+    source /dune/modules/dune-common/build-cmake/dune-env/bin/activate
+else
+    if [ -L /dune/bin/setup-python ] && [ -e /dune/bin/setup-python ] ; then
+        dunecontrol bexec "echo -n :\$(pwd)/python >> $(pwd)/pythonpath.txt"
+        export PYTHONPATH=$PYTHONPATH$(cat pythonpath.txt)
+        rm pythonpath.txt
+        setup-python --opts=$DUNE_OPTS_FILE install
+    fi
 fi

python/CMakeLists.txt

 add_subdirectory(dumux)
 configure_file(setup.py.in setup.py)
+
+# link properties.hh needed by the Python bindings
+# to determine the list of properties
+# create copy for Windows and symlink otherwise
+if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
+  execute_process(COMMAND ${CMAKE_COMMAND} "-E" "copy" "${CMAKE_SOURCE_DIR}/dumux/common/properties.hh" "${CMAKE_CURRENT_BINARY_DIR}/properties.hh")
+else()
+  execute_process(COMMAND ${CMAKE_COMMAND} "-E" "create_symlink" "${CMAKE_SOURCE_DIR}/dumux/common/properties.hh" "${CMAKE_CURRENT_BINARY_DIR}/properties.hh")
+endif()
+
+if(${dune-common_VERSION} VERSION_GREATER_EQUAL 2.9)
+  dune_python_install_package(
+    PATH "."
+    CMAKE_METADATA_FILE dumux/metadata.cmake
+    DEPENDS _common
+    CMAKE_METADATA_FLAGS DUNE_OPTS_FILE
+  )
+endif()

python/dumux/__init__.py

@@ -10,4 +10,11 @@ DuMux is
 https://dumux.org/
 """
 
-__import__("pkg_resources").declare_namespace(__name__)
+try:
+    from dune.common import registerExternalModule
+
+    # register dumux to be recognized by dune-py (code generation module)
+    # as a module of the dune univers
+    registerExternalModule("dumux")
+except ImportError:
+    pass

python/dumux/common/CMakeLists.txt

@@ -4,4 +4,7 @@ add_python_targets(common
 )
 dune_add_pybind11_module(NAME _common)
 set_property(TARGET _common PROPERTY LINK_LIBRARIES dunecommon dunegrid APPEND)
-install(TARGETS _common LIBRARY DESTINATION python/dumux/common)
+
+if(SKBUILD)
+  install(TARGETS _common LIBRARY DESTINATION python/dumux/common)
+endif()

python/dumux/common/properties.py

@@ -7,6 +7,7 @@ import os
 from dataclasses import dataclass
 from typing import List, Union
 from dune.common.hashit import hashIt
+import dumux
 
 
 @dataclass
@@ -146,12 +147,46 @@ def listTypeTags():
     print("\n**********************************")
 
 
-def predefinedProperties():
-    """Create a list of properties defined in properties.hh"""
+def propertiesHeaderPath():
+    """Find the path to the properties.hh C++ header"""
+
+    path, _ = os.path.split(dumux.__file__)
+    metaDataFile = os.path.join(path, "data/metadata.cmake")
+    if os.path.exists(metaDataFile):
+        data = {}
+        with open(metaDataFile, "r") as metaData:
+            for line in metaData:
+                try:
+                    key, value = line.split("=", 1)
+                    data[key] = value.strip()
+                except ValueError:  # no '=' in line
+                    pass
+        return os.path.abspath(
+            os.path.join(
+                data["DEPBUILDDIRS"].split(";")[0],
+                "python",
+                "properties.hh",
+            )
+        )
 
+    # as fall-back try relative path
     propertiesHeader = os.path.abspath(
-        os.path.dirname(__file__) + "/../../../../dumux/common/properties.hh"
+        os.path.join(
+            os.path.dirname(__file__),
+            "../../../../dumux/common/properties.hh",
+        )
     )
+
+    if os.path.exists(propertiesHeader):
+        return propertiesHeader
+
+    raise RuntimeError("Could not find properties.hh header")
+
+
+def predefinedProperties():
+    """Create a list of properties defined in properties.hh"""
+
+    propertiesHeader = propertiesHeaderPath()
     with open(propertiesHeader, encoding="utf-8") as header:
         properties = []
         for line in header:

python/setup.py.in

-from setuptools import setup, find_namespace_packages
+from setuptools import setup
+
+REQUIRED_PACKAGES = '${RequiredPythonModules}'.replace(';',' ').split(' ')
 
 setup(
     name="${ProjectName}",
@@ -6,8 +8,9 @@ setup(
     version="${ProjectVersionString}",
     author="${ProjectAuthor}",
     author_email="${ProjectMaintainerEmail}",
-    packages=find_namespace_packages(include=["dumux.*"]),
+    packages=["dumux"],
     zip_safe=0,
     package_data={"": ["*.so"]},
-    install_requires="${ProjectPythonRequires}".split(" "),
+    install_requires=REQUIRED_PACKAGES,
+    include_package_data=True,
 )