From d8a6a26d35d7fbcb28de25f8e491e149a5cd019f Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Sun, 31 Oct 2021 01:15:56 +0200 Subject: [PATCH 1/8] [python] Use metadata file to find properties.hh if available --- python/CMakeLists.txt | 9 +++++++ python/dumux/common/properties.py | 41 ++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 0c4ddccf90..bbb5ac7774 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,2 +1,11 @@ add_subdirectory(dumux) configure_file(setup.py.in setup.py) + +# link properties.hh needed by the Python bindings +# to determine the list of properties +# create copy for Windows and symlink otherwise +if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + execute_process(COMMAND ${CMAKE_COMMAND} "-E" "copy" "${CMAKE_SOURCE_DIR}/dumux/common/properties.hh" "${CMAKE_CURRENT_BINARY_DIR}/properties.hh") +else() + execute_process(COMMAND ${CMAKE_COMMAND} "-E" "create_symlink" "${CMAKE_SOURCE_DIR}/dumux/common/properties.hh" "${CMAKE_CURRENT_BINARY_DIR}/properties.hh") +endif() diff --git a/python/dumux/common/properties.py b/python/dumux/common/properties.py index e72dc23f05..0e791e1edb 100644 --- a/python/dumux/common/properties.py +++ b/python/dumux/common/properties.py @@ -7,6 +7,7 @@ import os from dataclasses import dataclass from typing import List, Union from dune.common.hashit import hashIt +import dumux @dataclass @@ -146,12 +147,46 @@ def listTypeTags(): print("\n**********************************") -def predefinedProperties(): - """Create a list of properties defined in properties.hh""" +def propertiesHeaderPath(): + """Find the path to the properties.hh C++ header""" + + path, _ = os.path.split(dumux.__file__) + metaDataFile = os.path.join(path, "data/metadata.cmake") + if os.path.exists(metaDataFile): + data = {} + with open(metaDataFile, "r") as metaData: + for line in metaData: + try: + key, value = line.split("=", 1) + data[key] = value.strip() + except ValueError: # no '=' in line + pass + return os.path.abspath( + os.path.join( + data["DEPBUILDDIRS"].split(";")[0], + "python", + "properties.hh", + ) + ) + # as fall-back try relative path propertiesHeader = os.path.abspath( - os.path.dirname(__file__) + "/../../../../dumux/common/properties.hh" + os.path.join( + os.path.dirname(__file__), + "../../../../dumux/common/properties.hh", + ) ) + + if os.path.exists(propertiesHeader): + return propertiesHeader + + raise RuntimeError("Could not find properties.hh header") + + +def predefinedProperties(): + """Create a list of properties defined in properties.hh""" + + propertiesHeader = propertiesHeaderPath() with open(propertiesHeader, encoding="utf-8") as header: properties = [] for line in header: -- GitLab From e51ac2e482018dcffe6479e24762ee52c61a39f4 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Fri, 19 Nov 2021 17:00:59 +0100 Subject: [PATCH 2/8] [python] Only install lib when packacking --- python/dumux/common/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/dumux/common/CMakeLists.txt b/python/dumux/common/CMakeLists.txt index ffc35a5823..65f0a6b1f5 100644 --- a/python/dumux/common/CMakeLists.txt +++ b/python/dumux/common/CMakeLists.txt @@ -4,4 +4,7 @@ add_python_targets(common ) dune_add_pybind11_module(NAME _common) set_property(TARGET _common PROPERTY LINK_LIBRARIES dunecommon dunegrid APPEND) -install(TARGETS _common LIBRARY DESTINATION python/dumux/common) + +if(SKBUILD) + install(TARGETS _common LIBRARY DESTINATION python/dumux/common) +endif() -- GitLab From c73acb39fc96bbd1f56570b47143b6e4fd21bb8d Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Fri, 19 Nov 2021 17:05:24 +0100 Subject: [PATCH 3/8] [python] Include Python directory per default with dune 2.9 --- CMakeLists.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f0f1f229d8..257021890f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,9 +33,12 @@ add_subdirectory(dumux) add_subdirectory(test EXCLUDE_FROM_ALL) add_subdirectory(examples EXCLUDE_FROM_ALL) -# if Python bindings are enabled, include necessary sub directories. -if(DUNE_ENABLE_PYTHONBINDINGS) - if(${dune-common_VERSION} VERSION_GREATER_EQUAL 2.8) +# from Dune 2.9 on Python bindings are enabled per default +if(${dune-common_VERSION} VERSION_GREATER_EQUAL 2.9) + add_subdirectory(python) +else() + # with Dune 2.8, only if Python bindings are enabled + if(DUNE_ENABLE_PYTHONBINDINGS) add_subdirectory(python) dune_python_install_package(PATH "python") endif() -- GitLab From 16e64f1f1c3404406de844d8a12a33926a5f6512 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Fri, 19 Nov 2021 17:10:22 +0100 Subject: [PATCH 4/8] [python] Add new dune python install package command for dune 2.9 --- python/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index bbb5ac7774..f34a5c5d5c 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -9,3 +9,12 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") else() execute_process(COMMAND ${CMAKE_COMMAND} "-E" "create_symlink" "${CMAKE_SOURCE_DIR}/dumux/common/properties.hh" "${CMAKE_CURRENT_BINARY_DIR}/properties.hh") endif() + +if(${dune-common_VERSION} VERSION_GREATER_EQUAL 2.9) + dune_python_install_package( + PATH "." + CMAKE_METADATA_FILE dumux/metadata.cmake + DEPENDS _common + CMAKE_METADATA_FLAGS DUNE_OPTS_FILE + ) +endif() -- GitLab From bc85175d9ddb543802b60d67b297fba61636169e Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Fri, 19 Nov 2021 17:16:47 +0100 Subject: [PATCH 5/8] [python] Dumux is not a namespace package --- python/dumux/__init__.py | 2 -- python/setup.py.in | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/dumux/__init__.py b/python/dumux/__init__.py index bb4c6ae256..3a80e76b6c 100644 --- a/python/dumux/__init__.py +++ b/python/dumux/__init__.py @@ -9,5 +9,3 @@ DuMux is https://dumux.org/ """ - -__import__("pkg_resources").declare_namespace(__name__) diff --git a/python/setup.py.in b/python/setup.py.in index 970d78de29..5287131d00 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,4 +1,4 @@ -from setuptools import setup, find_namespace_packages +from setuptools import setup setup( name="${ProjectName}", @@ -6,7 +6,7 @@ setup( version="${ProjectVersionString}", author="${ProjectAuthor}", author_email="${ProjectMaintainerEmail}", - packages=find_namespace_packages(include=["dumux.*"]), + packages=["dumux"], zip_safe=0, package_data={"": ["*.so"]}, install_requires="${ProjectPythonRequires}".split(" "), -- GitLab From bb01836517281c23015a3e9f890ba9af1491e5b4 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Fri, 19 Nov 2021 17:17:19 +0100 Subject: [PATCH 6/8] [python] Try to register dumux as external module (dune 2.9) --- python/dumux/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/dumux/__init__.py b/python/dumux/__init__.py index 3a80e76b6c..5bcb3ab2d1 100644 --- a/python/dumux/__init__.py +++ b/python/dumux/__init__.py @@ -9,3 +9,12 @@ DuMux is https://dumux.org/ """ + +try: + from dune.common import registerExternalModule + + # register dumux to be recognized by dune-py (code generation module) + # as a module of the dune univers + registerExternalModule("dumux") +except ImportError: + pass -- GitLab From d622d536352a36b44664a2e16b66da40e7d9d72f Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Fri, 19 Nov 2021 18:41:18 +0100 Subject: [PATCH 7/8] [python] Update setup.py.in for dune 2.9 --- python/setup.py.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/setup.py.in b/python/setup.py.in index 5287131d00..8504ecc10b 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,5 +1,7 @@ from setuptools import setup +REQUIRED_PACKAGES = '${RequiredPythonModules}'.replace(';',' ').split(' ') + setup( name="${ProjectName}", description="${ProjectDescription}", @@ -9,5 +11,6 @@ setup( packages=["dumux"], zip_safe=0, package_data={"": ["*.so"]}, - install_requires="${ProjectPythonRequires}".split(" "), + install_requires=REQUIRED_PACKAGES, + include_package_data=True, ) -- GitLab From 4b79c15b3cf5522a33966cdbe53f2fac886517b8 Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Fri, 19 Nov 2021 18:43:53 +0100 Subject: [PATCH 8/8] [python][ci] Fix Python environment setup with dune 2.9 --- .gitlab-ci/default.yml | 29 ++++++++++++++++++++++++----- bin/testing/ci-setup-python-env.sh | 16 +++++++++++----- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml index 9ea9baf253..17f7d77393 100644 --- a/.gitlab-ci/default.yml +++ b/.gitlab-ci/default.yml @@ -24,8 +24,13 @@ variables: configure: stage: configure script: - - dunecontrol --opts=$DUNE_OPTS_FILE --current all - - source bin/testing/ci-setup-python-env.sh + - dunecontrol --opts=$DUNE_OPTS_FILE --current configure + - dunecontrol --opts=$DUNE_OPTS_FILE --current make -j8 + # cache state of the Dune virtual env for Python if it exists (Dune 2.9) + - | + if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then + cp -r /dune/modules/dune-common/build-cmake/dune-env build-cmake + fi artifacts: paths: - build-cmake @@ -46,9 +51,13 @@ black (python): pylint-flake8 (python): stage: linting script: - - source bin/testing/ci-setup-python-env.sh - | if [ -d build-cmake/python/dumux ] ; then + source bin/testing/ci-setup-python-env.sh + # if we are in venv (Dune 2.9) install linters + if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then + python -m pip install pylint flake8 + fi pylint --rcfile=.pylintrc build-cmake/python/dumux pylint --rcfile=.pylintrc bin flake8 build-cmake/python/dumux @@ -99,6 +108,11 @@ select tests: compile cpp: stage: build script: + # remove cached Python dune-env if existing (not needed for C++) (Dune 2.9) + - | + if [ -d "build-cmake/dune-env" ]; then + rm -r build-cmake/dune-env + fi - | pushd build-cmake make clean && make all @@ -148,7 +162,12 @@ test python: OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 script: - - source bin/testing/ci-setup-python-env.sh + # restore Python virtual env from cache (job:configure artifacts) (Dune 2.9) + - | + if [ -d "build-cmake/dune-env" ]; then + rm -r /dune/modules/dune-common/build-cmake/dune-env + mv build-cmake/dune-env /dune/modules/dune-common/build-cmake/dune-env + fi - | if ([ ! -s changedfiles.txt ] || grep -q python "changedfiles.txt"); then if [ ! -s changedfiles.txt ]; then @@ -158,7 +177,7 @@ test python: fi source bin/testing/ci-setup-python-env.sh pushd build-cmake - ctest --output-on-failure -L python + DUNE_LOG_LEVEL=DEBUG ctest --output-on-failure -L python popd else echo "No changes in the Python bindings/Python code detected: skipping tests." diff --git a/bin/testing/ci-setup-python-env.sh b/bin/testing/ci-setup-python-env.sh index dd20b89d61..5f9b133bfc 100755 --- a/bin/testing/ci-setup-python-env.sh +++ b/bin/testing/ci-setup-python-env.sh @@ -1,8 +1,14 @@ #!/bin/bash -if [ -L /dune/bin/setup-python ] && [ -e /dune/bin/setup-python ] ; then - dunecontrol bexec "echo -n :\$(pwd)/python >> $(pwd)/pythonpath.txt" - export PYTHONPATH=$PYTHONPATH$(cat pythonpath.txt) - rm pythonpath.txt - setup-python --opts=$DUNE_OPTS_FILE install +if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then + # Use internal venv of DUNE + echo "Activating the Python virtual environment of dune-common" + source /dune/modules/dune-common/build-cmake/dune-env/bin/activate +else + if [ -L /dune/bin/setup-python ] && [ -e /dune/bin/setup-python ] ; then + dunecontrol bexec "echo -n :\$(pwd)/python >> $(pwd)/pythonpath.txt" + export PYTHONPATH=$PYTHONPATH$(cat pythonpath.txt) + rm pythonpath.txt + setup-python --opts=$DUNE_OPTS_FILE install + fi fi -- GitLab