[python][ci] Fix Python environment setup with dune 2.9

.gitlab-ci/default.yml

@@ -24,8 +24,13 @@ variables:
 configure:
   stage: configure
   script:
-    - dunecontrol --opts=$DUNE_OPTS_FILE --current all
-    - source bin/testing/ci-setup-python-env.sh
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current configure
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current make -j8
+    # cache state of the Dune virtual env for Python if it exists (Dune 2.9)
+    - |
+      if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then
+        cp -r /dune/modules/dune-common/build-cmake/dune-env build-cmake
+      fi
   artifacts:
     paths:
       - build-cmake
@@ -46,9 +51,13 @@ black (python):
 pylint-flake8 (python):
   stage: linting
   script:
-    - source bin/testing/ci-setup-python-env.sh
     - |
       if [ -d build-cmake/python/dumux ] ; then
+        source bin/testing/ci-setup-python-env.sh
+        # if we are in venv (Dune 2.9) install linters
+        if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then
+          python -m pip install pylint flake8
+        fi
         pylint --rcfile=.pylintrc build-cmake/python/dumux
         pylint --rcfile=.pylintrc bin
         flake8 build-cmake/python/dumux
@@ -99,6 +108,11 @@ select tests:
 compile cpp:
   stage: build
   script:
+    # remove cached Python dune-env if existing (not needed for C++) (Dune 2.9)
+    - |
+      if [ -d "build-cmake/dune-env" ]; then
+        rm -r build-cmake/dune-env
+      fi
     - |
       pushd build-cmake
         make clean && make all
@@ -148,7 +162,12 @@ test python:
     OMPI_ALLOW_RUN_AS_ROOT: 1
     OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
   script:
-    - source bin/testing/ci-setup-python-env.sh
+    # restore Python virtual env from cache (job:configure artifacts) (Dune 2.9)
+    - |
+      if [ -d "build-cmake/dune-env" ]; then
+        rm -r /dune/modules/dune-common/build-cmake/dune-env
+        mv build-cmake/dune-env /dune/modules/dune-common/build-cmake/dune-env
+      fi
     - |
       if ([ ! -s changedfiles.txt ] || grep -q python "changedfiles.txt"); then
         if [ ! -s changedfiles.txt ]; then
@@ -158,7 +177,7 @@ test python:
         fi
         source bin/testing/ci-setup-python-env.sh
         pushd build-cmake
-          ctest --output-on-failure -L python
+          DUNE_LOG_LEVEL=DEBUG ctest --output-on-failure -L python
         popd
       else
         echo "No changes in the Python bindings/Python code detected: skipping tests."

bin/testing/ci-setup-python-env.sh

 #!/bin/bash
 
-if [ -L /dune/bin/setup-python ] && [ -e /dune/bin/setup-python ] ; then
-    dunecontrol bexec "echo -n :\$(pwd)/python >> $(pwd)/pythonpath.txt"
-    export PYTHONPATH=$PYTHONPATH$(cat pythonpath.txt)
-    rm pythonpath.txt
-    setup-python --opts=$DUNE_OPTS_FILE install
+if [ -d "/dune/modules/dune-common/build-cmake/dune-env" ]; then
+    # Use internal venv of DUNE
+    echo "Activating the Python virtual environment of dune-common"
+    source /dune/modules/dune-common/build-cmake/dune-env/bin/activate
+else
+    if [ -L /dune/bin/setup-python ] && [ -e /dune/bin/setup-python ] ; then
+        dunecontrol bexec "echo -n :\$(pwd)/python >> $(pwd)/pythonpath.txt"
+        export PYTHONPATH=$PYTHONPATH$(cat pythonpath.txt)
+        rm pythonpath.txt
+        setup-python --opts=$DUNE_OPTS_FILE install
+    fi
 fi