From 386168b89d41325c6d845dc16062e40548a28c12 Mon Sep 17 00:00:00 2001 From: "Dennis.Glaeser" Date: Fri, 20 Nov 2020 11:13:37 +0100 Subject: [PATCH 01/10] [ci] add yml file for pipeline config --- .gitlab-ci.yml | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..9377f4c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,60 @@ +default: + image: registry.dune-project.org/docker/ci/dune:2.7-debian-11-gcc-9-20 + +cache: + key: ${CI_COMMIT_REF_SLUG} + paths: + - dep_modules + +stages: + - build + - test + +# rules for build & test: +# - pipelines are built for commits to master, tags, merge requests +# - Within merge requests, we require manual trigger of the build stage +# to start off the pipeline +.build_rules: + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE=="pipeline" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual + +.test_rules: + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE=="pipeline" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + +build lecture: + extends: .build_rules + stage: build + before_script: + - mkdir -p dep_modules; cd dep_modules + - git clone -b releases/2.7 https://gitlab.dune-project.org/extensions/dune-foamgrid.git + - git clone -b releases/2.7 https://gitlab.dune-project.org/extensions/dune-alugrid.git + - git clone -b master https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git + - dunecontrol --opts=/duneci/dune.opts --only=dune-foamgrid all + - dunecontrol --opts=/duneci/dune.opts --only=dune-alugrid all + - dunecontrol --opts=/duneci/dune.opts --only=dumux all + - cd .. + script: + - cd .. + - dunecontrol --opts=/duneci/dune.opts --only=dumux-lecture all + - cd dumux-lecture/build-cmake + - make -j 8 build_tests + artifacts: + paths: + - build-cmake + expire_in: 3 hours + +test lecture: + extends: .test_rules + stage: test + script: + - cd build-cmake + - duneci-ctest -j4 --no-tests=error + needs: + - job: build lecture + artifacts: true -- GitLab From 635dc36068c303089cdc6a0f8ca945b3773ec63a Mon Sep 17 00:00:00 2001 From: Timo Koch Date: Sun, 22 Nov 2020 19:42:22 +0100 Subject: [PATCH 02/10] [ci] Trigger pipeline as a child pipeline --- .gitlab-ci.yml | 72 ++++++++++++------------------------------ .gitlab-ci/default.yml | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 52 deletions(-) create mode 100644 .gitlab-ci/default.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9377f4c..5393c57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,60 +1,28 @@ -default: - image: registry.dune-project.org/docker/ci/dune:2.7-debian-11-gcc-9-20 - -cache: - key: ${CI_COMMIT_REF_SLUG} - paths: - - dep_modules - stages: - - build - - test + - trigger pipelines -# rules for build & test: -# - pipelines are built for commits to master, tags, merge requests -# - Within merge requests, we require manual trigger of the build stage -# to start off the pipeline -.build_rules: +# rules for the default triggers: +# - pipelines are triggered for commits to master, tags, merge requests +# - Within merge requests, we require to start the pipeline manually +# by clicking play for the trigger +.default-trigger: + stage: trigger pipelines + trigger: + include: + - local: .gitlab-ci/default.yml + strategy: depend rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE=="pipeline" + - if: $CI_PIPELINE_SOURCE == "pipeline" - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual -.test_rules: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE=="pipeline" - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - -build lecture: - extends: .build_rules - stage: build - before_script: - - mkdir -p dep_modules; cd dep_modules - - git clone -b releases/2.7 https://gitlab.dune-project.org/extensions/dune-foamgrid.git - - git clone -b releases/2.7 https://gitlab.dune-project.org/extensions/dune-alugrid.git - - git clone -b master https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git - - dunecontrol --opts=/duneci/dune.opts --only=dune-foamgrid all - - dunecontrol --opts=/duneci/dune.opts --only=dune-alugrid all - - dunecontrol --opts=/duneci/dune.opts --only=dumux all - - cd .. - script: - - cd .. - - dunecontrol --opts=/duneci/dune.opts --only=dumux-lecture all - - cd dumux-lecture/build-cmake - - make -j 8 build_tests - artifacts: - paths: - - build-cmake - expire_in: 3 hours +debian11-gcc9-cpp20: + extends: .default-trigger + variables: + IMAGE: registry.dune-project.org/docker/ci/dune:2.7-debian-11-gcc-9-20 -test lecture: - extends: .test_rules - stage: test - script: - - cd build-cmake - - duneci-ctest -j4 --no-tests=error - needs: - - job: build lecture - artifacts: true +ubuntu18.04-clang6-cpp17: + extends: .default-trigger + variables: + IMAGE: registry.dune-project.org/docker/ci/dune:2.7-ubuntu-18.04-clang-6-17 diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml new file mode 100644 index 0000000..58bd022 --- /dev/null +++ b/.gitlab-ci/default.yml @@ -0,0 +1,55 @@ +default: + image: $IMAGE + +cache: + key: ${CI_COMMIT_REF_SLUG}-$IMAGE + paths: + - dep_modules + +stages: + - build + - test + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE=="parent_pipeline" + +build lecture: + stage: build + before_script: + - mkdir -p dep_modules + - pushd dep_modules + - > + if [ ! -d dune-foamgrid ]; then + git clone -b releases/2.7 --depth 1 https://gitlab.dune-project.org/extensions/dune-foamgrid.git; + dunecontrol --opts=/duneci/dune.opts --only=dune-foamgrid all; + fi + - > + if [ ! -d dune-alugrid ]; then + git clone -b releases/2.7 --depth 1 https://gitlab.dune-project.org/extensions/dune-alugrid.git; + dunecontrol --opts=/duneci/dune.opts --make-options="-j8" --only=dune-alugrid all; + fi + - > + if [ ! -d dumux ]; then + git clone -b master --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git; + dunecontrol --opts=/duneci/dune.opts --only=dumux all + fi + - popd + script: + - cd .. + - dunecontrol --opts=/duneci/dune.opts --only=dumux-lecture all + - cd dumux-lecture/build-cmake + - make -k -j 8 build_tests + artifacts: + paths: + - build-cmake + expire_in: 3 hours + +test lecture: + stage: test + script: + - cd build-cmake + - duneci-ctest -j4 --no-tests=error + needs: + - job: build lecture + artifacts: true -- GitLab From beca0aeb2fa57cf6251c760e6ca0a1c58d0237dc Mon Sep 17 00:00:00 2001 From: "Dennis.Glaeser" Date: Mon, 23 Nov 2020 18:25:59 +0100 Subject: [PATCH 03/10] [ci] adapt to images from docker-ci repository --- .gitlab-ci.yml | 17 +++++++---------- .gitlab-ci/default.yml | 43 +++++++++++++++--------------------------- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5393c57..cd23099 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,12 @@ stages: - trigger pipelines +variables: + IMAGE_REGISTRY_URL: $CI_REGISTRY/dumux-repositories/dumux-docker-ci + # rules for the default triggers: -# - pipelines are triggered for commits to master, tags, merge requests -# - Within merge requests, we require to start the pipeline manually -# by clicking play for the trigger +# - pipelines are triggered for commits to master, tags, merge requests, external triggers +# - Within merge requests, we require to start the pipeline manually by clicking play for the trigger .default-trigger: stage: trigger pipelines trigger: @@ -17,12 +19,7 @@ stages: - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual -debian11-gcc9-cpp20: - extends: .default-trigger - variables: - IMAGE: registry.dune-project.org/docker/ci/dune:2.7-debian-11-gcc-9-20 - -ubuntu18.04-clang6-cpp17: +full-dune-2.7-gcc: extends: .default-trigger variables: - IMAGE: registry.dune-project.org/docker/ci/dune:2.7-ubuntu-18.04-clang-6-17 + IMAGE: $IMAGE_REGISTRY_URL/full:dune-2.7-gcc-ubuntu-20.04 diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml index 58bd022..7908b0e 100644 --- a/.gitlab-ci/default.yml +++ b/.gitlab-ci/default.yml @@ -1,11 +1,6 @@ default: image: $IMAGE -cache: - key: ${CI_COMMIT_REF_SLUG}-$IMAGE - paths: - - dep_modules - stages: - build - test @@ -17,39 +12,31 @@ workflow: build lecture: stage: build before_script: - - mkdir -p dep_modules - - pushd dep_modules - - > - if [ ! -d dune-foamgrid ]; then - git clone -b releases/2.7 --depth 1 https://gitlab.dune-project.org/extensions/dune-foamgrid.git; - dunecontrol --opts=/duneci/dune.opts --only=dune-foamgrid all; - fi - - > - if [ ! -d dune-alugrid ]; then - git clone -b releases/2.7 --depth 1 https://gitlab.dune-project.org/extensions/dune-alugrid.git; - dunecontrol --opts=/duneci/dune.opts --make-options="-j8" --only=dune-alugrid all; + - | + if [ ! "${DUMUX_BRANCH}" ]; then + DUMUX_BRANCH=master; + else + echo "Received branch name ${DUMUX_BRANCH} from upstream" fi - - > - if [ ! -d dumux ]; then - git clone -b master --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git; - dunecontrol --opts=/duneci/dune.opts --only=dumux all - fi - - popd + - echo "Checking out branch ${DUMUX_BRANCH} in dumux" + - git clone -b ${DUMUX_BRANCH} --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git + - dunecontrol --opts=$DUNE_OPTS_FILE --only=dumux all script: - - cd .. - - dunecontrol --opts=/duneci/dune.opts --only=dumux-lecture all - - cd dumux-lecture/build-cmake - - make -k -j 8 build_tests + - dunecontrol --opts=$DUNE_OPTS_FILE --current all + - dunecontrol --opts=$DUNE_OPTS_FILE --current bexec make -k -j4 build_tests artifacts: paths: - build-cmake + - dumux expire_in: 3 hours test lecture: stage: test script: - - cd build-cmake - - duneci-ctest -j4 --no-tests=error + - dunecontrol --opts=$DUNE_OPTS_FILE --current bexec dune-ctest -j4 --output-on-failure needs: - job: build lecture artifacts: true + artifacts: + reports: + junit: junit/dumux-lecture-cmake.xml -- GitLab From f450f27ad8dbcd7384df2e34905b09be4538dd4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= Date: Wed, 21 Apr 2021 15:52:19 +0200 Subject: [PATCH 04/10] [ci] pass down upstream mr branch name --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd23099..9f60cc0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,8 @@ variables: # rules for the default triggers: # - pipelines are triggered for commits to master, tags, merge requests, external triggers # - Within merge requests, we require to start the pipeline manually by clicking play for the trigger +# - if the pipeline was triggered externally (from dumux), we expect the variable DUMUX_MERGE_REQUEST_BRANCH +# and pass it to this project's test pipeline as DUMUX_BRANCH variable .default-trigger: stage: trigger pipelines trigger: @@ -18,6 +20,8 @@ variables: - if: $CI_PIPELINE_SOURCE == "pipeline" - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual + variables: + DUMUX_BRANCH: $DUMUX_MERGE_REQUEST_BRANCH full-dune-2.7-gcc: extends: .default-trigger -- GitLab From 5d7eff39e8b78455779298b1878f3022f1538b05 Mon Sep 17 00:00:00 2001 From: Dennis <> Date: Tue, 25 May 2021 16:53:39 +0200 Subject: [PATCH 05/10] [cmake] use dumux_add_test This ensures that test meta data are written out upon configuration. --- lecture/efm/1p2c_2p_2p2c/CMakeLists.txt | 6 +++--- lecture/efm/1p2cvs2p/CMakeLists.txt | 4 ++-- lecture/efm/2p/CMakeLists.txt | 2 +- lecture/mhs/groundwater/CMakeLists.txt | 2 +- lecture/mm/buckleyleverett/CMakeLists.txt | 2 +- lecture/mm/co2plume/CMakeLists.txt | 2 +- lecture/mm/columnxylene/CMakeLists.txt | 2 +- lecture/mm/convectivemixing/CMakeLists.txt | 2 +- lecture/mm/fractures/CMakeLists.txt | 2 +- lecture/mm/fuelcell/CMakeLists.txt | 2 +- lecture/mm/heatpipe/CMakeLists.txt | 2 +- lecture/mm/heavyoil/sagd/CMakeLists.txt | 2 +- lecture/mm/heavyoil/sagdcyclic/CMakeLists.txt | 2 +- lecture/mm/heavyoil/sagdcyclichyst/CMakeLists.txt | 2 +- lecture/mm/henryproblem/henry1p2c/CMakeLists.txt | 2 +- lecture/mm/henryproblem/henry2p/CMakeLists.txt | 2 +- lecture/mm/mcwhorter/CMakeLists.txt | 2 +- lecture/mm/naplinfiltration/3p/CMakeLists.txt | 2 +- lecture/mm/naplinfiltration/3p3c/CMakeLists.txt | 2 +- lecture/mm/remediationscenarios/CMakeLists.txt | 2 +- 20 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lecture/efm/1p2c_2p_2p2c/CMakeLists.txt b/lecture/efm/1p2c_2p_2p2c/CMakeLists.txt index 21994f4..246e7ef 100644 --- a/lecture/efm/1p2c_2p_2p2c/CMakeLists.txt +++ b/lecture/efm/1p2c_2p_2p2c/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME lens2pexercise3 +dumux_add_test(NAME lens2pexercise3 SOURCES lens2pexercise3.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy @@ -8,7 +8,7 @@ dune_add_test(NAME lens2pexercise3 ${CMAKE_CURRENT_BINARY_DIR}/lens-2p-00011.vtu --command "${CMAKE_CURRENT_BINARY_DIR}/lens2pexercise3 -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/exercise3.input -Problem.Name lens-2p") -dune_add_test(NAME lens2p2cexercise3 +dumux_add_test(NAME lens2p2cexercise3 SOURCES lens2p2cexercise3.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy @@ -16,7 +16,7 @@ dune_add_test(NAME lens2p2cexercise3 ${CMAKE_CURRENT_BINARY_DIR}/lens-2p2c-00011.vtu --command "${CMAKE_CURRENT_BINARY_DIR}/lens2p2cexercise3 -ParameterFile ${CMAKE_CURRENT_SOURCE_DIR}/exercise3.input -Problem.Name lens-2p2c") -dune_add_test(NAME lens1p2cexercise3 +dumux_add_test(NAME lens1p2cexercise3 SOURCES lens1p2cexercise3.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/efm/1p2cvs2p/CMakeLists.txt b/lecture/efm/1p2cvs2p/CMakeLists.txt index 54cb2fc..cf38fb0 100644 --- a/lecture/efm/1p2cvs2p/CMakeLists.txt +++ b/lecture/efm/1p2cvs2p/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME lens1p2cexercise1 +dumux_add_test(NAME lens1p2cexercise1 SOURCES lens1p2cexercise1.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy @@ -9,7 +9,7 @@ dune_add_test(NAME lens1p2cexercise1 --command "${CMAKE_CURRENT_BINARY_DIR}/lens1p2cexercise1 ${CMAKE_CURRENT_SOURCE_DIR}/exercise1.input -Problem.Name lens-1p2c" --zeroThreshold {"x^N2_liq":1e-18,"X^N2_liq":1e-18}) -dune_add_test(NAME lens2pexercise1 +dumux_add_test(NAME lens2pexercise1 SOURCES lens2pexercise1.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/efm/2p/CMakeLists.txt b/lecture/efm/2p/CMakeLists.txt index d90e917..e43dcbd 100644 --- a/lecture/efm/2p/CMakeLists.txt +++ b/lecture/efm/2p/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME lens2pexercise2 +dumux_add_test(NAME lens2pexercise2 SOURCES lens2pexercise2.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mhs/groundwater/CMakeLists.txt b/lecture/mhs/groundwater/CMakeLists.txt index efd16dd..ff1a264 100644 --- a/lecture/mhs/groundwater/CMakeLists.txt +++ b/lecture/mhs/groundwater/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME groundwater +dumux_add_test(NAME groundwater SOURCES groundwater.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/buckleyleverett/CMakeLists.txt b/lecture/mm/buckleyleverett/CMakeLists.txt index b905401..ecc346e 100644 --- a/lecture/mm/buckleyleverett/CMakeLists.txt +++ b/lecture/mm/buckleyleverett/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME buckleyleverettexercise +dumux_add_test(NAME buckleyleverettexercise SOURCES buckleyleverettexercise.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/co2plume/CMakeLists.txt b/lecture/mm/co2plume/CMakeLists.txt index 4841ec0..167a7ee 100644 --- a/lecture/mm/co2plume/CMakeLists.txt +++ b/lecture/mm/co2plume/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME co2plumeshapeexercise +dumux_add_test(NAME co2plumeshapeexercise SOURCES co2plumeshapeexercise.cc COMPILE_DEFINITIONS TYPETAG=PlumeShapeBoxTypeTag TIMEOUT 1800 diff --git a/lecture/mm/columnxylene/CMakeLists.txt b/lecture/mm/columnxylene/CMakeLists.txt index dc69964..1aa99f3 100644 --- a/lecture/mm/columnxylene/CMakeLists.txt +++ b/lecture/mm/columnxylene/CMakeLists.txt @@ -1,7 +1,7 @@ add_input_file_links() dune_symlink_to_source_files(FILES grids) -dune_add_test(NAME columnxyleneexercise +dumux_add_test(NAME columnxyleneexercise SOURCES columnxyleneexercise.cc COMPILE_DEFINITIONS TYPETAG=ColumnProblemBoxTypeTag COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py diff --git a/lecture/mm/convectivemixing/CMakeLists.txt b/lecture/mm/convectivemixing/CMakeLists.txt index 17461a6..5fefd10 100644 --- a/lecture/mm/convectivemixing/CMakeLists.txt +++ b/lecture/mm/convectivemixing/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(COMPILE_ONLY # the test produces different results due to the fingers on different machines +dumux_add_test(COMPILE_ONLY # the test produces different results due to the fingers on different machines NAME convmixexercise SOURCES convmixexercise.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py diff --git a/lecture/mm/fractures/CMakeLists.txt b/lecture/mm/fractures/CMakeLists.txt index e7be2a5..64a7575 100644 --- a/lecture/mm/fractures/CMakeLists.txt +++ b/lecture/mm/fractures/CMakeLists.txt @@ -1,7 +1,7 @@ dune_symlink_to_source_files(FILES "grids" "fracture_exercise.input" "plot.p") # test for the exercise -dune_add_test(NAME fracture_exercise +dumux_add_test(NAME fracture_exercise CMAKE_GUARD "( dune-foamgrid_FOUND AND dune-alugrid_FOUND )" SOURCES fractures.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py diff --git a/lecture/mm/fuelcell/CMakeLists.txt b/lecture/mm/fuelcell/CMakeLists.txt index 8f815a3..29e3580 100644 --- a/lecture/mm/fuelcell/CMakeLists.txt +++ b/lecture/mm/fuelcell/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME fuelcell +dumux_add_test(NAME fuelcell TIMEOUT 1800 SOURCES fuelcell.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py diff --git a/lecture/mm/heatpipe/CMakeLists.txt b/lecture/mm/heatpipe/CMakeLists.txt index 957a787..bd97ed9 100644 --- a/lecture/mm/heatpipe/CMakeLists.txt +++ b/lecture/mm/heatpipe/CMakeLists.txt @@ -1,7 +1,7 @@ add_input_file_links() dune_symlink_to_source_files(FILES grids) -dune_add_test(NAME heatpipe +dumux_add_test(NAME heatpipe SOURCES heatpipe.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/heavyoil/sagd/CMakeLists.txt b/lecture/mm/heavyoil/sagd/CMakeLists.txt index a11e9f5..8fae54e 100644 --- a/lecture/mm/heavyoil/sagd/CMakeLists.txt +++ b/lecture/mm/heavyoil/sagd/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME sagd +dumux_add_test(NAME sagd SOURCES sagd.cc TIMEOUT 1800 COMMAND ${CMAKE_CURRENT_BINARY_DIR}/sagd diff --git a/lecture/mm/heavyoil/sagdcyclic/CMakeLists.txt b/lecture/mm/heavyoil/sagdcyclic/CMakeLists.txt index 0d84e1f..208f394 100644 --- a/lecture/mm/heavyoil/sagdcyclic/CMakeLists.txt +++ b/lecture/mm/heavyoil/sagdcyclic/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME sagd_cyclic +dumux_add_test(NAME sagd_cyclic SOURCES sagd_cyclic.cc TIMEOUT 1800 COMMAND ${CMAKE_CURRENT_BINARY_DIR}/sagd_cyclic diff --git a/lecture/mm/heavyoil/sagdcyclichyst/CMakeLists.txt b/lecture/mm/heavyoil/sagdcyclichyst/CMakeLists.txt index d372728..cdab635 100644 --- a/lecture/mm/heavyoil/sagdcyclichyst/CMakeLists.txt +++ b/lecture/mm/heavyoil/sagdcyclichyst/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME sagd_cyclic_hyst +dumux_add_test(NAME sagd_cyclic_hyst SOURCES sagd_cyclic_hyst.cc TIMEOUT 1800 COMMAND ${CMAKE_CURRENT_BINARY_DIR}/sagd_cyclic_hyst diff --git a/lecture/mm/henryproblem/henry1p2c/CMakeLists.txt b/lecture/mm/henryproblem/henry1p2c/CMakeLists.txt index 0d6cb33..fb633db 100644 --- a/lecture/mm/henryproblem/henry1p2c/CMakeLists.txt +++ b/lecture/mm/henryproblem/henry1p2c/CMakeLists.txt @@ -1,7 +1,7 @@ add_input_file_links() dune_symlink_to_source_files(FILES grids) -dune_add_test(NAME henry1p2c +dumux_add_test(NAME henry1p2c SOURCES henry1p2c.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/henryproblem/henry2p/CMakeLists.txt b/lecture/mm/henryproblem/henry2p/CMakeLists.txt index 753fd58..7cdfbd6 100644 --- a/lecture/mm/henryproblem/henry2p/CMakeLists.txt +++ b/lecture/mm/henryproblem/henry2p/CMakeLists.txt @@ -1,7 +1,7 @@ add_input_file_links() dune_symlink_to_source_files(FILES grids) -dune_add_test(NAME henry2p +dumux_add_test(NAME henry2p SOURCES henry2p.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/mcwhorter/CMakeLists.txt b/lecture/mm/mcwhorter/CMakeLists.txt index c78e7f9..7d3ff65 100644 --- a/lecture/mm/mcwhorter/CMakeLists.txt +++ b/lecture/mm/mcwhorter/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links() -dune_add_test(NAME mcwhorterexercise +dumux_add_test(NAME mcwhorterexercise SOURCES mcwhorterexercise.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/naplinfiltration/3p/CMakeLists.txt b/lecture/mm/naplinfiltration/3p/CMakeLists.txt index 1103704..e73378e 100644 --- a/lecture/mm/naplinfiltration/3p/CMakeLists.txt +++ b/lecture/mm/naplinfiltration/3p/CMakeLists.txt @@ -1,7 +1,7 @@ add_input_file_links() # for the test only simulate one week -dune_add_test(NAME naplinfiltration3p +dumux_add_test(NAME naplinfiltration3p SOURCES naplinfiltration3p.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/naplinfiltration/3p3c/CMakeLists.txt b/lecture/mm/naplinfiltration/3p3c/CMakeLists.txt index 8360898..565d7f7 100644 --- a/lecture/mm/naplinfiltration/3p3c/CMakeLists.txt +++ b/lecture/mm/naplinfiltration/3p3c/CMakeLists.txt @@ -1,7 +1,7 @@ add_input_file_links() # for the test only simulate one week -dune_add_test(NAME naplinfiltration3p3c +dumux_add_test(NAME naplinfiltration3p3c SOURCES naplinfiltration3p3c.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py CMD_ARGS --script fuzzy diff --git a/lecture/mm/remediationscenarios/CMakeLists.txt b/lecture/mm/remediationscenarios/CMakeLists.txt index 3657666..9733426 100644 --- a/lecture/mm/remediationscenarios/CMakeLists.txt +++ b/lecture/mm/remediationscenarios/CMakeLists.txt @@ -1,6 +1,6 @@ add_input_file_links(remediationscenariosexercise.input) -dune_add_test(NAME remediationscenariosexercise +dumux_add_test(NAME remediationscenariosexercise TIMEOUT 1800 SOURCES remediationscenariosexercise.cc COMMAND ${dumux_INCLUDE_DIRS}/bin/testing/runtest.py -- GitLab From 34294ef212e1c25bb125a68a5c9ea00f1a95591c Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 19 May 2021 19:13:56 +0200 Subject: [PATCH 06/10] [ci] add test selection stage --- .gitlab-ci.yml | 7 +++-- .gitlab-ci/default.yml | 60 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f60cc0..dea92b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,10 +5,9 @@ variables: IMAGE_REGISTRY_URL: $CI_REGISTRY/dumux-repositories/dumux-docker-ci # rules for the default triggers: -# - pipelines are triggered for commits to master, tags, merge requests, external triggers +# - pipelines are triggered for merge requests and external triggers (e.g. nightly build in dumux) # - Within merge requests, we require to start the pipeline manually by clicking play for the trigger -# - if the pipeline was triggered externally (from dumux), we expect the variable DUMUX_MERGE_REQUEST_BRANCH -# and pass it to this project's test pipeline as DUMUX_BRANCH variable +# - if the pipeline was triggered externally (from dumux), we accept the variable DUMUX_MERGE_REQUEST_BRANCH .default-trigger: stage: trigger pipelines trigger: @@ -21,7 +20,7 @@ variables: - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual variables: - DUMUX_BRANCH: $DUMUX_MERGE_REQUEST_BRANCH + DUMUX_MR_BRANCH: $DUMUX_MERGE_REQUEST_BRANCH full-dune-2.7-gcc: extends: .default-trigger diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml index 7908b0e..1649ab9 100644 --- a/.gitlab-ci/default.yml +++ b/.gitlab-ci/default.yml @@ -2,6 +2,7 @@ default: image: $IMAGE stages: + - configure - build - test @@ -9,31 +10,68 @@ workflow: rules: - if: $CI_PIPELINE_SOURCE=="parent_pipeline" -build lecture: - stage: build +select tests: + stage: configure before_script: - | - if [ ! "${DUMUX_BRANCH}" ]; then - DUMUX_BRANCH=master; - else - echo "Received branch name ${DUMUX_BRANCH} from upstream" + DUMUX_CLONE_BRANCH=master + if [[ -n "${DUMUX_MR_BRANCH}" ]]; then + DUMUX_CLONE_BRANCH=${DUMUX_MR_BRANCH} + echo "Received upstream merge request branch ${DUMUX_MR_BRANCH}" fi - - echo "Checking out branch ${DUMUX_BRANCH} in dumux" - - git clone -b ${DUMUX_BRANCH} --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git + - echo "Checking out branch ${DUMUX_CLONE_BRANCH} in dumux" + - git clone -b ${DUMUX_CLONE_BRANCH} --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git - dunecontrol --opts=$DUNE_OPTS_FILE --only=dumux all script: - - dunecontrol --opts=$DUNE_OPTS_FILE --current all - - dunecontrol --opts=$DUNE_OPTS_FILE --current bexec make -k -j4 build_tests + - | + dunecontrol --opts=$DUNE_OPTS_FILE --current all + if [[ -n "${DUMUX_MR_BRANCH}" ]]; then + python3 dumux/bin/testing/getchangedfiles.py -o changedfiles.txt -t origin/$DUMUX_MR_BRANCH + python3 dumux/bin/testing/findtests.py -o affectedtests.json --file-list changedfiles.txt + else + echo "Skipping test selection, build/test stages will consider all tests!" + echo "{}" >> affectedtests.json + fi + artifacts: + paths: + - dumux + - build-cmake + - affectedtests.json + expire_in: 3 hours + +build lecture: + stage: build + script: + - | + pushd build-cmake + make clean + if [[ -n "${DUMUX_MR_BRANCH}" ]]; then + python3 ../dumux/bin/testing/runselectedtests.py -c ../affectedtests.json -b + else + python3 ../dumux/bin/testing/runselectedtests.py --all -b + fi + popd artifacts: paths: - build-cmake - dumux + - affectedtests.json expire_in: 3 hours test lecture: stage: test + variables: + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 script: - - dunecontrol --opts=$DUNE_OPTS_FILE --current bexec dune-ctest -j4 --output-on-failure + - | + pushd build-cmake + if [[ -n "${DUMUX_MR_BRANCH}" ]]; then + python3 ../dumux/bin/testing/runselectedtests.py -c ../affectedtests.json -t + else + python3 ../dumux/bin/testing/runselectedtests.py --all -t + fi + popd needs: - job: build lecture artifacts: true -- GitLab From 49ccd33c9768a0604c13fd20b40b469e87213328 Mon Sep 17 00:00:00 2001 From: Dennis Date: Wed, 19 May 2021 19:20:56 +0200 Subject: [PATCH 07/10] [ci] do test selection also for mrs in lecture --- .gitlab-ci.yml | 2 ++ .gitlab-ci/default.yml | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dea92b2..671cf2d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,8 @@ variables: when: manual variables: DUMUX_MR_BRANCH: $DUMUX_MERGE_REQUEST_BRANCH + TRIGGER_SOURCE: $CI_PIPELINE_SOURCE + MR_TARGET_BRANCH_NAME: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME full-dune-2.7-gcc: extends: .default-trigger diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml index 1649ab9..2f7812a 100644 --- a/.gitlab-ci/default.yml +++ b/.gitlab-ci/default.yml @@ -10,6 +10,10 @@ workflow: rules: - if: $CI_PIPELINE_SOURCE=="parent_pipeline" +variables: + TRIGGER_SOURCE: "unknown" + MR_TARGET_BRANCH_NAME: "master" + select tests: stage: configure before_script: @@ -26,8 +30,13 @@ select tests: - | dunecontrol --opts=$DUNE_OPTS_FILE --current all if [[ -n "${DUMUX_MR_BRANCH}" ]]; then - python3 dumux/bin/testing/getchangedfiles.py -o changedfiles.txt -t origin/$DUMUX_MR_BRANCH - python3 dumux/bin/testing/findtests.py -o affectedtests.json --file-list changedfiles.txt + echo "Detecting test affected by changes in Dumux" + python3 dumux/bin/testing/getchangedfiles.py -f dumux -o changedfiles.txt -t origin/$DUMUX_MR_BRANCH + python3 dumux/bin/testing/findtests.py -o affectedtests.json --file-list changedfiles.txt --build-dir build-cmake + elif [[ "$TRIGGER_SOURCE" == "merge_request_event" ]]; then + echo "Detecting test affected by changes in this merge request, using target branch $MR_TARGET_BRANCH_NAME" + python3 dumux/bin/testing/getchangedfiles.py -o changedfiles.txt -t origin/$MR_TARGET_BRANCH_NAME + python3 dumux/bin/testing/findtests.py -o affectedtests.json --file-list changedfiles.txt --build-dir build-cmake else echo "Skipping test selection, build/test stages will consider all tests!" echo "{}" >> affectedtests.json -- GitLab From 8f2b4fe988e314136ba30f0025083f36d0423c1f Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 20 May 2021 08:20:01 +0200 Subject: [PATCH 08/10] [ci] receive dumux mr info from upstream --- .gitlab-ci.yml | 7 ++++--- .gitlab-ci/default.yml | 46 +++++++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 671cf2d..7cc5a7c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: # rules for the default triggers: # - pipelines are triggered for merge requests and external triggers (e.g. nightly build in dumux) # - Within merge requests, we require to start the pipeline manually by clicking play for the trigger -# - if the pipeline was triggered externally (from dumux), we accept the variable DUMUX_MERGE_REQUEST_BRANCH +# - if the pipeline was triggered externally (from dumux), we may receive merge-request-related variables .default-trigger: stage: trigger pipelines trigger: @@ -15,14 +15,15 @@ variables: - local: .gitlab-ci/default.yml strategy: depend rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == "pipeline" - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: manual variables: - DUMUX_MR_BRANCH: $DUMUX_MERGE_REQUEST_BRANCH TRIGGER_SOURCE: $CI_PIPELINE_SOURCE MR_TARGET_BRANCH_NAME: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + DUMUX_PIPELINE_SOURCE: $DUMUX_PIPELINE_SOURCE + DUMUX_MR_SOURCE_BRANCH: $DUMUX_MERGE_REQUEST_SOURCE_BRANCH + DUMUX_MR_TARGET_BRANCH: $DUMUX_MERGE_REQUEST_TARGET_BRANCH full-dune-2.7-gcc: extends: .default-trigger diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml index 2f7812a..7926e12 100644 --- a/.gitlab-ci/default.yml +++ b/.gitlab-ci/default.yml @@ -10,36 +10,54 @@ workflow: rules: - if: $CI_PIPELINE_SOURCE=="parent_pipeline" +# variables that may be overwritten by the trigger variables: TRIGGER_SOURCE: "unknown" MR_TARGET_BRANCH_NAME: "master" + DUMUX_PIPELINE_SOURCE: "unknown" + DUMUX_MR_SOURCE_BRANCH: "unknown" + DUMUX_MR_TARGET_BRANCH: "unknown" select tests: stage: configure before_script: - | DUMUX_CLONE_BRANCH=master - if [[ -n "${DUMUX_MR_BRANCH}" ]]; then - DUMUX_CLONE_BRANCH=${DUMUX_MR_BRANCH} - echo "Received upstream merge request branch ${DUMUX_MR_BRANCH}" + if [[ -n "${DUMUX_MR_SOURCE_BRANCH}" ]]; then + DUMUX_CLONE_BRANCH=${DUMUX_MR_SOURCE_BRANCH} + echo "Received upstream merge request branch ${DUMUX_MR_SOURCE_BRANCH}" fi - echo "Checking out branch ${DUMUX_CLONE_BRANCH} in dumux" - git clone -b ${DUMUX_CLONE_BRANCH} --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git - dunecontrol --opts=$DUNE_OPTS_FILE --only=dumux all script: + - dunecontrol --opts=$DUNE_OPTS_FILE --current all - | - dunecontrol --opts=$DUNE_OPTS_FILE --current all - if [[ -n "${DUMUX_MR_BRANCH}" ]]; then - echo "Detecting test affected by changes in Dumux" - python3 dumux/bin/testing/getchangedfiles.py -f dumux -o changedfiles.txt -t origin/$DUMUX_MR_BRANCH - python3 dumux/bin/testing/findtests.py -o affectedtests.json --file-list changedfiles.txt --build-dir build-cmake + echo "Check if tests affected by changes have to be identified" + if [[ "$DUMUX_PIPELINE_SOURCE" == "merge_request_event" ]]; then + CHECK_FOLDER="dumux" + SOURCE_TREE=origin/$DUMUX_MR_SOURCE_BRANCH + TARGET_TREE=origin/$DUMUX_MR_TARGET_BRANCH elif [[ "$TRIGGER_SOURCE" == "merge_request_event" ]]; then - echo "Detecting test affected by changes in this merge request, using target branch $MR_TARGET_BRANCH_NAME" - python3 dumux/bin/testing/getchangedfiles.py -o changedfiles.txt -t origin/$MR_TARGET_BRANCH_NAME - python3 dumux/bin/testing/findtests.py -o affectedtests.json --file-list changedfiles.txt --build-dir build-cmake + CHECK_FOLDER="." + SOURCE_TREE="HEAD" + TARGET_TREE=origin/$MR_TARGET_BRANCH_NAME + fi + - | + if [[ -n "$SOURCE_TREE" ]]; then + echo "Detecting tests affected by changes in folder $CHECK_FOLDER," + echo "using source/target trees: $SOURCE_TREE / $TARGET_TREE" + python3 dumux/bin/testing/getchangedfiles.py --folder $CHECK_FOLDER \ + --source-tree $SOURCE_TREE \ + --target-tree $TARGET_TREE \ + --outfile changedfiles.txt + python3 dumux/bin/testing/findtests.py --outfile affectedtests.json \ + --file-list changedfiles.txt \ + --build-dir build-cmake else + echo "Received trigger source / dumux pipeline source: $TRIGGER_SOURCE / $DUMUX_PIPELINE_SOURCE" echo "Skipping test selection, build/test stages will consider all tests!" - echo "{}" >> affectedtests.json + touch affectedtests.json fi artifacts: paths: @@ -54,7 +72,7 @@ build lecture: - | pushd build-cmake make clean - if [[ -n "${DUMUX_MR_BRANCH}" ]]; then + if [ -s ../affectedtests.json ]; then python3 ../dumux/bin/testing/runselectedtests.py -c ../affectedtests.json -b else python3 ../dumux/bin/testing/runselectedtests.py --all -b @@ -75,7 +93,7 @@ test lecture: script: - | pushd build-cmake - if [[ -n "${DUMUX_MR_BRANCH}" ]]; then + if [ -s ../affectedtests.json ]; then python3 ../dumux/bin/testing/runselectedtests.py -c ../affectedtests.json -t else python3 ../dumux/bin/testing/runselectedtests.py --all -t -- GitLab From 6ac653eea850a170744a58c1e5b0f0156bed3a96 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 20 May 2021 09:46:50 +0200 Subject: [PATCH 09/10] [ci] streamline logic between scripts --- .gitlab-ci/default.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml index 7926e12..be1f9da 100644 --- a/.gitlab-ci/default.yml +++ b/.gitlab-ci/default.yml @@ -23,9 +23,10 @@ select tests: before_script: - | DUMUX_CLONE_BRANCH=master - if [[ -n "${DUMUX_MR_SOURCE_BRANCH}" ]]; then - DUMUX_CLONE_BRANCH=${DUMUX_MR_SOURCE_BRANCH} + if [[ "$DUMUX_PIPELINE_SOURCE" == "merge_request_event" ]]; then + echo "Starting pipeline triggered by upstream merge request event" echo "Received upstream merge request branch ${DUMUX_MR_SOURCE_BRANCH}" + DUMUX_CLONE_BRANCH=${DUMUX_MR_SOURCE_BRANCH} fi - echo "Checking out branch ${DUMUX_CLONE_BRANCH} in dumux" - git clone -b ${DUMUX_CLONE_BRANCH} --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git -- GitLab From d1b0957b1504159146033dfad2a60cf34e367020 Mon Sep 17 00:00:00 2001 From: Dennis Date: Thu, 20 May 2021 10:22:50 +0200 Subject: [PATCH 10/10] [ci] ensure that upstream mr branches are present --- .gitlab-ci/default.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci/default.yml b/.gitlab-ci/default.yml index be1f9da..633672b 100644 --- a/.gitlab-ci/default.yml +++ b/.gitlab-ci/default.yml @@ -22,23 +22,24 @@ select tests: stage: configure before_script: - | - DUMUX_CLONE_BRANCH=master if [[ "$DUMUX_PIPELINE_SOURCE" == "merge_request_event" ]]; then - echo "Starting pipeline triggered by upstream merge request event" - echo "Received upstream merge request branch ${DUMUX_MR_SOURCE_BRANCH}" - DUMUX_CLONE_BRANCH=${DUMUX_MR_SOURCE_BRANCH} + echo "Cloning upstream merge request source/target branches ${DUMUX_MR_SOURCE_BRANCH}/${DUMUX_MR_TARGET_BRANCH}" + git clone -b ${DUMUX_MR_SOURCE_BRANCH} --depth=1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git + pushd dumux + git fetch --depth=1 origin ${DUMUX_MR_TARGET_BRANCH}:${DUMUX_MR_TARGET_BRANCH} + popd + else + echo "Cloning into Dumux master" + git clone -b master --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git fi - - echo "Checking out branch ${DUMUX_CLONE_BRANCH} in dumux" - - git clone -b ${DUMUX_CLONE_BRANCH} --depth 1 https://git.iws.uni-stuttgart.de/dumux-repositories/dumux.git - dunecontrol --opts=$DUNE_OPTS_FILE --only=dumux all script: - dunecontrol --opts=$DUNE_OPTS_FILE --current all - | - echo "Check if tests affected by changes have to be identified" if [[ "$DUMUX_PIPELINE_SOURCE" == "merge_request_event" ]]; then CHECK_FOLDER="dumux" - SOURCE_TREE=origin/$DUMUX_MR_SOURCE_BRANCH - TARGET_TREE=origin/$DUMUX_MR_TARGET_BRANCH + SOURCE_TREE=$DUMUX_MR_SOURCE_BRANCH + TARGET_TREE=$DUMUX_MR_TARGET_BRANCH elif [[ "$TRIGGER_SOURCE" == "merge_request_event" ]]; then CHECK_FOLDER="." SOURCE_TREE="HEAD" -- GitLab