[ci] avoid pipeline scripting

.gitlab-ci.yml

@@ -6,8 +6,9 @@ stages:
 variables:
   IMAGE_REGISTRY_URL: $CI_REGISTRY/dumux-repositories/dumux-docker-ci
 
-# Cases in which to create a pipeline. The `generate-config` job further
-# specifies the situations in which they must be started manually.
+# Cases in which to create a pipeline. The `select-pipeline` job further
+# specifies the situations in which they must be started manually. Currently,
+# we only have automatic pipeline triggers for scheduled pipelines.
 workflow:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -21,19 +22,19 @@ workflow:
 # the build stage to identify the tests affected by changes introduced in the  #
 # merge request. In all other cases, we use the default which runs all tests.  #
 ################################################################################
-generate-config:
+select-pipeline:
   image: $IMAGE_REGISTRY_URL/full:dune-2.7-gcc-ubuntu-20.04
   stage: configure
   script:
     - |
       if [ $CI_PIPELINE_SOURCE == "merge_request_event" ]; then
-          python3 .gitlab-ci/makepipelineconfig.py -o generated-config.yml --affectedtestsonly
+          cp .gitlab-ci/affectedtestsonly.yml pipeline-config.yml
       else
-          python3 .gitlab-ci/makepipelineconfig.py -o generated-config.yml
+          cp .gitlab-ci/default.yml pipeline-config.yml
       fi
   artifacts:
     paths:
-      - generated-config.yml
+      - pipeline-config.yml
     expire_in: 3 hours
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -53,11 +54,11 @@ generate-config:
 .base-trigger:
   stage: trigger
   needs:
-    - generate-config
+    - select-pipeline
   trigger:
     include:
-      - artifact: generated-config.yml
-        job: generate-config
+      - artifact: pipeline-config.yml
+        job: select-pipeline
     strategy: depend
 
 # trigger for jobs that should not be created in merge requests

.gitlab-ci/affectedtestsonly.yml

+default:
+  image: $IMAGE
+
+stages:
+  - configure
+  - build
+  - test
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE=="parent_pipeline"
+
+select tests:
+  stage: configure
+  script:
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current all
+    - |
+      pushd build-cmake
+        python3 ../bin/testing/findtests.py -f ../affectedtests.json -t origin/master
+      popd
+  artifacts:
+    paths:
+      - affectedtests.json
+    expire_in: 3 hours
+
+build dumux:
+  stage: build
+  script:
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current all
+    - cp affectedtests.json build-cmake
+    - |
+      pushd build-cmake
+        python3 ../bin/testing/runselectedtests.py -c affectedtests.json -b
+      popd
+  artifacts:
+    paths:
+      - build-cmake
+    expire_in: 3 hours
+  needs:
+    - job: select tests
+      artifacts: true
+
+test dumux:
+  stage: test
+  script:
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current all
+    - |
+      pushd build-cmake
+        python3 ../bin/testing/runselectedtests.py -c affectedtests.json -t
+      popd
+  needs:
+    - job: build dumux
+      artifacts: true
+  artifacts:
+    reports:
+      junit: junit/dumux-cmake.xml

.gitlab-ci/default.yml.template → .gitlab-ci/default.yml

 default:
-  image: $$IMAGE
+  image: $IMAGE
 
 stages:
-${stages}
+  - build
+  - test
 
 workflow:
   rules:
-    - if: $$CI_PIPELINE_SOURCE=="parent_pipeline"
-
-${test_select_job}
+    - if: $CI_PIPELINE_SOURCE=="parent_pipeline"
 
 build dumux:
   stage: build
   script:
-${build_script}
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current all
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current bexec make -k -j4 build_tests
   artifacts:
     paths:
       - build-cmake
     expire_in: 3 hours
-${build_needs}
 
 test dumux:
   stage: test
   script:
-${test_script}
+    - dunecontrol --opts=$DUNE_OPTS_FILE --current bexec dune-ctest -j4 --output-on-failure
   needs:
     - job: build dumux
       artifacts: true

.gitlab-ci/makepipelineconfig.py

-#!/usr/bin/env python3
-
-import os
-import sys
-import string
-from argparse import ArgumentParser
-
-# require Python 3
-if sys.version_info.major < 3:
-    sys.exit('Python 3 required')
-
-parser = ArgumentParser(description='Generate dumux test pipeline .yml file')
-parser.add_argument('-o', '--outfile',
-                    required=True,
-                    help='Specify the file to write the pipeline definition')
-parser.add_argument('-a', '--affectedtestsonly',
-                    required=False,
-                    action='store_true',
-                    help='Use this flag to create a pipeline that runs only '
-                         'those tests that are affected by changes w.r.t to '
-                         'the origin/master branch')
-parser.add_argument('-t', '--template',
-                    required=False,
-                    default='.gitlab-ci/default.yml.template',
-                    help='Specify the template .yml file to be used')
-parser.add_argument('-i', '--indentation',
-                    required=False,
-                    default=4,
-                    help='Specify the indentation for the script commands')
-args = vars(parser.parse_args())
-
-
-# substitute content from template and write to target
-def substituteAndWrite(mapping):
-
-    template = args['template']
-    if not os.path.exists(template):
-        sys.exit("Template file '" + template + "' could not be found")
-
-    with open(args['outfile'], 'w') as ymlFile:
-        raw = string.Template(open(template).read())
-        ymlFile.write(raw.substitute(**mapping))
-
-
-commandIndentation = ' '*args['indentation']
-with open(args['outfile'], 'w') as ymlFile:
-
-    def wrapDuneControl(command):
-        return 'dunecontrol --opts=$DUNE_OPTS_FILE --current ' + command
-
-    def makeYamlList(commands, indent=commandIndentation):
-        commands = [indent + '- ' + comm for comm in commands]
-        return '\n'.join(commands)
-
-    # if no configuration is given, build and run all tests (skip select stage)
-    if not args['affectedtestsonly']:
-        buildCommand = [wrapDuneControl('all'),
-                        wrapDuneControl('bexec make -k -j4 build_tests')]
-        testCommand = [wrapDuneControl('bexec dune-ctest'
-                                       ' -j4 --output-on-failure')]
-
-        substituteAndWrite({'build_script': makeYamlList(buildCommand),
-                            'test_script': makeYamlList(testCommand),
-                            'stages': makeYamlList(['build', 'test'], '  '),
-                            'test_select_job': '',
-                            'build_needs': ''})
-
-    # otherwise, add a stage that detects the tests to be run first
-    else:
-        selectStageName = 'configure'
-        selectJobName = 'select tests'
-
-        stages = makeYamlList([selectStageName, 'build', 'test'], '  ')
-        buildNeeds = '\n'.join(['  needs:',
-                                '    - job: {}'.format(selectJobName),
-                                '      artifacts: true'])
-
-        selectJob = '\n'.join([selectJobName + ':',
-                               '  stage: {}'.format(selectStageName),
-                               '  script:'])
-        selectJob += '\n'
-        selectJob += makeYamlList([wrapDuneControl('all'),
-                                   'pushd build-cmake',
-                                   'python3 ../bin/testing/findtests.py'
-                                   ' -f ../affectedtests.json'
-                                   ' -t origin/master',
-                                   'popd'])
-        selectJob += '\n'
-        selectJob += '\n'.join(['  artifacts:',
-                                '    paths:',
-                                '      - affectedtests.json',
-                                '    expire_in: 3 hours'])
-
-        buildCommand = [wrapDuneControl('all'),
-                        'cp affectedtests.json build-cmake',
-                        'pushd build-cmake',
-                        'python3 ../bin/testing/runselectedtests.py '
-                        ' -c affectedtests.json -b',
-                        'popd']
-        testCommand = [wrapDuneControl('all'),
-                       'pushd build-cmake',
-                       'python3 ../bin/testing/runselectedtests.py '
-                       ' -c affectedtests.json -t',
-                       'popd']
-
-        substituteAndWrite({'build_script': makeYamlList(buildCommand),
-                            'test_script': makeYamlList(testCommand),
-                            'stages': stages,
-                            'test_select_job': selectJob,
-                            'build_needs': buildNeeds})