Commit ee20fc79 authored by Dennis Gläser's avatar Dennis Gläser
Browse files

[ci] avoid pipeline scripting

parent 43c45b3b
......@@ -6,8 +6,9 @@ stages:
variables:
IMAGE_REGISTRY_URL: $CI_REGISTRY/dumux-repositories/dumux-docker-ci
# Cases in which to create a pipeline. The `generate-config` job further
# specifies the situations in which they must be started manually.
# Cases in which to create a pipeline. The `select-pipeline` job further
# specifies the situations in which they must be started manually. Currently,
# we only have automatic pipeline triggers for scheduled pipelines.
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
......@@ -21,19 +22,19 @@ workflow:
# the build stage to identify the tests affected by changes introduced in the #
# merge request. In all other cases, we use the default which runs all tests. #
################################################################################
generate-config:
select-pipeline:
image: $IMAGE_REGISTRY_URL/full:dune-2.7-gcc-ubuntu-20.04
stage: configure
script:
- |
if [ $CI_PIPELINE_SOURCE == "merge_request_event" ]; then
python3 .gitlab-ci/makepipelineconfig.py -o generated-config.yml --affectedtestsonly
cp .gitlab-ci/affectedtestsonly.yml pipeline-config.yml
else
python3 .gitlab-ci/makepipelineconfig.py -o generated-config.yml
cp .gitlab-ci/default.yml pipeline-config.yml
fi
artifacts:
paths:
- generated-config.yml
- pipeline-config.yml
expire_in: 3 hours
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
......@@ -53,11 +54,11 @@ generate-config:
.base-trigger:
stage: trigger
needs:
- generate-config
- select-pipeline
trigger:
include:
- artifact: generated-config.yml
job: generate-config
- artifact: pipeline-config.yml
job: select-pipeline
strategy: depend
# trigger for jobs that should not be created in merge requests
......
default:
image: $IMAGE
stages:
- configure
- build
- test
workflow:
rules:
- if: $CI_PIPELINE_SOURCE=="parent_pipeline"
select tests:
stage: configure
script:
- dunecontrol --opts=$DUNE_OPTS_FILE --current all
- |
pushd build-cmake
python3 ../bin/testing/findtests.py -f ../affectedtests.json -t origin/master
popd
artifacts:
paths:
- affectedtests.json
expire_in: 3 hours
build dumux:
stage: build
script:
- dunecontrol --opts=$DUNE_OPTS_FILE --current all
- cp affectedtests.json build-cmake
- |
pushd build-cmake
python3 ../bin/testing/runselectedtests.py -c affectedtests.json -b
popd
artifacts:
paths:
- build-cmake
expire_in: 3 hours
needs:
- job: select tests
artifacts: true
test dumux:
stage: test
script:
- dunecontrol --opts=$DUNE_OPTS_FILE --current all
- |
pushd build-cmake
python3 ../bin/testing/runselectedtests.py -c affectedtests.json -t
popd
needs:
- job: build dumux
artifacts: true
artifacts:
reports:
junit: junit/dumux-cmake.xml
default:
image: $$IMAGE
image: $IMAGE
stages:
${stages}
- build
- test
workflow:
rules:
- if: $$CI_PIPELINE_SOURCE=="parent_pipeline"
${test_select_job}
- if: $CI_PIPELINE_SOURCE=="parent_pipeline"
build dumux:
stage: build
script:
${build_script}
- dunecontrol --opts=$DUNE_OPTS_FILE --current all
- dunecontrol --opts=$DUNE_OPTS_FILE --current bexec make -k -j4 build_tests
artifacts:
paths:
- build-cmake
expire_in: 3 hours
${build_needs}
test dumux:
stage: test
script:
${test_script}
- dunecontrol --opts=$DUNE_OPTS_FILE --current bexec dune-ctest -j4 --output-on-failure
needs:
- job: build dumux
artifacts: true
......
#!/usr/bin/env python3
import os
import sys
import string
from argparse import ArgumentParser
# require Python 3
if sys.version_info.major < 3:
sys.exit('Python 3 required')
parser = ArgumentParser(description='Generate dumux test pipeline .yml file')
parser.add_argument('-o', '--outfile',
required=True,
help='Specify the file to write the pipeline definition')
parser.add_argument('-a', '--affectedtestsonly',
required=False,
action='store_true',
help='Use this flag to create a pipeline that runs only '
'those tests that are affected by changes w.r.t to '
'the origin/master branch')
parser.add_argument('-t', '--template',
required=False,
default='.gitlab-ci/default.yml.template',
help='Specify the template .yml file to be used')
parser.add_argument('-i', '--indentation',
required=False,
default=4,
help='Specify the indentation for the script commands')
args = vars(parser.parse_args())
# substitute content from template and write to target
def substituteAndWrite(mapping):
template = args['template']
if not os.path.exists(template):
sys.exit("Template file '" + template + "' could not be found")
with open(args['outfile'], 'w') as ymlFile:
raw = string.Template(open(template).read())
ymlFile.write(raw.substitute(**mapping))
commandIndentation = ' '*args['indentation']
with open(args['outfile'], 'w') as ymlFile:
def wrapDuneControl(command):
return 'dunecontrol --opts=$DUNE_OPTS_FILE --current ' + command
def makeYamlList(commands, indent=commandIndentation):
commands = [indent + '- ' + comm for comm in commands]
return '\n'.join(commands)
# if no configuration is given, build and run all tests (skip select stage)
if not args['affectedtestsonly']:
buildCommand = [wrapDuneControl('all'),
wrapDuneControl('bexec make -k -j4 build_tests')]
testCommand = [wrapDuneControl('bexec dune-ctest'
' -j4 --output-on-failure')]
substituteAndWrite({'build_script': makeYamlList(buildCommand),
'test_script': makeYamlList(testCommand),
'stages': makeYamlList(['build', 'test'], ' '),
'test_select_job': '',
'build_needs': ''})
# otherwise, add a stage that detects the tests to be run first
else:
selectStageName = 'configure'
selectJobName = 'select tests'
stages = makeYamlList([selectStageName, 'build', 'test'], ' ')
buildNeeds = '\n'.join([' needs:',
' - job: {}'.format(selectJobName),
' artifacts: true'])
selectJob = '\n'.join([selectJobName + ':',
' stage: {}'.format(selectStageName),
' script:'])
selectJob += '\n'
selectJob += makeYamlList([wrapDuneControl('all'),
'pushd build-cmake',
'python3 ../bin/testing/findtests.py'
' -f ../affectedtests.json'
' -t origin/master',
'popd'])
selectJob += '\n'
selectJob += '\n'.join([' artifacts:',
' paths:',
' - affectedtests.json',
' expire_in: 3 hours'])
buildCommand = [wrapDuneControl('all'),
'cp affectedtests.json build-cmake',
'pushd build-cmake',
'python3 ../bin/testing/runselectedtests.py '
' -c affectedtests.json -b',
'popd']
testCommand = [wrapDuneControl('all'),
'pushd build-cmake',
'python3 ../bin/testing/runselectedtests.py '
' -c affectedtests.json -t',
'popd']
substituteAndWrite({'build_script': makeYamlList(buildCommand),
'test_script': makeYamlList(testCommand),
'stages': stages,
'test_select_job': selectJob,
'build_needs': buildNeeds})
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment