From 43c45b3bdda7f5b41693e444a6fa8a582c7d3b30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dennis=20Gl=C3=A4ser?= <dennis.glaeser@iws.uni-stuttgart.de>
Date: Wed, 14 Apr 2021 18:54:02 +0200
Subject: [PATCH] [ci] make test selection part of test pipeline
---
.gitlab-ci.yml | 4 +-
.gitlab-ci/default.yml.template | 6 +-
.gitlab-ci/makepipelineconfig.py | 137 ++++++++++++++++---------------
bin/testing/runselectedtests.py | 0
4 files changed, 76 insertions(+), 71 deletions(-)
mode change 100644 => 100755 bin/testing/runselectedtests.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 008eea11a8..a6587d0097 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -27,9 +27,7 @@ generate-config:
script:
- |
if [ $CI_PIPELINE_SOURCE == "merge_request_event" ]; then
- dunecontrol --opts=$DUNE_OPTS_FILE --current all
- pushd build-cmake && python3 ../bin/testing/findtests.py -f affectedtests.json -t origin/master && popd
- python3 .gitlab-ci/makepipelineconfig.py -c build-cmake/affectedtests.json -o generated-config.yml
+ python3 .gitlab-ci/makepipelineconfig.py -o generated-config.yml --affectedtestsonly
else
python3 .gitlab-ci/makepipelineconfig.py -o generated-config.yml
fi
diff --git a/.gitlab-ci/default.yml.template b/.gitlab-ci/default.yml.template
index 29095ff05d..90135ea3f4 100644
--- a/.gitlab-ci/default.yml.template
+++ b/.gitlab-ci/default.yml.template
@@ -2,13 +2,14 @@ default:
image: $$IMAGE
stages:
- - build
- - test
+${stages}
workflow:
rules:
- if: $$CI_PIPELINE_SOURCE=="parent_pipeline"
+${test_select_job}
+
build dumux:
stage: build
script:
@@ -17,6 +18,7 @@ ${build_script}
paths:
- build-cmake
expire_in: 3 hours
+${build_needs}
test dumux:
stage: test
diff --git a/.gitlab-ci/makepipelineconfig.py b/.gitlab-ci/makepipelineconfig.py
index 14e7221a6e..d5ea63bb13 100644
--- a/.gitlab-ci/makepipelineconfig.py
+++ b/.gitlab-ci/makepipelineconfig.py
@@ -2,7 +2,6 @@
import os
import sys
-import json
import string
from argparse import ArgumentParser
@@ -11,15 +10,22 @@ if sys.version_info.major < 3:
sys.exit('Python 3 required')
parser = ArgumentParser(description='Generate dumux test pipeline .yml file')
-parser.add_argument('-o', '--outfile', required=True,
+parser.add_argument('-o', '--outfile',
+ required=True,
help='Specify the file to write the pipeline definition')
-parser.add_argument('-c', '--testconfig', required=False,
- help='Specify a test configuration file containing the '
- 'tests that should be run within the test pipeline')
-parser.add_argument('-t', '--template', required=False,
+parser.add_argument('-a', '--affectedtestsonly',
+ required=False,
+ action='store_true',
+ help='Use this flag to create a pipeline that runs only '
+ 'those tests that are affected by changes w.r.t to '
+ 'the origin/master branch')
+parser.add_argument('-t', '--template',
+ required=False,
default='.gitlab-ci/default.yml.template',
help='Specify the template .yml file to be used')
-parser.add_argument('-i', '--indentation', required=False, default=4,
+parser.add_argument('-i', '--indentation',
+ required=False,
+ default=4,
help='Specify the indentation for the script commands')
args = vars(parser.parse_args())
@@ -37,69 +43,68 @@ def substituteAndWrite(mapping):
commandIndentation = ' '*args['indentation']
-duneConfigCommand = 'dunecontrol --opts=$DUNE_OPTS_FILE --current all'
with open(args['outfile'], 'w') as ymlFile:
- def makeScriptString(commands):
- commands = [commandIndentation + '- ' + comm for comm in commands]
- return '\n'.join(commands)
-
- def makeMultiLineCommand(commandParts):
+ def wrapDuneControl(command):
+ return 'dunecontrol --opts=$DUNE_OPTS_FILE --current ' + command
- # add indentation to each part
- result = [commandIndentation + ' ' + cp for cp in commandParts]
-
- # add line break token at the end of each part
- result = ' \\\n'.join(result)
+ def makeYamlList(commands, indent=commandIndentation):
+ commands = [indent + '- ' + comm for comm in commands]
+ return '\n'.join(commands)
- # add multiline trigger token at the beginning
- return '|\n' + result
+ # if no configuration is given, build and run all tests (skip select stage)
+ if not args['affectedtestsonly']:
+ buildCommand = [wrapDuneControl('all'),
+ wrapDuneControl('bexec make -k -j4 build_tests')]
+ testCommand = [wrapDuneControl('bexec dune-ctest'
+ ' -j4 --output-on-failure')]
- # if no configuration is given, build and run all tests
- if not args['testconfig']:
- buildCommand = [duneConfigCommand,
- 'dunecontrol --opts=$DUNE_OPTS_FILE --current '
- 'make -k -j4 build_tests']
- testCommand = ['cd build-cmake', 'dune-ctest -j4 --output-on-failure']
+ substituteAndWrite({'build_script': makeYamlList(buildCommand),
+ 'test_script': makeYamlList(testCommand),
+ 'stages': makeYamlList(['build', 'test'], ' '),
+ 'test_select_job': '',
+ 'build_needs': ''})
- # otherwise, parse data from the given configuration file
+ # otherwise, add a stage that detects the tests to be run first
else:
- with open(args['testconfig']) as configFile:
- config = json.load(configFile)
-
- testNames = list(config.keys())
- targetNames = [tc['target'] for tc in config.values()]
-
- if not targetNames:
- buildCommand = [duneConfigCommand,
- 'echo "No tests to be built."']
- else:
- # The MakeFile generated by cmake contains .NOTPARALLEL,
- # as it only allows a single call to `CMakeFiles/Makefile2`.
- # Parallelism is taken care of within that latter Makefile.
- # We let the script create a small custom makeFile here on top
- # of `Makefile2`, defining a new target to be built in parallel
- buildCommand = [
- duneConfigCommand,
- 'cd build-cmake',
- 'rm -f TestMakefile && touch TestMakefile',
- 'echo "include CMakeFiles/Makefile2" >> TestMakefile',
- 'echo "" >> TestMakefile',
- makeMultiLineCommand(['echo "build_selected_tests:"']
- + targetNames
- + ['>> TestMakefile']),
- 'make -f TestMakefile -j4 build_selected_tests']
-
- if not testNames:
- testCommand = ['echo "No tests to be run, make empty report."',
- 'cd build-cmake',
- 'dune-ctest -R NOOP']
- else:
- testCommand = ['cd build-cmake',
- makeMultiLineCommand(['dune-ctest -j4 '
- '--output-on-failure '
- '-R']
- + testNames)]
-
- substituteAndWrite({'build_script': makeScriptString(buildCommand),
- 'test_script': makeScriptString(testCommand)})
+ selectStageName = 'configure'
+ selectJobName = 'select tests'
+
+ stages = makeYamlList([selectStageName, 'build', 'test'], ' ')
+ buildNeeds = '\n'.join([' needs:',
+ ' - job: {}'.format(selectJobName),
+ ' artifacts: true'])
+
+ selectJob = '\n'.join([selectJobName + ':',
+ ' stage: {}'.format(selectStageName),
+ ' script:'])
+ selectJob += '\n'
+ selectJob += makeYamlList([wrapDuneControl('all'),
+ 'pushd build-cmake',
+ 'python3 ../bin/testing/findtests.py'
+ ' -f ../affectedtests.json'
+ ' -t origin/master',
+ 'popd'])
+ selectJob += '\n'
+ selectJob += '\n'.join([' artifacts:',
+ ' paths:',
+ ' - affectedtests.json',
+ ' expire_in: 3 hours'])
+
+ buildCommand = [wrapDuneControl('all'),
+ 'cp affectedtests.json build-cmake',
+ 'pushd build-cmake',
+ 'python3 ../bin/testing/runselectedtests.py '
+ ' -c affectedtests.json -b',
+ 'popd']
+ testCommand = [wrapDuneControl('all'),
+ 'pushd build-cmake',
+ 'python3 ../bin/testing/runselectedtests.py '
+ ' -c affectedtests.json -t',
+ 'popd']
+
+ substituteAndWrite({'build_script': makeYamlList(buildCommand),
+ 'test_script': makeYamlList(testCommand),
+ 'stages': stages,
+ 'test_select_job': selectJob,
+ 'build_needs': buildNeeds})
diff --git a/bin/testing/runselectedtests.py b/bin/testing/runselectedtests.py
old mode 100644
new mode 100755
--
GitLab