Commit 56a4ade7 authored by Timo Koch's avatar Timo Koch
Browse files

Merge branch 'feature/continuous-master-pipeline-status' into 'master'

Feature/continuous master pipeline status

See merge request !2644
parents f950c025 34c2513d
stages:
- check-status
- trigger
- downstream modules
......@@ -10,7 +11,36 @@ variables:
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_COMMIT_BRANCH == "master"
# for commits happening on master, we check if there was a successful
# pipeline on a related merge request already. If yes, we simply return
# to propagate that pipeline status on master. Otherwise, we trigger a new run.
check-pipeline-status:
image: $IMAGE_REGISTRY_URL/full:dune-2.7-gcc-ubuntu-20.04
stage: check-status
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: never
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_COMMIT_BRANCH == "master"
when: always
script:
- |
if ! python3 .gitlab-ci/getpipelineinfo.py --access-token $CI_JOB_TOKEN \
--look-for HEAD \
--print-format pipeline-id; then
echo "No successful pipeline found. Triggering new pipeline..."
curl --request POST --form "token=$CI_JOB_TOKEN" \
--form ref=$CI_COMMIT_BRANCH \
--form "variables[CI_TEST_AGAINST_LAST_SUCCESSFUL]=true" \
"https://git.iws.uni-stuttgart.de/api/v4/projects/31/trigger/pipeline"
else
echo "Found successful pipeline for the current state of the branch. Not testing again."
fi
###################################################################################
# Stage 1: trigger the Dumux test pipelines #
......@@ -28,10 +58,14 @@ workflow:
strategy: depend
variables:
TRIGGER_SOURCE: $CI_PIPELINE_SOURCE
COMMIT_BRANCH: $CI_COMMIT_BRANCH
MR_TARGET_BRANCH_NAME: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
TEST_AGAINST_LAST_SUCCESSFUL: $CI_TEST_AGAINST_LAST_SUCCESSFUL
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: always
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
when: manual
......@@ -39,7 +73,7 @@ workflow:
.non-mr-trigger:
extends: .base-trigger
rules:
- if: $CI_PIPELINE_SOURCE != "merge_request_event"
- if: $CI_PIPELINE_SOURCE == "schedule"
#############################################
# pipelines to be created in merge requests #
......@@ -77,6 +111,10 @@ full-dune-master-clang:
# trigger lecture test
trigger lecture:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
stage: downstream modules
trigger:
project: dumux-repositories/dumux-lecture
......
......@@ -10,22 +10,56 @@ workflow:
rules:
- if: $CI_PIPELINE_SOURCE=="parent_pipeline"
# variables that may be overwritten by the trigger
# variables that should be overwritten by the trigger
variables:
TRIGGER_SOURCE: "undefined"
COMMIT_BRANCH: "undefined"
MR_TARGET_BRANCH_NAME: "undefined"
select tests:
stage: configure
script:
- dunecontrol --opts=$DUNE_OPTS_FILE --current all
- |
dunecontrol --opts=$DUNE_OPTS_FILE --current all
getLastSuccesful() {
python3 .gitlab-ci/getpipelineinfo.py \
--access-token $CI_JOB_TOKEN \
--look-for latest \
--print-format commit-sha
}
if [[ "$TRIGGER_SOURCE" == "merge_request_event" ]]; then
echo "Detecting changes w.r.t to target branch '$MR_TARGET_BRANCH_NAME'"
python3 bin/testing/getchangedfiles.py -o changedfiles.txt -t origin/$MR_TARGET_BRANCH_NAME
python3 bin/testing/findtests.py -o affectedtests.json --file-list changedfiles.txt --build-dir build-cmake
python3 bin/testing/getchangedfiles.py --outfile changedfiles.txt \
--target.tree origin/$MR_TARGET_BRANCH_NAME
python3 bin/testing/findtests.py --outfile affectedtests.json \
--file-list changedfiles.txt \
--build-dir build-cmake
elif [[ "$TRIGGER_SOURCE" == "schedule" ]]; then
echo "Starting scheduled pipeline"
echo "Skipping test selection, build/test stages will consider all tests!"
touch affectedtests.json
elif [ -n $TEST_AGAINST_LAST_SUCCESSFUL ]; then
echo "Determining sha of the last successful pipeline to test against"
if ! getLastSuccesful; then
echo "Could not find a successful pipeline, will build/run all tests"
touch affectedtests.json
else
COMMIT_SHA=$(getLastSuccesful)
echo "Comparing against sha $COMMIT_SHA"
python3 bin/testing/getchangedfiles.py --outfile changedfiles.txt \
--source-tree HEAD \
--target-tree $COMMIT_SHA
python3 bin/testing/findtests.py --outfile affectedtests.json \
--file-list changedfiles.txt \
--build-dir build-cmake
fi
else
echo "Received '$TRIGGER_SOURCE' as pipeline trigger event"
echo "Unknown pipeline trigger event"
echo "Skipping test selection, build/test stages will consider all tests!"
touch affectedtests.json
fi
......
import json
import sys
import os
from argparse import ArgumentParser
try:
path = os.path.split(os.path.abspath(__file__))[0]
sys.path.append(os.path.join(path, '../bin/util'))
from common import runCommand
except Exception:
sys.exit('Could not import common module')
def performApiQuery(command, err='API query unsuccessful'):
return runCommand(command, suppressTraceBack=True, errorMessage=err)
def getPipeLinesApiURL(apiURL):
return apiURL.rstrip('/') + '/pipelines/'
def getPipelines(apiURL, token, filter=''):
queryURL = getPipeLinesApiURL(apiURL) + filter
queryCmd = 'curl --header "token={}" "{}"'.format(token, queryURL)
pl = performApiQuery(queryCmd, 'Could not retrieve pipelines')
return json.loads(pl)
def getPipelineInfo(apiURL, token, pipeLineId, infoString):
queryURL = getPipeLinesApiURL(apiURL) + str(pipeLineId) + '/' + infoString
queryCmd = 'curl --header "token={}" "{}"'.format(token, queryURL)
pl = performApiQuery(queryCmd, 'Could not retrieve pipeline info')
return json.loads(pl)
def getPipeLineJobs(apiURL, token, pipeLineId):
return getPipelineInfo(apiURL, token, pipeLineId, 'jobs')
def findPipeline(pipeLines, predicate):
for pipeLine in pipeLines:
if predicate(pipeLine):
return pipeLine
return None
parser = ArgumentParser(
description='Find and print information on a previously run pipeline'
)
parser.add_argument('-p', '--print-format',
required=True, choices=['pipeline-id', 'commit-sha'],
help='Switch between reporting the pipeline-id/commit-sha')
parser.add_argument('-l', '--look-for',
required=True, choices=['latest', 'HEAD'],
help='Define how to search for pipelines')
parser.add_argument('-t', '--access-token',
required=True,
help='The token to post read requests to the GitLab API')
parser.add_argument('-u', '--project-api-url',
required=False,
default='https://git.iws.uni-stuttgart.de/api/v4/projects/31/',
help='The token to post read requests to the GitLab API')
parser.add_argument('-f', '--filter',
required=False, default='?status=success',
help='Pipeline query filter (default: "?status=success"')
parser.add_argument('-e', '--exclude-jobs',
required=False, nargs='*',
default=['check-pipeline-status'],
help='Exclude pipelines that contain the given jobs')
args = vars(parser.parse_args())
apiURL = args['project_api_url']
token = args['access_token']
pipeLines = getPipelines(apiURL, token, args['filter'])
currentBranch = runCommand('git branch --show-current').strip('\n')
currentCommitInfo = runCommand('git show HEAD').split('\n')
headIsMergeCommit = 'Merge:' in currentCommitInfo[1]
headSHA = runCommand('git rev-list HEAD --max-count=1').strip('\n')
if headIsMergeCommit:
preSHA = runCommand('git rev-list HEAD --max-count=2').split('\n')[1]
mrSHA = currentCommitInfo[1].split()[2]
def checkBranch(pipeLine):
return pipeLine['ref'] == currentBranch
def checkCommit(pipeLine):
sha = pipeLine['sha']
if headIsMergeCommit:
return sha == headSHA or sha == preSHA or mrSHA in sha
return sha == headSHA
def skip(pipeLine):
jobs = getPipeLineJobs(apiURL, token, pipeLine['id'])
jobNames = [j['name'] for j in jobs]
return any(j in jobNames for j in args['exclude_jobs'])
if args['look_for'] == 'HEAD':
pipeLine = findPipeline(
pipeLines, lambda p: checkCommit(p) and not skip(p)
)
elif args['look_for'] == 'latest':
pipeLine = findPipeline(
pipeLines, lambda p: checkBranch(p) and not skip(p)
)
if pipeLine is not None:
if args['print_format'] == 'pipeline-id':
print(pipeLine['id'])
elif args['print_format'] == 'commit-sha':
print(pipeLine['sha'])
else:
sys.exit('Could not find a succesful pipeline')
......@@ -2,24 +2,38 @@ import os
import sys
import functools
import subprocess
import traceback
def getCommandErrorHints(command):
if "git " in command:
return "It seems that a git command failed. Please check:\n" \
" -- is the module registered as git repository?\n" \
" -- is upstream defined for the branch?"
return None
# execute a command and retrieve the output
def runCommand(command):
def runCommand(command, suppressTraceBack=False, errorMessage=''):
try:
return subprocess.run(command, shell=True, check=True,
text=True, capture_output=True).stdout
except Exception as e:
print()
print("An error occurred during subprocess run:")
print("-- command: {}".format(command))
print("-- folder: {}".format(os.getcwd()))
print("-- error: {}".format(sys.exc_info()[1]))
if "git " in command:
print()
print("It seems that a git command failed. Please check:\n" \
" -- is the module registered as git repository?\n" \
" -- is upstream defined for the branch?\n")
raise
return subprocess.run(command,
shell=True, check=True,
text=True, capture_output=True).stdout
except Exception:
eType, eValue, eTraceback = sys.exc_info()
if suppressTraceBack:
traceback.print_exception(eType, eType(errorMessage), None)
elif errorMessage:
traceback.print_exception(eType, eType(errorMessage), eTraceback)
else:
print("An error occurred during subprocess run:")
print("-- command: {}".format(command))
print("-- folder: {}".format(os.getcwd()))
traceback.print_exception(eType, eValue, eTraceback)
hints = getCommandErrorHints(command)
if hints is not None:
print(hints)
# decorator to call function from within the given path
def callFromPath(path):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment