[ci][getpipelineinfo] allow exlusion of pls with certain jobs

.gitlab-ci.yml

@@ -30,11 +30,7 @@ check-pipeline-status:
       when: always
   script:
     - |
-      curl --header "token=$CI_JOB_TOKEN" \
-                    "https://git.iws.uni-stuttgart.de/api/v4/projects/31/pipelines/?status=success" \
-                    > pipeline_status.json
-    - |
-      if ! python3 .gitlab-ci/getpipelineinfo.py --status-file pipeline_status.json \
+      if ! python3 .gitlab-ci/getpipelineinfo.py --access-token $CI_JOB_TOKEN \
                                                  --look-for HEAD \
                                                  --print-format pipeline-id; then
           echo "No successful pipeline found. Triggering new pipeline..."

.gitlab-ci/default.yml

@@ -23,7 +23,7 @@ select tests:
     - |
       getLastSuccesful() {
           python3 .gitlab-ci/getpipelineinfo.py \
-                      --status-file $1 \
+                      --access-token $CI_JOB_TOKEN \
                       --look-for latest \
                       --print-format commit-sha
       }
@@ -43,15 +43,12 @@ select tests:
 
       elif [ -n $TEST_AGAINST_LAST_SUCCESSFUL ]; then
           echo "Determining sha of the last successful pipeline to test against"
-          curl --header "token=$CI_JOB_TOKEN" \
-                        "https://git.iws.uni-stuttgart.de/api/v4/projects/31/pipelines/?status=success" \
-                        > pipeline_status.json
 
-          if ! getLastSuccesful pipeline_status.json; then
+          if ! getLastSuccesful; then
               echo "Could not find a successful pipeline, will build/run all tests"
               touch affectedtests.json
           else
-              COMMIT_SHA=$(getLastSuccesful pipeline_status.json)
+              COMMIT_SHA=$(getLastSuccesful)
               echo "Comparing against sha $COMMIT_SHA"
               python3 bin/testing/getchangedfiles.py --outfile changedfiles.txt \
                                                      --source-tree HEAD \

.gitlab-ci/getpipelineinfo.py

@@ -11,11 +11,36 @@ except Exception:
     sys.exit('Could not import common module')
 
 
-def findPipeline(pipelineFile, predicate):
-    with open(pipelineFile) as pipeLines:
-        for pipeLine in json.load(pipeLines):
-            if predicate(pipeLine):
-                return pipeLine
+def performApiQuery(command, err='API query unsuccessful'):
+    return runCommand(command, suppressTraceBack=True, errorMessage=err)
+
+
+def getPipeLinesApiURL(apiURL):
+    return apiURL.rstrip('/') + '/pipelines/'
+
+
+def getPipelines(apiURL, token, filter=''):
+    queryURL = getPipeLinesApiURL(apiURL) + filter
+    queryCmd = 'curl --header "token={}" "{}"'.format(token, queryURL)
+    pl = performApiQuery(queryCmd, 'Could not retrieve pipelines')
+    return json.loads(pl)
+
+
+def getPipelineInfo(apiURL, token, pipeLineId, infoString):
+    queryURL = getPipeLinesApiURL(apiURL) + str(pipeLineId) + '/' + infoString
+    queryCmd = 'curl --header "token={}" "{}"'.format(token, queryURL)
+    pl = performApiQuery(queryCmd, 'Could not retrieve pipeline info')
+    return json.loads(pl)
+
+
+def getPipeLineJobs(apiURL, token, pipeLineId):
+    return getPipelineInfo(apiURL, token, pipeLineId, 'jobs')
+
+
+def findPipeline(pipeLines, predicate):
+    for pipeLine in pipeLines:
+        if predicate(pipeLine):
+            return pipeLine
     return None
 
 
@@ -23,20 +48,32 @@ parser = ArgumentParser(
     description='Find and print information on a previously run pipeline'
 )
 
-parser.add_argument('-s', '--status',
-                    required=False, default='success',
-                    help='Define the status of the pipeline to be found')
 parser.add_argument('-p', '--print-format',
                     required=True, choices=['pipeline-id', 'commit-sha'],
                     help='Switch between reporting the pipeline-id/commit-sha')
 parser.add_argument('-l', '--look-for',
                     required=True, choices=['latest', 'HEAD'],
                     help='Define how to search for pipelines')
-parser.add_argument('-f', '--status-file',
+parser.add_argument('-t', '--access-token',
                     required=True,
-                    help='The .json file with the latest pipeline statuses')
+                    help='The token to post read requests to the GitLab API')
+parser.add_argument('-u', '--project-api-url',
+                    required=False,
+                    default='https://git.iws.uni-stuttgart.de/api/v4/projects/31/',
+                    help='The token to post read requests to the GitLab API')
+parser.add_argument('-f', '--filter',
+                    required=False, default='?status=success',
+                    help='Pipeline query filter (default: "?status=success"')
+parser.add_argument('-e', '--exclude-jobs',
+                    required=False, nargs='*',
+                    default=['check-pipeline-status'],
+                    help='Exclude pipelines that contain the given jobs')
 args = vars(parser.parse_args())
 
+apiURL = args['project_api_url']
+token = args['access_token']
+pipeLines = getPipelines(apiURL, token, args['filter'])
+
 currentBranch = runCommand('git branch --show-current').strip('\n')
 currentCommitInfo = runCommand('git show HEAD').split('\n')
 headIsMergeCommit = 'Merge:' in currentCommitInfo[1]
@@ -44,25 +81,34 @@ headIsMergeCommit = 'Merge:' in currentCommitInfo[1]
 headSHA = runCommand('git rev-list HEAD --max-count=1').strip('\n')
 if headIsMergeCommit:
     preSHA = runCommand('git rev-list HEAD --max-count=2').split('\n')[1]
-    mrSha = currentCommitInfo[1].split()[2]
+    mrSHA = currentCommitInfo[1].split()[2]
+
+
+def checkBranch(pipeLine):
+    return pipeLine['ref'] == currentBranch
 
 
-def checkCommitSHA(pipeLine):
+def checkCommit(pipeLine):
     sha = pipeLine['sha']
     if headIsMergeCommit:
-        return sha == headSHA or sha == preSHA or mrSha in sha
+        return sha == headSHA or sha == preSHA or mrSHA in sha
     return sha == headSHA
 
 
-def checkStatus(pipeLine): return pipeLine['status'] == args['status']
-def checkBranch(pipeLine): return pipeLine['ref'] == currentBranch
-def find(predicate): return findPipeline(args['status_file'], predicate)
+def skip(pipeLine):
+    jobs = getPipeLineJobs(apiURL, token, pipeLine['id'])
+    jobNames = [j['name'] for j in jobs]
+    return any(j in jobNames for j in args['exclude_jobs'])
 
 
 if args['look_for'] == 'HEAD':
-    pipeLine = find(lambda p: checkStatus(p) and checkCommitSHA(p))
+    pipeLine = findPipeline(
+        pipeLines, lambda p: checkCommit(p) and not skip(p)
+    )
 elif args['look_for'] == 'latest':
-    pipeLine = find(lambda p: checkStatus(p) and checkBranch(p))
+    pipeLine = findPipeline(
+        pipeLines, lambda p: checkBranch(p) and not skip(p)
+    )
 
 if pipeLine is not None:
     if args['print_format'] == 'pipeline-id':