From 051871d93c7c6183ea611100d37adb56ad332a27 Mon Sep 17 00:00:00 2001 From: Tim Jaacks <tim.jaacks@garz-fricke.com> Date: Tue, 23 Aug 2022 16:43:34 +0200 Subject: [PATCH] Retrigger MRs from all integrating projects Instead of getting the projects from the manifest file, search them via their INTEGRATION variable. Improve log messages on this occasion: print URL instead of just IDs. --- manifest-pipeline-ci-test.yml | 3 + manifest-pipeline-yocto.yml | 3 + manifest-pipeline.yml | 28 ++---- scripts/get_manifest_projects.py | 67 ------------- scripts/retrigger_integrating_projects.py | 112 ++++++++++++++++++++++ scripts/retrigger_mr_pipeline_jobs.py | 112 ---------------------- scripts/retrigger_pipeline_jobs.py | 2 +- 7 files changed, 125 insertions(+), 202 deletions(-) delete mode 100755 scripts/get_manifest_projects.py create mode 100755 scripts/retrigger_integrating_projects.py delete mode 100755 scripts/retrigger_mr_pipeline_jobs.py diff --git a/manifest-pipeline-ci-test.yml b/manifest-pipeline-ci-test.yml index 9ecd5a27..c296a4af 100644 --- a/manifest-pipeline-ci-test.yml +++ b/manifest-pipeline-ci-test.yml @@ -13,6 +13,9 @@ variables: # named differently, so we need a variable that may be overriden. CI_PARAM_SECO_REMOTE: ci-test + # GitLab group to search for projects to retrigger + RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test + BUILD_TIMEOUT: 2m # This is the jinja2 template file used to generate the build jobs diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml index e0fc9a76..290c5ce3 100644 --- a/manifest-pipeline-yocto.yml +++ b/manifest-pipeline-yocto.yml @@ -13,6 +13,9 @@ variables: # named differently, so we need a variable that may be overriden. CI_PARAM_SECO_REMOTE: seco-ne + # GitLab group to search for projects to retrigger + RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE} + BUILD_TIMEOUT: 1h # This is the jinja2 template file used to generate the build jobs diff --git a/manifest-pipeline.yml b/manifest-pipeline.yml index ae394457..983a10c6 100644 --- a/manifest-pipeline.yml +++ b/manifest-pipeline.yml @@ -60,28 +60,12 @@ retrigger: - .short_master_pipeline stage: retrigger script: - - PROJECTS=$( - .gitlab-ci/scripts/get_manifest_projects.py - --manifest=default.xml - --remote=${CI_PARAM_SECO_REMOTE} - --concat-namespaces - ) - # Add the gitlab-ci project - - PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci" - # TODO retrigger gitlab-ci integration also - # Retrigger also project in SRCREV - - echo -e "Projects:\n${PROJECTS}" - - for PROJECT in ${PROJECTS}; do - .gitlab-ci/scripts/retrigger_mr_pipeline_jobs.py - --gitlab-url=${CI_SERVER_URL} - --token=${GITBOT_TOKEN} - --project=${PROJECT} - --state=opened - --target-branch=${MASTER_BRANCH} - --job=check - --include-children - ; - done + - .gitlab-ci/scripts/retrigger_integrating_projects.py + --gitlab-url=${CI_SERVER_URL} + --token=${GITBOT_TOKEN} + --manifest-project=${CI_PROJECT_PATH} + --manifest-branch=${MASTER_BRANCH} + --group=${RETRIGGER_GROUP} # -------------------------------------------------------------------------------------- # Stage: infrastructure diff --git a/scripts/get_manifest_projects.py b/scripts/get_manifest_projects.py deleted file mode 100755 index 03e5798c..00000000 --- a/scripts/get_manifest_projects.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import sys -from furl import furl -from lxml import etree - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--manifest", - help="""manifest file to parse projects from""", - dest="manifest", - required=True, - ) - parser.add_argument( - "--remote", - help="""get only projects with this remote""", - dest="remote", - required=False, - ) - parser.add_argument( - "--concat-namespaces", - help="""parse namespace from fetch URL and prepend it to project names""", - dest="concat_namespaces", - action="store_true", - required=False, - ) - - args, _ = parser.parse_known_args() - - # Parse manifest file - try: - manifest = etree.parse(args.manifest) - except FileNotFoundError: - sys.exit("ERROR: file '%s' not found" % args.manifest) - - # Get namespace from remote - # This is needed for cases where the remote URL contains a part of the project - # namespace (e.g. "ssh://git@gitlab.com/garz-fricke/yocto") and the project name - # contains another part of it (e.g. "layers/meta-seconorth-machine"). There is no - # GitLab API call which will find a project given this information. - # Thus we are adding a possibility to parse the namespace from the remote and pre- - # pend it to the project name in order to return it including its complete namespace - # (e.g. "garz-fricke/yocto/layers/meta-seconorth-machine"). - if args.concat_namespaces: - remote = manifest.find("remote[@name='%s']" % args.remote) - if remote is None: - sys.exit("ERROR: remote '%s' not found in manifest" % args.remote) - path = furl(remote.get("fetch")).path - prefix = str(path).strip("/") + "/" - else: - prefix = "" - - # Find project references in manifest - if args.remote: - find_expression = "project[@remote='%s']" % args.remote - else: - find_expression = "project" - projects = manifest.findall(find_expression) - - for project in projects: - print(prefix + project.get("name")) - - -if __name__ == "__main__": - main() diff --git a/scripts/retrigger_integrating_projects.py b/scripts/retrigger_integrating_projects.py new file mode 100755 index 00000000..512cb67d --- /dev/null +++ b/scripts/retrigger_integrating_projects.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +import argparse +import sys +from gitlab import Gitlab, GitlabJobRetryError + +from get_integrating_projects import get_integrating_projects +from get_merge_requests import get_merge_requests +from retrigger_pipeline_jobs import retrigger_pipeline_jobs + + +def main(): + """ + Retrigger a given job in all open MRs of projects in the given group with a target + branch that is configured for automatic integration into the given manifest project + and branch. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--gitlab-url", + help="""URL to the GitLab instance""", + dest="gitlab_url", + required=True, + ) + parser.add_argument( + "--token", + help="""GitLab REST API private access token""", + dest="token", + required=True, + ) + parser.add_argument( + "--manifest-project", + help="""name of the GitLab manifest project""", + dest="manifest_project", + required=True, + ) + parser.add_argument( + "--manifest-branch", + help="""target integration branch""", + dest="manifest_branch", + required=True, + ) + parser.add_argument( + "--group", + help="""group path or id to limit search scope to""", + dest="group", + required=True, + ) + parser.add_argument( + "--job", + help="""job to retrigger""", + dest="job", + required=False, + default="check", + ) + + args, _ = parser.parse_known_args() + + gitlab = Gitlab(args.gitlab_url, private_token=args.token) + group = gitlab.groups.get(args.group, retry_transient_errors=True) + + projects = get_integrating_projects( + args.manifest_project, args.manifest_branch, group + ) + + failed = 0 + + for p in projects: + + project = gitlab.projects.get(p["project"], retry_transient_errors=True) + + mrs = get_merge_requests( + project, + state="opened", + target_branch=p["branch"], + ) + + for mr in mrs: + # Get pipeline + if not mr.pipeline: + print("No pipeline in %s" % mr.web_url) + continue + pipeline = project.pipelines.get( + mr.pipeline.get("id"), + retry_transient_errors=True, + ) + try: + jobs = retrigger_pipeline_jobs( + project, + pipeline, + args.job, + ["success", "running"], + include_children=True, + ) + if not jobs: + print( + "Could not find any jobs named '%s' in %s" + % (args.job, pipeline.web_url) + ) + except GitlabJobRetryError as e: + print( + "ERROR: Could not retrigger job '%s' in %s: %s" + % (args.job, pipeline.web_url, e) + ) + failed = failed + 1 + continue + + if failed > 0: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/retrigger_mr_pipeline_jobs.py b/scripts/retrigger_mr_pipeline_jobs.py deleted file mode 100755 index 8879b2d1..00000000 --- a/scripts/retrigger_mr_pipeline_jobs.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -import common - -import argparse -import sys -from gitlab import Gitlab, GitlabJobRetryError - -from get_merge_requests import get_merge_requests -from retrigger_pipeline_jobs import retrigger_pipeline_jobs - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--gitlab-url", - help="""URL to the GitLab instance""", - dest="gitlab_url", - required=True, - ) - parser.add_argument( - "--token", - help="""GitLab REST API private access token""", - dest="token", - required=True, - ) - parser.add_argument( - "--project", - help="""name of the GitLab project""", - dest="project", - required=True, - ) - parser.add_argument( - "--state", - help="""state of the merge request (opened, closed, locked, or merged)""", - dest="state", - required=True, - ) - parser.add_argument( - "--source-branch", - help="""source branch of the merge request""", - dest="source_branch", - required=False, - ) - parser.add_argument( - "--target-branch", - help="""target branch of the merge request""", - dest="target_branch", - required=False, - ) - parser.add_argument( - "--commit", - help="""commit sha of the merge request""", - dest="commit", - required=False, - ) - parser.add_argument( - "--job", - help="""job to retrigger""", - dest="job", - required=True, - ) - - args, _ = parser.parse_known_args() - - gitlab = Gitlab(args.gitlab_url, private_token=args.token) - project = common.get_project(gitlab, args.project) - - mrs = get_merge_requests( - project, - state=args.state, - source_branch=args.source_branch, - target_branch=args.target_branch, - commit=args.commit, - ) - - failed = 0 - for mr in mrs: - # Get pipeline - if not mr.pipeline: - print("No pipeline in !%s" % mr.iid) - continue - pipeline = project.pipelines.get( - mr.pipeline.get("id"), - retry_transient_errors=True, - ) - try: - jobs = retrigger_pipeline_jobs( - project, - pipeline, - args.job, - ["success", "running"], - include_children=True, - ) - if not jobs: - print( - "Could not find any jobs named '%s' in pipeline of !%s" - % (args.job, mr.iid) - ) - except GitlabJobRetryError as e: - print( - "ERROR: Could not retrigger job '%s' of %s!%s: %s" - % (args.job, args.project, mr.iid, e) - ) - failed = failed + 1 - continue - - if failed > 0: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/scripts/retrigger_pipeline_jobs.py b/scripts/retrigger_pipeline_jobs.py index 21b9e263..c55fa86e 100755 --- a/scripts/retrigger_pipeline_jobs.py +++ b/scripts/retrigger_pipeline_jobs.py @@ -47,7 +47,7 @@ def retrigger_pipeline_jobs( # Retrigger job job.retry() - print("Retrigger job '%s' of pipeline #%s:" % (job_name, pipeline.id)) + print("Retrigger job '%s' in %s:" % (job_name, pipeline.web_url)) job = project.jobs.get(job.id, retry_transient_errors=True) print(job.web_url) jobs.append(job) -- GitLab