Retrigger MRs from all integrating projects

Instead of getting the projects from the manifest file, search them via
their INTEGRATION variable.

Improve log messages on this occasion: print URL instead of just IDs.

manifest-pipeline-ci-test.yml

@@ -13,6 +13,9 @@ variables:
   # named differently, so we need a variable that may be overriden.
   CI_PARAM_SECO_REMOTE: ci-test
 
+  # GitLab group to search for projects to retrigger
+  RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
+
   BUILD_TIMEOUT: 2m
 
   # This is the jinja2 template file used to generate the build jobs

manifest-pipeline-yocto.yml

@@ -13,6 +13,9 @@ variables:
   # named differently, so we need a variable that may be overriden.
   CI_PARAM_SECO_REMOTE: seco-ne
 
+  # GitLab group to search for projects to retrigger
+  RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}
+
   BUILD_TIMEOUT: 1h
 
   # This is the jinja2 template file used to generate the build jobs

manifest-pipeline.yml

@@ -60,28 +60,12 @@ retrigger:
     - .short_master_pipeline
   stage: retrigger
   script:
-    - PROJECTS=$(
-        .gitlab-ci/scripts/get_manifest_projects.py
-        --manifest=default.xml
-        --remote=${CI_PARAM_SECO_REMOTE}
-        --concat-namespaces
-      )
-    # Add the gitlab-ci project
-    - PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci"
-    # TODO retrigger gitlab-ci integration also
-    # Retrigger also project in SRCREV
-    - echo -e "Projects:\n${PROJECTS}"
-    - for PROJECT in ${PROJECTS}; do
-        .gitlab-ci/scripts/retrigger_mr_pipeline_jobs.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${PROJECT}
-          --state=opened
-          --target-branch=${MASTER_BRANCH}
-          --job=check
-          --include-children
-        ;
-      done
+    - .gitlab-ci/scripts/retrigger_integrating_projects.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${CI_PROJECT_PATH}
+        --manifest-branch=${MASTER_BRANCH}
+        --group=${RETRIGGER_GROUP}
 
 # --------------------------------------------------------------------------------------
 # Stage: infrastructure

scripts/get_manifest_projects.py

-#!/usr/bin/env python3
-import argparse
-import sys
-from furl import furl
-from lxml import etree
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--manifest",
-        help="""manifest file to parse projects from""",
-        dest="manifest",
-        required=True,
-    )
-    parser.add_argument(
-        "--remote",
-        help="""get only projects with this remote""",
-        dest="remote",
-        required=False,
-    )
-    parser.add_argument(
-        "--concat-namespaces",
-        help="""parse namespace from fetch URL and prepend it to project names""",
-        dest="concat_namespaces",
-        action="store_true",
-        required=False,
-    )
-
-    args, _ = parser.parse_known_args()
-
-    # Parse manifest file
-    try:
-        manifest = etree.parse(args.manifest)
-    except FileNotFoundError:
-        sys.exit("ERROR: file '%s' not found" % args.manifest)
-
-    # Get namespace from remote
-    # This is needed for cases where the remote URL contains a part of the project
-    # namespace (e.g. "ssh://git@gitlab.com/garz-fricke/yocto") and the project name
-    # contains another part of it (e.g. "layers/meta-seconorth-machine"). There is no
-    # GitLab API call which will find a project given this information.
-    # Thus we are adding a possibility to parse the namespace from the remote and pre-
-    # pend it to the project name in order to return it including its complete namespace
-    # (e.g. "garz-fricke/yocto/layers/meta-seconorth-machine").
-    if args.concat_namespaces:
-        remote = manifest.find("remote[@name='%s']" % args.remote)
-        if remote is None:
-            sys.exit("ERROR: remote '%s' not found in manifest" % args.remote)
-        path = furl(remote.get("fetch")).path
-        prefix = str(path).strip("/") + "/"
-    else:
-        prefix = ""
-
-    # Find project references in manifest
-    if args.remote:
-        find_expression = "project[@remote='%s']" % args.remote
-    else:
-        find_expression = "project"
-    projects = manifest.findall(find_expression)
-
-    for project in projects:
-        print(prefix + project.get("name"))
-
-
-if __name__ == "__main__":
-    main()

scripts/retrigger_mr_pipeline_jobs.py → scripts/retrigger_integrating_projects.py

 #!/usr/bin/env python3
-import common
-
 import argparse
 import sys
 from gitlab import Gitlab, GitlabJobRetryError
 
+from get_integrating_projects import get_integrating_projects
 from get_merge_requests import get_merge_requests
 from retrigger_pipeline_jobs import retrigger_pipeline_jobs
 
 
 def main():
+    """
+    Retrigger a given job in all open MRs of projects in the given group with a target
+    branch that is configured for automatic integration into the given manifest project
+    and branch.
+    """
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--gitlab-url",
@@ -24,85 +28,81 @@ def main():
         required=True,
     )
     parser.add_argument(
-        "--project",
-        help="""name of the GitLab project""",
-        dest="project",
+        "--manifest-project",
+        help="""name of the GitLab manifest project""",
+        dest="manifest_project",
         required=True,
     )
     parser.add_argument(
-        "--state",
-        help="""state of the merge request (opened, closed, locked, or merged)""",
-        dest="state",
+        "--manifest-branch",
+        help="""target integration branch""",
+        dest="manifest_branch",
         required=True,
     )
     parser.add_argument(
-        "--source-branch",
-        help="""source branch of the merge request""",
-        dest="source_branch",
-        required=False,
-    )
-    parser.add_argument(
-        "--target-branch",
-        help="""target branch of the merge request""",
-        dest="target_branch",
-        required=False,
-    )
-    parser.add_argument(
-        "--commit",
-        help="""commit sha of the merge request""",
-        dest="commit",
-        required=False,
+        "--group",
+        help="""group path or id to limit search scope to""",
+        dest="group",
+        required=True,
     )
     parser.add_argument(
         "--job",
         help="""job to retrigger""",
         dest="job",
-        required=True,
+        required=False,
+        default="check",
     )
 
     args, _ = parser.parse_known_args()
 
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
-    project = common.get_project(gitlab, args.project)
+    group = gitlab.groups.get(args.group, retry_transient_errors=True)
 
-    mrs = get_merge_requests(
-        project,
-        state=args.state,
-        source_branch=args.source_branch,
-        target_branch=args.target_branch,
-        commit=args.commit,
+    projects = get_integrating_projects(
+        args.manifest_project, args.manifest_branch, group
     )
 
     failed = 0
-    for mr in mrs:
-        # Get pipeline
-        if not mr.pipeline:
-            print("No pipeline in !%s" % mr.iid)
-            continue
-        pipeline = project.pipelines.get(
-            mr.pipeline.get("id"),
-            retry_transient_errors=True,
+
+    for p in projects:
+
+        project = gitlab.projects.get(p["project"], retry_transient_errors=True)
+
+        mrs = get_merge_requests(
+            project,
+            state="opened",
+            target_branch=p["branch"],
         )
-        try:
-            jobs = retrigger_pipeline_jobs(
-                project,
-                pipeline,
-                args.job,
-                ["success", "running"],
-                include_children=True,
+
+        for mr in mrs:
+            # Get pipeline
+            if not mr.pipeline:
+                print("No pipeline in %s" % mr.web_url)
+                continue
+            pipeline = project.pipelines.get(
+                mr.pipeline.get("id"),
+                retry_transient_errors=True,
             )
-            if not jobs:
+            try:
+                jobs = retrigger_pipeline_jobs(
+                    project,
+                    pipeline,
+                    args.job,
+                    ["success", "running"],
+                    include_children=True,
+                )
+                if not jobs:
+                    print(
+                        "Could not find any jobs named '%s' in %s"
+                        % (args.job, pipeline.web_url)
+                    )
+            except GitlabJobRetryError as e:
                 print(
-                    "Could not find any jobs named '%s' in pipeline of !%s"
-                    % (args.job, mr.iid)
+                    "ERROR: Could not retrigger job '%s' in %s: %s"
+                    % (args.job, pipeline.web_url, e)
                 )
-        except GitlabJobRetryError as e:
-            print(
-                "ERROR: Could not retrigger job '%s' of %s!%s: %s"
-                % (args.job, args.project, mr.iid, e)
-            )
-            failed = failed + 1
-            continue
+                failed = failed + 1
+                continue
 
     if failed > 0:
         sys.exit(1)

scripts/retrigger_pipeline_jobs.py

@@ -47,7 +47,7 @@ def retrigger_pipeline_jobs(
 
     # Retrigger job
     job.retry()
-    print("Retrigger job '%s' of pipeline #%s:" % (job_name, pipeline.id))
+    print("Retrigger job '%s' in %s:" % (job_name, pipeline.web_url))
     job = project.jobs.get(job.id, retry_transient_errors=True)
     print(job.web_url)
     jobs.append(job)