deploy_gitlab_ci: remove hard-coded integration targets

Get targets dynamically using the INTEGRATION variable instead.

.gitlab-ci.yml

@@ -67,42 +67,19 @@ yamllint:
 # ---------------------------------------------------------------------------------------
 .ci-test-projects:
   variables:
-    PROJECT_ROOT:
+    PROJECT_GROUP:
       ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
     MANIFEST_PROJECT:
-      ${PROJECT_ROOT}/minimal-manifest
-
-    INTEGRATE_INTO:
-      ${PROJECT_ROOT}/minimal-foo
-      ${PROJECT_ROOT}/minimal-bar
-      ${PROJECT_ROOT}/minimal-srcrev
+      ${PROJECT_GROUP}/minimal-manifest
+    MANIFEST_BRANCH: primary
 
 .yocto-projects:
   variables:
-    PROJECT_ROOT:
-      ${CI_PROJECT_ROOT_NAMESPACE}
+    PROJECT_GROUP:
+      ${CI_PROJECT_ROOT_NAMESPACE}/yocto
     MANIFEST_PROJECT:
-      ${PROJECT_ROOT}/yocto/manifest
-    INTEGRATE_INTO:
-      ${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk
-      ${PROJECT_ROOT}/kernel/linux-guf
-      ${PROJECT_ROOT}/kernel/linux-imx-kuk
-      ${PROJECT_ROOT}/kernel/modules/egalaxi2c
-      ${PROJECT_ROOT}/kernel/modules/gfplatdetect
-      ${PROJECT_ROOT}/tools/gf-emc-test-suite
-      ${PROJECT_ROOT}/tools/gf-productiontests
-      ${PROJECT_ROOT}/tools/gfeeprom
-      ${PROJECT_ROOT}/tools/gfxml2dto
-      ${PROJECT_ROOT}/tools/guf-show-demo
-      ${PROJECT_ROOT}/tools/libmdb
-      ${PROJECT_ROOT}/tools/touchcal-conv
-      ${PROJECT_ROOT}/tools/xconfig
-      ${PROJECT_ROOT}/tools/qt-multi-screen-compositor
-      ${PROJECT_ROOT}/yocto/config
-      ${PROJECT_ROOT}/yocto/custom/dual-espresso/meta-seconorth-dual-espresso
-      ${PROJECT_ROOT}/yocto/layers/meta-seconorth-distro
-      ${PROJECT_ROOT}/yocto/layers/meta-seconorth-machine
-      ${PROJECT_ROOT}/yocto/layers/meta-seconorth-nogplv3
+      ${PROJECT_ROOT}/manifest
+    MANIFEST_BRANCH: dunfell
 
 .integrate:
   stage: integrate
@@ -118,11 +95,12 @@ yamllint:
       --gitlab-url=${CI_SERVER_URL}
       --token=${GITBOT_TOKEN}
       --manifest-project=${MANIFEST_PROJECT}
+      --manifest-branch=${MANIFEST_BRANCH}
       --submodule=.gitlab-ci
       --revision=${CI_COMMIT_SHA}
+      --group=${PROJECT_GROUP}
       --verbose
       ${MERGE}
-      ${INTEGRATE_INTO}
 
 integrate-yocto:
   extends:

scripts/deploy_gitlab_ci.py

@@ -9,6 +9,7 @@ from gitlab import Gitlab
 
 from accept_merge_request import accept_merge_request
 from create_merge_request import create_merge_request
+from get_integrating_projects import get_integrating_projects
 from get_merge_requests import get_merge_requests
 from update_submodule import (
     update_submodule_and_include_ref,
@@ -108,10 +109,16 @@ def main():
         required=True,
     )
     parser.add_argument(
-        "--project",
         "--manifest-project",
-        help="""name of the GitLab project""",
-        dest="project",
+        help="""name of the manifest project""",
+        dest="manifest_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--manifest-branch",
+        help="""manifest branch to integrate changes into""",
+        dest="manifest_branch",
+        required=True,
     )
     parser.add_argument(
         "--submodule",
@@ -125,13 +132,6 @@ def main():
         dest="revision",
         required=True,
     )
-    parser.add_argument(
-        "--branch",
-        help="""project branch (if not default branch)""",
-        dest="branch",
-        required=False,
-        default=None,
-    )
     parser.add_argument(
         "--merge",
         help="""if set, perform merge after integration""",
@@ -155,10 +155,10 @@ def main():
         required=False,
     )
     parser.add_argument(
-        "projects",
-        help="""List of projects the change should be deployed to additionally
-                to the manifest project given as named parameter.""",
-        nargs="*",
+        "--group",
+        help="""group path or id to limit search scope to""",
+        dest="group",
+        required=True,
     )
     parser.add_argument(
         "-v",
@@ -176,37 +176,41 @@ def main():
         )
 
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+    group = gitlab.groups.get(args.group)
 
     # =======================================================
     # Create integration branches and commits with updates
     # submodule in all projects
     # =======================================================
     project_integration = {}
+    projects = get_integrating_projects(
+        args.manifest_project, args.manifest_branch, group
+    )
     # Update submodule in all 'child' project
-    for p in args.projects:
-        print("Create integration commit in", p)
+    for p in projects:
+        print("Create integration commit in", p["project"])
 
         res = integrate_submodule_into(
-            gitlab, p, args.submodule, args.revision, args.branch
+            gitlab, p["project"], args.submodule, args.revision, p["branch"]
         )
         # Store in the list if commit is set (meaning there was an update or
         #   an exising integration branch)
         if res["commit"] is not None:
-            project_integration[p] = res
+            project_integration[p["project"]] = res
 
-    print("Create integration commit in", args.project)
+    print("Create integration commit in", args.manifest_project)
     # Update submodule in manifest project
     manifest_project = integrate_submodule_into(
         gitlab,
-        args.project,
+        args.manifest_project,
         args.submodule,
         args.revision,
-        args.branch,
+        args.manifest_branch,
         commit_and_push=False,
         force_clone=True,
     )
 
-    branch = args.branch
+    branch = args.manifest_branch
     if branch is None:
         branch = manifest_project["project"].default_branch
     # =======================================================
@@ -349,7 +353,7 @@ def main():
 
     print(
         "Successfully create integration commit {} in {}".format(
-            integration_commit, args.project
+            integration_commit, args.manifest_project
         )
     )
 
@@ -369,7 +373,7 @@ def main():
     # =================================================
     # The manifest needs to be merged at last
     mr = manifest_project["mr"]
-    logging.debug("Merge %s!%s", args.project, mr.iid)
+    logging.debug("Merge %s!%s", args.manifest_project, mr.iid)
 
     # Wait until GitLab has checked merge status
     common.wait_until_merge_status_is_set(manifest_project["project"], mr)

scripts/get_integrating_projects.py

+#!/usr/bin/env python3
+import argparse
+import re
+import sys
+from gitlab import Gitlab, GitlabGetError
+from gitlab.v4.objects import Group
+
+
+def get_integrating_projects(manifest_project: str, manifest_branch: str, group: Group):
+    """
+    Get a list of projects in the given group which are automatically integrated into
+    the given branch of the given manifest project.
+    """
+    projects = []
+    gitlab = group.manager.gitlab
+
+    # Recurse into subgroups
+    for g in group.subgroups.list():
+        subgroup = gitlab.groups.get(g.id)
+        projects += get_integrating_projects(
+            manifest_project, manifest_branch, subgroup
+        )
+
+    # Regex to check INTEGRATION variable against
+    regex = f":{manifest_project}:{manifest_branch}$"
+
+    for project in group.projects.list():
+        try:
+            project = gitlab.projects.get(project.id)
+            if not project.archived:
+                integrations = project.variables.get("INTEGRATION").value
+                for integration in integrations.splitlines():
+                    if re.search(regex, integration):
+                        source_branch = integration.split(":")[0]
+                        projects.append(
+                            {
+                                "project": project.path_with_namespace,
+                                "branch": source_branch,
+                            }
+                        )
+        except GitlabGetError as e:
+            if e.response_code == 404:  # not found
+                pass
+            elif e.response_code == 403:  # forbidden
+                sys.exit(
+                    (
+                        "ERROR: could not get INTEGRATION variable of project %s\n"
+                        % project.path_with_namespace
+                    )
+                    + e.error_message
+                )
+            else:
+                raise
+
+    return projects
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        required=True,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--manifest-project",
+        help="""name of the manifest project""",
+        dest="manifest_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--manifest-branch",
+        help="""manifest branch""",
+        dest="manifest_branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--group",
+        help="""group path or id to limit search scope to""",
+        dest="group",
+        required=True,
+    )
+
+    args, _ = parser.parse_known_args()
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+    group = gitlab.groups.get(args.group)
+
+    projects = get_integrating_projects(
+        args.manifest_project,
+        args.manifest_branch,
+        group,
+    )
+
+    for project in projects:
+        print(project)
+
+
+if __name__ == "__main__":
+    main()