From 1e3136cd92a943b65831571f36504fb59b287695 Mon Sep 17 00:00:00 2001
From: Tim Jaacks <tim.jaacks@garz-fricke.com>
Date: Tue, 16 Aug 2022 17:03:11 +0200
Subject: [PATCH] deploy_gitlab_ci: remove hard-coded integration targets
Get targets dynamically using the INTEGRATION variable instead.
---
.gitlab-ci.yml | 40 +++--------
scripts/deploy_gitlab_ci.py | 52 +++++++-------
scripts/get_integrating_projects.py | 107 ++++++++++++++++++++++++++++
3 files changed, 144 insertions(+), 55 deletions(-)
create mode 100755 scripts/get_integrating_projects.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d2b7d363..b6ea9fcc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -67,42 +67,19 @@ yamllint:
# ---------------------------------------------------------------------------------------
.ci-test-projects:
variables:
- PROJECT_ROOT:
+ PROJECT_GROUP:
${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
MANIFEST_PROJECT:
- ${PROJECT_ROOT}/minimal-manifest
-
- INTEGRATE_INTO:
- ${PROJECT_ROOT}/minimal-foo
- ${PROJECT_ROOT}/minimal-bar
- ${PROJECT_ROOT}/minimal-srcrev
+ ${PROJECT_GROUP}/minimal-manifest
+ MANIFEST_BRANCH: primary
.yocto-projects:
variables:
- PROJECT_ROOT:
- ${CI_PROJECT_ROOT_NAMESPACE}
+ PROJECT_GROUP:
+ ${CI_PROJECT_ROOT_NAMESPACE}/yocto
MANIFEST_PROJECT:
- ${PROJECT_ROOT}/yocto/manifest
- INTEGRATE_INTO:
- ${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk
- ${PROJECT_ROOT}/kernel/linux-guf
- ${PROJECT_ROOT}/kernel/linux-imx-kuk
- ${PROJECT_ROOT}/kernel/modules/egalaxi2c
- ${PROJECT_ROOT}/kernel/modules/gfplatdetect
- ${PROJECT_ROOT}/tools/gf-emc-test-suite
- ${PROJECT_ROOT}/tools/gf-productiontests
- ${PROJECT_ROOT}/tools/gfeeprom
- ${PROJECT_ROOT}/tools/gfxml2dto
- ${PROJECT_ROOT}/tools/guf-show-demo
- ${PROJECT_ROOT}/tools/libmdb
- ${PROJECT_ROOT}/tools/touchcal-conv
- ${PROJECT_ROOT}/tools/xconfig
- ${PROJECT_ROOT}/tools/qt-multi-screen-compositor
- ${PROJECT_ROOT}/yocto/config
- ${PROJECT_ROOT}/yocto/custom/dual-espresso/meta-seconorth-dual-espresso
- ${PROJECT_ROOT}/yocto/layers/meta-seconorth-distro
- ${PROJECT_ROOT}/yocto/layers/meta-seconorth-machine
- ${PROJECT_ROOT}/yocto/layers/meta-seconorth-nogplv3
+ ${PROJECT_ROOT}/manifest
+ MANIFEST_BRANCH: dunfell
.integrate:
stage: integrate
@@ -118,11 +95,12 @@ yamllint:
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
+ --manifest-branch=${MANIFEST_BRANCH}
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
+ --group=${PROJECT_GROUP}
--verbose
${MERGE}
- ${INTEGRATE_INTO}
integrate-yocto:
extends:
diff --git a/scripts/deploy_gitlab_ci.py b/scripts/deploy_gitlab_ci.py
index b05e7707..e673d61d 100755
--- a/scripts/deploy_gitlab_ci.py
+++ b/scripts/deploy_gitlab_ci.py
@@ -9,6 +9,7 @@ from gitlab import Gitlab
from accept_merge_request import accept_merge_request
from create_merge_request import create_merge_request
+from get_integrating_projects import get_integrating_projects
from get_merge_requests import get_merge_requests
from update_submodule import (
update_submodule_and_include_ref,
@@ -108,10 +109,16 @@ def main():
required=True,
)
parser.add_argument(
- "--project",
"--manifest-project",
- help="""name of the GitLab project""",
- dest="project",
+ help="""name of the manifest project""",
+ dest="manifest_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-branch",
+ help="""manifest branch to integrate changes into""",
+ dest="manifest_branch",
+ required=True,
)
parser.add_argument(
"--submodule",
@@ -125,13 +132,6 @@ def main():
dest="revision",
required=True,
)
- parser.add_argument(
- "--branch",
- help="""project branch (if not default branch)""",
- dest="branch",
- required=False,
- default=None,
- )
parser.add_argument(
"--merge",
help="""if set, perform merge after integration""",
@@ -155,10 +155,10 @@ def main():
required=False,
)
parser.add_argument(
- "projects",
- help="""List of projects the change should be deployed to additionally
- to the manifest project given as named parameter.""",
- nargs="*",
+ "--group",
+ help="""group path or id to limit search scope to""",
+ dest="group",
+ required=True,
)
parser.add_argument(
"-v",
@@ -176,37 +176,41 @@ def main():
)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ group = gitlab.groups.get(args.group)
# =======================================================
# Create integration branches and commits with updates
# submodule in all projects
# =======================================================
project_integration = {}
+ projects = get_integrating_projects(
+ args.manifest_project, args.manifest_branch, group
+ )
# Update submodule in all 'child' project
- for p in args.projects:
- print("Create integration commit in", p)
+ for p in projects:
+ print("Create integration commit in", p["project"])
res = integrate_submodule_into(
- gitlab, p, args.submodule, args.revision, args.branch
+ gitlab, p["project"], args.submodule, args.revision, p["branch"]
)
# Store in the list if commit is set (meaning there was an update or
# an exising integration branch)
if res["commit"] is not None:
- project_integration[p] = res
+ project_integration[p["project"]] = res
- print("Create integration commit in", args.project)
+ print("Create integration commit in", args.manifest_project)
# Update submodule in manifest project
manifest_project = integrate_submodule_into(
gitlab,
- args.project,
+ args.manifest_project,
args.submodule,
args.revision,
- args.branch,
+ args.manifest_branch,
commit_and_push=False,
force_clone=True,
)
- branch = args.branch
+ branch = args.manifest_branch
if branch is None:
branch = manifest_project["project"].default_branch
# =======================================================
@@ -349,7 +353,7 @@ def main():
print(
"Successfully create integration commit {} in {}".format(
- integration_commit, args.project
+ integration_commit, args.manifest_project
)
)
@@ -369,7 +373,7 @@ def main():
# =================================================
# The manifest needs to be merged at last
mr = manifest_project["mr"]
- logging.debug("Merge %s!%s", args.project, mr.iid)
+ logging.debug("Merge %s!%s", args.manifest_project, mr.iid)
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(manifest_project["project"], mr)
diff --git a/scripts/get_integrating_projects.py b/scripts/get_integrating_projects.py
new file mode 100755
index 00000000..45513c3f
--- /dev/null
+++ b/scripts/get_integrating_projects.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+import argparse
+import re
+import sys
+from gitlab import Gitlab, GitlabGetError
+from gitlab.v4.objects import Group
+
+
+def get_integrating_projects(manifest_project: str, manifest_branch: str, group: Group):
+ """
+ Get a list of projects in the given group which are automatically integrated into
+ the given branch of the given manifest project.
+ """
+ projects = []
+ gitlab = group.manager.gitlab
+
+ # Recurse into subgroups
+ for g in group.subgroups.list():
+ subgroup = gitlab.groups.get(g.id)
+ projects += get_integrating_projects(
+ manifest_project, manifest_branch, subgroup
+ )
+
+ # Regex to check INTEGRATION variable against
+ regex = f":{manifest_project}:{manifest_branch}$"
+
+ for project in group.projects.list():
+ try:
+ project = gitlab.projects.get(project.id)
+ if not project.archived:
+ integrations = project.variables.get("INTEGRATION").value
+ for integration in integrations.splitlines():
+ if re.search(regex, integration):
+ source_branch = integration.split(":")[0]
+ projects.append(
+ {
+ "project": project.path_with_namespace,
+ "branch": source_branch,
+ }
+ )
+ except GitlabGetError as e:
+ if e.response_code == 404: # not found
+ pass
+ elif e.response_code == 403: # forbidden
+ sys.exit(
+ (
+ "ERROR: could not get INTEGRATION variable of project %s\n"
+ % project.path_with_namespace
+ )
+ + e.error_message
+ )
+ else:
+ raise
+
+ return projects
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-project",
+ help="""name of the manifest project""",
+ dest="manifest_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-branch",
+ help="""manifest branch""",
+ dest="manifest_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--group",
+ help="""group path or id to limit search scope to""",
+ dest="group",
+ required=True,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ group = gitlab.groups.get(args.group)
+
+ projects = get_integrating_projects(
+ args.manifest_project,
+ args.manifest_branch,
+ group,
+ )
+
+ for project in projects:
+ print(project)
+
+
+if __name__ == "__main__":
+ main()
--
GitLab