From 25b37b9207dbb527c933885c3d08e496eba7d240 Mon Sep 17 00:00:00 2001
From: Lorenzo Pagliai <lorenzo.pagliai@seco.com>
Date: Thu, 15 Dec 2022 12:34:36 +0100
Subject: [PATCH] Insert new script for .gitlab-ci submodule integration
* The script is identical to the 'scripts/deploy_gitlab_ci_projects.py'
apart from the fact that does look for a manifest project to
integrate/merge
* In the future the two scripts may be merged to treat both cases
correctly, for now we skip this part
---
scripts/deploy_gitlab_ci_projects.py | 307 +++++++++++++++++++++++++++
1 file changed, 307 insertions(+)
create mode 100755 scripts/deploy_gitlab_ci_projects.py
diff --git a/scripts/deploy_gitlab_ci_projects.py b/scripts/deploy_gitlab_ci_projects.py
new file mode 100755
index 0000000..25d5f96
--- /dev/null
+++ b/scripts/deploy_gitlab_ci_projects.py
@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import os
+from gitlab import Gitlab
+from gitlab.v4.objects import Project, MergeRequest
+
+from accept_merge_request import accept_merge_request
+from create_merge_request import create_merge_request
+from get_integration_sources import get_integration_sources
+from get_merge_requests import get_merge_requests
+from update_submodule import update_submodule_and_include_ref
+from integrate_into_manifest import update_manifest, update_srcrev
+
+from ruamel.yaml import YAML
+
+
+def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
+
+ # Read values from existing file
+ yaml = YAML()
+ data = yaml.load(gitlab_ci_yml)
+ logging.debug("Yaml: %s", data)
+
+ try:
+ recipe = data["variables"]["BB_RECIPE_NAME"]
+ logging.debug("Recipe %s", recipe)
+ except KeyError:
+ recipe = None
+ return {"recipe": recipe}
+
+
+def integrate_submodule_into(
+ gitlab,
+ project_name,
+ submodule_name,
+ new_revision,
+ branch,
+ commit_and_push=True,
+ force_clone=False,
+):
+
+ gitlab_project = common.get_project(gitlab, project_name)
+
+ (
+ project_repo,
+ project_dir,
+ integration_branch_name,
+ integration_commit,
+ message,
+ ) = update_submodule_and_include_ref(
+ gitlab_project,
+ submodule_name,
+ new_revision,
+ branch,
+ commit_and_push=commit_and_push,
+ force_clone=force_clone,
+ )
+ # ======================================
+ # Store the references for creating the integration
+ # commit in the manifest later
+ # ======================================
+ ret = {
+ "project": gitlab_project,
+ "repo": project_repo,
+ "dir": project_dir,
+ "integration_branch": integration_branch_name,
+ "master_branch": branch,
+ "commit": integration_commit,
+ "message": message,
+ }
+ logging.debug(
+ "Integration branch: %s (%s)",
+ integration_branch_name,
+ integration_commit,
+ )
+ return ret
+
+
+def create_integration_merge_request(
+ project: Project,
+ integration_branch: str,
+ target_branch: str,
+ source_mr: MergeRequest = None,
+) -> MergeRequest:
+ # Create merge request
+ # This should be optional
+ mr, created = create_merge_request(project, integration_branch, target_branch)
+ if created:
+ if source_mr is not None:
+ common.crosslink_merge_requests(source_mr, mr)
+ print("Created new merge request:\n%s" % mr.web_url)
+ else:
+ print("Existing integration merge request:\n%s" % mr.web_url)
+ return mr
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-project",
+ help="""name of the manifest project""",
+ dest="manifest_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-branch",
+ help="""manifest branch to integrate changes into (can be a comma-separated list)""",
+ dest="manifest_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--submodule",
+ help="""submodule to update""",
+ dest="submodule",
+ required=True,
+ )
+ parser.add_argument(
+ "--revision",
+ help="""new revision for submodule""",
+ dest="revision",
+ required=True,
+ )
+ parser.add_argument(
+ "--merge",
+ help="""if set, perform merge after integration""",
+ dest="merge",
+ action="store_true",
+ required=False,
+ default=False,
+ )
+ parser.add_argument(
+ "--project",
+ help="""gitlab-ci project path or id""",
+ dest="project",
+ default=os.environ.get("CI_PROJECT_PATH"),
+ required=False,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""gitlab-ci branch that we're merging into""",
+ dest="branch",
+ default="master",
+ required=False,
+ )
+ parser.add_argument(
+ "--manifest-file",
+ help="""manifest file name (default: 'default.xml')""",
+ dest="manifest_file",
+ default=common.manifest_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--srcrev-file",
+ help="""source revision file name (default: 'SRCREV.conf')""",
+ dest="srcrev_file",
+ default=common.srcrev_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--group",
+ help="""group path or id to limit search scope to""",
+ dest="group",
+ required=True,
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ help="""Increase verbosity.""",
+ )
+
+ args, _ = parser.parse_known_args()
+ if args.verbose:
+ logging.basicConfig(
+ level=logging.DEBUG,
+ format="%(asctime)s %(levelname)-8s %(message)s",
+ datefmt="%H:%M:%S",
+ )
+
+ manifest_branches = args.manifest_branch.split(",")
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ group = gitlab.groups.get(args.group)
+
+ # =======================================================
+ # Create integration branches and commits with updates
+ # submodule in all projects
+ # =======================================================
+ integration_sources = {}
+ all_integration_sources = []
+ for manifest_branch in manifest_branches:
+ print(
+ "Searching for projects in %s that are configured for automatic integration into %s:%s"
+ % (args.group, args.manifest_project, manifest_branch)
+ )
+ integration_sources[manifest_branch] = get_integration_sources(
+ args.manifest_project, manifest_branch, group
+ )
+ for s in integration_sources[manifest_branch]:
+ if s not in all_integration_sources:
+ all_integration_sources.append(s)
+
+ # Update submodule in all integration sources
+ project_integrations = []
+ for s in all_integration_sources:
+ print("Create integration commit in %s:%s" % (s["project"], s["branch"]))
+
+ integration = integrate_submodule_into(
+ gitlab, s["project"], args.submodule, args.revision, s["branch"]
+ )
+ # Store in the list if commit is set (meaning there was an update or
+ # an exising integration branch)
+ if integration["commit"] is not None:
+ project_integrations.append(integration)
+
+ # Update submodule in all manifest branches
+ manifest_integrations = []
+ for manifest_branch in manifest_branches:
+ print(
+ "Create integration commit in %s:%s"
+ % (args.manifest_project, manifest_branch),
+ )
+ manifest_integrations.append(
+ integrate_submodule_into(
+ gitlab,
+ args.manifest_project,
+ args.submodule,
+ args.revision,
+ manifest_branch,
+ commit_and_push=False,
+ force_clone=True,
+ )
+ )
+
+ # =======================================================
+ # Create and merge merge_requests if needed
+ # =======================================================
+ if args.merge:
+ # Get source merge request (the one in the gitlab-ci repo)
+ gitlab_ci_project = common.get_project(gitlab, args.project)
+ mrs = get_merge_requests(
+ project=gitlab_ci_project,
+ target_branch=args.branch,
+ state="merged",
+ commit=args.revision,
+ )
+ if not mrs:
+ sys.exit(
+ "ERROR: could not determine source merge request for commit %s"
+ % args.revision
+ )
+ source_mr = mrs[0]
+
+ for project_integration in project_integrations:
+ logging.debug("Create MR in %s", project_integration["project"].name)
+ mr = create_integration_merge_request(
+ project_integration["project"],
+ project_integration["integration_branch"],
+ project_integration["master_branch"],
+ source_mr,
+ )
+ # Now merge
+ logging.debug("Merge %s!%s", project_integration["project"], mr.iid)
+
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(project_integration["project"], mr)
+
+ # Attempt to merge
+ merged, integration_commit = accept_merge_request(
+ project_integration["project"], mr, rebase=True
+ )
+ # if this has rebased the integration commit needs to be adapted:
+ project_integration["commit"] = integration_commit
+ # Save the target branch here, as the source branch gets deleted
+ # during merge
+ project_integration["integration_branch"] = mr.target_branch
+
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged:\n"
+ "%s\n"
+ "This can probably be resolved by creating a new commit in "
+ "gitlab-ci and merging it. The above MR can be closed then."
+ % mr.web_url
+ )
+
+ if not args.merge:
+ sys.exit(0)
+
+if __name__ == "__main__":
+ main()
--
GitLab