From 6c707e3875704f4d7128a68f3684fea42f87dd6f Mon Sep 17 00:00:00 2001
From: Lorenzo Pagliai <lorenzo.pagliai@seco.com>
Date: Thu, 15 Dec 2022 14:54:46 +0100
Subject: [PATCH] Include integrate_into_layer.py script
* The script is derived from the integrate_into_manifest file and is
used to create the integration branch on layers
* The script create a new branch with the modifications on the
SRCREV.conf indicated as parameter and on the bitbake recipe indicated
* In a future this file could be merged with the integrate_inot_manifest
one
---
scripts/integrate_into_layer.py | 233 ++++++++++++++++++++++++++++++++
1 file changed, 233 insertions(+)
create mode 100755 scripts/integrate_into_layer.py
diff --git a/scripts/integrate_into_layer.py b/scripts/integrate_into_layer.py
new file mode 100755
index 0000000..03e4e82
--- /dev/null
+++ b/scripts/integrate_into_layer.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import tempfile
+import re
+from pathlib import Path
+from furl import furl
+from git import GitCommandError, Repo
+from gitlab import Gitlab
+from gitlab.v4.objects import Project
+from lxml import etree
+
+def update_srcrev(srcrev, recipe_name, new_revision):
+ # Check if project is referenced in SRCREV.conf
+ # Match "...RECIPE_NAME ="
+ pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+
+ project_line = None
+ for line in srcrev.splitlines():
+ if pattern.search(line):
+ project_line = line
+ break
+ if project_line is None:
+ return None
+
+ # Get current project revision from SRCREV file
+ # Assuming notation: <project> = "<hash>"
+ old_revision = project_line.split('"')[1]
+
+ # Update SRCREV file
+ srcrev = srcrev.replace(old_revision, new_revision)
+ return srcrev
+
+
+def integrate_into_layer(
+ layer_project: Project,
+ layer_branch,
+ srcrev_file,
+ recipe_name,
+ project: Project,
+ merge_request,
+):
+ gitlab = layer_project.manager.gitlab
+
+ with tempfile.TemporaryDirectory() as layer_dir:
+ srcrev_filepath = Path(layer_dir) / srcrev_file
+
+ # Construct clone url containing access token
+ clone_url = furl(layer_project.http_url_to_repo)
+ print("This is the url to clone", clone_url.url)
+ clone_url.username = "gitlab-ci"
+ clone_url.password = gitlab.private_token
+
+ # Checkout layer
+ # TODO replace checkout with gitlab api access
+ print("Cloning layer repo: %s" % layer_project.http_url_to_repo)
+ try:
+ layer_repo = Repo.clone_from(
+ clone_url.url, layer_dir, branch=layer_branch
+ )
+ except GitCommandError as e:
+ sys.exit("ERROR: could not clone layer repository\n" + str(e))
+ except IndexError:
+ sys.exit("ERROR: branch '%s' not found" % layer_branch)
+
+ # Special handling for the gitlab-ci integration
+ # When the branch 'merge_request.source_branch' already starts with
+ # integrate/gitlab-ci we add our new commit to this branch
+ # Otherwise (normal behaviour) a new integration branch is created
+ integration_branch = common.find_gitlab_ci_integration_branch(
+ layer_repo, merge_request.source_branch
+ )
+
+ if integration_branch is not None:
+ layer_repo.git.checkout(
+ "-b", integration_branch, "origin/{}".format(integration_branch)
+ )
+ logging.debug("Heads: %s", layer_repo.heads)
+ layer_repo.heads[integration_branch].checkout()
+ logging.debug(layer_repo.git.log("--oneline", "-n", "5"))
+ print("Using existing integration branch: %s" % integration_branch)
+ else:
+ # Create integration branch (delete former one if already exists)
+ integration_branch = recipe_name + "/" + merge_request.source_branch
+ for ref in layer_repo.references:
+ if integration_branch == ref.name:
+ layer_repo.delete_head(ref)
+
+ print("Creating integration branch: %s" % integration_branch)
+ layer_repo.head.set_reference(
+ layer_repo.create_head(integration_branch)
+ )
+ with open('integration_branch_file', "w", encoding="utf-8") as file:
+ file.write(integration_branch)
+ # Get new project revision from merge request
+ new_revision = merge_request.sha
+
+ with open(srcrev_filepath, "r", encoding="utf8") as fp:
+ srcrev = fp.read()
+ new_srcrev = update_srcrev(srcrev, recipe_name, new_revision)
+ # write file
+ if new_srcrev is None:
+ sys.exit(
+ "ERROR: project '%s' not found in layer and "
+ "no recipe name is specified" % project.path
+ )
+ with open(srcrev_filepath.as_posix(), "w", encoding="utf8") as fp:
+ fp.write(new_srcrev)
+ layer_repo.index.add([srcrev_file])
+
+ # Make an API request to create the gitlab.user object
+ gitlab.auth()
+
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s\n%s" % (
+ project.path,
+ merge_request.source_branch,
+ common.list_commits(merge_request.commits()),
+ )
+ layer_revision = common.commit_and_push(
+ layer_project,
+ layer_repo,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ )
+ logging.debug("New revision in layer: %s", layer_revision)
+
+ return layer_revision
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token for the layer group""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--layer-project",
+ help="""name of the yocto layer project""",
+ dest="layer_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--layer-branch",
+ help="""yocto layer branch to branch off from""",
+ dest="layer_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--srcrev-file",
+ help="""source revision file name (default: 'SRCREV.conf')""",
+ dest="srcrev_file",
+ default=common.srcrev_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--recipe-name",
+ help="""recipe name to resolve project in 'SRCREV.conf'""",
+ dest="recipe_name",
+ default=None,
+ required=False,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the project, as specified in the layer""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--merge-request",
+ help="""project merge request IID or link containing the
+ changes to be integrated""",
+ dest="merge_request",
+ required=True,
+ )
+ parser.add_argument(
+ "--save-revision-to",
+ help="""path to a file where the new layer revision is stored""",
+ dest="revision_file",
+ required=False,
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ help="""Increase verbosity.""",
+ )
+
+ args, _ = parser.parse_known_args()
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+
+ logging.debug(args)
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ layer_project = common.get_project(gitlab, args.layer_project)
+
+ project = common.get_project(gitlab, args.project)
+ logging.debug("Project: %s", project.name)
+ logging.debug("Merge Request: %s", args.merge_request)
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+ merge_request = common.get_merge_request(project, args.merge_request)
+ if merge_request is None:
+ sys.exit("ERROR: could not get %s %s" % (project.name, args.merge_request))
+
+ layer_revision = integrate_into_layer(
+ layer_project=layer_project,
+ layer_branch=args.layer_branch,
+ srcrev_file=args.srcrev_file,
+ recipe_name=args.recipe_name,
+ project=project,
+ merge_request=merge_request,
+ )
+
+ if args.revision_file:
+ with open(args.revision_file, "w", encoding="utf-8") as file:
+ file.write(layer_revision)
+
+if __name__ == "__main__":
+ main()
--
GitLab