CI: Create integration commits in deploy_gitlab_ci directly

Split out functions to change manifest and srcrev from integrate_into_manifest
and reuse it in deploy_gitlab_ci

deploy_gitlab_ci.py

@@ -12,10 +12,31 @@ from create_merge_request import create_merge_request
 from get_merge_requests import get_merge_requests
 from update_submodule import update_submodule
 from update_gitlab_ci import update_gitlab_ci_include
+from integrate_into_manifest import update_manifest, update_srcrev
 
 from ruamel.yaml import YAML
 
 
+def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
+
+    # Read values from existing file
+    yaml = YAML()
+    data = yaml.load(gitlab_ci_yml)
+    logging.debug("Yaml: %s", data)
+
+    try:
+        masterbranch = data["variables"]["MASTER_BRANCH_PROJECT"]
+        logging.debug("Masterbranch %s", masterbranch)
+    except KeyError:
+        masterbranch = None
+    try:
+        recipe = data["variables"]["BB_RECIPE_NAME"]
+        logging.debug("Recipe %s", recipe)
+    except KeyError:
+        recipe = None
+    return {"recipe": recipe, "masterbranch": masterbranch}
+
+
 def create_gitlab_ci_yml(repo, gitlab_ci_yml):
     """This code snippet was used to initially populate
     all repos with the minial .gitlab-ci.yml that
@@ -209,6 +230,20 @@ def main():
         required=False,
         default=False,
     )
+    parser.add_argument(
+        "--manifest-file",
+        help="""manifest file name (default: 'default.xml')""",
+        dest="manifest_file",
+        default=common.manifest_file,
+        required=False,
+    )
+    parser.add_argument(
+        "--srcrev-file",
+        help="""source revision file name (default: 'SRCREV.conf')""",
+        dest="srcrev_file",
+        default=common.srcrev_file,
+        required=False,
+    )
     parser.add_argument(
         "projects",
         help="""List of projects the change should be deployed to additionally
@@ -254,9 +289,77 @@ def main():
             )
 
     # If submodule is already at specified revision in all projects, exit successfully
-    if not len(project_integration) == 0:
+    if len(project_integration) == 0:
+        print("No integration done, changes are already included in all projects.")
         sys.exit(0)
 
+    # Now create the integration commit in the manifest
+    # for all subprojects at once
+    manifest_project = project_integration[args.project]
+    manifest = common.get_repository_file_raw(
+        manifest_project["project"], args.manifest_file, ref=manifest_project["branch"]
+    )
+    logging.debug(manifest)
+    srcrev = common.get_repository_file_raw(
+        manifest_project["project"], args.srcrev_file, ref=manifest_project["branch"]
+    )
+    logging.debug(srcrev)
+
+    for p in args.projects:
+        integration = project_integration[p]
+        logging.debug(
+            "Update %s to %s", integration["project"].name, integration["commit"]
+        )
+
+        new_manifest = update_manifest(
+            manifest, integration["project"], integration["commit"]
+        )
+        if new_manifest is not None:
+            manifest = new_manifest
+            logging.debug(manifest)
+            continue
+
+        # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
+        gitlab_ci_yml = common.get_repository_file_raw(
+            integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
+        )
+        project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
+
+        new_srcrev = update_srcrev(
+            srcrev, project_keys["recipe"], integration["commit"]
+        )
+        if new_srcrev is not None:
+            srcrev = new_srcrev
+            logging.debug(srcrev)
+        else:
+            logging.debug("Project %s not found in xml or srcrev file", p)
+
+    # Now manifest and srcrev contain the updated revisions created above
+    # Send them to gitlab as patch
+    patch = {
+        "branch": manifest_project["branch"],
+        "commit_message": "This is a test",  # TODO
+        "actions": [
+            {
+                "action": "update",
+                "file_path": args.manifest_file,
+                "content": manifest,
+            },
+            {
+                "action": "update",
+                "file_path": args.srcrev_file,
+                "content": srcrev,
+            },
+        ],
+    }
+    logging.debug("Send patch: %s", patch)
+    commit = manifest_project["project"].commits.create(
+        patch, retry_transient_errors=True
+    )
+    logging.debug("Create new commit: %s", commit)
+
+    sys.exit(0)
+
     if not args.merge:
         print(
             "Skipping automatic merge in MR context. If you like to extend the "

integrate_into_manifest.py

@@ -10,9 +10,66 @@ from pathlib import Path
 from furl import furl
 from git import GitCommandError, Repo
 from gitlab import Gitlab
+from gitlab.v4.objects import Project
 from lxml import etree
 
 
+def update_manifest(manifest, project: Project, new_revision):
+    """Returns updated version of the manifest or None id project_name
+    was not found in the manifest
+    """
+    # Parse manifest
+    try:
+        manifestxml = etree.fromstring(manifest.encode())
+    except etree.XMLSyntaxError:
+        sys.exit("ERROR: Failed to parse given manifest")
+
+    # Find project reference in manifest
+    # We are using str.endswith() for this in order to support sub-projects as well
+    # (e.g."mygroup/myproject", when only "myproject" is given)
+    project_node = None
+    project_nodes = manifestxml.findall("project")
+    for node in project_nodes:
+        name = node.get("name")
+        if name is not None and name.endswith(project.path):
+            project_node = node
+    if project_node is None:
+        return None
+
+    # Get current project revision from manifest
+    old_revision = project_node.get("revision")
+    logging.debug("Replace %s with %s", old_revision, new_revision)
+
+    # Update manifest file
+    # We are doing this using a plain text replace action. Unfortunately
+    # all python libraries for handling XML data are not able to preserve
+    # the file layout, and we want a minimal diff.
+    manifest = manifest.replace(old_revision, new_revision)
+    return manifest
+
+
+def update_srcrev(srcrev, recipe_name, new_revision):
+    # Check if project is referenced in SRCREV.conf
+    # Match "...RECIPE_NAME ="
+    pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+
+    project_line = None
+    for line in srcrev.splitlines():
+        if pattern.search(line):
+            project_line = line
+            break
+    if project_line is None:
+        return None
+
+    # Get current project revision from SRCREV file
+    # Assuming notation: <project> = "<hash>"
+    old_revision = project_line.split('"')[1]
+
+    # Update SRCREV file
+    srcrev = srcrev.replace(old_revision, new_revision)
+    return srcrev
+
+
 def integrate_into_manifest(
     manifest_project,
     integration_base,
@@ -34,6 +91,7 @@ def integrate_into_manifest(
         clone_url.password = gitlab.private_token
 
         # Checkout manifest
+        # TODO replace checkout with gitlab api access
         try:
             manifest_repo = Repo.clone_from(
                 clone_url.url, manifest_dir, branch=integration_base
@@ -71,68 +129,42 @@ def integrate_into_manifest(
             manifest_repo.head.set_reference(
                 manifest_repo.create_head(integration_branch)
             )
+        # Get new project revision from merge request
+        new_revision = merge_request.sha
 
         # Parse manifest file
         try:
-            manifest = etree.parse(manifest_filepath.as_posix())
+            with open(manifest_filepath.as_posix(), "r", encoding="utf8") as fp:
+                manifest = fp.read_text()
         except FileNotFoundError:
             sys.exit("ERROR: file '%s' not found in manifest repo" % manifest_file)
 
-        # Find project reference in manifest
-        # We are using str.endswith() for this in order to support sub-projects as well
-        # (e.g."mygroup/myproject", when only "myproject" is given)
-        project_node = None
-        project_nodes = manifest.findall("project")
-        for node in project_nodes:
-            name = node.get("name")
-            if name is not None and name.endswith(project.path):
-                project_node = node
-        if project_node is None:
+        new_manifest = update_manifest(manifest, project, new_revision)
+        if new_manifest is not None:
+            # write file
+            with open(manifest_filepath.as_posix(), "w", encoding="utf8") as fp:
+                fp.write_text(new_manifest)
+            manifest_repo.index.add([manifest_file])
+        else:
+            # Look for project in SRCREV as it has not been found in the manifest
             if recipe_name is None:
                 sys.exit(
                     "ERROR: project '%s' not found in manifest and "
                     "no recipe name is specified" % project.path
                 )
-            # Check if project is referenced in SRCREV.conf
-            project_line = None
-            content = srcrev_filepath.read_text()
-            # Match "...RECIPE_NAME ="
-            pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
-            for line in content.splitlines():
-                if pattern.search(line):
-                    project_line = line
-            if project_line is None:
+
+            with open(srcrev_filepath, "r", encoding="utf8") as fp:
+                srcrev = fp.read_text()
+            new_srcrev = update_srcrev(srcrev, recipe_name, new_revision)
+            # write file
+            if new_srcrev is None:
                 sys.exit(
                     "ERROR: project '%s' not found in manifest and "
-                    "SRCREV file" % project.path
+                    "no recipe name is specified" % project.path
                 )
-
-            # Get current project revision from SRCREV file
-            # Assuming notation: <project> = "<hash>"
-            old_revision = project_line.split('"')[1]
-
-            # Get new project revision from merge request
-            new_revision = merge_request.sha
-
-            # Update SRCREV file
-            content = content.replace(old_revision, new_revision)
-            srcrev_filepath.write_text(content)
+            with open(srcrev_filepath.as_posix(), "w", encoding="utf8") as fp:
+                fp.write_text(new_manifest)
             manifest_repo.index.add([srcrev_file])
-        else:
-            # Get current project revision from manifest
-            old_revision = project_node.get("revision")
-
-            # Get new project revision from merge request
-            new_revision = merge_request.sha
-
-            # Update manifest file
-            # We are doing this using a plain text replace action. Unfortunately
-            # all python libraries for handling XML data are not able to preserve
-            # the file layout, and we want a minimal diff.
-            content = manifest_filepath.read_text()
-            content = content.replace(old_revision, new_revision)
-            manifest_filepath.write_text(content)
-            manifest_repo.index.add([manifest_file])
 
         # Make an API request to create the gitlab.user object
         gitlab.auth()