Draft: CI: Rewrite update_submodule using gitlab api: DOES NOT WORK!

deploy_gitlab_ci.py

@@ -4,14 +4,15 @@ import common
 import argparse
 import logging
 import sys
-import os
-from gitlab import Gitlab
+from gitlab import Gitlab, GitlabDeleteError
 
 # from accept_merge_request import accept_merge_request
 from create_merge_request import create_merge_request
 from get_merge_requests import get_merge_requests
 from update_submodule import update_submodule
-from update_gitlab_ci import update_gitlab_ci_include
+from update_submodule import get_submodule_project_path_and_revision
+
+# from update_gitlab_ci import update_gitlab_ci_include
 from integrate_into_manifest import update_manifest, update_srcrev
 
 from ruamel.yaml import YAML
@@ -37,6 +38,40 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
     return {"recipe": recipe, "masterbranch": masterbranch}
 
 
+def update_gitlab_ci_include(content, include_project, new_revision):
+
+    # Remove the SECO-Northern-Europe part from the project filter
+    # as it is normally specified by $CI_PROJECT_ROOT_NAMESPACE
+    include_project = include_project.split("/", 1)[1]
+
+    yaml = YAML()
+    data = yaml.load(content)
+    logging.debug("Yaml: %s", data)
+    try:
+        includes = data["include"]
+    except KeyError:
+        logging.debug("No include statement found")
+        return None
+    current_revision = None
+    for entry in includes:
+        try:
+            if include_project in entry["project"]:
+                current_revision = entry["ref"]
+                break
+        except KeyError:
+            logging.debug("Failed to parse include statement")
+            return None
+    if current_revision is None:
+        logging.debug("Failed to find %s in include statement", include_project)
+        return None
+
+    # Use plain replacement to keep the content of the file
+    # Yes, this may fail if the 'current_revision' is used multiple
+    # time is this fail. But probably this will not ever happen
+    logging.debug("Replace %s with %s", current_revision, new_revision)
+    return content.replace(current_revision, new_revision)
+
+
 def create_gitlab_ci_yml(repo, gitlab_ci_yml):
     """This code snippet was used to initially populate
     all repos with the minial .gitlab-ci.yml that
@@ -106,25 +141,7 @@ include:
     # ======================================
 
 
-def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
-
-    # Add changed revision also to .gitlab-ci.yml
-    gitlab_ci_yml = os.path.join(repo.working_tree_dir, ".gitlab-ci.yml")
-
-    if update_gitlab_ci_include(
-        gitlab_ci_yml,
-        submodule_project.web_url.split("//")[1].split("/", 1)[1],
-        submodule_revision,
-    ):
-        repo.git.add(gitlab_ci_yml)
-
-    with open(gitlab_ci_yml, "r", encoding="utf8") as fp:
-        logging.debug(fp.read())
-
-
-def deploy_into(
-    project, submodule, revision, branch, replace_existing_branch=False, create_mr=False
-):
+def deploy_into(project, submodule, revision, branch, create_mr=False):
     """Update the submodule and include refs to the submodule in the given project.
         Create mergerequest if needed.
 
@@ -133,12 +150,151 @@ def deploy_into(
         submodule_name (string): The name of the submodule to pull
         submodule_revision (hex string): The sha hash of the commit to update the submodule to
         branch (string): branch to update, if None, the projects default branch is used
-        replace_existing_branch: When an existing integration branch is found it is always replaced.
 
     Returns: tuple of:
         branch (string): Name of the newly created integration branch
         merge_request (gitlab mr): Mergerequest for the integration branch
     """
+
+    # Rewrite update_submodule without git access
+    gitlab = project.manager.gitlab
+    if branch is None:
+        branch = project.default_branch
+    logging.debug("Branch: %s", branch)
+
+    (
+        submodule_project_path,
+        submodule_current_rev,
+    ) = get_submodule_project_path_and_revision(project, submodule, branch)
+
+    # Get submodule project
+    submodule_project = common.get_project(gitlab, submodule_project_path)
+
+    # Check if revisions are different
+    if submodule_current_rev == revision:
+        print("Submodule is already at %s" % revision)
+        return (None, None, None)
+
+    # Get commits between current and new revision
+    revision_range = submodule_current_rev + ".." + revision
+    commits = submodule_project.commits.list(
+        ref_name=revision_range, retry_transient_errors=True
+    )
+    if not commits:
+        sys.exit("ERROR: no commits found in range %s" % revision_range)
+    logging.debug("New commits: %s", commits)
+
+    # Find out if top commit is part of a merge request
+    # If so, use source branch of this MR as integration branch name
+    # Else use commit sha instead
+    integration_branch_suffix = revision
+    for mr in commits[0].merge_requests():
+        if mr["target_branch"] == submodule_project.default_branch:
+            integration_branch_suffix = mr["source_branch"]
+            break
+    logging.debug("Integration branch suffix: %s", integration_branch_suffix)
+
+    # Check if integration branch already exists and if it is up to date
+    integration_branch_name = common.integration_branch_name(
+        submodule_project.name, integration_branch_suffix
+    )
+
+    integration_branch = None
+    for b in project.branches.list(retry_transient_errors=True):
+        if b.name == integration_branch_name:
+            integration_branch = b
+            break
+
+    logging.debug("Existing integration_branch: %s", integration_branch_name)
+
+    if integration_branch:
+        print("Replacing integration branch %s" % integration_branch_name)
+
+        try:
+            project.branches.delete(integration_branch.get_id())
+        except GitlabDeleteError:
+            print(
+                "Failed to delete branch %s, possible you'll find additional commits in the branch.",
+                integration_branch_name,
+            )
+    else:
+        print("Creating integration branch %s" % integration_branch_name)
+
+    logging.debug(
+        "Creating integration branch %s from %s",
+        integration_branch_name,
+        branch,
+    )
+    project.branches.create({"branch": integration_branch_name, "ref": branch})
+
+    # Update the submodule
+    # Write the new revision to the submodule file
+    # Construct commit message and commit the change
+    message = "Integrate %s/%s%s\n%s" % (
+        submodule_project.name,
+        integration_branch_suffix,
+        " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
+        common.list_commits(commits),
+    )
+
+    # ".gitlab-ci",
+    project.update_submodule(
+        submodule=".gitlab-ci", 
+        branch=integration_branch_name,
+        commit_sha=revision,
+        commit_message=message,
+        retry_transient_errors=True,
+    )
+    #     patch = {
+    #         "branch": integration_branch_name,
+    #         "commit_message": message,
+    #         "actions": [
+    #             {
+    #                 "action": "update",
+    #                 "file_path": submodule,
+    #                 "content": "1"
+    #                 # revision
+    #                 #"bbc15f3863fbcfaab266360e45b54a20c8733bdd"
+    #                 #revision,
+    #             }
+    #         ],
+    #     }
+
+    # Now also update the project '.gitlab-ci.yml' file
+    gitlab_ci_yml = common.get_repository_file_raw(
+        project, ".gitlab-ci.yml", ref=integration_branch_name
+    )
+    logging.debug(gitlab_ci_yml)
+
+    new_gitlab_ci_yml = update_gitlab_ci_include(
+        gitlab_ci_yml,
+        submodule_project.web_url.split("//")[1].split("/", 1)[1],
+        revision,
+    )
+    if new_gitlab_ci_yml is None:
+        print("Failed to update the include revision in '.gitlab-ci.yml'")
+    else:
+        logging.debug(new_gitlab_ci_yml)
+        # Add the change to the commit list
+        #         patch = {
+        #             "branch": integration_branch_name,
+        #             "commit_message": message,
+        #             "actions": [
+        #             ],
+        #         }
+        patch["actions"].append(
+            {
+                "action": "update",
+                "file_path": ".gitlab-ci.yml",
+                "content": new_gitlab_ci_yml,
+            }
+        )
+
+    logging.debug("Send patch: %s", patch["actions"])
+    commit = project.commits.create(patch, retry_transient_errors=True)
+    logging.debug("Create new commit: %s", commit)
+    print()
+    sys.exit("So long and thanks for all the fish")
     # Update submodule
     integration_branch, integration_commit, submodule_project = update_submodule(
         project,
@@ -146,7 +302,6 @@ def deploy_into(
         revision,
         branch,
         pre_commit_hook=update_rev_in_gitlab_ci,
-        replace_existing_branch=replace_existing_branch,
     )
 
     logging.debug("Integration branch: %s, %s", integration_branch, integration_commit)
@@ -275,7 +430,6 @@ def main():
             args.submodule,
             args.revision,
             args.branch,
-            replace_existing_branch=len(args.projects) > 0,
         )
         if integration_branch is not None:
             project_integration[p] = {
@@ -356,7 +510,7 @@ def main():
     commit = manifest_project["project"].commits.create(
         patch, retry_transient_errors=True
     )
-    logging.debug("Create new commit: %s", commit)
+    logging.debug("Created new commit: %s", commit)
 
     sys.exit(0)
 

update_gitlab_ci.py

@@ -4,11 +4,11 @@ import logging
 import re
 
 
-def update_gitlab_ci_include(filename, include_project, new_revision):
+def update_gitlab_ci_include(content, include_project, new_revision):
     """Update the include statement in a gitlab-ci yml to a given revision
 
     Parameters:
-        filename( string): The path to the file to change.
+        content: The content of the gitlab-ci.yml file to change
         include_project( string): The path used to reference the project the include points to.
         new_revision (string): The hex sha to set the include to.
 
@@ -121,7 +121,16 @@ def main():
     if args.verbose:
         logging.basicConfig(level=logging.DEBUG)
 
-    update_gitlab_ci_include(args.filename, args.include_project, args.revision)
+    with open(args.filename, "r", encoding="UTF-8") as fp:
+        gitlab_ci_yml = fp.readtext()
+
+    new_gitlab_ci_yml = update_gitlab_ci_include(
+        gitlab_ci_yml, args.include_project, args.revision
+    )
+
+    if new_gitlab_ci_yml is not None:
+        with open(args.filename, "w", encoding="UTF-8") as fp:
+            fp.writetext(new_gitlab_ci_yml)
 
 
 if __name__ == "__main__":