CI: Reuse existing integration branch preserving manual changes

common.py

@@ -239,6 +239,7 @@ def clone_project(project: Project, into, branch=None):
 def get_repository_file_raw(project: Project, filename, ref=None):
     # TODO tree objects are not supported
     fileobj = get_repository_file_obj(project, filename, ref)
+    logging.debug("Read file '%s' from '%s' at ref %s", filename, project.name, ref)
     return project.repository_raw_blob(
         fileobj["id"], retry_transient_errors=True
     ).decode()
@@ -253,7 +254,7 @@ def get_repository_file_obj(project: Project, filename, ref=None):
     repository_tree = project.repository_tree(
         ref=ref, all=True, retry_transient_errors=True
     )
-    logging.debug(repository_tree)
+    # logging.debug(repository_tree)
     fileobj = [f for f in repository_tree if f["name"] == filename]
 
     if len(fileobj) == 0:

deploy_gitlab_ci.py

@@ -40,7 +40,13 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
 
 
 def integrate_submodule_into(
-    gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
+    gitlab,
+    project_name,
+    submodule_name,
+    new_revision,
+    branch,
+    commit_and_push=True,
+    force_clone=False,
 ):
 
     gitlab_project = common.get_project(gitlab, project_name)
@@ -56,9 +62,8 @@ def integrate_submodule_into(
         new_revision,
         branch,
         commit_and_push=commit_and_push,
+        force_clone=force_clone,
     )
-    if integration_branch_name is None:
-        return None
     # ======================================
     # Store the references for creating the integration
     # commit in the manifest later
@@ -185,7 +190,9 @@ def main():
         res = integrate_submodule_into(
             gitlab, p, args.submodule, args.revision, args.branch
         )
-        if res is not None:
+        # Store in the list if commit is set (meaning there was an update or
+        #   an exising integration branch)
+        if res["commit"] is not None:
             project_integration[p] = res
 
     # Update submodule in manifest project
@@ -196,22 +203,19 @@ def main():
         args.revision,
         args.branch,
         commit_and_push=False,
+        force_clone=True,
     )
-    if manifest_project is not None:
-        project_integration[args.project] = manifest_project
-
-    # If submodule is already at specified revision in all projects, exit successfully
-    if len(project_integration) == 0:
-        print("No integration done, changes are already included in all projects.")
-        sys.exit(0)
 
+    branch = args.branch
+    if branch is None:
+        branch = manifest_project["project"].default_branch
     # =======================================================
     # Create and merge merge_requests if needed
     # =======================================================
     if args.merge:
         # Get source merge request ( the one in the gitlab-ci repo)
         submodule_project_path, _ = get_submodule_project_path_and_revision(
-            manifest_project["project"], args.submodule, args.branch
+            manifest_project["project"], args.submodule, branch
         )
         submodule_project = common.get_project(gitlab, submodule_project_path)
         mrs = get_merge_requests(
@@ -292,12 +296,11 @@ def main():
             continue
 
         # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
-        gitlab_ci_yml_file = os.path.join(
-            integration["repo"].working_tree_dir, ".gitlab-ci.yml"
+        # Use direct read from gitlab as we have not checked out
+        # the repo if the branch is already up to date
+        gitlab_ci_yml = common.get_repository_file_raw(
+            integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
         )
-        logging.debug("Read recipe name from %s", gitlab_ci_yml_file)
-        with open(gitlab_ci_yml_file, "r", encoding="utf8") as fp:
-            gitlab_ci_yml = fp.read()
         project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
 
         new_srcrev = update_srcrev(
@@ -319,6 +322,13 @@ def main():
     manifest_project["repo"].git.add(args.srcrev_file)
     logging.debug(srcrev)
 
+    # ========================================================
+    # Squash all commits on the integration branch to one
+    # ========================================================
+    manifest_project["repo"].remotes.origin.fetch(branch)
+    manifest_master = manifest_project["project"].branches.get(branch)
+    manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
+
     # ========================================================
     # Now commit and push the changes to the manifest repo
     # ========================================================

update_submodule.py

@@ -9,7 +9,7 @@ import tempfile
 from configparser import ConfigParser
 from furl import furl
 from git import GitCommandError, Repo
-from gitlab import Gitlab
+from gitlab import Gitlab, GitlabGetError
 from gitlab.v4.objects import Project
 from ruamel.yaml import YAML
 
@@ -21,7 +21,7 @@ def get_submodule_project_path_and_revision(project: Project, submodule, branch=
         logging.error("Submodule %s not found in %s.", submodule, project.name)
         return None, None
 
-    logging.debug("Gitmodules: %s", gitmodules)
+    # logging.debug("Gitmodules: %s", gitmodules)
 
     cfgparse = ConfigParser()
     cfgparse.read_string(gitmodules)
@@ -121,7 +121,12 @@ def clone_project_and_submodule(project: Project, submodule_name, branch=None):
     submodule_relative_url = submodule.url
     with submodule.config_writer() as writer:
         writer.set("url", submodule_clone_url.url)
-    submodule.update(init=True)
+    try:
+        submodule.update(init=True)
+    except GitCommandError:
+        # This seems to happen when a not existing commit is referenced
+        logging.error("Failed to initialize submodule %s", submodule_name)
+
     with submodule.config_writer() as writer:
         writer.set("url", submodule_relative_url)
 
@@ -181,6 +186,7 @@ def update_submodule_and_include_ref(
     new_revision,
     branch=None,
     commit_and_push=True,
+    force_clone=False,
 ):
     """Update the submodule and include refs to the submodule in the given project.
         Create mergerequest if needed.
@@ -191,6 +197,7 @@ def update_submodule_and_include_ref(
         new_revision (hex string): The sha hash of the commit to update the submodule to
         branch (string): branch to update, if None, the projects default branch is used
         commit_and_push: Set to false if no commit should be created. Changes are left in staging.
+        force_clone: Checkout repo and setup integration branch even if no update is needed
 
     Returns: tuple of:
         project_repo (Repo): GitPython repo with the cloned project
@@ -198,23 +205,23 @@ def update_submodule_and_include_ref(
         integration_commit (hexsha): Hash of the newly created commit
         message: Commit message based on the integrated changes.
     """
+    gitlab = project.manager.gitlab
+
+    submodule_update_needed = True
+    project_repo = None
+    integration_commit = None
 
     if branch is None:
         branch = project.default_branch
     logging.debug("Branch: %s", branch)
 
-    _, submodule_current_rev = get_submodule_project_path_and_revision(
-        project, submodule_name, branch
-    )
-
-    # Check if revisions are different
-    if submodule_current_rev == new_revision:
-        print("Submodule is already at %s" % new_revision)
-        return (None, None, None, None)
+    (
+        submodule_project_path,
+        submodule_current_rev,
+    ) = get_submodule_project_path_and_revision(project, submodule_name, branch)
 
-    project_repo, submodule_project = clone_project_and_submodule(
-        project, submodule_name, branch
-    )
+    # Get submodule project
+    submodule_project = common.get_project(gitlab, submodule_project_path)
 
     # Get commits between current and new revision
     revision_range = submodule_current_rev + ".." + new_revision
@@ -235,64 +242,134 @@ def update_submodule_and_include_ref(
             break
     logging.debug("Integration branch suffix: %s", integration_branch_suffix)
 
-    # Setup integration branch
+    # Construct integration branch name
     integration_branch_name = common.integration_branch_name(
         submodule_project.name, integration_branch_suffix
     )
-    # Create integration branch
-    print("Creating/replacing integration branch %s" % integration_branch_name)
-    project_repo.head.set_reference(project_repo.create_head(integration_branch_name))
-
-    # Update submodule to new revision
-    submodule_repo = common.get_submodule(project_repo, submodule_name)
-    update_submodule_in_repo(project_repo, submodule_repo, new_revision)
-
-    # Update the gitlab-ci.yml file to the new revision
-    # Now also update the project '.gitlab-ci.yml' file
-    gitlab_ci_yml_filename = os.path.join(
-        project_repo.working_tree_dir, ".gitlab-ci.yml"
-    )
-    with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
-        gitlab_ci_yml = fp.read()
-    logging.debug(gitlab_ci_yml)
-
-    new_gitlab_ci_yml = update_gitlab_ci_include(
-        gitlab_ci_yml,
-        submodule_project.web_url.split("//")[1].split("/", 1)[1],
-        new_revision,
-    )
-    if new_gitlab_ci_yml is None:
-        print("Failed to update the include revision in '.gitlab-ci.yml'")
-    else:
-        logging.debug(new_gitlab_ci_yml)
-        with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
-            fp.write(new_gitlab_ci_yml)
-        project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
-
-    # Construct commit message and commit the change
+
+    # Construct commit message
     message = "Integrate %s/%s%s\n%s" % (
         submodule_project.name,
         integration_branch_suffix,
         " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
         common.list_commits(commits),
     )
-    # Commit the changes
-    if commit_and_push:
-
-        # Make an API request to create the gitlab.user object
-        gitlab = project.manager.gitlab
-        gitlab.auth()
-        integration_commit = common.commit_and_push(
-            project,
-            project_repo,
-            integration_branch_name,
-            message,
-            gitlab.user.username,
-            gitlab.user.email,
-            less_verbose=True,
+
+    # Check if revisions are different
+    if submodule_current_rev == new_revision:
+        print("Submodule is already at %s" % new_revision)
+        submodule_update_needed = False
+
+    # Check if we already have an integration branch (before we actually do the checkout)
+    # Check if integration branch already exists and if it is up to date
+    # This is needed for one use case:
+    #   It is possible to amend changes to the integration branch manually
+    #   outside the pipeline.
+    #   When the submodule revision has not changed and the pipeline is run
+    #   again (due to merge or manually triggered) the manual change persists
+    #   in the final commit
+    #   For example rename a file in gitlab-ci repo that is included in a
+    #   subproject, requires an adapted include statement here.
+    #   To get this change 'atomic' the updated include statement should be
+    #   in the same commit as the update of the submodule
+
+    if submodule_update_needed or force_clone:
+        existing_branch = None
+        try:
+            existing_branch = project.branches.get(integration_branch_name)
+        except GitlabGetError:
+            # Branch not found
+            pass
+
+        if existing_branch:
+            (
+                _,
+                integration_branch_submodule_rev,
+            ) = get_submodule_project_path_and_revision(
+                project, submodule_name, integration_branch_name
+            )
+            logging.debug(
+                "Revision in integration branch '%s', new_revision '%s'",
+                integration_branch_submodule_rev,
+                new_revision,
+            )
+
+            if integration_branch_submodule_rev == new_revision:
+                print(
+                    "Submodule is already at %s on branch %s"
+                    % (new_revision, integration_branch_name)
+                )
+                integration_commit = existing_branch.commit["id"]
+                submodule_update_needed = False
+
+    # Clone the project, we need to do changes
+    if submodule_update_needed or force_clone:
+        clone_branch = branch
+        if existing_branch:
+            clone_branch = integration_branch_name
+
+        # Actually clone
+        project_repo, submodule_project = clone_project_and_submodule(
+            project, submodule_name, clone_branch
+        )
+
+        if existing_branch:
+            print("Using existing integration branch %s" % integration_branch_name)
+        else:
+            # Create branch
+            print("Creating integration branch %s" % integration_branch_name)
+            project_repo.head.set_reference(
+                project_repo.create_head(integration_branch_name)
+            )
+
+    if submodule_update_needed:
+        # Update submodule to new revision
+        submodule_repo = common.get_submodule(project_repo, submodule_name)
+        update_submodule_in_repo(project_repo, submodule_repo, new_revision)
+
+        # Update the gitlab-ci.yml file to the new revision
+        # Now also update the project '.gitlab-ci.yml' file
+        gitlab_ci_yml_filename = os.path.join(
+            project_repo.working_tree_dir, ".gitlab-ci.yml"
+        )
+        with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
+            gitlab_ci_yml = fp.read()
+        logging.debug(gitlab_ci_yml)
+
+        new_gitlab_ci_yml = update_gitlab_ci_include(
+            gitlab_ci_yml,
+            submodule_project.web_url.split("//")[1].split("/", 1)[1],
+            new_revision,
         )
-    else:
-        integration_commit = None
+        if new_gitlab_ci_yml is None:
+            print("Failed to update the include revision in '.gitlab-ci.yml'")
+        else:
+            logging.debug(new_gitlab_ci_yml)
+            with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
+                fp.write(new_gitlab_ci_yml)
+            project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
+
+        # Commit the changes
+        if commit_and_push:
+            # ========================================================
+            # Squash all commits on the integration branch to one
+            # ========================================================
+            project_repo.remotes.origin.fetch(branch)
+            gitlab_branch = project.branches.get(branch)
+            project_repo.git.reset("--soft", gitlab_branch.commit["id"])
+
+            # Make an API request to create the gitlab.user object
+            gitlab.auth()
+            # Push the changes
+            integration_commit = common.commit_and_push(
+                project,
+                project_repo,
+                integration_branch_name,
+                message,
+                gitlab.user.username,
+                gitlab.user.email,
+                less_verbose=True,
+            )
 
     return project_repo, integration_branch_name, integration_commit, message