CI: Reuse existing integration branch if submodule is already on new revision

common.py

@@ -239,6 +239,7 @@ def clone_project(project: Project, into, branch=None):
 def get_repository_file_raw(project: Project, filename, ref=None):
     # TODO tree objects are not supported
     fileobj = get_repository_file_obj(project, filename, ref)
+    logging.debug("Read file '%s' from '%s' at ref %s", filename, project.name, ref)
     return project.repository_raw_blob(
         fileobj["id"], retry_transient_errors=True
     ).decode()

deploy_gitlab_ci.py

@@ -40,7 +40,13 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
 
 
 def integrate_submodule_into(
-    gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
+    gitlab,
+    project_name,
+    submodule_name,
+    new_revision,
+    branch,
+    commit_and_push=True,
+    force_replace_of_existing_branch=False,
 ):
 
     gitlab_project = common.get_project(gitlab, project_name)
@@ -56,6 +62,7 @@ def integrate_submodule_into(
         new_revision,
         branch,
         commit_and_push=commit_and_push,
+        force_replace_of_existing_branch=force_replace_of_existing_branch,
     )
     if integration_branch_name is None:
         return None
@@ -196,6 +203,11 @@ def main():
         args.revision,
         args.branch,
         commit_and_push=False,
+        # We need the checkout and integration
+        # branch in the manifest commit
+        # TODO develop a way to allow amend to
+        # this branch also
+        force_replace_of_existing_branch=True,
     )
     if manifest_project is not None:
         project_integration[args.project] = manifest_project
@@ -292,12 +304,11 @@ def main():
             continue
 
         # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
-        gitlab_ci_yml_file = os.path.join(
-            integration["repo"].working_tree_dir, ".gitlab-ci.yml"
+        # Use direct read from gitlab as we have not checked out
+        # the repo if the branch is already up to date
+        gitlab_ci_yml = common.get_repository_file_raw(
+            integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
         )
-        logging.debug("Read recipe name from %s", gitlab_ci_yml_file)
-        with open(gitlab_ci_yml_file, "r", encoding="utf8") as fp:
-            gitlab_ci_yml = fp.read()
         project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
 
         new_srcrev = update_srcrev(

update_submodule.py

@@ -9,7 +9,7 @@ import tempfile
 from configparser import ConfigParser
 from furl import furl
 from git import GitCommandError, Repo
-from gitlab import Gitlab
+from gitlab import Gitlab, GitlabGetError
 from gitlab.v4.objects import Project
 from ruamel.yaml import YAML
 
@@ -21,7 +21,7 @@ def get_submodule_project_path_and_revision(project: Project, submodule, branch=
         logging.error("Submodule %s not found in %s.", submodule, project.name)
         return None, None
 
-    logging.debug("Gitmodules: %s", gitmodules)
+    # logging.debug("Gitmodules: %s", gitmodules)
 
     cfgparse = ConfigParser()
     cfgparse.read_string(gitmodules)
@@ -181,6 +181,7 @@ def update_submodule_and_include_ref(
     new_revision,
     branch=None,
     commit_and_push=True,
+    force_replace_of_existing_branch=False,
 ):
     """Update the submodule and include refs to the submodule in the given project.
         Create mergerequest if needed.
@@ -198,23 +199,24 @@ def update_submodule_and_include_ref(
         integration_commit (hexsha): Hash of the newly created commit
         message: Commit message based on the integrated changes.
     """
+    gitlab = project.manager.gitlab
 
     if branch is None:
         branch = project.default_branch
     logging.debug("Branch: %s", branch)
 
-    _, submodule_current_rev = get_submodule_project_path_and_revision(
-        project, submodule_name, branch
-    )
+    (
+        submodule_project_path,
+        submodule_current_rev,
+    ) = get_submodule_project_path_and_revision(project, submodule_name, branch)
 
     # Check if revisions are different
-    if submodule_current_rev == new_revision:
+    if submodule_current_rev == new_revision and not force_replace_of_existing_branch:
         print("Submodule is already at %s" % new_revision)
         return (None, None, None, None)
 
-    project_repo, submodule_project = clone_project_and_submodule(
-        project, submodule_name, branch
-    )
+    # Get submodule project
+    submodule_project = common.get_project(gitlab, submodule_project_path)
 
     # Get commits between current and new revision
     revision_range = submodule_current_rev + ".." + new_revision
@@ -235,12 +237,70 @@ def update_submodule_and_include_ref(
             break
     logging.debug("Integration branch suffix: %s", integration_branch_suffix)
 
-    # Setup integration branch
+    # Construct integration branch name
     integration_branch_name = common.integration_branch_name(
         submodule_project.name, integration_branch_suffix
     )
-    # Create integration branch
-    print("Creating/replacing integration branch %s" % integration_branch_name)
+
+    # Construct commit message
+    message = "Integrate %s/%s%s\n%s" % (
+        submodule_project.name,
+        integration_branch_suffix,
+        " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
+        common.list_commits(commits),
+    )
+
+    # Check if we already have an integration branch (before we actually do the checkout)
+    # Check if integration branch already exists and if it is up to date
+    # This is needed for one use case:
+    #   It is possible to amend changes to the integration branch manually
+    #   outside the pipeline.
+    #   When the submodule revision has not changed and the pipeline is run
+    #   again (due to merge or manually triggered) the manual change persists
+    #   in the final commit
+    #   For example rename a file in gitlab-ci repo that is included in a
+    #   subproject, requires an adapted include statement here.
+    #   To get this change 'atomic' the updated include statement should be
+    #   in the same commit as the update of the submodule
+
+    existing_branch = None
+    if not force_replace_of_existing_branch:
+        try:
+            existing_branch = project.branches.get(integration_branch_name)
+        except GitlabGetError:
+            # Branch not found
+            pass
+
+        if existing_branch:
+            (
+                _,
+                integration_branch_submodule_rev,
+            ) = get_submodule_project_path_and_revision(
+                project, submodule_name, integration_branch_name
+            )
+            logging.debug(
+                "Revision in integration branch '%s', new_revision '%s'", integration_branch_submodule_rev, new_revision
+            )
+
+            if integration_branch_submodule_rev == new_revision:
+                print(
+                    "Submodule is already at %s on branch %s"
+                    % (new_revision, integration_branch_name)
+                )
+                integration_commit = existing_branch.commit["id"]
+                return None, integration_branch_name, integration_commit, message
+
+    # Clone the project, we need to do changes
+    project_repo, submodule_project = clone_project_and_submodule(
+        project, submodule_name, branch
+    )
+
+    if existing_branch:
+        print("Replacing outdated integration branch %s" % integration_branch_name)
+    else:
+        print("Creating integration branch %s" % integration_branch_name)
+
+    # Create branch
     project_repo.head.set_reference(project_repo.create_head(integration_branch_name))
 
     # Update submodule to new revision
@@ -269,18 +329,9 @@ def update_submodule_and_include_ref(
             fp.write(new_gitlab_ci_yml)
         project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
 
-    # Construct commit message and commit the change
-    message = "Integrate %s/%s%s\n%s" % (
-        submodule_project.name,
-        integration_branch_suffix,
-        " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
-        common.list_commits(commits),
-    )
     # Commit the changes
     if commit_and_push:
-
         # Make an API request to create the gitlab.user object
-        gitlab = project.manager.gitlab
         gitlab.auth()
         integration_commit = common.commit_and_push(
             project,