CI: Reuse integration branch on manifest also

common.py

@@ -254,7 +254,7 @@ def get_repository_file_obj(project: Project, filename, ref=None):
     repository_tree = project.repository_tree(
         ref=ref, all=True, retry_transient_errors=True
     )
-    logging.debug(repository_tree)
+    # logging.debug(repository_tree)
     fileobj = [f for f in repository_tree if f["name"] == filename]
 
     if len(fileobj) == 0:

deploy_gitlab_ci.py

@@ -46,7 +46,7 @@ def integrate_submodule_into(
     new_revision,
     branch,
     commit_and_push=True,
-    force_replace_of_existing_branch=False,
+    force_clone=False,
 ):
 
     gitlab_project = common.get_project(gitlab, project_name)
@@ -62,10 +62,8 @@ def integrate_submodule_into(
         new_revision,
         branch,
         commit_and_push=commit_and_push,
-        force_replace_of_existing_branch=force_replace_of_existing_branch,
+        force_clone=force_clone,
     )
-    if integration_branch_name is None:
-        return None
     # ======================================
     # Store the references for creating the integration
     # commit in the manifest later
@@ -192,7 +190,9 @@ def main():
         res = integrate_submodule_into(
             gitlab, p, args.submodule, args.revision, args.branch
         )
-        if res is not None:
+        # Store in the list if commit is set (meaning there was an update or
+        #   an exising integration branch)
+        if res["commit"] is not None:
             project_integration[p] = res
 
     # Update submodule in manifest project
@@ -203,27 +203,19 @@ def main():
         args.revision,
         args.branch,
         commit_and_push=False,
-        # We need the checkout and integration
-        # branch in the manifest commit
-        # TODO develop a way to allow amend to
-        # this branch also
-        force_replace_of_existing_branch=True,
+        force_clone=True,
     )
-    if manifest_project is not None:
-        project_integration[args.project] = manifest_project
-
-    # If submodule is already at specified revision in all projects, exit successfully
-    if len(project_integration) == 0:
-        print("No integration done, changes are already included in all projects.")
-        sys.exit(0)
 
+    branch = args.branch
+    if branch is None:
+        branch = manifest_project["project"].default_branch
     # =======================================================
     # Create and merge merge_requests if needed
     # =======================================================
     if args.merge:
         # Get source merge request ( the one in the gitlab-ci repo)
         submodule_project_path, _ = get_submodule_project_path_and_revision(
-            manifest_project["project"], args.submodule, args.branch
+            manifest_project["project"], args.submodule, branch
         )
         submodule_project = common.get_project(gitlab, submodule_project_path)
         mrs = get_merge_requests(
@@ -330,6 +322,13 @@ def main():
     manifest_project["repo"].git.add(args.srcrev_file)
     logging.debug(srcrev)
 
+    # ========================================================
+    # Squash all commits on the integration branch to one
+    # ========================================================
+    manifest_project["repo"].remotes.origin.fetch(branch)
+    manifest_master = manifest_project["project"].branches.get(branch)
+    manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
+
     # ========================================================
     # Now commit and push the changes to the manifest repo
     # ========================================================

update_submodule.py

@@ -121,7 +121,12 @@ def clone_project_and_submodule(project: Project, submodule_name, branch=None):
     submodule_relative_url = submodule.url
     with submodule.config_writer() as writer:
         writer.set("url", submodule_clone_url.url)
-    submodule.update(init=True)
+    try:
+        submodule.update(init=True)
+    except GitCommandError:
+        # This seems to happen when a not existing commit is referenced
+        logging.error("Failed to initialize submodule %s", submodule_name)
+
     with submodule.config_writer() as writer:
         writer.set("url", submodule_relative_url)
 
@@ -181,7 +186,7 @@ def update_submodule_and_include_ref(
     new_revision,
     branch=None,
     commit_and_push=True,
-    force_replace_of_existing_branch=False,
+    force_clone=False,
 ):
     """Update the submodule and include refs to the submodule in the given project.
         Create mergerequest if needed.
@@ -192,6 +197,7 @@ def update_submodule_and_include_ref(
         new_revision (hex string): The sha hash of the commit to update the submodule to
         branch (string): branch to update, if None, the projects default branch is used
         commit_and_push: Set to false if no commit should be created. Changes are left in staging.
+        force_clone: Checkout repo and setup integration branch even if no update is needed
 
     Returns: tuple of:
         project_repo (Repo): GitPython repo with the cloned project
@@ -201,6 +207,10 @@ def update_submodule_and_include_ref(
     """
     gitlab = project.manager.gitlab
 
+    submodule_update_needed = True
+    project_repo = None
+    integration_commit = None
+
     if branch is None:
         branch = project.default_branch
     logging.debug("Branch: %s", branch)
@@ -210,11 +220,6 @@ def update_submodule_and_include_ref(
         submodule_current_rev,
     ) = get_submodule_project_path_and_revision(project, submodule_name, branch)
 
-    # Check if revisions are different
-    if submodule_current_rev == new_revision and not force_replace_of_existing_branch:
-        print("Submodule is already at %s" % new_revision)
-        return (None, None, None, None)
-
     # Get submodule project
     submodule_project = common.get_project(gitlab, submodule_project_path)
 
@@ -250,6 +255,11 @@ def update_submodule_and_include_ref(
         common.list_commits(commits),
     )
 
+    # Check if revisions are different
+    if submodule_current_rev == new_revision:
+        print("Submodule is already at %s" % new_revision)
+        submodule_update_needed = False
+
     # Check if we already have an integration branch (before we actually do the checkout)
     # Check if integration branch already exists and if it is up to date
     # This is needed for one use case:
@@ -263,8 +273,8 @@ def update_submodule_and_include_ref(
     #   To get this change 'atomic' the updated include statement should be
     #   in the same commit as the update of the submodule
 
-    existing_branch = None
-    if not force_replace_of_existing_branch:
+    if submodule_update_needed or force_clone:
+        existing_branch = None
         try:
             existing_branch = project.branches.get(integration_branch_name)
         except GitlabGetError:
@@ -290,62 +300,76 @@ def update_submodule_and_include_ref(
                     % (new_revision, integration_branch_name)
                 )
                 integration_commit = existing_branch.commit["id"]
-                return None, integration_branch_name, integration_commit, message
+                submodule_update_needed = False
 
     # Clone the project, we need to do changes
-    project_repo, submodule_project = clone_project_and_submodule(
-        project, submodule_name, branch
-    )
+    if submodule_update_needed or force_clone:
+        clone_branch = branch
+        if existing_branch:
+            clone_branch = integration_branch_name
 
-    if existing_branch:
-        print("Replacing outdated integration branch %s" % integration_branch_name)
-    else:
-        print("Creating integration branch %s" % integration_branch_name)
+        # Actually clone
+        project_repo, submodule_project = clone_project_and_submodule(
+            project, submodule_name, clone_branch
+        )
 
-    # Create branch
-    project_repo.head.set_reference(project_repo.create_head(integration_branch_name))
+        if existing_branch:
+            print("Using existing integration branch %s" % integration_branch_name)
+        else:
+            # Create branch
+            print("Creating integration branch %s" % integration_branch_name)
+            project_repo.head.set_reference(
+                project_repo.create_head(integration_branch_name)
+            )
 
-    # Update submodule to new revision
-    submodule_repo = common.get_submodule(project_repo, submodule_name)
-    update_submodule_in_repo(project_repo, submodule_repo, new_revision)
+    if submodule_update_needed:
+        # Update submodule to new revision
+        submodule_repo = common.get_submodule(project_repo, submodule_name)
+        update_submodule_in_repo(project_repo, submodule_repo, new_revision)
 
-    # Update the gitlab-ci.yml file to the new revision
-    # Now also update the project '.gitlab-ci.yml' file
-    gitlab_ci_yml_filename = os.path.join(
-        project_repo.working_tree_dir, ".gitlab-ci.yml"
-    )
-    with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
-        gitlab_ci_yml = fp.read()
-    logging.debug(gitlab_ci_yml)
-
-    new_gitlab_ci_yml = update_gitlab_ci_include(
-        gitlab_ci_yml,
-        submodule_project.web_url.split("//")[1].split("/", 1)[1],
-        new_revision,
-    )
-    if new_gitlab_ci_yml is None:
-        print("Failed to update the include revision in '.gitlab-ci.yml'")
-    else:
-        logging.debug(new_gitlab_ci_yml)
-        with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
-            fp.write(new_gitlab_ci_yml)
-        project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
-
-    # Commit the changes
-    if commit_and_push:
-        # Make an API request to create the gitlab.user object
-        gitlab.auth()
-        integration_commit = common.commit_and_push(
-            project,
-            project_repo,
-            integration_branch_name,
-            message,
-            gitlab.user.username,
-            gitlab.user.email,
-            less_verbose=True,
+        # Update the gitlab-ci.yml file to the new revision
+        # Now also update the project '.gitlab-ci.yml' file
+        gitlab_ci_yml_filename = os.path.join(
+            project_repo.working_tree_dir, ".gitlab-ci.yml"
         )
-    else:
-        integration_commit = None
+        with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
+            gitlab_ci_yml = fp.read()
+        logging.debug(gitlab_ci_yml)
+
+        new_gitlab_ci_yml = update_gitlab_ci_include(
+            gitlab_ci_yml,
+            submodule_project.web_url.split("//")[1].split("/", 1)[1],
+            new_revision,
+        )
+        if new_gitlab_ci_yml is None:
+            print("Failed to update the include revision in '.gitlab-ci.yml'")
+        else:
+            logging.debug(new_gitlab_ci_yml)
+            with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
+                fp.write(new_gitlab_ci_yml)
+            project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
+
+        # Commit the changes
+        if commit_and_push:
+            # ========================================================
+            # Squash all commits on the integration branch to one
+            # ========================================================
+            project_repo.remotes.origin.fetch(branch)
+            gitlab_branch = project.branches.get(branch)
+            project_repo.git.reset("--soft", gitlab_branch.commit["id"])
+
+            # Make an API request to create the gitlab.user object
+            gitlab.auth()
+            # Push the changes
+            integration_commit = common.commit_and_push(
+                project,
+                project_repo,
+                integration_branch_name,
+                message,
+                gitlab.user.username,
+                gitlab.user.email,
+                less_verbose=True,
+            )
 
     return project_repo, integration_branch_name, integration_commit, message