From 4bcae2a139cf9415b8ef01498efa49dbe7082109 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20H=C3=B6ppner?= <jonas.hoeppner@garz-fricke.com>
Date: Fri, 1 Apr 2022 16:14:08 +0200
Subject: [PATCH] CI: Reuse integration branch on manifest also
---
common.py | 2 +-
deploy_gitlab_ci.py | 35 ++++++-----
update_submodule.py | 141 ++++++++++++++++++++++++++------------------
3 files changed, 101 insertions(+), 77 deletions(-)
diff --git a/common.py b/common.py
index 7fbc86f5..f93f5e1e 100755
--- a/common.py
+++ b/common.py
@@ -254,7 +254,7 @@ def get_repository_file_obj(project: Project, filename, ref=None):
repository_tree = project.repository_tree(
ref=ref, all=True, retry_transient_errors=True
)
- logging.debug(repository_tree)
+ # logging.debug(repository_tree)
fileobj = [f for f in repository_tree if f["name"] == filename]
if len(fileobj) == 0:
diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
index f2caf724..75938a54 100755
--- a/deploy_gitlab_ci.py
+++ b/deploy_gitlab_ci.py
@@ -46,7 +46,7 @@ def integrate_submodule_into(
new_revision,
branch,
commit_and_push=True,
- force_replace_of_existing_branch=False,
+ force_clone=False,
):
gitlab_project = common.get_project(gitlab, project_name)
@@ -62,10 +62,8 @@ def integrate_submodule_into(
new_revision,
branch,
commit_and_push=commit_and_push,
- force_replace_of_existing_branch=force_replace_of_existing_branch,
+ force_clone=force_clone,
)
- if integration_branch_name is None:
- return None
# ======================================
# Store the references for creating the integration
# commit in the manifest later
@@ -192,7 +190,9 @@ def main():
res = integrate_submodule_into(
gitlab, p, args.submodule, args.revision, args.branch
)
- if res is not None:
+ # Store in the list if commit is set (meaning there was an update or
+ # an exising integration branch)
+ if res["commit"] is not None:
project_integration[p] = res
# Update submodule in manifest project
@@ -203,27 +203,19 @@ def main():
args.revision,
args.branch,
commit_and_push=False,
- # We need the checkout and integration
- # branch in the manifest commit
- # TODO develop a way to allow amend to
- # this branch also
- force_replace_of_existing_branch=True,
+ force_clone=True,
)
- if manifest_project is not None:
- project_integration[args.project] = manifest_project
-
- # If submodule is already at specified revision in all projects, exit successfully
- if len(project_integration) == 0:
- print("No integration done, changes are already included in all projects.")
- sys.exit(0)
+ branch = args.branch
+ if branch is None:
+ branch = manifest_project["project"].default_branch
# =======================================================
# Create and merge merge_requests if needed
# =======================================================
if args.merge:
# Get source merge request ( the one in the gitlab-ci repo)
submodule_project_path, _ = get_submodule_project_path_and_revision(
- manifest_project["project"], args.submodule, args.branch
+ manifest_project["project"], args.submodule, branch
)
submodule_project = common.get_project(gitlab, submodule_project_path)
mrs = get_merge_requests(
@@ -330,6 +322,13 @@ def main():
manifest_project["repo"].git.add(args.srcrev_file)
logging.debug(srcrev)
+ # ========================================================
+ # Squash all commits on the integration branch to one
+ # ========================================================
+ manifest_project["repo"].remotes.origin.fetch(branch)
+ manifest_master = manifest_project["project"].branches.get(branch)
+ manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
+
# ========================================================
# Now commit and push the changes to the manifest repo
# ========================================================
diff --git a/update_submodule.py b/update_submodule.py
index 6dbf0792..c5a422f1 100755
--- a/update_submodule.py
+++ b/update_submodule.py
@@ -121,7 +121,13 @@ def clone_project_and_submodule(project: Project, submodule_name, branch=None):
submodule_relative_url = submodule.url
with submodule.config_writer() as writer:
writer.set("url", submodule_clone_url.url)
- submodule.update(init=True)
+ try:
+ submodule.update(init=True)
+ except GitCommandError:
+ # This seems to happen when a not existing commit is referenced
+ logging.error("Failed to initialize submodule %s", submodule_name)
+ pass
+
with submodule.config_writer() as writer:
writer.set("url", submodule_relative_url)
@@ -181,7 +187,7 @@ def update_submodule_and_include_ref(
new_revision,
branch=None,
commit_and_push=True,
- force_replace_of_existing_branch=False,
+ force_clone=False,
):
"""Update the submodule and include refs to the submodule in the given project.
Create mergerequest if needed.
@@ -192,6 +198,7 @@ def update_submodule_and_include_ref(
new_revision (hex string): The sha hash of the commit to update the submodule to
branch (string): branch to update, if None, the projects default branch is used
commit_and_push: Set to false if no commit should be created. Changes are left in staging.
+ force_clone: Checkout repo and setup integration branch even if no update is needed
Returns: tuple of:
project_repo (Repo): GitPython repo with the cloned project
@@ -201,6 +208,10 @@ def update_submodule_and_include_ref(
"""
gitlab = project.manager.gitlab
+ submodule_update_needed = True
+ project_repo = None
+ integration_commit = None
+
if branch is None:
branch = project.default_branch
logging.debug("Branch: %s", branch)
@@ -210,11 +221,6 @@ def update_submodule_and_include_ref(
submodule_current_rev,
) = get_submodule_project_path_and_revision(project, submodule_name, branch)
- # Check if revisions are different
- if submodule_current_rev == new_revision and not force_replace_of_existing_branch:
- print("Submodule is already at %s" % new_revision)
- return (None, None, None, None)
-
# Get submodule project
submodule_project = common.get_project(gitlab, submodule_project_path)
@@ -250,6 +256,11 @@ def update_submodule_and_include_ref(
common.list_commits(commits),
)
+ # Check if revisions are different
+ if submodule_current_rev == new_revision:
+ print("Submodule is already at %s" % new_revision)
+ submodule_update_needed = False
+
# Check if we already have an integration branch (before we actually do the checkout)
# Check if integration branch already exists and if it is up to date
# This is needed for one use case:
@@ -263,8 +274,8 @@ def update_submodule_and_include_ref(
# To get this change 'atomic' the updated include statement should be
# in the same commit as the update of the submodule
- existing_branch = None
- if not force_replace_of_existing_branch:
+ if submodule_update_needed or force_clone:
+ existing_branch = None
try:
existing_branch = project.branches.get(integration_branch_name)
except GitlabGetError:
@@ -290,62 +301,76 @@ def update_submodule_and_include_ref(
% (new_revision, integration_branch_name)
)
integration_commit = existing_branch.commit["id"]
- return None, integration_branch_name, integration_commit, message
+ submodule_update_needed = False
# Clone the project, we need to do changes
- project_repo, submodule_project = clone_project_and_submodule(
- project, submodule_name, branch
- )
+ if submodule_update_needed or force_clone:
+ clone_branch = branch
+ if existing_branch:
+ clone_branch = integration_branch_name
- if existing_branch:
- print("Replacing outdated integration branch %s" % integration_branch_name)
- else:
- print("Creating integration branch %s" % integration_branch_name)
+ # Actually clone
+ project_repo, submodule_project = clone_project_and_submodule(
+ project, submodule_name, clone_branch
+ )
- # Create branch
- project_repo.head.set_reference(project_repo.create_head(integration_branch_name))
+ if existing_branch:
+ print("Using existing integration branch %s" % integration_branch_name)
+ else:
+ # Create branch
+ print("Creating integration branch %s" % integration_branch_name)
+ project_repo.head.set_reference(
+ project_repo.create_head(integration_branch_name)
+ )
- # Update submodule to new revision
- submodule_repo = common.get_submodule(project_repo, submodule_name)
- update_submodule_in_repo(project_repo, submodule_repo, new_revision)
+ if submodule_update_needed:
+ # Update submodule to new revision
+ submodule_repo = common.get_submodule(project_repo, submodule_name)
+ update_submodule_in_repo(project_repo, submodule_repo, new_revision)
- # Update the gitlab-ci.yml file to the new revision
- # Now also update the project '.gitlab-ci.yml' file
- gitlab_ci_yml_filename = os.path.join(
- project_repo.working_tree_dir, ".gitlab-ci.yml"
- )
- with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
- gitlab_ci_yml = fp.read()
- logging.debug(gitlab_ci_yml)
-
- new_gitlab_ci_yml = update_gitlab_ci_include(
- gitlab_ci_yml,
- submodule_project.web_url.split("//")[1].split("/", 1)[1],
- new_revision,
- )
- if new_gitlab_ci_yml is None:
- print("Failed to update the include revision in '.gitlab-ci.yml'")
- else:
- logging.debug(new_gitlab_ci_yml)
- with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
- fp.write(new_gitlab_ci_yml)
- project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
-
- # Commit the changes
- if commit_and_push:
- # Make an API request to create the gitlab.user object
- gitlab.auth()
- integration_commit = common.commit_and_push(
- project,
- project_repo,
- integration_branch_name,
- message,
- gitlab.user.username,
- gitlab.user.email,
- less_verbose=True,
+ # Update the gitlab-ci.yml file to the new revision
+ # Now also update the project '.gitlab-ci.yml' file
+ gitlab_ci_yml_filename = os.path.join(
+ project_repo.working_tree_dir, ".gitlab-ci.yml"
)
- else:
- integration_commit = None
+ with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
+ gitlab_ci_yml = fp.read()
+ logging.debug(gitlab_ci_yml)
+
+ new_gitlab_ci_yml = update_gitlab_ci_include(
+ gitlab_ci_yml,
+ submodule_project.web_url.split("//")[1].split("/", 1)[1],
+ new_revision,
+ )
+ if new_gitlab_ci_yml is None:
+ print("Failed to update the include revision in '.gitlab-ci.yml'")
+ else:
+ logging.debug(new_gitlab_ci_yml)
+ with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
+ fp.write(new_gitlab_ci_yml)
+ project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
+
+ # Commit the changes
+ if commit_and_push:
+ # ========================================================
+ # Squash all commits on the integration branch to one
+ # ========================================================
+ project_repo.remotes.origin.fetch(branch)
+ gitlab_branch = project.branches.get(branch)
+ project_repo.git.reset("--soft", gitlab_branch.commit["id"])
+
+ # Make an API request to create the gitlab.user object
+ gitlab.auth()
+ # Push the changes
+ integration_commit = common.commit_and_push(
+ project,
+ project_repo,
+ integration_branch_name,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ less_verbose=True,
+ )
return project_repo, integration_branch_name, integration_commit, message
--
GitLab