From 2ae874f98ad8e48c7689e862739326d9f44bb212 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20H=C3=B6ppner?= <jonas.hoeppner@garz-fricke.com>
Date: Fri, 1 Apr 2022 13:43:07 +0200
Subject: [PATCH] CI: Reuse existing integration branch if submodule is already
on new revision
---
common.py | 1 +
deploy_gitlab_ci.py | 23 ++++++++---
update_submodule.py | 93 +++++++++++++++++++++++++++++++++++----------
3 files changed, 90 insertions(+), 27 deletions(-)
diff --git a/common.py b/common.py
index 8f77c6d7..7fbc86f5 100755
--- a/common.py
+++ b/common.py
@@ -239,6 +239,7 @@ def clone_project(project: Project, into, branch=None):
def get_repository_file_raw(project: Project, filename, ref=None):
# TODO tree objects are not supported
fileobj = get_repository_file_obj(project, filename, ref)
+ logging.debug("Read file '%s' from '%s' at ref %s", filename, project.name, ref)
return project.repository_raw_blob(
fileobj["id"], retry_transient_errors=True
).decode()
diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
index 19000e9b..f2caf724 100755
--- a/deploy_gitlab_ci.py
+++ b/deploy_gitlab_ci.py
@@ -40,7 +40,13 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
def integrate_submodule_into(
- gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
+ gitlab,
+ project_name,
+ submodule_name,
+ new_revision,
+ branch,
+ commit_and_push=True,
+ force_replace_of_existing_branch=False,
):
gitlab_project = common.get_project(gitlab, project_name)
@@ -56,6 +62,7 @@ def integrate_submodule_into(
new_revision,
branch,
commit_and_push=commit_and_push,
+ force_replace_of_existing_branch=force_replace_of_existing_branch,
)
if integration_branch_name is None:
return None
@@ -196,6 +203,11 @@ def main():
args.revision,
args.branch,
commit_and_push=False,
+ # We need the checkout and integration
+ # branch in the manifest commit
+ # TODO develop a way to allow amend to
+ # this branch also
+ force_replace_of_existing_branch=True,
)
if manifest_project is not None:
project_integration[args.project] = manifest_project
@@ -292,12 +304,11 @@ def main():
continue
# get BB_RECIPE_NAME from the projects .gitlab-ci.yml
- gitlab_ci_yml_file = os.path.join(
- integration["repo"].working_tree_dir, ".gitlab-ci.yml"
+ # Use direct read from gitlab as we have not checked out
+ # the repo if the branch is already up to date
+ gitlab_ci_yml = common.get_repository_file_raw(
+ integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
)
- logging.debug("Read recipe name from %s", gitlab_ci_yml_file)
- with open(gitlab_ci_yml_file, "r", encoding="utf8") as fp:
- gitlab_ci_yml = fp.read()
project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
new_srcrev = update_srcrev(
diff --git a/update_submodule.py b/update_submodule.py
index f74518d0..bfd939be 100755
--- a/update_submodule.py
+++ b/update_submodule.py
@@ -9,7 +9,7 @@ import tempfile
from configparser import ConfigParser
from furl import furl
from git import GitCommandError, Repo
-from gitlab import Gitlab
+from gitlab import Gitlab, GitlabGetError
from gitlab.v4.objects import Project
from ruamel.yaml import YAML
@@ -21,7 +21,7 @@ def get_submodule_project_path_and_revision(project: Project, submodule, branch=
logging.error("Submodule %s not found in %s.", submodule, project.name)
return None, None
- logging.debug("Gitmodules: %s", gitmodules)
+ # logging.debug("Gitmodules: %s", gitmodules)
cfgparse = ConfigParser()
cfgparse.read_string(gitmodules)
@@ -181,6 +181,7 @@ def update_submodule_and_include_ref(
new_revision,
branch=None,
commit_and_push=True,
+ force_replace_of_existing_branch=False,
):
"""Update the submodule and include refs to the submodule in the given project.
Create mergerequest if needed.
@@ -198,23 +199,24 @@ def update_submodule_and_include_ref(
integration_commit (hexsha): Hash of the newly created commit
message: Commit message based on the integrated changes.
"""
+ gitlab = project.manager.gitlab
if branch is None:
branch = project.default_branch
logging.debug("Branch: %s", branch)
- _, submodule_current_rev = get_submodule_project_path_and_revision(
- project, submodule_name, branch
- )
+ (
+ submodule_project_path,
+ submodule_current_rev,
+ ) = get_submodule_project_path_and_revision(project, submodule_name, branch)
# Check if revisions are different
- if submodule_current_rev == new_revision:
+ if submodule_current_rev == new_revision and not force_replace_of_existing_branch:
print("Submodule is already at %s" % new_revision)
return (None, None, None, None)
- project_repo, submodule_project = clone_project_and_submodule(
- project, submodule_name, branch
- )
+ # Get submodule project
+ submodule_project = common.get_project(gitlab, submodule_project_path)
# Get commits between current and new revision
revision_range = submodule_current_rev + ".." + new_revision
@@ -235,12 +237,70 @@ def update_submodule_and_include_ref(
break
logging.debug("Integration branch suffix: %s", integration_branch_suffix)
- # Setup integration branch
+ # Construct integration branch name
integration_branch_name = common.integration_branch_name(
submodule_project.name, integration_branch_suffix
)
- # Create integration branch
- print("Creating/replacing integration branch %s" % integration_branch_name)
+
+ # Construct commit message
+ message = "Integrate %s/%s%s\n%s" % (
+ submodule_project.name,
+ integration_branch_suffix,
+ " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
+ common.list_commits(commits),
+ )
+
+ # Check if we already have an integration branch (before we actually do the checkout)
+ # Check if integration branch already exists and if it is up to date
+ # This is needed for one use case:
+ # It is possible to amend changes to the integration branch manually
+ # outside the pipeline.
+ # When the submodule revision has not changed and the pipeline is run
+ # again (due to merge or manually triggered) the manual change persists
+ # in the final commit
+ # For example rename a file in gitlab-ci repo that is included in a
+ # subproject, requires an adapted include statement here.
+ # To get this change 'atomic' the updated include statement should be
+ # in the same commit as the update of the submodule
+
+ existing_branch = None
+ if not force_replace_of_existing_branch:
+ try:
+ existing_branch = project.branches.get(integration_branch_name)
+ except GitlabGetError:
+ # Branch not found
+ pass
+
+ if existing_branch:
+ (
+ _,
+ integration_branch_submodule_rev,
+ ) = get_submodule_project_path_and_revision(
+ project, submodule_name, integration_branch_name
+ )
+ logging.debug(
+ "Revision in integration branch '%s', new_revision '%s'", integration_branch_submodule_rev, new_revision
+ )
+
+ if integration_branch_submodule_rev == new_revision:
+ print(
+ "Submodule is already at %s on branch %s"
+ % (new_revision, integration_branch_name)
+ )
+ integration_commit = existing_branch.commit["id"]
+ return None, integration_branch_name, integration_commit, message
+
+ # Clone the project, we need to do changes
+ project_repo, submodule_project = clone_project_and_submodule(
+ project, submodule_name, branch
+ )
+
+ if existing_branch:
+ print("Replacing outdated integration branch %s" % integration_branch_name)
+ else:
+ print("Creating integration branch %s" % integration_branch_name)
+
+ # Create branch
project_repo.head.set_reference(project_repo.create_head(integration_branch_name))
# Update submodule to new revision
@@ -269,18 +329,9 @@ def update_submodule_and_include_ref(
fp.write(new_gitlab_ci_yml)
project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
- # Construct commit message and commit the change
- message = "Integrate %s/%s%s\n%s" % (
- submodule_project.name,
- integration_branch_suffix,
- " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
- common.list_commits(commits),
- )
# Commit the changes
if commit_and_push:
-
# Make an API request to create the gitlab.user object
- gitlab = project.manager.gitlab
gitlab.auth()
integration_commit = common.commit_and_push(
project,
--
GitLab