diff --git a/common.py b/common.py
index 8f77c6d737613d2ec5ffd925d39a073123ae0281..f93f5e1e95f661db8ab81417b0113236b75b2280 100755
--- a/common.py
+++ b/common.py
@@ -239,6 +239,7 @@ def clone_project(project: Project, into, branch=None):
def get_repository_file_raw(project: Project, filename, ref=None):
# TODO tree objects are not supported
fileobj = get_repository_file_obj(project, filename, ref)
+ logging.debug("Read file '%s' from '%s' at ref %s", filename, project.name, ref)
return project.repository_raw_blob(
fileobj["id"], retry_transient_errors=True
).decode()
@@ -253,7 +254,7 @@ def get_repository_file_obj(project: Project, filename, ref=None):
repository_tree = project.repository_tree(
ref=ref, all=True, retry_transient_errors=True
)
- logging.debug(repository_tree)
+ # logging.debug(repository_tree)
fileobj = [f for f in repository_tree if f["name"] == filename]
if len(fileobj) == 0:
diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
index 19000e9bae4a42d501561a4f4ea710827da3fc41..75938a540326d2ffd2f320e53e58b57398a505c4 100755
--- a/deploy_gitlab_ci.py
+++ b/deploy_gitlab_ci.py
@@ -40,7 +40,13 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
def integrate_submodule_into(
- gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
+ gitlab,
+ project_name,
+ submodule_name,
+ new_revision,
+ branch,
+ commit_and_push=True,
+ force_clone=False,
):
gitlab_project = common.get_project(gitlab, project_name)
@@ -56,9 +62,8 @@ def integrate_submodule_into(
new_revision,
branch,
commit_and_push=commit_and_push,
+ force_clone=force_clone,
)
- if integration_branch_name is None:
- return None
# ======================================
# Store the references for creating the integration
# commit in the manifest later
@@ -185,7 +190,9 @@ def main():
res = integrate_submodule_into(
gitlab, p, args.submodule, args.revision, args.branch
)
- if res is not None:
+ # Store in the list if commit is set (meaning there was an update or
+ # an exising integration branch)
+ if res["commit"] is not None:
project_integration[p] = res
# Update submodule in manifest project
@@ -196,22 +203,19 @@ def main():
args.revision,
args.branch,
commit_and_push=False,
+ force_clone=True,
)
- if manifest_project is not None:
- project_integration[args.project] = manifest_project
-
- # If submodule is already at specified revision in all projects, exit successfully
- if len(project_integration) == 0:
- print("No integration done, changes are already included in all projects.")
- sys.exit(0)
+ branch = args.branch
+ if branch is None:
+ branch = manifest_project["project"].default_branch
# =======================================================
# Create and merge merge_requests if needed
# =======================================================
if args.merge:
# Get source merge request ( the one in the gitlab-ci repo)
submodule_project_path, _ = get_submodule_project_path_and_revision(
- manifest_project["project"], args.submodule, args.branch
+ manifest_project["project"], args.submodule, branch
)
submodule_project = common.get_project(gitlab, submodule_project_path)
mrs = get_merge_requests(
@@ -292,12 +296,11 @@ def main():
continue
# get BB_RECIPE_NAME from the projects .gitlab-ci.yml
- gitlab_ci_yml_file = os.path.join(
- integration["repo"].working_tree_dir, ".gitlab-ci.yml"
+ # Use direct read from gitlab as we have not checked out
+ # the repo if the branch is already up to date
+ gitlab_ci_yml = common.get_repository_file_raw(
+ integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
)
- logging.debug("Read recipe name from %s", gitlab_ci_yml_file)
- with open(gitlab_ci_yml_file, "r", encoding="utf8") as fp:
- gitlab_ci_yml = fp.read()
project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
new_srcrev = update_srcrev(
@@ -319,6 +322,13 @@ def main():
manifest_project["repo"].git.add(args.srcrev_file)
logging.debug(srcrev)
+ # ========================================================
+ # Squash all commits on the integration branch to one
+ # ========================================================
+ manifest_project["repo"].remotes.origin.fetch(branch)
+ manifest_master = manifest_project["project"].branches.get(branch)
+ manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
+
# ========================================================
# Now commit and push the changes to the manifest repo
# ========================================================
diff --git a/update_submodule.py b/update_submodule.py
index f74518d00536771cab5a82164b8f74e1073aab4d..4f18263944fec9a38177fbe33c92653bd5040465 100755
--- a/update_submodule.py
+++ b/update_submodule.py
@@ -9,7 +9,7 @@ import tempfile
from configparser import ConfigParser
from furl import furl
from git import GitCommandError, Repo
-from gitlab import Gitlab
+from gitlab import Gitlab, GitlabGetError
from gitlab.v4.objects import Project
from ruamel.yaml import YAML
@@ -21,7 +21,7 @@ def get_submodule_project_path_and_revision(project: Project, submodule, branch=
logging.error("Submodule %s not found in %s.", submodule, project.name)
return None, None
- logging.debug("Gitmodules: %s", gitmodules)
+ # logging.debug("Gitmodules: %s", gitmodules)
cfgparse = ConfigParser()
cfgparse.read_string(gitmodules)
@@ -121,7 +121,12 @@ def clone_project_and_submodule(project: Project, submodule_name, branch=None):
submodule_relative_url = submodule.url
with submodule.config_writer() as writer:
writer.set("url", submodule_clone_url.url)
- submodule.update(init=True)
+ try:
+ submodule.update(init=True)
+ except GitCommandError:
+ # This seems to happen when a not existing commit is referenced
+ logging.error("Failed to initialize submodule %s", submodule_name)
+
with submodule.config_writer() as writer:
writer.set("url", submodule_relative_url)
@@ -181,6 +186,7 @@ def update_submodule_and_include_ref(
new_revision,
branch=None,
commit_and_push=True,
+ force_clone=False,
):
"""Update the submodule and include refs to the submodule in the given project.
Create mergerequest if needed.
@@ -191,6 +197,7 @@ def update_submodule_and_include_ref(
new_revision (hex string): The sha hash of the commit to update the submodule to
branch (string): branch to update, if None, the projects default branch is used
commit_and_push: Set to false if no commit should be created. Changes are left in staging.
+ force_clone: Checkout repo and setup integration branch even if no update is needed
Returns: tuple of:
project_repo (Repo): GitPython repo with the cloned project
@@ -198,23 +205,23 @@ def update_submodule_and_include_ref(
integration_commit (hexsha): Hash of the newly created commit
message: Commit message based on the integrated changes.
"""
+ gitlab = project.manager.gitlab
+
+ submodule_update_needed = True
+ project_repo = None
+ integration_commit = None
if branch is None:
branch = project.default_branch
logging.debug("Branch: %s", branch)
- _, submodule_current_rev = get_submodule_project_path_and_revision(
- project, submodule_name, branch
- )
-
- # Check if revisions are different
- if submodule_current_rev == new_revision:
- print("Submodule is already at %s" % new_revision)
- return (None, None, None, None)
+ (
+ submodule_project_path,
+ submodule_current_rev,
+ ) = get_submodule_project_path_and_revision(project, submodule_name, branch)
- project_repo, submodule_project = clone_project_and_submodule(
- project, submodule_name, branch
- )
+ # Get submodule project
+ submodule_project = common.get_project(gitlab, submodule_project_path)
# Get commits between current and new revision
revision_range = submodule_current_rev + ".." + new_revision
@@ -235,64 +242,134 @@ def update_submodule_and_include_ref(
break
logging.debug("Integration branch suffix: %s", integration_branch_suffix)
- # Setup integration branch
+ # Construct integration branch name
integration_branch_name = common.integration_branch_name(
submodule_project.name, integration_branch_suffix
)
- # Create integration branch
- print("Creating/replacing integration branch %s" % integration_branch_name)
- project_repo.head.set_reference(project_repo.create_head(integration_branch_name))
-
- # Update submodule to new revision
- submodule_repo = common.get_submodule(project_repo, submodule_name)
- update_submodule_in_repo(project_repo, submodule_repo, new_revision)
-
- # Update the gitlab-ci.yml file to the new revision
- # Now also update the project '.gitlab-ci.yml' file
- gitlab_ci_yml_filename = os.path.join(
- project_repo.working_tree_dir, ".gitlab-ci.yml"
- )
- with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
- gitlab_ci_yml = fp.read()
- logging.debug(gitlab_ci_yml)
-
- new_gitlab_ci_yml = update_gitlab_ci_include(
- gitlab_ci_yml,
- submodule_project.web_url.split("//")[1].split("/", 1)[1],
- new_revision,
- )
- if new_gitlab_ci_yml is None:
- print("Failed to update the include revision in '.gitlab-ci.yml'")
- else:
- logging.debug(new_gitlab_ci_yml)
- with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
- fp.write(new_gitlab_ci_yml)
- project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
-
- # Construct commit message and commit the change
+
+ # Construct commit message
message = "Integrate %s/%s%s\n%s" % (
submodule_project.name,
integration_branch_suffix,
" and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
common.list_commits(commits),
)
- # Commit the changes
- if commit_and_push:
-
- # Make an API request to create the gitlab.user object
- gitlab = project.manager.gitlab
- gitlab.auth()
- integration_commit = common.commit_and_push(
- project,
- project_repo,
- integration_branch_name,
- message,
- gitlab.user.username,
- gitlab.user.email,
- less_verbose=True,
+
+ # Check if revisions are different
+ if submodule_current_rev == new_revision:
+ print("Submodule is already at %s" % new_revision)
+ submodule_update_needed = False
+
+ # Check if we already have an integration branch (before we actually do the checkout)
+ # Check if integration branch already exists and if it is up to date
+ # This is needed for one use case:
+ # It is possible to amend changes to the integration branch manually
+ # outside the pipeline.
+ # When the submodule revision has not changed and the pipeline is run
+ # again (due to merge or manually triggered) the manual change persists
+ # in the final commit
+ # For example rename a file in gitlab-ci repo that is included in a
+ # subproject, requires an adapted include statement here.
+ # To get this change 'atomic' the updated include statement should be
+ # in the same commit as the update of the submodule
+
+ if submodule_update_needed or force_clone:
+ existing_branch = None
+ try:
+ existing_branch = project.branches.get(integration_branch_name)
+ except GitlabGetError:
+ # Branch not found
+ pass
+
+ if existing_branch:
+ (
+ _,
+ integration_branch_submodule_rev,
+ ) = get_submodule_project_path_and_revision(
+ project, submodule_name, integration_branch_name
+ )
+ logging.debug(
+ "Revision in integration branch '%s', new_revision '%s'",
+ integration_branch_submodule_rev,
+ new_revision,
+ )
+
+ if integration_branch_submodule_rev == new_revision:
+ print(
+ "Submodule is already at %s on branch %s"
+ % (new_revision, integration_branch_name)
+ )
+ integration_commit = existing_branch.commit["id"]
+ submodule_update_needed = False
+
+ # Clone the project, we need to do changes
+ if submodule_update_needed or force_clone:
+ clone_branch = branch
+ if existing_branch:
+ clone_branch = integration_branch_name
+
+ # Actually clone
+ project_repo, submodule_project = clone_project_and_submodule(
+ project, submodule_name, clone_branch
+ )
+
+ if existing_branch:
+ print("Using existing integration branch %s" % integration_branch_name)
+ else:
+ # Create branch
+ print("Creating integration branch %s" % integration_branch_name)
+ project_repo.head.set_reference(
+ project_repo.create_head(integration_branch_name)
+ )
+
+ if submodule_update_needed:
+ # Update submodule to new revision
+ submodule_repo = common.get_submodule(project_repo, submodule_name)
+ update_submodule_in_repo(project_repo, submodule_repo, new_revision)
+
+ # Update the gitlab-ci.yml file to the new revision
+ # Now also update the project '.gitlab-ci.yml' file
+ gitlab_ci_yml_filename = os.path.join(
+ project_repo.working_tree_dir, ".gitlab-ci.yml"
+ )
+ with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
+ gitlab_ci_yml = fp.read()
+ logging.debug(gitlab_ci_yml)
+
+ new_gitlab_ci_yml = update_gitlab_ci_include(
+ gitlab_ci_yml,
+ submodule_project.web_url.split("//")[1].split("/", 1)[1],
+ new_revision,
)
- else:
- integration_commit = None
+ if new_gitlab_ci_yml is None:
+ print("Failed to update the include revision in '.gitlab-ci.yml'")
+ else:
+ logging.debug(new_gitlab_ci_yml)
+ with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
+ fp.write(new_gitlab_ci_yml)
+ project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
+
+ # Commit the changes
+ if commit_and_push:
+ # ========================================================
+ # Squash all commits on the integration branch to one
+ # ========================================================
+ project_repo.remotes.origin.fetch(branch)
+ gitlab_branch = project.branches.get(branch)
+ project_repo.git.reset("--soft", gitlab_branch.commit["id"])
+
+ # Make an API request to create the gitlab.user object
+ gitlab.auth()
+ # Push the changes
+ integration_commit = common.commit_and_push(
+ project,
+ project_repo,
+ integration_branch_name,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ less_verbose=True,
+ )
return project_repo, integration_branch_name, integration_commit, message