diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
index ea83d1e44d955143d23a1695482c947206ed327c..1354149f1ac735bb61478c87409696303e65d081 100755
--- a/deploy_gitlab_ci.py
+++ b/deploy_gitlab_ci.py
@@ -4,14 +4,15 @@ import common
import argparse
import logging
import sys
-import os
-from gitlab import Gitlab
+from gitlab import Gitlab, GitlabDeleteError
# from accept_merge_request import accept_merge_request
from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
from update_submodule import update_submodule
-from update_gitlab_ci import update_gitlab_ci_include
+from update_submodule import get_submodule_project_path_and_revision
+
+# from update_gitlab_ci import update_gitlab_ci_include
from integrate_into_manifest import update_manifest, update_srcrev
from ruamel.yaml import YAML
@@ -37,6 +38,40 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
return {"recipe": recipe, "masterbranch": masterbranch}
+def update_gitlab_ci_include(content, include_project, new_revision):
+
+ # Remove the SECO-Northern-Europe part from the project filter
+ # as it is normally specified by $CI_PROJECT_ROOT_NAMESPACE
+ include_project = include_project.split("/", 1)[1]
+
+ yaml = YAML()
+ data = yaml.load(content)
+ logging.debug("Yaml: %s", data)
+ try:
+ includes = data["include"]
+ except KeyError:
+ logging.debug("No include statement found")
+ return None
+ current_revision = None
+ for entry in includes:
+ try:
+ if include_project in entry["project"]:
+ current_revision = entry["ref"]
+ break
+ except KeyError:
+ logging.debug("Failed to parse include statement")
+ return None
+ if current_revision is None:
+ logging.debug("Failed to find %s in include statement", include_project)
+ return None
+
+ # Use plain replacement to keep the content of the file
+ # Yes, this may fail if the 'current_revision' is used multiple
+ # time is this fail. But probably this will not ever happen
+ logging.debug("Replace %s with %s", current_revision, new_revision)
+ return content.replace(current_revision, new_revision)
+
+
def create_gitlab_ci_yml(repo, gitlab_ci_yml):
"""This code snippet was used to initially populate
all repos with the minial .gitlab-ci.yml that
@@ -106,25 +141,7 @@ include:
# ======================================
-def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
-
- # Add changed revision also to .gitlab-ci.yml
- gitlab_ci_yml = os.path.join(repo.working_tree_dir, ".gitlab-ci.yml")
-
- if update_gitlab_ci_include(
- gitlab_ci_yml,
- submodule_project.web_url.split("//")[1].split("/", 1)[1],
- submodule_revision,
- ):
- repo.git.add(gitlab_ci_yml)
-
- with open(gitlab_ci_yml, "r", encoding="utf8") as fp:
- logging.debug(fp.read())
-
-
-def deploy_into(
- project, submodule, revision, branch, replace_existing_branch=False, create_mr=False
-):
+def deploy_into(project, submodule, revision, branch, create_mr=False):
"""Update the submodule and include refs to the submodule in the given project.
Create mergerequest if needed.
@@ -133,12 +150,151 @@ def deploy_into(
submodule_name (string): The name of the submodule to pull
submodule_revision (hex string): The sha hash of the commit to update the submodule to
branch (string): branch to update, if None, the projects default branch is used
- replace_existing_branch: When an existing integration branch is found it is always replaced.
Returns: tuple of:
branch (string): Name of the newly created integration branch
merge_request (gitlab mr): Mergerequest for the integration branch
"""
+
+ # Rewrite update_submodule without git access
+ gitlab = project.manager.gitlab
+ if branch is None:
+ branch = project.default_branch
+ logging.debug("Branch: %s", branch)
+
+ (
+ submodule_project_path,
+ submodule_current_rev,
+ ) = get_submodule_project_path_and_revision(project, submodule, branch)
+
+ # Get submodule project
+ submodule_project = common.get_project(gitlab, submodule_project_path)
+
+ # Check if revisions are different
+ if submodule_current_rev == revision:
+ print("Submodule is already at %s" % revision)
+ return (None, None, None)
+
+ # Get commits between current and new revision
+ revision_range = submodule_current_rev + ".." + revision
+ commits = submodule_project.commits.list(
+ ref_name=revision_range, retry_transient_errors=True
+ )
+ if not commits:
+ sys.exit("ERROR: no commits found in range %s" % revision_range)
+ logging.debug("New commits: %s", commits)
+
+ # Find out if top commit is part of a merge request
+ # If so, use source branch of this MR as integration branch name
+ # Else use commit sha instead
+ integration_branch_suffix = revision
+ for mr in commits[0].merge_requests():
+ if mr["target_branch"] == submodule_project.default_branch:
+ integration_branch_suffix = mr["source_branch"]
+ break
+ logging.debug("Integration branch suffix: %s", integration_branch_suffix)
+
+ # Check if integration branch already exists and if it is up to date
+ integration_branch_name = common.integration_branch_name(
+ submodule_project.name, integration_branch_suffix
+ )
+
+ integration_branch = None
+ for b in project.branches.list(retry_transient_errors=True):
+ if b.name == integration_branch_name:
+ integration_branch = b
+ break
+
+ logging.debug("Existing integration_branch: %s", integration_branch_name)
+
+ if integration_branch:
+ print("Replacing integration branch %s" % integration_branch_name)
+
+ try:
+ project.branches.delete(integration_branch.get_id())
+ except GitlabDeleteError:
+ print(
+ "Failed to delete branch %s, possible you'll find additional commits in the branch.",
+ integration_branch_name,
+ )
+ else:
+ print("Creating integration branch %s" % integration_branch_name)
+
+ logging.debug(
+ "Creating integration branch %s from %s",
+ integration_branch_name,
+ branch,
+ )
+ project.branches.create({"branch": integration_branch_name, "ref": branch})
+
+ # Update the submodule
+ # Write the new revision to the submodule file
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s%s\n%s" % (
+ submodule_project.name,
+ integration_branch_suffix,
+ " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
+ common.list_commits(commits),
+ )
+
+ # ".gitlab-ci",
+ project.update_submodule(
+ submodule=".gitlab-ci",
+ branch=integration_branch_name,
+ commit_sha=revision,
+ commit_message=message,
+ retry_transient_errors=True,
+ )
+ # patch = {
+ # "branch": integration_branch_name,
+ # "commit_message": message,
+ # "actions": [
+ # {
+ # "action": "update",
+ # "file_path": submodule,
+ # "content": "1"
+ # # revision
+ # #"bbc15f3863fbcfaab266360e45b54a20c8733bdd"
+ # #revision,
+ # }
+ # ],
+ # }
+
+ # Now also update the project '.gitlab-ci.yml' file
+ gitlab_ci_yml = common.get_repository_file_raw(
+ project, ".gitlab-ci.yml", ref=integration_branch_name
+ )
+ logging.debug(gitlab_ci_yml)
+
+ new_gitlab_ci_yml = update_gitlab_ci_include(
+ gitlab_ci_yml,
+ submodule_project.web_url.split("//")[1].split("/", 1)[1],
+ revision,
+ )
+ if new_gitlab_ci_yml is None:
+ print("Failed to update the include revision in '.gitlab-ci.yml'")
+ else:
+ logging.debug(new_gitlab_ci_yml)
+ # Add the change to the commit list
+ # patch = {
+ # "branch": integration_branch_name,
+ # "commit_message": message,
+ # "actions": [
+ # ],
+ # }
+ patch["actions"].append(
+ {
+ "action": "update",
+ "file_path": ".gitlab-ci.yml",
+ "content": new_gitlab_ci_yml,
+ }
+ )
+
+ logging.debug("Send patch: %s", patch["actions"])
+ commit = project.commits.create(patch, retry_transient_errors=True)
+ logging.debug("Create new commit: %s", commit)
+ print()
+ sys.exit("So long and thanks for all the fish")
# Update submodule
integration_branch, integration_commit, submodule_project = update_submodule(
project,
@@ -146,7 +302,6 @@ def deploy_into(
revision,
branch,
pre_commit_hook=update_rev_in_gitlab_ci,
- replace_existing_branch=replace_existing_branch,
)
logging.debug("Integration branch: %s, %s", integration_branch, integration_commit)
@@ -275,7 +430,6 @@ def main():
args.submodule,
args.revision,
args.branch,
- replace_existing_branch=len(args.projects) > 0,
)
if integration_branch is not None:
project_integration[p] = {
@@ -356,7 +510,7 @@ def main():
commit = manifest_project["project"].commits.create(
patch, retry_transient_errors=True
)
- logging.debug("Create new commit: %s", commit)
+ logging.debug("Created new commit: %s", commit)
sys.exit(0)
diff --git a/update_gitlab_ci.py b/update_gitlab_ci.py
index c1c1057f03de970b25814c426b553ab9c695505e..b23c603f976a7d6ef8316b14debef9eb41ed9cbc 100755
--- a/update_gitlab_ci.py
+++ b/update_gitlab_ci.py
@@ -4,11 +4,11 @@ import logging
import re
-def update_gitlab_ci_include(filename, include_project, new_revision):
+def update_gitlab_ci_include(content, include_project, new_revision):
"""Update the include statement in a gitlab-ci yml to a given revision
Parameters:
- filename( string): The path to the file to change.
+ content: The content of the gitlab-ci.yml file to change
include_project( string): The path used to reference the project the include points to.
new_revision (string): The hex sha to set the include to.
@@ -121,7 +121,16 @@ def main():
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
- update_gitlab_ci_include(args.filename, args.include_project, args.revision)
+ with open(args.filename, "r", encoding="UTF-8") as fp:
+ gitlab_ci_yml = fp.readtext()
+
+ new_gitlab_ci_yml = update_gitlab_ci_include(
+ gitlab_ci_yml, args.include_project, args.revision
+ )
+
+ if new_gitlab_ci_yml is not None:
+ with open(args.filename, "w", encoding="UTF-8") as fp:
+ fp.writetext(new_gitlab_ci_yml)
if __name__ == "__main__":