diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
index b49bea3540d062f0fd079a8caa1612e01e38c94b..ea83d1e44d955143d23a1695482c947206ed327c 100755
--- a/deploy_gitlab_ci.py
+++ b/deploy_gitlab_ci.py
@@ -12,10 +12,31 @@ from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
from update_submodule import update_submodule
from update_gitlab_ci import update_gitlab_ci_include
+from integrate_into_manifest import update_manifest, update_srcrev
from ruamel.yaml import YAML
+def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
+
+ # Read values from existing file
+ yaml = YAML()
+ data = yaml.load(gitlab_ci_yml)
+ logging.debug("Yaml: %s", data)
+
+ try:
+ masterbranch = data["variables"]["MASTER_BRANCH_PROJECT"]
+ logging.debug("Masterbranch %s", masterbranch)
+ except KeyError:
+ masterbranch = None
+ try:
+ recipe = data["variables"]["BB_RECIPE_NAME"]
+ logging.debug("Recipe %s", recipe)
+ except KeyError:
+ recipe = None
+ return {"recipe": recipe, "masterbranch": masterbranch}
+
+
def create_gitlab_ci_yml(repo, gitlab_ci_yml):
"""This code snippet was used to initially populate
all repos with the minial .gitlab-ci.yml that
@@ -209,6 +230,20 @@ def main():
required=False,
default=False,
)
+ parser.add_argument(
+ "--manifest-file",
+ help="""manifest file name (default: 'default.xml')""",
+ dest="manifest_file",
+ default=common.manifest_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--srcrev-file",
+ help="""source revision file name (default: 'SRCREV.conf')""",
+ dest="srcrev_file",
+ default=common.srcrev_file,
+ required=False,
+ )
parser.add_argument(
"projects",
help="""List of projects the change should be deployed to additionally
@@ -254,9 +289,77 @@ def main():
)
# If submodule is already at specified revision in all projects, exit successfully
- if not len(project_integration) == 0:
+ if len(project_integration) == 0:
+ print("No integration done, changes are already included in all projects.")
sys.exit(0)
+ # Now create the integration commit in the manifest
+ # for all subprojects at once
+ manifest_project = project_integration[args.project]
+ manifest = common.get_repository_file_raw(
+ manifest_project["project"], args.manifest_file, ref=manifest_project["branch"]
+ )
+ logging.debug(manifest)
+ srcrev = common.get_repository_file_raw(
+ manifest_project["project"], args.srcrev_file, ref=manifest_project["branch"]
+ )
+ logging.debug(srcrev)
+
+ for p in args.projects:
+ integration = project_integration[p]
+ logging.debug(
+ "Update %s to %s", integration["project"].name, integration["commit"]
+ )
+
+ new_manifest = update_manifest(
+ manifest, integration["project"], integration["commit"]
+ )
+ if new_manifest is not None:
+ manifest = new_manifest
+ logging.debug(manifest)
+ continue
+
+ # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
+ gitlab_ci_yml = common.get_repository_file_raw(
+ integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
+ )
+ project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
+
+ new_srcrev = update_srcrev(
+ srcrev, project_keys["recipe"], integration["commit"]
+ )
+ if new_srcrev is not None:
+ srcrev = new_srcrev
+ logging.debug(srcrev)
+ else:
+ logging.debug("Project %s not found in xml or srcrev file", p)
+
+ # Now manifest and srcrev contain the updated revisions created above
+ # Send them to gitlab as patch
+ patch = {
+ "branch": manifest_project["branch"],
+ "commit_message": "This is a test", # TODO
+ "actions": [
+ {
+ "action": "update",
+ "file_path": args.manifest_file,
+ "content": manifest,
+ },
+ {
+ "action": "update",
+ "file_path": args.srcrev_file,
+ "content": srcrev,
+ },
+ ],
+ }
+ logging.debug("Send patch: %s", patch)
+ commit = manifest_project["project"].commits.create(
+ patch, retry_transient_errors=True
+ )
+ logging.debug("Create new commit: %s", commit)
+
+ sys.exit(0)
+
if not args.merge:
print(
"Skipping automatic merge in MR context. If you like to extend the "
diff --git a/integrate_into_manifest.py b/integrate_into_manifest.py
index b4c19315c9d45df1cf552d2c5ab5954ac1b0f4b3..fe7d688ff9f00455bf0c9edc7cef9975a408de43 100755
--- a/integrate_into_manifest.py
+++ b/integrate_into_manifest.py
@@ -10,9 +10,66 @@ from pathlib import Path
from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab
+from gitlab.v4.objects import Project
from lxml import etree
+def update_manifest(manifest, project: Project, new_revision):
+ """Returns updated version of the manifest or None id project_name
+ was not found in the manifest
+ """
+ # Parse manifest
+ try:
+ manifestxml = etree.fromstring(manifest.encode())
+ except etree.XMLSyntaxError:
+ sys.exit("ERROR: Failed to parse given manifest")
+
+ # Find project reference in manifest
+ # We are using str.endswith() for this in order to support sub-projects as well
+ # (e.g."mygroup/myproject", when only "myproject" is given)
+ project_node = None
+ project_nodes = manifestxml.findall("project")
+ for node in project_nodes:
+ name = node.get("name")
+ if name is not None and name.endswith(project.path):
+ project_node = node
+ if project_node is None:
+ return None
+
+ # Get current project revision from manifest
+ old_revision = project_node.get("revision")
+ logging.debug("Replace %s with %s", old_revision, new_revision)
+
+ # Update manifest file
+ # We are doing this using a plain text replace action. Unfortunately
+ # all python libraries for handling XML data are not able to preserve
+ # the file layout, and we want a minimal diff.
+ manifest = manifest.replace(old_revision, new_revision)
+ return manifest
+
+
+def update_srcrev(srcrev, recipe_name, new_revision):
+ # Check if project is referenced in SRCREV.conf
+ # Match "...RECIPE_NAME ="
+ pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+
+ project_line = None
+ for line in srcrev.splitlines():
+ if pattern.search(line):
+ project_line = line
+ break
+ if project_line is None:
+ return None
+
+ # Get current project revision from SRCREV file
+ # Assuming notation: <project> = "<hash>"
+ old_revision = project_line.split('"')[1]
+
+ # Update SRCREV file
+ srcrev = srcrev.replace(old_revision, new_revision)
+ return srcrev
+
+
def integrate_into_manifest(
manifest_project,
integration_base,
@@ -34,6 +91,7 @@ def integrate_into_manifest(
clone_url.password = gitlab.private_token
# Checkout manifest
+ # TODO replace checkout with gitlab api access
try:
manifest_repo = Repo.clone_from(
clone_url.url, manifest_dir, branch=integration_base
@@ -71,68 +129,42 @@ def integrate_into_manifest(
manifest_repo.head.set_reference(
manifest_repo.create_head(integration_branch)
)
+ # Get new project revision from merge request
+ new_revision = merge_request.sha
# Parse manifest file
try:
- manifest = etree.parse(manifest_filepath.as_posix())
+ with open(manifest_filepath.as_posix(), "r", encoding="utf8") as fp:
+ manifest = fp.read_text()
except FileNotFoundError:
sys.exit("ERROR: file '%s' not found in manifest repo" % manifest_file)
- # Find project reference in manifest
- # We are using str.endswith() for this in order to support sub-projects as well
- # (e.g."mygroup/myproject", when only "myproject" is given)
- project_node = None
- project_nodes = manifest.findall("project")
- for node in project_nodes:
- name = node.get("name")
- if name is not None and name.endswith(project.path):
- project_node = node
- if project_node is None:
+ new_manifest = update_manifest(manifest, project, new_revision)
+ if new_manifest is not None:
+ # write file
+ with open(manifest_filepath.as_posix(), "w", encoding="utf8") as fp:
+ fp.write_text(new_manifest)
+ manifest_repo.index.add([manifest_file])
+ else:
+ # Look for project in SRCREV as it has not been found in the manifest
if recipe_name is None:
sys.exit(
"ERROR: project '%s' not found in manifest and "
"no recipe name is specified" % project.path
)
- # Check if project is referenced in SRCREV.conf
- project_line = None
- content = srcrev_filepath.read_text()
- # Match "...RECIPE_NAME ="
- pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
- for line in content.splitlines():
- if pattern.search(line):
- project_line = line
- if project_line is None:
+
+ with open(srcrev_filepath, "r", encoding="utf8") as fp:
+ srcrev = fp.read_text()
+ new_srcrev = update_srcrev(srcrev, recipe_name, new_revision)
+ # write file
+ if new_srcrev is None:
sys.exit(
"ERROR: project '%s' not found in manifest and "
- "SRCREV file" % project.path
+ "no recipe name is specified" % project.path
)
-
- # Get current project revision from SRCREV file
- # Assuming notation: <project> = "<hash>"
- old_revision = project_line.split('"')[1]
-
- # Get new project revision from merge request
- new_revision = merge_request.sha
-
- # Update SRCREV file
- content = content.replace(old_revision, new_revision)
- srcrev_filepath.write_text(content)
+ with open(srcrev_filepath.as_posix(), "w", encoding="utf8") as fp:
+ fp.write_text(new_manifest)
manifest_repo.index.add([srcrev_file])
- else:
- # Get current project revision from manifest
- old_revision = project_node.get("revision")
-
- # Get new project revision from merge request
- new_revision = merge_request.sha
-
- # Update manifest file
- # We are doing this using a plain text replace action. Unfortunately
- # all python libraries for handling XML data are not able to preserve
- # the file layout, and we want a minimal diff.
- content = manifest_filepath.read_text()
- content = content.replace(old_revision, new_revision)
- manifest_filepath.write_text(content)
- manifest_repo.index.add([manifest_file])
# Make an API request to create the gitlab.user object
gitlab.auth()