CI: Update include file's ref in gitlab-ci.yml when deploying changes

Add update gitlab-ci file with function to adapt the include ref to a
given revision.
Add a 'pre-commit-hook' to the update-submodule function.
Adapt deploy_gitlab_ci to use these to update the include statement in
the base project .gitlab-ci.yml to use the same ref as the submodule
is set to.

BCS 746-000646

deploy_gitlab_ci.py

@@ -4,11 +4,25 @@ import common
 import argparse
 import logging
 import sys
+import os
 from gitlab import Gitlab
 from accept_merge_request import accept_merge_request
 from create_merge_request import create_merge_request
 from get_merge_requests import get_merge_requests
 from update_submodule import update_submodule
+from update_gitlab_ci import update_gitlab_ci_include
+
+
+def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
+
+    # Add changed revision also to .gitlab-ci.yml
+    gitlab_ci_yml = os.path.join(repo.working_tree_dir, ".gitlab-ci.yml")
+    if update_gitlab_ci_include(
+        gitlab_ci_yml,
+        submodule_project.web_url.split("//")[1].split("/", 1)[1],
+        submodule_revision,
+    ):
+        repo.git.add(gitlab_ci_yml)
 
 
 def main():
@@ -74,7 +88,11 @@ def main():
 
     # Update submodule
     integration_branch, _, submodule_project = update_submodule(
-        project, args.submodule, args.revision, args.branch
+        project,
+        args.submodule,
+        args.revision,
+        args.branch,
+        pre_commit_hook=update_rev_in_gitlab_ci,
     )
 
     logging.debug("Integration branch: %s", integration_branch)

update_gitlab_ci.py

+#!/usr/bin/env python3
+import argparse
+import logging
+import re
+
+
+def update_gitlab_ci_include(filename, include_project, new_revision):
+    """Update the include statement in a gitlab-ci yml to a given revision
+
+    Parameters:
+        filename( string): The path to the file to change.
+        include_project( string): The path used to reference the project the include points to.
+        new_revision (string): The hex sha to set the include to.
+
+    Returns: True if the file was changed.
+    """
+
+    # Set the possible include in the local .gitlab.yml file
+    # to the new revision. The include needs to have the revision
+    # specified directly in the file, as it is parsed before the
+    # submodule checkout is done
+
+    # Use custom read write method as I didn't got ruamel yaml
+    # to keep all costum formating (linebreaks ...)
+    # This assumes following format of the include block
+    # include:
+    #   - project: 'SECO-Northern-Europe/yocto/infrastructure/gitlab-ci'
+    #     ref: c5a3793e783fcb364c7f3bda73e8cd7c08a08804
+    #     file: 'manifest-childs.yml'
+
+    # Verify hash format:
+    if re.match(r"\A[0-9a-fA-F]{40}\Z", new_revision) is None:
+        raise TypeError("Format of specified revision is not correct")
+
+    parsestate = 0
+    changed = False
+
+    # Remove the SECO-Northern-Europe part from the priject filter
+    # as it is normally specified by $CI_PROJECT_ROOT_NAMESPACE
+    include_project = include_project.split("/", 1)[1]
+    logging.debug("Include project: %s", include_project)
+    logging.debug("New revision: %s", new_revision)
+
+    with open(filename, "r+", encoding="UTF-8") as fp:
+        while True:
+            linestart = fp.tell()
+            line = fp.readline()
+            parts = line.partition(":")
+            logging.debug("Splitted input line: %s", parts)
+            if len(line) == 0:
+                break  # End of file
+            if parsestate == 0:
+                if parts[0] == "include":
+                    # Found include block
+                    parsestate = 1
+                    logging.debug("Found 'include' block at %d", linestart)
+            elif parsestate == 1:
+                if parts[0] == "\n":
+                    break  # End of include block
+
+                if (
+                    parts[0].endswith(" - project")
+                    and parts[2].find(include_project) >= 0
+                ):
+                    # Found the correct project
+                    parsestate = 2
+                    logging.debug("Found 'project' entry at %d", linestart)
+            elif parsestate == 2:
+                if parts[0].endswith(" ref"):
+                    # Found the ref: entry, compare the revision
+                    logging.debug("Found 'ref' entry at %d", linestart)
+                    parsestate = 1
+                    if parts[2].find(new_revision) >= 0:
+                        print(
+                            "Revision in {} is already set to {}".format(
+                                filename, new_revision
+                            )
+                        )
+                    else:
+                        print(
+                            "Changed revision in {} to {}".format(
+                                filename, new_revision
+                            )
+                        )
+                        fp.seek(linestart)
+                        fp.write("{}: {}".format(parts[0], new_revision))
+                        fp.flush()
+                        changed = True
+                elif parts[0].find("- ") >= 0 or not parts[0].startswith("  "):
+                    # Format of the line is not 'name: value' assume end of block
+                    # Block was not found
+                    break
+    return changed
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--filename",
+        help="""File to change""",
+        required=True,
+    )
+    parser.add_argument(
+        "--include-project",
+        help="""The path to the included project as used in 'filename'""",
+        required=True,
+    )
+    parser.add_argument(
+        "--revision",
+        help="""new revision for submodule""",
+        required=True,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
+
+    args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    update_gitlab_ci_include(args.filename, args.include_project, args.revision)
+
+
+if __name__ == "__main__":
+    main()

update_submodule.py

@@ -11,8 +11,24 @@ from git import GitCommandError, Repo
 from gitlab import Gitlab
 
 
-def update_submodule(project, submodule_name, submodule_revision, branch=None):
-    """Update submodule of gitlab project to given revision"""
+def update_submodule(
+    project, submodule_name, submodule_revision, branch=None, pre_commit_hook=None
+):
+    """Update submodule of gitlab project to given revision
+
+    Parameters:
+        project (gitlab project): The project which's submodule should be updated
+        submodule_name (string): The name of the submodule to pull
+        submodule_revision (hex string): The sha hash of the commit to update the submodule to
+        branch (string): branch to update, if None, the projects default branch is used
+        pre_commit_hook: Function to be called before the actual commit is done, to add additional changes.
+                         Arguments passed: ( repo, submodule_project, submodule_revision)
+    Returns: tuple of:
+        branch (string): Name of the newly created integration branch
+        revision (string): hexsha of the new commit
+        submodule_project ( gitlab project): The submodule as gitlab priject instance
+
+    """
 
     gitlab = project.manager.gitlab
 
@@ -123,6 +139,9 @@ def update_submodule(project, submodule_name, submodule_revision, branch=None):
             sys.exit("ERROR: could not checkout commit\n" + str(e))
         repo.git.add(submodule.path)
 
+        if pre_commit_hook is not None:
+            pre_commit_hook(repo, submodule_project, submodule_revision)
+
         # Make an API request to create the gitlab.user object
         gitlab.auth()