.gitlab-ci.yml

@@ -91,9 +91,6 @@ yamllint:
       ${PROJECT_ROOT}/tools/touchcal-conv
       ${PROJECT_ROOT}/tools/xconfig
       ${PROJECT_ROOT}/yocto/config
-      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-bar
-      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-foo
-      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-manifest
       ${PROJECT_ROOT}/yocto/layers/meta-guf-distro
       ${PROJECT_ROOT}/yocto/layers/meta-guf-machine
 

check_if_integration_branch_is_up_to_date.py

@@ -46,20 +46,10 @@ def check_if_integration_branch_is_up_to_date(
 
     integration_base_id = integration_base_branch.commit["id"]
 
-    parent = integration_branch.commit
-    parent = manifest_project.commits.get(parent["id"], retry_transient_errors=True)
     # Loop over the commits until the integration_branch head id is found
-    while True:
-        # The integration branch is up to date if its parent is the integration base
-        logging.debug(parent.id)
-        if parent.id == integration_base_id:
-            return True
-        if len(parent.parent_ids) == 0:
-            break
-        parent_id = parent.parent_ids[0]  # Assume linear history
-        parent = manifest_project.commits.get(parent_id, retry_transient_errors=True)
-
-    return False
+    return common.is_commit_parent_of_project_commit(
+        manifest_project, integration_branch.commit["id"], integration_base_id
+    )
 
 
 def main():

common.py

@@ -255,3 +255,34 @@ def get_repository_file_obj(project: Project, filename, ref=None):
         return None
     fileobj = fileobj[0]
     return fileobj
+
+
+def is_commit_parent_of_project_commit(project: Project, project_commit, commit):
+    """Walks through the commits of project, starting with project_commit
+    and compares its sha with the given commit.
+    Both commits are specified as sha
+    """
+    try:
+        _ = project.commits.get(commit, retry_transient_errors=True)
+    except GitlabGetError as e:
+        raise Exception(
+            "Failed to find commit {} in {}".format(project_commit, project.name)
+        ) from e
+
+    # Loop over the parent commits until commit is found
+    parent_id = project_commit
+    while True:
+        try:
+            parent = project.commits.get(parent_id, retry_transient_errors=True)
+        except GitlabGetError as e:
+            raise Exception(
+                "Failed to find commit {} in {}".format(parent_id, project.name)
+            ) from e
+
+        # The integration branch is up to date if its parent is the integration base
+        logging.debug(parent.id)
+        if parent.id == commit:
+            return True
+        if len(parent.parent_ids) == 0:
+            return False
+        parent_id = parent.parent_ids[0]  # Assume linear history

deploy_gitlab_ci.py

@@ -15,14 +15,14 @@ from update_gitlab_ci import update_gitlab_ci_include
 from ruamel.yaml import YAML
 
 
-def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
-
-    # Add changed revision also to .gitlab-ci.yml
-    gitlab_ci_yml = os.path.join(repo.working_tree_dir, ".gitlab-ci.yml")
-
-    # ======================================
-    # Hack to deploy the .gitlab-ci.yml which does the include
-    # to the child project
+def create_gitlab_ci_yml(repo, gitlab_ci_yml):
+    """This code snippet was used to initially populate
+    all repos with the minial .gitlab-ci.yml that
+    includes the files from this repo.
+    Currently it is not used and only the revision is
+    changed.
+    It has also never been test as this function.
+    """
     logging.debug("Origin: %s", repo.remotes.origin.url)
 
     # Contains the base file to be used in the subprojects
@@ -83,6 +83,12 @@ include:
 
     # ======================================
 
+
+def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
+
+    # Add changed revision also to .gitlab-ci.yml
+    gitlab_ci_yml = os.path.join(repo.working_tree_dir, ".gitlab-ci.yml")
+
     if update_gitlab_ci_include(
         gitlab_ci_yml,
         submodule_project.web_url.split("//")[1].split("/", 1)[1],
@@ -231,7 +237,6 @@ def main():
     merge_requests = []
 
     # If submodule is already at specified revision, exit successfully
-    # TODO: is this correct for multi deploy also?
     if not integration_branch:
         sys.exit(0)
 

get_current_revision_from_manifest.py

+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import re
+from gitlab import Gitlab
+from gitlab.v4.objects import Project
+
+from lxml import etree
+
+
+def get_current_revision_from_manifest(
+    manifest_project: Project,
+    manifest_branch,
+    project: Project,
+    recipe_name=None,
+    srcrev_file=common.srcrev_file,
+):
+    """
+    Reads the xml manifest an the SRC_REVC file in the given manifest project
+    at given branch and returns the current revision of the given project.
+    Returns a dict containing the file referencing the project as key and the revision
+    """
+
+    if manifest_branch is None:
+        manifest_branch = manifest_project.default_branch
+        logging.debug("Using default branch %s", manifest_project)
+
+    # Get all manifest xml
+    repository_tree = manifest_project.repository_tree(
+        ref=manifest_branch, all=True, retry_transient_errors=True
+    )
+    logging.debug(repository_tree)
+    manifest_files = [
+        f
+        for f in repository_tree
+        if (f["name"].endswith(".xml") or f["name"] == srcrev_file)
+        and f["type"] == "blob"
+    ]
+    results = {}
+    for f in manifest_files:
+        file_name = f["name"]
+        logging.debug(file_name)
+        content = common.get_repository_file_raw(
+            manifest_project, file_name, ref=manifest_branch
+        )
+        if content is None:
+            logging.error("Failed to read %s.", file_name)
+            continue
+        logging.debug(content)
+        revision = None
+        if file_name.endswith(".xml"):
+            manifest = etree.fromstring(content.encode())
+
+            for node in manifest.findall("project"):
+                project_name = node.get("name")
+                if project_name is not None and project_name.endswith(project.path):
+                    logging.debug(node)
+                    revision = node.get("revision")
+
+        elif recipe_name is not None:
+            # read from SRCREV
+            project_line = None
+            # Match "...RECIPE_NAME ="
+            pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+            for line in content.splitlines():
+                if pattern.search(line):
+                    project_line = line
+            if project_line is None:
+                continue
+            # Get current project revision from SRCREV file
+            # Assuming notation: <project> = "<hash>"
+            revision = project_line.split('"')[1]
+
+        if revision is not None:
+            results[file_name] = revision
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        required=True,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--manifest-project",
+        help="""name of the manifest project""",
+        dest="manifest_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--manifest-branch",
+        help="""the branch in the manifest repo to read, use project's default if not set""",
+        dest="manifest_branch",
+        default=None,
+    )
+    parser.add_argument(
+        "--project",
+        help="""name of the project get the current revision for""",
+        dest="project",
+        required=True,
+    )
+    parser.add_argument(
+        "--srcrev-file",
+        help="""source revision file name (default: 'SRCREV.conf')""",
+        dest="srcrev_file",
+        default=common.srcrev_file,
+        required=False,
+    )
+    parser.add_argument(
+        "--recipe-name",
+        help="""recipe name to resolve the project in 'SRCREV.conf'""",
+        dest="recipe_name",
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
+
+    args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+    project = common.get_project(gitlab, args.project)
+    manifest_project = common.get_project(gitlab, args.manifest_project)
+
+    current_revisions = get_current_revision_from_manifest(
+        manifest_project=manifest_project,
+        manifest_branch=args.manifest_branch,
+        project=project,
+        recipe_name=args.recipe_name,
+        srcrev_file=args.srcrev_file,
+    )
+    print(
+        "The manifest repo {} includes the project {} at revision {}".format(
+            manifest_project.name, project.name, current_revisions
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

merge_gitlab_ci.py

@@ -139,9 +139,20 @@ def main():
         if branch is None:
             branch = project.default_branch
 
+        print(
+            "Try to merge {}({}) into {} ({})".format(
+                args.submodule, args.revision, project.name, branch
+            )
+        )
+
         mr = find_integration_merge_request(
             project, args.submodule, args.revision, branch
         )
+        if mr is None:
+            sys.exit("ERROR: Failed to find the integration MR.")
+            # TODO if this ever happens, why ever, we could call
+            # deploy_gitlab_ci again to create a new integration
+            # commit or make sure the change is already integrated.
 
         print("Merge {}!{}: {}".format(project.name, mr.iid, mr.title))
         # Wait until GitLab has checked merge status
@@ -157,13 +168,24 @@ def main():
         )
 
         if not merged:
-            sys.exit(
-                "Integration MR could not be merged. You have two possibilities to fix "
-                "this:\n"
-                "  1. Checkout the MR and rebase it on the current master manually, or\n"
-                "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
-                "In either case restart this job afterwards in order to get it merged."
+            print(
+                "Integration MR could not be merged."
+                "To fix this:\n"
+                "  1. Checkout the MR {}!{} and merge it manually.\n"
+                "  2. Manually merge the follow up MRs in the following project:".format(
+                    project.name, mr.iid
+                )
             )
+            found = False
+            for p2 in args.projects + [args.project]:
+                if p2 == p:
+                    found = True
+                    continue
+                if not found:
+                    continue
+                print("     {}".format(p2.name))
+            sys.exit()
+
     print("Successfully merged")
 
     exit()

merge_into_manifest.py

@@ -9,6 +9,7 @@ from accept_merge_request import accept_merge_request
 from create_merge_request import create_merge_request
 from get_merge_requests import get_merge_requests
 from integrate_into_manifest import integrate_into_manifest
+from get_current_revision_from_manifest import get_current_revision_from_manifest
 
 
 def merge_into_manifest(
@@ -26,6 +27,27 @@ def merge_into_manifest(
     the current manifest master.
     """
 
+    # Check if the commit is already merged
+    # Currently this is especially true if the gitlab-ci
+    # pipeline already has merged an gitlab-ci update
+    current_revisions = get_current_revision_from_manifest(
+        manifest_project=manifest_project,
+        manifest_branch=master_branch,
+        project=project,
+        recipe_name=recipe_name,
+        srcrev_file=srcrev_file,
+    )
+    # This commit is contained there if current_revisions contains
+    # it directly or any parrent of current_revisions is commit
+    for rev in current_revisions:
+        if common.is_commit_parent_of_project_commit(project, rev, commit):
+            print(
+                "Commit {} is already merged into {}".format(
+                    commit, manifest_project.name
+                )
+            )
+            return
+
     # Get source merge request
     mrs = get_merge_requests(
         project,