CI: Add function to merge the gitlab-ci MRs

.gitlab-ci.yml

@@ -20,6 +20,7 @@ stages:
   - analyze
   - deploy
   - check
+  - merge
 
 workflow:
   rules:
@@ -57,12 +58,51 @@ yamllint:
 # ---------------------------------------------------------------------------------------
 # Stage: deploy-test
 # ---------------------------------------------------------------------------------------
+
+.foobar-manifest-projects:
+  variables:
+    PROJECT_ROOT:
+      ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
+    MANIFEST_PROJECT:
+      ${PROJECT_ROOT}/minimal-manifest
+
+    DEPLOY_TO:
+      ${PROJECT_ROOT}/minimal-foo
+      ${PROJECT_ROOT}/minimal-bar
+
+.manifest-projects:
+  variables:
+    PROJECT_ROOT:
+      ${CI_PROJECT_ROOT_NAMESPACE}
+    MANIFEST_PROJECT:
+      ${PROJECT_ROOT}/yocto/manifest
+    DEPLOY_TO:
+      ${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk
+      ${PROJECT_ROOT}/kernel/linux-guf
+      ${PROJECT_ROOT}/kernel/linux-imx-kuk
+      ${PROJECT_ROOT}/kernel/modules/egalaxi2c
+      ${PROJECT_ROOT}/kernel/modules/gfplatdetect
+      ${PROJECT_ROOT}/tools/gf-emc-test-suite
+      ${PROJECT_ROOT}/tools/gf-productiontests
+      ${PROJECT_ROOT}/tools/gfeeprom
+      ${PROJECT_ROOT}/tools/gfxml2dto
+      ${PROJECT_ROOT}/tools/guf-show-demo
+      ${PROJECT_ROOT}/tools/libmdb
+      ${PROJECT_ROOT}/tools/touchcal-conv
+      ${PROJECT_ROOT}/tools/xconfig
+      ${PROJECT_ROOT}/yocto/config
+      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-bar
+      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-foo
+      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-manifest
+      ${PROJECT_ROOT}/yocto/layers/meta-guf-distro
+      ${PROJECT_ROOT}/yocto/layers/meta-guf-machine
+
 .deploy:
   stage: deploy
+  rules:
+    - if: $CI_MERGE_REQUEST_IID
   script:
     - cd ${CI_PROJECT_DIR}
-    - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then
-      MERGE="--merge"; else MERGE=""; fi
     - ./deploy_gitlab_ci.py
       --gitlab-url=${CI_SERVER_URL}
       --token=${GITBOT_TOKEN}
@@ -70,7 +110,6 @@ yamllint:
       --submodule=.gitlab-ci
       --revision=${CI_COMMIT_SHA}
       --verbose
-      ${MERGE}
       ${DEPLOY_TO}
 
     - ./generate_job_from_template.py
@@ -88,51 +127,23 @@ yamllint:
       - integration.yml
 
 deploy:
-  extends: .deploy
+  extends:
+    - .deploy
+    - .manifest-projects
   rules:
     # For now to test the merge step
     - when: manual
       allow_failure: true
-  variables:
-    PROJECT_ROOT:
-      ${CI_PROJECT_ROOT_NAMESPACE}
-    MANIFEST_PROJECT:
-      ${PROJECT_ROOT}/yocto/manifest
-    DEPLOY_TO:
-      ${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk
-      ${PROJECT_ROOT}/kernel/linux-guf
-      ${PROJECT_ROOT}/kernel/linux-imx-kuk
-      ${PROJECT_ROOT}/kernel/modules/egalaxi2c
-      ${PROJECT_ROOT}/kernel/modules/gfplatdetect
-      ${PROJECT_ROOT}/tools/gf-emc-test-suite
-      ${PROJECT_ROOT}/tools/gf-productiontests
-      ${PROJECT_ROOT}/tools/gfeeprom
-      ${PROJECT_ROOT}/tools/gfxml2dto
-      ${PROJECT_ROOT}/tools/guf-show-demo
-      ${PROJECT_ROOT}/tools/libmdb
-      ${PROJECT_ROOT}/tools/touchcal-conv
-      ${PROJECT_ROOT}/tools/xconfig
-      ${PROJECT_ROOT}/yocto/config
-      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-bar
-      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-foo
-      ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-manifest
-      ${PROJECT_ROOT}/yocto/layers/meta-guf-distro
-      ${PROJECT_ROOT}/yocto/layers/meta-guf-machine
 
 deploy-foobar:
-  extends: .deploy
-  variables:
-    PROJECT_ROOT:
-      ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
-    MANIFEST_PROJECT:
-      ${PROJECT_ROOT}/minimal-manifest
-
-    DEPLOY_TO:
-      ${PROJECT_ROOT}/minimal-foo
-      ${PROJECT_ROOT}/minimal-bar
+  extends:
+    - .deploy
+    - .foobar-manifest-projects
 
 trigger:
   stage: deploy
+  rules:
+    - if: $CI_MERGE_REQUEST_IID
   needs: [deploy]
   trigger:
     include:
@@ -142,12 +153,36 @@ trigger:
 
 trigger-foobar:
   stage: deploy
+  rules:
+    - if: $CI_MERGE_REQUEST_IID
   needs: [deploy-foobar]
   trigger:
     include:
       - artifact: integration.yml
         job: deploy-foobar
     strategy: depend
+# --------------------------------------------------------------------------------------
+# Stage: merge
+# --------------------------------------------------------------------------------------
+.merge:
+  stage: merge
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - ./merge_gitlab_ci.py
+      --gitlab-url=${CI_SERVER_URL}
+      --token=${GITBOT_TOKEN}
+      --manifest-project=${MANIFEST_PROJECT}
+      --submodule=.gitlab-ci
+      --revision=${CI_COMMIT_SHA}
+      --verbose
+      ${DEPLOY_TO}
+
+merge-foobar:
+  extends:
+    - .merge
+    - .foobar-manifest-projects
 
 # --------------------------------------------------------------------------------------
 # Stage: check

accept_merge_request.py

@@ -2,6 +2,7 @@
 import common
 
 import argparse
+import logging
 import sys
 import time
 from gitlab import (
@@ -51,6 +52,7 @@ def accept_merge_request(project, mr, rebase=False):
         except GitlabMRClosedError as e:
             # See HTTP error codes for merge requests here:
             # https://docs.gitlab.com/ce/api/merge_requests.html#accept-mr
+            logging.debug("Error from gitlab: %d", e.response_code)
 
             if e.response_code == 405:
                 # Not allowed (draft, closed, pipeline pending or failed)

common.py

 #!/usr/bin/env python3
 
+import logging
 import requests
 import sys
-import logging
 import time
+from furl import furl
 from git import Actor, GitCommandError
 from git.repo.base import Repo
 from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
@@ -203,3 +204,48 @@ def get_merge_request(project: Project, merge_request):
     except GitlabGetError:
         return None
     return mr
+
+
+def clone_project(project: Project, into, branch=None):
+
+    gitlab = project.manager.gitlab
+    # If no branch is given, use project's default branch
+    if branch is None:
+        branch = project.default_branch
+
+    # Construct clone url containing access token
+    clone_url = furl(project.http_url_to_repo)
+    clone_url.username = "gitlab-ci"
+    clone_url.password = gitlab.private_token
+
+    # Checkout project
+    try:
+        repo = Repo.clone_from(clone_url.url, into, branch=branch, depth=1)
+    except GitCommandError as e:
+        raise Exception("could not clone repository\n" + str(e)) from e
+    except IndexError:
+        raise Exception("branch '%s' not found" % branch) from e
+    return repo
+
+
+def get_repository_file_raw(project: Project, filename, ref=None):
+    # TODO tree objects are not supported
+    fileobj = get_repository_file_obj(project, filename, ref)
+    return project.repository_raw_blob(
+        fileobj["id"], retry_transient_errors=True
+    ).decode()
+
+
+def get_repository_file_obj(project: Project, filename, ref=None):
+    # TODO tree objects are not supported
+    if ref is None:
+        ref = project.default_branch
+        logging.debug("Using default branch %s", ref)
+    repository_tree = project.repository_tree(ref=ref, retry_transient_errors=True)
+    logging.debug(repository_tree)
+    fileobj = [f for f in repository_tree if f["name"] == filename]
+
+    if len(fileobj) == 0:
+        return None
+    fileobj = fileobj[0]
+    return fileobj

merge_gitlab_ci.py

+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+from gitlab import Gitlab, GitlabGetError
+from accept_merge_request import accept_merge_request
+import update_submodule
+from gitlab.v4.objects import Project
+
+
+def find_integration_merge_request(
+    project: Project, submodule, revision, target_branch
+):
+
+    (
+        submodule_path,
+        submodule_revision,
+    ) = update_submodule.get_submodule_project_path_and_revision(
+        project, submodule, target_branch
+    )
+    logging.debug("Module: %s, Revision: %s", submodule_path, submodule_revision)
+
+    # Get submodule project
+    gitlab = project.manager.gitlab
+    submodule_project = common.get_project(gitlab, submodule_path)
+
+    # Get the integration branch name
+    integration_branch_suffix = (
+        update_submodule.get_submodule_integration_branch_suffix(
+            submodule_project, revision
+        )
+    )
+    integration_branch_name = common.integration_branch_name(
+        submodule_project.name, integration_branch_suffix
+    )
+    logging.debug(integration_branch_name)
+
+    # Get the merge request for the branch TODO catch exception
+    try:
+        integration_branch = project.branches.get(
+            integration_branch_name, retry_transient_errors=True
+        )
+    except GitlabGetError:
+        print("ERROR: Failed to find integration branch for {}".format(submodule))
+        return None
+
+    logging.debug(integration_branch)
+    commit = project.commits.get(
+        integration_branch.commit["id"], retry_transient_errors=True
+    )
+    for mr in commit.merge_requests(retry_transient_errors=True):
+        if mr["target_branch"] == target_branch:
+            integration_mr = mr
+            break
+    if integration_mr is None:
+        print(
+            "ERROR: Failed to find integration merge request for %s",
+            integration_branch_name,
+        )
+        return None
+    logging.debug(integration_mr["iid"])
+    mr = project.mergerequests.get(integration_mr["iid"], retry_transient_errors=True)
+
+    return mr
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        required=True,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--project",
+        "--manifest-project",
+        help="""name of the GitLab project""",
+        dest="project",
+    )
+    parser.add_argument(
+        "--submodule",
+        help="""submodule to update""",
+        dest="submodule",
+        required=True,
+    )
+    parser.add_argument(
+        "--revision",
+        help="""new revision for submodule""",
+        dest="revision",
+        required=True,
+    )
+    parser.add_argument(
+        "--branch",
+        help="""project branch (if not default branch)""",
+        dest="branch",
+        required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "--merge",
+        help="""if set, perform merge after integration""",
+        dest="merge",
+        action="store_true",
+        required=False,
+        default=False,
+    )
+    parser.add_argument(
+        "projects",
+        help="""List of projects the change should be deployed to additionally
+                to the manifest project given as named parameter.""",
+        nargs="*",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
+
+    args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+    for p in args.projects + [args.project]:
+        project = common.get_project(gitlab, p)
+        branch = args.branch
+
+        if branch is None:
+            branch = project.default_branch
+
+        mr = find_integration_merge_request(
+            project, args.submodule, args.revision, branch
+        )
+
+        logging.debug("Merge %s!%d: %s", project.name, mr.iid, mr.title)
+        # Wait until GitLab has checked merge status
+        common.wait_until_merge_status_is_set(project, mr)
+
+        # Attempt to merge
+        merged = accept_merge_request(
+            project,
+            mr,
+            # Only the file manifest commit may be rebased
+            # other rebase would require new integration commits
+            rebase=(p == args.project),
+        )
+
+        if not merged:
+            sys.exit(
+                "Integration MR could not be merged. You have two possibilities to fix "
+                "this:\n"
+                "  1. Checkout the MR and rebase it on the current master manually, or\n"
+                "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
+                "In either case restart this job afterwards in order to get it merged."
+            )
+    print("Successfully merged")
+
+    exit()
+
+
+if __name__ == "__main__":
+    main()

update_submodule.py

@@ -6,9 +6,63 @@ import logging
 import os
 import sys
 import tempfile
+from configparser import ConfigParser
 from furl import furl
 from git import GitCommandError, Repo
 from gitlab import Gitlab
+from gitlab.v4.objects import Project
+
+
+def get_submodule_project_path_and_revision(project: Project, submodule, branch=None):
+
+    gitmodules = common.get_repository_file_raw(project, ".gitmodules", ref=branch)
+    if gitmodules is None:
+        logging.error("Submodule %s not found in %s.", submodule, project.name)
+        return None, None
+
+    logging.debug("Gitmodules: %s", gitmodules)
+
+    cfgparse = ConfigParser()
+    cfgparse.read_string(gitmodules)
+    try:
+        section = cfgparse['submodule "{}"'.format(submodule)]
+    except KeyError:
+        logging.error("Submodule %s not found in %s.", submodule, project.name)
+        return None, None
+
+    submodule_url = section["url"]
+    # absolut path to a relative submodule
+    # Check for relative path
+    if not submodule_url.startswith(".."):
+        logging.error("absolute submodule paths are not supported (%s)", submodule_url)
+        return None, None
+
+    # Get absolute project path
+    # This cannot be done with gitpython directly due to issue:
+    # https://github.com/gitpython-developers/GitPython/issues/730
+    relative_path = os.path.splitext(submodule_url)[0]  # remove .git
+    project_path = project.path_with_namespace
+    while relative_path.startswith(".."):
+        relative_path = relative_path[3:]  # strip off '../'
+        project_path, _ = os.path.split(project_path)  # remove last part
+    submodule_project_path = os.path.join(project_path, relative_path)
+
+    # Get current revision
+    gitmodule_rev = common.get_repository_file_obj(project, submodule, ref=branch)
+
+    return submodule_project_path, gitmodule_rev["id"]
+
+
+def get_submodule_integration_branch_suffix(submodule_project: Project, revision):
+    # Find out if top commit is part of a merge request
+    # If so, use source branch of this MR as integration branch name
+    # Else use commit sha instead
+    integration_branch_suffix = revision
+    for mr in submodule_project.commits.get(revision).merge_requests():
+        if mr["target_branch"] == submodule_project.default_branch:
+            integration_branch_suffix = mr["source_branch"]
+            break
+    return integration_branch_suffix
 
 
 def update_submodule(