From b9cb9a29a80973721956fb6c153c5bc9bfb51dde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20H=C3=B6ppner?= <jonas.hoeppner@garz-fricke.com>
Date: Thu, 24 Mar 2022 16:38:22 +0100
Subject: [PATCH] CI: Add function to merge the gitlab-ci MRs
---
.gitlab-ci.yml | 113 +++++++++++++++++---------
accept_merge_request.py | 2 +
common.py | 48 ++++++++++-
merge_gitlab_ci.py | 173 ++++++++++++++++++++++++++++++++++++++++
update_submodule.py | 54 +++++++++++++
5 files changed, 350 insertions(+), 40 deletions(-)
create mode 100755 merge_gitlab_ci.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7c570c4c..5022ee43 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,6 +20,7 @@ stages:
- analyze
- deploy
- check
+ - merge
workflow:
rules:
@@ -57,12 +58,51 @@ yamllint:
# ---------------------------------------------------------------------------------------
# Stage: deploy-test
# ---------------------------------------------------------------------------------------
+
+.foobar-manifest-projects:
+ variables:
+ PROJECT_ROOT:
+ ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
+ MANIFEST_PROJECT:
+ ${PROJECT_ROOT}/minimal-manifest
+
+ DEPLOY_TO:
+ ${PROJECT_ROOT}/minimal-foo
+ ${PROJECT_ROOT}/minimal-bar
+
+.manifest-projects:
+ variables:
+ PROJECT_ROOT:
+ ${CI_PROJECT_ROOT_NAMESPACE}
+ MANIFEST_PROJECT:
+ ${PROJECT_ROOT}/yocto/manifest
+ DEPLOY_TO:
+ ${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk
+ ${PROJECT_ROOT}/kernel/linux-guf
+ ${PROJECT_ROOT}/kernel/linux-imx-kuk
+ ${PROJECT_ROOT}/kernel/modules/egalaxi2c
+ ${PROJECT_ROOT}/kernel/modules/gfplatdetect
+ ${PROJECT_ROOT}/tools/gf-emc-test-suite
+ ${PROJECT_ROOT}/tools/gf-productiontests
+ ${PROJECT_ROOT}/tools/gfeeprom
+ ${PROJECT_ROOT}/tools/gfxml2dto
+ ${PROJECT_ROOT}/tools/guf-show-demo
+ ${PROJECT_ROOT}/tools/libmdb
+ ${PROJECT_ROOT}/tools/touchcal-conv
+ ${PROJECT_ROOT}/tools/xconfig
+ ${PROJECT_ROOT}/yocto/config
+ ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-bar
+ ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-foo
+ ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-manifest
+ ${PROJECT_ROOT}/yocto/layers/meta-guf-distro
+ ${PROJECT_ROOT}/yocto/layers/meta-guf-machine
+
.deploy:
stage: deploy
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
script:
- cd ${CI_PROJECT_DIR}
- - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then
- MERGE="--merge"; else MERGE=""; fi
- ./deploy_gitlab_ci.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
@@ -70,7 +110,6 @@ yamllint:
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
--verbose
- ${MERGE}
${DEPLOY_TO}
- ./generate_job_from_template.py
@@ -88,51 +127,23 @@ yamllint:
- integration.yml
deploy:
- extends: .deploy
+ extends:
+ - .deploy
+ - .manifest-projects
rules:
# For now to test the merge step
- when: manual
allow_failure: true
- variables:
- PROJECT_ROOT:
- ${CI_PROJECT_ROOT_NAMESPACE}
- MANIFEST_PROJECT:
- ${PROJECT_ROOT}/yocto/manifest
- DEPLOY_TO:
- ${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk
- ${PROJECT_ROOT}/kernel/linux-guf
- ${PROJECT_ROOT}/kernel/linux-imx-kuk
- ${PROJECT_ROOT}/kernel/modules/egalaxi2c
- ${PROJECT_ROOT}/kernel/modules/gfplatdetect
- ${PROJECT_ROOT}/tools/gf-emc-test-suite
- ${PROJECT_ROOT}/tools/gf-productiontests
- ${PROJECT_ROOT}/tools/gfeeprom
- ${PROJECT_ROOT}/tools/gfxml2dto
- ${PROJECT_ROOT}/tools/guf-show-demo
- ${PROJECT_ROOT}/tools/libmdb
- ${PROJECT_ROOT}/tools/touchcal-conv
- ${PROJECT_ROOT}/tools/xconfig
- ${PROJECT_ROOT}/yocto/config
- ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-bar
- ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-foo
- ${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-manifest
- ${PROJECT_ROOT}/yocto/layers/meta-guf-distro
- ${PROJECT_ROOT}/yocto/layers/meta-guf-machine
deploy-foobar:
- extends: .deploy
- variables:
- PROJECT_ROOT:
- ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
- MANIFEST_PROJECT:
- ${PROJECT_ROOT}/minimal-manifest
-
- DEPLOY_TO:
- ${PROJECT_ROOT}/minimal-foo
- ${PROJECT_ROOT}/minimal-bar
+ extends:
+ - .deploy
+ - .foobar-manifest-projects
trigger:
stage: deploy
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
needs: [deploy]
trigger:
include:
@@ -142,12 +153,36 @@ trigger:
trigger-foobar:
stage: deploy
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
needs: [deploy-foobar]
trigger:
include:
- artifact: integration.yml
job: deploy-foobar
strategy: depend
+# --------------------------------------------------------------------------------------
+# Stage: merge
+# --------------------------------------------------------------------------------------
+.merge:
+ stage: merge
+ rules:
+ - if: $CI_COMMIT_BRANCH == "master"
+ script:
+ - cd ${CI_PROJECT_DIR}
+ - ./merge_gitlab_ci.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --manifest-project=${MANIFEST_PROJECT}
+ --submodule=.gitlab-ci
+ --revision=${CI_COMMIT_SHA}
+ --verbose
+ ${DEPLOY_TO}
+
+merge-foobar:
+ extends:
+ - .merge
+ - .foobar-manifest-projects
# --------------------------------------------------------------------------------------
# Stage: check
diff --git a/accept_merge_request.py b/accept_merge_request.py
index b9df28d5..a193e7a1 100755
--- a/accept_merge_request.py
+++ b/accept_merge_request.py
@@ -2,6 +2,7 @@
import common
import argparse
+import logging
import sys
import time
from gitlab import (
@@ -51,6 +52,7 @@ def accept_merge_request(project, mr, rebase=False):
except GitlabMRClosedError as e:
# See HTTP error codes for merge requests here:
# https://docs.gitlab.com/ce/api/merge_requests.html#accept-mr
+ logging.debug("Error from gitlab: %d", e.response_code)
if e.response_code == 405:
# Not allowed (draft, closed, pipeline pending or failed)
diff --git a/common.py b/common.py
index 8642798f..144c7f8d 100755
--- a/common.py
+++ b/common.py
@@ -1,9 +1,10 @@
#!/usr/bin/env python3
+import logging
import requests
import sys
-import logging
import time
+from furl import furl
from git import Actor, GitCommandError
from git.repo.base import Repo
from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
@@ -203,3 +204,48 @@ def get_merge_request(project: Project, merge_request):
except GitlabGetError:
return None
return mr
+
+
+def clone_project(project: Project, into, branch=None):
+
+ gitlab = project.manager.gitlab
+ # If no branch is given, use project's default branch
+ if branch is None:
+ branch = project.default_branch
+
+ # Construct clone url containing access token
+ clone_url = furl(project.http_url_to_repo)
+ clone_url.username = "gitlab-ci"
+ clone_url.password = gitlab.private_token
+
+ # Checkout project
+ try:
+ repo = Repo.clone_from(clone_url.url, into, branch=branch, depth=1)
+ except GitCommandError as e:
+ raise Exception("could not clone repository\n" + str(e)) from e
+ except IndexError:
+ raise Exception("branch '%s' not found" % branch) from e
+ return repo
+
+
+def get_repository_file_raw(project: Project, filename, ref=None):
+ # TODO tree objects are not supported
+ fileobj = get_repository_file_obj(project, filename, ref)
+ return project.repository_raw_blob(
+ fileobj["id"], retry_transient_errors=True
+ ).decode()
+
+
+def get_repository_file_obj(project: Project, filename, ref=None):
+ # TODO tree objects are not supported
+ if ref is None:
+ ref = project.default_branch
+ logging.debug("Using default branch %s", ref)
+ repository_tree = project.repository_tree(ref=ref, retry_transient_errors=True)
+ logging.debug(repository_tree)
+ fileobj = [f for f in repository_tree if f["name"] == filename]
+
+ if len(fileobj) == 0:
+ return None
+ fileobj = fileobj[0]
+ return fileobj
diff --git a/merge_gitlab_ci.py b/merge_gitlab_ci.py
new file mode 100755
index 00000000..bd380b7d
--- /dev/null
+++ b/merge_gitlab_ci.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+from gitlab import Gitlab, GitlabGetError
+from accept_merge_request import accept_merge_request
+import update_submodule
+from gitlab.v4.objects import Project
+
+
+def find_integration_merge_request(
+ project: Project, submodule, revision, target_branch
+):
+
+ (
+ submodule_path,
+ submodule_revision,
+ ) = update_submodule.get_submodule_project_path_and_revision(
+ project, submodule, target_branch
+ )
+ logging.debug("Module: %s, Revision: %s", submodule_path, submodule_revision)
+
+ # Get submodule project
+ gitlab = project.manager.gitlab
+ submodule_project = common.get_project(gitlab, submodule_path)
+
+ # Get the integration branch name
+ integration_branch_suffix = (
+ update_submodule.get_submodule_integration_branch_suffix(
+ submodule_project, revision
+ )
+ )
+ integration_branch_name = common.integration_branch_name(
+ submodule_project.name, integration_branch_suffix
+ )
+ logging.debug(integration_branch_name)
+
+ # Get the merge request for the branch TODO catch exception
+ try:
+ integration_branch = project.branches.get(
+ integration_branch_name, retry_transient_errors=True
+ )
+ except GitlabGetError:
+ print("ERROR: Failed to find integration branch for {}".format(submodule))
+ return None
+
+ logging.debug(integration_branch)
+ commit = project.commits.get(
+ integration_branch.commit["id"], retry_transient_errors=True
+ )
+ for mr in commit.merge_requests(retry_transient_errors=True):
+ if mr["target_branch"] == target_branch:
+ integration_mr = mr
+ break
+ if integration_mr is None:
+ print(
+ "ERROR: Failed to find integration merge request for %s",
+ integration_branch_name,
+ )
+ return None
+ logging.debug(integration_mr["iid"])
+ mr = project.mergerequests.get(integration_mr["iid"], retry_transient_errors=True)
+
+ return mr
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ "--manifest-project",
+ help="""name of the GitLab project""",
+ dest="project",
+ )
+ parser.add_argument(
+ "--submodule",
+ help="""submodule to update""",
+ dest="submodule",
+ required=True,
+ )
+ parser.add_argument(
+ "--revision",
+ help="""new revision for submodule""",
+ dest="revision",
+ required=True,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""project branch (if not default branch)""",
+ dest="branch",
+ required=False,
+ default=None,
+ )
+ parser.add_argument(
+ "--merge",
+ help="""if set, perform merge after integration""",
+ dest="merge",
+ action="store_true",
+ required=False,
+ default=False,
+ )
+ parser.add_argument(
+ "projects",
+ help="""List of projects the change should be deployed to additionally
+ to the manifest project given as named parameter.""",
+ nargs="*",
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ help="""Increase verbosity.""",
+ )
+
+ args, _ = parser.parse_known_args()
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+ for p in args.projects + [args.project]:
+ project = common.get_project(gitlab, p)
+ branch = args.branch
+
+ if branch is None:
+ branch = project.default_branch
+
+ mr = find_integration_merge_request(
+ project, args.submodule, args.revision, branch
+ )
+
+ logging.debug("Merge %s!%d: %s", project.name, mr.iid, mr.title)
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(project, mr)
+
+ # Attempt to merge
+ merged = accept_merge_request(
+ project,
+ mr,
+ # Only the file manifest commit may be rebased
+ # other rebase would require new integration commits
+ rebase=(p == args.project),
+ )
+
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged. You have two possibilities to fix "
+ "this:\n"
+ " 1. Checkout the MR and rebase it on the current master manually, or\n"
+ " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
+ "In either case restart this job afterwards in order to get it merged."
+ )
+ print("Successfully merged")
+
+ exit()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/update_submodule.py b/update_submodule.py
index d749fa57..17167ef9 100755
--- a/update_submodule.py
+++ b/update_submodule.py
@@ -6,9 +6,63 @@ import logging
import os
import sys
import tempfile
+from configparser import ConfigParser
from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab
+from gitlab.v4.objects import Project
+
+
+def get_submodule_project_path_and_revision(project: Project, submodule, branch=None):
+
+ gitmodules = common.get_repository_file_raw(project, ".gitmodules", ref=branch)
+ if gitmodules is None:
+ logging.error("Submodule %s not found in %s.", submodule, project.name)
+ return None, None
+
+ logging.debug("Gitmodules: %s", gitmodules)
+
+ cfgparse = ConfigParser()
+ cfgparse.read_string(gitmodules)
+ try:
+ section = cfgparse['submodule "{}"'.format(submodule)]
+ except KeyError:
+ logging.error("Submodule %s not found in %s.", submodule, project.name)
+ return None, None
+
+ submodule_url = section["url"]
+ # absolut path to a relative submodule
+ # Check for relative path
+ if not submodule_url.startswith(".."):
+ logging.error("absolute submodule paths are not supported (%s)", submodule_url)
+ return None, None
+
+ # Get absolute project path
+ # This cannot be done with gitpython directly due to issue:
+ # https://github.com/gitpython-developers/GitPython/issues/730
+ relative_path = os.path.splitext(submodule_url)[0] # remove .git
+ project_path = project.path_with_namespace
+ while relative_path.startswith(".."):
+ relative_path = relative_path[3:] # strip off '../'
+ project_path, _ = os.path.split(project_path) # remove last part
+ submodule_project_path = os.path.join(project_path, relative_path)
+
+ # Get current revision
+ gitmodule_rev = common.get_repository_file_obj(project, submodule, ref=branch)
+
+ return submodule_project_path, gitmodule_rev["id"]
+
+
+def get_submodule_integration_branch_suffix(submodule_project: Project, revision):
+ # Find out if top commit is part of a merge request
+ # If so, use source branch of this MR as integration branch name
+ # Else use commit sha instead
+ integration_branch_suffix = revision
+ for mr in submodule_project.commits.get(revision).merge_requests():
+ if mr["target_branch"] == submodule_project.default_branch:
+ integration_branch_suffix = mr["source_branch"]
+ break
+ return integration_branch_suffix
def update_submodule(
--
GitLab