From b40e6093c8940f5035de5028d0c48fa201ec924f Mon Sep 17 00:00:00 2001
From: Tim Jaacks <tim.jaacks@garz-fricke.com>
Date: Mon, 16 Nov 2020 11:04:40 +0100
Subject: [PATCH] Add gitlab ci scripts
---
.gitlab-ci.yml | 32 ++++++
accept_merge_request.py | 144 ++++++++++++++++++++++++++
check_pipeline_status.py | 66 ++++++++++++
common.py | 107 +++++++++++++++++++
integrate_into_manifest.py | 203 +++++++++++++++++++++++++++++++++++++
merge_into_manifest.py | 179 ++++++++++++++++++++++++++++++++
pylintrc | 10 ++
test.py | 24 +++++
8 files changed, 765 insertions(+)
create mode 100644 .gitlab-ci.yml
create mode 100755 accept_merge_request.py
create mode 100755 check_pipeline_status.py
create mode 100755 common.py
create mode 100755 integrate_into_manifest.py
create mode 100755 merge_into_manifest.py
create mode 100644 pylintrc
create mode 100755 test.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000..0b035fb5
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------------------------
+# Global
+#-------------------------------------------------------------------------------
+image:
+ name: "python:3.9"
+
+stages:
+ - analyze
+
+#-------------------------------------------------------------------------------
+# Stage: analyze
+#-------------------------------------------------------------------------------
+pylint:
+ stage: analyze
+ timeout: 2m
+ before_script:
+ # FIXME: prepare docker image with modules already installed
+ - pip3 install furl
+ - pip3 install gitpython
+ - pip3 install lxml
+ - pip3 install python-gitlab
+ - pip3 install pylint
+ script:
+ - pylint --rcfile=pylintrc *.py
+
+black:
+ stage: analyze
+ timeout: 2m
+ before_script:
+ - pip3 install black
+ script:
+ - black --diff --check *.py
diff --git a/accept_merge_request.py b/accept_merge_request.py
new file mode 100755
index 00000000..b3006c32
--- /dev/null
+++ b/accept_merge_request.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+import time
+from gitlab import (
+ Gitlab,
+ GitlabGetError,
+ GitlabMRClosedError,
+)
+
+critical_error = (
+ "This is a critical error! Please make sure to:\n"
+ " 1. merge the above-mentioned merge request by hand as soon as possible\n"
+ " (this is important, otherwise following merge requests will get stuck, too)\n"
+ " 2. examine why this has happened and fix it in the CI pipeline"
+)
+
+
+def accept_merge_request(gitlab, project, mr, rebase=False):
+ """Attempt to merge a merge request, rebase if necessary"""
+ merged = False
+ pipeline_pending = False
+
+ while not merged:
+ # Update merge request before trying to merge it in order to get the latest
+ # pipeline status
+ try:
+ updated_mr = project.mergerequests.get(id=mr.iid)
+ mr = updated_mr
+ except GitlabGetError as e:
+ print("WARNING: Could not update merge request object: %s" % e)
+
+ # Try to merge the merge request
+ try:
+ mr.merge()
+ merged = True
+ if pipeline_pending:
+ print("")
+
+ except GitlabMRClosedError as e:
+ # See HTTP error codes for merge requests here:
+ # https://docs.gitlab.com/ce/api/merge_requests.html#accept-mr
+
+ if e.response_code == 405:
+ # Not allowed (draft, closed, pipeline pending or failed)
+ # If pipeline is running, wait for completion
+ if not mr.head_pipeline:
+ # No pipeline created yet
+ print("No pipeline created yet")
+ time.sleep(1)
+ elif mr.head_pipeline["status"] in common.pending_states:
+ # Pipeline pending
+ if not pipeline_pending:
+ print("Waiting for pending pipeline", end="", flush=True)
+ pipeline_pending = True
+ print(".", end="", flush=True)
+ time.sleep(1)
+ else:
+ # Merge conflict, automatic rebase not possible
+ if pipeline_pending:
+ print("")
+ print("Merge not possible, has to be rebased manually")
+ return False
+
+ elif e.response_code == 406:
+ # Merge conflict, automatic rebase is possible
+ if pipeline_pending:
+ print("")
+ pipeline_pending = False
+ print("Merge not possible, but branch can be automatically rebased")
+ if rebase:
+ print("Trying to rebase...")
+ mr = common.rebase_merge_request(gitlab, project, mr)
+ if mr.merge_error:
+ print("ERROR: rebase not possible\n'%s'" % mr.merge_error)
+ sys.exit(critical_error)
+ print("Sucessfully rebased")
+ else:
+ return False
+
+ else:
+ if pipeline_pending:
+ print("")
+ print("ERROR: merge not possible: %s" % e)
+ sys.exit(critical_error)
+
+ return True
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--merge-request",
+ help="""id of the merge request""",
+ dest="merge_request",
+ required=True,
+ )
+ parser.add_argument(
+ "--rebase",
+ help="""attempt to automatically rebase merge request if necessary""",
+ dest="rebase",
+ action="store_true",
+ required=False,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+
+ with gitlab:
+ try:
+ merge_request = project.mergerequests.get(id=args.merge_request)
+ except GitlabGetError as e:
+ sys.exit("Could not get merge request: %s" % e)
+
+ if accept_merge_request(gitlab, project, merge_request, rebase=args.rebase):
+ print("Successfully merged")
+ else:
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/check_pipeline_status.py b/check_pipeline_status.py
new file mode 100755
index 00000000..c02108d8
--- /dev/null
+++ b/check_pipeline_status.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+import time
+from gitlab import Gitlab
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--commit",
+ help="""sha of the project commit to check""",
+ dest="commit",
+ required=True,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ with Gitlab(args.gitlab_url, private_token=args.token) as gitlab:
+
+ # Find project
+ project = common.get_project(gitlab, args.project)
+
+ # Find pipeline for commit
+ pipelines = project.pipelines.list(sha=args.commit)
+ if not pipelines:
+ print("ERROR: no pipeline for commit '%s' found" % args.commit)
+ sys.exit(1)
+ pipeline = pipelines[0]
+
+ # Wait for pipeline termination
+ print("Waiting for pipeline %s" % pipeline.web_url)
+ terminated_states = ["success", "failed", "canceled", "skipped"]
+ while pipeline.status not in terminated_states:
+ print(".", end="", flush=True)
+ time.sleep(1)
+ pipeline = project.pipelines.get(pipeline.id)
+ print("")
+
+ print("Result: %s" % pipeline.status)
+ if pipeline.status != "success":
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/common.py b/common.py
new file mode 100755
index 00000000..f48a9575
--- /dev/null
+++ b/common.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import requests
+import sys
+import time
+from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
+
+
+manifest_file = "default.xml"
+pending_states = ["created", "waiting_for_resource", "preparing", "pending", "running"]
+
+
+def integration_branch_name(project_name, branch_name):
+ """Get integration branch name"""
+ return "integrate/" + project_name + "/" + branch_name
+
+
+def get_project(gitlab, project_name):
+ """Get a GitLab project by its name"""
+ with gitlab:
+ project = None
+ try:
+ for p in gitlab.projects.list(
+ search=project_name, retry_transient_errors=True
+ ):
+ if p.name == project_name:
+ project = p
+ if not project:
+ sys.exit("ERROR: project '%s' not found" % project_name)
+ except requests.ConnectionError:
+ sys.exit("ERROR: could not connect to GitLab server")
+ except GitlabAuthenticationError:
+ sys.exit("ERROR: authentication failed")
+ return project
+
+
+def get_latest_commit(gitlab, project, branch_name):
+ """Get latest commit on a given project branch"""
+ with gitlab:
+ try:
+ branch = project.branches.get(branch_name)
+ except GitlabGetError as e:
+ sys.exit(
+ "ERROR: could not get branch '%s' for project '%s': %s"
+ % (branch_name, project.name, e)
+ )
+ if not branch:
+ sys.exit(
+ "ERROR: branch '%s' not found in project %s"
+ % (branch_name, project.name)
+ )
+ return branch.commit
+
+
+def get_merge_request(
+ gitlab, project, state, source_branch=None, target_branch=None, commit=None
+):
+ """Get merge request by source and target branch and optionally commit sha"""
+ merge_request = None
+ with gitlab:
+ try:
+ merge_requests = project.mergerequests.list(
+ source_branch=source_branch,
+ target_branch=target_branch,
+ state=state,
+ all=True,
+ retry_transient_errors=True,
+ )
+ except GitlabGetError as e:
+ sys.exit(
+ "ERROR: could not list merge requests for project '%s': %s"
+ % (project.name, e)
+ )
+ if commit:
+ for mr in merge_requests:
+ if mr.sha == commit:
+ merge_request = mr
+ elif merge_requests:
+ merge_request = merge_requests[0]
+ return merge_request
+
+
+def rebase_merge_request(gitlab, project, merge_request):
+ """Attempt to rebase a merge request and return the updated merge request object"""
+ # Rebasing takes more than one API call, see:
+ # https://docs.gitlab.com/ce/api/merge_requests.html#rebase-a-merge-request
+ with gitlab:
+ try:
+ merge_request.rebase()
+ except GitlabMRRebaseError as e:
+ merge_request.merge_error = "Could not rebase merge request: %s" % e
+ return merge_request
+ rebase_in_progress = True
+ while rebase_in_progress:
+ time.sleep(1)
+ try:
+ updated_merge_request = project.mergerequests.get(
+ id=merge_request.iid,
+ query_parameters={"include_rebase_in_progress": "True"},
+ )
+ except GitlabGetError as e:
+ merge_request.merge_error = (
+ "Could not get updated merge request: %s" % e
+ )
+ return merge_request
+ rebase_in_progress = updated_merge_request.rebase_in_progress
+ return updated_merge_request
diff --git a/integrate_into_manifest.py b/integrate_into_manifest.py
new file mode 100755
index 00000000..101bfeb4
--- /dev/null
+++ b/integrate_into_manifest.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+import tempfile
+from furl import furl
+from git import Actor, GitCommandError, Repo
+from gitlab import Gitlab, GitlabGetError
+from lxml import etree
+
+
+def integrate_into_manifest(
+ gitlab,
+ manifest_project,
+ integration_base,
+ manifest_file,
+ project,
+ branch,
+ commit=None,
+):
+ with tempfile.TemporaryDirectory() as manifest_dir:
+ manifest_filepath = manifest_dir + "/" + manifest_file
+
+ # Construct clone url containing access token
+ clone_url = furl(manifest_project.http_url_to_repo)
+ clone_url.username = "gitlab-ci"
+ clone_url.password = gitlab.private_token
+
+ # Checkout manifest
+ try:
+ manifest_repo = Repo.clone_from(clone_url.url, manifest_dir)
+ manifest_repo.heads[integration_base].checkout()
+ except GitCommandError as e:
+ sys.exit("ERROR: could not clone manifest repository\n" + str(e))
+ except IndexError as e:
+ sys.exit("ERROR: branch '%s' not found" % integration_base)
+
+ # Create integration branch (delete former one if already exists)
+ integration_branch = common.integration_branch_name(project.name, branch)
+ integration_base = None
+ for ref in manifest_repo.references:
+ if integration_branch == ref.name:
+ manifest_repo.delete_head(ref)
+ integration_base = manifest_repo.create_head(integration_branch)
+ manifest_repo.head.reference = integration_base
+
+ # Parse manifest file
+ try:
+ manifest = etree.parse(manifest_filepath)
+ except FileNotFoundError:
+ sys.exit("ERROR: file '%s' not found in manifest repo" % manifest_file)
+
+ # Find project references in manifest
+ project_nodes = manifest.findall("project[@name='%s']" % project.path)
+ if not project_nodes:
+ sys.exit("ERROR: project '%s' not found in manifest" % project.path)
+ project_node = project_nodes[0]
+
+ # Get current project revision
+ old_revision = project_node.get("revision")
+
+ # Get project revision
+ if commit:
+ try:
+ project_commit = project.commits.get(commit)
+ except GitlabGetError as e:
+ sys.exit(
+ "ERROR: could not get commit '%s' for project '%s': %s"
+ % (commit, project.name, e)
+ )
+ # project_commit is a Gitlab.ProjectCommit object here
+ new_revision = project_commit.id
+ commit_message = project_commit.message
+ else:
+ project_commit = common.get_latest_commit(gitlab, project, branch)
+ # project_commit is a simple dictionary here
+ new_revision = project_commit["id"]
+ commit_message = project_commit["message"]
+
+ # Update manifest file
+ # We are doing this using a plain text replace action. Unfortunately
+ # all python libraries for handling XML data are not able to preserve
+ # the file layout, and we want a minimal diff.
+ with open(manifest_filepath, "r") as file:
+ content = file.read()
+ content = content.replace(old_revision, new_revision)
+ with open(manifest_filepath, "w") as file:
+ file.write(content)
+
+ # Construct commit object and commit change
+ gitlab.auth()
+ author = Actor(gitlab.user.username, gitlab.user.email)
+ commit_link = project.web_url + "/-/commit/" + new_revision
+ manifest_repo.index.add([manifest_file])
+ manifest_repo.index.commit(
+ "Integrate %s/%s\n\n" % (project.path, branch)
+ + "Commit: %s\n\n" % (commit_link)
+ + commit_message,
+ author=author,
+ committer=author,
+ )
+
+ # Push commit
+ try:
+ origin = manifest_repo.remote("origin")
+ origin.push(integration_branch, force=True)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not commit changes\n" + str(e))
+
+ # Print commit information
+ manifest_revision = manifest_repo.head.commit.hexsha
+ print("Pushed new commit:")
+ print(manifest_project.web_url + "/-/commit/" + manifest_revision)
+ print(manifest_repo.git.show("--summary", "--decorate"))
+
+ return manifest_revision
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-project",
+ help="""name of the manifest project""",
+ dest="manifest_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--integration-base",
+ help="""manifest branch to branch off from""",
+ dest="integration_base",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-file",
+ help="""manifest file name (default: 'default.xml')""",
+ dest="manifest_file",
+ default=common.manifest_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the project, as specified in the manifest""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""project branch to be integrated""",
+ dest="branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--commit",
+ help="""project commit""",
+ dest="commit",
+ default=None,
+ required=False,
+ )
+ parser.add_argument(
+ "--save-revision-to",
+ help="""path to a file where the new manifest revision is stored""",
+ dest="revision_file",
+ required=False,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+ manifest_project = common.get_project(gitlab, args.manifest_project)
+ project = common.get_project(gitlab, args.project)
+
+ with gitlab:
+ manifest_revision = integrate_into_manifest(
+ gitlab=gitlab,
+ manifest_project=manifest_project,
+ integration_base=args.integration_base,
+ manifest_file=args.manifest_file,
+ project=project,
+ branch=args.branch,
+ commit=args.commit,
+ )
+
+ if args.revision_file:
+ with open(args.revision_file, "w") as file:
+ file.write(manifest_revision)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/merge_into_manifest.py b/merge_into_manifest.py
new file mode 100755
index 00000000..98a87bec
--- /dev/null
+++ b/merge_into_manifest.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+from gitlab import (
+ Gitlab,
+ GitlabGetError,
+)
+from accept_merge_request import accept_merge_request
+from integrate_into_manifest import integrate_into_manifest
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-project",
+ help="""name of the manifest project""",
+ dest="manifest_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--master-branch",
+ help="""master branch to merge changes into""",
+ dest="master_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the project containing the commit to be merged""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--commit",
+ help="""sha of the commit to be merged""",
+ dest="commit",
+ required=True,
+ )
+ parser.add_argument(
+ "--save-revision-to",
+ help="""path to a file where the new manifest revision is stored""",
+ dest="revision_file",
+ required=False,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+ manifest_project = common.get_project(gitlab, args.manifest_project)
+
+ # Get source merge request
+ source_mr = common.get_merge_request(
+ gitlab,
+ project,
+ target_branch=args.master_branch,
+ state="merged",
+ commit=args.commit,
+ )
+ if not source_mr:
+ sys.exit(
+ "ERROR: could not determine source merge request for commit %s"
+ % args.commit
+ )
+
+ # Get original branch for commit
+ original_branch = source_mr.source_branch
+ integration_branch = common.integration_branch_name(project.name, original_branch)
+ target_branch = args.master_branch
+
+ # Check if merge request already exists
+ mr = common.get_merge_request(
+ gitlab,
+ manifest_project,
+ source_branch=integration_branch,
+ target_branch=target_branch,
+ state="opened",
+ )
+ if mr:
+ sys.exit("ERROR: There is already an open merge request:\n%s" % mr.web_url)
+
+ with gitlab:
+
+ # Check if branches exist
+ try:
+ manifest_project.branches.get(integration_branch)
+ except GitlabGetError:
+ sys.exit("ERROR: source branch '%s' does not exist." % integration_branch)
+ try:
+ manifest_project.branches.get(target_branch)
+ except GitlabGetError:
+ sys.exit("ERROR: target branch '%s' does not exist." % target_branch)
+
+ # Create new merge request
+ mr = manifest_project.mergerequests.create(
+ {
+ "source_branch": integration_branch,
+ "target_branch": target_branch,
+ "remove_source_branch": True,
+ "title": "Merge " + integration_branch,
+ }
+ )
+
+ print("Created new merge request:")
+ print(mr.web_url)
+
+ # Insert cross-links in both merge requests
+ mr.notes.create({"body": "Source merge request: %s" % source_mr.web_url})
+ source_mr.notes.create({"body": "Integration merge request: %s" % mr.web_url})
+
+ # Attempt to merge, reintegrate if necessary
+ manifest_revision = mr.sha
+ merged = False
+ while not merged:
+ merged = accept_merge_request(gitlab, manifest_project, mr)
+ if not merged:
+ # Note: if reintegration is necessary here, the source merge request was
+ # merged without running the pipeline on the latest manifest, i.e. some
+ # other project has been merged in between. Automatic rebase is not
+ # possible, though, because the altered manifest lines are part of the
+ # diff context, so we have to create a new integration commit. This
+ # possibly might result in a failing pipeline state on the master.
+ manifest_revision = integrate_into_manifest(
+ gitlab=gitlab,
+ manifest_project=manifest_project,
+ integration_base=target_branch,
+ manifest_file=common.manifest_file,
+ project=project,
+ branch=original_branch,
+ commit=args.commit,
+ )
+
+ print("Successfully merged")
+
+ # Check if there is a running pipeline on the integration branch and cancel it.
+ # This happens when a reintegration was necessary before merging. The instant
+ # merge afterwards deletes the integration branch, so the pipeline on the branch
+ # will fail, because repo cannot checkout the code anymore. In order not to
+ # confuse people, we better cancel it. Since there is a pipeline running on the
+ # master after merging, we don't need the branch results anyway.
+ try:
+ pipelines = manifest_project.pipelines.list(
+ sha=manifest_revision,
+ ref=integration_branch,
+ retry_transient_errors=True,
+ )
+ except GitlabGetError:
+ print("WARNING: could not list pipelines for project '%s'" % project.name)
+ if pipelines:
+ pipeline = pipelines[0]
+ if pipeline.status in common.pending_states:
+ pipeline.cancel()
+ print(
+ "Cancelling running pipeline for integration branch '%s':"
+ % integration_branch
+ )
+ print(pipeline.web_url)
+
+ # Write new manifest revision to file for next pipeline stage
+ if args.revision_file:
+ with open(args.revision_file, "w") as file:
+ file.write(manifest_revision)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pylintrc b/pylintrc
new file mode 100644
index 00000000..6bc07365
--- /dev/null
+++ b/pylintrc
@@ -0,0 +1,10 @@
+[MESSAGES CONTROL]
+
+# Disable some verifications
+disable =
+ C, # conventions
+ R, # refactoring
+ W0511, # fixme warnings
+
+# Whitelist lxml package to disable I1101
+extension-pkg-whitelist = lxml
diff --git a/test.py b/test.py
new file mode 100755
index 00000000..e33c1951
--- /dev/null
+++ b/test.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+import common
+
+from gitlab import Gitlab
+
+
+def main():
+ gitlab = Gitlab("https://gitlab.com", private_token="9q7XsWMpiv_MM5z6HiHw")
+ project = common.get_project(gitlab, "minimal-foo")
+
+ with gitlab:
+ merge_request = common.get_merge_request(
+ gitlab,
+ project,
+ state="merged",
+ source_branch="remove-file-3",
+ target_branch="master",
+ commit="1d5c8da0a4826cceef1abf99c23d7c3585754f19",
+ )
+ print(merge_request)
+
+
+if __name__ == "__main__":
+ main()
--
GitLab