From 00f04bda40df1e8066c71b89cc0e3dda7f697389 Mon Sep 17 00:00:00 2001
From: Tim Jaacks <tim.jaacks@garz-fricke.com>
Date: Tue, 15 Jun 2021 16:09:08 +0200
Subject: [PATCH] Add deploy stage
The jobs in the deploy stage have to be triggered manually in GitLab.
There is one deploy job for each project which uses the gitlab-ci
scripts as a submodule, so that the deployment can be performed step by
step.
If executed within MR context, an integration MR is created and left
open. The user can extend this integration MR, e.g. if CI scripts have
been renamed, changed command line arguments or other changes requiring
updates of the correspronding .gitlab-ci.yml file. Subsequent runs of
this job will re-create the integration branch, so manual changes are
lost in this case.
If executed on the master branch (i.e. after the source MR has been
merged), the job does exactly the same, plus the integration MR will be
automatically merged. If this fails, the job will fail as well.
---
.gitlab-ci.yml | 34 +++++++
common.py | 65 +++++++++++++
create_merge_request.py | 92 ++++++++++++++++++
deploy_gitlab_ci.py | 116 ++++++++++++++++++++++
get_merge_requests.py | 4 +-
integrate_into_manifest.py | 41 ++++----
merge_into_manifest.py | 55 +++--------
update_submodule.py | 194 +++++++++++++++++++++++++++++++++++++
8 files changed, 532 insertions(+), 69 deletions(-)
create mode 100755 create_merge_request.py
create mode 100755 deploy_gitlab_ci.py
create mode 100755 update_submodule.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cb98394a..632b1258 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,6 +10,11 @@ default:
stages:
- analyze
+ - deploy
+
+workflow:
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
#---------------------------------------------------------------------------------------
# Stage: analyze
@@ -31,3 +36,32 @@ executable:
timeout: 2m
script:
- (! find ! -executable -name "*.py" -exec echo "not executable:"" {}" \; | grep .)
+
+#---------------------------------------------------------------------------------------
+# Stage: deploy
+#---------------------------------------------------------------------------------------
+.deploy: &deploy
+ stage: deploy
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
+ when: manual
+ - if: $CI_COMMIT_BRANCH == "master"
+ when: manual
+ allow_failure: yes
+ script:
+ - cd ${CI_PROJECT_DIR}
+ - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi
+ - ./deploy_gitlab_ci.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --project=${CI_JOB_NAME}
+ --submodule=.gitlab-ci
+ --revision=${CI_COMMIT_SHA}
+ ${MERGE}
+
+garz-fricke/yocto/infrastructure/ci-test/minimal-bar: *deploy
+garz-fricke/yocto/infrastructure/ci-test/minimal-foo: *deploy
+garz-fricke/yocto/infrastructure/ci-test/minimal-manifest: *deploy
+garz-fricke/yocto/layers/meta-guf-distro: *deploy
+garz-fricke/yocto/layers/meta-guf-machine: *deploy
+garz-fricke/yocto/manifest: *deploy
diff --git a/common.py b/common.py
index 10b23fe7..4b7e1f74 100755
--- a/common.py
+++ b/common.py
@@ -3,7 +3,11 @@
import requests
import sys
import time
+from git import Actor, GitCommandError
+from git.repo.base import Repo
from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
+from gitlab.v4.objects import Project
+from gitlab.v4.objects import MergeRequest
manifest_file = "default.xml"
@@ -79,3 +83,64 @@ def rebase_merge_request(project, merge_request):
return merge_request
rebase_in_progress = updated_merge_request.rebase_in_progress
return updated_merge_request
+
+
+def crosslink_merge_requests(source_mr: MergeRequest, integration_mr: MergeRequest):
+ """Insert cross-links in merge requests"""
+ integration_mr.notes.create(
+ {"body": "Source merge request: %s" % source_mr.web_url}
+ )
+ source_mr.notes.create(
+ {"body": "Integration merge request: %s" % integration_mr.web_url}
+ )
+
+
+def wait_until_merge_status_is_set(project: Project, mr: MergeRequest):
+ """Periodically query MR until GitLab has checked its merge status"""
+ print("Waiting until merge status has been checked", end="", flush=True)
+ unchecked_states = ["unchecked", "checking", "cannot_be_merged_recheck"]
+ while mr.merge_status in unchecked_states:
+ print(".", end="", flush=True)
+ time.sleep(1)
+ mr = project.mergerequests.get(mr.iid, retry_transient_errors=True)
+ print(" -> %s" % mr.merge_status)
+
+
+def list_commits(commits):
+ """Create a list of commits along with the commit messages"""
+ commit_list = ""
+ for commit in commits:
+ commit_list += "\n---\n\nCommit: %s\n\n%s" % (commit.web_url, commit.message)
+ return commit_list
+
+
+def commit_and_push(project: Project, repo: Repo, branch, message, name, email):
+ """Commit and push to a repo branch"""
+ author = Actor(name, email)
+ repo.index.commit(message, author=author, committer=author)
+
+ # Push commit
+ try:
+ origin = repo.remote("origin")
+ origin.push(branch, force=True)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not commit changes\n" + str(e))
+
+ # Print commit information
+ revision = repo.head.commit.hexsha
+ print("Pushed new commit:")
+ print(project.web_url + "/-/commit/" + revision)
+ print(repo.git.show("--summary", "--decorate"))
+
+ return revision
+
+
+def get_submodule(repo: Repo, submodule_name):
+ """Find a submodule in a Git repository by its name"""
+ submodule = None
+ for sm in repo.submodules:
+ if sm.name == submodule_name:
+ submodule = sm
+ if submodule is None:
+ sys.exit("ERROR: submodule '%s' not found" % submodule_name)
+ return submodule
diff --git a/create_merge_request.py b/create_merge_request.py
new file mode 100755
index 00000000..62123cec
--- /dev/null
+++ b/create_merge_request.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+from gitlab import Gitlab, GitlabGetError
+from gitlab.v4.objects import Project
+from get_merge_requests import get_merge_requests
+
+
+def create_merge_request(project: Project, source_branch, target_branch):
+ """Create a merge request"""
+ # Check if merge request already exists
+ mrs = get_merge_requests(
+ project,
+ source_branch=source_branch,
+ target_branch=target_branch,
+ state="opened",
+ )
+ if mrs:
+ return mrs[0], False
+
+ # Check if branches exist
+ try:
+ project.branches.get(source_branch, retry_transient_errors=True)
+ except GitlabGetError:
+ sys.exit("ERROR: source branch '%s' does not exist." % source_branch)
+ try:
+ project.branches.get(target_branch, retry_transient_errors=True)
+ except GitlabGetError:
+ sys.exit("ERROR: target branch '%s' does not exist." % target_branch)
+
+ # Create new merge request
+ mr = project.mergerequests.create(
+ {
+ "source_branch": source_branch,
+ "target_branch": target_branch,
+ "remove_source_branch": True,
+ "title": "Merge " + source_branch,
+ }
+ )
+
+ return mr, True
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--source-branch",
+ help="""source branch of the merge request""",
+ dest="source_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--target-branch",
+ help="""target branch of the merge request""",
+ dest="target_branch",
+ required=True,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+
+ create_merge_request(
+ project,
+ source_branch=args.source_branch,
+ target_branch=args.target_branch,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
new file mode 100755
index 00000000..30a1dad7
--- /dev/null
+++ b/deploy_gitlab_ci.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+from gitlab import Gitlab
+from accept_merge_request import accept_merge_request
+from create_merge_request import create_merge_request
+from get_merge_requests import get_merge_requests
+from update_submodule import update_submodule
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--submodule",
+ help="""submodule to update""",
+ dest="submodule",
+ required=True,
+ )
+ parser.add_argument(
+ "--revision",
+ help="""new revision for submodule""",
+ dest="revision",
+ required=True,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""project branch (if not default branch)""",
+ dest="branch",
+ required=False,
+ default=None,
+ )
+ parser.add_argument(
+ "--merge",
+ help="""if set, perform merge after integration""",
+ dest="merge",
+ action="store_true",
+ required=False,
+ default=False,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+
+ # Update submodule
+ integration_branch, _, submodule_project = update_submodule(
+ project, args.submodule, args.revision, args.branch
+ )
+
+ # Get source merge request
+ mrs = get_merge_requests(
+ submodule_project,
+ target_branch="master",
+ commit=args.revision,
+ )
+ if not mrs:
+ sys.exit(
+ "ERROR: could not determine source merge request for commit %s"
+ % args.revision
+ )
+ source_mr = mrs[0]
+
+ # Create merge request
+ mr, created = create_merge_request(
+ project, integration_branch, project.default_branch
+ )
+ if created:
+ common.crosslink_merge_requests(source_mr, mr)
+ print("Created new merge request:\n%s" % mr.web_url)
+ else:
+ print("Existing integration merge request:\n%s" % mr.web_url)
+
+ if not args.merge:
+ print(
+ "Skipping automatic merge in MR context. If you like to extend the "
+ "integration MR by hand, please do it now. Afterwards you can either merge "
+ "it by hand or re-run this job on the master branch after the source MR "
+ "has been merged."
+ )
+ sys.exit(0)
+
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(project, mr)
+
+ # Attempt to merge
+ merged = accept_merge_request(project, mr)
+
+ if not merged:
+ sys.exit("Integration MR could not be merged")
+
+ print("Successfully merged")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/get_merge_requests.py b/get_merge_requests.py
index 8846d5af..c1fc7c4d 100755
--- a/get_merge_requests.py
+++ b/get_merge_requests.py
@@ -7,7 +7,7 @@ from gitlab import Gitlab, GitlabGetError
def get_merge_requests(
- project, state, source_branch=None, target_branch=None, commit=None
+ project, state=None, source_branch=None, target_branch=None, commit=None
):
"""Get merge request by source and target branch and optionally commit sha"""
merge_requests = []
@@ -15,7 +15,7 @@ def get_merge_requests(
all_merge_requests = project.mergerequests.list(
source_branch=source_branch,
target_branch=target_branch,
- state=state,
+ state=state if state else "all",
all=True,
retry_transient_errors=True,
)
diff --git a/integrate_into_manifest.py b/integrate_into_manifest.py
index 68bb9391..986b98ed 100755
--- a/integrate_into_manifest.py
+++ b/integrate_into_manifest.py
@@ -6,7 +6,7 @@ import sys
import tempfile
from pathlib import Path
from furl import furl
-from git import Actor, GitCommandError, Repo
+from git import GitCommandError, Repo
from gitlab import Gitlab, GitlabGetError
from lxml import etree
@@ -42,12 +42,10 @@ def integrate_into_manifest(
integration_branch = common.integration_branch_name(
project.name, merge_request.source_branch
)
- integration_base = None
for ref in manifest_repo.references:
if integration_branch == ref.name:
manifest_repo.delete_head(ref)
- integration_base = manifest_repo.create_head(integration_branch)
- manifest_repo.head.reference = integration_base
+ manifest_repo.head.reference = manifest_repo.create_head(integration_branch)
# Parse manifest file
try:
@@ -80,30 +78,25 @@ def integrate_into_manifest(
content = manifest_filepath.read_text()
content = content.replace(old_revision, new_revision)
manifest_filepath.write_text(content)
+ manifest_repo.index.add([manifest_file])
# Make an API request to create the gitlab.user object
gitlab.auth()
- # Construct commit object and commit change
- author = Actor(gitlab.user.username, gitlab.user.email)
- message = "Integrate %s/%s\n" % (project.path, merge_request.source_branch)
- for commit in merge_request.commits():
- message += "\nCommit: %s\n\n%s" % (commit.web_url, commit.message)
- manifest_repo.index.add([manifest_file])
- manifest_repo.index.commit(message, author=author, committer=author)
-
- # Push commit
- try:
- origin = manifest_repo.remote("origin")
- origin.push(integration_branch, force=True)
- except GitCommandError as e:
- sys.exit("ERROR: could not commit changes\n" + str(e))
-
- # Print commit information
- manifest_revision = manifest_repo.head.commit.hexsha
- print("Pushed new commit:")
- print(manifest_project.web_url + "/-/commit/" + manifest_revision)
- print(manifest_repo.git.show("--summary", "--decorate"))
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s\n%s" % (
+ project.path,
+ merge_request.source_branch,
+ common.list_commits(merge_request.commits()),
+ )
+ manifest_revision = common.commit_and_push(
+ manifest_project,
+ manifest_repo,
+ integration_branch,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ )
return manifest_revision
diff --git a/merge_into_manifest.py b/merge_into_manifest.py
index 531cc6e5..f580af81 100755
--- a/merge_into_manifest.py
+++ b/merge_into_manifest.py
@@ -4,11 +4,9 @@ import common
import argparse
import sys
import time
-from gitlab import (
- Gitlab,
- GitlabGetError,
-)
+from gitlab import Gitlab
from accept_merge_request import accept_merge_request
+from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
from integrate_into_manifest import integrate_into_manifest
@@ -38,44 +36,19 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
integration_branch = common.integration_branch_name(project.name, original_branch)
target_branch = master_branch
- # Check if merge request already exists
- mrs = get_merge_requests(
- manifest_project,
- source_branch=integration_branch,
- target_branch=target_branch,
- state="opened",
+ # Create merge request. If there already is a merge request, we assume that it has
+ # been opened manually and thus will be merged manually as well, so we fail here.
+ mr, created = create_merge_request(
+ manifest_project, integration_branch, target_branch
)
- # If there already is a merge request, we assume that it has been opened manually
- # and thus will be merged manually as well, so we exit with a failure here.
- if mrs:
+ if not created:
sys.exit("ERROR: There is already an open merge request:\n%s" % mrs[0].web_url)
- # Check if branches exist
- try:
- manifest_project.branches.get(integration_branch, retry_transient_errors=True)
- except GitlabGetError:
- sys.exit("ERROR: source branch '%s' does not exist." % integration_branch)
- try:
- manifest_project.branches.get(target_branch, retry_transient_errors=True)
- except GitlabGetError:
- sys.exit("ERROR: target branch '%s' does not exist." % target_branch)
-
- # Create new merge request
- mr = manifest_project.mergerequests.create(
- {
- "source_branch": integration_branch,
- "target_branch": target_branch,
- "remove_source_branch": True,
- "title": "Merge " + integration_branch,
- }
- )
-
print("Created new merge request:")
print(mr.web_url)
# Insert cross-links in both merge requests
- mr.notes.create({"body": "Source merge request: %s" % source_mr.web_url})
- source_mr.notes.create({"body": "Integration merge request: %s" % mr.web_url})
+ common.crosslink_merge_requests(source_mr, mr)
# Loop until MR has been merged successfully
manifest_revision = mr.sha
@@ -83,13 +56,7 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
while not merged:
# Wait until GitLab has checked merge status
- print("Waiting until merge status has been checked", end="", flush=True)
- unchecked_states = ["unchecked", "checking", "cannot_be_merged_recheck"]
- while mr.merge_status in unchecked_states:
- print(".", end="", flush=True)
- time.sleep(1)
- mr = manifest_project.mergerequests.get(mr.iid)
- print(" -> %s" % mr.merge_status)
+ common.wait_until_merge_status_is_set(manifest_project, mr)
# Attempt to merge
merged = accept_merge_request(manifest_project, mr)
@@ -117,7 +84,9 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
while mr.sha != manifest_revision:
print(".", end="", flush=True)
time.sleep(1)
- mr = manifest_project.mergerequests.get(mr.iid)
+ mr = manifest_project.mergerequests.get(
+ mr.iid, retry_transient_errors=True
+ )
print("")
print("Successfully merged")
diff --git a/update_submodule.py b/update_submodule.py
new file mode 100755
index 00000000..091fec94
--- /dev/null
+++ b/update_submodule.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import os
+import sys
+import tempfile
+from furl import furl
+from git import GitCommandError, Repo
+from gitlab import Gitlab
+
+
+def update_submodule(project, submodule_name, submodule_revision, branch=None):
+ """Update submodule of gitlab project to given revision"""
+
+ gitlab = project.manager.gitlab
+
+ # If no branch is given, use project's default branch
+ if branch is None:
+ branch = project.default_branch
+
+ with tempfile.TemporaryDirectory() as project_dir:
+
+ # Construct clone url containing access token
+ clone_url = furl(project.http_url_to_repo)
+ clone_url.username = "gitlab-ci"
+ clone_url.password = gitlab.private_token
+
+ # Checkout project
+ try:
+ repo = Repo.clone_from(clone_url.url, project_dir, branch=branch)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not clone repository\n" + str(e))
+ except IndexError as e:
+ sys.exit("ERROR: branch '%s' not found" % branch)
+
+ # Find submodule
+ submodule = common.get_submodule(repo, submodule_name)
+
+ # Check if revisions are different
+ if submodule.hexsha == submodule_revision:
+ print("Submodule is already at %s" % submodule_revision)
+ return 0
+
+ # Check for relative path
+ if not submodule.url.startswith(".."):
+ sys.exit(
+ "ERROR: absolute submodule paths are not supported (%s)" % submodule.url
+ )
+
+ # Get absolute project path
+ # This cannot be done with gitpython directly due to issue:
+ # https://github.com/gitpython-developers/GitPython/issues/730
+ relative_path = os.path.splitext(submodule.url)[0] # remove .git
+ project_path = project.path_with_namespace
+ while relative_path.startswith(".."):
+ relative_path = relative_path[3:] # strip off '../'
+ project_path, _ = os.path.split(project_path) # remove last part
+ submodule_project_path = os.path.join(project_path, relative_path)
+
+ # Get submodule project
+ submodule_project = common.get_project(gitlab, submodule_project_path)
+
+ # Get commits between current and new revision
+ revision_range = submodule.hexsha + ".." + submodule_revision
+ commits = submodule_project.commits.list(ref_name=revision_range)
+ if not commits:
+ sys.exit("ERROR: no commits found in range %s" % revision_range)
+
+ # Find out if top commit is part of a merge request
+ # If so, use source branch of this MR as integration branch name
+ # Else use commit sha instead
+ integration_branch_suffix = submodule_revision
+ for mr in commits[0].merge_requests():
+ if mr["target_branch"] == submodule_project.default_branch:
+ integration_branch_suffix = mr["source_branch"]
+ break
+
+ # Initialize submodule
+ # Hack due to issue above: change to absolute path and switch back afterwards
+ submodule_clone_url = furl(submodule_project.http_url_to_repo)
+ submodule_clone_url.username = "gitlab-ci"
+ submodule_clone_url.password = gitlab.private_token
+ submodule_relative_url = submodule.url
+ with submodule.config_writer() as writer:
+ writer.set("url", submodule_clone_url.url)
+ submodule.update(init=True)
+ with submodule.config_writer() as writer:
+ writer.set("url", submodule_relative_url)
+
+ # Check if integration branch already exists and if it is up to date
+ integration_branch = common.integration_branch_name(
+ submodule_project.name, integration_branch_suffix
+ )
+ existing_branch = None
+ for ref in repo.references:
+ if "origin/" + integration_branch == ref.name:
+ existing_branch = ref
+ if existing_branch:
+ repo.head.reference = existing_branch
+ submodule = common.get_submodule(repo, submodule_name)
+ if submodule.hexsha == submodule_revision:
+ print(
+ "Submodule is already at %s on branch %s"
+ % (submodule_revision, integration_branch)
+ )
+ return (integration_branch, existing_branch.commit, submodule_project)
+ else:
+ print("Replacing outdated integration branch %s" % integration_branch)
+ else:
+ print("Creating integration branch %s" % integration_branch)
+
+ # Create integration branch
+ repo.head.reference = repo.create_head(integration_branch)
+
+ # Update submodule
+ try:
+ submodule.module().git.checkout(submodule_revision)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not checkout commit\n" + str(e))
+ repo.git.add(submodule.path)
+
+ # Make an API request to create the gitlab.user object
+ gitlab.auth()
+
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s%s\n%s" % (
+ submodule_project.name,
+ integration_branch_suffix,
+ " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
+ common.list_commits(commits),
+ )
+ project_revision = common.commit_and_push(
+ project,
+ repo,
+ integration_branch,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ )
+
+ return (integration_branch, project_revision, submodule_project)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--submodule",
+ help="""submodule to update""",
+ dest="submodule",
+ required=True,
+ )
+ parser.add_argument(
+ "--revision",
+ help="""new revision for submodule""",
+ dest="revision",
+ required=True,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""project branch (if not default branch)""",
+ dest="branch",
+ required=False,
+ default=None,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+
+ update_submodule(project, args.submodule, args.revision, args.branch)
+
+
+if __name__ == "__main__":
+ main()
--
GitLab