diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cb98394af493f9c5a5e699e1bc14eee47c4c98ad..632b1258ffb217d2bb142d490c61f9d010921e25 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,6 +10,11 @@ default:
stages:
- analyze
+ - deploy
+
+workflow:
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
#---------------------------------------------------------------------------------------
# Stage: analyze
@@ -31,3 +36,32 @@ executable:
timeout: 2m
script:
- (! find ! -executable -name "*.py" -exec echo "not executable:"" {}" \; | grep .)
+
+#---------------------------------------------------------------------------------------
+# Stage: deploy
+#---------------------------------------------------------------------------------------
+.deploy: &deploy
+ stage: deploy
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
+ when: manual
+ - if: $CI_COMMIT_BRANCH == "master"
+ when: manual
+ allow_failure: yes
+ script:
+ - cd ${CI_PROJECT_DIR}
+ - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi
+ - ./deploy_gitlab_ci.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --project=${CI_JOB_NAME}
+ --submodule=.gitlab-ci
+ --revision=${CI_COMMIT_SHA}
+ ${MERGE}
+
+garz-fricke/yocto/infrastructure/ci-test/minimal-bar: *deploy
+garz-fricke/yocto/infrastructure/ci-test/minimal-foo: *deploy
+garz-fricke/yocto/infrastructure/ci-test/minimal-manifest: *deploy
+garz-fricke/yocto/layers/meta-guf-distro: *deploy
+garz-fricke/yocto/layers/meta-guf-machine: *deploy
+garz-fricke/yocto/manifest: *deploy
diff --git a/common.py b/common.py
index 10b23fe7d1744022f2159319b7bf323093f96c14..4b7e1f74fd024c107bcab7b3e122c5a686c527a1 100755
--- a/common.py
+++ b/common.py
@@ -3,7 +3,11 @@
import requests
import sys
import time
+from git import Actor, GitCommandError
+from git.repo.base import Repo
from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
+from gitlab.v4.objects import Project
+from gitlab.v4.objects import MergeRequest
manifest_file = "default.xml"
@@ -79,3 +83,64 @@ def rebase_merge_request(project, merge_request):
return merge_request
rebase_in_progress = updated_merge_request.rebase_in_progress
return updated_merge_request
+
+
+def crosslink_merge_requests(source_mr: MergeRequest, integration_mr: MergeRequest):
+ """Insert cross-links in merge requests"""
+ integration_mr.notes.create(
+ {"body": "Source merge request: %s" % source_mr.web_url}
+ )
+ source_mr.notes.create(
+ {"body": "Integration merge request: %s" % integration_mr.web_url}
+ )
+
+
+def wait_until_merge_status_is_set(project: Project, mr: MergeRequest):
+ """Periodically query MR until GitLab has checked its merge status"""
+ print("Waiting until merge status has been checked", end="", flush=True)
+ unchecked_states = ["unchecked", "checking", "cannot_be_merged_recheck"]
+ while mr.merge_status in unchecked_states:
+ print(".", end="", flush=True)
+ time.sleep(1)
+ mr = project.mergerequests.get(mr.iid, retry_transient_errors=True)
+ print(" -> %s" % mr.merge_status)
+
+
+def list_commits(commits):
+ """Create a list of commits along with the commit messages"""
+ commit_list = ""
+ for commit in commits:
+ commit_list += "\n---\n\nCommit: %s\n\n%s" % (commit.web_url, commit.message)
+ return commit_list
+
+
+def commit_and_push(project: Project, repo: Repo, branch, message, name, email):
+ """Commit and push to a repo branch"""
+ author = Actor(name, email)
+ repo.index.commit(message, author=author, committer=author)
+
+ # Push commit
+ try:
+ origin = repo.remote("origin")
+ origin.push(branch, force=True)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not commit changes\n" + str(e))
+
+ # Print commit information
+ revision = repo.head.commit.hexsha
+ print("Pushed new commit:")
+ print(project.web_url + "/-/commit/" + revision)
+ print(repo.git.show("--summary", "--decorate"))
+
+ return revision
+
+
+def get_submodule(repo: Repo, submodule_name):
+ """Find a submodule in a Git repository by its name"""
+ submodule = None
+ for sm in repo.submodules:
+ if sm.name == submodule_name:
+ submodule = sm
+ if submodule is None:
+ sys.exit("ERROR: submodule '%s' not found" % submodule_name)
+ return submodule
diff --git a/create_merge_request.py b/create_merge_request.py
new file mode 100755
index 0000000000000000000000000000000000000000..62123cec92ca7aaab0c5693dab865208232f0734
--- /dev/null
+++ b/create_merge_request.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+from gitlab import Gitlab, GitlabGetError
+from gitlab.v4.objects import Project
+from get_merge_requests import get_merge_requests
+
+
+def create_merge_request(project: Project, source_branch, target_branch):
+ """Create a merge request"""
+ # Check if merge request already exists
+ mrs = get_merge_requests(
+ project,
+ source_branch=source_branch,
+ target_branch=target_branch,
+ state="opened",
+ )
+ if mrs:
+ return mrs[0], False
+
+ # Check if branches exist
+ try:
+ project.branches.get(source_branch, retry_transient_errors=True)
+ except GitlabGetError:
+ sys.exit("ERROR: source branch '%s' does not exist." % source_branch)
+ try:
+ project.branches.get(target_branch, retry_transient_errors=True)
+ except GitlabGetError:
+ sys.exit("ERROR: target branch '%s' does not exist." % target_branch)
+
+ # Create new merge request
+ mr = project.mergerequests.create(
+ {
+ "source_branch": source_branch,
+ "target_branch": target_branch,
+ "remove_source_branch": True,
+ "title": "Merge " + source_branch,
+ }
+ )
+
+ return mr, True
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--source-branch",
+ help="""source branch of the merge request""",
+ dest="source_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--target-branch",
+ help="""target branch of the merge request""",
+ dest="target_branch",
+ required=True,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+
+ create_merge_request(
+ project,
+ source_branch=args.source_branch,
+ target_branch=args.target_branch,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
new file mode 100755
index 0000000000000000000000000000000000000000..30a1dad75b91796b834bda5527a5dbcc6cbd768f
--- /dev/null
+++ b/deploy_gitlab_ci.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+from gitlab import Gitlab
+from accept_merge_request import accept_merge_request
+from create_merge_request import create_merge_request
+from get_merge_requests import get_merge_requests
+from update_submodule import update_submodule
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--submodule",
+ help="""submodule to update""",
+ dest="submodule",
+ required=True,
+ )
+ parser.add_argument(
+ "--revision",
+ help="""new revision for submodule""",
+ dest="revision",
+ required=True,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""project branch (if not default branch)""",
+ dest="branch",
+ required=False,
+ default=None,
+ )
+ parser.add_argument(
+ "--merge",
+ help="""if set, perform merge after integration""",
+ dest="merge",
+ action="store_true",
+ required=False,
+ default=False,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+
+ # Update submodule
+ integration_branch, _, submodule_project = update_submodule(
+ project, args.submodule, args.revision, args.branch
+ )
+
+ # Get source merge request
+ mrs = get_merge_requests(
+ submodule_project,
+ target_branch="master",
+ commit=args.revision,
+ )
+ if not mrs:
+ sys.exit(
+ "ERROR: could not determine source merge request for commit %s"
+ % args.revision
+ )
+ source_mr = mrs[0]
+
+ # Create merge request
+ mr, created = create_merge_request(
+ project, integration_branch, project.default_branch
+ )
+ if created:
+ common.crosslink_merge_requests(source_mr, mr)
+ print("Created new merge request:\n%s" % mr.web_url)
+ else:
+ print("Existing integration merge request:\n%s" % mr.web_url)
+
+ if not args.merge:
+ print(
+ "Skipping automatic merge in MR context. If you like to extend the "
+ "integration MR by hand, please do it now. Afterwards you can either merge "
+ "it by hand or re-run this job on the master branch after the source MR "
+ "has been merged."
+ )
+ sys.exit(0)
+
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(project, mr)
+
+ # Attempt to merge
+ merged = accept_merge_request(project, mr)
+
+ if not merged:
+ sys.exit("Integration MR could not be merged")
+
+ print("Successfully merged")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/get_merge_requests.py b/get_merge_requests.py
index 8846d5afe653bee9d6ae22ce5091db50114badc4..c1fc7c4dcd945d788d0ce072fc1b300d62adfac3 100755
--- a/get_merge_requests.py
+++ b/get_merge_requests.py
@@ -7,7 +7,7 @@ from gitlab import Gitlab, GitlabGetError
def get_merge_requests(
- project, state, source_branch=None, target_branch=None, commit=None
+ project, state=None, source_branch=None, target_branch=None, commit=None
):
"""Get merge request by source and target branch and optionally commit sha"""
merge_requests = []
@@ -15,7 +15,7 @@ def get_merge_requests(
all_merge_requests = project.mergerequests.list(
source_branch=source_branch,
target_branch=target_branch,
- state=state,
+ state=state if state else "all",
all=True,
retry_transient_errors=True,
)
diff --git a/integrate_into_manifest.py b/integrate_into_manifest.py
index 68bb9391ca08c2e53385b71c6af48abbd93524b9..986b98ed9c81b9ee5fcfb75408256ae812d7c07d 100755
--- a/integrate_into_manifest.py
+++ b/integrate_into_manifest.py
@@ -6,7 +6,7 @@ import sys
import tempfile
from pathlib import Path
from furl import furl
-from git import Actor, GitCommandError, Repo
+from git import GitCommandError, Repo
from gitlab import Gitlab, GitlabGetError
from lxml import etree
@@ -42,12 +42,10 @@ def integrate_into_manifest(
integration_branch = common.integration_branch_name(
project.name, merge_request.source_branch
)
- integration_base = None
for ref in manifest_repo.references:
if integration_branch == ref.name:
manifest_repo.delete_head(ref)
- integration_base = manifest_repo.create_head(integration_branch)
- manifest_repo.head.reference = integration_base
+ manifest_repo.head.reference = manifest_repo.create_head(integration_branch)
# Parse manifest file
try:
@@ -80,30 +78,25 @@ def integrate_into_manifest(
content = manifest_filepath.read_text()
content = content.replace(old_revision, new_revision)
manifest_filepath.write_text(content)
+ manifest_repo.index.add([manifest_file])
# Make an API request to create the gitlab.user object
gitlab.auth()
- # Construct commit object and commit change
- author = Actor(gitlab.user.username, gitlab.user.email)
- message = "Integrate %s/%s\n" % (project.path, merge_request.source_branch)
- for commit in merge_request.commits():
- message += "\nCommit: %s\n\n%s" % (commit.web_url, commit.message)
- manifest_repo.index.add([manifest_file])
- manifest_repo.index.commit(message, author=author, committer=author)
-
- # Push commit
- try:
- origin = manifest_repo.remote("origin")
- origin.push(integration_branch, force=True)
- except GitCommandError as e:
- sys.exit("ERROR: could not commit changes\n" + str(e))
-
- # Print commit information
- manifest_revision = manifest_repo.head.commit.hexsha
- print("Pushed new commit:")
- print(manifest_project.web_url + "/-/commit/" + manifest_revision)
- print(manifest_repo.git.show("--summary", "--decorate"))
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s\n%s" % (
+ project.path,
+ merge_request.source_branch,
+ common.list_commits(merge_request.commits()),
+ )
+ manifest_revision = common.commit_and_push(
+ manifest_project,
+ manifest_repo,
+ integration_branch,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ )
return manifest_revision
diff --git a/merge_into_manifest.py b/merge_into_manifest.py
index 531cc6e5dcd8c6680a8fb8029581bc70771d15d6..f580af81f9022e79bc3c7b095351b149bc5dd88b 100755
--- a/merge_into_manifest.py
+++ b/merge_into_manifest.py
@@ -4,11 +4,9 @@ import common
import argparse
import sys
import time
-from gitlab import (
- Gitlab,
- GitlabGetError,
-)
+from gitlab import Gitlab
from accept_merge_request import accept_merge_request
+from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
from integrate_into_manifest import integrate_into_manifest
@@ -38,44 +36,19 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
integration_branch = common.integration_branch_name(project.name, original_branch)
target_branch = master_branch
- # Check if merge request already exists
- mrs = get_merge_requests(
- manifest_project,
- source_branch=integration_branch,
- target_branch=target_branch,
- state="opened",
+ # Create merge request. If there already is a merge request, we assume that it has
+ # been opened manually and thus will be merged manually as well, so we fail here.
+ mr, created = create_merge_request(
+ manifest_project, integration_branch, target_branch
)
- # If there already is a merge request, we assume that it has been opened manually
- # and thus will be merged manually as well, so we exit with a failure here.
- if mrs:
+ if not created:
sys.exit("ERROR: There is already an open merge request:\n%s" % mrs[0].web_url)
- # Check if branches exist
- try:
- manifest_project.branches.get(integration_branch, retry_transient_errors=True)
- except GitlabGetError:
- sys.exit("ERROR: source branch '%s' does not exist." % integration_branch)
- try:
- manifest_project.branches.get(target_branch, retry_transient_errors=True)
- except GitlabGetError:
- sys.exit("ERROR: target branch '%s' does not exist." % target_branch)
-
- # Create new merge request
- mr = manifest_project.mergerequests.create(
- {
- "source_branch": integration_branch,
- "target_branch": target_branch,
- "remove_source_branch": True,
- "title": "Merge " + integration_branch,
- }
- )
-
print("Created new merge request:")
print(mr.web_url)
# Insert cross-links in both merge requests
- mr.notes.create({"body": "Source merge request: %s" % source_mr.web_url})
- source_mr.notes.create({"body": "Integration merge request: %s" % mr.web_url})
+ common.crosslink_merge_requests(source_mr, mr)
# Loop until MR has been merged successfully
manifest_revision = mr.sha
@@ -83,13 +56,7 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
while not merged:
# Wait until GitLab has checked merge status
- print("Waiting until merge status has been checked", end="", flush=True)
- unchecked_states = ["unchecked", "checking", "cannot_be_merged_recheck"]
- while mr.merge_status in unchecked_states:
- print(".", end="", flush=True)
- time.sleep(1)
- mr = manifest_project.mergerequests.get(mr.iid)
- print(" -> %s" % mr.merge_status)
+ common.wait_until_merge_status_is_set(manifest_project, mr)
# Attempt to merge
merged = accept_merge_request(manifest_project, mr)
@@ -117,7 +84,9 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
while mr.sha != manifest_revision:
print(".", end="", flush=True)
time.sleep(1)
- mr = manifest_project.mergerequests.get(mr.iid)
+ mr = manifest_project.mergerequests.get(
+ mr.iid, retry_transient_errors=True
+ )
print("")
print("Successfully merged")
diff --git a/update_submodule.py b/update_submodule.py
new file mode 100755
index 0000000000000000000000000000000000000000..091fec94dac36d9a53808b8ff531af943aa80272
--- /dev/null
+++ b/update_submodule.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import os
+import sys
+import tempfile
+from furl import furl
+from git import GitCommandError, Repo
+from gitlab import Gitlab
+
+
+def update_submodule(project, submodule_name, submodule_revision, branch=None):
+ """Update submodule of gitlab project to given revision"""
+
+ gitlab = project.manager.gitlab
+
+ # If no branch is given, use project's default branch
+ if branch is None:
+ branch = project.default_branch
+
+ with tempfile.TemporaryDirectory() as project_dir:
+
+ # Construct clone url containing access token
+ clone_url = furl(project.http_url_to_repo)
+ clone_url.username = "gitlab-ci"
+ clone_url.password = gitlab.private_token
+
+ # Checkout project
+ try:
+ repo = Repo.clone_from(clone_url.url, project_dir, branch=branch)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not clone repository\n" + str(e))
+ except IndexError as e:
+ sys.exit("ERROR: branch '%s' not found" % branch)
+
+ # Find submodule
+ submodule = common.get_submodule(repo, submodule_name)
+
+ # Check if revisions are different
+ if submodule.hexsha == submodule_revision:
+ print("Submodule is already at %s" % submodule_revision)
+ return 0
+
+ # Check for relative path
+ if not submodule.url.startswith(".."):
+ sys.exit(
+ "ERROR: absolute submodule paths are not supported (%s)" % submodule.url
+ )
+
+ # Get absolute project path
+ # This cannot be done with gitpython directly due to issue:
+ # https://github.com/gitpython-developers/GitPython/issues/730
+ relative_path = os.path.splitext(submodule.url)[0] # remove .git
+ project_path = project.path_with_namespace
+ while relative_path.startswith(".."):
+ relative_path = relative_path[3:] # strip off '../'
+ project_path, _ = os.path.split(project_path) # remove last part
+ submodule_project_path = os.path.join(project_path, relative_path)
+
+ # Get submodule project
+ submodule_project = common.get_project(gitlab, submodule_project_path)
+
+ # Get commits between current and new revision
+ revision_range = submodule.hexsha + ".." + submodule_revision
+ commits = submodule_project.commits.list(ref_name=revision_range)
+ if not commits:
+ sys.exit("ERROR: no commits found in range %s" % revision_range)
+
+ # Find out if top commit is part of a merge request
+ # If so, use source branch of this MR as integration branch name
+ # Else use commit sha instead
+ integration_branch_suffix = submodule_revision
+ for mr in commits[0].merge_requests():
+ if mr["target_branch"] == submodule_project.default_branch:
+ integration_branch_suffix = mr["source_branch"]
+ break
+
+ # Initialize submodule
+ # Hack due to issue above: change to absolute path and switch back afterwards
+ submodule_clone_url = furl(submodule_project.http_url_to_repo)
+ submodule_clone_url.username = "gitlab-ci"
+ submodule_clone_url.password = gitlab.private_token
+ submodule_relative_url = submodule.url
+ with submodule.config_writer() as writer:
+ writer.set("url", submodule_clone_url.url)
+ submodule.update(init=True)
+ with submodule.config_writer() as writer:
+ writer.set("url", submodule_relative_url)
+
+ # Check if integration branch already exists and if it is up to date
+ integration_branch = common.integration_branch_name(
+ submodule_project.name, integration_branch_suffix
+ )
+ existing_branch = None
+ for ref in repo.references:
+ if "origin/" + integration_branch == ref.name:
+ existing_branch = ref
+ if existing_branch:
+ repo.head.reference = existing_branch
+ submodule = common.get_submodule(repo, submodule_name)
+ if submodule.hexsha == submodule_revision:
+ print(
+ "Submodule is already at %s on branch %s"
+ % (submodule_revision, integration_branch)
+ )
+ return (integration_branch, existing_branch.commit, submodule_project)
+ else:
+ print("Replacing outdated integration branch %s" % integration_branch)
+ else:
+ print("Creating integration branch %s" % integration_branch)
+
+ # Create integration branch
+ repo.head.reference = repo.create_head(integration_branch)
+
+ # Update submodule
+ try:
+ submodule.module().git.checkout(submodule_revision)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not checkout commit\n" + str(e))
+ repo.git.add(submodule.path)
+
+ # Make an API request to create the gitlab.user object
+ gitlab.auth()
+
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s%s\n%s" % (
+ submodule_project.name,
+ integration_branch_suffix,
+ " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
+ common.list_commits(commits),
+ )
+ project_revision = common.commit_and_push(
+ project,
+ repo,
+ integration_branch,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ )
+
+ return (integration_branch, project_revision, submodule_project)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--submodule",
+ help="""submodule to update""",
+ dest="submodule",
+ required=True,
+ )
+ parser.add_argument(
+ "--revision",
+ help="""new revision for submodule""",
+ dest="revision",
+ required=True,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""project branch (if not default branch)""",
+ dest="branch",
+ required=False,
+ default=None,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+
+ update_submodule(project, args.submodule, args.revision, args.branch)
+
+
+if __name__ == "__main__":
+ main()