Skip to content
Snippets Groups Projects
Commit 6e881999 authored by Jonas Höppner's avatar Jonas Höppner
Browse files

CI: Use new deploy_gitlab_ci in adapted pipeline

parent 29d4940e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 25
− 71
View file @ 6e881999
......@@ -19,7 +19,7 @@ default:
stages:
- analyze
- integrate
- check
- build
- merge
workflow:
......@@ -62,7 +62,6 @@ yamllint:
# ---------------------------------------------------------------------------------------
# Stage: integrate
# ---------------------------------------------------------------------------------------
.ci-test-projects:
variables:
PROJECT_ROOT:
......@@ -104,6 +103,8 @@ yamllint:
- if: $CI_MERGE_REQUEST_IID
when: manual
allow_failure: true
variables:
MERGE: ""
script:
- cd ${CI_PROJECT_DIR}
- ./deploy_gitlab_ci.py
......@@ -113,18 +114,8 @@ yamllint:
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
--verbose
${MERGE}
${INTEGRATE_INTO}
- ./generate_job_from_template.py
--template=gitlab-ci-integration.jinja2
--image=${CI_IMAGE_PYTHON}
--branch="integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
--manifest-project=${MANIFEST_PROJECT}
--parent_merge_request="${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
--verbose
${INTEGRATE_INTO}
> integration.yml
- cat integration.yml
artifacts:
paths:
- integration.yml
......@@ -139,47 +130,47 @@ integrate-ci-test:
- .integrate
- .ci-test-projects
trigger-yocto:
stage: integrate
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
build-yocto:
stage: build
needs: [integrate-yocto]
rules:
- if: $CI_MERGE_REQUEST_IID
allow_failure: true
needs: [integrate-yocto]
- if: $CI_COMMIT_BRANCH == "master"
when: manual
trigger:
include:
- artifact: integration.yml
job: integrate-yocto
project: SECO-Northern-Europe/yocto/manifest
branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
trigger-ci-test:
stage: integrate
build-ci-test:
stage: build
needs: [integrate-ci-test]
rules:
- if: $CI_MERGE_REQUEST_IID
needs: [integrate-ci-test]
allow_failure: true
- if: $CI_COMMIT_BRANCH == "master"
when: manual
trigger:
include:
- artifact: integration.yml
job: integrate-ci-test
project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
.merge:
extends: .integrate
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == "master"
when: manual
allow_failure: true
script:
- cd ${CI_PROJECT_DIR}
- ./merge_gitlab_ci.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
${INTEGRATE_INTO}
variables:
MERGE: --merge
merge-ci-test:
extends:
......@@ -190,40 +181,3 @@ merge-yocto:
extends:
- .merge
- .yocto-projects
# --------------------------------------------------------------------------------------
# Stage: check
# --------------------------------------------------------------------------------------
check:
stage: check
needs: [integrate-yocto, integrate-ci-test]
rules:
# Probably this job gets removed
- when: never
- if: $CI_MERGE_REQUEST_IID
tags:
- infrastructure
timeout: 2m
script:
- cd ${CI_PROJECT_DIR}
- MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
- MASTER_BRANCH=dunfell
- ./check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${CI_PROJECT_ROOT_NAMESPACE}/yocto/manifest
--integration-base=${MASTER_BRANCH}
--project=${CI_PROJECT_PATH}
--merge-request=${CI_MERGE_REQUEST_IID}
# The check is done for both manifests in one job as the retrigger
# looks for jobs named check, though there can only be one
- ./check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project
${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test/minimal-manifest
--integration-base=master
--project=${CI_PROJECT_PATH}
--merge-request=${CI_MERGE_REQUEST_IID}
accept_merge_request.py
+ 4
− 4
View file @ 6e881999
......@@ -47,7 +47,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
print("Merge error: %s" % mr.merge_error)
else:
print("Merge reported success, but MR state is '%s'" % mr.state)
return False
return False, mr.sha
except GitlabMRClosedError as e:
# See HTTP error codes for merge requests here:
......@@ -73,7 +73,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
if pipeline_pending:
print("")
print("Merge not possible, has to be rebased manually")
return False
return False, mr.sha
elif e.response_code == 406:
# Merge conflict, automatic rebase is possible
......@@ -82,7 +82,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
pipeline_pending = False
print("Merge not possible, but branch can be automatically rebased")
if not rebase:
return False
return False, mr.sha
print("Trying to rebase...")
mr = common.rebase_merge_request(project, mr)
if mr.merge_error:
......@@ -96,7 +96,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
print("ERROR: merge not possible: %s" % e)
sys.exit(critical_error)
return True
return True, mr.sha
def main():
......
deploy_gitlab_ci.py
+ 134
− 78
View file @ 6e881999
......@@ -39,6 +39,45 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
return {"recipe": recipe, "masterbranch": masterbranch}
def integrate_submodule_into(
gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
):
gitlab_project = common.get_project(gitlab, project_name)
(
project_repo,
integration_branch_name,
integration_commit,
message,
) = update_submodule_and_include_ref(
gitlab_project,
submodule_name,
new_revision,
branch,
commit_and_push=commit_and_push,
)
if integration_branch_name is None:
return None
# ======================================
# Store the references for creating the integration
# commit in the manifest later
# ======================================
ret = {
"project": gitlab_project,
"repo": project_repo,
"branch": integration_branch_name,
"commit": integration_commit,
"message": message,
}
logging.debug(
"Integration branch: %s (%s)",
integration_branch_name,
integration_commit,
)
return ret
def create_integration_merge_request(project, integration_branch_name, source_mr=None):
# Create merge request
# This should be optional
......@@ -134,54 +173,95 @@ def main():
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
# =======================================================
# Create integration branches and commits with updates
# submodule in all projects
# =======================================================
project_integration = {}
# Update submodule in this project, create MR if needed TODO
for p in args.projects + [args.project]:
# Update submodule in all 'child' project
for p in args.projects:
logging.debug("Integrate into: %s", p)
gitlab_project = common.get_project(gitlab, p)
(
project_repo,
integration_branch_name,
integration_commit,
message,
) = update_submodule_and_include_ref(
gitlab_project,
args.submodule,
args.revision,
args.branch,
# Commit in all projects except the manifest
commit_and_push=(p != args.project),
res = integrate_submodule_into(
gitlab, p, args.submodule, args.revision, args.branch
)
if integration_branch_name is not None:
# ======================================
# Store the references for creating the integration
# commit in the manifest later
# ======================================
project_integration[p] = {
"project": gitlab_project,
"repo": project_repo,
"branch": integration_branch_name,
"commit": integration_commit,
"message": message,
}
logging.debug(
"Integration branch: %s (%s)",
integration_branch_name,
integration_commit,
)
if res is not None:
project_integration[p] = res
# Update submodule in manifest project
manifest_project = integrate_submodule_into(
gitlab,
args.project,
args.submodule,
args.revision,
args.branch,
commit_and_push=False,
)
if manifest_project is not None:
project_integration[args.project] = manifest_project
# If submodule is already at specified revision in all projects, exit successfully
if len(project_integration) == 0:
print("No integration done, changes are already included in all projects.")
sys.exit(0)
# =======================================================
# Create and merge merge_requests if needed
# =======================================================
if args.merge:
# Get source merge request ( the one in the gitlab-ci repo)
submodule_project_path, _ = get_submodule_project_path_and_revision(
manifest_project["project"], args.submodule, args.branch
)
submodule_project = common.get_project(gitlab, submodule_project_path)
mrs = get_merge_requests(
submodule_project,
# TODO should this be submodule_project's default branch?
target_branch="master",
commit=args.revision,
)
if not mrs:
sys.exit(
"ERROR: could not determine source merge request for commit %s"
% args.revision
)
source_mr = mrs[0]
for p in args.projects:
integration = project_integration[p]
logging.debug("Create MR in %s", integration["project"].name)
mr = create_integration_merge_request(
integration["project"], integration["branch"], source_mr
)
integration["mr"] = mr
# Now merge
logging.debug("Merge %s!%s", p, mr.iid)
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(integration["project"], mr)
# Attempt to merge
merged, integration_commit = accept_merge_request(
integration["project"], mr, rebase=True
)
# if this has rebased the integration commit needs to be adapted:
project_integration[p]["commit"] = integration_commit
if not merged:
sys.exit(
"Integration MR could not be merged. You have two possibilities to fix "
"this:\n"
" 1. Checkout the MR and rebase it on the current master manually, or\n"
" 2. Delete the MR (Edit -> Delete in the MR UI)\n"
"In either case restart this job afterwards in order to get it merged."
)
print("Successfully merged")
# =======================================================
# Now create the integration commit in the manifest
# for all subprojects at once
# =======================================================
manifest_project = project_integration[args.project]
manifest_file_abs = os.path.join(
manifest_project["repo"].working_tree_dir, args.manifest_file
)
......@@ -275,58 +355,34 @@ def main():
sys.exit(0)
# ============================================
# Create merge requests if needed
# Create merge requests for the manifest
# ============================================
# Get source merge request ( the one in the gitlab-ci repo)
submodule_project_path, _ = get_submodule_project_path_and_revision(
manifest_project["project"], args.submodule, args.branch
)
submodule_project = common.get_project(gitlab, submodule_project_path)
mrs = get_merge_requests(
submodule_project,
# TODO should this be submodule_project's default branch?
target_branch="master",
commit=args.revision,
)
if not mrs:
sys.exit(
"ERROR: could not determine source merge request for commit %s"
% args.revision
)
source_mr = mrs[0]
for p in args.projects + [args.project]:
integration = project_integration[p]
logging.debug("Create MR in %s", integration["project"].name)
integration["mr"] = create_integration_merge_request(
integration["project"], integration["branch"], source_mr
)
logging.debug("Create MR in %s", manifest_project["project"].name)
manifest_project["mr"] = create_integration_merge_request(
manifest_project["project"], manifest_project["branch"], source_mr
)
# =================================================
# Now merge them all
# Now merge it
# =================================================
# The manifest needs to be merged at last
for p in args.projects + [args.project]:
integration = project_integration[p]
mr = integration["mr"]
logging.debug("Merge %s!%s", p, mr.iid)
mr = manifest_project["mr"]
logging.debug("Merge %s!%s", args.project, mr.iid)
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(integration["project"], mr)
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(manifest_project["project"], mr)
# Attempt to merge
# TODO if this has rebased the integration commit needs to be adapted
# change the sequence
merged = accept_merge_request(integration["project"], mr, rebase=True)
# Attempt to merge
merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
if not merged:
sys.exit(
"Integration MR could not be merged. You have two possibilities to fix "
"this:\n"
" 1. Checkout the MR and rebase it on the current master manually, or\n"
" 2. Delete the MR (Edit -> Delete in the MR UI)\n"
"In either case restart this job afterwards in order to get it merged."
)
if not merged:
sys.exit(
"Integration MR could not be merged. You have two possibilities to fix "
"this:\n"
" 1. Checkout the MR and rebase it on the current master manually, or\n"
" 2. Delete the MR (Edit -> Delete in the MR UI)\n"
"In either case restart this job afterwards in order to get it merged."
)
print("Successfully merged")
......
merge_gitlab_ci.py deleted 100755 → 0
+ 0
− 198
View file @ 29d4940e
#!/usr/bin/env python3
import common
import argparse
import logging
import sys
from gitlab import Gitlab, GitlabGetError
from accept_merge_request import accept_merge_request
import update_submodule
from gitlab.v4.objects import Project
def find_integration_merge_request(
project: Project, submodule, revision, target_branch
):
(
submodule_path,
submodule_revision,
) = update_submodule.get_submodule_project_path_and_revision(
project, submodule, target_branch
)
logging.debug("Module: %s, Revision: %s", submodule_path, submodule_revision)
# Get submodule project
gitlab = project.manager.gitlab
submodule_project = common.get_project(gitlab, submodule_path)
# Get the integration branch name
integration_branch_suffix = (
update_submodule.get_submodule_integration_branch_suffix(
submodule_project, revision
)
)
integration_branch_name = common.integration_branch_name(
submodule_project.name, integration_branch_suffix
)
logging.debug(integration_branch_name)
# Get the merge request for the branch TODO catch exception
try:
integration_branch = project.branches.get(
integration_branch_name, retry_transient_errors=True
)
except GitlabGetError:
print("ERROR: Failed to find integration branch for {}".format(submodule))
return None
logging.debug(integration_branch)
commit = project.commits.get(
integration_branch.commit["id"], retry_transient_errors=True
)
for mr in commit.merge_requests(retry_transient_errors=True):
if mr["target_branch"] == target_branch:
integration_mr = mr
break
if integration_mr is None:
print(
"ERROR: Failed to find integration merge request for %s",
integration_branch_name,
)
return None
logging.debug(integration_mr["iid"])
mr = project.mergerequests.get(integration_mr["iid"], retry_transient_errors=True)
return mr
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
required=True,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
"--manifest-project",
help="""name of the GitLab project""",
dest="project",
)
parser.add_argument(
"--submodule",
help="""submodule to update""",
dest="submodule",
required=True,
)
parser.add_argument(
"--revision",
help="""new revision for submodule""",
dest="revision",
required=True,
)
parser.add_argument(
"--branch",
help="""project branch (if not default branch)""",
dest="branch",
required=False,
default=None,
)
parser.add_argument(
"--merge",
help="""if set, perform merge after integration""",
dest="merge",
action="store_true",
required=False,
default=False,
)
parser.add_argument(
"projects",
help="""List of projects the change should be deployed to additionally
to the manifest project given as named parameter.""",
nargs="*",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
args, _ = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
# Start with the manifest here, so the subproject
# can see that they are already integrated
for p in args.projects + [args.project]:
project = common.get_project(gitlab, p)
branch = args.branch
if branch is None:
branch = project.default_branch
print(
"Try to merge {}({}) into {} ({})".format(
args.submodule, args.revision, project.name, branch
)
)
mr = find_integration_merge_request(
project, args.submodule, args.revision, branch
)
if mr is None:
sys.exit("ERROR: Failed to find the integration MR.")
# TODO if this ever happens, why ever, we could call
# deploy_gitlab_ci again to create a new integration
# commit or make sure the change is already integrated.
print("Merge {}!{}: {}".format(project.name, mr.iid, mr.title))
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(project, mr)
# Attempt to merge
merged = accept_merge_request(
project,
mr,
# Only the file manifest commit may be rebased
# other rebase would require new integration commits
# TODO implement rebase
rebase=(p == args.project),
)
if not merged:
print(
"Integration MR could not be merged."
"To fix this:\n"
" 1. Checkout the MR {}!{} and merge it manually.\n"
" 2. Manually merge the follow up MRs in the following project:".format(
project.name, mr.iid
)
)
found = False
for p2 in [args.project] + args.projects:
if p2 == p:
found = True
continue
if not found:
continue
print(" {}".format(p2))
sys.exit()
print("Successfully merged")
exit()
if __name__ == "__main__":
main()
merge_into_manifest.py
+ 1
− 1
View file @ 6e881999
......@@ -78,7 +78,7 @@ def merge_into_manifest(
common.wait_until_merge_status_is_set(manifest_project, mr)
# Attempt to merge
merged = accept_merge_request(manifest_project, mr)
merged, _ = accept_merge_request(manifest_project, mr)
if not merged:
# Merge failed, reintegrate the source merge request into the manifest.
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment