diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8f48a3415a2e8ca2bdc81bff57ac775d24d4dd16..bb561ca5464f4472d64d07aeb48049fd2f0da127 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,7 @@ default:
stages:
- analyze
- integrate
- - check
+ - build
- merge
workflow:
@@ -62,7 +62,6 @@ yamllint:
# ---------------------------------------------------------------------------------------
# Stage: integrate
# ---------------------------------------------------------------------------------------
-
.ci-test-projects:
variables:
PROJECT_ROOT:
@@ -104,6 +103,8 @@ yamllint:
- if: $CI_MERGE_REQUEST_IID
when: manual
allow_failure: true
+ variables:
+ MERGE: ""
script:
- cd ${CI_PROJECT_DIR}
- ./deploy_gitlab_ci.py
@@ -113,18 +114,8 @@ yamllint:
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
--verbose
+ ${MERGE}
${INTEGRATE_INTO}
-
- - ./generate_job_from_template.py
- --template=gitlab-ci-integration.jinja2
- --image=${CI_IMAGE_PYTHON}
- --branch="integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
- --manifest-project=${MANIFEST_PROJECT}
- --parent_merge_request="${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
- --verbose
- ${INTEGRATE_INTO}
- > integration.yml
- - cat integration.yml
artifacts:
paths:
- integration.yml
@@ -139,47 +130,47 @@ integrate-ci-test:
- .integrate
- .ci-test-projects
-trigger-yocto:
- stage: integrate
+# --------------------------------------------------------------------------------------
+# Stage: build
+# --------------------------------------------------------------------------------------
+build-yocto:
+ stage: build
+ needs: [integrate-yocto]
rules:
- if: $CI_MERGE_REQUEST_IID
allow_failure: true
- needs: [integrate-yocto]
+ - if: $CI_COMMIT_BRANCH == "master"
+ when: manual
trigger:
- include:
- - artifact: integration.yml
- job: integrate-yocto
+ project: SECO-Northern-Europe/yocto/manifest
+ branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
-trigger-ci-test:
- stage: integrate
+build-ci-test:
+ stage: build
+ needs: [integrate-ci-test]
rules:
- if: $CI_MERGE_REQUEST_IID
- needs: [integrate-ci-test]
+ allow_failure: true
+ - if: $CI_COMMIT_BRANCH == "master"
+ when: manual
trigger:
- include:
- - artifact: integration.yml
- job: integrate-ci-test
+ project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
+ branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
.merge:
+ extends: .integrate
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == "master"
when: manual
allow_failure: true
- script:
- - cd ${CI_PROJECT_DIR}
- - ./merge_gitlab_ci.py
- --gitlab-url=${CI_SERVER_URL}
- --token=${GITBOT_TOKEN}
- --manifest-project=${MANIFEST_PROJECT}
- --submodule=.gitlab-ci
- --revision=${CI_COMMIT_SHA}
- ${INTEGRATE_INTO}
+ variables:
+ MERGE: --merge
merge-ci-test:
extends:
@@ -190,40 +181,3 @@ merge-yocto:
extends:
- .merge
- .yocto-projects
-
-# --------------------------------------------------------------------------------------
-# Stage: check
-# --------------------------------------------------------------------------------------
-check:
- stage: check
- needs: [integrate-yocto, integrate-ci-test]
- rules:
- # Probably this job gets removed
- - when: never
- - if: $CI_MERGE_REQUEST_IID
- tags:
- - infrastructure
- timeout: 2m
- script:
- - cd ${CI_PROJECT_DIR}
- - MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-
- - MASTER_BRANCH=dunfell
- - ./check_if_integration_branch_is_up_to_date.py
- --gitlab-url=${CI_SERVER_URL}
- --token=${GITBOT_TOKEN}
- --manifest-project=${CI_PROJECT_ROOT_NAMESPACE}/yocto/manifest
- --integration-base=${MASTER_BRANCH}
- --project=${CI_PROJECT_PATH}
- --merge-request=${CI_MERGE_REQUEST_IID}
-
- # The check is done for both manifests in one job as the retrigger
- # looks for jobs named check, though there can only be one
- - ./check_if_integration_branch_is_up_to_date.py
- --gitlab-url=${CI_SERVER_URL}
- --token=${GITBOT_TOKEN}
- --manifest-project
- ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test/minimal-manifest
- --integration-base=master
- --project=${CI_PROJECT_PATH}
- --merge-request=${CI_MERGE_REQUEST_IID}
diff --git a/accept_merge_request.py b/accept_merge_request.py
index 779d6117857b3ed5ea5be77a36267ccf0484fbc8..7500bbe98fc152075fcb66c9c6623223f623090f 100755
--- a/accept_merge_request.py
+++ b/accept_merge_request.py
@@ -47,7 +47,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
print("Merge error: %s" % mr.merge_error)
else:
print("Merge reported success, but MR state is '%s'" % mr.state)
- return False
+ return False, mr.sha
except GitlabMRClosedError as e:
# See HTTP error codes for merge requests here:
@@ -73,7 +73,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
if pipeline_pending:
print("")
print("Merge not possible, has to be rebased manually")
- return False
+ return False, mr.sha
elif e.response_code == 406:
# Merge conflict, automatic rebase is possible
@@ -82,7 +82,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
pipeline_pending = False
print("Merge not possible, but branch can be automatically rebased")
if not rebase:
- return False
+ return False, mr.sha
print("Trying to rebase...")
mr = common.rebase_merge_request(project, mr)
if mr.merge_error:
@@ -96,7 +96,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
print("ERROR: merge not possible: %s" % e)
sys.exit(critical_error)
- return True
+ return True, mr.sha
def main():
diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
index 4952e63b914f0652b1aae01556b9ec8651cdf88b..7b2609e3744415a2fc16d7f2515328d43a29f8c0 100755
--- a/deploy_gitlab_ci.py
+++ b/deploy_gitlab_ci.py
@@ -39,6 +39,45 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
return {"recipe": recipe, "masterbranch": masterbranch}
+def integrate_submodule_into(
+ gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
+):
+
+ gitlab_project = common.get_project(gitlab, project_name)
+
+ (
+ project_repo,
+ integration_branch_name,
+ integration_commit,
+ message,
+ ) = update_submodule_and_include_ref(
+ gitlab_project,
+ submodule_name,
+ new_revision,
+ branch,
+ commit_and_push=commit_and_push,
+ )
+ if integration_branch_name is None:
+ return None
+ # ======================================
+ # Store the references for creating the integration
+ # commit in the manifest later
+ # ======================================
+ ret = {
+ "project": gitlab_project,
+ "repo": project_repo,
+ "branch": integration_branch_name,
+ "commit": integration_commit,
+ "message": message,
+ }
+ logging.debug(
+ "Integration branch: %s (%s)",
+ integration_branch_name,
+ integration_commit,
+ )
+ return ret
+
+
def create_integration_merge_request(project, integration_branch_name, source_mr=None):
# Create merge request
# This should be optional
@@ -134,54 +173,95 @@ def main():
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ # =======================================================
+ # Create integration branches and commits with updates
+ # submodule in all projects
+ # =======================================================
project_integration = {}
- # Update submodule in this project, create MR if needed TODO
- for p in args.projects + [args.project]:
+ # Update submodule in all 'child' project
+ for p in args.projects:
logging.debug("Integrate into: %s", p)
- gitlab_project = common.get_project(gitlab, p)
-
- (
- project_repo,
- integration_branch_name,
- integration_commit,
- message,
- ) = update_submodule_and_include_ref(
- gitlab_project,
- args.submodule,
- args.revision,
- args.branch,
- # Commit in all projects except the manifest
- commit_and_push=(p != args.project),
+ res = integrate_submodule_into(
+ gitlab, p, args.submodule, args.revision, args.branch
)
- if integration_branch_name is not None:
- # ======================================
- # Store the references for creating the integration
- # commit in the manifest later
- # ======================================
- project_integration[p] = {
- "project": gitlab_project,
- "repo": project_repo,
- "branch": integration_branch_name,
- "commit": integration_commit,
- "message": message,
- }
- logging.debug(
- "Integration branch: %s (%s)",
- integration_branch_name,
- integration_commit,
- )
+ if res is not None:
+ project_integration[p] = res
+
+ # Update submodule in manifest project
+ manifest_project = integrate_submodule_into(
+ gitlab,
+ args.project,
+ args.submodule,
+ args.revision,
+ args.branch,
+ commit_and_push=False,
+ )
+ if manifest_project is not None:
+ project_integration[args.project] = manifest_project
# If submodule is already at specified revision in all projects, exit successfully
if len(project_integration) == 0:
print("No integration done, changes are already included in all projects.")
sys.exit(0)
+ # =======================================================
+ # Create and merge merge_requests if needed
+ # =======================================================
+ if args.merge:
+ # Get source merge request ( the one in the gitlab-ci repo)
+ submodule_project_path, _ = get_submodule_project_path_and_revision(
+ manifest_project["project"], args.submodule, args.branch
+ )
+ submodule_project = common.get_project(gitlab, submodule_project_path)
+ mrs = get_merge_requests(
+ submodule_project,
+ # TODO should this be submodule_project's default branch?
+ target_branch="master",
+ commit=args.revision,
+ )
+ if not mrs:
+ sys.exit(
+ "ERROR: could not determine source merge request for commit %s"
+ % args.revision
+ )
+ source_mr = mrs[0]
+
+ for p in args.projects:
+ integration = project_integration[p]
+ logging.debug("Create MR in %s", integration["project"].name)
+ mr = create_integration_merge_request(
+ integration["project"], integration["branch"], source_mr
+ )
+ integration["mr"] = mr
+ # Now merge
+ logging.debug("Merge %s!%s", p, mr.iid)
+
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(integration["project"], mr)
+
+ # Attempt to merge
+ merged, integration_commit = accept_merge_request(
+ integration["project"], mr, rebase=True
+ )
+ # if this has rebased the integration commit needs to be adapted:
+ project_integration[p]["commit"] = integration_commit
+
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged. You have two possibilities to fix "
+ "this:\n"
+ " 1. Checkout the MR and rebase it on the current master manually, or\n"
+ " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
+ "In either case restart this job afterwards in order to get it merged."
+ )
+
+ print("Successfully merged")
+
# =======================================================
# Now create the integration commit in the manifest
# for all subprojects at once
# =======================================================
- manifest_project = project_integration[args.project]
manifest_file_abs = os.path.join(
manifest_project["repo"].working_tree_dir, args.manifest_file
)
@@ -275,58 +355,34 @@ def main():
sys.exit(0)
# ============================================
- # Create merge requests if needed
+ # Create merge requests for the manifest
# ============================================
- # Get source merge request ( the one in the gitlab-ci repo)
- submodule_project_path, _ = get_submodule_project_path_and_revision(
- manifest_project["project"], args.submodule, args.branch
- )
- submodule_project = common.get_project(gitlab, submodule_project_path)
- mrs = get_merge_requests(
- submodule_project,
- # TODO should this be submodule_project's default branch?
- target_branch="master",
- commit=args.revision,
- )
- if not mrs:
- sys.exit(
- "ERROR: could not determine source merge request for commit %s"
- % args.revision
- )
- source_mr = mrs[0]
-
- for p in args.projects + [args.project]:
- integration = project_integration[p]
- logging.debug("Create MR in %s", integration["project"].name)
- integration["mr"] = create_integration_merge_request(
- integration["project"], integration["branch"], source_mr
- )
+ logging.debug("Create MR in %s", manifest_project["project"].name)
+ manifest_project["mr"] = create_integration_merge_request(
+ manifest_project["project"], manifest_project["branch"], source_mr
+ )
# =================================================
- # Now merge them all
+ # Now merge it
# =================================================
# The manifest needs to be merged at last
- for p in args.projects + [args.project]:
- integration = project_integration[p]
- mr = integration["mr"]
- logging.debug("Merge %s!%s", p, mr.iid)
+ mr = manifest_project["mr"]
+ logging.debug("Merge %s!%s", args.project, mr.iid)
- # Wait until GitLab has checked merge status
- common.wait_until_merge_status_is_set(integration["project"], mr)
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(manifest_project["project"], mr)
- # Attempt to merge
- # TODO if this has rebased the integration commit needs to be adapted
- # change the sequence
- merged = accept_merge_request(integration["project"], mr, rebase=True)
+ # Attempt to merge
+ merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
- if not merged:
- sys.exit(
- "Integration MR could not be merged. You have two possibilities to fix "
- "this:\n"
- " 1. Checkout the MR and rebase it on the current master manually, or\n"
- " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
- "In either case restart this job afterwards in order to get it merged."
- )
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged. You have two possibilities to fix "
+ "this:\n"
+ " 1. Checkout the MR and rebase it on the current master manually, or\n"
+ " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
+ "In either case restart this job afterwards in order to get it merged."
+ )
print("Successfully merged")
diff --git a/merge_gitlab_ci.py b/merge_gitlab_ci.py
deleted file mode 100755
index 217419dbcb0e23a91a14a20953a69bb12ac0dc89..0000000000000000000000000000000000000000
--- a/merge_gitlab_ci.py
+++ /dev/null
@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-import common
-
-import argparse
-import logging
-import sys
-from gitlab import Gitlab, GitlabGetError
-from accept_merge_request import accept_merge_request
-import update_submodule
-from gitlab.v4.objects import Project
-
-
-def find_integration_merge_request(
- project: Project, submodule, revision, target_branch
-):
-
- (
- submodule_path,
- submodule_revision,
- ) = update_submodule.get_submodule_project_path_and_revision(
- project, submodule, target_branch
- )
- logging.debug("Module: %s, Revision: %s", submodule_path, submodule_revision)
-
- # Get submodule project
- gitlab = project.manager.gitlab
- submodule_project = common.get_project(gitlab, submodule_path)
-
- # Get the integration branch name
- integration_branch_suffix = (
- update_submodule.get_submodule_integration_branch_suffix(
- submodule_project, revision
- )
- )
- integration_branch_name = common.integration_branch_name(
- submodule_project.name, integration_branch_suffix
- )
- logging.debug(integration_branch_name)
-
- # Get the merge request for the branch TODO catch exception
- try:
- integration_branch = project.branches.get(
- integration_branch_name, retry_transient_errors=True
- )
- except GitlabGetError:
- print("ERROR: Failed to find integration branch for {}".format(submodule))
- return None
-
- logging.debug(integration_branch)
- commit = project.commits.get(
- integration_branch.commit["id"], retry_transient_errors=True
- )
- for mr in commit.merge_requests(retry_transient_errors=True):
- if mr["target_branch"] == target_branch:
- integration_mr = mr
- break
- if integration_mr is None:
- print(
- "ERROR: Failed to find integration merge request for %s",
- integration_branch_name,
- )
- return None
- logging.debug(integration_mr["iid"])
- mr = project.mergerequests.get(integration_mr["iid"], retry_transient_errors=True)
-
- return mr
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "--gitlab-url",
- help="""URL to the GitLab instance""",
- dest="gitlab_url",
- required=True,
- )
- parser.add_argument(
- "--token",
- help="""GitLab REST API private access token""",
- dest="token",
- required=True,
- )
- parser.add_argument(
- "--project",
- "--manifest-project",
- help="""name of the GitLab project""",
- dest="project",
- )
- parser.add_argument(
- "--submodule",
- help="""submodule to update""",
- dest="submodule",
- required=True,
- )
- parser.add_argument(
- "--revision",
- help="""new revision for submodule""",
- dest="revision",
- required=True,
- )
- parser.add_argument(
- "--branch",
- help="""project branch (if not default branch)""",
- dest="branch",
- required=False,
- default=None,
- )
- parser.add_argument(
- "--merge",
- help="""if set, perform merge after integration""",
- dest="merge",
- action="store_true",
- required=False,
- default=False,
- )
- parser.add_argument(
- "projects",
- help="""List of projects the change should be deployed to additionally
- to the manifest project given as named parameter.""",
- nargs="*",
- )
- parser.add_argument(
- "-v",
- "--verbose",
- action="store_true",
- help="""Increase verbosity.""",
- )
-
- args, _ = parser.parse_known_args()
- if args.verbose:
- logging.basicConfig(level=logging.DEBUG)
-
- gitlab = Gitlab(args.gitlab_url, private_token=args.token)
-
- # Start with the manifest here, so the subproject
- # can see that they are already integrated
- for p in args.projects + [args.project]:
- project = common.get_project(gitlab, p)
- branch = args.branch
-
- if branch is None:
- branch = project.default_branch
-
- print(
- "Try to merge {}({}) into {} ({})".format(
- args.submodule, args.revision, project.name, branch
- )
- )
-
- mr = find_integration_merge_request(
- project, args.submodule, args.revision, branch
- )
- if mr is None:
- sys.exit("ERROR: Failed to find the integration MR.")
- # TODO if this ever happens, why ever, we could call
- # deploy_gitlab_ci again to create a new integration
- # commit or make sure the change is already integrated.
-
- print("Merge {}!{}: {}".format(project.name, mr.iid, mr.title))
- # Wait until GitLab has checked merge status
- common.wait_until_merge_status_is_set(project, mr)
-
- # Attempt to merge
- merged = accept_merge_request(
- project,
- mr,
- # Only the file manifest commit may be rebased
- # other rebase would require new integration commits
- # TODO implement rebase
- rebase=(p == args.project),
- )
-
- if not merged:
- print(
- "Integration MR could not be merged."
- "To fix this:\n"
- " 1. Checkout the MR {}!{} and merge it manually.\n"
- " 2. Manually merge the follow up MRs in the following project:".format(
- project.name, mr.iid
- )
- )
- found = False
- for p2 in [args.project] + args.projects:
- if p2 == p:
- found = True
- continue
- if not found:
- continue
- print(" {}".format(p2))
- sys.exit()
-
- print("Successfully merged")
-
- exit()
-
-
-if __name__ == "__main__":
- main()
diff --git a/merge_into_manifest.py b/merge_into_manifest.py
index 603a9ccd6deb2bc1759bf6cf8a2e978b2b84224e..37205cf7f31f3f7364dd1ba5e10d96c11090bb73 100755
--- a/merge_into_manifest.py
+++ b/merge_into_manifest.py
@@ -78,7 +78,7 @@ def merge_into_manifest(
common.wait_until_merge_status_is_set(manifest_project, mr)
# Attempt to merge
- merged = accept_merge_request(manifest_project, mr)
+ merged, _ = accept_merge_request(manifest_project, mr)
if not merged:
# Merge failed, reintegrate the source merge request into the manifest.