diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b6ea9fcc868ce14d93c3bd111a24f8c63bb12f87..833113758812b28c973090ed8559a8a1c5805d8a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,12 +1,12 @@
+---
# ---------------------------------------------------------------------------------------
# Global
# ---------------------------------------------------------------------------------------
----
variables:
# CI_IMAGES_BASEPATH: Environment variable configure in gitlab
CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
- CI_IMAGES_REVISION: 44965ccdd847f1e077670f49d546047f8ad0110c
+ CI_IMAGES_REVISION: 5dfddb02d67bbd16beb09ff4e31afcd5380f5788
CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REVISION}"
CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REVISION}"
@@ -65,22 +65,6 @@ yamllint:
# ---------------------------------------------------------------------------------------
# Stage: integrate
# ---------------------------------------------------------------------------------------
-.ci-test-projects:
- variables:
- PROJECT_GROUP:
- ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
- MANIFEST_PROJECT:
- ${PROJECT_GROUP}/minimal-manifest
- MANIFEST_BRANCH: primary
-
-.yocto-projects:
- variables:
- PROJECT_GROUP:
- ${CI_PROJECT_ROOT_NAMESPACE}/yocto
- MANIFEST_PROJECT:
- ${PROJECT_ROOT}/manifest
- MANIFEST_BRANCH: dunfell
-
.integrate:
stage: integrate
rules:
@@ -99,49 +83,102 @@ yamllint:
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
--group=${PROJECT_GROUP}
- --verbose
${MERGE}
-integrate-yocto:
- extends:
- - .integrate
- - .yocto-projects
+# Running multiple integration jobs for the same manifest in parallel can lead to race
+# conditions if there are projects integrating a single branch to different manifest
+# branches. Use resource groups to force execution one by one.
+.integrate-ci-test:
+ extends: .integrate
+ resource_group: integrate-ci-test
+ variables:
+ PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
+ MANIFEST_PROJECT: ${PROJECT_GROUP}/minimal-manifest
+
+.integrate-yocto:
+ extends: .integrate
+ resource_group: integrate-yocto
+ variables:
+ PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}
+ MANIFEST_PROJECT: ${PROJECT_GROUP}/yocto/manifest
+
+# Jobs
+
+integrate-ci-test:primary:
+ extends: .integrate-ci-test
+ variables:
+ MANIFEST_BRANCH: primary
+
+integrate-ci-test:secondary:
+ extends: .integrate-ci-test
+ variables:
+ MANIFEST_BRANCH: secondary
+
+integrate-yocto:dunfell:
+ extends: .integrate-yocto
+ variables:
+ MANIFEST_BRANCH: dunfell
+
+integrate-yocto:kirkstone:
+ extends: .integrate-yocto
+ variables:
+ MANIFEST_BRANCH: kirkstone
-integrate-ci-test:
- extends:
- - .integrate
- - .ci-test-projects
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
-build-yocto:
+.build:
stage: build
- needs: [integrate-yocto]
rules:
- if: $CI_MERGE_REQUEST_IID
allow_failure: true
+
+.build-ci-test:
+ extends: .build
trigger:
- project: seco-ne/yocto/manifest
- branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+ project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
+ branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"
strategy: depend
-build-ci-test:
- stage: build
- needs: [integrate-ci-test]
- rules:
- - if: $CI_MERGE_REQUEST_IID
- allow_failure: true
+.build-yocto:
+ extends: .build
trigger:
- project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
- branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+ project: seco-ne/yocto/manifest
+ branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"
strategy: depend
+# Jobs
+
+build-ci-test:primary:
+ extends: .build-ci-test
+ needs: ["integrate-ci-test:primary"]
+ variables:
+ MANIFEST_BRANCH: primary
+
+build-ci-test:secondary:
+ extends: .build-ci-test
+ needs: ["integrate-ci-test:secondary"]
+ variables:
+ MANIFEST_BRANCH: secondary
+
+build-yocto:dunfell:
+ extends: .build-yocto
+ needs: ["integrate-yocto:dunfell"]
+ variables:
+ MANIFEST_BRANCH: dunfell
+
+build-yocto:kirkstone:
+ extends: .build-yocto
+ needs: ["integrate-yocto:kirkstone"]
+ variables:
+ MANIFEST_BRANCH: kirkstone
+
+
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
.merge:
- extends: .integrate
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == "master"
@@ -152,35 +189,62 @@ build-ci-test:
merge-ci-test:
extends:
+ - .integrate-ci-test
- .merge
- - .ci-test-projects
+ variables:
+ MANIFEST_BRANCH: primary,secondary
merge-yocto:
extends:
+ - .integrate-yocto
- .merge
- - .yocto-projects
+ variables:
+ MANIFEST_BRANCH: dunfell,kirkstone
+
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
-build-master-yocto:
+.build-master:
stage: build
- needs: [merge-yocto]
rules:
- if: $CI_COMMIT_BRANCH == "master"
when: manual
+
+.build-master-ci-test:
+ extends: .build-master
+ needs: ["merge-ci-test"]
trigger:
- project: seco-ne/yocto/manifest
- branch: "dunfell"
+ project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
+ branch: "${MANIFEST_BRANCH}"
strategy: depend
-build-master-ci-test:
- stage: build
- needs: [merge-ci-test]
- rules:
- - if: $CI_COMMIT_BRANCH == "master"
- when: manual
+.build-master-yocto:
+ extends: .build-master
+ needs: ["merge-yocto"]
trigger:
- project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
- branch: "master"
+ project: seco-ne/yocto/manifest
+ branch: "${MANIFEST_BRANCH}"
strategy: depend
+
+# Jobs
+
+build-master-ci-test:primary:
+ extends: .build-master-ci-test
+ variables:
+ MANIFEST_BRANCH: primary
+
+build-master-ci-test:secondary:
+ extends: .build-master-ci-test
+ variables:
+ MANIFEST_BRANCH: secondary
+
+build-master-yocto:dunfell:
+ extends: .build-master-yocto
+ variables:
+ MANIFEST_BRANCH: dunfell
+
+build-master-yocto:kirkstone:
+ extends: .build-master-yocto
+ variables:
+ MANIFEST_BRANCH: kirkstone
diff --git a/scripts/deploy_gitlab_ci.py b/scripts/deploy_gitlab_ci.py
index c3a1bf0a001916d925fe3070829c33e40804bb08..5f1f0d428a70105a15e2bdf43e6484c5270c6e5b 100755
--- a/scripts/deploy_gitlab_ci.py
+++ b/scripts/deploy_gitlab_ci.py
@@ -69,7 +69,8 @@ def integrate_submodule_into(
"project": gitlab_project,
"repo": project_repo,
"dir": project_dir,
- "branch": integration_branch_name,
+ "integration_branch": integration_branch_name,
+ "master_branch": branch,
"commit": integration_commit,
"message": message,
}
@@ -118,7 +119,7 @@ def main():
)
parser.add_argument(
"--manifest-branch",
- help="""manifest branch to integrate changes into""",
+ help="""manifest branch to integrate changes into (can be a comma-separated list)""",
dest="manifest_branch",
required=True,
)
@@ -177,6 +178,8 @@ def main():
datefmt="%H:%M:%S",
)
+ manifest_branches = args.manifest_branch.split(",")
+
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
group = gitlab.groups.get(args.group)
@@ -184,44 +187,60 @@ def main():
# Create integration branches and commits with updates
# submodule in all projects
# =======================================================
- project_integration = {}
- projects = get_integrating_projects(
- args.manifest_project, args.manifest_branch, group
- )
- # Update submodule in all 'child' project
+ projects = []
+ for manifest_branch in manifest_branches:
+ print(
+ "Searching for projects in %s that are configured for automatic integration into %s:%s"
+ % (args.group, args.manifest_project, manifest_branch)
+ )
+ for p in get_integrating_projects(
+ args.manifest_project, manifest_branch, group
+ ):
+ if p not in projects:
+ projects.append(p)
+
+ # Update submodule in all integrating projects
+ project_integrations = []
for p in projects:
- print("Create integration commit in", p["project"])
+ print("Create integration commit in %s:%s" % (p["project"], p["branch"]))
- res = integrate_submodule_into(
+ integration = integrate_submodule_into(
gitlab, p["project"], args.submodule, args.revision, p["branch"]
)
# Store in the list if commit is set (meaning there was an update or
# an exising integration branch)
- if res["commit"] is not None:
- project_integration[p["project"]] = res
-
- print("Create integration commit in", args.manifest_project)
- # Update submodule in manifest project
- manifest_project = integrate_submodule_into(
- gitlab,
- args.manifest_project,
- args.submodule,
- args.revision,
- args.manifest_branch,
- commit_and_push=False,
- force_clone=True,
- )
+ if integration["commit"] is not None:
+ project_integrations.append(integration)
+
+ manifest_projects = []
+ for manifest_branch in manifest_branches:
+ print(
+ "Create integration commit in %s:%s"
+ % (args.manifest_project, manifest_branch),
+ )
+ # Update submodule in manifest project
+ manifest_projects.append(
+ integrate_submodule_into(
+ gitlab,
+ args.manifest_project,
+ args.submodule,
+ args.revision,
+ manifest_branch,
+ commit_and_push=False,
+ force_clone=True,
+ )
+ )
- branch = args.manifest_branch
- if branch is None:
- branch = manifest_project["project"].default_branch
# =======================================================
# Create and merge merge_requests if needed
# =======================================================
if args.merge:
- # Get source merge request ( the one in the gitlab-ci repo)
+ # Get source merge request (the one in the gitlab-ci repo)
+ branch = manifest_projects[0]["master_branch"]
+ if branch is None:
+ branch = manifest_projects[0]["project"].default_branch
submodule_project_path, _ = get_submodule_project_path_and_revision(
- manifest_project["project"], args.submodule, branch
+ manifest_projects[0]["project"], args.submodule, branch
)
submodule_project = common.get_project(gitlab, submodule_project_path)
mrs = get_merge_requests(
@@ -237,11 +256,10 @@ def main():
)
source_mr = mrs[0]
- for p in project_integration:
- integration = project_integration[p]
+ for integration in project_integrations:
logging.debug("Create MR in %s", integration["project"].name)
mr = create_integration_merge_request(
- integration["project"], integration["branch"], source_mr
+ integration["project"], integration["integration_branch"], source_mr
)
integration["mr"] = mr
# Now merge
@@ -255,10 +273,10 @@ def main():
integration["project"], mr, rebase=True
)
# if this has rebased the integration commit needs to be adapted:
- project_integration[p]["commit"] = integration_commit
+ integration["commit"] = integration_commit
# Save the target branch here, as the source branch gets deleted
# during merge
- project_integration[p]["branch"] = mr.target_branch
+ integration["integration_branch"] = mr.target_branch
if not merged:
sys.exit(
@@ -275,89 +293,87 @@ def main():
# Now create the integration commit in the manifest
# for all subprojects at once
# =======================================================
- manifest_file_abs = os.path.join(
- manifest_project["repo"].working_tree_dir, args.manifest_file
- )
- logging.debug("Read manifest from: %s", manifest_file_abs)
- with open(manifest_file_abs, "r", encoding="utf8") as fp:
- manifest = fp.read()
- logging.debug(manifest)
- srcrev_file_abs = os.path.join(
- manifest_project["repo"].working_tree_dir, args.srcrev_file
- )
- logging.debug("Read manifest from: %s", srcrev_file_abs)
- with open(srcrev_file_abs, "r", encoding="utf8") as fp:
- srcrev = fp.read()
- logging.debug(srcrev)
-
- for p in project_integration:
- integration = project_integration[p]
- logging.debug(
- "Update %s to %s", integration["project"].name, integration["commit"]
+ for manifest_project in manifest_projects:
+ manifest_file_abs = os.path.join(
+ manifest_project["repo"].working_tree_dir, args.manifest_file
)
-
- new_manifest = update_manifest(
- manifest, integration["project"], integration["commit"]
+ logging.debug("Read manifest from: %s", manifest_file_abs)
+ with open(manifest_file_abs, "r", encoding="utf8") as fp:
+ manifest = fp.read()
+ logging.debug(manifest)
+ srcrev_file_abs = os.path.join(
+ manifest_project["repo"].working_tree_dir, args.srcrev_file
)
- if new_manifest is not None:
- manifest = new_manifest
- logging.debug(manifest)
- continue
-
- # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
- # Use direct read from gitlab as we have not checked out
- # the repo if the branch is already up to date
+ logging.debug("Read manifest from: %s", srcrev_file_abs)
+ with open(srcrev_file_abs, "r", encoding="utf8") as fp:
+ srcrev = fp.read()
+ logging.debug(srcrev)
+
+ for integration in project_integrations:
+ logging.debug(
+ "Update %s to %s", integration["project"].name, integration["commit"]
+ )
- gitlab_ci_yml = common.get_repository_file_raw(
- integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
- )
- project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
+ new_manifest = update_manifest(
+ manifest, integration["project"], integration["commit"]
+ )
+ if new_manifest is not None:
+ manifest = new_manifest
+ logging.debug(manifest)
+ continue
+
+ # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
+ # Use direct read from gitlab as we have not checked out
+ # the repo if the branch is already up to date
+
+ gitlab_ci_yml = common.get_repository_file_raw(
+ integration["project"],
+ ".gitlab-ci.yml",
+ ref=integration["integration_branch"],
+ )
+ project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
- new_srcrev = update_srcrev(
- srcrev, project_keys["recipe"], integration["commit"]
+ new_srcrev = update_srcrev(
+ srcrev, project_keys["recipe"], integration["commit"]
+ )
+ if new_srcrev is not None:
+ srcrev = new_srcrev
+ logging.debug(srcrev)
+ else:
+ logging.debug("Project %s not found in xml or srcrev file", p)
+
+ # Write manifest
+ with open(manifest_file_abs, "w", encoding="utf8") as fp:
+ fp.write(manifest)
+ manifest_project["repo"].git.add(args.manifest_file)
+ logging.debug(manifest)
+ with open(srcrev_file_abs, "w", encoding="utf8") as fp:
+ fp.write(srcrev)
+ manifest_project["repo"].git.add(args.srcrev_file)
+ logging.debug(srcrev)
+
+ # ========================================================
+ # Squash all commits on the integration branch to one
+ # ========================================================
+ manifest_project["repo"].remotes.origin.fetch(manifest_project["master_branch"])
+ manifest_master = manifest_project["project"].branches.get(
+ manifest_project["master_branch"]
)
- if new_srcrev is not None:
- srcrev = new_srcrev
- logging.debug(srcrev)
- else:
- logging.debug("Project %s not found in xml or srcrev file", p)
-
- # Write manifest
- with open(manifest_file_abs, "w", encoding="utf8") as fp:
- fp.write(manifest)
- manifest_project["repo"].git.add(args.manifest_file)
- logging.debug(manifest)
- with open(srcrev_file_abs, "w", encoding="utf8") as fp:
- fp.write(srcrev)
- manifest_project["repo"].git.add(args.srcrev_file)
- logging.debug(srcrev)
-
- # ========================================================
- # Squash all commits on the integration branch to one
- # ========================================================
- manifest_project["repo"].remotes.origin.fetch(branch)
- manifest_master = manifest_project["project"].branches.get(branch)
- manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
-
- # ========================================================
- # Now commit and push the changes to the manifest repo
- # ========================================================
- # Make an API request to create the gitlab.user object
- gitlab = manifest_project["project"].manager.gitlab
- gitlab.auth()
- integration_commit = common.commit_and_push(
- manifest_project["project"],
- manifest_project["repo"],
- manifest_project["message"],
- gitlab.user.username,
- gitlab.user.email,
- )
-
- print(
- "Successfully create integration commit {} in {}".format(
- integration_commit, args.manifest_project
+ manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
+
+ # ========================================================
+ # Now commit and push the changes to the manifest repo
+ # ========================================================
+ # Make an API request to create the gitlab.user object
+ gitlab = manifest_project["project"].manager.gitlab
+ gitlab.auth()
+ integration_commit = common.commit_and_push(
+ manifest_project["project"],
+ manifest_project["repo"],
+ manifest_project["message"],
+ gitlab.user.username,
+ gitlab.user.email,
)
- )
if not args.merge:
sys.exit(0)
@@ -365,34 +381,36 @@ def main():
# ============================================
# Create merge requests for the manifest
# ============================================
-
- logging.debug("Create MR in %s", manifest_project["project"].name)
- manifest_project["mr"] = create_integration_merge_request(
- manifest_project["project"], manifest_project["branch"], source_mr
- )
- # =================================================
- # Now merge it
- # =================================================
- # The manifest needs to be merged at last
- mr = manifest_project["mr"]
- logging.debug("Merge %s!%s", args.manifest_project, mr.iid)
-
- # Wait until GitLab has checked merge status
- common.wait_until_merge_status_is_set(manifest_project["project"], mr)
-
- # Attempt to merge
- merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
-
- if not merged:
- sys.exit(
- "Integration MR could not be merged. You have two possibilities to fix "
- "this:\n"
- " 1. Checkout the MR and rebase it on the current master manually, or\n"
- " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
- "In either case restart this job afterwards in order to get it merged."
+ for manifest_project in manifest_projects:
+ logging.debug("Create MR in %s", manifest_project["project"].name)
+ manifest_project["mr"] = create_integration_merge_request(
+ manifest_project["project"],
+ manifest_project["integration_branch"],
+ source_mr,
)
+ # =================================================
+ # Now merge it
+ # =================================================
+ # The manifest needs to be merged at last
+ mr = manifest_project["mr"]
+ logging.debug("Merge %s!%s", args.manifest_project, mr.iid)
- print("Successfully merged")
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(manifest_project["project"], mr)
+
+ # Attempt to merge
+ merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
+
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged. You have two possibilities to fix "
+ "this:\n"
+ " 1. Checkout the MR and rebase it on the current master manually, or\n"
+ " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
+ "In either case restart this job afterwards in order to get it merged."
+ )
+
+ print("Successfully merged")
if __name__ == "__main__":