Deploy gitlab-ci to multiple branches

Add different integration jobs for different manifest branches. The
merge stage only has one job for each manifest, though, because
otherwise we cannot guarantee a consistent state for all branches.

Extend the deploy script for this purpose, so that it can deploy to
multiple manifest branches at once.

.gitlab-ci.yml

+---
 # ---------------------------------------------------------------------------------------
 # Global
 # ---------------------------------------------------------------------------------------
----
 
 variables:
   # CI_IMAGES_BASEPATH: Environment variable configure in gitlab
   CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
-  CI_IMAGES_REVISION: 44965ccdd847f1e077670f49d546047f8ad0110c
+  CI_IMAGES_REVISION: 5dfddb02d67bbd16beb09ff4e31afcd5380f5788
   CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REVISION}"
   CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REVISION}"
 
@@ -65,22 +65,6 @@ yamllint:
 # ---------------------------------------------------------------------------------------
 # Stage: integrate
 # ---------------------------------------------------------------------------------------
-.ci-test-projects:
-  variables:
-    PROJECT_GROUP:
-      ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
-    MANIFEST_PROJECT:
-      ${PROJECT_GROUP}/minimal-manifest
-    MANIFEST_BRANCH: primary
-
-.yocto-projects:
-  variables:
-    PROJECT_GROUP:
-      ${CI_PROJECT_ROOT_NAMESPACE}/yocto
-    MANIFEST_PROJECT:
-      ${PROJECT_ROOT}/manifest
-    MANIFEST_BRANCH: dunfell
-
 .integrate:
   stage: integrate
   rules:
@@ -99,49 +83,102 @@ yamllint:
       --submodule=.gitlab-ci
       --revision=${CI_COMMIT_SHA}
       --group=${PROJECT_GROUP}
-      --verbose
       ${MERGE}
 
-integrate-yocto:
-  extends:
-    - .integrate
-    - .yocto-projects
+# Running multiple integration jobs for the same manifest in parallel can lead to race
+# conditions if there are projects integrating a single branch to different manifest
+# branches. Use resource groups to force execution one by one.
+.integrate-ci-test:
+  extends: .integrate
+  resource_group: integrate-ci-test
+  variables:
+    PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
+    MANIFEST_PROJECT: ${PROJECT_GROUP}/minimal-manifest
+
+.integrate-yocto:
+  extends: .integrate
+  resource_group: integrate-yocto
+  variables:
+    PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}
+    MANIFEST_PROJECT: ${PROJECT_GROUP}/yocto/manifest
+
+# Jobs
+
+integrate-ci-test:primary:
+  extends: .integrate-ci-test
+  variables:
+    MANIFEST_BRANCH: primary
+
+integrate-ci-test:secondary:
+  extends: .integrate-ci-test
+  variables:
+    MANIFEST_BRANCH: secondary
+
+integrate-yocto:dunfell:
+  extends: .integrate-yocto
+  variables:
+    MANIFEST_BRANCH: dunfell
+
+integrate-yocto:kirkstone:
+  extends: .integrate-yocto
+  variables:
+    MANIFEST_BRANCH: kirkstone
 
-integrate-ci-test:
-  extends:
-    - .integrate
-    - .ci-test-projects
 
 # --------------------------------------------------------------------------------------
 # Stage: build
 # --------------------------------------------------------------------------------------
-build-yocto:
+.build:
   stage: build
-  needs: [integrate-yocto]
   rules:
     - if: $CI_MERGE_REQUEST_IID
       allow_failure: true
+
+.build-ci-test:
+  extends: .build
   trigger:
-    project: seco-ne/yocto/manifest
-    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
+    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"
     strategy: depend
 
-build-ci-test:
-  stage: build
-  needs: [integrate-ci-test]
-  rules:
-    - if: $CI_MERGE_REQUEST_IID
-      allow_failure: true
+.build-yocto:
+  extends: .build
   trigger:
-    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
-    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+    project: seco-ne/yocto/manifest
+    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"
     strategy: depend
 
+# Jobs
+
+build-ci-test:primary:
+  extends: .build-ci-test
+  needs: ["integrate-ci-test:primary"]
+  variables:
+    MANIFEST_BRANCH: primary
+
+build-ci-test:secondary:
+  extends: .build-ci-test
+  needs: ["integrate-ci-test:secondary"]
+  variables:
+    MANIFEST_BRANCH: secondary
+
+build-yocto:dunfell:
+  extends: .build-yocto
+  needs: ["integrate-yocto:dunfell"]
+  variables:
+    MANIFEST_BRANCH: dunfell
+
+build-yocto:kirkstone:
+  extends: .build-yocto
+  needs: ["integrate-yocto:kirkstone"]
+  variables:
+    MANIFEST_BRANCH: kirkstone
+
+
 # --------------------------------------------------------------------------------------
 # Stage: merge
 # --------------------------------------------------------------------------------------
 .merge:
-  extends: .integrate
   stage: merge
   rules:
     - if: $CI_COMMIT_BRANCH == "master"
@@ -152,35 +189,62 @@ build-ci-test:
 
 merge-ci-test:
   extends:
+    - .integrate-ci-test
     - .merge
-    - .ci-test-projects
+  variables:
+    MANIFEST_BRANCH: primary,secondary
 
 merge-yocto:
   extends:
+    - .integrate-yocto
     - .merge
-    - .yocto-projects
+  variables:
+    MANIFEST_BRANCH: dunfell,kirkstone
+
 
 # --------------------------------------------------------------------------------------
 # Stage: build
 # --------------------------------------------------------------------------------------
-build-master-yocto:
+.build-master:
   stage: build
-  needs: [merge-yocto]
   rules:
     - if: $CI_COMMIT_BRANCH == "master"
       when: manual
+
+.build-master-ci-test:
+  extends: .build-master
+  needs: ["merge-ci-test"]
   trigger:
-    project: seco-ne/yocto/manifest
-    branch: "dunfell"
+    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
+    branch: "${MANIFEST_BRANCH}"
     strategy: depend
 
-build-master-ci-test:
-  stage: build
-  needs: [merge-ci-test]
-  rules:
-    - if: $CI_COMMIT_BRANCH == "master"
-      when: manual
+.build-master-yocto:
+  extends: .build-master
+  needs: ["merge-yocto"]
   trigger:
-    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
-    branch: "master"
+    project: seco-ne/yocto/manifest
+    branch: "${MANIFEST_BRANCH}"
     strategy: depend
+
+# Jobs
+
+build-master-ci-test:primary:
+  extends: .build-master-ci-test
+  variables:
+    MANIFEST_BRANCH: primary
+
+build-master-ci-test:secondary:
+  extends: .build-master-ci-test
+  variables:
+    MANIFEST_BRANCH: secondary
+
+build-master-yocto:dunfell:
+  extends: .build-master-yocto
+  variables:
+    MANIFEST_BRANCH: dunfell
+
+build-master-yocto:kirkstone:
+  extends: .build-master-yocto
+  variables:
+    MANIFEST_BRANCH: kirkstone

scripts/deploy_gitlab_ci.py

@@ -69,7 +69,8 @@ def integrate_submodule_into(
         "project": gitlab_project,
         "repo": project_repo,
         "dir": project_dir,
-        "branch": integration_branch_name,
+        "integration_branch": integration_branch_name,
+        "master_branch": branch,
         "commit": integration_commit,
         "message": message,
     }
@@ -118,7 +119,7 @@ def main():
     )
     parser.add_argument(
         "--manifest-branch",
-        help="""manifest branch to integrate changes into""",
+        help="""manifest branch to integrate changes into (can be a comma-separated list)""",
         dest="manifest_branch",
         required=True,
     )
@@ -177,6 +178,8 @@ def main():
             datefmt="%H:%M:%S",
         )
 
+    manifest_branches = args.manifest_branch.split(",")
+
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
     group = gitlab.groups.get(args.group)
 
@@ -184,44 +187,60 @@ def main():
     # Create integration branches and commits with updates
     # submodule in all projects
     # =======================================================
-    project_integration = {}
-    projects = get_integrating_projects(
-        args.manifest_project, args.manifest_branch, group
-    )
-    # Update submodule in all 'child' project
+    projects = []
+    for manifest_branch in manifest_branches:
+        print(
+            "Searching for projects in %s that are configured for automatic integration into %s:%s"
+            % (args.group, args.manifest_project, manifest_branch)
+        )
+        for p in get_integrating_projects(
+            args.manifest_project, manifest_branch, group
+        ):
+            if p not in projects:
+                projects.append(p)
+
+    # Update submodule in all integrating projects
+    project_integrations = []
     for p in projects:
-        print("Create integration commit in", p["project"])
+        print("Create integration commit in %s:%s" % (p["project"], p["branch"]))
 
-        res = integrate_submodule_into(
+        integration = integrate_submodule_into(
             gitlab, p["project"], args.submodule, args.revision, p["branch"]
         )
         # Store in the list if commit is set (meaning there was an update or
         #   an exising integration branch)
-        if res["commit"] is not None:
-            project_integration[p["project"]] = res
-
-    print("Create integration commit in", args.manifest_project)
-    # Update submodule in manifest project
-    manifest_project = integrate_submodule_into(
-        gitlab,
-        args.manifest_project,
-        args.submodule,
-        args.revision,
-        args.manifest_branch,
-        commit_and_push=False,
-        force_clone=True,
-    )
+        if integration["commit"] is not None:
+            project_integrations.append(integration)
+
+    manifest_projects = []
+    for manifest_branch in manifest_branches:
+        print(
+            "Create integration commit in %s:%s"
+            % (args.manifest_project, manifest_branch),
+        )
+        # Update submodule in manifest project
+        manifest_projects.append(
+            integrate_submodule_into(
+                gitlab,
+                args.manifest_project,
+                args.submodule,
+                args.revision,
+                manifest_branch,
+                commit_and_push=False,
+                force_clone=True,
+            )
+        )
 
-    branch = args.manifest_branch
-    if branch is None:
-        branch = manifest_project["project"].default_branch
     # =======================================================
     # Create and merge merge_requests if needed
     # =======================================================
     if args.merge:
-        # Get source merge request ( the one in the gitlab-ci repo)
+        # Get source merge request (the one in the gitlab-ci repo)
+        branch = manifest_projects[0]["master_branch"]
+        if branch is None:
+            branch = manifest_projects[0]["project"].default_branch
         submodule_project_path, _ = get_submodule_project_path_and_revision(
-            manifest_project["project"], args.submodule, branch
+            manifest_projects[0]["project"], args.submodule, branch
         )
         submodule_project = common.get_project(gitlab, submodule_project_path)
         mrs = get_merge_requests(
@@ -237,11 +256,10 @@ def main():
             )
         source_mr = mrs[0]
 
-        for p in project_integration:
-            integration = project_integration[p]
+        for integration in project_integrations:
             logging.debug("Create MR in %s", integration["project"].name)
             mr = create_integration_merge_request(
-                integration["project"], integration["branch"], source_mr
+                integration["project"], integration["integration_branch"], source_mr
             )
             integration["mr"] = mr
             # Now merge
@@ -255,10 +273,10 @@ def main():
                 integration["project"], mr, rebase=True
             )
             # if this has rebased the integration commit needs to be adapted:
-            project_integration[p]["commit"] = integration_commit
+            integration["commit"] = integration_commit
             # Save the target branch here, as the source branch gets deleted
             # during merge
-            project_integration[p]["branch"] = mr.target_branch
+            integration["integration_branch"] = mr.target_branch
 
             if not merged:
                 sys.exit(
@@ -275,89 +293,87 @@ def main():
     # Now create the integration commit in the manifest
     # for all subprojects at once
     # =======================================================
-    manifest_file_abs = os.path.join(
-        manifest_project["repo"].working_tree_dir, args.manifest_file
-    )
-    logging.debug("Read manifest from: %s", manifest_file_abs)
-    with open(manifest_file_abs, "r", encoding="utf8") as fp:
-        manifest = fp.read()
-    logging.debug(manifest)
-    srcrev_file_abs = os.path.join(
-        manifest_project["repo"].working_tree_dir, args.srcrev_file
-    )
-    logging.debug("Read manifest from: %s", srcrev_file_abs)
-    with open(srcrev_file_abs, "r", encoding="utf8") as fp:
-        srcrev = fp.read()
-    logging.debug(srcrev)
-
-    for p in project_integration:
-        integration = project_integration[p]
-        logging.debug(
-            "Update %s to %s", integration["project"].name, integration["commit"]
+    for manifest_project in manifest_projects:
+        manifest_file_abs = os.path.join(
+            manifest_project["repo"].working_tree_dir, args.manifest_file
         )
-
-        new_manifest = update_manifest(
-            manifest, integration["project"], integration["commit"]
+        logging.debug("Read manifest from: %s", manifest_file_abs)
+        with open(manifest_file_abs, "r", encoding="utf8") as fp:
+            manifest = fp.read()
+        logging.debug(manifest)
+        srcrev_file_abs = os.path.join(
+            manifest_project["repo"].working_tree_dir, args.srcrev_file
         )
-        if new_manifest is not None:
-            manifest = new_manifest
-            logging.debug(manifest)
-            continue
-
-        # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
-        # Use direct read from gitlab as we have not checked out
-        # the repo if the branch is already up to date
+        logging.debug("Read manifest from: %s", srcrev_file_abs)
+        with open(srcrev_file_abs, "r", encoding="utf8") as fp:
+            srcrev = fp.read()
+        logging.debug(srcrev)
+
+        for integration in project_integrations:
+            logging.debug(
+                "Update %s to %s", integration["project"].name, integration["commit"]
+            )
 
-        gitlab_ci_yml = common.get_repository_file_raw(
-            integration["project"], ".gitlab-ci.yml", ref=integration["branch"]
-        )
-        project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
+            new_manifest = update_manifest(
+                manifest, integration["project"], integration["commit"]
+            )
+            if new_manifest is not None:
+                manifest = new_manifest
+                logging.debug(manifest)
+                continue
+
+            # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
+            # Use direct read from gitlab as we have not checked out
+            # the repo if the branch is already up to date
+
+            gitlab_ci_yml = common.get_repository_file_raw(
+                integration["project"],
+                ".gitlab-ci.yml",
+                ref=integration["integration_branch"],
+            )
+            project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
 
-        new_srcrev = update_srcrev(
-            srcrev, project_keys["recipe"], integration["commit"]
+            new_srcrev = update_srcrev(
+                srcrev, project_keys["recipe"], integration["commit"]
+            )
+            if new_srcrev is not None:
+                srcrev = new_srcrev
+                logging.debug(srcrev)
+            else:
+                logging.debug("Project %s not found in xml or srcrev file", p)
+
+        # Write manifest
+        with open(manifest_file_abs, "w", encoding="utf8") as fp:
+            fp.write(manifest)
+        manifest_project["repo"].git.add(args.manifest_file)
+        logging.debug(manifest)
+        with open(srcrev_file_abs, "w", encoding="utf8") as fp:
+            fp.write(srcrev)
+        manifest_project["repo"].git.add(args.srcrev_file)
+        logging.debug(srcrev)
+
+        # ========================================================
+        # Squash all commits on the integration branch to one
+        # ========================================================
+        manifest_project["repo"].remotes.origin.fetch(manifest_project["master_branch"])
+        manifest_master = manifest_project["project"].branches.get(
+            manifest_project["master_branch"]
         )
-        if new_srcrev is not None:
-            srcrev = new_srcrev
-            logging.debug(srcrev)
-        else:
-            logging.debug("Project %s not found in xml or srcrev file", p)
-
-    # Write manifest
-    with open(manifest_file_abs, "w", encoding="utf8") as fp:
-        fp.write(manifest)
-    manifest_project["repo"].git.add(args.manifest_file)
-    logging.debug(manifest)
-    with open(srcrev_file_abs, "w", encoding="utf8") as fp:
-        fp.write(srcrev)
-    manifest_project["repo"].git.add(args.srcrev_file)
-    logging.debug(srcrev)
-
-    # ========================================================
-    # Squash all commits on the integration branch to one
-    # ========================================================
-    manifest_project["repo"].remotes.origin.fetch(branch)
-    manifest_master = manifest_project["project"].branches.get(branch)
-    manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
-
-    # ========================================================
-    # Now commit and push the changes to the manifest repo
-    # ========================================================
-    # Make an API request to create the gitlab.user object
-    gitlab = manifest_project["project"].manager.gitlab
-    gitlab.auth()
-    integration_commit = common.commit_and_push(
-        manifest_project["project"],
-        manifest_project["repo"],
-        manifest_project["message"],
-        gitlab.user.username,
-        gitlab.user.email,
-    )
-
-    print(
-        "Successfully create integration commit {} in {}".format(
-            integration_commit, args.manifest_project
+        manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])
+
+        # ========================================================
+        # Now commit and push the changes to the manifest repo
+        # ========================================================
+        # Make an API request to create the gitlab.user object
+        gitlab = manifest_project["project"].manager.gitlab
+        gitlab.auth()
+        integration_commit = common.commit_and_push(
+            manifest_project["project"],
+            manifest_project["repo"],
+            manifest_project["message"],
+            gitlab.user.username,
+            gitlab.user.email,
         )
-    )
 
     if not args.merge:
         sys.exit(0)
@@ -365,34 +381,36 @@ def main():
     # ============================================
     # Create merge requests for the manifest
     # ============================================
-
-    logging.debug("Create MR in %s", manifest_project["project"].name)
-    manifest_project["mr"] = create_integration_merge_request(
-        manifest_project["project"], manifest_project["branch"], source_mr
-    )
-    # =================================================
-    # Now merge it
-    # =================================================
-    # The manifest needs to be merged at last
-    mr = manifest_project["mr"]
-    logging.debug("Merge %s!%s", args.manifest_project, mr.iid)
-
-    # Wait until GitLab has checked merge status
-    common.wait_until_merge_status_is_set(manifest_project["project"], mr)
-
-    # Attempt to merge
-    merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
-
-    if not merged:
-        sys.exit(
-            "Integration MR could not be merged. You have two possibilities to fix "
-            "this:\n"
-            "  1. Checkout the MR and rebase it on the current master manually, or\n"
-            "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
-            "In either case restart this job afterwards in order to get it merged."
+    for manifest_project in manifest_projects:
+        logging.debug("Create MR in %s", manifest_project["project"].name)
+        manifest_project["mr"] = create_integration_merge_request(
+            manifest_project["project"],
+            manifest_project["integration_branch"],
+            source_mr,
         )
+        # =================================================
+        # Now merge it
+        # =================================================
+        # The manifest needs to be merged at last
+        mr = manifest_project["mr"]
+        logging.debug("Merge %s!%s", args.manifest_project, mr.iid)
 
-    print("Successfully merged")
+        # Wait until GitLab has checked merge status
+        common.wait_until_merge_status_is_set(manifest_project["project"], mr)
+
+        # Attempt to merge
+        merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
+
+        if not merged:
+            sys.exit(
+                "Integration MR could not be merged. You have two possibilities to fix "
+                "this:\n"
+                "  1. Checkout the MR and rebase it on the current master manually, or\n"
+                "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
+                "In either case restart this job afterwards in order to get it merged."
+            )
+
+        print("Successfully merged")
 
 
 if __name__ == "__main__":