CI: Test 'multi-deploy' job

.gitlab-ci.yml

@@ -18,6 +18,7 @@ default:
 
 stages:
   - analyze
+  - deploy-test
   - deploy
 
 workflow:
@@ -53,6 +54,28 @@ yamllint:
     - yamllint -c .yamllint.yml *.yml .*.yml
 
 
+# ---------------------------------------------------------------------------------------
+# Stage: deploy-test
+# ---------------------------------------------------------------------------------------
+deploy-foobar-manifest:
+  stage: deploy-test
+  # when: manual
+  allow_failure: true
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi
+    - PROJECT_ROOT="${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test"
+    - ./deploy_gitlab_ci.py
+      --gitlab-url=${CI_SERVER_URL}
+      --token=${GITBOT_TOKEN}
+      --submodule=.gitlab-ci
+      --revision=${CI_COMMIT_SHA}
+      --verbose
+      ${MERGE}
+      --manifest-project=${PROJECT_ROOT}/minimal-manifest
+      ${PROJECT_ROOT}/minimal-foo
+      ${PROJECT_ROOT}/minimal-bar
+
 # ---------------------------------------------------------------------------------------
 # Stage: deploy
 # ---------------------------------------------------------------------------------------

deploy_gitlab_ci.py

@@ -41,9 +41,9 @@ def main():
     )
     parser.add_argument(
         "--project",
+        "--manifest-project",
         help="""name of the GitLab project""",
         dest="project",
-        required=True,
     )
     parser.add_argument(
         "--submodule",
@@ -72,6 +72,12 @@ def main():
         required=False,
         default=False,
     )
+    parser.add_argument(
+        "projects",
+        help="""List of projects the change should be deployed to additionally
+                to the manifest project given as named parameter.""",
+        nargs="*",
+    )
     parser.add_argument(
         "-v",
         "--verbose",
@@ -86,6 +92,8 @@ def main():
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
     project = common.get_project(gitlab, args.project)
 
+    logging.debug("Integrate into: %s", args.project)
+
     # Update submodule
     integration_branch, _, submodule_project = update_submodule(
         project,
@@ -93,6 +101,7 @@ def main():
         args.revision,
         args.branch,
         pre_commit_hook=update_rev_in_gitlab_ci,
+        replace_exising_branch=len(args.projects) > 0,
     )
 
     logging.debug("Integration branch: %s", integration_branch)
@@ -100,6 +109,20 @@ def main():
     if not integration_branch:
         sys.exit(0)
 
+    for p in args.projects:
+        gitlab_p = common.get_project(gitlab, p)
+        logging.debug("Integrate into: %s", p)
+
+        # Update submodule
+        integration_branch, _, submodule_project = update_submodule(
+            gitlab_p,
+            args.submodule,
+            args.revision,
+            args.branch,
+            pre_commit_hook=update_rev_in_gitlab_ci,
+        )
+        logging.debug("Integration branch: %s", integration_branch)
+
     # Get source merge request
     mrs = get_merge_requests(
         submodule_project,

foobar-manifest-integration.yml

@@ -62,6 +62,7 @@ integrate:
   cache:
     policy: push
   script:
+    - printenv
     - cd ${CI_PROJECT_DIR}
     - .gitlab-ci/integrate_into_manifest.py
         --gitlab-url=${CI_SERVER_URL}
@@ -71,6 +72,7 @@ integrate:
         --project=${CI_PROJECT_PATH}
         --merge-request=${CI_MERGE_REQUEST_IID}
         --save-revision-to=manifest_revision
+        --verbose
   artifacts:
     paths:
       - manifest_revision
@@ -105,7 +107,10 @@ merge:
 build:
   stage: build
   rules:
-    - if: $CI_MERGE_REQUEST_IID
+    # execute this in MR only and not for integrate/gitlab-ci/ integrations
+    # branches. These are build after the integration has been done in all
+    # projects
+    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
   trigger:
     project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
     branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"

integrate_into_manifest.py

@@ -2,6 +2,7 @@
 import common
 
 import argparse
+import logging
 import sys
 import tempfile
 import re
@@ -42,14 +43,36 @@ def integrate_into_manifest(
         except IndexError:
             sys.exit("ERROR: branch '%s' not found" % integration_base)
 
-        # Create integration branch (delete former one if already exists)
-        integration_branch = common.integration_branch_name(
-            project.name, merge_request.source_branch
-        )
-        for ref in manifest_repo.references:
-            if integration_branch == ref.name:
-                manifest_repo.delete_head(ref)
-        manifest_repo.head.set_reference(manifest_repo.create_head(integration_branch))
+        # Special handling for the gitlab-ci integration
+        # When the branch 'merge_request.source_branch' already starts with
+        # integrate/gitlab-ci we add our new commit to this branch
+        # Otherwise (normal behaviour) a new integration branch is created
+        integration_branch = None
+        if merge_request.source_branch.startswith("integrate/gitlab-ci"):
+            logging.debug("Integration of gitlab-ci: %s", merge_request.source_branch)
+            for ref in manifest_repo.references:
+                # remove 'origin/' from the ref before compare
+                refname = ref.name.split("/", 1)[1]
+                logging.debug("Found ref: %s", refname)
+                if merge_request.source_branch == ref.name:
+                    logging.debug("Found integration for gitlab-ci")
+                    manifest_repo.head.set_reference(ref)
+                    integration_branch = merge_request.source_branch
+                    break
+
+        if integration_branch is None:
+            # Create integration branch (delete former one if already exists)
+            integration_branch = common.integration_branch_name(
+                project.name, merge_request.source_branch
+            )
+            for ref in manifest_repo.references:
+                if integration_branch == ref.name:
+                    manifest_repo.delete_head(ref)
+            manifest_repo.head.set_reference(
+                manifest_repo.create_head(integration_branch)
+            )
+
+        logging.debug("Integration branch: %s", integration_branch)
 
         # Parse manifest file
         try:
@@ -199,8 +222,16 @@ def main():
         dest="revision_file",
         required=False,
     )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
 
     args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
 
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
 

update_submodule.py

@@ -12,7 +12,12 @@ from gitlab import Gitlab
 
 
 def update_submodule(
-    project, submodule_name, submodule_revision, branch=None, pre_commit_hook=None
+    project,
+    submodule_name,
+    submodule_revision,
+    branch=None,
+    pre_commit_hook=None,
+    replace_exising_branch=False,
 ):
     """Update submodule of gitlab project to given revision
 
@@ -23,6 +28,7 @@ def update_submodule(
         branch (string): branch to update, if None, the projects default branch is used
         pre_commit_hook: Function to be called before the actual commit is done, to add additional changes.
                          Arguments passed: ( repo, submodule_project, submodule_revision)
+        replace_exising_branch: When an existing integration branch is found it is always replaced.
     Returns: tuple of:
         branch (string): Name of the newly created integration branch
         revision (string): hexsha of the new commit
@@ -116,16 +122,16 @@ def update_submodule(
         if existing_branch:
             repo.head.set_reference(existing_branch)
             submodule = common.get_submodule(repo, submodule_name)
-            if submodule.hexsha == submodule_revision:
+            if replace_exising_branch or submodule.hexsha != submodule_revision:
+                print("Replacing outdated integration branch %s" % integration_branch)
+                repo.head.set_reference(branch)
+                submodule = common.get_submodule(repo, submodule_name)
+            else:
                 print(
                     "Submodule is already at %s on branch %s"
                     % (submodule_revision, integration_branch)
                 )
                 return (integration_branch, existing_branch.commit, submodule_project)
-            else:
-                print("Replacing outdated integration branch %s" % integration_branch)
-                repo.head.set_reference(branch)
-                submodule = common.get_submodule(repo, submodule_name)
         else:
             print("Creating integration branch %s" % integration_branch)