CI: Test 'multi-deploy' job

.gitlab-ci.yml

@@ -18,6 +18,7 @@ default:
 
 stages:
   - analyze
+  - deploy-test
   - deploy
 
 workflow:
@@ -53,6 +54,28 @@ yamllint:
     - yamllint -c .yamllint.yml *.yml .*.yml
 
 
+# ---------------------------------------------------------------------------------------
+# Stage: deploy-test
+# ---------------------------------------------------------------------------------------
+deploy-foobar-manifest:
+  stage: deploy-test
+  # when: manual
+  allow_failure: true
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi
+    - PROJECT_ROOT="${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test"
+    - ./deploy_gitlab_ci.py
+      --gitlab-url=${CI_SERVER_URL}
+      --token=${GITBOT_TOKEN}
+      --submodule=.gitlab-ci
+      --revision=${CI_COMMIT_SHA}
+      --verbose
+      ${MERGE}
+      --manifest-project=${PROJECT_ROOT}/minimal-manifest
+      ${PROJECT_ROOT}/minimal-foo
+      ${PROJECT_ROOT}/minimal-bar
+
 # ---------------------------------------------------------------------------------------
 # Stage: deploy
 # ---------------------------------------------------------------------------------------

deploy_gitlab_ci.py

@@ -25,6 +25,62 @@ def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
         repo.git.add(gitlab_ci_yml)
 
 
+def deploy_into(project, submodule, revision, branch, replace_exising_branch=False):
+    """Update the submodule and include refs to the submodule in the given project.
+        Create mergerequest if needed.
+
+    Parameters:
+        project ( gitlab project): The project which's submodule should be updated
+        submodule_name (string): The name of the submodule to pull
+        submodule_revision (hex string): The sha hash of the commit to update the submodule to
+        branch (string): branch to update, if None, the projects default branch is used
+        replace_exising_branch: When an existing integration branch is found it is always replaced.
+
+    Returns: tuple of:
+        branch (string): Name of the newly created integration branch
+        merge_request (gitlab mr): Mergerequest for the integration branch
+    """
+    # Update submodule
+    integration_branch, _, submodule_project = update_submodule(
+        project,
+        submodule,
+        revision,
+        branch,
+        pre_commit_hook=update_rev_in_gitlab_ci,
+        replace_exising_branch=replace_exising_branch,
+    )
+
+    logging.debug("Integration branch: %s", integration_branch)
+
+    # If submodule is already at specified revision, return directly
+    if not integration_branch:
+        return None, submodule_project
+
+    # Get source merge request
+    mrs = get_merge_requests(
+        submodule_project,
+        # TODO should this be submodule_project's default branch?
+        target_branch="master",
+        commit=revision,
+    )
+    if not mrs:
+        sys.exit(
+            "ERROR: could not determine source merge request for commit %s" % revision
+        )
+    source_mr = mrs[0]
+
+    # Create merge request
+    mr, created = create_merge_request(
+        project, integration_branch, project.default_branch
+    )
+    if created:
+        common.crosslink_merge_requests(source_mr, mr)
+        print("Created new merge request:\n%s" % mr.web_url)
+    else:
+        print("Existing integration merge request:\n%s" % mr.web_url)
+    return integration_branch, mr
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -41,9 +97,9 @@ def main():
     )
     parser.add_argument(
         "--project",
+        "--manifest-project",
         help="""name of the GitLab project""",
         dest="project",
-        required=True,
     )
     parser.add_argument(
         "--submodule",
@@ -72,6 +128,12 @@ def main():
         required=False,
         default=False,
     )
+    parser.add_argument(
+        "projects",
+        help="""List of projects the change should be deployed to additionally
+                to the manifest project given as named parameter.""",
+        nargs="*",
+    )
     parser.add_argument(
         "-v",
         "--verbose",
@@ -86,42 +148,37 @@ def main():
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
     project = common.get_project(gitlab, args.project)
 
-    # Update submodule
-    integration_branch, _, submodule_project = update_submodule(
+    logging.debug("Integrate into: %s", args.project)
+
+    # Update submodule in this project, create MR
+    integration_branch, mr = deploy_into(
         project,
         args.submodule,
         args.revision,
         args.branch,
-        pre_commit_hook=update_rev_in_gitlab_ci,
+        replace_exising_branch=len(args.projects) > 0,
     )
+    merge_requests = []
+    merge_requests.append(mr)
 
-    logging.debug("Integration branch: %s", integration_branch)
     # If submodule is already at specified revision, exit successfully
+    # TODO: is this correct for multi deploy also?
     if not integration_branch:
         sys.exit(0)
 
-    # Get source merge request
-    mrs = get_merge_requests(
-        submodule_project,
-        target_branch="master",
-        commit=args.revision,
-    )
-    if not mrs:
-        sys.exit(
-            "ERROR: could not determine source merge request for commit %s"
-            % args.revision
+    for p in args.projects:
+        gitlab_p = common.get_project(gitlab, p)
+        logging.debug("Integrate into: %s", p)
+
+        integration_branch, mr = deploy_into(
+            gitlab_p,
+            args.submodule,
+            args.revision,
+            args.branch,
         )
-    source_mr = mrs[0]
+        merge_requests.append(mr)
 
-    # Create merge request
-    mr, created = create_merge_request(
-        project, integration_branch, project.default_branch
-    )
-    if created:
-        common.crosslink_merge_requests(source_mr, mr)
-        print("Created new merge request:\n%s" % mr.web_url)
-    else:
-        print("Existing integration merge request:\n%s" % mr.web_url)
+        logging.debug("Integration branch: %s", integration_branch)
 
     if not args.merge:
         print(
@@ -132,11 +189,12 @@ def main():
         )
         sys.exit(0)
 
+    # TODO handle all MRs here
     # Wait until GitLab has checked merge status
-    common.wait_until_merge_status_is_set(project, mr)
+    common.wait_until_merge_status_is_set(project, merge_requests[0])
 
     # Attempt to merge
-    merged = accept_merge_request(project, mr, rebase=True)
+    merged = accept_merge_request(project, merge_requests[0], rebase=True)
 
     if not merged:
         sys.exit(

foobar-manifest-integration.yml

@@ -29,7 +29,23 @@ stages:
 
 workflow:
   rules:
-    # Do not run pipelines on forked projects
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+    # Do not run pipelines for merge requests for integrate/gitlab-ci/ branches
+    # These are trigger explicitly from the integration pipeline in gitlab-ci repo
+    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    # Do not run pipelines on forked projects.
+    # The pipelines would not work anyway because of the users permissions.
+    # There are two cases catched here:
+    # 1. The project is forked into someones gitlab namespace and a MR to
+    #    include a change into this forked project is created. In this case
+    #    is the CI_PROJECT_ROOT_NAMESPACE not SECO-Northern-Europe but the
+    #    namespace the fork lives in.
+    # 2. The MR from the forked project is created to merge the change into this
+    #    the project in the SECO-Northern-Europe namespace (customer sending
+    #    change to us). Here the the IDs used below differ.
+    #
     - if: $CI_PROJECT_ROOT_NAMESPACE == "SECO-Northern-Europe"
         && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID
 
@@ -62,6 +78,7 @@ integrate:
   cache:
     policy: push
   script:
+    - printenv
     - cd ${CI_PROJECT_DIR}
     - .gitlab-ci/integrate_into_manifest.py
         --gitlab-url=${CI_SERVER_URL}
@@ -71,6 +88,7 @@ integrate:
         --project=${CI_PROJECT_PATH}
         --merge-request=${CI_MERGE_REQUEST_IID}
         --save-revision-to=manifest_revision
+        --verbose
   artifacts:
     paths:
       - manifest_revision
@@ -105,7 +123,10 @@ merge:
 build:
   stage: build
   rules:
-    - if: $CI_MERGE_REQUEST_IID
+    # execute this in MR only and not for integrate/gitlab-ci/ integrations
+    # branches. These are build after the integration has been done in all
+    # projects
+    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
   trigger:
     project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
     branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"

integrate_into_manifest.py

@@ -2,6 +2,7 @@
 import common
 
 import argparse
+import logging
 import sys
 import tempfile
 import re
@@ -42,14 +43,36 @@ def integrate_into_manifest(
         except IndexError:
             sys.exit("ERROR: branch '%s' not found" % integration_base)
 
-        # Create integration branch (delete former one if already exists)
-        integration_branch = common.integration_branch_name(
-            project.name, merge_request.source_branch
-        )
-        for ref in manifest_repo.references:
-            if integration_branch == ref.name:
-                manifest_repo.delete_head(ref)
-        manifest_repo.head.set_reference(manifest_repo.create_head(integration_branch))
+        # Special handling for the gitlab-ci integration
+        # When the branch 'merge_request.source_branch' already starts with
+        # integrate/gitlab-ci we add our new commit to this branch
+        # Otherwise (normal behaviour) a new integration branch is created
+        integration_branch = None
+        if merge_request.source_branch.startswith("integrate/gitlab-ci"):
+            logging.debug("Integration of gitlab-ci: %s", merge_request.source_branch)
+            for ref in manifest_repo.references:
+                # remove 'origin/' from the ref before compare
+                refname = ref.name.split("/", 1)[1]
+                logging.debug("Found ref: %s", refname)
+                if merge_request.source_branch == ref.name:
+                    logging.debug("Found integration for gitlab-ci")
+                    manifest_repo.head.set_reference(ref)
+                    integration_branch = merge_request.source_branch
+                    break
+
+        if integration_branch is None:
+            # Create integration branch (delete former one if already exists)
+            integration_branch = common.integration_branch_name(
+                project.name, merge_request.source_branch
+            )
+            for ref in manifest_repo.references:
+                if integration_branch == ref.name:
+                    manifest_repo.delete_head(ref)
+            manifest_repo.head.set_reference(
+                manifest_repo.create_head(integration_branch)
+            )
+
+        logging.debug("Integration branch: %s", integration_branch)
 
         # Parse manifest file
         try:
@@ -199,8 +222,16 @@ def main():
         dest="revision_file",
         required=False,
     )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
 
     args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
 
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
 

update_submodule.py

@@ -12,7 +12,12 @@ from gitlab import Gitlab
 
 
 def update_submodule(
-    project, submodule_name, submodule_revision, branch=None, pre_commit_hook=None
+    project,
+    submodule_name,
+    submodule_revision,
+    branch=None,
+    pre_commit_hook=None,
+    replace_exising_branch=False,
 ):
     """Update submodule of gitlab project to given revision
 
@@ -23,6 +28,7 @@ def update_submodule(
         branch (string): branch to update, if None, the projects default branch is used
         pre_commit_hook: Function to be called before the actual commit is done, to add additional changes.
                          Arguments passed: ( repo, submodule_project, submodule_revision)
+        replace_exising_branch: When an existing integration branch is found it is always replaced.
     Returns: tuple of:
         branch (string): Name of the newly created integration branch
         revision (string): hexsha of the new commit
@@ -116,16 +122,16 @@ def update_submodule(
         if existing_branch:
             repo.head.set_reference(existing_branch)
             submodule = common.get_submodule(repo, submodule_name)
-            if submodule.hexsha == submodule_revision:
+            if replace_exising_branch or submodule.hexsha != submodule_revision:
+                print("Replacing outdated integration branch %s" % integration_branch)
+                repo.head.set_reference(branch)
+                submodule = common.get_submodule(repo, submodule_name)
+            else:
                 print(
                     "Submodule is already at %s on branch %s"
                     % (submodule_revision, integration_branch)
                 )
                 return (integration_branch, existing_branch.commit, submodule_project)
-            else:
-                print("Replacing outdated integration branch %s" % integration_branch)
-                repo.head.set_reference(branch)
-                submodule = common.get_submodule(repo, submodule_name)
         else:
             print("Creating integration branch %s" % integration_branch)