CI: Use new deploy_gitlab_ci in adapted pipeline

.gitlab-ci.yml

@@ -19,7 +19,7 @@ default:
 stages:
   - analyze
   - integrate
-  - check
+  - build
   - merge
 
 workflow:
@@ -62,7 +62,6 @@ yamllint:
 # ---------------------------------------------------------------------------------------
 # Stage: integrate
 # ---------------------------------------------------------------------------------------
-
 .ci-test-projects:
   variables:
     PROJECT_ROOT:
@@ -104,6 +103,8 @@ yamllint:
     - if: $CI_MERGE_REQUEST_IID
       when: manual
       allow_failure: true
+  variables:
+    MERGE: ""
   script:
     - cd ${CI_PROJECT_DIR}
     - ./deploy_gitlab_ci.py
@@ -113,18 +114,8 @@ yamllint:
       --submodule=.gitlab-ci
       --revision=${CI_COMMIT_SHA}
       --verbose
+      ${MERGE}
       ${INTEGRATE_INTO}
-
-    - ./generate_job_from_template.py
-      --template=gitlab-ci-integration.jinja2
-      --image=${CI_IMAGE_PYTHON}
-      --branch="integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
-      --manifest-project=${MANIFEST_PROJECT}
-      --parent_merge_request="${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
-      --verbose
-      ${INTEGRATE_INTO}
-      > integration.yml
-    - cat integration.yml
   artifacts:
     paths:
       - integration.yml
@@ -139,47 +130,47 @@ integrate-ci-test:
     - .integrate
     - .ci-test-projects
 
-trigger-yocto:
-  stage: integrate
+# --------------------------------------------------------------------------------------
+# Stage: build
+# --------------------------------------------------------------------------------------
+build-yocto:
+  stage: build
+  needs: [integrate-yocto]
   rules:
     - if: $CI_MERGE_REQUEST_IID
       allow_failure: true
-  needs: [integrate-yocto]
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: manual
   trigger:
-    include:
-      - artifact: integration.yml
-        job: integrate-yocto
+    project: SECO-Northern-Europe/yocto/manifest
+    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
     strategy: depend
 
-trigger-ci-test:
-  stage: integrate
+build-ci-test:
+  stage: build
+  needs: [integrate-ci-test]
   rules:
     - if: $CI_MERGE_REQUEST_IID
-  needs: [integrate-ci-test]
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: manual
   trigger:
-    include:
-      - artifact: integration.yml
-        job: integrate-ci-test
+    project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
+    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
     strategy: depend
 
 # --------------------------------------------------------------------------------------
 # Stage: merge
 # --------------------------------------------------------------------------------------
 .merge:
+  extends: .integrate
   stage: merge
   rules:
     - if: $CI_COMMIT_BRANCH == "master"
       when: manual
       allow_failure: true
-  script:
-    - cd ${CI_PROJECT_DIR}
-    - ./merge_gitlab_ci.py
-      --gitlab-url=${CI_SERVER_URL}
-      --token=${GITBOT_TOKEN}
-      --manifest-project=${MANIFEST_PROJECT}
-      --submodule=.gitlab-ci
-      --revision=${CI_COMMIT_SHA}
-      ${INTEGRATE_INTO}
+  variables:
+    MERGE: --merge
 
 merge-ci-test:
   extends:
@@ -190,40 +181,3 @@ merge-yocto:
   extends:
     - .merge
     - .yocto-projects
-
-# --------------------------------------------------------------------------------------
-# Stage: check
-# --------------------------------------------------------------------------------------
-check:
-  stage: check
-  needs: [integrate-yocto, integrate-ci-test]
-  rules:
-    # Probably this job gets removed
-    - when: never
-    - if: $CI_MERGE_REQUEST_IID
-  tags:
-    - infrastructure
-  timeout: 2m
-  script:
-    - cd ${CI_PROJECT_DIR}
-    - MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-
-    - MASTER_BRANCH=dunfell
-    - ./check_if_integration_branch_is_up_to_date.py
-      --gitlab-url=${CI_SERVER_URL}
-      --token=${GITBOT_TOKEN}
-      --manifest-project=${CI_PROJECT_ROOT_NAMESPACE}/yocto/manifest
-      --integration-base=${MASTER_BRANCH}
-      --project=${CI_PROJECT_PATH}
-      --merge-request=${CI_MERGE_REQUEST_IID}
-
-    # The check is done for both manifests in one job as the retrigger
-    # looks for jobs named check, though there can only be one
-    - ./check_if_integration_branch_is_up_to_date.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --manifest-project
-          ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test/minimal-manifest
-        --integration-base=master
-        --project=${CI_PROJECT_PATH}
-        --merge-request=${CI_MERGE_REQUEST_IID}

accept_merge_request.py

@@ -47,7 +47,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
                     print("Merge error: %s" % mr.merge_error)
                 else:
                     print("Merge reported success, but MR state is '%s'" % mr.state)
-                return False
+                return False, mr.sha
 
         except GitlabMRClosedError as e:
             # See HTTP error codes for merge requests here:
@@ -73,7 +73,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
                     if pipeline_pending:
                         print("")
                     print("Merge not possible, has to be rebased manually")
-                    return False
+                    return False, mr.sha
 
             elif e.response_code == 406:
                 # Merge conflict, automatic rebase is possible
@@ -82,7 +82,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
                     pipeline_pending = False
                 print("Merge not possible, but branch can be automatically rebased")
                 if not rebase:
-                    return False
+                    return False, mr.sha
                 print("Trying to rebase...")
                 mr = common.rebase_merge_request(project, mr)
                 if mr.merge_error:
@@ -96,7 +96,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
                 print("ERROR: merge not possible: %s" % e)
                 sys.exit(critical_error)
 
-    return True
+    return True, mr.sha
 
 
 def main():

deploy_gitlab_ci.py

@@ -39,6 +39,45 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
     return {"recipe": recipe, "masterbranch": masterbranch}
 
 
+def integrate_submodule_into(
+    gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
+):
+
+    gitlab_project = common.get_project(gitlab, project_name)
+
+    (
+        project_repo,
+        integration_branch_name,
+        integration_commit,
+        message,
+    ) = update_submodule_and_include_ref(
+        gitlab_project,
+        submodule_name,
+        new_revision,
+        branch,
+        commit_and_push=commit_and_push,
+    )
+    if integration_branch_name is None:
+        return None
+    # ======================================
+    # Store the references for creating the integration
+    # commit in the manifest later
+    # ======================================
+    ret = {
+        "project": gitlab_project,
+        "repo": project_repo,
+        "branch": integration_branch_name,
+        "commit": integration_commit,
+        "message": message,
+    }
+    logging.debug(
+        "Integration branch: %s (%s)",
+        integration_branch_name,
+        integration_commit,
+    )
+    return ret
+
+
 def create_integration_merge_request(project, integration_branch_name, source_mr=None):
     # Create merge request
     # This should be optional
@@ -134,54 +173,95 @@ def main():
 
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
 
+    # =======================================================
+    # Create integration branches and commits with updates
+    # submodule in all projects
+    # =======================================================
     project_integration = {}
-    # Update submodule in this project, create MR if needed TODO
-    for p in args.projects + [args.project]:
+    # Update submodule in all 'child' project
+    for p in args.projects:
         logging.debug("Integrate into: %s", p)
 
-        gitlab_project = common.get_project(gitlab, p)
-
-        (
-            project_repo,
-            integration_branch_name,
-            integration_commit,
-            message,
-        ) = update_submodule_and_include_ref(
-            gitlab_project,
-            args.submodule,
-            args.revision,
-            args.branch,
-            # Commit in all projects except the manifest
-            commit_and_push=(p != args.project),
+        res = integrate_submodule_into(
+            gitlab, p, args.submodule, args.revision, args.branch
         )
-        if integration_branch_name is not None:
-            # ======================================
-            # Store the references for creating the integration
-            # commit in the manifest later
-            # ======================================
-            project_integration[p] = {
-                "project": gitlab_project,
-                "repo": project_repo,
-                "branch": integration_branch_name,
-                "commit": integration_commit,
-                "message": message,
-            }
-            logging.debug(
-                "Integration branch: %s (%s)",
-                integration_branch_name,
-                integration_commit,
-            )
+        if res is not None:
+            project_integration[p] = res
+
+    # Update submodule in manifest project
+    manifest_project = integrate_submodule_into(
+        gitlab,
+        args.project,
+        args.submodule,
+        args.revision,
+        args.branch,
+        commit_and_push=False,
+    )
+    if manifest_project is not None:
+        project_integration[args.project] = manifest_project
 
     # If submodule is already at specified revision in all projects, exit successfully
     if len(project_integration) == 0:
         print("No integration done, changes are already included in all projects.")
         sys.exit(0)
 
+    # =======================================================
+    # Create and merge merge_requests if needed
+    # =======================================================
+    if args.merge:
+        # Get source merge request ( the one in the gitlab-ci repo)
+        submodule_project_path, _ = get_submodule_project_path_and_revision(
+            manifest_project["project"], args.submodule, args.branch
+        )
+        submodule_project = common.get_project(gitlab, submodule_project_path)
+        mrs = get_merge_requests(
+            submodule_project,
+            # TODO should this be submodule_project's default branch?
+            target_branch="master",
+            commit=args.revision,
+        )
+        if not mrs:
+            sys.exit(
+                "ERROR: could not determine source merge request for commit %s"
+                % args.revision
+            )
+        source_mr = mrs[0]
+
+        for p in args.projects:
+            integration = project_integration[p]
+            logging.debug("Create MR in %s", integration["project"].name)
+            mr = create_integration_merge_request(
+                integration["project"], integration["branch"], source_mr
+            )
+            integration["mr"] = mr
+            # Now merge
+            logging.debug("Merge %s!%s", p, mr.iid)
+
+            # Wait until GitLab has checked merge status
+            common.wait_until_merge_status_is_set(integration["project"], mr)
+
+            # Attempt to merge
+            merged, integration_commit = accept_merge_request(
+                integration["project"], mr, rebase=True
+            )
+            # if this has rebased the integration commit needs to be adapted:
+            project_integration[p]["commit"] = integration_commit
+
+            if not merged:
+                sys.exit(
+                    "Integration MR could not be merged. You have two possibilities to fix "
+                    "this:\n"
+                    "  1. Checkout the MR and rebase it on the current master manually, or\n"
+                    "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
+                    "In either case restart this job afterwards in order to get it merged."
+                )
+
+        print("Successfully merged")
+
     # =======================================================
     # Now create the integration commit in the manifest
     # for all subprojects at once
     # =======================================================
-    manifest_project = project_integration[args.project]
     manifest_file_abs = os.path.join(
         manifest_project["repo"].working_tree_dir, args.manifest_file
     )
@@ -275,58 +355,34 @@ def main():
         sys.exit(0)
 
     # ============================================
-    # Create merge requests if needed
+    # Create merge requests for the manifest
     # ============================================
-    # Get source merge request ( the one in the gitlab-ci repo)
-    submodule_project_path, _ = get_submodule_project_path_and_revision(
-        manifest_project["project"], args.submodule, args.branch
-    )
-    submodule_project = common.get_project(gitlab, submodule_project_path)
-    mrs = get_merge_requests(
-        submodule_project,
-        # TODO should this be submodule_project's default branch?
-        target_branch="master",
-        commit=args.revision,
-    )
-    if not mrs:
-        sys.exit(
-            "ERROR: could not determine source merge request for commit %s"
-            % args.revision
-        )
-    source_mr = mrs[0]
-
-    for p in args.projects + [args.project]:
-        integration = project_integration[p]
-        logging.debug("Create MR in %s", integration["project"].name)
-        integration["mr"] = create_integration_merge_request(
-            integration["project"], integration["branch"], source_mr
-        )
 
+    logging.debug("Create MR in %s", manifest_project["project"].name)
+    manifest_project["mr"] = create_integration_merge_request(
+        manifest_project["project"], manifest_project["branch"], source_mr
+    )
     # =================================================
-    # Now merge them all
+    # Now merge it
     # =================================================
     # The manifest needs to be merged at last
-    for p in args.projects + [args.project]:
-        integration = project_integration[p]
-        mr = integration["mr"]
-        logging.debug("Merge %s!%s", p, mr.iid)
+    mr = manifest_project["mr"]
+    logging.debug("Merge %s!%s", args.project, mr.iid)
 
-        # Wait until GitLab has checked merge status
-        common.wait_until_merge_status_is_set(integration["project"], mr)
+    # Wait until GitLab has checked merge status
+    common.wait_until_merge_status_is_set(manifest_project["project"], mr)
 
-        # Attempt to merge
-        # TODO if this has rebased the integration commit needs to be adapted
-        #      change the sequence
-        merged = accept_merge_request(integration["project"], mr, rebase=True)
+    # Attempt to merge
+    merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
 
-        if not merged:
-            sys.exit(
-                "Integration MR could not be merged. You have two possibilities to fix "
-                "this:\n"
-                "  1. Checkout the MR and rebase it on the current master manually, or\n"
-                "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
-                "In either case restart this job afterwards in order to get it merged."
-            )
+    if not merged:
+        sys.exit(
+            "Integration MR could not be merged. You have two possibilities to fix "
+            "this:\n"
+            "  1. Checkout the MR and rebase it on the current master manually, or\n"
+            "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
+            "In either case restart this job afterwards in order to get it merged."
+        )
 
     print("Successfully merged")
 

merge_gitlab_ci.py

-#!/usr/bin/env python3
-import common
-
-import argparse
-import logging
-import sys
-from gitlab import Gitlab, GitlabGetError
-from accept_merge_request import accept_merge_request
-import update_submodule
-from gitlab.v4.objects import Project
-
-
-def find_integration_merge_request(
-    project: Project, submodule, revision, target_branch
-):
-
-    (
-        submodule_path,
-        submodule_revision,
-    ) = update_submodule.get_submodule_project_path_and_revision(
-        project, submodule, target_branch
-    )
-    logging.debug("Module: %s, Revision: %s", submodule_path, submodule_revision)
-
-    # Get submodule project
-    gitlab = project.manager.gitlab
-    submodule_project = common.get_project(gitlab, submodule_path)
-
-    # Get the integration branch name
-    integration_branch_suffix = (
-        update_submodule.get_submodule_integration_branch_suffix(
-            submodule_project, revision
-        )
-    )
-    integration_branch_name = common.integration_branch_name(
-        submodule_project.name, integration_branch_suffix
-    )
-    logging.debug(integration_branch_name)
-
-    # Get the merge request for the branch TODO catch exception
-    try:
-        integration_branch = project.branches.get(
-            integration_branch_name, retry_transient_errors=True
-        )
-    except GitlabGetError:
-        print("ERROR: Failed to find integration branch for {}".format(submodule))
-        return None
-
-    logging.debug(integration_branch)
-    commit = project.commits.get(
-        integration_branch.commit["id"], retry_transient_errors=True
-    )
-    for mr in commit.merge_requests(retry_transient_errors=True):
-        if mr["target_branch"] == target_branch:
-            integration_mr = mr
-            break
-    if integration_mr is None:
-        print(
-            "ERROR: Failed to find integration merge request for %s",
-            integration_branch_name,
-        )
-        return None
-    logging.debug(integration_mr["iid"])
-    mr = project.mergerequests.get(integration_mr["iid"], retry_transient_errors=True)
-
-    return mr
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--gitlab-url",
-        help="""URL to the GitLab instance""",
-        dest="gitlab_url",
-        required=True,
-    )
-    parser.add_argument(
-        "--token",
-        help="""GitLab REST API private access token""",
-        dest="token",
-        required=True,
-    )
-    parser.add_argument(
-        "--project",
-        "--manifest-project",
-        help="""name of the GitLab project""",
-        dest="project",
-    )
-    parser.add_argument(
-        "--submodule",
-        help="""submodule to update""",
-        dest="submodule",
-        required=True,
-    )
-    parser.add_argument(
-        "--revision",
-        help="""new revision for submodule""",
-        dest="revision",
-        required=True,
-    )
-    parser.add_argument(
-        "--branch",
-        help="""project branch (if not default branch)""",
-        dest="branch",
-        required=False,
-        default=None,
-    )
-    parser.add_argument(
-        "--merge",
-        help="""if set, perform merge after integration""",
-        dest="merge",
-        action="store_true",
-        required=False,
-        default=False,
-    )
-    parser.add_argument(
-        "projects",
-        help="""List of projects the change should be deployed to additionally
-                to the manifest project given as named parameter.""",
-        nargs="*",
-    )
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        action="store_true",
-        help="""Increase verbosity.""",
-    )
-
-    args, _ = parser.parse_known_args()
-    if args.verbose:
-        logging.basicConfig(level=logging.DEBUG)
-
-    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
-
-    # Start with the manifest here, so the subproject
-    # can see that they are already integrated
-    for p in args.projects + [args.project]:
-        project = common.get_project(gitlab, p)
-        branch = args.branch
-
-        if branch is None:
-            branch = project.default_branch
-
-        print(
-            "Try to merge {}({}) into {} ({})".format(
-                args.submodule, args.revision, project.name, branch
-            )
-        )
-
-        mr = find_integration_merge_request(
-            project, args.submodule, args.revision, branch
-        )
-        if mr is None:
-            sys.exit("ERROR: Failed to find the integration MR.")
-            # TODO if this ever happens, why ever, we could call
-            # deploy_gitlab_ci again to create a new integration
-            # commit or make sure the change is already integrated.
-
-        print("Merge {}!{}: {}".format(project.name, mr.iid, mr.title))
-        # Wait until GitLab has checked merge status
-        common.wait_until_merge_status_is_set(project, mr)
-
-        # Attempt to merge
-        merged = accept_merge_request(
-            project,
-            mr,
-            # Only the file manifest commit may be rebased
-            # other rebase would require new integration commits
-            # TODO implement rebase
-            rebase=(p == args.project),
-        )
-
-        if not merged:
-            print(
-                "Integration MR could not be merged."
-                "To fix this:\n"
-                "  1. Checkout the MR {}!{} and merge it manually.\n"
-                "  2. Manually merge the follow up MRs in the following project:".format(
-                    project.name, mr.iid
-                )
-            )
-            found = False
-            for p2 in [args.project] + args.projects:
-                if p2 == p:
-                    found = True
-                    continue
-                if not found:
-                    continue
-                print("     {}".format(p2))
-            sys.exit()
-
-    print("Successfully merged")
-
-    exit()
-
-
-if __name__ == "__main__":
-    main()

merge_into_manifest.py

@@ -78,7 +78,7 @@ def merge_into_manifest(
         common.wait_until_merge_status_is_set(manifest_project, mr)
 
         # Attempt to merge
-        merged = accept_merge_request(manifest_project, mr)
+        merged, _ = accept_merge_request(manifest_project, mr)
 
         if not merged:
             # Merge failed, reintegrate the source merge request into the manifest.