Merge branch 'manifest_branch_fix' of...

Merge branch 'manifest_branch_fix' of git.seco.com:yocto_ng/infrastructure/gitlab-ci into manifest_branch_fix

.gitlab-ci.yml

@@ -9,6 +9,7 @@ variables:
   CI_IMAGES_REV: latest
   CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/infrastructure/python3.9:${CI_IMAGES_REV}"
   CI_IMAGE_YOCTO: "secodocker/edgehog-builder:${CI_IMAGES_REV}"
+  PROJECT_INTEGRATION_GROUPS: "arm/nxp/imx/develop/yocto/5.x" #arm/rockchip
 
 image: "${CI_IMAGE_PYTHON}"
 
@@ -75,14 +76,26 @@ executable:
     MERGE: ""
   script:
     - cd ${CI_PROJECT_DIR}
-    - echo 
-    - echo ${CI_SERVER_URL}
     - echo ${GITBOT_TOKEN}
     - echo ${MANIFEST_PROJECT}
     - echo ${MANIFEST_BRANCH}
     - echo ${CI_COMMIT_SHA}
     - echo ${PROJECT_GROUP}
     - echo ${MERGE}
+    - |
+      for PROJECT_INTEGRATION in ${PROJECT_INTEGRATION_GROUPS}; do
+        echo ${PROJECT_INTEGRATION}
+        scripts/deploy_gitlab_ci_projects.py \
+        --gitlab-url=${CI_SERVER_URL} \
+        --token=${GITBOT_TOKEN} \
+        --manifest-project=${MANIFEST_PROJECT} \
+        --manifest-branch=${MANIFEST_BRANCH} \
+        --submodule=.gitlab-ci \
+        --revision=${CI_COMMIT_SHA} \
+        --group="${PROJECT_INTEGRATION}" \
+        --verbose \
+        ${MERGE}
+      done
     - scripts/deploy_gitlab_ci.py
       --gitlab-url=${CI_SERVER_URL}
       --token=${GITBOT_TOKEN}
@@ -116,7 +129,7 @@ executable:
 #  extends: .integrate-ci-test
 #  variables:
 #    MANIFEST_BRANCH: primary
-#
+
 #integrate-ci-test:secondary:
 #  extends: .integrate-ci-test
 #  variables:
@@ -166,7 +179,7 @@ integrate-yocto:kirkstone:
 #  needs: ["integrate-ci-test:primary"]
 #  variables:
 #    MANIFEST_BRANCH: primary
-#
+
 #build-ci-test:secondary:
 #  extends: .build-ci-test
 #  needs: ["integrate-ci-test:secondary"]

layers-integration.yml

+---
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+
+include:
+  - local: common.yml
+
+stages:
+  - infrastructure
+  - integrate-into-layer
+  - create-merge
+  - check
+  - accept-merge
+
+variables:
+  # The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
+  # to match the repository and the bitbake recipe name.
+  # We set it here to none, as every project needing it
+  # has to specify it in its own gitlab-ci.yml file.
+  # The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
+  # used for projects referenced in the manifest file.
+  # FIXME: This is only necessary due to the following GitLab limitation:
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/209904
+  # As soon as this gets fixed upstream, the hard-coded branch name should be removed.
+  MANIFEST_PROJECT: yocto_ng/seco-manifest
+  MASTER_BRANCH_LAYER: kirkstone/develop
+  BB_RECIPE_NAME: none
+  SRCREV_FILE: conf/SRCREV.conf
+
+  DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/"
+  GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
+  GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
+  TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
+ 
+# --------------------------------------------------------------------------------------
+# Stage: infrastructure
+# --------------------------------------------------------------------------------------
+integrate-into-layer:
+  extends: .infrastructure
+  rules:
+    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+    # The integration is done from the pipeline in gitlab-ci already
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    # We have to make sure that the pipeline runs for the current manifest
+    # master at the time a merge request is created. Otherwise we cannot
+    # guarantee a green master after merging.
+    - if: $CI_MERGE_REQUEST_IID
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+  cache:
+    policy: push
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
+    - .gitlab-ci/scripts/integrate_into_layer.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --layer-project=${LAYER_PROJECT_PATH}
+        --layer-branch=${MASTER_BRANCH_LAYER}
+        --project=${CI_PROJECT_PATH}
+        --srcrev-file=${SRCREV_FILE}
+        --merge-request=${MERGE_REQUEST}
+        --save-revision-to=srcrev_revision
+        --recipe-name=${BB_RECIPE_NAME}
+        --verbose
+  artifacts:
+    paths:
+      - srcrev_revision
+      - integration_branch_file
+
+# --------------------------------------------------------------------------------------
+# Stage: create-merge-request
+# --------------------------------------------------------------------------------------
+create-merge-request:
+  stage: create-merge
+  extends: .infrastructure
+  rules:
+    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+    # The integration is done from the pipeline in gitlab-ci already
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    # We have to make sure that the pipeline runs for the current manifest
+    # master at the time a merge request is created. Otherwise we cannot
+    # guarantee a green master after merging.
+    - if: $CI_MERGE_REQUEST_IID
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+  cache:
+    policy: push
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - INTEGRATION_BRANCH=$(cat integration_branch_file)
+    - .gitlab-ci/scripts/create_merge_request.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --project=${LAYER_PROJECT_PATH}
+        --source-branch=${INTEGRATION_BRANCH}
+        --target-branch=${MASTER_BRANCH_LAYER}
+  needs: 
+   - job: integrate-into-layer
+     artifacts: true
+
+#yamllint:
+#  extends: .yamllint
+ 
+# --------------------------------------------------------------------------------------
+# Stage: check
+# --------------------------------------------------------------------------------------
+check:
+  extends: .infrastructure
+  stage: check
+  rules:
+    # Do not run check if the "skip build" label is set on the merge request
+    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
+      when: never
+    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+    # The integration is done from the pipeline in gitlab-ci already
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    - if: $CI_MERGE_REQUEST_IID
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+  needs: ["integrate-into-layer"]
+  allow_failure: true
+  script:
+    - cd ${CI_PROJECT_DIR}
+    # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
+    # but CI_OPEN_MERGE_REQUESTS. We use  the first of this comma separated list
+    # in this case
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
+    # The 'parent_merge_request' is passed from the trigger
+    # in case this check job is part of a gitlab-ci integration
+    # pipeline. It is only used to display the correct MR to run again
+    # in a failed check
+    - if [ -n "${parent_merge_request}" ];then
+        PARENT_MR="--parent-merge-request=${parent_merge_request}";
+      fi
+    - while read -r integration; do
+        SOURCE_BRANCH=$(echo $integration | cut -d':' -f1);
+        TARGET_PROJECT=${LAYER_PROJECT_PATH};
+        TARGET_BRANCH=$(echo $integration | cut -d':' -f3);
+        if [[ "$SOURCE_BRANCH" == "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]]; then
+          .gitlab-ci/scripts/check_if_layer_branch_is_up_to_date.py
+            --gitlab-url=${CI_SERVER_URL}
+            --token=${GITBOT_TOKEN}
+            --target-project=${TARGET_PROJECT}
+            --target-branch=${TARGET_BRANCH}
+            --source-project=${CI_PROJECT_PATH}
+            --recipe-name=${BB_RECIPE_NAME}
+            --merge-request=${CI_MERGE_REQUEST_IID}
+            ;
+        fi;
+      done <<< "$INTEGRATION"
+
+merge-into-layer:
+  extends: .infrastructure
+  stage: accept-merge
+  timeout: 4h
+  rules:
+     # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+     # The integration is done from the pipeline in gitlab-ci already
+     - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+       when: never
+     - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
+       when: always
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - echo ${CI_COMMIT_REF_NAME}
+    - .gitlab-ci/scripts/accept_layer_merge_request.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --project=${CI_PROJECT_PATH}
+        --layer-project=${LAYER_PROJECT_PATH}
+        --target-branch=${MASTER_BRANCH}
+        --layer-target-branch=${MASTER_BRANCH_LAYER}
+        --recipe-name=${BB_RECIPE_NAME}
+        --rebase
\ No newline at end of file

manifest-integration.yml

@@ -24,6 +24,7 @@ variables:
   # https://gitlab.com/gitlab-org/gitlab/-/issues/209904
   # As soon as this gets fixed upstream, the hard-coded branch name should be removed.
   MANIFEST_PROJECT: seco-manifest
+  MANIFEST_GROUP: yocto_ng
   MASTER_BRANCH_MANIFEST: kirkstone/develop
   BB_RECIPE_NAME: none
 
@@ -61,7 +62,7 @@ integrate:
     - .gitlab-ci/scripts/integrate_into_manifest.py
         --gitlab-url=${CI_SERVER_URL}
         --token=${GITBOT_TOKEN}
-        --manifest-project=${MANIFEST_PROJECT}
+        --manifest-project=${MANIFEST_GROUP}/${MANIFEST_PROJECT}
         --manifest-file=${MANIFEST_FILE}
         --manifest-branch=${MASTER_BRANCH_MANIFEST}
         --project=${CI_PROJECT_PATH}
@@ -73,7 +74,6 @@ integrate:
     paths:
       - manifest_revision
 
-
 #yamllint:
 #  extends: .yamllint
 
@@ -92,16 +92,25 @@ merge:
        when: always
   script:
     - cd ${CI_PROJECT_DIR}
+    - echo ${CI_SERVER_URL}
+    - echo ${GITBOT_TOKEN}
+    - echo ${MANIFEST_PROJECT}
+    - echo ${MASTER_BRANCH_MANIFEST}
+    - echo ${CI_PROJECT_PATH}
+    - echo ${MASTER_BRANCH}
+    - echo ${CI_COMMIT_SOURCE}
+    - echo ${BB_RECIPE_NAME}
     - .gitlab-ci/scripts/merge_into_manifest.py
         --gitlab-url=${CI_SERVER_URL}
         --token=${GITBOT_TOKEN}
-        --manifest-project=${MANIFEST_PROJECT}
+        --manifest-project=${MANIFEST_GROUP}/${MANIFEST_PROJECT}
         --manifest-branch=${MASTER_BRANCH_MANIFEST}
         --project=${CI_PROJECT_PATH}
         --project-branch=${MASTER_BRANCH}
         --commit=${CI_COMMIT_SOURCE}
         --save-revision-to=manifest_revision
         --recipe-name=${BB_RECIPE_NAME}
+        --verbose
   artifacts:
     paths:
       - manifest_revision

manifest-pipeline-yocto.yml

@@ -191,7 +191,10 @@ retrigger:
          if [ ${CI_PIPELINE_SOURCE} == "pipeline" ]; then
             echo "This is the manifest branch used for the build ${CI_COMMIT_REF_NAME}"
             BUILD_BRANCH="${CI_COMMIT_REF_NAME}"
+<<<<<<< HEAD
          fi
+=======
+>>>>>>> c0c25b3b80578877a4c95177a8158f4c2adf5245
       - |
         su secous -c "
         repo init -u ${CI_REPOSITORY_URL} -b ${BUILD_BRANCH}; 

scripts/accept_layer_merge_request.py

+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import time
+from gitlab import (
+    Gitlab,
+    GitlabGetError,
+    GitlabMRClosedError,
+)
+from get_merge_requests import get_merge_requests
+
+critical_error = (
+    "This is a critical error! Please make sure to:\n"
+    " 1. merge the above-mentioned merge request by hand as soon as possible\n"
+    "    (this is important, otherwise following merge requests will get stuck, too)\n"
+    " 2. examine why this has happened and fix it in the CI pipeline"
+)
+
+
+def get_source_integration_requests(
+    project, state=None, target_branch=None, commit=None
+):
+    """Get merge request by target branch and optionally commit sha"""
+    merge_requests = []
+    try:
+        all_merge_requests = project.mergerequests.list(
+            target_branch=target_branch,
+            state=state if state else "all",
+            all=True,
+            retry_transient_errors=True,
+            order_by='updated_at'
+        )
+    except GitlabGetError as e:
+        sys.exit(
+            "ERROR: could not list merge requests for project '%s': %s"
+            % (project.name, e)
+        )
+    if commit:
+        for mr in all_merge_requests:
+            if mr.sha == commit or mr.squash_commit_sha == commit:
+                merge_requests.append(mr)
+    elif all_merge_requests:
+        merge_requests = all_merge_requests
+
+    # Get complete objects
+    full_merge_requests = []
+    for mr in merge_requests:
+        mr = project.mergerequests.get(mr.iid, retry_transient_errors=True)
+        full_merge_requests.append(mr)
+
+    return full_merge_requests[0].source_branch
+
+def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=True):
+    """Attempt to merge a merge request, rebase if necessary"""
+    merged = False
+    pipeline_pending = False
+
+    while not merged:
+        # Update merge request before trying to merge it in order to get the latest
+        # pipeline status
+        try:
+            updated_mr = project.mergerequests.get(
+                id=mr.iid, retry_transient_errors=True
+            )
+            mr = updated_mr
+        except GitlabGetError as e:
+            print("WARNING: Could not update merge request object: %s" % e)
+
+        # Try to merge the merge request
+        try:
+            mr.merge(should_remove_source_branch=should_remove_source_branch)
+            if pipeline_pending:
+                print("")
+            if mr.state == "merged":
+                merged = True
+            else:
+                if mr.merge_error:
+                    print("Merge error: %s" % mr.merge_error)
+                else:
+                    print("Merge reported success, but MR state is '%s'" % mr.state)
+                return False, mr.sha
+
+        except GitlabMRClosedError as e:
+            # See HTTP error codes for merge requests here:
+            # https://docs.gitlab.com/ce/api/merge_requests.html#accept-mr
+            logging.debug("Error from gitlab: %d", e.response_code)
+
+            if e.response_code == 405:
+                # Not allowed (draft, closed, pipeline pending or failed)
+                # Contrary to the documentation, this response is also issued when the
+                # merge failed due to a merge conflict. See GitLab issue:
+                # https://gitlab.com/gitlab-org/gitlab/-/issues/364102
+
+                if mr.has_conflicts:
+                    # Merge conflict, automatic rebase not possible
+                    if pipeline_pending:
+                        print("")
+                    print("Merge not possible, has to be rebased manually")
+                    return False, mr.sha
+
+                # If pipeline is running, wait for completion
+                if not mr.head_pipeline:
+                    # No pipeline created yet
+                    print("No pipeline created yet")
+                    time.sleep(1)
+                elif mr.head_pipeline["status"] in common.pending_states:
+                    # Pipeline pending
+                    if not pipeline_pending:
+                        print("Waiting for pending pipeline", end="", flush=True)
+                        pipeline_pending = True
+                    print(".", end="", flush=True)
+                    time.sleep(1)
+                else:
+                    # Merge failed due to some other reason
+                    if pipeline_pending:
+                        print("")
+                    print("Merge not possible for unkown reason")
+                    return False, mr.sha
+
+            elif e.response_code == 406:
+                # Merge conflict, automatic rebase is possible
+                if pipeline_pending:
+                    print("")
+                    pipeline_pending = False
+                print("Merge not possible, but branch can be automatically rebased")
+                if not rebase:
+                    return False, mr.sha
+                print("Trying to rebase...")
+                mr = common.rebase_merge_request(project, mr)
+                if mr.merge_error:
+                    print("ERROR: rebase not possible\n'%s'" % mr.merge_error)
+                    sys.exit(critical_error)
+                print("Sucessfully rebased")
+
+            else:
+                if pipeline_pending:
+                    print("")
+                print("ERROR: merge not possible: %s" % e)
+                sys.exit(critical_error)
+
+    return True, mr.sha
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        required=True,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--project",
+        help="""name of the GitLab project""",
+        dest="project",
+        required=True,
+    )
+    parser.add_argument(
+        "--layer-project",
+        help="""name of the GitLab layer project""",
+        dest="layer_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--target-branch",
+        help="""target branch of the merge request""",
+        dest="target_branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--layer-target-branch",
+        help="""target branch of the layer merge request""",
+        dest="layer_target_branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--recipe-name",
+        help="""recipe name of the merge request""",
+        dest="recipe_name",
+        required=True,
+    )
+    parser.add_argument(
+        "--rebase",
+        help="""attempt to automatically rebase merge request if necessary""",
+        dest="rebase",
+        action="store_true",
+        required=False,
+    )
+
+    args, _ = parser.parse_known_args()
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+    project = common.get_project(gitlab, args.project)
+    layer_project = common.get_project(gitlab, args.layer_project)
+
+    #Retrieving the source branch from the last MR merged in the project
+    try:
+        integration_branch_name = get_source_integration_requests(
+            project,
+            target_branch=args.target_branch,
+            state="merged",
+        )
+        print("This is the source branch of the latest MR merged: ", integration_branch_name)
+        
+        #If the last merged branch is the gitlab-ci integration do nothing
+        if "integrate/gitlab-ci" in integration_branch_name:
+            print("The job was triggered by a merge of the gitlab-ci projects into the master branch, doing nothing!")
+            return ""
+
+    except GitlabGetError as e:
+        sys.exit("Could not get integration branch name for latest project MR: %s" % e)
+    
+    integration_branch_name = args.recipe_name + "/" + integration_branch_name
+
+    try:
+        merge_request = get_merge_requests(
+            layer_project,
+            source_branch=integration_branch_name,
+            target_branch=args.layer_target_branch,
+            state="opened",
+        )
+    except GitlabGetError as e:
+        sys.exit("Could not get merge request: %s" % e)
+
+    print("This job is going to merge MR %s in project %s", merge_request[0], layer_project)
+    if accept_merge_request(layer_project, merge_request[0], rebase=args.rebase):
+        print("Successfully merged")
+    else:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/check_if_layer_branch_is_up_to_date.py

+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+import logging
+from gitlab import Gitlab, GitlabGetError
+from gitlab.v4.objects import Project
+
+
+def check_if_layer_branch_is_up_to_date(
+    target_project: Project,
+    target_branch_name: str,
+    integration_branch_name: str,
+):
+    try:
+        integration_branch = target_project.branches.get(
+            integration_branch_name, retry_transient_errors=True
+        )
+    except GitlabGetError:
+        sys.exit(
+            "ERROR: could not find integration branch {} in {}.".format(
+                integration_branch_name, target_project.name
+            )
+        )
+
+    try:
+        target_branch = target_project.branches.get(
+            target_branch_name, retry_transient_errors=True
+        )
+    except GitlabGetError:
+        sys.exit(
+            "ERROR: could not find target branch {} in {}.".format(
+                target_branch_name, target_project.name
+            )
+        )
+
+    # Loop over the commits until the integration_branch head id is found
+    return common.is_commit_parent_of_project_commit(
+        target_project,
+        integration_branch.commit["id"],
+        target_branch.commit["id"],
+        limit=10,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        required=True,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--target-project",
+        help="""name of the target project""",
+        dest="target_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--target-branch",
+        help="""target branch to integrate into""",
+        dest="target_branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--source-project",
+        help="""name of the source project""",
+        dest="source_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--recipe-name",
+        help="""name of the BB recipe (for integration branch name)""",
+        dest="recipe_name",
+        required=True,
+    )
+    parser.add_argument(
+        "--merge-request",
+        help="""source project merge request IID containing the changes to be integrated""",
+        dest="merge_request",
+        required=True,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
+
+    args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+    logging.debug(args)
+    target_project = common.get_project(gitlab, args.target_project)
+    source_project = common.get_project(gitlab, args.source_project)
+    merge_request = common.get_merge_request(source_project, args.merge_request)
+    if merge_request is None:
+        sys.exit(
+            "ERROR: could not get %s  %s" % (source_project.name, args.merge_request)
+        )
+
+    integration_branch_name = args.recipe_name + "/" + merge_request.source_branch
+
+    if check_if_layer_branch_is_up_to_date(
+        target_project=target_project,
+        target_branch_name=args.target_branch,
+        integration_branch_name=integration_branch_name,
+    ):
+        print(
+            "Integration branch {} in {} is up to date.".format(
+                integration_branch_name, target_project.name
+            )
+        )
+    else:
+        sys.exit(
+            "Integration branch {} in {} is not up to date.\n"
+            "Please re-run the MR pipeline:\n"
+            "  1. Open the MR pipelines page:\n"
+            "     {}\n"
+            "  2. Click 'Run Pipeline'".format(
+                integration_branch_name,
+                target_project.name,
+                merge_request.web_url + "/pipelines",
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

scripts/deploy_gitlab_ci_projects.py

+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import os
+from gitlab import Gitlab
+from gitlab.v4.objects import Project, MergeRequest
+
+from accept_merge_request import accept_merge_request
+from create_merge_request import create_merge_request
+from get_integration_sources import get_integration_sources
+from get_merge_requests import get_merge_requests
+from update_submodule import update_submodule_and_include_ref
+from integrate_into_manifest import update_manifest, update_srcrev
+
+from ruamel.yaml import YAML
+
+
+def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
+
+    # Read values from existing file
+    yaml = YAML()
+    data = yaml.load(gitlab_ci_yml)
+    logging.debug("Yaml: %s", data)
+
+    try:
+        recipe = data["variables"]["BB_RECIPE_NAME"]
+        logging.debug("Recipe %s", recipe)
+    except KeyError:
+        recipe = None
+    return {"recipe": recipe}
+
+
+def integrate_submodule_into(
+    gitlab,
+    project_name,
+    submodule_name,
+    new_revision,
+    branch,
+    commit_and_push=True,
+    force_clone=False,
+):
+
+    gitlab_project = common.get_project(gitlab, project_name)
+
+    (
+        project_repo,
+        project_dir,
+        integration_branch_name,
+        integration_commit,
+        message,
+    ) = update_submodule_and_include_ref(
+        gitlab_project,
+        submodule_name,
+        new_revision,
+        branch,
+        commit_and_push=commit_and_push,
+        force_clone=force_clone,
+    )
+    # ======================================
+    # Store the references for creating the integration
+    # commit in the manifest later
+    # ======================================
+    ret = {
+        "project": gitlab_project,
+        "repo": project_repo,
+        "dir": project_dir,
+        "integration_branch": integration_branch_name,
+        "master_branch": branch,
+        "commit": integration_commit,
+        "message": message,
+    }
+    logging.debug(
+        "Integration branch: %s (%s)",
+        integration_branch_name,
+        integration_commit,
+    )
+    return ret
+
+
+def create_integration_merge_request(
+    project: Project,
+    integration_branch: str,
+    target_branch: str,
+    source_mr: MergeRequest = None,
+) -> MergeRequest:
+    # Create merge request
+    # This should be optional
+    mr, created = create_merge_request(project, integration_branch, target_branch)
+    if created:
+        if source_mr is not None:
+            common.crosslink_merge_requests(source_mr, mr)
+        print("Created new merge request:\n%s" % mr.web_url)
+    else:
+        print("Existing integration merge request:\n%s" % mr.web_url)
+    return mr
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        required=True,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--manifest-project",
+        help="""name of the manifest project""",
+        dest="manifest_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--manifest-branch",
+        help="""manifest branch to integrate changes into (can be a comma-separated list)""",
+        dest="manifest_branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--submodule",
+        help="""submodule to update""",
+        dest="submodule",
+        required=True,
+    )
+    parser.add_argument(
+        "--revision",
+        help="""new revision for submodule""",
+        dest="revision",
+        required=True,
+    )
+    parser.add_argument(
+        "--merge",
+        help="""if set, perform merge after integration""",
+        dest="merge",
+        action="store_true",
+        required=False,
+        default=False,
+    )
+    parser.add_argument(
+        "--project",
+        help="""gitlab-ci project path or id""",
+        dest="project",
+        default=os.environ.get("CI_PROJECT_PATH"),
+        required=False,
+    )
+    parser.add_argument(
+        "--branch",
+        help="""gitlab-ci branch that we're merging into""",
+        dest="branch",
+        default="master",
+        required=False,
+    )
+    parser.add_argument(
+        "--manifest-file",
+        help="""manifest file name (default: 'default.xml')""",
+        dest="manifest_file",
+        default=common.manifest_file,
+        required=False,
+    )
+    parser.add_argument(
+        "--srcrev-file",
+        help="""source revision file name (default: 'SRCREV.conf')""",
+        dest="srcrev_file",
+        default=common.srcrev_file,
+        required=False,
+    )
+    parser.add_argument(
+        "--group",
+        help="""group path or id to limit search scope to""",
+        dest="group",
+        required=True,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
+
+    args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(
+            level=logging.DEBUG,
+            format="%(asctime)s %(levelname)-8s %(message)s",
+            datefmt="%H:%M:%S",
+        )
+
+    manifest_branches = args.manifest_branch.split(",")
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+    group = gitlab.groups.get(args.group)
+
+    # =======================================================
+    # Create integration branches and commits with updates
+    # submodule in all projects
+    # =======================================================
+    integration_sources = {}
+    all_integration_sources = []
+    for manifest_branch in manifest_branches:
+        print(
+            "Searching for projects in %s that are configured for automatic integration into %s:%s"
+            % (args.group, args.manifest_project, manifest_branch)
+        )
+        integration_sources[manifest_branch] = get_integration_sources(
+            args.manifest_project, manifest_branch, group
+        )
+        for s in integration_sources[manifest_branch]:
+            if s not in all_integration_sources:
+                all_integration_sources.append(s)
+
+    # Update submodule in all integration sources
+    project_integrations = []
+    for s in all_integration_sources:
+        print("Create integration commit in %s:%s" % (s["project"], s["branch"]))
+
+        integration = integrate_submodule_into(
+            gitlab, s["project"], args.submodule, args.revision, s["branch"]
+        )
+        # Store in the list if commit is set (meaning there was an update or
+        #   an exising integration branch)
+        if integration["commit"] is not None:
+            project_integrations.append(integration)
+
+    # Update submodule in all manifest branches
+    manifest_integrations = []
+    for manifest_branch in manifest_branches:
+        print(
+            "Create integration commit in %s:%s"
+            % (args.manifest_project, manifest_branch),
+        )
+        manifest_integrations.append(
+            integrate_submodule_into(
+                gitlab,
+                args.manifest_project,
+                args.submodule,
+                args.revision,
+                manifest_branch,
+                commit_and_push=False,
+                force_clone=True,
+            )
+        )
+
+    # =======================================================
+    # Create and merge merge_requests if needed
+    # =======================================================
+    if args.merge:
+        # Get source merge request (the one in the gitlab-ci repo)
+        gitlab_ci_project = common.get_project(gitlab, args.project)
+        mrs = get_merge_requests(
+            project=gitlab_ci_project,
+            target_branch=args.branch,
+            state="merged",
+            commit=args.revision,
+        )
+        if not mrs:
+            sys.exit(
+                "ERROR: could not determine source merge request for commit %s"
+                % args.revision
+            )
+        source_mr = mrs[0]
+
+        for project_integration in project_integrations:
+            logging.debug("Create MR in %s", project_integration["project"].name)
+            mr = create_integration_merge_request(
+                project_integration["project"],
+                project_integration["integration_branch"],
+                project_integration["master_branch"],
+                source_mr,
+            )
+            # Now merge
+            logging.debug("Merge %s!%s", project_integration["project"], mr.iid)
+
+            # Wait until GitLab has checked merge status
+            common.wait_until_merge_status_is_set(project_integration["project"], mr)
+
+            # Attempt to merge
+            merged, integration_commit = accept_merge_request(
+                project_integration["project"], mr, rebase=True
+            )
+            # if this has rebased the integration commit needs to be adapted:
+            project_integration["commit"] = integration_commit
+            # Save the target branch here, as the source branch gets deleted
+            # during merge
+            project_integration["integration_branch"] = mr.target_branch
+
+            if not merged:
+                sys.exit(
+                    "Integration MR could not be merged:\n"
+                    "%s\n"
+                    "This can probably be resolved by creating a new commit in "
+                    "gitlab-ci and merging it. The above MR can be closed then."
+                    % mr.web_url
+                )
+
+    if not args.merge:
+        sys.exit(0)
+
+if __name__ == "__main__":
+    main()

scripts/integrate_into_layer.py

+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import tempfile
+import re
+from pathlib import Path
+from furl import furl
+from git import GitCommandError, Repo
+from gitlab import Gitlab
+from gitlab.v4.objects import Project
+from lxml import etree
+
+def update_srcrev(srcrev, recipe_name, new_revision):
+    # Check if project is referenced in SRCREV.conf
+    # Match "...RECIPE_NAME ="
+    pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+
+    project_line = None
+    for line in srcrev.splitlines():
+        if pattern.search(line):
+            project_line = line
+            break
+    if project_line is None:
+        return None
+
+    # Get current project revision from SRCREV file
+    # Assuming notation: <project> = "<hash>"
+    old_revision = project_line.split('"')[1]
+
+    # Update SRCREV file
+    srcrev = srcrev.replace(old_revision, new_revision)
+    return srcrev
+
+
+def integrate_into_layer(
+    layer_project: Project,
+    layer_branch,
+    srcrev_file,
+    recipe_name,
+    project: Project,
+    merge_request,
+):
+    gitlab = layer_project.manager.gitlab
+
+    with tempfile.TemporaryDirectory() as layer_dir:
+        srcrev_filepath = Path(layer_dir) / srcrev_file
+
+        # Construct clone url containing access token
+        clone_url = furl(layer_project.http_url_to_repo)
+        print("This is the url to clone", clone_url.url)
+        clone_url.username = "gitlab-ci"
+        clone_url.password = gitlab.private_token
+
+        # Checkout layer
+        # TODO replace checkout with gitlab api access
+        print("Cloning layer repo: %s" % layer_project.http_url_to_repo)
+        try:
+            layer_repo = Repo.clone_from(
+                clone_url.url, layer_dir, branch=layer_branch
+            )
+        except GitCommandError as e:
+            sys.exit("ERROR: could not clone layer repository\n" + str(e))
+        except IndexError:
+            sys.exit("ERROR: branch '%s' not found" % layer_branch)
+
+        # Special handling for the gitlab-ci integration
+        # When the branch 'merge_request.source_branch' already starts with
+        # integrate/gitlab-ci we add our new commit to this branch
+        # Otherwise (normal behaviour) a new integration branch is created
+        integration_branch = common.find_gitlab_ci_integration_branch(
+            layer_repo, merge_request.source_branch
+        )
+
+        if integration_branch is not None:
+            layer_repo.git.checkout(
+                "-b", integration_branch, "origin/{}".format(integration_branch)
+            )
+            logging.debug("Heads: %s", layer_repo.heads)
+            layer_repo.heads[integration_branch].checkout()
+            logging.debug(layer_repo.git.log("--oneline", "-n", "5"))
+            print("Using existing integration branch: %s" % integration_branch)
+        else:
+            # Create integration branch (delete former one if already exists)
+            integration_branch = recipe_name + "/" + merge_request.source_branch
+            for ref in layer_repo.references:
+                if integration_branch == ref.name:
+                    layer_repo.delete_head(ref)
+
+            print("Creating integration branch: %s" % integration_branch)
+            layer_repo.head.set_reference(
+                layer_repo.create_head(integration_branch)
+            )
+            with open('integration_branch_file', "w", encoding="utf-8") as file:
+                file.write(integration_branch)
+        # Get new project revision from merge request
+        new_revision = merge_request.sha
+
+        with open(srcrev_filepath, "r", encoding="utf8") as fp:
+            srcrev = fp.read()
+        new_srcrev = update_srcrev(srcrev, recipe_name, new_revision)
+        # write file
+        if new_srcrev is None:
+            sys.exit(
+                "ERROR: project '%s' not found in layer and "
+                "no recipe name is specified" % project.path
+            )
+        with open(srcrev_filepath.as_posix(), "w", encoding="utf8") as fp:
+            fp.write(new_srcrev)
+        layer_repo.index.add([srcrev_file])
+
+        # Make an API request to create the gitlab.user object
+        gitlab.auth()
+
+        # Construct commit message and commit the change
+        message = "Integrate %s/%s\n%s" % (
+            project.path,
+            merge_request.source_branch,
+            common.list_commits(merge_request.commits()),
+        )
+        layer_revision = common.commit_and_push(
+            layer_project,
+            layer_repo,
+            message,
+            gitlab.user.username,
+            gitlab.user.email,
+        )
+        logging.debug("New revision in layer: %s", layer_revision)
+
+        return layer_revision
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        required=True,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token for the layer group""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--layer-project",
+        help="""name of the yocto layer project""",
+        dest="layer_project",
+        required=True,
+    )
+    parser.add_argument(
+        "--layer-branch",
+        help="""yocto layer branch to branch off from""",
+        dest="layer_branch",
+        required=True,
+    )
+    parser.add_argument(
+        "--srcrev-file",
+        help="""source revision file name (default: 'SRCREV.conf')""",
+        dest="srcrev_file",
+        default=common.srcrev_file,
+        required=False,
+    )
+    parser.add_argument(
+        "--recipe-name",
+        help="""recipe name to resolve project in 'SRCREV.conf'""",
+        dest="recipe_name",
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        "--project",
+        help="""name of the project, as specified in the layer""",
+        dest="project",
+        required=True,
+    )
+    parser.add_argument(
+        "--merge-request",
+        help="""project merge request IID or link containing the
+                changes to be integrated""",
+        dest="merge_request",
+        required=True,
+    )
+    parser.add_argument(
+        "--save-revision-to",
+        help="""path to a file where the new layer revision is stored""",
+        dest="revision_file",
+        required=False,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
+
+    args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    logging.debug(args)
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+    layer_project = common.get_project(gitlab, args.layer_project)
+    
+    project = common.get_project(gitlab, args.project)
+    logging.debug("Project: %s", project.name)
+    logging.debug("Merge Request: %s", args.merge_request)
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+    merge_request = common.get_merge_request(project, args.merge_request)
+    if merge_request is None:
+        sys.exit("ERROR: could not get %s  %s" % (project.name, args.merge_request))
+
+    layer_revision = integrate_into_layer(
+        layer_project=layer_project,
+        layer_branch=args.layer_branch,
+        srcrev_file=args.srcrev_file,
+        recipe_name=args.recipe_name,
+        project=project,
+        merge_request=merge_request,
+    )
+
+    if args.revision_file:
+        with open(args.revision_file, "w", encoding="utf-8") as file:
+            file.write(layer_revision)
+
+if __name__ == "__main__":
+    main()