diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0fcdf03e67f2ce0e23404d1d8347331e1d1eb181..81216c79ef1deeab1279274212dbd3fa1bfb2d46 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,6 +9,7 @@ variables:
CI_IMAGES_REV: latest
CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/infrastructure/python3.9:${CI_IMAGES_REV}"
CI_IMAGE_YOCTO: "secodocker/edgehog-builder:${CI_IMAGES_REV}"
+ PROJECT_INTEGRATION_GROUPS: "arm/nxp/imx/develop/yocto/5.x" #arm/rockchip
image: "${CI_IMAGE_PYTHON}"
@@ -75,14 +76,26 @@ executable:
MERGE: ""
script:
- cd ${CI_PROJECT_DIR}
- - echo
- - echo ${CI_SERVER_URL}
- echo ${GITBOT_TOKEN}
- echo ${MANIFEST_PROJECT}
- echo ${MANIFEST_BRANCH}
- echo ${CI_COMMIT_SHA}
- echo ${PROJECT_GROUP}
- echo ${MERGE}
+ - |
+ for PROJECT_INTEGRATION in ${PROJECT_INTEGRATION_GROUPS}; do
+ echo ${PROJECT_INTEGRATION}
+ scripts/deploy_gitlab_ci_projects.py \
+ --gitlab-url=${CI_SERVER_URL} \
+ --token=${GITBOT_TOKEN} \
+ --manifest-project=${MANIFEST_PROJECT} \
+ --manifest-branch=${MANIFEST_BRANCH} \
+ --submodule=.gitlab-ci \
+ --revision=${CI_COMMIT_SHA} \
+ --group="${PROJECT_INTEGRATION}" \
+ --verbose \
+ ${MERGE}
+ done
- scripts/deploy_gitlab_ci.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
@@ -116,7 +129,7 @@ executable:
# extends: .integrate-ci-test
# variables:
# MANIFEST_BRANCH: primary
-#
+
#integrate-ci-test:secondary:
# extends: .integrate-ci-test
# variables:
@@ -166,7 +179,7 @@ integrate-yocto:kirkstone:
# needs: ["integrate-ci-test:primary"]
# variables:
# MANIFEST_BRANCH: primary
-#
+
#build-ci-test:secondary:
# extends: .build-ci-test
# needs: ["integrate-ci-test:secondary"]
diff --git a/layers-integration.yml b/layers-integration.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ca522a0a60e7fc145aaf915e0c9fd1d930a43bf1
--- /dev/null
+++ b/layers-integration.yml
@@ -0,0 +1,186 @@
+---
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+
+include:
+ - local: common.yml
+
+stages:
+ - infrastructure
+ - integrate-into-layer
+ - create-merge
+ - check
+ - accept-merge
+
+variables:
+ # The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
+ # to match the repository and the bitbake recipe name.
+ # We set it here to none, as every project needing it
+ # has to specify it in its own gitlab-ci.yml file.
+ # The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
+ # used for projects referenced in the manifest file.
+ # FIXME: This is only necessary due to the following GitLab limitation:
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/209904
+ # As soon as this gets fixed upstream, the hard-coded branch name should be removed.
+ MANIFEST_PROJECT: yocto_ng/seco-manifest
+ MASTER_BRANCH_LAYER: kirkstone/develop
+ BB_RECIPE_NAME: none
+ SRCREV_FILE: conf/SRCREV.conf
+
+ DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/"
+ GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
+ GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
+ TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
+
+# --------------------------------------------------------------------------------------
+# Stage: infrastructure
+# --------------------------------------------------------------------------------------
+integrate-into-layer:
+ extends: .infrastructure
+ rules:
+ # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+ # The integration is done from the pipeline in gitlab-ci already
+ - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+ when: never
+ # We have to make sure that the pipeline runs for the current manifest
+ # master at the time a merge request is created. Otherwise we cannot
+ # guarantee a green master after merging.
+ - if: $CI_MERGE_REQUEST_IID
+ # Explicitly allow externally triggered pipelines in every case
+ - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+ cache:
+ policy: push
+ script:
+ - cd ${CI_PROJECT_DIR}
+ - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+ MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+ else
+ MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+ fi
+ - .gitlab-ci/scripts/integrate_into_layer.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --layer-project=${LAYER_PROJECT_PATH}
+ --layer-branch=${MASTER_BRANCH_LAYER}
+ --project=${CI_PROJECT_PATH}
+ --srcrev-file=${SRCREV_FILE}
+ --merge-request=${MERGE_REQUEST}
+ --save-revision-to=srcrev_revision
+ --recipe-name=${BB_RECIPE_NAME}
+ --verbose
+ artifacts:
+ paths:
+ - srcrev_revision
+ - integration_branch_file
+
+# --------------------------------------------------------------------------------------
+# Stage: create-merge-request
+# --------------------------------------------------------------------------------------
+create-merge-request:
+ stage: create-merge
+ extends: .infrastructure
+ rules:
+ # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+ # The integration is done from the pipeline in gitlab-ci already
+ - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+ when: never
+ # We have to make sure that the pipeline runs for the current manifest
+ # master at the time a merge request is created. Otherwise we cannot
+ # guarantee a green master after merging.
+ - if: $CI_MERGE_REQUEST_IID
+ - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+ cache:
+ policy: push
+ script:
+ - cd ${CI_PROJECT_DIR}
+ - INTEGRATION_BRANCH=$(cat integration_branch_file)
+ - .gitlab-ci/scripts/create_merge_request.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --project=${LAYER_PROJECT_PATH}
+ --source-branch=${INTEGRATION_BRANCH}
+ --target-branch=${MASTER_BRANCH_LAYER}
+ needs:
+ - job: integrate-into-layer
+ artifacts: true
+
+#yamllint:
+# extends: .yamllint
+
+# --------------------------------------------------------------------------------------
+# Stage: check
+# --------------------------------------------------------------------------------------
+check:
+ extends: .infrastructure
+ stage: check
+ rules:
+ # Do not run check if the "skip build" label is set on the merge request
+ - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
+ when: never
+ # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+ # The integration is done from the pipeline in gitlab-ci already
+ - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+ when: never
+ - if: $CI_MERGE_REQUEST_IID
+ # Explicitly allow externally triggered pipelines in every case
+ - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+ needs: ["integrate-into-layer"]
+ allow_failure: true
+ script:
+ - cd ${CI_PROJECT_DIR}
+ # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
+ # but CI_OPEN_MERGE_REQUESTS. We use the first of this comma separated list
+ # in this case
+ - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+ MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+ else
+ MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+ fi
+ # The 'parent_merge_request' is passed from the trigger
+ # in case this check job is part of a gitlab-ci integration
+ # pipeline. It is only used to display the correct MR to run again
+ # in a failed check
+ - if [ -n "${parent_merge_request}" ];then
+ PARENT_MR="--parent-merge-request=${parent_merge_request}";
+ fi
+ - while read -r integration; do
+ SOURCE_BRANCH=$(echo $integration | cut -d':' -f1);
+ TARGET_PROJECT=${LAYER_PROJECT_PATH};
+ TARGET_BRANCH=$(echo $integration | cut -d':' -f3);
+ if [[ "$SOURCE_BRANCH" == "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]]; then
+ .gitlab-ci/scripts/check_if_layer_branch_is_up_to_date.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --target-project=${TARGET_PROJECT}
+ --target-branch=${TARGET_BRANCH}
+ --source-project=${CI_PROJECT_PATH}
+ --recipe-name=${BB_RECIPE_NAME}
+ --merge-request=${CI_MERGE_REQUEST_IID}
+ ;
+ fi;
+ done <<< "$INTEGRATION"
+
+merge-into-layer:
+ extends: .infrastructure
+ stage: accept-merge
+ timeout: 4h
+ rules:
+ # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+ # The integration is done from the pipeline in gitlab-ci already
+ - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+ when: never
+ - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
+ when: always
+ script:
+ - cd ${CI_PROJECT_DIR}
+ - echo ${CI_COMMIT_REF_NAME}
+ - .gitlab-ci/scripts/accept_layer_merge_request.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --project=${CI_PROJECT_PATH}
+ --layer-project=${LAYER_PROJECT_PATH}
+ --target-branch=${MASTER_BRANCH}
+ --layer-target-branch=${MASTER_BRANCH_LAYER}
+ --recipe-name=${BB_RECIPE_NAME}
+ --rebase
\ No newline at end of file
diff --git a/manifest-integration.yml b/manifest-integration.yml
index 605a2c3a7df82ac28ad95e1c873d91fc13ab27fa..70f3c463d0a1095176511c076b023acc09dd059b 100644
--- a/manifest-integration.yml
+++ b/manifest-integration.yml
@@ -24,6 +24,7 @@ variables:
# https://gitlab.com/gitlab-org/gitlab/-/issues/209904
# As soon as this gets fixed upstream, the hard-coded branch name should be removed.
MANIFEST_PROJECT: seco-manifest
+ MANIFEST_GROUP: yocto_ng
MASTER_BRANCH_MANIFEST: kirkstone/develop
BB_RECIPE_NAME: none
@@ -61,7 +62,7 @@ integrate:
- .gitlab-ci/scripts/integrate_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
- --manifest-project=${MANIFEST_PROJECT}
+ --manifest-project=${MANIFEST_GROUP}/${MANIFEST_PROJECT}
--manifest-file=${MANIFEST_FILE}
--manifest-branch=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
@@ -73,7 +74,6 @@ integrate:
paths:
- manifest_revision
-
#yamllint:
# extends: .yamllint
@@ -92,16 +92,25 @@ merge:
when: always
script:
- cd ${CI_PROJECT_DIR}
+ - echo ${CI_SERVER_URL}
+ - echo ${GITBOT_TOKEN}
+ - echo ${MANIFEST_PROJECT}
+ - echo ${MASTER_BRANCH_MANIFEST}
+ - echo ${CI_PROJECT_PATH}
+ - echo ${MASTER_BRANCH}
+ - echo ${CI_COMMIT_SOURCE}
+ - echo ${BB_RECIPE_NAME}
- .gitlab-ci/scripts/merge_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
- --manifest-project=${MANIFEST_PROJECT}
+ --manifest-project=${MANIFEST_GROUP}/${MANIFEST_PROJECT}
--manifest-branch=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--project-branch=${MASTER_BRANCH}
--commit=${CI_COMMIT_SOURCE}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
+ --verbose
artifacts:
paths:
- manifest_revision
diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml
index c0d96905f407ba209b9a50a864964a525218b814..46c34f37ccbb7a3b80a2c3c810f9979c2761b26d 100644
--- a/manifest-pipeline-yocto.yml
+++ b/manifest-pipeline-yocto.yml
@@ -191,7 +191,10 @@ retrigger:
if [ ${CI_PIPELINE_SOURCE} == "pipeline" ]; then
echo "This is the manifest branch used for the build ${CI_COMMIT_REF_NAME}"
BUILD_BRANCH="${CI_COMMIT_REF_NAME}"
+<<<<<<< HEAD
fi
+=======
+>>>>>>> c0c25b3b80578877a4c95177a8158f4c2adf5245
- |
su secous -c "
repo init -u ${CI_REPOSITORY_URL} -b ${BUILD_BRANCH};
diff --git a/scripts/accept_layer_merge_request.py b/scripts/accept_layer_merge_request.py
new file mode 100755
index 0000000000000000000000000000000000000000..ecc0453b9c6b792df645078a30396fac44fc944b
--- /dev/null
+++ b/scripts/accept_layer_merge_request.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import time
+from gitlab import (
+ Gitlab,
+ GitlabGetError,
+ GitlabMRClosedError,
+)
+from get_merge_requests import get_merge_requests
+
+critical_error = (
+ "This is a critical error! Please make sure to:\n"
+ " 1. merge the above-mentioned merge request by hand as soon as possible\n"
+ " (this is important, otherwise following merge requests will get stuck, too)\n"
+ " 2. examine why this has happened and fix it in the CI pipeline"
+)
+
+
+def get_source_integration_requests(
+ project, state=None, target_branch=None, commit=None
+):
+ """Get merge request by target branch and optionally commit sha"""
+ merge_requests = []
+ try:
+ all_merge_requests = project.mergerequests.list(
+ target_branch=target_branch,
+ state=state if state else "all",
+ all=True,
+ retry_transient_errors=True,
+ order_by='updated_at'
+ )
+ except GitlabGetError as e:
+ sys.exit(
+ "ERROR: could not list merge requests for project '%s': %s"
+ % (project.name, e)
+ )
+ if commit:
+ for mr in all_merge_requests:
+ if mr.sha == commit or mr.squash_commit_sha == commit:
+ merge_requests.append(mr)
+ elif all_merge_requests:
+ merge_requests = all_merge_requests
+
+ # Get complete objects
+ full_merge_requests = []
+ for mr in merge_requests:
+ mr = project.mergerequests.get(mr.iid, retry_transient_errors=True)
+ full_merge_requests.append(mr)
+
+ return full_merge_requests[0].source_branch
+
+def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=True):
+ """Attempt to merge a merge request, rebase if necessary"""
+ merged = False
+ pipeline_pending = False
+
+ while not merged:
+ # Update merge request before trying to merge it in order to get the latest
+ # pipeline status
+ try:
+ updated_mr = project.mergerequests.get(
+ id=mr.iid, retry_transient_errors=True
+ )
+ mr = updated_mr
+ except GitlabGetError as e:
+ print("WARNING: Could not update merge request object: %s" % e)
+
+ # Try to merge the merge request
+ try:
+ mr.merge(should_remove_source_branch=should_remove_source_branch)
+ if pipeline_pending:
+ print("")
+ if mr.state == "merged":
+ merged = True
+ else:
+ if mr.merge_error:
+ print("Merge error: %s" % mr.merge_error)
+ else:
+ print("Merge reported success, but MR state is '%s'" % mr.state)
+ return False, mr.sha
+
+ except GitlabMRClosedError as e:
+ # See HTTP error codes for merge requests here:
+ # https://docs.gitlab.com/ce/api/merge_requests.html#accept-mr
+ logging.debug("Error from gitlab: %d", e.response_code)
+
+ if e.response_code == 405:
+ # Not allowed (draft, closed, pipeline pending or failed)
+ # Contrary to the documentation, this response is also issued when the
+ # merge failed due to a merge conflict. See GitLab issue:
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/364102
+
+ if mr.has_conflicts:
+ # Merge conflict, automatic rebase not possible
+ if pipeline_pending:
+ print("")
+ print("Merge not possible, has to be rebased manually")
+ return False, mr.sha
+
+ # If pipeline is running, wait for completion
+ if not mr.head_pipeline:
+ # No pipeline created yet
+ print("No pipeline created yet")
+ time.sleep(1)
+ elif mr.head_pipeline["status"] in common.pending_states:
+ # Pipeline pending
+ if not pipeline_pending:
+ print("Waiting for pending pipeline", end="", flush=True)
+ pipeline_pending = True
+ print(".", end="", flush=True)
+ time.sleep(1)
+ else:
+ # Merge failed due to some other reason
+ if pipeline_pending:
+ print("")
+ print("Merge not possible for unkown reason")
+ return False, mr.sha
+
+ elif e.response_code == 406:
+ # Merge conflict, automatic rebase is possible
+ if pipeline_pending:
+ print("")
+ pipeline_pending = False
+ print("Merge not possible, but branch can be automatically rebased")
+ if not rebase:
+ return False, mr.sha
+ print("Trying to rebase...")
+ mr = common.rebase_merge_request(project, mr)
+ if mr.merge_error:
+ print("ERROR: rebase not possible\n'%s'" % mr.merge_error)
+ sys.exit(critical_error)
+ print("Sucessfully rebased")
+
+ else:
+ if pipeline_pending:
+ print("")
+ print("ERROR: merge not possible: %s" % e)
+ sys.exit(critical_error)
+
+ return True, mr.sha
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the GitLab project""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--layer-project",
+ help="""name of the GitLab layer project""",
+ dest="layer_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--target-branch",
+ help="""target branch of the merge request""",
+ dest="target_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--layer-target-branch",
+ help="""target branch of the layer merge request""",
+ dest="layer_target_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--recipe-name",
+ help="""recipe name of the merge request""",
+ dest="recipe_name",
+ required=True,
+ )
+ parser.add_argument(
+ "--rebase",
+ help="""attempt to automatically rebase merge request if necessary""",
+ dest="rebase",
+ action="store_true",
+ required=False,
+ )
+
+ args, _ = parser.parse_known_args()
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ project = common.get_project(gitlab, args.project)
+ layer_project = common.get_project(gitlab, args.layer_project)
+
+ #Retrieving the source branch from the last MR merged in the project
+ try:
+ integration_branch_name = get_source_integration_requests(
+ project,
+ target_branch=args.target_branch,
+ state="merged",
+ )
+ print("This is the source branch of the latest MR merged: ", integration_branch_name)
+
+ #If the last merged branch is the gitlab-ci integration do nothing
+ if "integrate/gitlab-ci" in integration_branch_name:
+ print("The job was triggered by a merge of the gitlab-ci projects into the master branch, doing nothing!")
+ return ""
+
+ except GitlabGetError as e:
+ sys.exit("Could not get integration branch name for latest project MR: %s" % e)
+
+ integration_branch_name = args.recipe_name + "/" + integration_branch_name
+
+ try:
+ merge_request = get_merge_requests(
+ layer_project,
+ source_branch=integration_branch_name,
+ target_branch=args.layer_target_branch,
+ state="opened",
+ )
+ except GitlabGetError as e:
+ sys.exit("Could not get merge request: %s" % e)
+
+ print("This job is going to merge MR %s in project %s", merge_request[0], layer_project)
+ if accept_merge_request(layer_project, merge_request[0], rebase=args.rebase):
+ print("Successfully merged")
+ else:
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/check_if_layer_branch_is_up_to_date.py b/scripts/check_if_layer_branch_is_up_to_date.py
new file mode 100755
index 0000000000000000000000000000000000000000..e3c302478834676fea97e684c46fb1b12f0dc9f4
--- /dev/null
+++ b/scripts/check_if_layer_branch_is_up_to_date.py
@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import sys
+import logging
+from gitlab import Gitlab, GitlabGetError
+from gitlab.v4.objects import Project
+
+
+def check_if_layer_branch_is_up_to_date(
+ target_project: Project,
+ target_branch_name: str,
+ integration_branch_name: str,
+):
+ try:
+ integration_branch = target_project.branches.get(
+ integration_branch_name, retry_transient_errors=True
+ )
+ except GitlabGetError:
+ sys.exit(
+ "ERROR: could not find integration branch {} in {}.".format(
+ integration_branch_name, target_project.name
+ )
+ )
+
+ try:
+ target_branch = target_project.branches.get(
+ target_branch_name, retry_transient_errors=True
+ )
+ except GitlabGetError:
+ sys.exit(
+ "ERROR: could not find target branch {} in {}.".format(
+ target_branch_name, target_project.name
+ )
+ )
+
+ # Loop over the commits until the integration_branch head id is found
+ return common.is_commit_parent_of_project_commit(
+ target_project,
+ integration_branch.commit["id"],
+ target_branch.commit["id"],
+ limit=10,
+ )
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--target-project",
+ help="""name of the target project""",
+ dest="target_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--target-branch",
+ help="""target branch to integrate into""",
+ dest="target_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--source-project",
+ help="""name of the source project""",
+ dest="source_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--recipe-name",
+ help="""name of the BB recipe (for integration branch name)""",
+ dest="recipe_name",
+ required=True,
+ )
+ parser.add_argument(
+ "--merge-request",
+ help="""source project merge request IID containing the changes to be integrated""",
+ dest="merge_request",
+ required=True,
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ help="""Increase verbosity.""",
+ )
+
+ args, _ = parser.parse_known_args()
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+ logging.debug(args)
+ target_project = common.get_project(gitlab, args.target_project)
+ source_project = common.get_project(gitlab, args.source_project)
+ merge_request = common.get_merge_request(source_project, args.merge_request)
+ if merge_request is None:
+ sys.exit(
+ "ERROR: could not get %s %s" % (source_project.name, args.merge_request)
+ )
+
+ integration_branch_name = args.recipe_name + "/" + merge_request.source_branch
+
+ if check_if_layer_branch_is_up_to_date(
+ target_project=target_project,
+ target_branch_name=args.target_branch,
+ integration_branch_name=integration_branch_name,
+ ):
+ print(
+ "Integration branch {} in {} is up to date.".format(
+ integration_branch_name, target_project.name
+ )
+ )
+ else:
+ sys.exit(
+ "Integration branch {} in {} is not up to date.\n"
+ "Please re-run the MR pipeline:\n"
+ " 1. Open the MR pipelines page:\n"
+ " {}\n"
+ " 2. Click 'Run Pipeline'".format(
+ integration_branch_name,
+ target_project.name,
+ merge_request.web_url + "/pipelines",
+ )
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/deploy_gitlab_ci_projects.py b/scripts/deploy_gitlab_ci_projects.py
new file mode 100755
index 0000000000000000000000000000000000000000..25d5f9652ed4c253972cc5c4b28a3bf5c8eedbb6
--- /dev/null
+++ b/scripts/deploy_gitlab_ci_projects.py
@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import os
+from gitlab import Gitlab
+from gitlab.v4.objects import Project, MergeRequest
+
+from accept_merge_request import accept_merge_request
+from create_merge_request import create_merge_request
+from get_integration_sources import get_integration_sources
+from get_merge_requests import get_merge_requests
+from update_submodule import update_submodule_and_include_ref
+from integrate_into_manifest import update_manifest, update_srcrev
+
+from ruamel.yaml import YAML
+
+
+def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
+
+ # Read values from existing file
+ yaml = YAML()
+ data = yaml.load(gitlab_ci_yml)
+ logging.debug("Yaml: %s", data)
+
+ try:
+ recipe = data["variables"]["BB_RECIPE_NAME"]
+ logging.debug("Recipe %s", recipe)
+ except KeyError:
+ recipe = None
+ return {"recipe": recipe}
+
+
+def integrate_submodule_into(
+ gitlab,
+ project_name,
+ submodule_name,
+ new_revision,
+ branch,
+ commit_and_push=True,
+ force_clone=False,
+):
+
+ gitlab_project = common.get_project(gitlab, project_name)
+
+ (
+ project_repo,
+ project_dir,
+ integration_branch_name,
+ integration_commit,
+ message,
+ ) = update_submodule_and_include_ref(
+ gitlab_project,
+ submodule_name,
+ new_revision,
+ branch,
+ commit_and_push=commit_and_push,
+ force_clone=force_clone,
+ )
+ # ======================================
+ # Store the references for creating the integration
+ # commit in the manifest later
+ # ======================================
+ ret = {
+ "project": gitlab_project,
+ "repo": project_repo,
+ "dir": project_dir,
+ "integration_branch": integration_branch_name,
+ "master_branch": branch,
+ "commit": integration_commit,
+ "message": message,
+ }
+ logging.debug(
+ "Integration branch: %s (%s)",
+ integration_branch_name,
+ integration_commit,
+ )
+ return ret
+
+
+def create_integration_merge_request(
+ project: Project,
+ integration_branch: str,
+ target_branch: str,
+ source_mr: MergeRequest = None,
+) -> MergeRequest:
+ # Create merge request
+ # This should be optional
+ mr, created = create_merge_request(project, integration_branch, target_branch)
+ if created:
+ if source_mr is not None:
+ common.crosslink_merge_requests(source_mr, mr)
+ print("Created new merge request:\n%s" % mr.web_url)
+ else:
+ print("Existing integration merge request:\n%s" % mr.web_url)
+ return mr
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-project",
+ help="""name of the manifest project""",
+ dest="manifest_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--manifest-branch",
+ help="""manifest branch to integrate changes into (can be a comma-separated list)""",
+ dest="manifest_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--submodule",
+ help="""submodule to update""",
+ dest="submodule",
+ required=True,
+ )
+ parser.add_argument(
+ "--revision",
+ help="""new revision for submodule""",
+ dest="revision",
+ required=True,
+ )
+ parser.add_argument(
+ "--merge",
+ help="""if set, perform merge after integration""",
+ dest="merge",
+ action="store_true",
+ required=False,
+ default=False,
+ )
+ parser.add_argument(
+ "--project",
+ help="""gitlab-ci project path or id""",
+ dest="project",
+ default=os.environ.get("CI_PROJECT_PATH"),
+ required=False,
+ )
+ parser.add_argument(
+ "--branch",
+ help="""gitlab-ci branch that we're merging into""",
+ dest="branch",
+ default="master",
+ required=False,
+ )
+ parser.add_argument(
+ "--manifest-file",
+ help="""manifest file name (default: 'default.xml')""",
+ dest="manifest_file",
+ default=common.manifest_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--srcrev-file",
+ help="""source revision file name (default: 'SRCREV.conf')""",
+ dest="srcrev_file",
+ default=common.srcrev_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--group",
+ help="""group path or id to limit search scope to""",
+ dest="group",
+ required=True,
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ help="""Increase verbosity.""",
+ )
+
+ args, _ = parser.parse_known_args()
+ if args.verbose:
+ logging.basicConfig(
+ level=logging.DEBUG,
+ format="%(asctime)s %(levelname)-8s %(message)s",
+ datefmt="%H:%M:%S",
+ )
+
+ manifest_branches = args.manifest_branch.split(",")
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ group = gitlab.groups.get(args.group)
+
+ # =======================================================
+ # Create integration branches and commits with updates
+ # submodule in all projects
+ # =======================================================
+ integration_sources = {}
+ all_integration_sources = []
+ for manifest_branch in manifest_branches:
+ print(
+ "Searching for projects in %s that are configured for automatic integration into %s:%s"
+ % (args.group, args.manifest_project, manifest_branch)
+ )
+ integration_sources[manifest_branch] = get_integration_sources(
+ args.manifest_project, manifest_branch, group
+ )
+ for s in integration_sources[manifest_branch]:
+ if s not in all_integration_sources:
+ all_integration_sources.append(s)
+
+ # Update submodule in all integration sources
+ project_integrations = []
+ for s in all_integration_sources:
+ print("Create integration commit in %s:%s" % (s["project"], s["branch"]))
+
+ integration = integrate_submodule_into(
+ gitlab, s["project"], args.submodule, args.revision, s["branch"]
+ )
+ # Store in the list if commit is set (meaning there was an update or
+ # an exising integration branch)
+ if integration["commit"] is not None:
+ project_integrations.append(integration)
+
+ # Update submodule in all manifest branches
+ manifest_integrations = []
+ for manifest_branch in manifest_branches:
+ print(
+ "Create integration commit in %s:%s"
+ % (args.manifest_project, manifest_branch),
+ )
+ manifest_integrations.append(
+ integrate_submodule_into(
+ gitlab,
+ args.manifest_project,
+ args.submodule,
+ args.revision,
+ manifest_branch,
+ commit_and_push=False,
+ force_clone=True,
+ )
+ )
+
+ # =======================================================
+ # Create and merge merge_requests if needed
+ # =======================================================
+ if args.merge:
+ # Get source merge request (the one in the gitlab-ci repo)
+ gitlab_ci_project = common.get_project(gitlab, args.project)
+ mrs = get_merge_requests(
+ project=gitlab_ci_project,
+ target_branch=args.branch,
+ state="merged",
+ commit=args.revision,
+ )
+ if not mrs:
+ sys.exit(
+ "ERROR: could not determine source merge request for commit %s"
+ % args.revision
+ )
+ source_mr = mrs[0]
+
+ for project_integration in project_integrations:
+ logging.debug("Create MR in %s", project_integration["project"].name)
+ mr = create_integration_merge_request(
+ project_integration["project"],
+ project_integration["integration_branch"],
+ project_integration["master_branch"],
+ source_mr,
+ )
+ # Now merge
+ logging.debug("Merge %s!%s", project_integration["project"], mr.iid)
+
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(project_integration["project"], mr)
+
+ # Attempt to merge
+ merged, integration_commit = accept_merge_request(
+ project_integration["project"], mr, rebase=True
+ )
+ # if this has rebased the integration commit needs to be adapted:
+ project_integration["commit"] = integration_commit
+ # Save the target branch here, as the source branch gets deleted
+ # during merge
+ project_integration["integration_branch"] = mr.target_branch
+
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged:\n"
+ "%s\n"
+ "This can probably be resolved by creating a new commit in "
+ "gitlab-ci and merging it. The above MR can be closed then."
+ % mr.web_url
+ )
+
+ if not args.merge:
+ sys.exit(0)
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/integrate_into_layer.py b/scripts/integrate_into_layer.py
new file mode 100755
index 0000000000000000000000000000000000000000..03e4e82c48e64387fb2e4593a3cb4b5fc2c309a5
--- /dev/null
+++ b/scripts/integrate_into_layer.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+import common
+
+import argparse
+import logging
+import sys
+import tempfile
+import re
+from pathlib import Path
+from furl import furl
+from git import GitCommandError, Repo
+from gitlab import Gitlab
+from gitlab.v4.objects import Project
+from lxml import etree
+
+def update_srcrev(srcrev, recipe_name, new_revision):
+ # Check if project is referenced in SRCREV.conf
+ # Match "...RECIPE_NAME ="
+ pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+
+ project_line = None
+ for line in srcrev.splitlines():
+ if pattern.search(line):
+ project_line = line
+ break
+ if project_line is None:
+ return None
+
+ # Get current project revision from SRCREV file
+ # Assuming notation: <project> = "<hash>"
+ old_revision = project_line.split('"')[1]
+
+ # Update SRCREV file
+ srcrev = srcrev.replace(old_revision, new_revision)
+ return srcrev
+
+
+def integrate_into_layer(
+ layer_project: Project,
+ layer_branch,
+ srcrev_file,
+ recipe_name,
+ project: Project,
+ merge_request,
+):
+ gitlab = layer_project.manager.gitlab
+
+ with tempfile.TemporaryDirectory() as layer_dir:
+ srcrev_filepath = Path(layer_dir) / srcrev_file
+
+ # Construct clone url containing access token
+ clone_url = furl(layer_project.http_url_to_repo)
+ print("This is the url to clone", clone_url.url)
+ clone_url.username = "gitlab-ci"
+ clone_url.password = gitlab.private_token
+
+ # Checkout layer
+ # TODO replace checkout with gitlab api access
+ print("Cloning layer repo: %s" % layer_project.http_url_to_repo)
+ try:
+ layer_repo = Repo.clone_from(
+ clone_url.url, layer_dir, branch=layer_branch
+ )
+ except GitCommandError as e:
+ sys.exit("ERROR: could not clone layer repository\n" + str(e))
+ except IndexError:
+ sys.exit("ERROR: branch '%s' not found" % layer_branch)
+
+ # Special handling for the gitlab-ci integration
+ # When the branch 'merge_request.source_branch' already starts with
+ # integrate/gitlab-ci we add our new commit to this branch
+ # Otherwise (normal behaviour) a new integration branch is created
+ integration_branch = common.find_gitlab_ci_integration_branch(
+ layer_repo, merge_request.source_branch
+ )
+
+ if integration_branch is not None:
+ layer_repo.git.checkout(
+ "-b", integration_branch, "origin/{}".format(integration_branch)
+ )
+ logging.debug("Heads: %s", layer_repo.heads)
+ layer_repo.heads[integration_branch].checkout()
+ logging.debug(layer_repo.git.log("--oneline", "-n", "5"))
+ print("Using existing integration branch: %s" % integration_branch)
+ else:
+ # Create integration branch (delete former one if already exists)
+ integration_branch = recipe_name + "/" + merge_request.source_branch
+ for ref in layer_repo.references:
+ if integration_branch == ref.name:
+ layer_repo.delete_head(ref)
+
+ print("Creating integration branch: %s" % integration_branch)
+ layer_repo.head.set_reference(
+ layer_repo.create_head(integration_branch)
+ )
+ with open('integration_branch_file', "w", encoding="utf-8") as file:
+ file.write(integration_branch)
+ # Get new project revision from merge request
+ new_revision = merge_request.sha
+
+ with open(srcrev_filepath, "r", encoding="utf8") as fp:
+ srcrev = fp.read()
+ new_srcrev = update_srcrev(srcrev, recipe_name, new_revision)
+ # write file
+ if new_srcrev is None:
+ sys.exit(
+ "ERROR: project '%s' not found in layer and "
+ "no recipe name is specified" % project.path
+ )
+ with open(srcrev_filepath.as_posix(), "w", encoding="utf8") as fp:
+ fp.write(new_srcrev)
+ layer_repo.index.add([srcrev_file])
+
+ # Make an API request to create the gitlab.user object
+ gitlab.auth()
+
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s\n%s" % (
+ project.path,
+ merge_request.source_branch,
+ common.list_commits(merge_request.commits()),
+ )
+ layer_revision = common.commit_and_push(
+ layer_project,
+ layer_repo,
+ message,
+ gitlab.user.username,
+ gitlab.user.email,
+ )
+ logging.debug("New revision in layer: %s", layer_revision)
+
+ return layer_revision
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--gitlab-url",
+ help="""URL to the GitLab instance""",
+ dest="gitlab_url",
+ required=True,
+ )
+ parser.add_argument(
+ "--token",
+ help="""GitLab REST API private access token for the layer group""",
+ dest="token",
+ required=True,
+ )
+ parser.add_argument(
+ "--layer-project",
+ help="""name of the yocto layer project""",
+ dest="layer_project",
+ required=True,
+ )
+ parser.add_argument(
+ "--layer-branch",
+ help="""yocto layer branch to branch off from""",
+ dest="layer_branch",
+ required=True,
+ )
+ parser.add_argument(
+ "--srcrev-file",
+ help="""source revision file name (default: 'SRCREV.conf')""",
+ dest="srcrev_file",
+ default=common.srcrev_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--recipe-name",
+ help="""recipe name to resolve project in 'SRCREV.conf'""",
+ dest="recipe_name",
+ default=None,
+ required=False,
+ )
+ parser.add_argument(
+ "--project",
+ help="""name of the project, as specified in the layer""",
+ dest="project",
+ required=True,
+ )
+ parser.add_argument(
+ "--merge-request",
+ help="""project merge request IID or link containing the
+ changes to be integrated""",
+ dest="merge_request",
+ required=True,
+ )
+ parser.add_argument(
+ "--save-revision-to",
+ help="""path to a file where the new layer revision is stored""",
+ dest="revision_file",
+ required=False,
+ )
+ parser.add_argument(
+ "-v",
+ "--verbose",
+ action="store_true",
+ help="""Increase verbosity.""",
+ )
+
+ args, _ = parser.parse_known_args()
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+
+ logging.debug(args)
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+ layer_project = common.get_project(gitlab, args.layer_project)
+
+ project = common.get_project(gitlab, args.project)
+ logging.debug("Project: %s", project.name)
+ logging.debug("Merge Request: %s", args.merge_request)
+
+ gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+
+ merge_request = common.get_merge_request(project, args.merge_request)
+ if merge_request is None:
+ sys.exit("ERROR: could not get %s %s" % (project.name, args.merge_request))
+
+ layer_revision = integrate_into_layer(
+ layer_project=layer_project,
+ layer_branch=args.layer_branch,
+ srcrev_file=args.srcrev_file,
+ recipe_name=args.recipe_name,
+ project=project,
+ merge_request=merge_request,
+ )
+
+ if args.revision_file:
+ with open(args.revision_file, "w", encoding="utf-8") as file:
+ file.write(layer_revision)
+
+if __name__ == "__main__":
+ main()