Skip to content
Snippets Groups Projects
Commit c25eb909 authored by Lorenzo Pagliai's avatar Lorenzo Pagliai
Browse files

[CICD] Updating manifest-integration.yml file for edgehog

parent 38b88aed
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
common.yml
+ 4
− 4
View file @ c25eb909
......@@ -5,9 +5,9 @@
variables:
# CI_IMAGES_BASEPATH: Environment variable configure in gitlab
CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
CI_IMAGES_REV: e6a404c3e1a919f54feb83f1bc061ce171220437
CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REV}"
CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REV}"
CI_IMAGES_REV: test
CI_IMAGE_PYTHON: "git.seco.com:5050/edgehog/yocto_ng/seco-base:${CI_IMAGES_REV}"
CI_IMAGE_YOCTO: "secodocker/edgehog-builder:latest"
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
# Reduced depth as checkout of larger projects (like the kernel)
......@@ -22,7 +22,7 @@ variables:
tags:
- infrastructure
timeout: 10m
image: secodocker/manifest-test:vtemp
image: $CI_IMAGE_PYTHON
needs: []
variables:
# Include git submodules
......
manifest-integration.yml
+ 137
− 106
View file @ c25eb909
......@@ -2,12 +2,16 @@
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- local: common.yml
stages:
- testing
- manifest-integration
- infrastructure
- integrate
- merge
- build
- check
variables:
MANIFEST_FILE: "distro/seco-distro.xml"
......@@ -17,128 +21,155 @@ variables:
# has to specify it in its own gitlab-ci.yml file.
# The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
# used for projects referenced in the manifest file.
#GITLAB_CI_REVISION: 80380e9065c04e7983fa36d451a9a62d96e5addb
# FIXME: This is only necessary due to the following GitLab limitation:
# https://gitlab.com/gitlab-org/gitlab/-/issues/209904
# As soon as this gets fixed upstream, the hard-coded branch name should be removed.
#MASTER_BRANCH: kirkstone/develop
MANIFEST_PROJECT: seco-manifest
MASTER_BRANCH_MANIFEST: kirkstone/develop
MASTER_BRANCH_PROJECT: main
MASTER_BRANCH_PROJECT: $MASTER_BRANCH
BB_RECIPE_NAME: none
workflow:
DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/"
GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
# --------------------------------------------------------------------------------------
# Stage: infrastructure
# --------------------------------------------------------------------------------------
integrate:
extends: .infrastructure
rules:
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
# Do not run pipelines on forked projects.
# The pipelines would not work anyway because of the users permissions.
# There are two cases catched here:
# 1. The project is forked into someones gitlab namespace and a MR to
# include a change into this forked project is created. In this case
# is the CI_PROJECT_ROOT_NAMESPACE not seco-ne but the
# namespace the fork lives in.
# 2. The MR from the forked project is created to merge the change into this
# the project in the seco-ne namespace (customer sending
# change to us). Here the the IDs used below differ.
#
- if: $CI_PROJECT_ROOT_NAMESPACE != "seco-ne"
# Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
# The integration is done from the pipeline in gitlab-ci already
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
- if: $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != $CI_MERGE_REQUEST_PROJECT_ID
when: never
# FIXME: Unfortunately we cannot use variables in regular expressions due to a
# GitLab limitation: https://gitlab.com/gitlab-org/gitlab/-/issues/209904
# As soon as this get fixed, use the regex based rules below instead of checking
# against the MASTER_BRANCH variable.
# Run pipeline if target branch of the merge request has an integration target, i.e.
# INTEGRATION contains a line beginning with the target branch followed by a colon.
# This also implies that the pipeline runs in merge request context only.
# - if: $INTEGRATION =~ /^$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:/m
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $MASTER_BRANCH
# Run pipeline on target branch after merging a merge request.
# - if: $INTEGRATION =~ /^$CI_COMMIT_BRANCH:/m
- if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
# We have to make sure that the pipeline runs for the current manifest
# master at the time a merge request is created. Otherwise we cannot
# guarantee a green master after merging.
#- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- if: $CI_MERGE_REQUEST_IID
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
cache:
policy: push
script:
- cd ${CI_PROJECT_DIR}
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
- python3 scripts/integrate_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--manifest-file=${MANIFEST_FILE}
--integration-base=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--merge-request=${MERGE_REQUEST}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
--verbose
artifacts:
paths:
- manifest_revision
.skip-for-gitlab-ci-integrations:
rules:
- if: $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
yamllint:
extends: .yamllint
# --------------------------------------------------------------------------------------
# Manifest integration jobs
# Stage: merge
# --------------------------------------------------------------------------------------
testing:
tags:
- infrastructure
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
stage: testing
timeout: 10 m
image: secodocker/manifest-test:vtemp
needs: []
generate-pipelines:
merge:
extends: .infrastructure
stage: merge
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
extends:
- .infrastructure
- .skip-for-gitlab-ci-integrations
stage: manifest-integration
- if: $CI_COMMIT_BRANCH == "$MASTER_BRANCH_PROJECT"
when: always
script:
# The job generation script implicitly passes the OS environment to the template, so
# that the template has access to all GitLab CI variables. Hence there is no need
# to explicitly pass any of them as command line arguments.
- .gitlab-ci/scripts/generate_job_from_template.py
--template=.gitlab-ci/manifest-integration-pipelines.yml.jinja2
> manifest-integration-pipelines.yml
- cd ${CI_PROJECT_DIR}
- python3 scripts/merge_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--master-branch=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--master-branch-project=${MASTER_BRANCH_PROJECT}
--commit=${CI_COMMIT_SOURCE}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
artifacts:
expire_in: 4 weeks
paths:
- manifest-integration-pipelines.yml
trigger-pipelines:
- manifest_revision
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
build:
stage: build
tags:
- infrastructure
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
extends:
- .skip-for-gitlab-ci-integrations
stage: manifest-integration
needs:
- generate-pipelines
trigger:
include:
- artifact: manifest-integration-pipelines.yml
job: generate-pipelines
strategy: depend
# Do not run build if the "skip build" label is set on the merge request
- if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
when: never
# execute this in MR only and not for integrate/gitlab-ci/ integrations
# branches. These are build after the integration has been done in all
# projects
- if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
script:
- echo "Building"
#trigger:
# project: !reference [variables, MANIFEST_PROJECT]
# branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
# strategy: depend
#
# --------------------------------------------------------------------------------------
# Stage: check
# --------------------------------------------------------------------------------------
check:
extends:
- .infrastructure
- .skip-for-gitlab-ci-integrations
stage: manifest-integration
extends: .infrastructure
stage: check
rules:
- if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
needs:
- trigger-pipelines
# Do not run check if the "skip build" label is set on the merge request
- if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
when: never
# Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
# The integration is done from the pipeline in gitlab-ci already
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
- if: $CI_MERGE_REQUEST_IID
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
needs: ["integrate"]
allow_failure: true
script:
- cd ${CI_PROJECT_DIR}
# Loop over all integrations and check each integration branch
- while read -r integration; do
SOURCE_BRANCH=$(echo $integration | cut -d':' -f1);
TARGET_PROJECT=$(echo $integration | cut -d':' -f2);
TARGET_BRANCH=$(echo $integration | cut -d':' -f3);
if [[ "$SOURCE_BRANCH" == "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]]; then
.gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--target-project=${TARGET_PROJECT}
--target-branch=${TARGET_BRANCH}
--source-project=${CI_PROJECT_PATH}
--merge-request=${CI_MERGE_REQUEST_IID}
;
fi;
done <<< "$INTEGRATION"
yamllint:
extends: .yamllint
stage: manifest-integration
# When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
# but CI_OPEN_MERGE_REQUESTS. We use the first of this comma separated list
# in this case
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
# The 'parent_merge_request' is passed from the trigger
# in case this check job is part of a gitlab-ci integration
# pipeline. It is only used to display the correct MR to run again
# in a failed check
- if [ -n "${parent_merge_request}" ];then
PARENT_MR="--parent-merge-request=${parent_merge_request}";
fi
- python3 scripts/check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--integration-base=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--merge-request=${MERGE_REQUEST}
--verbose
${PARENT_MR}
manifest-integration_GF.yml 0 → 100644
+ 144
− 0
View file @ c25eb909
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- local: common.yml
stages:
- testing
- manifest-integration
variables:
MANIFEST_FILE: "distro/seco-distro.xml"
# The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
# to match the repository and the bitbake recipe name.
# We set it here to none, as every project needing it
# has to specify it in its own gitlab-ci.yml file.
# The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
# used for projects referenced in the manifest file.
MANIFEST_PROJECT: seco-manifest
MASTER_BRANCH_MANIFEST: kirkstone/develop
MASTER_BRANCH_PROJECT: main
BB_RECIPE_NAME: none
workflow:
rules:
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
# Do not run pipelines on forked projects.
# The pipelines would not work anyway because of the users permissions.
# There are two cases catched here:
# 1. The project is forked into someones gitlab namespace and a MR to
# include a change into this forked project is created. In this case
# is the CI_PROJECT_ROOT_NAMESPACE not seco-ne but the
# namespace the fork lives in.
# 2. The MR from the forked project is created to merge the change into this
# the project in the seco-ne namespace (customer sending
# change to us). Here the the IDs used below differ.
#
- if: $CI_PROJECT_ROOT_NAMESPACE != "seco-ne"
when: never
- if: $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != $CI_MERGE_REQUEST_PROJECT_ID
when: never
# FIXME: Unfortunately we cannot use variables in regular expressions due to a
# GitLab limitation: https://gitlab.com/gitlab-org/gitlab/-/issues/209904
# As soon as this get fixed, use the regex based rules below instead of checking
# against the MASTER_BRANCH variable.
# Run pipeline if target branch of the merge request has an integration target, i.e.
# INTEGRATION contains a line beginning with the target branch followed by a colon.
# This also implies that the pipeline runs in merge request context only.
# - if: $INTEGRATION =~ /^$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:/m
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $MASTER_BRANCH
# Run pipeline on target branch after merging a merge request.
# - if: $INTEGRATION =~ /^$CI_COMMIT_BRANCH:/m
- if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
- if: $CI_MERGE_REQUEST_IID
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
.skip-for-gitlab-ci-integrations:
rules:
- if: $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
# --------------------------------------------------------------------------------------
# Manifest integration jobs
# --------------------------------------------------------------------------------------
testing:
tags:
- infrastructure
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
stage: testing
timeout: 10 m
image: secodocker/manifest-test:vtemp
needs: []
generate-pipelines:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
extends:
- .infrastructure
- .skip-for-gitlab-ci-integrations
stage: manifest-integration
script:
# The job generation script implicitly passes the OS environment to the template, so
# that the template has access to all GitLab CI variables. Hence there is no need
# to explicitly pass any of them as command line arguments.
- .gitlab-ci/scripts/generate_job_from_template.py
--template=.gitlab-ci/manifest-integration-pipelines.yml.jinja2
> manifest-integration-pipelines.yml
artifacts:
expire_in: 4 weeks
paths:
- manifest-integration-pipelines.yml
trigger-pipelines:
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
extends:
- .skip-for-gitlab-ci-integrations
stage: manifest-integration
needs:
- generate-pipelines
trigger:
include:
- artifact: manifest-integration-pipelines.yml
job: generate-pipelines
strategy: depend
check:
extends:
- .infrastructure
- .skip-for-gitlab-ci-integrations
stage: manifest-integration
rules:
- if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
needs:
- trigger-pipelines
allow_failure: true
script:
- cd ${CI_PROJECT_DIR}
# Loop over all integrations and check each integration branch
- while read -r integration; do
SOURCE_BRANCH=$(echo $integration | cut -d':' -f1);
TARGET_PROJECT=$(echo $integration | cut -d':' -f2);
TARGET_BRANCH=$(echo $integration | cut -d':' -f3);
if [[ "$SOURCE_BRANCH" == "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]]; then
.gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--target-project=${TARGET_PROJECT}
--target-branch=${TARGET_BRANCH}
--source-project=${CI_PROJECT_PATH}
--merge-request=${CI_MERGE_REQUEST_IID}
;
fi;
done <<< "$INTEGRATION"
yamllint:
extends: .yamllint
stage: manifest-integration
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment