Skip to content
Snippets Groups Projects
Commit 2cbac87d authored by Lorenzo Pagliai's avatar Lorenzo Pagliai
Browse files

[LAYERS] Major refactoring of layers integration

This commit introduces major changes in the layers integration steps
performed every time a new MR is opened in one of the BSP repositories.
More in detail:

* Introduces jinja2 templating tool to perform dynamic child pipeline
and dinamically assign the INTEGRATION variable value to the CI jobs.
* The main layers-integration.yml file only contains check and trigger
pipelines to create a dynamic downstream pipeline.
* Introduces layers-integration-jobs.yml that is dinamically assigned
with the variables from the INTEGRATION variable and that executes all
the main tasks of layer integration pipelines
parent b0ad83c8
No related branches found
No related tags found
No related merge requests found
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- project: '${CI_PROJECT_ROOT_NAMESPACE}/infrastructure/gitlab-ci'
ref: ${GITLAB_CI_REVISION}
file: common.yml
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "parent_pipeline"
stages:
- layers-integration-jobs
# --------------------------------------------------------------------------------------
# Merge request pipeline
# --------------------------------------------------------------------------------------
integrate-into-layer:
extends: .infrastructure
stage: layers-integration-jobs
rules:
- if: $CI_MERGE_REQUEST_IID
cache:
policy: push
script:
- cd ${CI_PROJECT_DIR}
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
- .gitlab-ci/scripts/integrate_into_layer.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--layer-project=${TARGET_PROJECT}
--srcrev-file=${SRCREV_FILE}
--layer-branch=${TARGET_BRANCH}
--project=${CI_PROJECT_PATH}
--merge-request=${MERGE_REQUEST}
--save-revision-to=srcrev_revision
--recipe-name=${BB_RECIPE_NAME}
--verbose
artifacts:
paths:
- srcrev_revision
- integration_branch_file
create-merge-request:
extends: .infrastructure
stage: layers-integration-jobs
rules:
# execute this in MR only and not for integrate/gitlab-ci/ integrations
# branches. These are build after the integration has been done in all
# projects
- if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
cache:
policy: push
script:
- cd ${CI_PROJECT_DIR}
- INTEGRATION_BRANCH=$(cat integration_branch_file)
- .gitlab-ci/scripts/create_merge_request.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${TARGET_PROJECT}
--source-branch=${INTEGRATION_BRANCH}
--target-branch=${TARGET_BRANCH}
needs: ["integrate-into-layer"]
# --------------------------------------------------------------------------------------
# Master pipeline
# --------------------------------------------------------------------------------------
merge-into-layer:
extends: .infrastructure
stage: layers-integration-jobs
timeout: 4h
rules:
- if: $CI_COMMIT_BRANCH == $SOURCE_BRANCH
script:
- cd ${CI_PROJECT_DIR}
- .gitlab-ci/scripts/accept_layer_merge_request.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--layer-project=${TARGET_PROJECT}
--target-branch=${SOURCE_BRANCH}
--layer-target-branch=${TARGET_BRANCH}
--recipe-name=${BB_RECIPE_NAME}
--rebase
\ No newline at end of file
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- project: {{ CI_PROJECT_ROOT_NAMESPACE }}/infrastructure/gitlab-ci
ref: {{ GITLAB_CI_REVISION }}
file: common.yml
workflow:
rules:
- if: $CI_PIPELINE_SOURCE == "parent_pipeline"
stages:
- layers-integration-pipelines
# --------------------------------------------------------------------------------------
# Trigger jobs
# --------------------------------------------------------------------------------------
{% for integration in INTEGRATION.split('\n') %}
{% set SOURCE_BRANCH, TARGET_PROJECT, TARGET_BRANCH = integration.split(':') %}
{% if (CI_MERGE_REQUEST_TARGET_BRANCH_NAME is defined
and SOURCE_BRANCH == CI_MERGE_REQUEST_TARGET_BRANCH_NAME)
or SOURCE_BRANCH == CI_COMMIT_REF_NAME %}
{{ TARGET_PROJECT }}:{{ TARGET_BRANCH }}:
stage: layers-integration-pipelines
variables:
SOURCE_BRANCH: {{ SOURCE_BRANCH }}
TARGET_PROJECT: {{ TARGET_PROJECT }}
TARGET_BRANCH: {{ TARGET_BRANCH }}
GITLAB_CI_REVISION: {{ GITLAB_CI_REVISION }}
SRCREV_FILE: {{ SRCREV_FILE }}
BB_RECIPE_NAME: {{ BB_RECIPE_NAME }}
trigger:
include:
project: '${CI_PROJECT_ROOT_NAMESPACE}/infrastructure/gitlab-ci'
ref: ${GITLAB_CI_REVISION}
file: layers-integration-jobs.yml
strategy: depend
{% endif %}
{% endfor %}
......@@ -7,11 +7,7 @@ include:
- local: common.yml
stages:
- infrastructure
- integrate-into-layer
- create-merge
- check
- accept-merge
- layers-integration
variables:
# The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
......@@ -19,134 +15,98 @@ variables:
# We set it here to none, as every project needing it
# has to specify it in its own gitlab-ci.yml file.
# The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
# used for projects referenced in the manifest file.
# used for projects referenced in the layers file.
# FIXME: This is only necessary due to the following GitLab limitation:
# https://gitlab.com/gitlab-org/gitlab/-/issues/209904
# As soon as this gets fixed upstream, the hard-coded branch name should be removed.
MANIFEST_PROJECT: edgehog/seco-manifest
MASTER_BRANCH_LAYER: kirkstone
BB_RECIPE_NAME: none
SRCREV_FILE: conf/SRCREV.conf
DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/"
GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
# --------------------------------------------------------------------------------------
# Stage: infrastructure
# --------------------------------------------------------------------------------------
integrate-into-layer:
extends: .infrastructure
workflow:
rules:
# Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
# The integration is done from the pipeline in gitlab-ci already
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
# We have to make sure that the pipeline runs for the current manifest
# master at the time a merge request is created. Otherwise we cannot
# guarantee a green master after merging.
- if: $CI_MERGE_REQUEST_IID
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
cache:
policy: push
script:
- cd ${CI_PROJECT_DIR}
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
- .gitlab-ci/scripts/integrate_into_layer.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--layer-project=${LAYER_PROJECT_PATH}
--layer-branch=${MASTER_BRANCH_LAYER}
--project=${CI_PROJECT_PATH}
--srcrev-file=${SRCREV_FILE}
--merge-request=${MERGE_REQUEST}
--save-revision-to=srcrev_revision
--recipe-name=${BB_RECIPE_NAME}
--verbose
artifacts:
paths:
- srcrev_revision
- integration_branch_file
# Do not run pipelines on forked projects.
# The pipelines would not work anyway because of the users permissions.
# There are two cases catched here:
# 1. The project is forked into someones gitlab namespace and a MR to
# include a change into this forked project is created. In this case
# is the CI_PROJECT_ROOT_NAMESPACE not seco-ne but the
# namespace the fork lives in.
# 2. The MR from the forked project is created to merge the change into this
# the project in the seco-ne namespace (customer sending
# change to us). Here the the IDs used below differ.
#
- if: $CI_PROJECT_ROOT_NAMESPACE != "edgehog"
when: never
- if: $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != $CI_MERGE_REQUEST_PROJECT_ID
when: never
# FIXME: Unfortunately we cannot use variables in regular expressions due to a
# GitLab limitation: https://gitlab.com/gitlab-org/gitlab/-/issues/209904
# As soon as this get fixed, use the regex based rules below instead of checking
# if branch is protected.
# Run pipeline if target branch of the merge request has an integration target, i.e.
# INTEGRATION contains a line beginning with the target branch followed by a colon.
# This also implies that the pipeline runs in merge request context only.
# - if: $INTEGRATION =~ /^$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:/m
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_PROTECTED == "true"
# Run pipeline on target branch after merging a merge request.
# - if: $INTEGRATION =~ /^$CI_COMMIT_BRANCH:/m
- if: $CI_COMMIT_REF_PROTECTED == "true"
# --------------------------------------------------------------------------------------
# Stage: create-merge-request
# --------------------------------------------------------------------------------------
create-merge-request:
stage: create-merge
extends: .infrastructure
.skip-for-gitlab-ci-integrations:
rules:
# Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
# The integration is done from the pipeline in gitlab-ci already
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
# We have to make sure that the pipeline runs for the current manifest
# master at the time a merge request is created. Otherwise we cannot
# guarantee a green master after merging.
- if: $CI_MERGE_REQUEST_IID
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
cache:
policy: push
script:
- cd ${CI_PROJECT_DIR}
- INTEGRATION_BRANCH=$(cat integration_branch_file)
- .gitlab-ci/scripts/create_merge_request.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${LAYER_PROJECT_PATH}
--source-branch=${INTEGRATION_BRANCH}
--target-branch=${MASTER_BRANCH_LAYER}
needs:
- job: integrate-into-layer
artifacts: true
- if: $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
#yamllint:
# extends: .yamllint
# --------------------------------------------------------------------------------------
# Stage: check
# Layer integration jobs
# --------------------------------------------------------------------------------------
generate-pipelines:
extends:
- .infrastructure
- .skip-for-gitlab-ci-integrations
stage: layers-integration
script:
# The Jinja2 render script implicitly passes the OS environment to the template, so
# that the template has access to all GitLab CI variables. Hence there is no need
# to explicitly pass any of them as command line arguments.
- .gitlab-ci/scripts/render_jinja2_template.py
--template=.gitlab-ci/layers-integration-pipelines.yml.jinja2
--format=yaml
> layers-integration-pipelines.yml
artifacts:
expire_in: 4 weeks
paths:
- layers-integration-pipelines.yml
trigger-pipelines:
extends:
- .skip-for-gitlab-ci-integrations
stage: layers-integration
needs:
- generate-pipelines
trigger:
include:
- artifact: layers-integration-pipelines.yml
job: generate-pipelines
strategy: depend
check:
extends: .infrastructure
stage: check
extends:
- .infrastructure
- .skip-for-gitlab-ci-integrations
stage: layers-integration
rules:
# Do not run check if the "skip build" label is set on the merge request
- if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
when: never
# Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
# The integration is done from the pipeline in gitlab-ci already
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
- if: $CI_MERGE_REQUEST_IID
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
needs: ["integrate-into-layer"]
- if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
needs:
- trigger-pipelines
allow_failure: true
script:
- cd ${CI_PROJECT_DIR}
# When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
# but CI_OPEN_MERGE_REQUESTS. We use the first of this comma separated list
# in this case
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
# The 'parent_merge_request' is passed from the trigger
# in case this check job is part of a gitlab-ci integration
# pipeline. It is only used to display the correct MR to run again
# in a failed check
- if [ -n "${parent_merge_request}" ];then
PARENT_MR="--parent-merge-request=${parent_merge_request}";
fi
# Loop over all integrations and check each integration branch
- while read -r integration; do
SOURCE_BRANCH=$(echo $integration | cut -d':' -f1);
TARGET_PROJECT=${LAYER_PROJECT_PATH};
TARGET_PROJECT=$(echo $integration | cut -d':' -f1);
TARGET_BRANCH=$(echo $integration | cut -d':' -f3);
if [[ "$SOURCE_BRANCH" == "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]]; then
.gitlab-ci/scripts/check_if_layer_branch_is_up_to_date.py
......@@ -161,26 +121,24 @@ check:
fi;
done <<< "$INTEGRATION"
merge-into-layer:
extends: .infrastructure
stage: accept-merge
timeout: 4h
rules:
# Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
# The integration is done from the pipeline in gitlab-ci already
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
- if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
when: always
script:
- cd ${CI_PROJECT_DIR}
- echo ${CI_COMMIT_REF_NAME}
- .gitlab-ci/scripts/accept_layer_merge_request.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--layer-project=${LAYER_PROJECT_PATH}
--target-branch=${MASTER_BRANCH}
--layer-target-branch=${MASTER_BRANCH_LAYER}
--recipe-name=${BB_RECIPE_NAME}
--rebase
\ No newline at end of file
# cancel-previous-pipelines:
# extends:
# - .infrastructure
# - .skip-for-gitlab-ci-integrations
# stage: layers-integration
# rules:
# - if: $CI_MERGE_REQUEST_IID
# allow_failure: true
# script:
# - .gitlab-ci/scripts/cancel_pipelines.py
# --gitlab-url=${CI_SERVER_URL}
# --token=${GITBOT_TOKEN}
# --project=${CI_PROJECT_PATH}
# --ref=${CI_MERGE_REQUEST_REF_PATH}
# --below-pipeline-id=${CI_PIPELINE_ID}
#
# yamllint:
# extends: .yamllint
# stage: layers-integration
# rules:
# - if: $CI_MERGE_REQUEST_IID
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment