Skip to content
Snippets Groups Projects
Commit b0ad83c8 authored by Lorenzo Pagliai's avatar Lorenzo Pagliai
Browse files

[MANIFEST] Major refactoring of manifest pipeline 

The ideas behind this refactoring are the following:

* Introduce dynamic child pipelines also for pipeline running on the
manifest. This in particular allows to dynamically creates a .yml
configuration file containing all the board jobs via jinja2 templating
tool. With this approach it will only be necessary to define three main
variables in the manifest or in the CI/CD variables of the manifest
repository:
	* MACHINES: a list of the machines of which the build is
necessacy
	* DISTRO: a list of the Edgehog distros
	* GRAPHIC_BACKEND: a list of the graphic backends supported
* Introduce a generic manifest-pipeline.yml that is used both by the
pipeline on the test and general manifest and that simply generates
the final .yml file doing all manifest CI/CD tasks.
* Introduce a generic build-pipeline.yml file containing only the
definition of the main jobs used in the manifest CI/CD (i.e. build,
deploy, test, changelog, etc.).
* Refactor manifest-pipeline-yocto.yml to simply stores variables and
include the other .yml files thus allowing the other steps to be better
tested in the testing group.
parent daa6b66c
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
build-pipeline-yocto.yml.jinja2 0 → 100644
+ 316
− 0
View file @ b0ad83c8
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
# As the trigger job is not executed in a environment with checked out repository, we
# need to get the includes directly from gitlab
include:
- project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/infrastructure/gitlab-ci'
ref: {{ GITLAB_CI_REVISION }}
file:
- build-pipeline.yml
- common.yml
#------------
# Stages
#------------
stages:
- build
- deploy
- changelog
- notify
- release
- test-probe
#----------------
# Variables
#----------------
variables:
MASTER_BRANCH: {{ MASTER_BRANCH }}
# --------------------------------------------------------------------------------------
# Stage: changelog
# --------------------------------------------------------------------------------------
changelog:
extends: .changelog
# --------------------------------------------------------------------------------------
# Stage: notify
# --------------------------------------------------------------------------------------
notify:
extends: .notify
# --------------------------------------------------------------------------------------
# Stage: release
# --------------------------------------------------------------------------------------
release:
extends: .release
# --------------------------------------------------------------------------------------
# Generated jobs
# --------------------------------------------------------------------------------------
{% if MACHINES %}
{% for machine in MACHINES.split(' ') %}
{% if CUSTOM is not defined %}
{% if '928' in machine %}
{% set PROCESSOR = "imx6" %}
{% set RAM_CONF = machine.split('-')[-1] %}
{% set MACHINE = "seco-imx6qdl-" + machine.split('-')[0] %}
{% set DEFCONFIG_FILE = "seco_q7_928_edgehog" %}
{% set BUILD_DIRECTORY = "build_928" %}
{% elif '962' in machine %}
{% set PROCESSOR = "imx6" %}
{% set RAM_CONF = machine.split('-')[-1] %}
{% set MACHINE = "seco-imx6qdl-" + machine.split('-')[0] %}
{% set DEFCONFIG_FILE = "seco_uq7_962_edgehog" %}
{% set BUILD_DIRECTORY = "build_962" %}
{% elif 'a62' in machine %}
{% set PROCESSOR = "imx6" %}
{% set RAM_CONF = machine.split('-')[-1] %}
{% set MACHINE = "seco-imx6qdl-" + machine.split('-')[0] %}
{% set DEFCONFIG_FILE = "seco_sbc_a62_edgehog" %}
{% set BUILD_DIRECTORY = "build_a62" %}
{% elif 'a75' in machine %}
{% set PROCESSOR = "imx6" %}
{% set RAM_CONF = machine.split('-')[-1] %}
{% set MACHINE = "seco-imx6qdl-" + machine.split('-')[0] %}
{% set DEFCONFIG_FILE = "seco_uq7_a75_edgehog" %}
{% set BUILD_DIRECTORY = "build_a75" %}
{% elif 'b08' in machine %}
{% set PROCESSOR = "imx6" %}
{% set RAM_CONF = machine.split('-')[-1] %}
{% set MACHINE = "seco-imx6sx-" + machine.split('-')[0] %}
{% set DEFCONFIG_FILE = "seco_sbc_b08_edgehog" %}
{% set BUILD_DIRECTORY = "build_b08" %}
{% elif 'c23' in machine %}
{% set PROCESSOR = "imx6" %}
{% set RAM_CONF = machine.split('-')[-1] %}
{% set MACHINE = "seco-imx6sx-" + machine.split('-')[0] %}
{% set DEFCONFIG_FILE = "seco_sbc_c23_edgehog" %}
{% set BUILD_DIRECTORY = "build_c23" %}
{% elif 'c12' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mq-" + machine %}
{% set DEFCONFIG_FILE = "seco_smarc_c12_edgehog" %}
{% set BUILD_DIRECTORY = "build_c12" %}
{% elif 'c20' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mq-" + machine %}
{% set DEFCONFIG_FILE = "seco_sbc_c20_edgehog" %}
{% set BUILD_DIRECTORY = "build_c20" %}
{% elif 'c25' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mq-" + machine %}
{% set DEFCONFIG_FILE = "seco_q7_c25_edgehog" %}
{% set BUILD_DIRECTORY = "build_c25" %}
{% elif 'c26' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8qm-" + machine %}
{% set DEFCONFIG_FILE = "seco_q7_c26_edgehog" %}
{% set BUILD_DIRECTORY = "build_c26" %}
{% elif 'c31' in machine %}
{% set PROCESSOR = "rk3399" %}
{% set MACHINE = "seco-rk3399-" + machine %}
{% set DEFCONFIG_FILE = "seco_sbc_c31_edgehog" %}
{% set BUILD_DIRECTORY = "build_c31" %}
{% set KERNEL = "4-19-111" %}
{% elif 'c43' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8qm-" + machine %}
{% set DEFCONFIG_FILE = "seco_sbc_c43_edgehog" %}
{% set BUILD_DIRECTORY = "build_c43" %}
{% elif 'c57' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8x-" + machine %}
{% set DEFCONFIG_FILE = "seco_sbc_c57_edgehog" %}
{% set BUILD_DIRECTORY = "build_c57" %}
{% elif 'c58' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8x-" + machine %}
{% set DEFCONFIG_FILE = "seco_q7_c58_edgehog" %}
{% set BUILD_DIRECTORY = "build_c58" %}
{% elif 'c61' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mm-" + machine %}
{% set DEFCONFIG_FILE = "seco_sbc_c61_edgehog" %}
{% set BUILD_DIRECTORY = "build_c61" %}
{% elif 'c72n' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mn-" + machine %}
{% set DEFCONFIG_FILE = "seco_q7_c72n_edgehog" %}
{% set BUILD_DIRECTORY = "build_c72n" %}
{% elif 'c72m' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mm-" + machine %}
{% set DEFCONFIG_FILE = "seco_q7_c72m_edgehog" %}
{% set BUILD_DIRECTORY = "build_c72m" %}
{% elif 'd16' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8x-" + machine %}
{% set DEFCONFIG_FILE = "seco_smarc_d16_edgehog" %}
{% set BUILD_DIRECTORY = "build_d16" %}
{% elif 'd18' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mp-" + machine %}
{% set DEFCONFIG_FILE = "seco_smarc_d18_edgehog" %}
{% set BUILD_DIRECTORY = "build_d18" %}
{% elif 'd23' in machine %}
{% set PROCESSOR = "px30-" %}
{% set MACHINE = "seco-px30" + machine %}
{% set DEFCONFIG_FILE = "seco_sbc_d23_edgehog" %}
{% set BUILD_DIRECTORY = "build_d23" %}
{% elif 'tanaro' in machine %}
{% set PROCESSOR = "imx8" %}
{% set MACHINE = "seco-imx8mm" + machine %}
{% set DEFCONFIG_FILE = "seco_sbc_tanaro_edgehog" %}
{% set BUILD_DIRECTORY = "build_tanaro" %}
{% elif 'apollolake' in machine %}
{% set PROCESSOR = "intel" %}
{% set MACHINE = "seco-" + PROCESSOR + "-" + machine %}
{% set DEFCONFIG_FILE = "seco_intel_apl_edgehog" %}
{% set BUILD_DIRECTORY = "build_intel_apl" %}
{% elif 'tigerlake' in machine %}
{% set PROCESSOR = "intel" %}
{% set MACHINE = "seco-" + PROCESSOR + "-" + machine %}
{% set DEFCONFIG_FILE = "seco_intel_tgl_edgehog" %}
{% set BUILD_DIRECTORY = "build_intel_tgl" %}
{% elif 'elkhartlake' in machine %}
{% set PROCESSOR = "intel" %}
{% set MACHINE = "seco-" + PROCESSOR + "-" + machine %}
{% set DEFCONFIG_FILE = "seco_intel_ehl_edgehog" %}
{% set BUILD_DIRECTORY = "build_intel_ehl" %}
{% elif 'coffeelake' in machine %}
{% set PROCESSOR = "intel" %}
{% set MACHINE = "seco-" + PROCESSOR + "-" + machine %}
{% set DEFCONFIG_FILE = "seco_intel_cfl_edgehog" %}
{% set BUILD_DIRECTORY = "build_intel_cfl" %}
{% elif 'whiskeylake' in machine %}
{% set PROCESSOR = "intel" %}
{% set MACHINE = "seco-" + PROCESSOR + "-" + machine %}
{% set DEFCONFIG_FILE = "seco_intel_wkl_edgehog" %}
{% set BUILD_DIRECTORY = "build_intel_wkl" %}
{% elif 'raptorlake' in machine %}
{% set PROCESSOR = "intel" %}
{% set MACHINE = "seco-" + PROCESSOR + "-" + machine %}
{% set DEFCONFIG_FILE = "seco_intel_rpl_edgehog" %}
{% set BUILD_DIRECTORY = "build_intel_rpl" %}
{% endif %}
{% if PROCESSOR == 'imx6' %}
{% set UBOOT_FILE = "u-boot.imx" %}
{% set KERNEL_FILE = "zImage" %}
{% set AZURE_PATH = PROCESSOR + "/" + machine.split('-')[0] %}
{% elif PROCESSOR == 'imx8' %}
{% set UBOOT_FILE = "imx-boot" %}
{% set KERNEL_FILE = "Image" %}
{% set AZURE_PATH = PROCESSOR + "/" + machine %}
{% elif "rk" in PROCESSOR or "px" in PROCESSOR %}
{% set UBOOT_FILE = "uboot.img trust.img idblock.img" %}
{% set KERNEL_FILE = "Image" %}
{% set AZURE_PATH = "rockchip/" + machine %}
{% elif PROCESSOR == 'intel' or PROCESSOR == 'amd' %}
{% set UBOOT_FILE = "none" %}
{% set KERNEL_FILE = "none" %}
{% set AZURE_PATH = machine + "/Yocto" %}
{% endif %}
{% endif %}
{% for distro in DISTRO.split(' ')%}
{% if distro == 'things' %}
{% set recipe = "seco-image-edgehog" %}
{% set image = "edgehog-things-image" %}
#----------------------------------------------------
# Edgehog-build-{{ machine }}-{{ distro }}
#----------------------------------------------------
build-{{ machine }}-{{ distro }}:
extends: .build
variables: &build-edgehog-{{ machine }}-{{ distro }}
BOARD: {{ machine }}
{% if RAM_CONF is defined %}
RAM_CONF: {{ RAM_CONF }}
{% endif %}
PROCESSOR: {{ PROCESSOR }}
BUILD_DIRECTORY: {{ BUILD_DIRECTORY }}
MACHINE: {{ MACHINE }}
DEFCONFIG_FILE: {{ DEFCONFIG_FILE }}
RECIPE_NAME: {{ recipe }}
IMAGE_NAME: {{ image }}
{% if KERNEL is defined %}
KERNEL: {{ KERNEL }}
{% endif %}
UBOOT_FILE: {{ UBOOT_FILE }}
KERNEL_FILE: {{ KERNEL_FILE }}
AZURE_PATH: {{ AZURE_PATH }}
#----------------------------------------------------
# Edgehog-deploy-{{ machine }}-{{ distro }}
#----------------------------------------------------
deploy-{{ machine }}-{{ distro }}:
extends: .deploy
needs:
- job: build-edgehog-{{ machine }}-{{ distro }}
artifacts: true
variables: &deploy-edgehog-{{ machine }}-{{ distro }}
<<: *build-edgehog-{{ machine }}-{{ distro }}
{% elif distro == 'embedded-base' %}
{% set DEFCONFIG_FILE = DEFCONFIG_FILE + "_embedded_" %}
{% set BUILD_DIRECTORY = BUILD_DIRECTORY + "_embedded_" %}
{% set recipe = "seco-image-edgehog-base" %}
{% set image = "edgehog-embedded-base-image" %}
{% elif distro == 'embedded-full' %}
{% set DEFCONFIG_FILE = DEFCONFIG_FILE + "_embedded_" %}
{% set BUILD_DIRECTORY = BUILD_DIRECTORY + "_embedded_" %}
{% set recipe = "seco-image-edgehog-full" %}
{% set image = "edgehog-embedded-full-image" %}
{% set testing = true %}
{% endif %}
{% if distro != 'things' %}
{% for backend in GRAPHIC_BACKEND.split(' ')%}
{% set DEFCONFIG_FILE = DEFCONFIG_FILE + backend %}
{% set BUILD_DIRECTORY = BUILD_DIRECTORY + backend %}
{% set distro = distro + "-" + backend %}
#----------------------------------------------------
# Edgehog-build-{{ machine }}-{{ distro }}
#----------------------------------------------------
build-{{ machine }}-{{ distro }}:
extends: .build
variables: &build-edgehog-{{ machine }}-{{ distro }}
BOARD: {{ machine }}
{% if RAM_CONF is defined %}
RAM_CONF: {{ RAM_CONF }}
{% endif %}
PROCESSOR: {{ PROCESSOR }}
BUILD_DIRECTORY: {{ BUILD_DIRECTORY }}
MACHINE: {{ MACHINE }}
DEFCONFIG_FILE: {{ DEFCONFIG_FILE }}
RECIPE_NAME: {{ recipe }}
IMAGE_NAME: {{ image }}
UBOOT_FILE: {{ UBOOT_FILE }}
KERNEL_FILE: {{ KERNEL_FILE }}
AZURE_PATH: {{ AZURE_PATH }}
#----------------------------------------------------
# Edgehog-deploy-{{ machine }}-{{ distro }}
#----------------------------------------------------
deploy-{{ machine }}-{{ distro }}:
extends: .deploy
needs:
- job: build-edgehog-{{ machine }}-{{ distro }}
artifacts: true
variables: &deploy-edgehog-{{ machine }}-{{ distro }}
<<: *build-edgehog-{{ machine }}-{{ distro }}
{% if testing is defined %}
#----------------------------------------------------
# Edgehog-test-{{ machine }}-{{ distro }}
#----------------------------------------------------
test-{{ machine }}-{{ distro }}:
extends: .test
needs:
- job: deploy-edgehog-{{ machine }}-{{ distro }}
artifacts: true
variables:
<<: *build-edgehog-{{ machine }}-{{ distro }}
CI_PARAM_TEST_SUITE: '{{ machine }}.jinja2'
{% endif %}
{% endfor %}
{% endif %}
{% endfor %}
{% endfor %}
{% endif %}
\ No newline at end of file
build-pipeline.yml 0 → 100644
+ 364
− 0
View file @ b0ad83c8
include:
- local: 'manifest-pipeline.yml'
- local: 'common.yml'
workflow:
rules:
# This rule is needed, as otherwise the workflow:rules from
# the parent job seem to be used and prevent the pipeline generation
- if: $CI_PIPELINE_SOURCE == "parent_pipeline"
.initENV: &initENV
- NEWUS='secous'
- groupadd $NEWUS -g 1002
- /usr/bin/restrict_useradd.sh 1002 1002 $NEWUS
- groupadd edgehog -g 1023
- usermod -a -G edgehog $NEWUS
- chown -R :edgehog /var/cache/edgehog_*
- chsh -s /bin/bash $NEWUS
- su $NEWUS
.initSSH: &initSSH
- mkdir -p ~/.ssh
- echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
- echo ~/.ssh/known_hosts
- echo "$SSH_CONFIG_FILE" > ~/.ssh/config
- echo ~/.ssh/config
- echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
#-----------------------------------
# Stage: build
#-----------------------------------
.build:
extends:
- .infrastructure
- .full_build_pipeline
rules: &build_rule
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: "manual"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_TAG
stage: build
timeout: !reference [variables, BUILD_TIMEOUT]
image:
name: ${CI_IMAGE_YOCTO}
entrypoint: [""]
cache: {}
retry : 2
before_script:
- *initENV
script:
# init repository and generate configuration files for bitbake
# checkout
- BASEDIR=$PWD
- export EULA=1
- BUILD_BRANCH=${MASTER_BRANCH}
- |
if [ ${CI_PIPELINE_SOURCE} == "pipeline" ]; then
BUILD_BRANCH="${CI_COMMIT_REF_NAME}"
elif [ ${CI_PIPELINE_SOURCE} == "schedule" ]; then
BUILD_BRANCH="${CI_COMMIT_BRANCH}"
elif [ ! -z ${CI_COMMIT_TAG} ]; then
BUILD_BRANCH="refs/tags/${CI_COMMIT_TAG}"
fi
echo "This is the manifest repository URL used for the build: ${CI_REPOSITORY_URL}"
echo "This is the manifest branch used for the build: ${BUILD_BRANCH}"
echo "This is the board for which the build will be performed: ${BOARD}"
echo "This is the bitbake recipe we will be using for the build: ${RECIPE_NAME}"
echo "This is the image name will be assigned to software artifacts: ${IMAGE_NAME}"
- |
su secous -c "
sh .gitlab-ci/scripts/build_setup.sh $GITBOT_TOKEN $CI_REPOSITORY_URL $BUILD_BRANCH $DEFCONFIG_FILE $RECIPE_NAME
. ./seco-setup.sh -d ${DEFCONFIG_FILE};
. ./seco-setup.sh -c;
if [ -z "$CI_COMMIT_TAG" ]; then sh ../.gitlab-ci/scripts/yocto_cache_setup.sh $PROCESSOR $NUM_MAX_CPU $CUSTOM; fi;
if [ ! -z "$RAM_CONF" ]; then sed -i -e '/UBOOT_CONFIG =/d' conf/local.conf; echo 'UBOOT_CONFIG = \"$RAM_CONF\"' >> conf/local.conf; cat conf/local.conf; fi;
time bitbake ${RECIPE_NAME};
if [[ "$CI_JOB_NAME" != *"embedded"* ]]; then time bitbake seco-bundle-edgehog; fi;
. ../.gitlab-ci/scripts/check_build_artifacts.sh;
"
- echo "$BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE" > $BASEDIR/pathname.txt
after_script:
- |
echo -e "### **"$BOARD"_"$IMAGE_NAME"**" >> Job_build_message_"$BOARD"_"$IMAGE_NAME".txt
echo "##################################################################" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
echo ""$BOARD"_"$IMAGE_NAME" JOB STATUS" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
echo "##################################################################" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
if [ $CI_JOB_STATUS == "failed" ]; then
echo -e "$CI_JOB_NAME: \t\t FAILED" >> Job_build_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME: \t \t FAILED" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
else
echo -e "$CI_JOB_NAME: \t\t $CI_JOB_STATUS" >> Job_build_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME \t \t $CI_JOB_STATUS" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
fi
artifacts:
expire_in: 24 hrs
paths:
- $BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE/
- pathname.txt
- Job_build_report_${BOARD}_${IMAGE_NAME}.txt
- Job_build_message_${BOARD}_${IMAGE_NAME}.txt
allow_failure: true
#-----------------------------------
# Stage: deploy
#-----------------------------------
.deploy:
extends:
- .infrastructure
- .full_build_pipeline
image: mcr.microsoft.com/azure-cli
stage: deploy
cache: {}
retry : 2
timeout: 1h
tags:
- azure_deploy
rules:
- *build_rule
script:
- echo "Deploying packages..."
- echo "This is the board for which the build has been performed... ${BOARD}"
- echo "This is the recipe name associated to that build... ${RECIPE_NAME}"
- echo "This is the image name will be assigned to software artifacts... ${IMAGE_NAME}"
- BASE_DIRECTORY=$PWD
- ART_PATH=$(cat pathname.txt)
- DEPLOY_DATE=$(date +%Y%m%d)
- DEPLOY_PATH="${TAG_NAME}/week_$(date +%U)"
- |
if [ -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PRIVATE_CONTAINER_NAME}"
fi
if [ -n "$CI_COMMIT_TAG" ]; then
TAG_NAME=${CI_COMMIT_TAG}
DEPLOY_PATH=${TAG_NAME}
if [ ! -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PUBLIC_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PUBLIC_CONTAINER_NAME}"
fi
fi
- |
echo "Starting the deploy of software artifacts"
- |
echo "Deploying software artifacts to Azure.."
. .gitlab-ci/scripts/artifacts_name.sh $TAG_NAME $DEPLOY_DATE
echo $uboot_filename
echo $kernel_filename
echo $filesystem_name
echo $image_name
echo $bmap_name
echo $bundle_filename
- sh .gitlab-ci/scripts/deploy_artifacts.sh $BASE_DIRECTORY $ART_PATH $DEPLOY_PATH $AZURE_STORAGE_SAS_TOKEN $AZURE_CONTAINER_NAME $AZURE_PATH
after_script:
- |
mv Job_build_message_"$BOARD"_"$IMAGE_NAME".txt Job_message_"$BOARD"_"$IMAGE_NAME".txt
mv Job_build_report_"$BOARD"_"$IMAGE_NAME".txt Job_report_"$BOARD"_"$IMAGE_NAME".txt
if [ $CI_JOB_STATUS == "failed" ]; then
echo -e "$CI_JOB_NAME: \t\t FAILED" >> Job_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME: \t \t FAILED" >> Job_report_"$BOARD"_"$IMAGE_NAME".txt
else
echo -e "$CI_JOB_NAME: \t\t $CI_JOB_STATUS" >> Job_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME \t $CI_JOB_STATUS" >> Job_report_"$BOARD"_"$IMAGE_NAME".txt
fi
DEPLOY_PATH="${TAG_NAME}/week_$(date +%U)"
DEPLOY_DATE=$(date +%Y%m%d)
if [ -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PRIVATE_CONTAINER_NAME}"
fi
if [ -n "$CI_COMMIT_TAG" ]; then
TAG_NAME=${CI_COMMIT_TAG}
DEPLOY_PATH=${TAG_NAME}
AZURE_PATH="${AZURE_PATH}/edgehog"
if [ ! -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PUBLIC_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PUBLIC_CONTAINER_NAME}"
fi
fi
echo "##################################################################" >> Link_report_"$BOARD"_"$IMAGE_NAME".txt
echo ""$BOARD"_"$IMAGE_NAME" URLs" >> Link_report_"$BOARD"_"$IMAGE_NAME".txt
echo "##################################################################" >> Link_report_"$BOARD"_"$IMAGE_NAME".txt
- |
echo "Deploying software artifacts to Azure.."
. .gitlab-ci/scripts/artifacts_name.sh $TAG_NAME $DEPLOY_DATE
- sh .gitlab-ci/scripts/generate_report.sh $DEPLOY_PATH $AZURE_STORAGE_SAS_TOKEN $AZURE_CONTAINER_NAME $AZURE_PATH
allow_failure: true
artifacts:
paths:
- Job_report_${BOARD}_${IMAGE_NAME}.txt
- Job_message_${BOARD}_${IMAGE_NAME}.txt
- Link_report_${BOARD}_${IMAGE_NAME}.txt
#-----------------------------------
# Stage: changelog
#-----------------------------------
.changelog:
stage: changelog
extends: .infrastructure
timeout: 4h
tags:
- send_notify
allow_failure: true
rules:
- *build_rule
script:
- PROJECT_ARGS=""
- for project in ${CHANGELOG_PROJECTS}; do
PROJECT_ARGS="${PROJECT_ARGS} --project=$project";
done
- echo ${PROJECT_ARGS}
- .gitlab-ci/scripts/changelog_generator.py
--token=${GITBOT_TOKEN}
--branch=${MASTER_BRANCH}
--project=${CI_PROJECT_PATH}
${PROJECT_ARGS}
> changelog.md
artifacts:
expire_in: 4 weeks
paths:
- "changelog.md"
#-----------------------------------
# Stage: notify
#-----------------------------------
.notify:
stage: notify
timeout: 4h
image:
name: mcr.microsoft.com/azure-cli
entrypoint: [""]
tags:
- send_notify
allow_failure: true
rules: &schedule_tag_rule
# Trigger deploy jobs only on a schedule basis (for weekly release) or on tag
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_TAG
script:
- |
echo -e "The job results for all boards are listed below (also available by clicking on the **Job Report** button)." >> Job_message.txt
echo -e "The paths to all packages can be accessed by clicking on the **Link Report** button." >> Job_message.txt
echo -e "The changelog can be accessed by clicking on the **Changelog** button." >> Job_message.txt
echo -e "*We kindly ask developers to check job results so that in case of failures a fix to eventual bugs can be promptly made.*" >> Job_message.txt
AZURE_PATH="edgehog/reports"
if [ -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PRIVATE_CONTAINER_NAME}"
fi
if [ -n "$CI_COMMIT_TAG" ]; then
TAG_NAME=${CI_COMMIT_TAG}
if [ ! -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PUBLIC_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PUBLIC_CONTAINER_NAME}"
fi
else
TAG_NAME=$(date +%Y%m%d)
fi
- sh .gitlab-ci/scripts/notify_message.sh $AZURE_PATH $TAG_NAME $AZURE_STORAGE_SAS_TOKEN $AZURE_CONTAINER_NAME
artifacts:
expire_in: 4 weeks
paths:
- Link_report.txt
- changelog_link.txt
#-----------------------------------
# Stage: release
#-----------------------------------
.release:
# Caution, as of 2021-02-02 these assets links require a login, see:
# https://gitlab.com/gitlab-org/gitlab/-/issues/299384
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
rules:
- if: $CI_COMMIT_TAG
script:
- |
rm -rf release.sh
echo "#!/bin/bash" >> release.sh
echo "" >> release.sh
command='release-cli create --name Release-'$CI_COMMIT_TAG' --tag-name '$CI_COMMIT_TAG' --assets-link='"'"'['
while IFS= read -r line || [ -n "$line" ]; do
# Reading each line
if [[ ${line} == *https* ]]; then
url=$line
name=$(echo ${line} | sed 's/^.*seco_/seco_/')
command=${command}'{"name": "'${name}'", "url":"'${url}'"}, '
fi
done < Link_report.txt
url="$(cat changelog_link.txt)"
command=${command}'{"name": "changelog", "url":"'${url}'"}, '
command="${command%??}"
command=${command}']'"'"''
echo $command >> release.sh
cat release.sh
chmod +x release.sh
sh release.sh
#-----------------------------------
# Stage: test
#-----------------------------------
.test:
image: lava-submitter:v0.1
tags:
- lava-server
stage: test-probe
timeout: 4h
cache: {}
rules:
- *build_rule
artifacts:
when: always
paths:
- "results/**"
reports:
junit: results/results-*.xml
allow_failure: true
variables:
GIT_STRATEGY: none
before_script:
- *initSSH
- git clone git@git.seco.com:devops/lava-test-suite/lava-test.git
script:
- ls -la
- link_report=$(find . -iname link*report*"${BOARD}"*.txt)
- echo ${link_report}
- sed -i 's/\"//g' ${link_report}
- cat ${link_report}
- image_url=$(cat ${link_report} | grep 'seco.*'"${BOARD}"'.*image.*.wic.bz2'| grep -v latest)
- echo ${image_url}
- |
if [ -n "$CUSTOM" ]; then
image_url="${image_url}?${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
echo "{image_url}"
fi
- |
echo "Submitting LAVA job .."
python3 lava-test/scripts/submit_test.py \
--image-update True \
--image "${image_url}" \
--lavaserver-ip 10.197.197.10 \
--webserver-ip lavalab.seco.com \
--jobname ${CI_JOB_NAME}_${CI_PIPELINE_ID} \
"Gitlab build test ${CI_PIPELINE_ID}" \
--results-path "results" \
--test-repo ${TEST_REPO_URL} \
--test-repo-branch ${TEST_BRANCH} \
--test-plan ${CI_PARAM_TEST_SUITE}
\ No newline at end of file
manifest-pipeline-yocto.yml
+ 15
− 488
View file @ b0ad83c8
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- local: 'common.yml'
- local: 'boards/*'
rules:
- if: $TARGET == "generic"
- local: manifest-pipeline.yml
variables:
# The id of the gitlab project used in the rules section to not run pipelines in
# forked projects. Using variable here, to allow override in other projects including
......@@ -18,17 +11,12 @@ variables:
# In the manifest, the remotes are specified by an identifier. This is used to find
# out included projects for the retrigger job. In custom manifests, the remote may be
# named differently, so we need a variable that may be overriden.
CI_PARAM_SECO_REMOTE: edgehog
SECO_REMOTE: edgehog
# GitLab group to search for projects to retrigger
RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}
BUILD_TIMEOUT: 6h
IMAGES_PATH: "tmp/deploy/images"
TAG_NAME: weekly
KERNEL: 5-10-x
GIT_SSL_NO_VERIFY: 1
FS_EXTENSION: rootfs.tar.bz2
IMAGE_EXTENSION: wic.bz2
# Projects to include in the changelog in addition to the manifest project
CHANGELOG_PROJECTS:
edgehog/layers/seco/meta-seco-imx
edgehog/layers/seco/meta-seco-rk
......@@ -36,480 +24,19 @@ variables:
edgehog/layers/seco/meta-seco-edgehog-things
edgehog/layers/seco/meta-seco-edgehog-embedded
edgehog/layers/external/poky
BUILD_TIMEOUT: 6h
IMAGES_PATH: "tmp/deploy/images"
TAG_NAME: weekly
KERNEL: 5-10-x
GIT_SSL_NO_VERIFY: 1
FS_EXTENSION: rootfs.tar.bz2
IMAGE_EXTENSION: wic.bz2
#GitLab group containing LAVA tests
TEST_REPO_URL: "https://gitlab-ci-token:${GITBOT_TOKEN}@git.seco.com/devops/lava-test-suite/lava-test.git"
TEST_BRANCH: main
NUM_MAX_CPU: 32
stages:
# - manifest-pipeline
- retrigger
- build
- deploy
- changelog
- notify
- release
- test-probe
workflow:
rules:
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "api"
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_TAG
# Do not run pipelines for merge requests
- if: $CI_MERGE_REQUEST_IID
when: never
# Do not run pipelines on forked projects
# (use id instead of name because of rename)
# - if: $CI_PROJECT_ID != $MANIFEST_PROJECT_ID
# when: never
# Do not run pipelines on integration branches
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
when: never
# In all other cases, run the pipeline automatically
.full_build_pipeline:
rules:
# Run the full build pipeline on non-master branches (i.e. in the merge request)
# or if explicitly triggered by the API or the web button.
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH
- if: $CI_PIPELINE_SOURCE == "api"
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_TAG
# Trigger the pipeline on schedule basis (for weekly release)
- if: $CI_PIPELINE_SOURCE == "schedule"
stage: manifest-pipeline
.short_master_pipeline:
rules:
# The short master pipeline does not execute a full build, but only mirrors the
# build result from the merge request. Run it on the master branch per default if
# not explicitly triggered.
- if: $CI_PIPELINE_SOURCE == "api"
when: never
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web"
when: never
- if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH
# --------------------------------------------------------------------------------------
# Full build pipeline (runs in merge requests, and on master if manually triggered)
# --------------------------------------------------------------------------------------
#generate-build-jobs:
# extends:
# - .infrastructure
# - .full_build_pipeline
# script:
# # The job generation script implicitly passes the OS environment to the template, so
# # that the template has access to all GitLab CI variables. Hence there is no need
# # to explicitly pass any of them as command line arguments.
# - .gitlab-ci/scripts/generate_job_from_template.py
# --template=.gitlab-ci/${BUILD_JOBS_TEMPLATE}
# > build-jobs.yml
# artifacts:
# expire_in: 4 weeks
# paths:
# - build-jobs.yml
# build-jobs:
# extends:
# - .full_build_pipeline
# needs: ["generate-build-jobs"]
# trigger:
# include:
# - artifact: build-jobs.yml
# job: generate-build-jobs
# strategy: depend
#
# yamllint:
# extends:
# - .yamllint
# - .full_build_pipeline
# --------------------------------------------------------------------------------------
# Short master pipeline (runs on master after merging a merge request)
# --------------------------------------------------------------------------------------
retrigger:
yocto-pipeline:
extends:
- .infrastructure
- .full_build_pipeline
stage: retrigger
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "schedule"
when: never
script:
- .gitlab-ci/scripts/retrigger_integrating_projects.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${CI_PROJECT_PATH}
--manifest-branch=${MASTER_BRANCH}
--group=${RETRIGGER_GROUP}
#############################
########## BUILD ############
#############################
.initENV: &initENV
- NEWUS='secous'
- groupadd $NEWUS -g 1002
- /usr/bin/restrict_useradd.sh 1002 1002 $NEWUS
- groupadd edgehog -g 1023
- usermod -a -G edgehog $NEWUS
- chown -R :edgehog /var/cache/edgehog_*
- chsh -s /bin/bash $NEWUS
- su $NEWUS
.build:
extends:
- .infrastructure
- .full_build_pipeline
rules: &build_rule
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: "manual"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_TAG
stage: build
timeout: !reference [variables, BUILD_TIMEOUT]
image:
name: ${CI_IMAGE_YOCTO}
entrypoint: [""]
cache: {}
retry : 2
before_script:
- *initENV
script:
# init repository and generate configuration files for bitbake
# checkout
- BASEDIR=$PWD
- export EULA=1
- BUILD_BRANCH=${MASTER_BRANCH}
- |
if [ ${CI_PIPELINE_SOURCE} == "pipeline" ]; then
BUILD_BRANCH="${CI_COMMIT_REF_NAME}"
elif [ ${CI_PIPELINE_SOURCE} == "schedule" ]; then
BUILD_BRANCH="${CI_COMMIT_BRANCH}"
elif [ ! -z ${CI_COMMIT_TAG} ]; then
BUILD_BRANCH="refs/tags/${CI_COMMIT_TAG}"
fi
echo "This is the manifest repository URL used for the build: ${CI_REPOSITORY_URL}"
echo "This is the manifest branch used for the build: ${BUILD_BRANCH}"
echo "This is the board for which the build will be performed: ${BOARD}"
echo "This is the bitbake recipe we will be using for the build: ${RECIPE_NAME}"
echo "This is the image name will be assigned to software artifacts: ${IMAGE_NAME}"
- |
su secous -c "
sh .gitlab-ci/scripts/build_setup.sh $GITBOT_TOKEN $CI_REPOSITORY_URL $BUILD_BRANCH $DEFCONFIG_FILE $RECIPE_NAME
. ./seco-setup.sh -d ${DEFCONFIG_FILE};
. ./seco-setup.sh -c;
if [ -z "$CI_COMMIT_TAG" ]; then sh ../.gitlab-ci/scripts/yocto_cache_setup.sh $PROCESSOR $NUM_MAX_CPU $CUSTOM; fi;
if [ ! -z "$RAM_CONF" ]; then sed -i -e '/UBOOT_CONFIG =/d' conf/local.conf; echo 'UBOOT_CONFIG = \"$RAM_CONF\"' >> conf/local.conf; cat conf/local.conf; fi;
time bitbake ${RECIPE_NAME};
if [[ "$CI_JOB_NAME" != *"embedded"* ]]; then time bitbake seco-bundle-edgehog; fi;
. ../.gitlab-ci/scripts/check_build_artifacts.sh;
"
- echo "$BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE" > $BASEDIR/pathname.txt
after_script:
- |
echo -e "### **"$BOARD"_"$IMAGE_NAME"**" >> Job_build_message_"$BOARD"_"$IMAGE_NAME".txt
echo "##################################################################" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
echo ""$BOARD"_"$IMAGE_NAME" JOB STATUS" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
echo "##################################################################" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
if [ $CI_JOB_STATUS == "failed" ]; then
echo -e "$CI_JOB_NAME: \t\t FAILED" >> Job_build_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME: \t \t FAILED" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
else
echo -e "$CI_JOB_NAME: \t\t $CI_JOB_STATUS" >> Job_build_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME \t \t $CI_JOB_STATUS" >> Job_build_report_"$BOARD"_"$IMAGE_NAME".txt
fi
artifacts:
expire_in: 24 hrs
paths:
- $BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE/
- pathname.txt
- Job_build_report_${BOARD}_${IMAGE_NAME}.txt
- Job_build_message_${BOARD}_${IMAGE_NAME}.txt
allow_failure: true
#############################
########## DEPLOY ###########
#############################
.deploy:
extends:
- .infrastructure
- .full_build_pipeline
image: mcr.microsoft.com/azure-cli
stage: deploy
cache: {}
retry : 2
timeout: 1h
tags:
- azure_deploy
rules:
- *build_rule
script:
- echo "Deploying packages..."
- echo "This is the board for which the build has been performed... ${BOARD}"
- echo "This is the recipe name associated to that build... ${RECIPE_NAME}"
- echo "This is the image name will be assigned to software artifacts... ${IMAGE_NAME}"
- BASE_DIRECTORY=$PWD
- ART_PATH=$(cat pathname.txt)
- DEPLOY_DATE=$(date +%Y%m%d)
- DEPLOY_PATH="${TAG_NAME}/week_$(date +%U)"
- |
if [ -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PRIVATE_CONTAINER_NAME}"
fi
if [ -n "$CI_COMMIT_TAG" ]; then
TAG_NAME=${CI_COMMIT_TAG}
DEPLOY_PATH=${TAG_NAME}
if [ ! -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PUBLIC_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PUBLIC_CONTAINER_NAME}"
fi
fi
- |
echo "Starting the deploy of software artifacts"
- |
echo "Deploying software artifacts to Azure.."
. .gitlab-ci/scripts/artifacts_name.sh $TAG_NAME $DEPLOY_DATE
echo $uboot_filename
echo $kernel_filename
echo $filesystem_name
echo $image_name
echo $bmap_name
echo $bundle_filename
- sh .gitlab-ci/scripts/deploy_artifacts.sh $BASE_DIRECTORY $ART_PATH $DEPLOY_PATH $AZURE_STORAGE_SAS_TOKEN $AZURE_CONTAINER_NAME $AZURE_PATH
after_script:
- |
mv Job_build_message_"$BOARD"_"$IMAGE_NAME".txt Job_message_"$BOARD"_"$IMAGE_NAME".txt
mv Job_build_report_"$BOARD"_"$IMAGE_NAME".txt Job_report_"$BOARD"_"$IMAGE_NAME".txt
if [ $CI_JOB_STATUS == "failed" ]; then
echo -e "$CI_JOB_NAME: \t\t FAILED" >> Job_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME: \t \t FAILED" >> Job_report_"$BOARD"_"$IMAGE_NAME".txt
else
echo -e "$CI_JOB_NAME: \t\t $CI_JOB_STATUS" >> Job_message_"$BOARD"_"$IMAGE_NAME".txt
echo -e "$CI_JOB_NAME \t $CI_JOB_STATUS" >> Job_report_"$BOARD"_"$IMAGE_NAME".txt
fi
DEPLOY_PATH="${TAG_NAME}/week_$(date +%U)"
DEPLOY_DATE=$(date +%Y%m%d)
if [ -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PRIVATE_CONTAINER_NAME}"
fi
if [ -n "$CI_COMMIT_TAG" ]; then
TAG_NAME=${CI_COMMIT_TAG}
DEPLOY_PATH=${TAG_NAME}
AZURE_PATH="${AZURE_PATH}/edgehog"
if [ ! -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PUBLIC_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PUBLIC_CONTAINER_NAME}"
fi
fi
echo "##################################################################" >> Link_report_"$BOARD"_"$IMAGE_NAME".txt
echo ""$BOARD"_"$IMAGE_NAME" URLs" >> Link_report_"$BOARD"_"$IMAGE_NAME".txt
echo "##################################################################" >> Link_report_"$BOARD"_"$IMAGE_NAME".txt
- |
echo "Deploying software artifacts to Azure.."
. .gitlab-ci/scripts/artifacts_name.sh $TAG_NAME $DEPLOY_DATE
- sh .gitlab-ci/scripts/generate_report.sh $DEPLOY_PATH $AZURE_STORAGE_SAS_TOKEN $AZURE_CONTAINER_NAME $AZURE_PATH
allow_failure: true
artifacts:
paths:
- Job_report_${BOARD}_${IMAGE_NAME}.txt
- Job_message_${BOARD}_${IMAGE_NAME}.txt
- Link_report_${BOARD}_${IMAGE_NAME}.txt
#############################
####### CHANGE LOG ##########
#############################
changelog:
stage: changelog
extends: .infrastructure
timeout: 4h
tags:
- send_notify
allow_failure: true
rules:
- *build_rule
script:
- PROJECT_ARGS=""
- for project in ${CHANGELOG_PROJECTS}; do
PROJECT_ARGS="${PROJECT_ARGS} --project=$project";
done
- echo ${PROJECT_ARGS}
- .gitlab-ci/scripts/changelog_generator.py
--token=${GITBOT_TOKEN}
--branch=${MASTER_BRANCH}
--project=${CI_PROJECT_PATH}
${PROJECT_ARGS}
> changelog.md
artifacts:
expire_in: 4 weeks
paths:
- "changelog.md"
#############################
########## NOTIFY ###########
#############################
notify:
stage: notify
timeout: 4h
image:
name: mcr.microsoft.com/azure-cli
entrypoint: [""]
tags:
- send_notify
allow_failure: true
rules: &schedule_tag_rule
# Trigger deploy jobs only on a schedule basis (for weekly release) or on tag
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_TAG
script:
- |
echo -e "The job results for all boards are listed below (also available by clicking on the **Job Report** button)." >> Job_message.txt
echo -e "The paths to all packages can be accessed by clicking on the **Link Report** button." >> Job_message.txt
echo -e "The changelog can be accessed by clicking on the **Changelog** button." >> Job_message.txt
echo -e "*We kindly ask developers to check job results so that in case of failures a fix to eventual bugs can be promptly made.*" >> Job_message.txt
AZURE_PATH="edgehog/reports"
if [ -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PRIVATE_CONTAINER_NAME}"
fi
if [ -n "$CI_COMMIT_TAG" ]; then
TAG_NAME=${CI_COMMIT_TAG}
if [ ! -n "$CUSTOM" ]; then
AZURE_STORAGE_SAS_TOKEN="${AZURE_STORAGE_PUBLIC_SAS_TOKEN}"
AZURE_CONTAINER_NAME="${AZURE_PUBLIC_CONTAINER_NAME}"
fi
else
TAG_NAME=$(date +%Y%m%d)
fi
- sh .gitlab-ci/scripts/notify_message.sh $AZURE_PATH $TAG_NAME $AZURE_STORAGE_SAS_TOKEN $AZURE_CONTAINER_NAME
artifacts:
expire_in: 4 weeks
paths:
- Link_report.txt
- changelog_link.txt
#############################
######### RELEASE ###########
#############################
release:
# Caution, as of 2021-02-02 these assets links require a login, see:
# https://gitlab.com/gitlab-org/gitlab/-/issues/299384
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
rules:
- if: $CI_COMMIT_TAG
script:
- |
rm -rf release.sh
echo "#!/bin/bash" >> release.sh
echo "" >> release.sh
command='release-cli create --name Release-'$CI_COMMIT_TAG' --tag-name '$CI_COMMIT_TAG' --assets-link='"'"'['
while IFS= read -r line || [ -n "$line" ]; do
# Reading each line
if [[ ${line} == *https* ]]; then
url=$line
name=$(echo ${line} | sed 's/^.*seco_/seco_/')
command=${command}'{"name": "'${name}'", "url":"'${url}'"}, '
fi
done < Link_report.txt
url="$(cat changelog_link.txt)"
command=${command}'{"name": "changelog", "url":"'${url}'"}, '
command="${command%??}"
command=${command}']'"'"''
echo $command >> release.sh
cat release.sh
chmod +x release.sh
sh release.sh
##########################
######### TEST ###########
##########################
.initSSH: &initSSH
- mkdir -p ~/.ssh
- echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
- echo ~/.ssh/known_hosts
- echo "$SSH_CONFIG_FILE" > ~/.ssh/config
- echo ~/.ssh/config
- echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
.test:
image: lava-submitter:v0.1
tags:
- lava-server
stage: test-probe
timeout: 4h
cache: {}
rules:
- *build_rule
artifacts:
when: always
paths:
- "results/**"
reports:
junit: results/results-*.xml
allow_failure: true
variables:
GIT_STRATEGY: none
before_script:
- *initSSH
- git clone git@git.seco.com:devops/lava-test-suite/lava-test.git
script:
- ls -la
- link_report=$(find . -iname link*report*"${BOARD}"*.txt)
- echo ${link_report}
- sed -i 's/\"//g' ${link_report}
- cat ${link_report}
- image_url=$(cat ${link_report} | grep 'seco.*'"${BOARD}"'.*image.*.wic.bz2'| grep -v latest)
- echo ${image_url}
- |
if [ -n "$CUSTOM" ]; then
image_url="${image_url}?${AZURE_STORAGE_PRIVATE_SAS_TOKEN}"
echo "{image_url}"
fi
- |
echo "Submitting LAVA job .."
python3 lava-test/scripts/submit_test.py \
--image-update True \
--image "${image_url}" \
--lavaserver-ip 10.197.197.10 \
--webserver-ip lavalab.seco.com \
--jobname ${CI_JOB_NAME}_${CI_PIPELINE_ID} \
"Gitlab build test ${CI_PIPELINE_ID}" \
--results-path "results" \
--test-repo ${TEST_REPO_URL} \
--test-repo-branch ${TEST_BRANCH} \
--test-plan ${CI_PARAM_TEST_SUITE}
\ No newline at end of file
- .build-pipeline
\ No newline at end of file
manifest-pipeline.yml
+ 30
− 42
View file @ b0ad83c8
......@@ -2,14 +2,14 @@
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- local: common.yml
- local: common.yml
stages:
- manifest-pipeline
- trigger
- retrigger
- build
workflow:
rules:
......@@ -17,32 +17,33 @@ workflow:
- if: $CI_PIPELINE_SOURCE == "api"
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_TAG
# Do not run pipelines for merge requests
- if: $CI_MERGE_REQUEST_IID
when: never
# Do not run pipelines on forked projects
# (use id instead of name because of rename)
- if: $CI_PROJECT_ID != $MANIFEST_PROJECT_ID
when: never
# - if: $CI_PROJECT_ID != $MANIFEST_PROJECT_ID
# when: never
# Do not run pipelines on integration branches
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
when: never
# Do not run pipeline if the "skip build" label is set on the merge request
- if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
when: never
# Run pipelines for merge requests
- if: $CI_MERGE_REQUEST_IID
# Run pipelines on the master branch
- if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH
# Run pipelines on tags
- if: $CI_COMMIT_TAG
# In all other cases, run the pipeline automatically
.full_build_pipeline:
rules:
# Run the full build pipeline on merge requests and tags
# Run the full build pipeline on non-master branches (i.e. in the merge request)
# or if explicitly triggered by the API or the web button.
- if: $CI_MERGE_REQUEST_IID
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH
- if: $CI_PIPELINE_SOURCE == "api"
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "web"
- if: $CI_COMMIT_TAG
# Trigger the pipeline on schedule basis (for weekly release)
- if: $CI_PIPELINE_SOURCE == "schedule"
stage: manifest-pipeline
.short_master_pipeline:
rules:
......@@ -56,7 +57,7 @@ workflow:
- if: $CI_PIPELINE_SOURCE == "web"
when: never
- if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH
# --------------------------------------------------------------------------------------
# Full build pipeline (runs in merge requests, and on master if manually triggered)
# --------------------------------------------------------------------------------------
......@@ -92,42 +93,29 @@ generate-build-pipeline:
job: generate-build-pipeline
strategy: depend
yamllint:
extends:
- .yamllint
- .full_build_pipeline
stage: manifest-pipeline
# yamllint:
# extends:
# - .yamllint
# - .full_build_pipeline
# stage: manifest-pipeline
# --------------------------------------------------------------------------------------
# Short master pipeline (runs on master after merging a merge request)
# --------------------------------------------------------------------------------------
retrigger:
extends:
- .infrastructure
- .short_master_pipeline
- .full_build_pipeline
stage: retrigger
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
- if: $CI_PIPELINE_SOURCE == "schedule"
when: never
script:
- .gitlab-ci/scripts/retrigger_integrating_projects.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${CI_PROJECT_PATH}
--manifest-branch=${MASTER_BRANCH}
--group=${RETRIGGER_GROUP}
build:merge_request:
extends:
- .infrastructure
- .short_master_pipeline
stage: build
# For some reason we cannot use a variable for the timeout, the CI linter reports
# 'jobs:build:merge_request:timeout config should be a duration' then.
# Hence we use GitLab's 'reference' syntax instead.
timeout: !reference [variables, BUILD_TIMEOUT]
script:
- cd ${CI_PROJECT_DIR}
- .gitlab-ci/scripts/mirror_mr_pipeline.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--commit=${CI_COMMIT_SHA}
--ref=${MASTER_BRANCH}
--group=${RETRIGGER_GROUP}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment