From e9b7c225e4eb0669cb7f88c8605d6a547e4f35bc Mon Sep 17 00:00:00 2001
From: Lorenzo Pagliai <lorenzo.pagliai@seco.com>
Date: Tue, 15 Nov 2022 09:10:01 +0100
Subject: [PATCH] Modify manifest pipeline to match Edgehog one
---
common.yml | 1 +
manifest-integration.yml | 2 +-
manifest-pipeline-yocto-original.yml | 38 +++++
manifest-pipeline-yocto.yml | 235 ++++++++++++++++++++++++---
4 files changed, 254 insertions(+), 22 deletions(-)
create mode 100644 manifest-pipeline-yocto-original.yml
diff --git a/common.yml b/common.yml
index 288d222..04cb7ff 100644
--- a/common.yml
+++ b/common.yml
@@ -27,6 +27,7 @@ variables:
variables:
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
+ CI_COMMIT_SOURCE: ${CI_COMMIT_SHA}
.yamllint:
extends: .infrastructure
diff --git a/manifest-integration.yml b/manifest-integration.yml
index 4f4c6da..be60aad 100644
--- a/manifest-integration.yml
+++ b/manifest-integration.yml
@@ -61,6 +61,7 @@ integrate:
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
+ - echo "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
- python3 scripts/integrate_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
@@ -101,7 +102,6 @@ merge:
--commit=${CI_COMMIT_SOURCE}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
- - echo "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
artifacts:
paths:
- manifest_revision
diff --git a/manifest-pipeline-yocto-original.yml b/manifest-pipeline-yocto-original.yml
new file mode 100644
index 0000000..631ebd9
--- /dev/null
+++ b/manifest-pipeline-yocto-original.yml
@@ -0,0 +1,38 @@
+---
+include:
+ - local: manifest-pipeline.yml
+
+variables:
+ # The id of the gitlab project used in the rules section to not run pipelines in
+ # forked projects. Using variable here, to allow override in other projects including
+ # this file.
+ MANIFEST_PROJECT_ID: 1725
+
+ # In the manifest, the remotes are specified by an identifier. This is used to find
+ # out included projects for the retrigger job. In custom manifests, the remote may be
+ # named differently, so we need a variable that may be overriden.
+ CI_PARAM_SECO_REMOTE: seco-ne
+
+ # GitLab group to search for projects to retrigger
+ RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}
+
+ BUILD_TIMEOUT: 1h
+
+ # This is the jinja2 template file used to generate the build jobs
+ BUILD_JOBS_TEMPLATE: build-jobs-yocto.yml.jinja2
+
+generate-build-jobs:
+ variables:
+ # Default image and distro
+ CI_PARAM_IMAGE: guf-image
+ CI_PARAM_DISTRO: guf-wayland
+
+ # Flash-N-Go image and distro
+ # In the past, the buildfng job overwrote the image and distro itself. Due to the
+ # transition to the new seconorth names, image and distro for the buildfng must be
+ # settable from outside of the job.
+ CI_PARAM_IMAGE_FNG: fngsystem-image
+ CI_PARAM_DISTRO_FNG: guf-fngsystem
+
+ # List of machines to build images for
+ CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml
index 631ebd9..ac945a7 100644
--- a/manifest-pipeline-yocto.yml
+++ b/manifest-pipeline-yocto.yml
@@ -1,38 +1,231 @@
---
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+
include:
- - local: manifest-pipeline.yml
+ - local: common.yml
variables:
# The id of the gitlab project used in the rules section to not run pipelines in
# forked projects. Using variable here, to allow override in other projects including
# this file.
- MANIFEST_PROJECT_ID: 1725
+ MANIFEST_PROJECT_ID: 2307
# In the manifest, the remotes are specified by an identifier. This is used to find
# out included projects for the retrigger job. In custom manifests, the remote may be
# named differently, so we need a variable that may be overriden.
- CI_PARAM_SECO_REMOTE: seco-ne
+ CI_PARAM_SECO_REMOTE: edgehog
# GitLab group to search for projects to retrigger
RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}
- BUILD_TIMEOUT: 1h
+ BUILD_TIMEOUT: 4h
# This is the jinja2 template file used to generate the build jobs
- BUILD_JOBS_TEMPLATE: build-jobs-yocto.yml.jinja2
-
-generate-build-jobs:
- variables:
- # Default image and distro
- CI_PARAM_IMAGE: guf-image
- CI_PARAM_DISTRO: guf-wayland
-
- # Flash-N-Go image and distro
- # In the past, the buildfng job overwrote the image and distro itself. Due to the
- # transition to the new seconorth names, image and distro for the buildfng must be
- # settable from outside of the job.
- CI_PARAM_IMAGE_FNG: fngsystem-image
- CI_PARAM_DISTRO_FNG: guf-fngsystem
-
- # List of machines to build images for
- CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
+ #BUILD_JOBS_TEMPLATE: build-jobs-yocto.yml.jinja2
+
+stages:
+# - manifest-pipeline
+ - retrigger
+ - build
+
+workflow:
+ rules:
+ # Explicitly allow externally triggered pipelines in every case
+ - if: $CI_PIPELINE_SOURCE == "api"
+ - if: $CI_PIPELINE_SOURCE == "pipeline"
+ - if: $CI_PIPELINE_SOURCE == "web"
+ # Do not run pipelines for merge requests
+ - if: $CI_MERGE_REQUEST_IID
+ when: never
+ # Do not run pipelines on forked projects
+ # (use id instead of name because of rename)
+ - if: $CI_PROJECT_ID != $MANIFEST_PROJECT_ID
+ when: never
+ # Do not run pipelines on integration branches
+ - if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
+ when: never
+ # In all other cases, run the pipeline automatically
+ - when: always
+
+.full_build_pipeline:
+ rules:
+ # Run the full build pipeline on non-master branches (i.e. in the merge request)
+ # or if explicitly triggered by the API or the web button.
+ - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH
+ - if: $CI_PIPELINE_SOURCE == "api"
+ - if: $CI_PIPELINE_SOURCE == "pipeline"
+ - if: $CI_PIPELINE_SOURCE == "web"
+ stage: manifest-pipeline
+
+.short_master_pipeline:
+ rules:
+ # The short master pipeline does not execute a full build, but only mirrors the
+ # build result from the merge request. Run it on the master branch per default if
+ # not explicitly triggered.
+ - if: $CI_PIPELINE_SOURCE == "api"
+ when: never
+ - if: $CI_PIPELINE_SOURCE == "pipeline"
+ when: never
+ - if: $CI_PIPELINE_SOURCE == "web"
+ when: never
+ - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH
+
+# --------------------------------------------------------------------------------------
+# Full build pipeline (runs in merge requests, and on master if manually triggered)
+# --------------------------------------------------------------------------------------
+#generate-build-jobs:
+# extends:
+# - .infrastructure
+# - .full_build_pipeline
+# script:
+# # The job generation script implicitly passes the OS environment to the template, so
+# # that the template has access to all GitLab CI variables. Hence there is no need
+# # to explicitly pass any of them as command line arguments.
+# - .gitlab-ci/scripts/generate_job_from_template.py
+# --template=.gitlab-ci/${BUILD_JOBS_TEMPLATE}
+# > build-jobs.yml
+# artifacts:
+# expire_in: 4 weeks
+# paths:
+# - build-jobs.yml
+
+# build-jobs:
+# extends:
+# - .full_build_pipeline
+# needs: ["generate-build-jobs"]
+# trigger:
+# include:
+# - artifact: build-jobs.yml
+# job: generate-build-jobs
+# strategy: depend
+#
+# yamllint:
+# extends:
+# - .yamllint
+# - .full_build_pipeline
+
+# --------------------------------------------------------------------------------------
+# Short master pipeline (runs on master after merging a merge request)
+# --------------------------------------------------------------------------------------
+retrigger:
+ extends:
+ - .infrastructure
+ - .short_master_pipeline
+ stage: retrigger
+ script:
+ - .gitlab-ci/scripts/retrigger_integrating_projects.py
+ --gitlab-url=${CI_SERVER_URL}
+ --token=${GITBOT_TOKEN}
+ --manifest-project=${CI_PROJECT_PATH}
+ --manifest-branch=${MASTER_BRANCH}
+ --group=${RETRIGGER_GROUP}
+
+.build:
+ extends:
+ - .infrastructure
+ - .short_master_pipeline
+ stage: build
+ image:
+ name: secodocker/edgehog-builder:v1.1
+ entrypoint: [""]
+ cache: {}
+ retry : 2
+ before_script:
+ - NEWUS='secous'; groupadd $NEWUS -g 1002; /usr/bin/restrict_useradd.sh 1002 1002 $NEWUS; chsh -s /bin/bash $NEWUS; su $NEWUS
+
+ script:
+ # init repository and generate configuration files for bitbake
+ # checkout
+ - BASEDIR=$PWD
+ - git config --global url.https://github.com/.insteadOf git://github.com/
+ - mkdir -p /home/secous/.ssh
+ - echo "$SSH_KNOWN_HOSTS" > /home/secous/.ssh/known_hosts
+ - echo "$SSH_CONFIG_FILE" > /home/secous/.ssh/config
+ - echo "$SSH_PRIVATE_KEY" > /home/secous/.ssh/id_rsa
+ - chown 1002:1002 /home/secous/.ssh/known_hosts
+ - chown 1002:1002 /home/secous/.ssh/config
+ - chown 1002:1002 /home/secous/.ssh/id_rsa
+ - chmod 600 /home/secous/.ssh/id_rsa
+ - ls -l /home/secous/.ssh/
+ - eval $(ssh-agent -s)
+ - ssh-add /home/secous/.ssh/id_rsa
+ - export EULA=1
+ - |
+ su secous -c "
+ repo init -u $CI_REPOSITORY_URL -b kirkstone/develop;
+ repo sync -j$(nproc) --fetch-submodules;
+ . ./seco-setup.sh -d $DEFCONFIG_FILE;
+ . ./seco-setup.sh -c;
+ time bitbake $RECIPE_NAME;
+ time bitbake seco-bundle-edgehog;
+ "
+ - ls -la
+ - pwd
+ - echo "$BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE"
+ - ls -la "$BUILD_DIRECTORY"
+ - ls -la "$BUILD_DIRECTORY/$IMAGES_PATH/"
+ - ls -la "$BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE"
+ - echo "$BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE" > $BASEDIR/pathname.txt
+
+ after_script:
+ - |
+ echo -e "### **$BOARD**" >> Job_message_$BOARD.txt
+ echo "##################################################################" >> Job_report_$BOARD.txt
+ echo "$BOARD JOB STATUS" >> Job_report_$BOARD.txt
+ echo "##################################################################" >> Job_report_$BOARD.txt
+ if [ $CI_JOB_STATUS == "failed" ]; then
+ echo -e "$CI_JOB_NAME: \t\t FAILED" >> Job_message_$BOARD.txt
+ echo -e "$CI_JOB_NAME: \t \t FAILED" >> Job_report_$BOARD.txt
+ else
+ echo -e "$CI_JOB_NAME: \t\t $CI_JOB_STATUS" >> Job_message_$BOARD.txt
+ echo -e "$CI_JOB_NAME \t \t $CI_JOB_STATUS" >> Job_report_$BOARD.txt
+ fi
+
+ artifacts:
+ paths:
+ - $BUILD_DIRECTORY/$IMAGES_PATH/$MACHINE/
+ - pathname.txt
+ - Job_report_$BOARD.txt
+ - Job_message_$BOARD.txt
+
+build:merge_request:
+ extends: .build
+ variables: &build-edgehog-a62
+ BOARD: a62-1G-4x256M
+ PROCESSOR: imx6
+ BUILD_DIRECTORY: build_a62
+ MACHINE: seco-imx6qdl-a62
+ DEFCONFIG_FILE: seco_sbc_a62_edgehog
+
+ # For some reason we cannot use a variable for the timeout, the CI linter reports
+ # 'jobs:build:merge_request:timeout config should be a duration' then.
+ # Hence we use GitLab's 'reference' syntax instead.
+ # timeout: !reference [variables, BUILD_TIMEOUT]
+ # script:
+ # - cd ${CI_PROJECT_DIR}
+ # # Get pipeline for merge request
+ # - MR_PIPELINE=$(.gitlab-ci/scripts/get_pipelines.py
+ # --gitlab-url=${CI_SERVER_URL}
+ # --token=${GITBOT_TOKEN}
+ # --project=${CI_PROJECT_PATH}
+ # --commit=${CI_COMMIT_SHA}
+ # --ref=^${MASTER_BRANCH} || true | head -1)
+ # # If pipeline exists, mirror its result
+ # - if [ ! -z "${MR_PIPELINE}" ]; then
+ # .gitlab-ci/scripts/mirror_pipeline_result.py
+ # --gitlab-url=${CI_SERVER_URL}
+ # --token=${GITBOT_TOKEN}
+ # --project=${CI_PROJECT_PATH}
+ # --pipeline=${MR_PIPELINE}
+ # # If no pipeline found, trigger a new one on the master
+ # - else
+ # .gitlab-ci/scripts/trigger_pipeline.py
+ # --gitlab-url=${CI_SERVER_URL}
+ # --token=${GITBOT_TOKEN}
+ # --project=${CI_PROJECT_PATH}
+ # --ref=${MASTER_BRANCH}
+ # - fi
+
+
--
GitLab