From 0f7ef6cd630b3077a01e4452116415740dea6d81 Mon Sep 17 00:00:00 2001
From: Lorenzo Pagliai <lorenzo.pagliai@seco.com>
Date: Tue, 12 Nov 2024 15:27:12 +0100
Subject: [PATCH] [PRE-BUILD] Introduce common preliminary job for repo init
and sync
* Right now each build job perform the same action of initialising repo.
This approach is time and resource consuming, so we introduce a
'repo_checkout' job to perform repo init and sync actions.
* The output of this job is stored as GitLab CI cache and pulled by each
build job in the build stage.
* Minor fixes in build setup configuration.
---
build-pipeline-yocto.yml.jinja2 | 18 +++------
build-pipeline.yml | 71 +++++++++++++++++++++++++--------
scripts/build_setup.sh | 4 +-
3 files changed, 62 insertions(+), 31 deletions(-)
diff --git a/build-pipeline-yocto.yml.jinja2 b/build-pipeline-yocto.yml.jinja2
index 9a73121..c45fcc0 100644
--- a/build-pipeline-yocto.yml.jinja2
+++ b/build-pipeline-yocto.yml.jinja2
@@ -15,6 +15,7 @@ include:
# Stages
#------------
stages:
+ - pre-build
- build
- deploy
- changelog
@@ -28,18 +29,11 @@ stages:
variables:
MASTER_BRANCH: {{ MASTER_BRANCH }}
-build:pass:
- stage: build
- tags:
- - infrastructure
- needs: []
- timeout: 2m
- image: {{ CI_IMAGE_PYTHON }}
- script:
- - printenv
- - echo "Build successful"
- - echo "This is the value of the MANUAL_BUILD variable '${MANUAL_BUILD}'"
-
+# --------------------------------------------------------------------------------------
+# Stage: pre-build
+# --------------------------------------------------------------------------------------
+repo_checkout:
+ extends: .repo_checkout
# --------------------------------------------------------------------------------------
# Stage: changelog
diff --git a/build-pipeline.yml b/build-pipeline.yml
index f32001e..fcd6195 100644
--- a/build-pipeline.yml
+++ b/build-pipeline.yml
@@ -14,6 +14,16 @@ workflow:
- chown -R :clea-os /var/cache/clea-os_*
- chsh -s /bin/bash $NEWUS
- su $NEWUS
+ - export BASEDIR=$PWD
+ - export EULA=1
+ - BUILD_BRANCH=${MASTER_BRANCH}
+ - |
+ if [ ${CI_PIPELINE_SOURCE} == "parent_pipeline" ]; then
+ BUILD_BRANCH="${CI_COMMIT_REF_NAME}"
+ fi
+ if [ ! -z ${CI_COMMIT_TAG} ]; then
+ BUILD_BRANCH="refs/tags/${CI_COMMIT_TAG}"
+ fi
.initSSH: &initSSH
- mkdir -p ~/.ssh
@@ -25,15 +35,50 @@ workflow:
- chmod 600 ~/.ssh/id_rsa
# -----------------------------------
-# Stage: build
+# Stage: pre-build
# -----------------------------------
-.build:
+
+# Checkout the repository and store it in the cache to use cached data
+# instead of fetching it from Git for each build job.
+.repo_checkout:
extends:
- .infrastructure
rules: &build_rule
- if: $MANUAL_BUILD == "true"
when: manual
- when: always
+ stage: pre-build
+ tags:
+ - pre-yocto_build
+ image:
+ name: ${CI_IMAGE_YOCTO}
+ entrypoint: [""]
+ before_script:
+ - *initENV
+ script:
+ - echo "Perform repo checkout"
+ - cd ${CI_PROJECT_DIR}
+ - |
+ su secous -c "
+ mkdir repo_checkout
+ cd repo_checkout
+ sh ../.gitlab-ci/scripts/build_setup.sh $GITBOT_TOKEN $CI_REPOSITORY_URL $BUILD_BRANCH
+ "
+ cache:
+ - key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
+ policy: push
+ paths:
+ - ${CI_PROJECT_DIR}/repo_checkout
+ allow_failure: true
+
+# -----------------------------------
+# Stage: build
+# -----------------------------------
+.build:
+ extends:
+ - .infrastructure
+ rules:
+ - *build_rule
stage: build
timeout: 6h
tags:
@@ -41,24 +86,18 @@ workflow:
image:
name: ${CI_IMAGE_YOCTO}
entrypoint: [""]
- cache: {}
+ cache: &cache
+ # Get prepared repo from cache
+ - key: ${CI_PIPELINE_ID}-repo_checkout
+ policy: pull
+ paths:
+ - ${CI_PROJECT_DIR}/repo_checkout
+ needs: ["repo_checkout"]
retry: 2
before_script:
- *initENV
-
script:
- # init repository and generate configuration files for bitbake
- # checkout
- - export BASEDIR=$PWD
- - export EULA=1
- - BUILD_BRANCH=${MASTER_BRANCH}
- |
- if [ ${CI_PIPELINE_SOURCE} == "parent_pipeline" ]; then
- BUILD_BRANCH="${CI_COMMIT_REF_NAME}"
- fi
- if [ ! -z ${CI_COMMIT_TAG} ]; then
- BUILD_BRANCH="refs/tags/${CI_COMMIT_TAG}"
- fi
echo "This is the manifest repository URL used for the build: ${CI_REPOSITORY_URL}"
echo "This is the manifest branch used for the build: ${BUILD_BRANCH}"
echo "This is the board for which the build will be performed: ${BOARD}"
@@ -66,7 +105,7 @@ workflow:
echo "This is the image name will be assigned to software artifacts: ${IMAGE_NAME}"
- |
su secous -c "
- sh .gitlab-ci/scripts/build_setup.sh $GITBOT_TOKEN $CI_REPOSITORY_URL $BUILD_BRANCH $DEFCONFIG_FILE $RECIPE_NAME
+ mv repo_checkout/* . && mv repo_checkout/.repo .
git -C .repo/manifests rev-parse --short HEAD > $BASEDIR/commit_sha.txt
. ./seco-setup.sh -d ${DEFCONFIG_FILE};
. ./seco-setup.sh -c;
diff --git a/scripts/build_setup.sh b/scripts/build_setup.sh
index bab5a24..2e433d9 100644
--- a/scripts/build_setup.sh
+++ b/scripts/build_setup.sh
@@ -3,8 +3,6 @@
GITBOT_TOKEN=$1
CI_REPOSITORY_URL=$2
BUILD_BRANCH=$3
-DEFCONFIG_FILE=$4
-RECIPE_NAME=$5
git config --global url.https://github.com/.insteadOf git://github.com/
git config --global url.https://gitlab-ci-token:${GITBOT_TOKEN}@git.seco.com/.insteadOf https://git.seco.com/
@@ -14,7 +12,7 @@ repo sync -j$(nproc) --fetch-submodules --no-clone-bundle
RECIPES="u-boot-seco-rk linux-seco-rk u-boot-seco-imx linux-seco-imx imx-mkimage u-boot-seco-mtk linux-seco-mtk"
for str in $RECIPES; do
if echo "$BUILD_BRANCH" | grep -q "$str"; then
- filename=$(find layers/meta-seco-imx layers/meta-seco-rk -name "*$str*" -type f -exec grep -l "SRCBRANCH = " {} \;)
+ filename=$(find layers/meta-seco/meta-seco-imx layers/meta-seco/meta-seco-rk layers/meta-seco/meta-seco-mtk layers/meta-seco/meta-seco-qcom -name "*$str*" -type f -exec grep -l "SRCBRANCH = " {} \;)
echo $filename
RECIPE_BRANCH=$(echo "$BUILD_BRANCH" | sed "s|.*$str/||"i | sed 's/\/into.*//')
sed -i "s/\(SRCBRANCH = \).*/\1\"$(echo "$RECIPE_BRANCH" | sed 's/[\/&]/\\&/g')\"/" $filename
--
GitLab