From 3353ee7ac12c59064ff9e9e705c9b7bb24e36731 Mon Sep 17 00:00:00 2001
From: Andrii Sosiuk <andrii.sosiuk@seco.com>
Date: Wed, 30 Oct 2024 07:14:08 +0000
Subject: [PATCH] [DEPLOY] Implement Custom files deployment
---
build-pipeline-yocto.yml.jinja2 | 19 +++++++++++++++++++
docs/add-new-project-to-pipeline.md | 14 ++++++++++++++
scripts/artifacts_name.sh | 1 +
scripts/check_build_artifacts.sh | 15 +++++++++++++++
scripts/deploy_artifacts.sh | 25 +++++++++++++++++++++++++
scripts/generate_report.sh | 7 +++++++
6 files changed, 81 insertions(+)
diff --git a/build-pipeline-yocto.yml.jinja2 b/build-pipeline-yocto.yml.jinja2
index 7cbf89d..558df97 100644
--- a/build-pipeline-yocto.yml.jinja2
+++ b/build-pipeline-yocto.yml.jinja2
@@ -40,6 +40,7 @@ build:pass:
- echo "Build successful"
- echo "This is the value of the MANUAL_BUILD variable '${MANUAL_BUILD}'"
+
# --------------------------------------------------------------------------------------
# Stage: changelog
# --------------------------------------------------------------------------------------
@@ -62,6 +63,7 @@ release:
# Generated jobs
# --------------------------------------------------------------------------------------
+{% set ca_namespace = namespace(CUSTOM_ARTIFACTS="") %}
{% if MACHINES %}
{% for machine in MACHINES.split(' ') %}
{% if CUSTOM is not defined %}
@@ -212,6 +214,15 @@ release:
{% endif %}
{% endif %}
{% for distro in DISTRO.split(' ')%}
+ {# ============== Set custom artifacts =================== #}
+ {% if DISTROS_CUSTOM_ARTIFACTS is defined and DISTROS_CUSTOM_ARTIFACTS %}
+ {% for entry in DISTROS_CUSTOM_ARTIFACTS.split(' ') %}
+ {% if entry.startswith(distro + ":") %}
+ {% set custom_artifacts = entry.split(':')[1].split(';') %}
+ {% set ca_namespace.CUSTOM_ARTIFACTS = " ".join(custom_artifacts) %}
+ {% endif %}
+ {% endfor %}
+ {% endif %}
{% if distro == 'things' %}
{% set recipe = "seco-image-" + SECO_REMOTE %}
{% set image = SECO_REMOTE + "-things-image" %}
@@ -258,6 +269,10 @@ build-{{ machine }}-{{ distro }}:
UBOOT_FILE: {{ UBOOT_FILE }}
KERNEL_FILE: {{ KERNEL_FILE }}
AZURE_PATH: {{ AZURE_PATH }}
+ {% if ca_namespace.CUSTOM_ARTIFACTS %}
+ CUSTOM_ARTIFACTS: >
+ {{ ca_namespace.CUSTOM_ARTIFACTS }}
+ {% endif %}
#----------------------------------------------------
# {{ SECO_REMOTE }}-deploy-{{ machine }}-{{ distro }}
#----------------------------------------------------
@@ -301,6 +316,10 @@ build-{{ machine }}-{{ distro }}:
UBOOT_FILE: {{ UBOOT_FILE }}
KERNEL_FILE: {{ KERNEL_FILE }}
AZURE_PATH: {{ AZURE_PATH }}
+ {% if ca_namespace.CUSTOM_ARTIFACTS %}
+ CUSTOM_ARTIFACTS: >
+ {{ ca_namespace.CUSTOM_ARTIFACTS }}
+ {% endif %}
#----------------------------------------------------
# {{ SECO_REMOTE }}-deploy-{{ machine }}-{{ distro }}
#----------------------------------------------------
diff --git a/docs/add-new-project-to-pipeline.md b/docs/add-new-project-to-pipeline.md
index 95d3700..5ecdbd4 100644
--- a/docs/add-new-project-to-pipeline.md
+++ b/docs/add-new-project-to-pipeline.md
@@ -162,4 +162,18 @@ level) in the custom manifest repository:<>
QT_EDITION = "commercial"
LICENSE_FLAGS_ACCEPTED += " commercial "
```
+* `DISTROS_CUSTOM_ARTIFACTS`
+ A list of additional files to be deployed alongside standard artifacts.
+ These files will be packaged in a separate archive for deployment.
+ The availability of these artifacts depends on the specific DISTRO.
+ Therefore, the format for DISTROS_CUSTOM_ARTIFACTS should be as follows:
+ ```distro_name1:file1_path1;file_path2 distro_name2:file_path3;file_path4;file_path5```
+
+ > **Hint:** For improved readability, you can use YAML multi-line block syntax to format the variable as:
+ ```
+ DISTROS_CUSTOM_ARTIFACTS: >
+ distro_name1:file1_path1;file_path2
+ distro_name2:file_path3;file_path4;file_path5
+ ```
+ > **Note:** The copying process uses `cp -r file_path destination`, which supports wildcards in the `file_path` following standard shell patterns. Be cautious when using wildcards to ensure the correct files are matched.
diff --git a/scripts/artifacts_name.sh b/scripts/artifacts_name.sh
index ea8afa4..40dc101 100644
--- a/scripts/artifacts_name.sh
+++ b/scripts/artifacts_name.sh
@@ -18,3 +18,4 @@ export image_name="seco_"$PROCESSOR""$board_suffix"_"$IMAGE_NAME"_"$KERNEL"_"$TA
export bmap_name="seco_"$PROCESSOR""$board_suffix"_bmapfile-"$IMAGE_NAME"_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".rootfs.wic.bmap"
export bundle_filename="seco_"$PROCESSOR""$board_suffix"_${SECO_REMOTE}-bundle_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".raucb"
export sdk_filename="seco_"$PROCESSOR""$board_suffix"_sdk-"$IMAGE_NAME"_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".tar.gz"
+export custom_artifacts_filename="seco_"$PROCESSOR""$board_suffix"_custom-artifacts-"$IMAGE_NAME"_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".tar.gz"
diff --git a/scripts/check_build_artifacts.sh b/scripts/check_build_artifacts.sh
index 09d5421..9f387bf 100755
--- a/scripts/check_build_artifacts.sh
+++ b/scripts/check_build_artifacts.sh
@@ -24,6 +24,21 @@ else
count_wic=`ls -1 $ARTIFACTS_DIR/*.$IMAGE_EXTENSION 2>/dev/null | wc -l`
count_bundle=`ls -1 $ARTIFACTS_DIR/*.raucb 2>/dev/null | wc -l`
+ # Check custom artifacts
+ if [ -n "$CUSTOM_ARTIFACTS" ]; then
+ echo "CUSTOM_ARTIFACTS: $CUSTOM_ARTIFACTS"
+ count_el=0
+ for custom_artifact in $CUSTOM_ARTIFACTS; do
+ c_artifacts=`ls -1 $ARTIFACTS_DIR/$custom_artifact 2>/dev/null | wc -l`
+ count_el=$((count_el+$c_artifacts))
+ done
+ if [ $count_el -gt 0 ]; then
+ echo "$count_el custom artifacts were found!"
+ else
+ echo "No custom artifacts were found, warning!"
+ fi
+ fi
+
if [[ $UBOOT_FILE != *"none"* ]]; then
count_el=0
for i in $UBOOT_FILE; do
diff --git a/scripts/deploy_artifacts.sh b/scripts/deploy_artifacts.sh
index 6c97cb3..8c50e61 100644
--- a/scripts/deploy_artifacts.sh
+++ b/scripts/deploy_artifacts.sh
@@ -161,4 +161,29 @@ else
--file $BASE_DIRECTORY/$bmap_name \
--name $AZURE_PATH/$DEPLOY_PATH/$bmap_name \
--overwrite
+
+ echo "################################################################"
+ echo "################### Custom artifacts #########################"
+ echo "################################################################"
+
+ if [ -n "$CUSTOM_ARTIFACTS" ]; then
+ echo "CUSTOM_ARTIFACTS: $CUSTOM_ARTIFACTS"
+ custom_artifacts_temp_dir="custom_artifacts_temp_dir"
+ mkdir -p $BASE_DIRECTORY/$custom_artifacts_temp_dir
+
+ for custom_artifact in $CUSTOM_ARTIFACTS; do
+ cp -r $BASE_DIRECTORY/$ART_PATH/$custom_artifact $BASE_DIRECTORY/$custom_artifacts_temp_dir/
+ done
+ tar -czvf $BASE_DIRECTORY/$custom_artifacts_temp_dir/$custom_artifacts_filename -C $BASE_DIRECTORY/$custom_artifacts_temp_dir .
+
+ az storage blob upload --account-name $AZURE_STORAGE_ACCOUNT \
+ --sas-token $AZURE_STORAGE_SAS_TOKEN \
+ --container-name $AZURE_CONTAINER_NAME \
+ --file $BASE_DIRECTORY/$custom_artifacts_temp_dir/$custom_artifacts_filename \
+ --name $AZURE_PATH/$DEPLOY_PATH/$custom_artifacts_filename \
+ --overwrite
+
+ rm -rf $custom_artifacts_temp_dir
+ fi
+
fi
diff --git a/scripts/generate_report.sh b/scripts/generate_report.sh
index 8deb874..bb6b992 100644
--- a/scripts/generate_report.sh
+++ b/scripts/generate_report.sh
@@ -72,6 +72,13 @@ if [ "$CI_JOB_STATUS" == "success" ]; then
artifacts="bundle uboot kernel filesystem image bmap"
+ if [ -n "$CUSTOM_ARTIFACTS" ]; then
+ # Generate read-only SAS token with 6-months expiration policy
+ sas_custom=$(az storage blob generate-sas --account-key $AZURE_STORAGE_KEY --container-name $AZURE_CONTAINER_NAME --name $AZURE_PATH/$DEPLOY_PATH/$custom_artifacts_filename --permissions r --expiry "$expire_date" --output tsv)
+ url_custom=$(az storage blob url --container-name $AZURE_CONTAINER_NAME --name $AZURE_PATH/$DEPLOY_PATH/$custom_artifacts_filename --output tsv | sed -E 's/\?s.*//')
+ artifacts="$artifacts custom"
+ fi
+
fi
case "$AZURE_CONTAINER_NAME" in
--
GitLab