[DEPLOY] Implement Custom files deployment

build-pipeline-yocto.yml.jinja2

@@ -40,6 +40,7 @@ build:pass:
     - echo "Build successful"
     - echo "This is the value of the MANUAL_BUILD variable '${MANUAL_BUILD}'"
 
+
 # --------------------------------------------------------------------------------------
 # Stage: changelog
 # --------------------------------------------------------------------------------------
@@ -62,6 +63,7 @@ release:
 # Generated jobs
 # --------------------------------------------------------------------------------------
 
+{% set ca_namespace = namespace(CUSTOM_ARTIFACTS="") %}
 {% if MACHINES %}
   {% for machine in MACHINES.split(' ') %}
     {% if CUSTOM is not defined %}
@@ -212,6 +214,15 @@ release:
       {% endif %}
     {% endif %}
     {% for distro in DISTRO.split(' ')%}
+      {# ============== Set custom artifacts =================== #}
+      {% if DISTROS_CUSTOM_ARTIFACTS is defined and DISTROS_CUSTOM_ARTIFACTS %}
+        {% for entry in DISTROS_CUSTOM_ARTIFACTS.split(' ') %}
+          {% if entry.startswith(distro + ":") %}
+            {% set custom_artifacts = entry.split(':')[1].split(';') %}
+            {% set ca_namespace.CUSTOM_ARTIFACTS = " ".join(custom_artifacts) %}
+          {% endif %}
+        {% endfor %}
+      {% endif %}
       {% if distro == 'things' %}
         {% set recipe = "seco-image-" + SECO_REMOTE %}
         {% set image = SECO_REMOTE + "-things-image" %}
@@ -258,6 +269,10 @@ build-{{ machine }}-{{ distro }}:
      UBOOT_FILE: {{ UBOOT_FILE }}
      KERNEL_FILE: {{ KERNEL_FILE }}
      AZURE_PATH: {{ AZURE_PATH }}
+     {% if ca_namespace.CUSTOM_ARTIFACTS %}
+     CUSTOM_ARTIFACTS: >
+       {{ ca_namespace.CUSTOM_ARTIFACTS }}
+     {% endif %}
 #----------------------------------------------------
 # {{ SECO_REMOTE }}-deploy-{{ machine }}-{{ distro }}
 #----------------------------------------------------
@@ -301,6 +316,10 @@ build-{{ machine }}-{{ distro }}:
      UBOOT_FILE: {{ UBOOT_FILE }}
      KERNEL_FILE: {{ KERNEL_FILE }}
      AZURE_PATH: {{ AZURE_PATH }}
+     {% if ca_namespace.CUSTOM_ARTIFACTS %}
+     CUSTOM_ARTIFACTS: >
+       {{ ca_namespace.CUSTOM_ARTIFACTS }}
+     {% endif %}
 #----------------------------------------------------
 # {{ SECO_REMOTE }}-deploy-{{ machine }}-{{ distro }}
 #----------------------------------------------------

docs/add-new-project-to-pipeline.md

@@ -162,4 +162,18 @@ level) in the custom manifest repository:<>
     QT_EDITION = "commercial"
     LICENSE_FLAGS_ACCEPTED += " commercial "
   ```
+* `DISTROS_CUSTOM_ARTIFACTS`
+  A list of additional files to be deployed alongside standard artifacts.
+  These files will be packaged in a separate archive for deployment.
+  The availability of these artifacts depends on the specific DISTRO.
+  Therefore, the format for DISTROS_CUSTOM_ARTIFACTS should be as follows:
+  ```distro_name1:file1_path1;file_path2 distro_name2:file_path3;file_path4;file_path5```
+
+  > **Hint:** For improved readability, you can use YAML multi-line block syntax to format the variable as:
+  ```
+  DISTROS_CUSTOM_ARTIFACTS: >
+      distro_name1:file1_path1;file_path2
+      distro_name2:file_path3;file_path4;file_path5
+  ```
 
+  >  **Note:** The copying process uses `cp -r file_path destination`, which supports wildcards in the `file_path` following standard shell patterns. Be cautious when using wildcards to ensure the correct files are matched.

scripts/artifacts_name.sh

@@ -18,3 +18,4 @@ export image_name="seco_"$PROCESSOR""$board_suffix"_"$IMAGE_NAME"_"$KERNEL"_"$TA
 export bmap_name="seco_"$PROCESSOR""$board_suffix"_bmapfile-"$IMAGE_NAME"_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".rootfs.wic.bmap"
 export bundle_filename="seco_"$PROCESSOR""$board_suffix"_${SECO_REMOTE}-bundle_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".raucb"
 export sdk_filename="seco_"$PROCESSOR""$board_suffix"_sdk-"$IMAGE_NAME"_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".tar.gz"
+export custom_artifacts_filename="seco_"$PROCESSOR""$board_suffix"_custom-artifacts-"$IMAGE_NAME"_"$KERNEL"_"$TAG_NAME"_"$DEPLOY_DATE".tar.gz"

scripts/check_build_artifacts.sh

@@ -24,6 +24,21 @@ else
   count_wic=`ls -1 $ARTIFACTS_DIR/*.$IMAGE_EXTENSION 2>/dev/null | wc -l`
   count_bundle=`ls -1 $ARTIFACTS_DIR/*.raucb 2>/dev/null | wc -l`
 
+  # Check custom artifacts
+  if [ -n "$CUSTOM_ARTIFACTS" ]; then
+    echo "CUSTOM_ARTIFACTS: $CUSTOM_ARTIFACTS"
+    count_el=0
+    for custom_artifact in $CUSTOM_ARTIFACTS; do
+      c_artifacts=`ls -1 $ARTIFACTS_DIR/$custom_artifact 2>/dev/null | wc -l`
+      count_el=$((count_el+$c_artifacts))
+    done
+    if [ $count_el -gt 0 ]; then
+        echo "$count_el custom artifacts were found!"
+    else
+        echo "No custom artifacts were found, warning!"
+    fi
+  fi
+
   if [[ $UBOOT_FILE != *"none"* ]]; then
     count_el=0
     for i in $UBOOT_FILE; do

scripts/deploy_artifacts.sh

@@ -161,4 +161,29 @@ else
         --file $BASE_DIRECTORY/$bmap_name \
         --name $AZURE_PATH/$DEPLOY_PATH/$bmap_name \
         --overwrite
+
+    echo "################################################################"
+    echo "###################  Custom artifacts  #########################"
+    echo "################################################################"
+
+    if [ -n "$CUSTOM_ARTIFACTS" ]; then
+        echo "CUSTOM_ARTIFACTS: $CUSTOM_ARTIFACTS"
+        custom_artifacts_temp_dir="custom_artifacts_temp_dir"
+        mkdir -p $BASE_DIRECTORY/$custom_artifacts_temp_dir
+
+        for custom_artifact in $CUSTOM_ARTIFACTS; do
+            cp -r $BASE_DIRECTORY/$ART_PATH/$custom_artifact $BASE_DIRECTORY/$custom_artifacts_temp_dir/
+        done
+        tar -czvf $BASE_DIRECTORY/$custom_artifacts_temp_dir/$custom_artifacts_filename -C $BASE_DIRECTORY/$custom_artifacts_temp_dir .
+
+        az storage blob upload --account-name $AZURE_STORAGE_ACCOUNT \
+            --sas-token $AZURE_STORAGE_SAS_TOKEN \
+            --container-name $AZURE_CONTAINER_NAME \
+            --file $BASE_DIRECTORY/$custom_artifacts_temp_dir/$custom_artifacts_filename \
+            --name $AZURE_PATH/$DEPLOY_PATH/$custom_artifacts_filename \
+            --overwrite
+
+        rm -rf $custom_artifacts_temp_dir
+    fi
+
 fi

scripts/generate_report.sh

@@ -72,6 +72,13 @@ if [ "$CI_JOB_STATUS" == "success" ]; then
 
       artifacts="bundle uboot kernel filesystem image bmap"
 
+      if [ -n "$CUSTOM_ARTIFACTS" ]; then
+          # Generate read-only SAS token with 6-months expiration policy
+          sas_custom=$(az storage blob generate-sas --account-key $AZURE_STORAGE_KEY --container-name $AZURE_CONTAINER_NAME --name $AZURE_PATH/$DEPLOY_PATH/$custom_artifacts_filename --permissions r --expiry "$expire_date" --output tsv)
+          url_custom=$(az storage blob url --container-name $AZURE_CONTAINER_NAME --name $AZURE_PATH/$DEPLOY_PATH/$custom_artifacts_filename --output tsv | sed -E 's/\?s.*//')
+          artifacts="$artifacts custom"
+      fi
+
     fi
 
     case "$AZURE_CONTAINER_NAME" in