From 9406ad752cfe28ca33556df57282c278ef9ece00 Mon Sep 17 00:00:00 2001
From: Tim Jaacks <tim.jaacks@seco.com>
Date: Thu, 27 Jul 2023 12:35:34 +0200
Subject: [PATCH] Yocto build: separate images in multiple pipelines

Instead of building the Yocto image, the Yocto SDK and the FNGSystem
image in one single pipeline, separate them into three independent
pipelines that are triggered in parallel.

This makes the concept more modular: we have a single general pipeline
now which is configurable from outside via variables. Hence we can have
a custom number of pipelines along with custom build targets in each
project without having to make code changes in the gitlab-ci project.

The default Yocto manifest pipeline configures three build pipelines:

- yocto-build-jobs
- sdk-build-jobs
- fngsystem-build-jobs

In a project including the Yocto manifest pipeline, we can disable
certain build pipelines using job rules, e.g. disabling the SDK build:

sdk-build-jobs:
  rules:
    - when: never

Furthermore we can add more pipelines by simply adding jobs extending
the '.build-jobs' class in the project's .gitlab-ci.yml:

yocto-custom-build-jobs:
  extends:
    - .build-jobs
  variables:
    BITBAKE_TASK: build
    CI_PARAM_IMAGE: custom-image
    CI_PARAM_DISTRO: custom-distro
    PACKAGE_TYPE: image
---
 build-common.yml              |  19 +++--
 build-jobs-yocto.yml.jinja2   | 150 +++++-----------------------------
 build-yocto.yml               |  64 ---------------
 manifest-pipeline-ci-test.yml |   4 +
 manifest-pipeline-yocto.yml   |  75 ++++++++++++++---
 manifest-pipeline.yml         |   8 +-
 6 files changed, 108 insertions(+), 212 deletions(-)

diff --git a/build-common.yml b/build-common.yml
index 53bef858..9b986c56 100644
--- a/build-common.yml
+++ b/build-common.yml
@@ -125,7 +125,8 @@ workflow:
     - .infrastructure
   timeout: 1h
   rules:
-    - when: manual
+    - if: $TEST_STAGE == "true"
+      when: manual
       allow_failure: true
   dependencies: []
   variables:
@@ -235,9 +236,14 @@ workflow:
     - echo "MACHINE=${MACHINE}" >> deploy.env
   script:
     # Expand eventual nested variables contained within the DEPLOY_* variables
-    - DEPLOY_SOURCE=$(eval echo "${DEPLOY_SOURCE}")
-    - DEPLOY_TARGET=$(eval echo "${DEPLOY_TARGET}")
-    - DEPLOY_TARGET_LINK=$(eval echo "${DEPLOY_TARGET_LINK}")
+    # FIXME: For now we need a double 'eval' here due to a GitLab bug:
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/273409
+    # Escaped variables are not correctly passed to child pipelines. Proposed workaround
+    # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we
+    # update our GitLab, we can use the former mechanism using a single 'eval'.
+    - DEPLOY_SOURCE=$(eval eval echo "${DEPLOY_SOURCE}")
+    - DEPLOY_TARGET=$(eval eval echo "${DEPLOY_TARGET}")
+    - DEPLOY_TARGET_LINK=$(eval eval echo "${DEPLOY_TARGET_LINK}")
     - printf "Syncing %s to %s\n" "${DEPLOY_SOURCE}" "${DEPLOY_TARGET}"
     # Capture rsync output, prefix it with the target link and save it to files.txt
     - rsync -rh --out-format="%n" --mkpath ${DEPLOY_SOURCE}/ ${DEPLOY_TARGET} |
@@ -264,6 +270,8 @@ workflow:
   extends:
     - .infrastructure
   stage: Alphaplan
+  rules:
+    - if: $ALPHAPLAN_STAGE == "true"
   script:
     # MACHINE and RELEASE_NAME are available from deploy.env
     - .gitlab-ci/scripts/generate_alphaplan_fwr_file.py
@@ -280,7 +288,8 @@ workflow:
     - .infrastructure
   stage: Alphaplan
   rules:
-    - when: manual
+    - if: $ALPHAPLAN_STAGE == "true"
+      when: manual
       allow_failure: true
   variables:
     # Most AP_WEBSERVICE_* variables are set in GitLab CI variables. We're defining the
diff --git a/build-jobs-yocto.yml.jinja2 b/build-jobs-yocto.yml.jinja2
index f156dc62..ee1ebd6f 100644
--- a/build-jobs-yocto.yml.jinja2
+++ b/build-jobs-yocto.yml.jinja2
@@ -25,13 +25,6 @@ stages:
 variables:
   MASTER_BRANCH: {{ MASTER_BRANCH }}
 
-{% if CI_PARAM_DISTRO is not defined %}
-{% set CI_PARAM_DISTRO = "guf-wayland" %}
-{% endif %}
-{% if CI_PARAM_DISTRO_FNG is not defined %}
-{% set CI_PARAM_DISTRO_FNG = "guf-fngsystem" %}
-{% endif %}
-
 
 # --------------------------------------------------------------------------------------
 # Stage: Infrastructure
@@ -49,41 +42,11 @@ changelog:
 # --------------------------------------------------------------------------------------
 # Stage: Build
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
 build-{{ machine }}:
   extends: .build_yocto
   variables:
     BITBAKE_TASK: build
     CI_PARAM_MACHINE: {{ machine }}
-    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
-    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
-    INSTALLSCRIPT: "fng-install.sh"
-    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
-
-# Build jobs for the sdk
-buildsdk-{{ machine }}:
-  extends: .build_yocto
-  variables:
-    BITBAKE_TASK: populate_sdk
-    CI_PARAM_MACHINE: {{ machine }}
-    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
-    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
-    ARTIFACTS_PATH: build-*/tmp/deploy/sdk/*
-    MANUAL_BUILD: "true"
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-# Build jobs for the fng system image
-build-{{ machine }}-fngsystem:
-  extends: .build_yocto
-  variables:
-    BITBAKE_TASK: build
-    CI_PARAM_MACHINE: {{ machine }}
-    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }}
-    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }}
-    INSTALLSCRIPT: "fngsystem-self-update.sh"
-    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
-{% endif %}
 
 
 # --------------------------------------------------------------------------------------
@@ -92,7 +55,6 @@ build-{{ machine }}-fngsystem:
 # Run platform tests for this machine which the yocto image
 # This is a little hacky as we need to match the machine name to
 # the available platforms
-{% if CI_PARAM_IMAGE %}
 
 {% if machine == 'seco-mx6' or machine == 'imx6guf' %}
   {% set platforms = "santaro santoka santino santino-lt" %}
@@ -134,14 +96,11 @@ platformtest:{{ machine }}:
     CI_PARAM_MACHINE: {{ lavamachine }}
     CI_PARAM_PLATFORMS: {{ platforms }}
 {% endif %}
-  
-{% endif %}
 
 
 # --------------------------------------------------------------------------------------
 # Stage: Package
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
 package-{{ machine }}:
   extends: .package
   variables:
@@ -152,77 +111,35 @@ package-{{ machine }}:
       artifacts: false
     - job: changelog
 
-packagesdk-{{ machine }}:
-  extends: .package
-  variables:
-    PACKAGE_TYPE: sdk
-    ASSOCIATED_BUILD_JOB: buildsdk-{{ machine }}
-  needs:
-    - job: buildsdk-{{ machine }}
-      artifacts: false
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-package-{{ machine }}-fngsystem:
-  extends: .package
-  variables:
-    PACKAGE_TYPE: image
-    ASSOCIATED_BUILD_JOB: build-{{ machine }}-fngsystem
-  needs:
-    - job: build-{{ machine }}-fngsystem
-      artifacts: false
-{% endif %}
-
 
 # --------------------------------------------------------------------------------------
 # Stage: Deploy SoftwareStore
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
+deploy-{{ machine }}:
+  extends: .deploy
 {% if CI_COMMIT_TAG is defined %}
-  {% set deploy_class = ".deploy_software_store_yocto" %}
+  stage: Deploy SoftwareStore
+  variables:
+    DEPLOY_SOURCE: release/$${RELEASE_NAME}
+    DEPLOY_TARGET: ${DEPLOY_RELEASE_TARGET}
+    DEPLOY_TARGET_LINK: ${DEPLOY_RELEASE_TARGET_LINK}
 {% else %}
-  {% set deploy_class = ".deploy_software_store_yocto_internal" %}
-{% endif %}
-deploy-{{ machine }}:
-  extends: {{ deploy_class }}
+  stage: Deploy SoftwareStore Internal
   variables:
+    DEPLOY_SOURCE: release/$${RELEASE_NAME}
+    DEPLOY_TARGET: ${DEPLOY_INTERNAL_RELEASE_TARGET}
+    DEPLOY_TARGET_LINK: ${DEPLOY_INTERNAL_RELEASE_TARGET_LINK}
+{% endif %}
     ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
   needs:
     - job: package-{{ machine }}
       artifacts: false
     - job: changelog
 
-deploysdk-{{ machine }}:
-  extends: {{ deploy_class }}
-  variables:
-    ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
-  needs:
-    - job: packagesdk-{{ machine }}
-      artifacts: false
-    - job: changelog
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-{% if CI_COMMIT_TAG is defined %}
-  {% set deploy_class = ".deploy_software_store_fngsystem" %}
-{% else %}
-  {% set deploy_class = ".deploy_software_store_fngsystem_internal" %}
-{% endif %}
-deploy-{{ machine }}-fngsystem:
-  extends: {{ deploy_class }}
-  variables:
-    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
-  needs:
-    - job: package-{{ machine }}-fngsystem
-      artifacts: false
-    - job: changelog
-{% endif %}
-
 
 # --------------------------------------------------------------------------------------
 # Stage: Alphaplan
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
 generate-alphaplan-data-{{ machine }}:
   extends: .generate_alphaplan_data
   needs:
@@ -232,19 +149,6 @@ import-alphaplan-data-{{ machine }}:
   extends: .import_alphaplan_data
   needs:
     - generate-alphaplan-data-{{ machine }}
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-generate-alphaplan-data-{{ machine }}-fngsystem:
-  extends: .generate_alphaplan_data
-  needs:
-    - deploy-{{ machine }}-fngsystem
-
-import-alphaplan-data-{{ machine }}-fngsystem:
-  extends: .import_alphaplan_data
-  needs:
-    - generate-alphaplan-data-{{ machine }}-fngsystem
-{% endif %}
 
 
 # --------------------------------------------------------------------------------------
@@ -256,37 +160,21 @@ import-alphaplan-data-{{ machine }}-fngsystem:
 {% if HIDE_FTP_UPLOAD_STAGE is not defined or not HIDE_FTP_UPLOAD_STAGE %}
 {% if CI_COMMIT_TAG is defined %}
 
-{% if CI_PARAM_IMAGE %}
 ftp-{{ machine }}:
-  extends: .deploy_ftp_yocto
+  extends: .deploy
+  stage: Deploy FTP
+  tags:
+    - ftp
   variables:
+    DEPLOY_SOURCE: release/$${RELEASE_NAME}
+    DEPLOY_TARGET: ${DEPLOY_FTP_TARGET}
+    DEPLOY_TARGET_LINK: ${DEPLOY_FTP_TARGET_LINK}
     ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
   needs:
     - job: package-{{ machine }}
       artifacts: false
     - job: changelog
 
-ftpsdk-{{ machine }}:
-  extends: .deploy_ftp_yocto
-  variables:
-    ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
-  needs:
-    - job: packagesdk-{{ machine }}
-      artifacts: false
-    - job: changelog
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-ftp-{{ machine }}-fngsystem:
-  extends: .deploy_ftp_fngsystem
-  variables:
-    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
-  needs:
-    - job: package-{{ machine }}-fngsystem
-      artifacts: false
-    - job: changelog
-{% endif %}
-
 {% endif %}
 {% endif %}
 
diff --git a/build-yocto.yml b/build-yocto.yml
index 180698a5..819d7d72 100644
--- a/build-yocto.yml
+++ b/build-yocto.yml
@@ -95,67 +95,3 @@
     - *build_script
     - *collect_srcrevs
     - *dump_install_command
-
-# --------------------------------------------------------------------------------------
-# Stage: Deploy SoftwareStore
-# --------------------------------------------------------------------------------------
-.deploy_software_store_yocto:
-  extends: .deploy
-  stage: Deploy SoftwareStore
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-yocto/Releases/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Linux-Yocto/Releases/$${RELEASE_NAME}
-
-.deploy_software_store_yocto_internal:
-  extends: .deploy
-  stage: Deploy SoftwareStore Internal
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-yocto/Interne_Releases/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/$${RELEASE_NAME}
-
-.deploy_software_store_fngsystem:
-  extends: .deploy
-  stage: Deploy SoftwareStore
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-fngsystem/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
-
-.deploy_software_store_fngsystem_internal:
-  extends: .deploy
-  stage: Deploy SoftwareStore Internal
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-fngsystem/CI_Builds/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/$${RELEASE_NAME}
-
-# --------------------------------------------------------------------------------------
-# Stage: Deploy FTP
-# --------------------------------------------------------------------------------------
-.deploy_ftp_yocto:
-  extends: .deploy
-  stage: Deploy FTP
-  tags:
-    - ftp
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-ftp-yocto/Releases/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      http://support.garz-fricke.com/projects/Linux-Yocto/Releases/$${RELEASE_NAME}
-
-.deploy_ftp_fngsystem:
-  extends: .deploy
-  stage: Deploy FTP
-  tags:
-    - ftp
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-ftp-fngsystem/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
diff --git a/manifest-pipeline-ci-test.yml b/manifest-pipeline-ci-test.yml
index 21d2a1f0..9cdbd6e7 100644
--- a/manifest-pipeline-ci-test.yml
+++ b/manifest-pipeline-ci-test.yml
@@ -29,3 +29,7 @@ variables:
   CHANGELOG_PROJECTS:
     seco-ne/yocto/infrastructure/ci-test/minimal-bar
     seco-ne/yocto/infrastructure/ci-test/minimal-foo
+
+build-jobs:
+  extends:
+    - .build-jobs
diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml
index a3fad6c1..28bb74cb 100644
--- a/manifest-pipeline-yocto.yml
+++ b/manifest-pipeline-yocto.yml
@@ -26,18 +26,73 @@ variables:
     seco-ne/yocto/layers/meta-seconorth-distro
     seco-ne/yocto/layers/meta-seconorth-machine
 
-generate-build-jobs:
+  # List of machines to build images for
+  CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
+
+.yocto-deploy:
+  variables:
+    # FIXME: For now we need a quoted dollar sign here due to a GitLab bug:
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/273409
+    # Escaped variables are not correctly passed to child pipelines. Proposed workaround
+    # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we
+    # update our GitLab, we can use the former mechanism using '$$RELEASE_NAME'.
+    DEPLOY_RELEASE_TARGET: /artifacts-yocto/Releases/"$"{RELEASE_NAME}
+    DEPLOY_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Linux-Yocto/Releases/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-yocto/Interne_Releases/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET: /artifacts-ftp-yocto/Releases/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET_LINK: >
+      http://support.garz-fricke.com/projects/Linux-Yocto/Releases/"$"{RELEASE_NAME}
+
+.fngsystem-deploy:
+  variables:
+    DEPLOY_RELEASE_TARGET: /artifacts-fngsystem/"$"{RELEASE_NAME}
+    DEPLOY_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-fngsystem/CI_Builds/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET: /artifacts-ftp-fngsystem/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET_LINK: >
+      http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME}
+
+yocto-build-jobs:
+  extends:
+    - .build-jobs
+    - .yocto-deploy
   variables:
-    # Default image and distro
+    BITBAKE_TASK: build
     CI_PARAM_IMAGE: guf-image
     CI_PARAM_DISTRO: guf-wayland
+    INSTALLSCRIPT: fng-install.sh
+    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
+    PACKAGE_TYPE: image
+    TEST_STAGE: "true"
+    ALPHAPLAN_STAGE: "true"
 
-    # Flash-N-Go image and distro
-    # In the past, the buildfng job overwrote the image and distro itself. Due to the
-    # transition to the new seconorth names, image and distro for the buildfng must be
-    # settable from outside of the job.
-    CI_PARAM_IMAGE_FNG: fngsystem-image
-    CI_PARAM_DISTRO_FNG: guf-fngsystem
+sdk-build-jobs:
+  extends:
+    - .build-jobs
+    - .yocto-deploy
+  variables:
+    BITBAKE_TASK: populate_sdk
+    CI_PARAM_IMAGE: guf-image
+    CI_PARAM_DISTRO: guf-wayland
+    ARTIFACTS_PATH: build-*/tmp/deploy/sdk/*
+    MANUAL_BUILD: "true"
+    PACKAGE_TYPE: sdk
 
-    # List of machines to build images for
-    CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
+fngsystem-build-jobs:
+  extends:
+    - .build-jobs
+    - .fngsystem-deploy
+  variables:
+    BITBAKE_TASK: build
+    CI_PARAM_IMAGE: fngsystem-image
+    CI_PARAM_DISTRO: guf-fngsystem
+    INSTALLSCRIPT: fngsystem-self-update.sh
+    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
+    PACKAGE_TYPE: image
+    ALPHAPLAN_STAGE: "true"
diff --git a/manifest-pipeline.yml b/manifest-pipeline.yml
index 88f79343..ddc2cd4d 100644
--- a/manifest-pipeline.yml
+++ b/manifest-pipeline.yml
@@ -7,6 +7,7 @@ include:
 
 stages:
   - manifest-pipeline
+  - trigger
   - retrigger
   - build
 
@@ -42,7 +43,6 @@ workflow:
     - if: $CI_PIPELINE_SOURCE == "api"
     - if: $CI_PIPELINE_SOURCE == "pipeline"
     - if: $CI_PIPELINE_SOURCE == "web"
-  stage: manifest-pipeline
 
 .short_master_pipeline:
   rules:
@@ -64,7 +64,9 @@ generate-build-jobs:
   extends:
     - .infrastructure
     - .full_build_pipeline
+  stage: manifest-pipeline
   script:
+    - echo "Generating build jobs from template file '${BUILD_JOBS_TEMPLATE}'"
     # The job generation script implicitly passes the OS environment to the template, so
     # that the template has access to all GitLab CI variables. Hence there is no need
     # to explicitly pass any of them as command line arguments.
@@ -76,9 +78,10 @@ generate-build-jobs:
     paths:
       - build-jobs.yml
 
-build-jobs:
+.build-jobs:
   extends:
     - .full_build_pipeline
+  stage: trigger
   needs: ["generate-build-jobs"]
   trigger:
     include:
@@ -90,6 +93,7 @@ yamllint:
   extends:
     - .yamllint
     - .full_build_pipeline
+  stage: manifest-pipeline
 
 # --------------------------------------------------------------------------------------
 # Short master pipeline (runs on master after merging a merge request)
-- 
GitLab