Compare revisions

Tim Jaacks · Tim Jaacks · 9406ad75 · 9406ad75 · 9406ad75 · 9406ad75
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,7 +36,7 @@ workflow:
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_BRANCH == "master"
      when: never
-  timeout: 2m
+  timeout: 3m

 black:
  extends: .analyze

--- a/build-common.yml
+++ b/build-common.yml
@@ -125,7 +125,8 @@ workflow:
    - .infrastructure
  timeout: 1h
  rules:
-    - when: manual
+    - if: $TEST_STAGE == "true"
+      when: manual
      allow_failure: true
  dependencies: []
  variables:
@@ -235,9 +236,14 @@ workflow:
    - echo "MACHINE=${MACHINE}" >> deploy.env
  script:
    # Expand eventual nested variables contained within the DEPLOY_* variables
-    - DEPLOY_SOURCE=$(eval echo "${DEPLOY_SOURCE}")
-    - DEPLOY_TARGET=$(eval echo "${DEPLOY_TARGET}")
-    - DEPLOY_TARGET_LINK=$(eval echo "${DEPLOY_TARGET_LINK}")
+    # FIXME: For now we need a double 'eval' here due to a GitLab bug:
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/273409
+    # Escaped variables are not correctly passed to child pipelines. Proposed workaround
+    # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we
+    # update our GitLab, we can use the former mechanism using a single 'eval'.
+    - DEPLOY_SOURCE=$(eval eval echo "${DEPLOY_SOURCE}")
+    - DEPLOY_TARGET=$(eval eval echo "${DEPLOY_TARGET}")
+    - DEPLOY_TARGET_LINK=$(eval eval echo "${DEPLOY_TARGET_LINK}")
    - printf "Syncing %s to %s\n" "${DEPLOY_SOURCE}" "${DEPLOY_TARGET}"
    # Capture rsync output, prefix it with the target link and save it to files.txt
    - rsync -rh --out-format="%n" --mkpath ${DEPLOY_SOURCE}/ ${DEPLOY_TARGET} |
@@ -264,6 +270,8 @@ workflow:
  extends:
    - .infrastructure
  stage: Alphaplan
+  rules:
+    - if: $ALPHAPLAN_STAGE == "true"
  script:
    # MACHINE and RELEASE_NAME are available from deploy.env
    - .gitlab-ci/scripts/generate_alphaplan_fwr_file.py
@@ -280,7 +288,8 @@ workflow:
    - .infrastructure
  stage: Alphaplan
  rules:
-    - when: manual
+    - if: $ALPHAPLAN_STAGE == "true"
+      when: manual
      allow_failure: true
  variables:
    # Most AP_WEBSERVICE_* variables are set in GitLab CI variables. We're defining the

--- a/build-jobs-yocto.yml.jinja2
+++ b/build-jobs-yocto.yml.jinja2
@@ -25,13 +25,6 @@ stages:
 variables:
  MASTER_BRANCH: {{ MASTER_BRANCH }}

-{% if CI_PARAM_DISTRO is not defined %}
-{% set CI_PARAM_DISTRO = "guf-wayland" %}
-{% endif %}
-{% if CI_PARAM_DISTRO_FNG is not defined %}
-{% set CI_PARAM_DISTRO_FNG = "guf-fngsystem" %}
-{% endif %}
-

 # --------------------------------------------------------------------------------------
 # Stage: Infrastructure
@@ -49,41 +42,11 @@ changelog:
 # --------------------------------------------------------------------------------------
 # Stage: Build
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
 build-{{ machine }}:
  extends: .build_yocto
  variables:
    BITBAKE_TASK: build
    CI_PARAM_MACHINE: {{ machine }}
-    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
-    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
-    INSTALLSCRIPT: "fng-install.sh"
-    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
-
-# Build jobs for the sdk
-buildsdk-{{ machine }}:
-  extends: .build_yocto
-  variables:
-    BITBAKE_TASK: populate_sdk
-    CI_PARAM_MACHINE: {{ machine }}
-    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
-    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
-    ARTIFACTS_PATH: build-*/tmp/deploy/sdk/*
-    MANUAL_BUILD: "true"
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-# Build jobs for the fng system image
-build-{{ machine }}-fngsystem:
-  extends: .build_yocto
-  variables:
-    BITBAKE_TASK: build
-    CI_PARAM_MACHINE: {{ machine }}
-    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }}
-    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }}
-    INSTALLSCRIPT: "fngsystem-self-update.sh"
-    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
-{% endif %}


 # --------------------------------------------------------------------------------------
@@ -92,7 +55,6 @@ build-{{ machine }}-fngsystem:
 # Run platform tests for this machine which the yocto image
 # This is a little hacky as we need to match the machine name to
 # the available platforms
-{% if CI_PARAM_IMAGE %}

 {% if machine == 'seco-mx6' or machine == 'imx6guf' %}
  {% set platforms = "santaro santoka santino santino-lt" %}
@@ -134,14 +96,11 @@ platformtest:{{ machine }}:
    CI_PARAM_MACHINE: {{ lavamachine }}
    CI_PARAM_PLATFORMS: {{ platforms }}
 {% endif %}
-  
-{% endif %}


 # --------------------------------------------------------------------------------------
 # Stage: Package
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
 package-{{ machine }}:
  extends: .package
  variables:
@@ -152,77 +111,35 @@ package-{{ machine }}:
      artifacts: false
    - job: changelog

-packagesdk-{{ machine }}:
-  extends: .package
-  variables:
-    PACKAGE_TYPE: sdk
-    ASSOCIATED_BUILD_JOB: buildsdk-{{ machine }}
-  needs:
-    - job: buildsdk-{{ machine }}
-      artifacts: false
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-package-{{ machine }}-fngsystem:
-  extends: .package
-  variables:
-    PACKAGE_TYPE: image
-    ASSOCIATED_BUILD_JOB: build-{{ machine }}-fngsystem
-  needs:
-    - job: build-{{ machine }}-fngsystem
-      artifacts: false
-{% endif %}
-

 # --------------------------------------------------------------------------------------
 # Stage: Deploy SoftwareStore
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
+deploy-{{ machine }}:
+  extends: .deploy
 {% if CI_COMMIT_TAG is defined %}
-  {% set deploy_class = ".deploy_software_store_yocto" %}
+  stage: Deploy SoftwareStore
+  variables:
+    DEPLOY_SOURCE: release/$${RELEASE_NAME}
+    DEPLOY_TARGET: ${DEPLOY_RELEASE_TARGET}
+    DEPLOY_TARGET_LINK: ${DEPLOY_RELEASE_TARGET_LINK}
 {% else %}
-  {% set deploy_class = ".deploy_software_store_yocto_internal" %}
-{% endif %}
-deploy-{{ machine }}:
-  extends: {{ deploy_class }}
+  stage: Deploy SoftwareStore Internal
  variables:
+    DEPLOY_SOURCE: release/$${RELEASE_NAME}
+    DEPLOY_TARGET: ${DEPLOY_INTERNAL_RELEASE_TARGET}
+    DEPLOY_TARGET_LINK: ${DEPLOY_INTERNAL_RELEASE_TARGET_LINK}
+{% endif %}
    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
  needs:
    - job: package-{{ machine }}
      artifacts: false
    - job: changelog

-deploysdk-{{ machine }}:
-  extends: {{ deploy_class }}
-  variables:
-    ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
-  needs:
-    - job: packagesdk-{{ machine }}
-      artifacts: false
-    - job: changelog
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-{% if CI_COMMIT_TAG is defined %}
-  {% set deploy_class = ".deploy_software_store_fngsystem" %}
-{% else %}
-  {% set deploy_class = ".deploy_software_store_fngsystem_internal" %}
-{% endif %}
-deploy-{{ machine }}-fngsystem:
-  extends: {{ deploy_class }}
-  variables:
-    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
-  needs:
-    - job: package-{{ machine }}-fngsystem
-      artifacts: false
-    - job: changelog
-{% endif %}
-

 # --------------------------------------------------------------------------------------
 # Stage: Alphaplan
 # --------------------------------------------------------------------------------------
-{% if CI_PARAM_IMAGE %}
 generate-alphaplan-data-{{ machine }}:
  extends: .generate_alphaplan_data
  needs:
@@ -232,19 +149,6 @@ import-alphaplan-data-{{ machine }}:
  extends: .import_alphaplan_data
  needs:
    - generate-alphaplan-data-{{ machine }}
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-generate-alphaplan-data-{{ machine }}-fngsystem:
-  extends: .generate_alphaplan_data
-  needs:
-    - deploy-{{ machine }}-fngsystem
-
-import-alphaplan-data-{{ machine }}-fngsystem:
-  extends: .import_alphaplan_data
-  needs:
-    - generate-alphaplan-data-{{ machine }}-fngsystem
-{% endif %}


 # --------------------------------------------------------------------------------------
@@ -256,37 +160,21 @@ import-alphaplan-data-{{ machine }}-fngsystem:
 {% if HIDE_FTP_UPLOAD_STAGE is not defined or not HIDE_FTP_UPLOAD_STAGE %}
 {% if CI_COMMIT_TAG is defined %}

-{% if CI_PARAM_IMAGE %}
 ftp-{{ machine }}:
-  extends: .deploy_ftp_yocto
+  extends: .deploy
+  stage: Deploy FTP
+  tags:
+    - ftp
  variables:
+    DEPLOY_SOURCE: release/$${RELEASE_NAME}
+    DEPLOY_TARGET: ${DEPLOY_FTP_TARGET}
+    DEPLOY_TARGET_LINK: ${DEPLOY_FTP_TARGET_LINK}
    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
  needs:
    - job: package-{{ machine }}
      artifacts: false
    - job: changelog

-ftpsdk-{{ machine }}:
-  extends: .deploy_ftp_yocto
-  variables:
-    ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
-  needs:
-    - job: packagesdk-{{ machine }}
-      artifacts: false
-    - job: changelog
-{% endif %}
-
-{% if CI_PARAM_IMAGE_FNG %}
-ftp-{{ machine }}-fngsystem:
-  extends: .deploy_ftp_fngsystem
-  variables:
-    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
-  needs:
-    - job: package-{{ machine }}-fngsystem
-      artifacts: false
-    - job: changelog
-{% endif %}
-
 {% endif %}
 {% endif %}


--- a/build-yocto.yml
+++ b/build-yocto.yml
@@ -95,67 +95,3 @@
    - *build_script
    - *collect_srcrevs
    - *dump_install_command
-
-# --------------------------------------------------------------------------------------
-# Stage: Deploy SoftwareStore
-# --------------------------------------------------------------------------------------
-.deploy_software_store_yocto:
-  extends: .deploy
-  stage: Deploy SoftwareStore
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-yocto/Releases/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Linux-Yocto/Releases/$${RELEASE_NAME}
-
-.deploy_software_store_yocto_internal:
-  extends: .deploy
-  stage: Deploy SoftwareStore Internal
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-yocto/Interne_Releases/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/$${RELEASE_NAME}
-
-.deploy_software_store_fngsystem:
-  extends: .deploy
-  stage: Deploy SoftwareStore
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-fngsystem/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
-
-.deploy_software_store_fngsystem_internal:
-  extends: .deploy
-  stage: Deploy SoftwareStore Internal
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-fngsystem/CI_Builds/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/$${RELEASE_NAME}
-
-# --------------------------------------------------------------------------------------
-# Stage: Deploy FTP
-# --------------------------------------------------------------------------------------
-.deploy_ftp_yocto:
-  extends: .deploy
-  stage: Deploy FTP
-  tags:
-    - ftp
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-ftp-yocto/Releases/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      http://support.garz-fricke.com/projects/Linux-Yocto/Releases/$${RELEASE_NAME}
-
-.deploy_ftp_fngsystem:
-  extends: .deploy
-  stage: Deploy FTP
-  tags:
-    - ftp
-  variables:
-    DEPLOY_SOURCE: release/$${RELEASE_NAME}
-    DEPLOY_TARGET: /artifacts-ftp-fngsystem/$${RELEASE_NAME}
-    DEPLOY_TARGET_LINK: >
-      http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
--- a/manifest-pipeline-ci-test.yml
+++ b/manifest-pipeline-ci-test.yml
@@ -29,3 +29,7 @@ variables:
  CHANGELOG_PROJECTS:
    seco-ne/yocto/infrastructure/ci-test/minimal-bar
    seco-ne/yocto/infrastructure/ci-test/minimal-foo
+
+build-jobs:
+  extends:
+    - .build-jobs
--- a/manifest-pipeline-yocto.yml
+++ b/manifest-pipeline-yocto.yml
@@ -26,18 +26,73 @@ variables:
    seco-ne/yocto/layers/meta-seconorth-distro
    seco-ne/yocto/layers/meta-seconorth-machine

-generate-build-jobs:
+  # List of machines to build images for
+  CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
+
+.yocto-deploy:
+  variables:
+    # FIXME: For now we need a quoted dollar sign here due to a GitLab bug:
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/273409
+    # Escaped variables are not correctly passed to child pipelines. Proposed workaround
+    # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we
+    # update our GitLab, we can use the former mechanism using '$$RELEASE_NAME'.
+    DEPLOY_RELEASE_TARGET: /artifacts-yocto/Releases/"$"{RELEASE_NAME}
+    DEPLOY_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Linux-Yocto/Releases/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-yocto/Interne_Releases/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET: /artifacts-ftp-yocto/Releases/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET_LINK: >
+      http://support.garz-fricke.com/projects/Linux-Yocto/Releases/"$"{RELEASE_NAME}
+
+.fngsystem-deploy:
+  variables:
+    DEPLOY_RELEASE_TARGET: /artifacts-fngsystem/"$"{RELEASE_NAME}
+    DEPLOY_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-fngsystem/CI_Builds/"$"{RELEASE_NAME}
+    DEPLOY_INTERNAL_RELEASE_TARGET_LINK: >
+      Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET: /artifacts-ftp-fngsystem/"$"{RELEASE_NAME}
+    DEPLOY_FTP_TARGET_LINK: >
+      http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME}
+
+yocto-build-jobs:
+  extends:
+    - .build-jobs
+    - .yocto-deploy
  variables:
-    # Default image and distro
+    BITBAKE_TASK: build
    CI_PARAM_IMAGE: guf-image
    CI_PARAM_DISTRO: guf-wayland
+    INSTALLSCRIPT: fng-install.sh
+    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
+    PACKAGE_TYPE: image
+    TEST_STAGE: "true"
+    ALPHAPLAN_STAGE: "true"

-    # Flash-N-Go image and distro
-    # In the past, the buildfng job overwrote the image and distro itself. Due to the
-    # transition to the new seconorth names, image and distro for the buildfng must be
-    # settable from outside of the job.
-    CI_PARAM_IMAGE_FNG: fngsystem-image
-    CI_PARAM_DISTRO_FNG: guf-fngsystem
+sdk-build-jobs:
+  extends:
+    - .build-jobs
+    - .yocto-deploy
+  variables:
+    BITBAKE_TASK: populate_sdk
+    CI_PARAM_IMAGE: guf-image
+    CI_PARAM_DISTRO: guf-wayland
+    ARTIFACTS_PATH: build-*/tmp/deploy/sdk/*
+    MANUAL_BUILD: "true"
+    PACKAGE_TYPE: sdk

-    # List of machines to build images for
-    CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
+fngsystem-build-jobs:
+  extends:
+    - .build-jobs
+    - .fngsystem-deploy
+  variables:
+    BITBAKE_TASK: build
+    CI_PARAM_IMAGE: fngsystem-image
+    CI_PARAM_DISTRO: guf-fngsystem
+    INSTALLSCRIPT: fngsystem-self-update.sh
+    ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
+    PACKAGE_TYPE: image
+    ALPHAPLAN_STAGE: "true"
--- a/manifest-pipeline.yml
+++ b/manifest-pipeline.yml
@@ -7,6 +7,7 @@ include:

 stages:
  - manifest-pipeline
+  - trigger
  - retrigger
  - build

@@ -42,7 +43,6 @@ workflow:
    - if: $CI_PIPELINE_SOURCE == "api"
    - if: $CI_PIPELINE_SOURCE == "pipeline"
    - if: $CI_PIPELINE_SOURCE == "web"
-  stage: manifest-pipeline

 .short_master_pipeline:
  rules:
@@ -64,7 +64,9 @@ generate-build-jobs:
  extends:
    - .infrastructure
    - .full_build_pipeline
+  stage: manifest-pipeline
  script:
+    - echo "Generating build jobs from template file '${BUILD_JOBS_TEMPLATE}'"
    # The job generation script implicitly passes the OS environment to the template, so
    # that the template has access to all GitLab CI variables. Hence there is no need
    # to explicitly pass any of them as command line arguments.
@@ -76,9 +78,10 @@ generate-build-jobs:
    paths:
      - build-jobs.yml

-build-jobs:
+.build-jobs:
  extends:
    - .full_build_pipeline
+  stage: trigger
  needs: ["generate-build-jobs"]
  trigger:
    include:
@@ -90,6 +93,7 @@ yamllint:
  extends:
    - .yamllint
    - .full_build_pipeline
+  stage: manifest-pipeline

 # --------------------------------------------------------------------------------------
 # Short master pipeline (runs on master after merging a merge request)
No results found