diff --git a/build-common.yml b/build-common.yml index 53bef85872770d110cb91378366a25683d8060bd..9b986c56df515e104dd648551b0215ea27167cc9 100644 --- a/build-common.yml +++ b/build-common.yml @@ -125,7 +125,8 @@ workflow: - .infrastructure timeout: 1h rules: - - when: manual + - if: $TEST_STAGE == "true" + when: manual allow_failure: true dependencies: [] variables: @@ -235,9 +236,14 @@ workflow: - echo "MACHINE=${MACHINE}" >> deploy.env script: # Expand eventual nested variables contained within the DEPLOY_* variables - - DEPLOY_SOURCE=$(eval echo "${DEPLOY_SOURCE}") - - DEPLOY_TARGET=$(eval echo "${DEPLOY_TARGET}") - - DEPLOY_TARGET_LINK=$(eval echo "${DEPLOY_TARGET_LINK}") + # FIXME: For now we need a double 'eval' here due to a GitLab bug: + # https://gitlab.com/gitlab-org/gitlab/-/issues/273409 + # Escaped variables are not correctly passed to child pipelines. Proposed workaround + # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we + # update our GitLab, we can use the former mechanism using a single 'eval'. + - DEPLOY_SOURCE=$(eval eval echo "${DEPLOY_SOURCE}") + - DEPLOY_TARGET=$(eval eval echo "${DEPLOY_TARGET}") + - DEPLOY_TARGET_LINK=$(eval eval echo "${DEPLOY_TARGET_LINK}") - printf "Syncing %s to %s\n" "${DEPLOY_SOURCE}" "${DEPLOY_TARGET}" # Capture rsync output, prefix it with the target link and save it to files.txt - rsync -rh --out-format="%n" --mkpath ${DEPLOY_SOURCE}/ ${DEPLOY_TARGET} | @@ -264,6 +270,8 @@ workflow: extends: - .infrastructure stage: Alphaplan + rules: + - if: $ALPHAPLAN_STAGE == "true" script: # MACHINE and RELEASE_NAME are available from deploy.env - .gitlab-ci/scripts/generate_alphaplan_fwr_file.py @@ -280,7 +288,8 @@ workflow: - .infrastructure stage: Alphaplan rules: - - when: manual + - if: $ALPHAPLAN_STAGE == "true" + when: manual allow_failure: true variables: # Most AP_WEBSERVICE_* variables are set in GitLab CI variables. We're defining the diff --git a/build-jobs-yocto.yml.jinja2 b/build-jobs-yocto.yml.jinja2 index f156dc6250edcc4c06c9efee6e08ad99b8bc3fe1..ee1ebd6f799033a707a8eff250cf11c758b0c1f1 100644 --- a/build-jobs-yocto.yml.jinja2 +++ b/build-jobs-yocto.yml.jinja2 @@ -25,13 +25,6 @@ stages: variables: MASTER_BRANCH: {{ MASTER_BRANCH }} -{% if CI_PARAM_DISTRO is not defined %} -{% set CI_PARAM_DISTRO = "guf-wayland" %} -{% endif %} -{% if CI_PARAM_DISTRO_FNG is not defined %} -{% set CI_PARAM_DISTRO_FNG = "guf-fngsystem" %} -{% endif %} - # -------------------------------------------------------------------------------------- # Stage: Infrastructure @@ -49,41 +42,11 @@ changelog: # -------------------------------------------------------------------------------------- # Stage: Build # -------------------------------------------------------------------------------------- -{% if CI_PARAM_IMAGE %} build-{{ machine }}: extends: .build_yocto variables: BITBAKE_TASK: build CI_PARAM_MACHINE: {{ machine }} - CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }} - CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }} - INSTALLSCRIPT: "fng-install.sh" - ARTIFACTS_PATH: build-*/tmp/deploy/images/**/* - -# Build jobs for the sdk -buildsdk-{{ machine }}: - extends: .build_yocto - variables: - BITBAKE_TASK: populate_sdk - CI_PARAM_MACHINE: {{ machine }} - CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }} - CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }} - ARTIFACTS_PATH: build-*/tmp/deploy/sdk/* - MANUAL_BUILD: "true" -{% endif %} - -{% if CI_PARAM_IMAGE_FNG %} -# Build jobs for the fng system image -build-{{ machine }}-fngsystem: - extends: .build_yocto - variables: - BITBAKE_TASK: build - CI_PARAM_MACHINE: {{ machine }} - CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }} - CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }} - INSTALLSCRIPT: "fngsystem-self-update.sh" - ARTIFACTS_PATH: build-*/tmp/deploy/images/**/* -{% endif %} # -------------------------------------------------------------------------------------- @@ -92,7 +55,6 @@ build-{{ machine }}-fngsystem: # Run platform tests for this machine which the yocto image # This is a little hacky as we need to match the machine name to # the available platforms -{% if CI_PARAM_IMAGE %} {% if machine == 'seco-mx6' or machine == 'imx6guf' %} {% set platforms = "santaro santoka santino santino-lt" %} @@ -134,14 +96,11 @@ platformtest:{{ machine }}: CI_PARAM_MACHINE: {{ lavamachine }} CI_PARAM_PLATFORMS: {{ platforms }} {% endif %} - -{% endif %} # -------------------------------------------------------------------------------------- # Stage: Package # -------------------------------------------------------------------------------------- -{% if CI_PARAM_IMAGE %} package-{{ machine }}: extends: .package variables: @@ -152,77 +111,35 @@ package-{{ machine }}: artifacts: false - job: changelog -packagesdk-{{ machine }}: - extends: .package - variables: - PACKAGE_TYPE: sdk - ASSOCIATED_BUILD_JOB: buildsdk-{{ machine }} - needs: - - job: buildsdk-{{ machine }} - artifacts: false -{% endif %} - -{% if CI_PARAM_IMAGE_FNG %} -package-{{ machine }}-fngsystem: - extends: .package - variables: - PACKAGE_TYPE: image - ASSOCIATED_BUILD_JOB: build-{{ machine }}-fngsystem - needs: - - job: build-{{ machine }}-fngsystem - artifacts: false -{% endif %} - # -------------------------------------------------------------------------------------- # Stage: Deploy SoftwareStore # -------------------------------------------------------------------------------------- -{% if CI_PARAM_IMAGE %} +deploy-{{ machine }}: + extends: .deploy {% if CI_COMMIT_TAG is defined %} - {% set deploy_class = ".deploy_software_store_yocto" %} + stage: Deploy SoftwareStore + variables: + DEPLOY_SOURCE: release/$${RELEASE_NAME} + DEPLOY_TARGET: ${DEPLOY_RELEASE_TARGET} + DEPLOY_TARGET_LINK: ${DEPLOY_RELEASE_TARGET_LINK} {% else %} - {% set deploy_class = ".deploy_software_store_yocto_internal" %} -{% endif %} -deploy-{{ machine }}: - extends: {{ deploy_class }} + stage: Deploy SoftwareStore Internal variables: + DEPLOY_SOURCE: release/$${RELEASE_NAME} + DEPLOY_TARGET: ${DEPLOY_INTERNAL_RELEASE_TARGET} + DEPLOY_TARGET_LINK: ${DEPLOY_INTERNAL_RELEASE_TARGET_LINK} +{% endif %} ASSOCIATED_PACKAGE_JOB: package-{{ machine }} needs: - job: package-{{ machine }} artifacts: false - job: changelog -deploysdk-{{ machine }}: - extends: {{ deploy_class }} - variables: - ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }} - needs: - - job: packagesdk-{{ machine }} - artifacts: false - - job: changelog -{% endif %} - -{% if CI_PARAM_IMAGE_FNG %} -{% if CI_COMMIT_TAG is defined %} - {% set deploy_class = ".deploy_software_store_fngsystem" %} -{% else %} - {% set deploy_class = ".deploy_software_store_fngsystem_internal" %} -{% endif %} -deploy-{{ machine }}-fngsystem: - extends: {{ deploy_class }} - variables: - ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem - needs: - - job: package-{{ machine }}-fngsystem - artifacts: false - - job: changelog -{% endif %} - # -------------------------------------------------------------------------------------- # Stage: Alphaplan # -------------------------------------------------------------------------------------- -{% if CI_PARAM_IMAGE %} generate-alphaplan-data-{{ machine }}: extends: .generate_alphaplan_data needs: @@ -232,19 +149,6 @@ import-alphaplan-data-{{ machine }}: extends: .import_alphaplan_data needs: - generate-alphaplan-data-{{ machine }} -{% endif %} - -{% if CI_PARAM_IMAGE_FNG %} -generate-alphaplan-data-{{ machine }}-fngsystem: - extends: .generate_alphaplan_data - needs: - - deploy-{{ machine }}-fngsystem - -import-alphaplan-data-{{ machine }}-fngsystem: - extends: .import_alphaplan_data - needs: - - generate-alphaplan-data-{{ machine }}-fngsystem -{% endif %} # -------------------------------------------------------------------------------------- @@ -256,37 +160,21 @@ import-alphaplan-data-{{ machine }}-fngsystem: {% if HIDE_FTP_UPLOAD_STAGE is not defined or not HIDE_FTP_UPLOAD_STAGE %} {% if CI_COMMIT_TAG is defined %} -{% if CI_PARAM_IMAGE %} ftp-{{ machine }}: - extends: .deploy_ftp_yocto + extends: .deploy + stage: Deploy FTP + tags: + - ftp variables: + DEPLOY_SOURCE: release/$${RELEASE_NAME} + DEPLOY_TARGET: ${DEPLOY_FTP_TARGET} + DEPLOY_TARGET_LINK: ${DEPLOY_FTP_TARGET_LINK} ASSOCIATED_PACKAGE_JOB: package-{{ machine }} needs: - job: package-{{ machine }} artifacts: false - job: changelog -ftpsdk-{{ machine }}: - extends: .deploy_ftp_yocto - variables: - ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }} - needs: - - job: packagesdk-{{ machine }} - artifacts: false - - job: changelog -{% endif %} - -{% if CI_PARAM_IMAGE_FNG %} -ftp-{{ machine }}-fngsystem: - extends: .deploy_ftp_fngsystem - variables: - ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem - needs: - - job: package-{{ machine }}-fngsystem - artifacts: false - - job: changelog -{% endif %} - {% endif %} {% endif %} diff --git a/build-yocto.yml b/build-yocto.yml index 180698a514742049e6694281e93d34715901e068..819d7d72312963145c1e77edfa0204ea05fa66f5 100644 --- a/build-yocto.yml +++ b/build-yocto.yml @@ -95,67 +95,3 @@ - *build_script - *collect_srcrevs - *dump_install_command - -# -------------------------------------------------------------------------------------- -# Stage: Deploy SoftwareStore -# -------------------------------------------------------------------------------------- -.deploy_software_store_yocto: - extends: .deploy - stage: Deploy SoftwareStore - variables: - DEPLOY_SOURCE: release/$${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-yocto/Releases/$${RELEASE_NAME} - DEPLOY_TARGET_LINK: > - Z:/Development/SoftwareStore/Linux-Yocto/Releases/$${RELEASE_NAME} - -.deploy_software_store_yocto_internal: - extends: .deploy - stage: Deploy SoftwareStore Internal - variables: - DEPLOY_SOURCE: release/$${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-yocto/Interne_Releases/$${RELEASE_NAME} - DEPLOY_TARGET_LINK: > - Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/$${RELEASE_NAME} - -.deploy_software_store_fngsystem: - extends: .deploy - stage: Deploy SoftwareStore - variables: - DEPLOY_SOURCE: release/$${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-fngsystem/$${RELEASE_NAME} - DEPLOY_TARGET_LINK: > - Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/$${RELEASE_NAME} - -.deploy_software_store_fngsystem_internal: - extends: .deploy - stage: Deploy SoftwareStore Internal - variables: - DEPLOY_SOURCE: release/$${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-fngsystem/CI_Builds/$${RELEASE_NAME} - DEPLOY_TARGET_LINK: > - Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/$${RELEASE_NAME} - -# -------------------------------------------------------------------------------------- -# Stage: Deploy FTP -# -------------------------------------------------------------------------------------- -.deploy_ftp_yocto: - extends: .deploy - stage: Deploy FTP - tags: - - ftp - variables: - DEPLOY_SOURCE: release/$${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-ftp-yocto/Releases/$${RELEASE_NAME} - DEPLOY_TARGET_LINK: > - http://support.garz-fricke.com/projects/Linux-Yocto/Releases/$${RELEASE_NAME} - -.deploy_ftp_fngsystem: - extends: .deploy - stage: Deploy FTP - tags: - - ftp - variables: - DEPLOY_SOURCE: release/$${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-ftp-fngsystem/$${RELEASE_NAME} - DEPLOY_TARGET_LINK: > - http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/$${RELEASE_NAME} diff --git a/manifest-pipeline-ci-test.yml b/manifest-pipeline-ci-test.yml index 21d2a1f08ff729bd5d06f37334a99418e2d58096..9cdbd6e7235e07f4a2a9a2aac1bbf21bc973d886 100644 --- a/manifest-pipeline-ci-test.yml +++ b/manifest-pipeline-ci-test.yml @@ -29,3 +29,7 @@ variables: CHANGELOG_PROJECTS: seco-ne/yocto/infrastructure/ci-test/minimal-bar seco-ne/yocto/infrastructure/ci-test/minimal-foo + +build-jobs: + extends: + - .build-jobs diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml index a3fad6c1887945ee9ec89bcda242df7d5d90c124..28bb74cb7923dfb148488e563998c908659869bc 100644 --- a/manifest-pipeline-yocto.yml +++ b/manifest-pipeline-yocto.yml @@ -26,18 +26,73 @@ variables: seco-ne/yocto/layers/meta-seconorth-distro seco-ne/yocto/layers/meta-seconorth-machine -generate-build-jobs: + # List of machines to build images for + CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf + +.yocto-deploy: + variables: + # FIXME: For now we need a quoted dollar sign here due to a GitLab bug: + # https://gitlab.com/gitlab-org/gitlab/-/issues/273409 + # Escaped variables are not correctly passed to child pipelines. Proposed workaround + # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we + # update our GitLab, we can use the former mechanism using '$$RELEASE_NAME'. + DEPLOY_RELEASE_TARGET: /artifacts-yocto/Releases/"$"{RELEASE_NAME} + DEPLOY_RELEASE_TARGET_LINK: > + Z:/Development/SoftwareStore/Linux-Yocto/Releases/"$"{RELEASE_NAME} + DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-yocto/Interne_Releases/"$"{RELEASE_NAME} + DEPLOY_INTERNAL_RELEASE_TARGET_LINK: > + Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/"$"{RELEASE_NAME} + DEPLOY_FTP_TARGET: /artifacts-ftp-yocto/Releases/"$"{RELEASE_NAME} + DEPLOY_FTP_TARGET_LINK: > + http://support.garz-fricke.com/projects/Linux-Yocto/Releases/"$"{RELEASE_NAME} + +.fngsystem-deploy: + variables: + DEPLOY_RELEASE_TARGET: /artifacts-fngsystem/"$"{RELEASE_NAME} + DEPLOY_RELEASE_TARGET_LINK: > + Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME} + DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-fngsystem/CI_Builds/"$"{RELEASE_NAME} + DEPLOY_INTERNAL_RELEASE_TARGET_LINK: > + Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/"$"{RELEASE_NAME} + DEPLOY_FTP_TARGET: /artifacts-ftp-fngsystem/"$"{RELEASE_NAME} + DEPLOY_FTP_TARGET_LINK: > + http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME} + +yocto-build-jobs: + extends: + - .build-jobs + - .yocto-deploy variables: - # Default image and distro + BITBAKE_TASK: build CI_PARAM_IMAGE: guf-image CI_PARAM_DISTRO: guf-wayland + INSTALLSCRIPT: fng-install.sh + ARTIFACTS_PATH: build-*/tmp/deploy/images/**/* + PACKAGE_TYPE: image + TEST_STAGE: "true" + ALPHAPLAN_STAGE: "true" - # Flash-N-Go image and distro - # In the past, the buildfng job overwrote the image and distro itself. Due to the - # transition to the new seconorth names, image and distro for the buildfng must be - # settable from outside of the job. - CI_PARAM_IMAGE_FNG: fngsystem-image - CI_PARAM_DISTRO_FNG: guf-fngsystem +sdk-build-jobs: + extends: + - .build-jobs + - .yocto-deploy + variables: + BITBAKE_TASK: populate_sdk + CI_PARAM_IMAGE: guf-image + CI_PARAM_DISTRO: guf-wayland + ARTIFACTS_PATH: build-*/tmp/deploy/sdk/* + MANUAL_BUILD: "true" + PACKAGE_TYPE: sdk - # List of machines to build images for - CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf +fngsystem-build-jobs: + extends: + - .build-jobs + - .fngsystem-deploy + variables: + BITBAKE_TASK: build + CI_PARAM_IMAGE: fngsystem-image + CI_PARAM_DISTRO: guf-fngsystem + INSTALLSCRIPT: fngsystem-self-update.sh + ARTIFACTS_PATH: build-*/tmp/deploy/images/**/* + PACKAGE_TYPE: image + ALPHAPLAN_STAGE: "true" diff --git a/manifest-pipeline.yml b/manifest-pipeline.yml index 88f79343bd2e68fdee67b8ed0a762e02e5a51f5e..ddc2cd4d6492e219fb0b7bd8affd8a8e15465072 100644 --- a/manifest-pipeline.yml +++ b/manifest-pipeline.yml @@ -7,6 +7,7 @@ include: stages: - manifest-pipeline + - trigger - retrigger - build @@ -42,7 +43,6 @@ workflow: - if: $CI_PIPELINE_SOURCE == "api" - if: $CI_PIPELINE_SOURCE == "pipeline" - if: $CI_PIPELINE_SOURCE == "web" - stage: manifest-pipeline .short_master_pipeline: rules: @@ -64,7 +64,9 @@ generate-build-jobs: extends: - .infrastructure - .full_build_pipeline + stage: manifest-pipeline script: + - echo "Generating build jobs from template file '${BUILD_JOBS_TEMPLATE}'" # The job generation script implicitly passes the OS environment to the template, so # that the template has access to all GitLab CI variables. Hence there is no need # to explicitly pass any of them as command line arguments. @@ -76,9 +78,10 @@ generate-build-jobs: paths: - build-jobs.yml -build-jobs: +.build-jobs: extends: - .full_build_pipeline + stage: trigger needs: ["generate-build-jobs"] trigger: include: @@ -90,6 +93,7 @@ yamllint: extends: - .yamllint - .full_build_pipeline + stage: manifest-pipeline # -------------------------------------------------------------------------------------- # Short master pipeline (runs on master after merging a merge request)