diff --git a/build-common.yml b/build-common.yml index 9b986c56df515e104dd648551b0215ea27167cc9..ed41e4afa9c0f485a89a13d27d471e627029c5b8 100644 --- a/build-common.yml +++ b/build-common.yml @@ -111,12 +111,32 @@ workflow: # exists, we're adding it to the artifacts. - build-*/tmp/deploy/licenses/**/license.manifest - build.env - cache: + cache: &cache # Cache the build artifacts for subsequent stages in the same pipeline - key: ${CI_PIPELINE_ID}-${CI_JOB_NAME} policy: push paths: *artifacts_paths +.simulate_build: + extends: + - .buildbase + tags: + - misc + needs: [] + timeout: 60m + variables: + BUILD_ARTIFACTS: + script: + - if compgen -G build-*; then echo "Using build from cache"; exit 0; fi + - echo "Getting Yocto build artifacts" + - wget -O artifacts.zip ${BUILD_ARTIFACTS} + - unzip artifacts.zip + cache: + - *cache + # Additionally cache the build artifacts for re-runs of this job in other pipelines + - key: ${CI_JOB_NAME}-${BUILD_ARTIFACTS} + paths: *artifacts_paths + # -------------------------------------------------------------------------------------- # Stage: test # -------------------------------------------------------------------------------------- diff --git a/build-pipeline-ci-test.yml b/build-pipeline-ci-test.yml new file mode 100644 index 0000000000000000000000000000000000000000..cfe4174d2840b890273e2cfc3bcef776765dda2d --- /dev/null +++ b/build-pipeline-ci-test.yml @@ -0,0 +1,112 @@ +--- +# -------------------------------------------------------------------------------------- +# Global +# -------------------------------------------------------------------------------------- +include: + - project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci' + ref: ${GITLAB_CI_REVISION} + file: + - build-common.yml + - common.yml + +stages: + - Build + - Test + +# -------------------------------------------------------------------------------------- +# Stage: Build +# -------------------------------------------------------------------------------------- +.build: &build + - cd ${CI_PROJECT_DIR} + - VERSION=$(cd .repo/manifests && git describe --tags) + - cat .repo/manifests/default.xml + - find foo -name "[0-9]*.txt" -printf '%P\n' | sort -V > files-foo-$VERSION.txt + - cat files-foo-$VERSION.txt + - find bar -name "[0-9]*.txt" -printf '%P\n' | sort -V > files-bar-$VERSION.txt + - cat files-bar-$VERSION.txt + - FOO_FILES=$(cat files-foo-$VERSION.txt | wc -l) + - BAR_FILES=$(cat files-bar-$VERSION.txt | wc -l) + - DIFF=$((BAR_FILES-FOO_FILES)) + - (($DIFF >= -1 && $DIFF <= 1)) + +build:files: + stage: Build + extends: + - .buildbase + tags: + - infrastructure + needs: [] + timeout: 2m + variables: + GIT_STRATEGY: none + LOGPREFIX: "CI:build:" + before_script: + - !reference [.docker_check] + - !reference [.setup_ssh] + - !reference [.repo_checkout] + script: + - *build + artifacts: + paths: + - files-* + +build:echo: + stage: Build + tags: + - infrastructure + needs: [] + timeout: 2m + image: ${CI_IMAGE_PYTHON} + script: + - printenv + - echo "Build successful" + +build:check-foo-branch: + stage: Build + extends: + - .buildbase + tags: + - infrastructure + needs: [] + timeout: 2m + variables: + GIT_STRATEGY: none + before_script: + - !reference [.docker_check] + - !reference [.setup_ssh] + - !reference [.repo_checkout] + script: | + echo "repo branch: $MASTER_BRANCH" + echo "foo branch: $(cat foo/branch.txt)" + if [[ "$MASTER_BRANCH" != $(cat foo/branch.txt) ]]; then + echo "ERROR: Branches do not match!" + exit 1 + fi + artifacts: null + cache: null + +# -------------------------------------------------------------------------------------- +# Stage: Test +# -------------------------------------------------------------------------------------- +.test_simple: + stage: Test + rules: + - when: manual + allow_failure: true + tags: + - infrastructure + image: ${CI_IMAGE_PYTHON} + script: + - exit ${RETURNCODE} + +test:pass: + extends: + - .test_simple + variables: + RETURNCODE: 0 + +test:fail: + extends: + - .test_simple + variables: + RETURNCODE: 1 diff --git a/build-pipeline-ci-test.yml.jinja2 b/build-pipeline-ci-test.yml.jinja2 deleted file mode 100644 index 8d8ea3d7dfeeef9d5d4795888f4d724eee642db1..0000000000000000000000000000000000000000 --- a/build-pipeline-ci-test.yml.jinja2 +++ /dev/null @@ -1,300 +0,0 @@ ---- -# -------------------------------------------------------------------------------------- -# Global -# -------------------------------------------------------------------------------------- -# As the trigger job is not executed in a environment with checked out repository, we -# need to get the includes directly from gitlab -include: - - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' - ref: {{ GITLAB_CI_REVISION }} - file: - - build-common.yml - - common.yml - -stages: - - Infrastructure - - Build - - Test - - Package - - Deploy SoftwareStore - - Deploy FTP - - Alphaplan - -# -------------------------------------------------------------------------------------- -# Stage: Infrastructure -# -------------------------------------------------------------------------------------- -changelog: - extends: .changelog - -# -------------------------------------------------------------------------------------- -# Stage: Build -# -------------------------------------------------------------------------------------- -.build: &build - - cd ${CI_PROJECT_DIR} - - VERSION=$(cd .repo/manifests && git describe --tags) - - cat .repo/manifests/default.xml - - find foo -name "[0-9]*.txt" -printf '%P\n' | sort -V > files-foo-$VERSION.txt - - cat files-foo-$VERSION.txt - - find bar -name "[0-9]*.txt" -printf '%P\n' | sort -V > files-bar-$VERSION.txt - - cat files-bar-$VERSION.txt - - FOO_FILES=$(cat files-foo-$VERSION.txt | wc -l) - - BAR_FILES=$(cat files-bar-$VERSION.txt | wc -l) - - DIFF=$((BAR_FILES-FOO_FILES)) - - (($DIFF >= -1 && $DIFF <= 1)) - -build:files: - stage: Build - extends: - - .buildbase - tags: - - infrastructure - needs: [] - timeout: 2m - variables: - GIT_STRATEGY: none - LOGPREFIX: "CI:build:" - before_script: - - !reference [.docker_check] - - !reference [.setup_ssh] - - !reference [.repo_checkout] - script: - - *build - artifacts: - paths: - - files-* - -build:echo: - stage: Build - tags: - - infrastructure - needs: [] - timeout: 2m - image: ${CI_IMAGE_PYTHON} - script: - - printenv - - echo "Build successful" - -build:check-foo-branch: - stage: Build - extends: - - .buildbase - tags: - - infrastructure - needs: [] - timeout: 2m - variables: - GIT_STRATEGY: none - before_script: - - !reference [.docker_check] - - !reference [.setup_ssh] - - !reference [.repo_checkout] - script: | - echo "repo branch: $MASTER_BRANCH" - echo "foo branch: $(cat foo/branch.txt)" - if [[ "$MASTER_BRANCH" != $(cat foo/branch.txt) ]]; then - echo "ERROR: Branches do not match!" - exit 1 - fi - artifacts: null - cache: null - -.simulate_build: - extends: - - .buildbase - tags: - - misc - needs: [] - timeout: 60m - variables: - BUILD_ARTIFACTS: - script: - - if compgen -G build-*; then echo "Using build from cache"; exit 0; fi - - echo "Getting Yocto build artifacts" - - wget -O artifacts.zip ${BUILD_ARTIFACTS} - - unzip artifacts.zip - cache: - - !reference [.buildbase, cache] - # Additionally cache the build artifacts for re-runs of this job in other pipelines - - key: ${CI_JOB_NAME}-${BUILD_ARTIFACTS} - paths: !reference [.buildbase, artifacts, paths] - -simulate-build-seco-mx6: - extends: - - .buildbase - - .simulate_build - variables: - # We have to specify a tag here instead of getting the artifacts from a master - # branch, because we don't always execute the full pipeline on the master branches. - # In those cases the actual build job does not exist, which results in a 404 error. - BUILD_ARTIFACTS: https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/7.0/download?job=build-seco-mx6 - ARTIFACTS_PATH: build-*/tmp/deploy/images/**/* - -simulate-buildsdk-seco-mx6: - extends: - - .buildbase - - .simulate_build - variables: - BUILD_ARTIFACTS: https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/7.0/download?job=buildsdk-seco-mx6 - ARTIFACTS_PATH: build-*/tmp/deploy/sdk/* - MANUAL_BUILD: "true" - -# -------------------------------------------------------------------------------------- -# Stage: Test -# -------------------------------------------------------------------------------------- -.test_simple: - stage: Test - rules: - - when: manual - allow_failure: true - tags: - - infrastructure - image: ${CI_IMAGE_PYTHON} - script: - - exit ${RETURNCODE} - -test:pass: - extends: - - .test_simple - variables: - RETURNCODE: 0 - -test:fail: - extends: - - .test_simple - variables: - RETURNCODE: 1 - -smoketest:seco-mx6: - extends: - - .test - stage: Test - needs: [simulate-build-seco-mx6] - variables: - TEST_REPO_BRANCH: dunfell - CI_PARAM_PLATFORMS: imx6guf - CI_PARAM_EXTRA: --all-devices - CI_PARAM_TEST_SUITE: boot.jinja2 - CI_PARAM_BUILDJOB: simulate-build-seco-mx6 - -# -------------------------------------------------------------------------------------- -# Stage: Package -# -------------------------------------------------------------------------------------- -package-seco-mx6: - extends: .package - variables: - PACKAGE_TYPE: image - ASSOCIATED_BUILD_JOB: simulate-build-seco-mx6 - needs: - - job: simulate-build-seco-mx6 - artifacts: false - -package-sdk-seco-mx6: - extends: .package - variables: - PACKAGE_TYPE: sdk - ASSOCIATED_BUILD_JOB: simulate-buildsdk-seco-mx6 - needs: - - job: simulate-buildsdk-seco-mx6 - artifacts: false - -# -------------------------------------------------------------------------------------- -# Stage: Deploy SoftwareStore -# -------------------------------------------------------------------------------------- -.deploy_software_store: - extends: .deploy - stage: Deploy SoftwareStore - variables: - # We can't use the RELEASE_NAME variable from package.env in the variables section, - # because we're loading the file from the cache. Usually it is downloaded via - # GitLab's standard artifact mechanism, which adds all dotenv variables to the - # pipeline scope. This doesn't work when using the cache, so we have to prevent - # GitLab from expanding these variables early by escaping the $ signs for those - # variables that are not available yet at this point. These will get evaluated - # within the script block of the .deploy class. - DEPLOY_SOURCE: release/$${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-yocto/Test/$${RELEASE_NAME} - DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Linux-Yocto/Test/$${RELEASE_NAME} - -deploy-files: - extends: .deploy_software_store - variables: - GIT_STRATEGY: none - DEPLOY_SOURCE: . - DEPLOY_TARGET: /artifacts-yocto/Test/${CI_JOB_ID} - DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Linux-Yocto/Test/${CI_JOB_ID} - needs: - - build:files - # Disable reading package.env and create empty deploy.env (not needed for this job) - before_script: - - touch deploy.env - -deploy-seco-mx6: - extends: .deploy_software_store - variables: - ASSOCIATED_PACKAGE_JOB: package-seco-mx6 - needs: - - job: package-seco-mx6 - artifacts: false - -deploy-sdk-seco-mx6: - extends: .deploy_software_store - variables: - ASSOCIATED_PACKAGE_JOB: package-sdk-seco-mx6 - needs: - - job: package-sdk-seco-mx6 - artifacts: false - -# -------------------------------------------------------------------------------------- -# Stage: Deploy FTP -# -------------------------------------------------------------------------------------- -.deploy_ftp: - extends: .deploy - stage: Deploy FTP - tags: - - ftp - variables: - # env variables are available via package.env from package job - DEPLOY_SOURCE: release/${RELEASE_NAME} - DEPLOY_TARGET: /artifacts-ftp-yocto/Test/${RELEASE_NAME} - DEPLOY_TARGET_LINK: http://support.garz-fricke.com/projects/Linux-Yocto/Test/${RELEASE_NAME} - -ftp-files: - extends: .deploy_ftp - variables: - GIT_STRATEGY: none - DEPLOY_SOURCE: . - DEPLOY_TARGET: /artifacts-ftp-yocto/Test/${CI_JOB_ID} - DEPLOY_TARGET_LINK: http://support.garz-fricke.com/projects/Linux-Yocto/Test/${CI_JOB_ID} - needs: - - build:files - -ftp-seco-mx6: - extends: .deploy_ftp - variables: - ASSOCIATED_PACKAGE_JOB: package-seco-mx6 - needs: - - job: package-seco-mx6 - artifacts: false - -ftp-sdk-seco-mx6: - extends: .deploy_ftp - variables: - ASSOCIATED_PACKAGE_JOB: package-sdk-seco-mx6 - needs: - - job: package-sdk-seco-mx6 - artifacts: false - -# -------------------------------------------------------------------------------------- -# Stage: Alphaplan -# -------------------------------------------------------------------------------------- -generate-alphaplan-data-seco-mx6: - extends: .generate_alphaplan_data - needs: - - deploy-seco-mx6 - -import-alphaplan-data-seco-mx6: - extends: .import_alphaplan_data - variables: - AP_WEBSERVICE_URL: https://SRV06.hamburg.garz-fricke.de/Test/Alphaplan-API/Artikel/CreateFirmware - needs: - - generate-alphaplan-data-seco-mx6 diff --git a/build-pipeline-yocto.yml.jinja2 b/build-pipeline-yocto.yml.jinja2 index ee1ebd6f799033a707a8eff250cf11c758b0c1f1..f53483529ad0f134b8347a17945086d1cbe4e873 100644 --- a/build-pipeline-yocto.yml.jinja2 +++ b/build-pipeline-yocto.yml.jinja2 @@ -42,12 +42,27 @@ changelog: # -------------------------------------------------------------------------------------- # Stage: Build # -------------------------------------------------------------------------------------- +{% if SIMULATE_BUILD is not defined or SIMULATE_BUILD != "true" %} + build-{{ machine }}: extends: .build_yocto variables: BITBAKE_TASK: build CI_PARAM_MACHINE: {{ machine }} +{% else %} + +build-{{ machine }}: + extends: + - .buildbase + - .simulate_build + variables: + # We have to specify a tag here instead of getting the artifacts from a master + # branch, because we don't always execute the full pipeline on the master branches. + # In those cases the actual build job does not exist, which results in a 404 error. + BUILD_ARTIFACTS: ${BUILD_ARTIFACTS_PREFIX}-{{ machine }} + +{% endif %} # -------------------------------------------------------------------------------------- # Stage: Test @@ -158,7 +173,8 @@ import-alphaplan-data-{{ machine }}: # public FTP area. It should be removed as soon as we support uploading to different # FTP target folders. {% if HIDE_FTP_UPLOAD_STAGE is not defined or not HIDE_FTP_UPLOAD_STAGE %} -{% if CI_COMMIT_TAG is defined %} +{% if SIMULATE_BUILD is defined and SIMULATE_BUILD == "true" + or CI_COMMIT_TAG is defined %} ftp-{{ machine }}: extends: .deploy diff --git a/manifest-pipeline-ci-test.yml b/manifest-pipeline-ci-test.yml index ce62aba38957186bd0df6e361cdc7d795e9a7187..ff22e5a437b179ad0deba1cd62931f263e7bf546 100644 --- a/manifest-pipeline-ci-test.yml +++ b/manifest-pipeline-ci-test.yml @@ -18,9 +18,6 @@ variables: BUILD_TIMEOUT: 2m - # This is the jinja2 template file used to generate the build pipeline - BUILD_PIPELINE_TEMPLATE: build-pipeline-ci-test.yml.jinja2 - # The master branch is hardcoded here, because it cannot be determined automatically. # Has to be modified for new branches, e.g. new Yocto versions or fix releases. MASTER_BRANCH: master @@ -30,6 +27,68 @@ variables: seco-ne/yocto/infrastructure/ci-test/minimal-bar seco-ne/yocto/infrastructure/ci-test/minimal-foo -build-pipeline: + # List of machines to simulate building images for + CI_PARAM_MACHINES: seco-mx6 + +.yocto-deploy: + variables: + # FIXME: For now we need a quoted dollar sign here due to a GitLab bug: + # https://gitlab.com/gitlab-org/gitlab/-/issues/273409 + # Escaped variables are not correctly passed to child pipelines. Proposed workaround + # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we + # update our GitLab, we can use the former mechanism using '$$RELEASE_NAME'. + DEPLOY_RELEASE_TARGET: /artifacts-yocto/Test/"$"{RELEASE_NAME} + DEPLOY_RELEASE_TARGET_LINK: >- + Z:/Development/SoftwareStore/Linux-Yocto/Test/"$"{RELEASE_NAME} + DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-yocto/Test/"$"{RELEASE_NAME} + DEPLOY_INTERNAL_RELEASE_TARGET_LINK: >- + Z:/Development/SoftwareStore/Linux-Yocto/Test/"$"{RELEASE_NAME} + DEPLOY_FTP_TARGET: /artifacts-ftp-yocto/Test/"$"{RELEASE_NAME} + DEPLOY_FTP_TARGET_LINK: >- + http://support.garz-fricke.com/projects/Linux-Yocto/Test/"$"{RELEASE_NAME} + +generate-build-pipeline: + variables: + SIMULATE_BUILD: "true" + +ci-test-pipeline: + stage: trigger + needs: [] + trigger: + include: + - project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci' + ref: ${GITLAB_CI_REVISION} + file: build-pipeline-ci-test.yml + strategy: depend + +yocto-simulation-pipeline: + extends: + - .build-pipeline + - .yocto-deploy + variables: + BUILD_ARTIFACTS_PREFIX: >- + https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/7.0/download?job=build + CI_PARAM_IMAGE: seconorth-image + CI_PARAM_DISTRO: seconorth-wayland + INSTALLSCRIPT: fng-install.sh + ARTIFACTS_PATH: build-*/tmp/deploy/images/**/* + PACKAGE_TYPE: image + TEST_STAGE: "true" + TEST_REPO_BRANCH: dunfell + ALPHAPLAN_STAGE: "true" + AP_WEBSERVICE_URL: >- + https://SRV06.hamburg.garz-fricke.de/Test/Alphaplan-API/Artikel/CreateFirmware + +sdk-simulation-pipeline: extends: - .build-pipeline + - .yocto-deploy + variables: + BUILD_ARTIFACTS_PREFIX: >- + https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/7.0/download?job=buildsdk + CI_PARAM_IMAGE: seconorth-image + CI_PARAM_DISTRO: seconorth-wayland + INSTALLSCRIPT: fng-install.sh + ARTIFACTS_PATH: build-*/tmp/deploy/sdk/* + MANUAL_BUILD: "true" + PACKAGE_TYPE: sdk diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml index 3eca87dfa59e5f8402af079da7ce9cb70dd24c40..c43a76f27091c4f015156671d451e40cb46e792d 100644 --- a/manifest-pipeline-yocto.yml +++ b/manifest-pipeline-yocto.yml @@ -18,9 +18,6 @@ variables: BUILD_TIMEOUT: 1h - # This is the jinja2 template file used to generate the build pipeline - BUILD_PIPELINE_TEMPLATE: build-pipeline-yocto.yml.jinja2 - # Projects to include in the changelog in addition to the manifest project CHANGELOG_PROJECTS: seco-ne/yocto/layers/meta-seconorth-distro @@ -37,25 +34,25 @@ variables: # is to use raw variables, but we need at least GitLab 15.6 for that. As soon as we # update our GitLab, we can use the former mechanism using '$$RELEASE_NAME'. DEPLOY_RELEASE_TARGET: /artifacts-yocto/Releases/"$"{RELEASE_NAME} - DEPLOY_RELEASE_TARGET_LINK: > + DEPLOY_RELEASE_TARGET_LINK: >- Z:/Development/SoftwareStore/Linux-Yocto/Releases/"$"{RELEASE_NAME} DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-yocto/Interne_Releases/"$"{RELEASE_NAME} - DEPLOY_INTERNAL_RELEASE_TARGET_LINK: > + DEPLOY_INTERNAL_RELEASE_TARGET_LINK: >- Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/"$"{RELEASE_NAME} DEPLOY_FTP_TARGET: /artifacts-ftp-yocto/Releases/"$"{RELEASE_NAME} - DEPLOY_FTP_TARGET_LINK: > + DEPLOY_FTP_TARGET_LINK: >- http://support.garz-fricke.com/projects/Linux-Yocto/Releases/"$"{RELEASE_NAME} .fngsystem-deploy: variables: DEPLOY_RELEASE_TARGET: /artifacts-fngsystem/"$"{RELEASE_NAME} - DEPLOY_RELEASE_TARGET_LINK: > + DEPLOY_RELEASE_TARGET_LINK: >- Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME} DEPLOY_INTERNAL_RELEASE_TARGET: /artifacts-fngsystem/CI_Builds/"$"{RELEASE_NAME} - DEPLOY_INTERNAL_RELEASE_TARGET_LINK: > + DEPLOY_INTERNAL_RELEASE_TARGET_LINK: >- Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/"$"{RELEASE_NAME} DEPLOY_FTP_TARGET: /artifacts-ftp-fngsystem/"$"{RELEASE_NAME} - DEPLOY_FTP_TARGET_LINK: > + DEPLOY_FTP_TARGET_LINK: >- http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/"$"{RELEASE_NAME} yocto-pipeline: diff --git a/manifest-pipeline.yml b/manifest-pipeline.yml index b4f7e3a3475fc05bdccb6823facaebb7f0f83414..da573c840e1e4cc341438bbf07d3f7ec8abd5eed 100644 --- a/manifest-pipeline.yml +++ b/manifest-pipeline.yml @@ -65,6 +65,8 @@ generate-build-pipeline: - .infrastructure - .full_build_pipeline stage: manifest-pipeline + variables: + BUILD_PIPELINE_TEMPLATE: build-pipeline-yocto.yml.jinja2 script: - echo "Generating build pipeline from template file '${BUILD_PIPELINE_TEMPLATE}'" # The job generation script implicitly passes the OS environment to the template, so