diff --git a/build-common.yml b/build-common.yml new file mode 100644 index 0000000000000000000000000000000000000000..4c49833f002e207b92a610ed1892be7d4af694ab --- /dev/null +++ b/build-common.yml @@ -0,0 +1,111 @@ +--- +workflow: + rules: + # This rule is needed, as otherwise the workflow:rules from + # the parent job seem to be used and prevent the pipeline generation + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" + +.docker_check: + # Check if the build folder is empty. Sometimes the docker volume for the build is not + # removed afterwards (e.g. in case of a timeout), then a follow-up build might fail. + - |- + if [ "$(ls -A)" ]; then + echo "ERROR: Build folder is not empty. This might be the case because the" \ + "docker volume has not been not removed in a previous job. Please check" \ + "the docker container \"$CI_JOB_IMAGE\" on the GitLab runner" \ + "\"$CI_RUNNER_SHORT_TOKEN\" ($CI_RUNNER_DESCRIPTION) and remove any" \ + "attached volumes from it." + exit 1 + fi + +.setup_ssh: + # Setup ssh key to access private repos + # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor + # An SSH keypair has been generated for the manifest's pipeline to be able to access + # all private repositories in the manifest. The private key has been stored in the + # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has + # been added as a deploy key in GitLab's repository settings for all contained + # repositories. + - echo "${LOGPREFIX} Setup SSH" + - eval $(ssh-agent -s) + - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add - + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + # Add GitLab server to known hosts + # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys + # In order to access the private repositories via SSH, the GitLab server has to be + # added to the known_hosts file. The host keys were determined using the command + # ssh-keyscan [-p port] <gitlab-server> + # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable. + - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + +.repo_checkout: &repo_checkout + - echo "${LOGPREFIX} Perform repo checkout" + - cd ${CI_PROJECT_DIR} + - repo init --submodules -u ${CI_REPOSITORY_URL} + -b refs/pipelines/${CI_PIPELINE_ID} + - repo sync --detach --current-branch --no-tags --force-remove-dirty + --optimized-fetch --force-sync + +.buildbase: + tags: + - builds + timeout: 8h + interruptible: true + image: + name: "${CI_IMAGE_YOCTO}" + # Override entrypoint so we can pass --id to set the UID and GID for the + # user that is created in the container. This is a feature of the + # crops/poky images. See poky-entry.py for details. + entrypoint: + - "/usr/bin/distro-entry.sh" + - "/usr/bin/dumb-init" + - "--" + - "/usr/bin/poky-entry.py" + - "--id=118:998" + artifacts: + expire_in: 4 weeks + +.test: + extends: + - .infrastructure + timeout: 1h + rules: + - when: manual + allow_failure: true + variables: + LOGPREFIX: "CI:test:" + # Include git submodules + GIT_SUBMODULE_STRATEGY: recursive + CI_PARAM_TEST_SUITE: '{platform}.jinja2' + CI_PARAM_EXTRA: --nop + DEPLOYPATH_TEST: /artifacts/${CI_JOB_ID}/ + GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}" + GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}" + TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git" + TEST_REPO_BRANCH: ${MASTER_BRANCH_MANIFEST} + artifacts: + when: always + paths: + - "results/**" + reports: + junit: results/results-*.xml + before_script: + - !reference [.setup_ssh] + script: + # Submit tests to lava server + - |- + RELEASE=${RELEASE:-$(ls ${DEPLOYPATH_TEST}/)} + INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh" + FNG_INSTALL_URL="${ARTIFACTS_HOST_URL}/${INSTALLSCRIPT_ABS#/*/}" + .gitlab-ci/scripts/submit_test.py \ + --fng-install "${FNG_INSTALL_URL}" \ + --name \ + "Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \ + --results-path "results" \ + --test-repo ${TESTS_GIT_URL} \ + --test-repo-branch ${TEST_REPO_BRANCH} \ + --test-plan ${CI_PARAM_TEST_SUITE} \ + ${CI_PARAM_EXTRA} \ + ${CI_PARAM_PLATFORMS} diff --git a/build-jobs-ci-test.jinja2 b/build-jobs-ci-test.jinja2 index 4aebd92c97568bd5c524ae5c5ced631fdd9dcaa7..cfad8a92c0de4e21536b5d2d7217d8a468d11f23 100644 --- a/build-jobs-ci-test.jinja2 +++ b/build-jobs-ci-test.jinja2 @@ -7,56 +7,17 @@ include: - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' ref: {{ GITLAB_CI_REVISION }} - file: 'common.yml' - -workflow: - rules: - # This rule is needed, as otherwise the workflow:rules from - # the parent job seem to be used and prevent the pipeline generation - - if: $CI_PIPELINE_SOURCE == "parent_pipeline" + file: + - build-common.yml + - common.yml stages: - - infrastructure - build - test -# -------------------------------------------------------------------------------------- -# Stage: infrastructure -# -------------------------------------------------------------------------------------- -yamllint: - extends: .yamllint - # -------------------------------------------------------------------------------------- # Stage: build # -------------------------------------------------------------------------------------- -.setup_ssh: &setup_ssh - # Setup ssh key to access private repos - # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor - # An SSH keypair has been generated for the manifest's pipeline to be able to access - # all private repositories in the manifest. The private key has been stored in the - # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has - # been added as a deploy key in GitLab's repository settings for all contained - # repositories. - - eval $(ssh-agent -s) - - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - # Add GitLab server to known hosts - # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys - # In order to access the private repositories via SSH, the GitLab server has to be - # added to the known_hosts file. The host keys were determined using the command - # ssh-keyscan [-p port] <gitlab-server> - # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable. - - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - -.repo_checkout: &repo_checkout - # setup build dir - - cd ${CI_PROJECT_DIR} - - repo init -u ${CI_REPOSITORY_URL} -b refs/pipelines/${CI_PIPELINE_ID} - - repo sync --detach --current-branch --no-tags --force-remove-dirty - --optimized-fetch --force-sync - .build: &build - cd ${CI_PROJECT_DIR} - VERSION=$(cd .repo/manifests && git describe --tags) @@ -70,30 +31,22 @@ yamllint: build:files: stage: build + extends: + - .buildbase needs: [] tags: - infrastructure timeout: 2m - image: - name: "${CI_IMAGE_YOCTO}" - # Override entrypoint so we can pass --id to set the UID and GID for the user that - # is created in the container. This is a feature of the crops/poky images. - # See poky-entry.py for details. - entrypoint: - - "/usr/bin/distro-entry.sh" - - "/usr/bin/dumb-init" - - "--" - - "/usr/bin/poky-entry.py" - - "--id=118:998" variables: GIT_STRATEGY: none + LOGPREFIX: "CI:build:" before_script: - - *setup_ssh + - !reference [.docker_check] + - !reference [.setup_ssh] + - !reference [.repo_checkout] script: - - *repo_checkout - *build artifacts: - expire_in: 7d paths: - files-* @@ -111,7 +64,7 @@ build:echo: # -------------------------------------------------------------------------------------- # Stage: test # -------------------------------------------------------------------------------------- -.test-simple: +.test_simple: stage: test rules: - when: manual @@ -124,79 +77,16 @@ build:echo: test:pass: extends: - - .test-simple + - .test_simple variables: RETURNCODE: 0 test:fail: extends: - - .test-simple + - .test_simple variables: RETURNCODE: 1 -# --------------------------------------------------------------------------------------- -# Stage: test -# --------------------------------------------------------------------------------------- -.test: - extends: - - .infrastructure - timeout: 1h - rules: - - when: manual - allow_failure: true - variables: - # Include git submodules - GIT_SUBMODULE_STRATEGY: recursive - CI_PARAM_TEST_SUITE: '{platform}.jinja2' - CI_PARAM_EXTRA: --nop - LOGPREFIX: "CI:test:" - artifacts: - when: always - paths: - - "results/**" - reports: - junit: results/results-*.xml - before_script: - # Setup ssh key to access private repos - # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor - # An SSH keypair has been generated for the manifest's pipeline to be able to access - # all private repositories in the manifest. The private key has been stored in the - # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has - # been added as a deploy key in GitLab's repository settings for all contained - # repositories. - - echo "${LOGPREFIX} Setup SSH" - - eval $(ssh-agent -s) - - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - # Add GitLab server to known hosts - # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys - # In order to access the private repositories via SSH, the GitLab server has to be - # added to the known_hosts file. The host keys were determined using the command - # ssh-keyscan [-p port] <gitlab-server> - # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable. - - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - - script: - - |- - # Submit tests to lava server - # Use existing release as we don't build in CI-test - DEPLOYPATH_TEST="http://srv73/SoftwareStore/Linux-Yocto/Releases/" - RELEASE="Yocto-dunfell-15.0/" - INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh" - FNG_INSTALL_URL="${INSTALLSCRIPT_ABS#/*/}" - .gitlab-ci/scripts/submit_test.py \ - --fng-install "${FNG_INSTALL_URL}" \ - --name \ - "Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \ - --results-path "results" \ - --test-repo ${TESTS_GIT_URL} \ - --test-repo-branch dunfell \ - --test-plan ${CI_PARAM_TEST_SUITE} \ - ${CI_PARAM_EXTRA} \ - ${CI_PARAM_PLATFORMS} - smoketest:imx6guf: extends: .test stage: test @@ -205,3 +95,8 @@ smoketest:imx6guf: CI_PARAM_PLATFORMS: imx6guf CI_PARAM_TEST_SUITE: boot.jinja2 CI_PARAM_EXTRA: --all-devices + # Use existing hard-coded release as we don't build in CI-test + ARTIFACTS_HOST_URL: http://srv73 + DEPLOYPATH_TEST: SoftwareStore/Linux-Yocto/Releases/ + RELEASE: Yocto-dunfell-15.0 + TEST_REPO_BRANCH: dunfell diff --git a/build-jobs-yocto.jinja2 b/build-jobs-yocto.jinja2 index 315a246e135f78d06571ec290ec5a2b133b0fbb4..05f0b6a1469a5a97b3c1824fb8481657eab6cfc1 100644 --- a/build-jobs-yocto.jinja2 +++ b/build-jobs-yocto.jinja2 @@ -7,22 +7,11 @@ include: - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' ref: {{ GITLAB_CI_REVISION }} - file: 'common.yml' - - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' - ref: {{ GITLAB_CI_REVISION }} - file: 'manifest-build.yml' - - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' - ref: {{ GITLAB_CI_REVISION }} - file: 'manifest-package.yml' - -variables: - MASTER_BRANCH_MANIFEST: {{ MASTER_BRANCH_MANIFEST }} - -workflow: - rules: - # This rule is needed, as otherwise the workflow:rules from - # the parent job seem to be used and prevent the pipeline generation - - if: $CI_PIPELINE_SOURCE == "parent_pipeline" + file: + - build-common.yml + - build-yocto.yml + - common.yml + - manifest-package.yml stages: - infrastructure @@ -31,10 +20,12 @@ stages: - deploy - uploadftp +variables: + MASTER_BRANCH_MANIFEST: {{ MASTER_BRANCH_MANIFEST }} + # -------------------------------------------------------------------------------------- # Stage: infrastructure # -------------------------------------------------------------------------------------- - changelog: extends: .infrastructure script: .gitlab-ci/scripts/changelog_generator.py @@ -49,7 +40,6 @@ changelog: # -------------------------------------------------------------------------------------- # Generated build jobs # -------------------------------------------------------------------------------------- - {% if CI_PARAM_DISTRO is not defined %} {% set CI_PARAM_DISTRO = "guf-wayland" %} {% endif %} @@ -132,7 +122,9 @@ uploadftp-sdk-{{ machine }}: {% if platforms %} # Run smoketests for this machine which the yocto image smoketest:{{ machine }}: - extends: .test + extends: + - .test + - .test_override stage: test needs: - job: build-{{ machine }} @@ -143,7 +135,9 @@ smoketest:{{ machine }}: CI_PARAM_EXTRA: --all-devices platformtest:{{ machine }}: - extends: .test + extends: + - .test + - .test_override stage: test needs: - job: build-{{ machine }} diff --git a/manifest-build.yml b/build-yocto.yml similarity index 51% rename from manifest-build.yml rename to build-yocto.yml index fb8a08652156c4ba8891fd67d9db4a463b28d701..21acf89df87a1326d4c2ca1dfba708fa4b4fafd1 100644 --- a/manifest-build.yml +++ b/build-yocto.yml @@ -6,49 +6,6 @@ variables: LICENSESPATH: "tmp/deploy/licenses" SDKPATH: "tmp/deploy/sdk/" -.docker_check: &docker_check - # Check if the build folder is empty. Sometimes the docker volume for the build is not - # removed afterwards (e.g. in case of a timeout), then a follow-up build might fail. - - |- - if [ "$(ls -A)" ]; then - echo "ERROR: Build folder is not empty. This might be the case because the" \ - "docker volume has not been not removed in a previous job. Please check" \ - "the docker container \"$CI_JOB_IMAGE\" on the GitLab runner" \ - "\"$CI_RUNNER_SHORT_TOKEN\" ($CI_RUNNER_DESCRIPTION) and remove any" \ - "attached volumes from it." - exit 1 - fi - -.setup_ssh: &setup_ssh - # Setup ssh key to access private repos - # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor - # An SSH keypair has been generated for the manifest's pipeline to be able to access - # all private repositories in the manifest. The private key has been stored in the - # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has - # been added as a deploy key in GitLab's repository settings for all contained - # repositories. - - echo "${LOGPREFIX} Setup SSH" - - eval $(ssh-agent -s) - - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - # Add GitLab server to known hosts - # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys - # In order to access the private repositories via SSH, the GitLab server has to be - # added to the known_hosts file. The host keys were determined using the command - # ssh-keyscan [-p port] <gitlab-server> - # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable. - - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - -.repo_checkout: &repo_checkout - - echo "${LOGPREFIX} Perform repo checkout" - - cd ${CI_PROJECT_DIR} - - repo init --submodules -u ${CI_REPOSITORY_URL} - -b refs/pipelines/${CI_PIPELINE_ID} - - repo sync --detach --current-branch --no-tags --force-remove-dirty - --optimized-fetch --force-sync - .collect_srcrevs: &collect_srcrevs # write all package AUTOREVS to file - |- @@ -103,9 +60,9 @@ variables: BITBAKE_TASK: "build" LOGPREFIX: "CI:build:" before_script: - - *docker_check - - *setup_ssh - - *repo_checkout + - !reference [.docker_check] + - !reference [.setup_ssh] + - !reference [.repo_checkout] script: - *build_script - *collect_srcrevs @@ -114,40 +71,9 @@ variables: reports: dotenv: build.env -.prepare_test: - variables: - LOGPREFIX: "CI:test:" - before_script: - - *setup_ssh - - |- - # Copy artifacts to local server for automated tests - echo "${LOGPREFIX} Copy files for automated tests using package_release" - .gitlab-ci/scripts/package_release.py \ - --images-dir="${BUILDPATH}/${IMAGEPATH}" \ - --outputdir-local="${DEPLOYPATH_TEST}" - # -------------------------------------------------------------------------------------- # Stage: build # -------------------------------------------------------------------------------------- -.buildbase: - tags: - - builds - timeout: 8h - interruptible: true - image: - name: "${CI_IMAGE_YOCTO}" - # Override entrypoint so we can pass --id to set the UID and GID for the - # user that is created in the container. This is a feature of the - # crops/poky images. See poky-entry.py for details. - entrypoint: - - "/usr/bin/distro-entry.sh" - - "/usr/bin/dumb-init" - - "--" - - "/usr/bin/poky-entry.py" - - "--id=118:998" - artifacts: - expire_in: 4 weeks - .buildimage: extends: - .buildbase @@ -171,48 +97,24 @@ variables: variables: BITBAKE_TASK: "populate_sdk" -# --------------------------------------------------------------------------------------- +# -------------------------------------------------------------------------------------- # Stage: test -# --------------------------------------------------------------------------------------- -.test: - extends: - - .infrastructure - - .prepare_test - timeout: 1h - rules: - - when: manual - allow_failure: true - variables: - # Include git submodules - GIT_SUBMODULE_STRATEGY: recursive - CI_PARAM_TEST_SUITE: '{platform}.jinja2' - CI_PARAM_EXTRA: --nop - artifacts: - when: always - paths: - - "results/**" - reports: - junit: results/results-*.xml +# -------------------------------------------------------------------------------------- +.test_override: + before_script: + - !reference [.setup_ssh] + # Copy artifacts to local server for automated tests + # This is necessary because the LAVA devices cannot install directly from GitLab + # due to missing HTTPS support in FNGSystem. + - |- + echo "${LOGPREFIX} Copy files for automated tests using package_release" + .gitlab-ci/scripts/package_release.py \ + --images-dir="${BUILDPATH}/${IMAGEPATH}" \ + --outputdir-local="${DEPLOYPATH_TEST}" after_script: + # Remove artifacts after testing - rm -r "${DEPLOYPATH_TEST}" - script: - - |- - # Submit tests to lava server - RELEASE=$(ls ${DEPLOYPATH_TEST}/) - INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh" - FNG_INSTALL_URL="${ARTIFACTS_HOST_URL}/${INSTALLSCRIPT_ABS#/*/}" - .gitlab-ci/scripts/submit_test.py \ - --fng-install "${FNG_INSTALL_URL}" \ - --name \ - "Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \ - --results-path "results" \ - --test-repo ${TESTS_GIT_URL} \ - --test-repo-branch ${MASTER_BRANCH_MANIFEST} \ - --test-plan ${CI_PARAM_TEST_SUITE} \ - ${CI_PARAM_EXTRA} \ - ${CI_PARAM_PLATFORMS} - # -------------------------------------------------------------------------------------- # Stage: deploy # -------------------------------------------------------------------------------------- diff --git a/common.yml b/common.yml index afbe59c1cf2797cf8aba541c6b849161a4748562..20f5cc499879e22792d85fb4ba1a4653dc9c78ba 100644 --- a/common.yml +++ b/common.yml @@ -14,12 +14,6 @@ variables: # may take too long GIT_DEPTH: 1 - DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/" - GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}" - GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}" - TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git" - - # -------------------------------------------------------------------------------------- # Common infrastructure settings # --------------------------------------------------------------------------------------