Refactoring: share build and test job basics for ci-test and Yocto

This shares YAML code for the following procedures:
- Build environment
- Source code checkout via repo
- SSH key and known hosts setup
- LAVA test submission
- Docker check if build folder is empty

build-common.yml

+---
+workflow:
+  rules:
+    # This rule is needed, as otherwise the workflow:rules from
+    # the parent job seem to be used and prevent the pipeline generation
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+
+.docker_check:
+  # Check if the build folder is empty. Sometimes the docker volume for the build is not
+  # removed afterwards (e.g. in case of a timeout), then a follow-up build might fail.
+  - |-
+    if [ "$(ls -A)" ]; then
+        echo "ERROR: Build folder is not empty. This might be the case because the" \
+             "docker volume has not been not removed in a previous job. Please check" \
+             "the docker container \"$CI_JOB_IMAGE\" on the GitLab runner" \
+             "\"$CI_RUNNER_SHORT_TOKEN\" ($CI_RUNNER_DESCRIPTION) and remove any" \
+             "attached volumes from it."
+        exit 1
+    fi
+
+.setup_ssh:
+  # Setup ssh key to access private repos
+  # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor
+  # An SSH keypair has been generated for the manifest's pipeline to be able to access
+  # all private repositories in the manifest. The private key has been stored in the
+  # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has
+  # been added as a deploy key in GitLab's repository settings for all contained
+  # repositories.
+  - echo "${LOGPREFIX} Setup SSH"
+  - eval $(ssh-agent -s)
+  - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  # Add GitLab server to known hosts
+  # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys
+  # In order to access the private repositories via SSH, the GitLab server has to be
+  # added to the known_hosts file. The host keys were determined using the command
+  #   ssh-keyscan [-p port] <gitlab-server>
+  # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable.
+  - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts
+  - chmod 644 ~/.ssh/known_hosts
+
+.repo_checkout: &repo_checkout
+  - echo "${LOGPREFIX} Perform repo checkout"
+  - cd ${CI_PROJECT_DIR}
+  - repo init --submodules -u ${CI_REPOSITORY_URL}
+      -b refs/pipelines/${CI_PIPELINE_ID}
+  - repo sync --detach --current-branch --no-tags --force-remove-dirty
+      --optimized-fetch --force-sync
+
+.buildbase:
+  tags:
+    - builds
+  timeout: 8h
+  interruptible: true
+  image:
+    name: "${CI_IMAGE_YOCTO}"
+    # Override entrypoint so we can pass --id to set the UID and GID for the
+    # user that is created in the container. This is a feature of the
+    # crops/poky images. See poky-entry.py for details.
+    entrypoint:
+      - "/usr/bin/distro-entry.sh"
+      - "/usr/bin/dumb-init"
+      - "--"
+      - "/usr/bin/poky-entry.py"
+      - "--id=118:998"
+  artifacts:
+    expire_in: 4 weeks
+
+.test:
+  extends:
+    - .infrastructure
+  timeout: 1h
+  rules:
+    - when: manual
+      allow_failure: true
+  variables:
+    LOGPREFIX: "CI:test:"
+    # Include git submodules
+    GIT_SUBMODULE_STRATEGY: recursive
+    CI_PARAM_TEST_SUITE: '{platform}.jinja2'
+    CI_PARAM_EXTRA: --nop
+    DEPLOYPATH_TEST: /artifacts/${CI_JOB_ID}/
+    GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
+    GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
+    TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
+    TEST_REPO_BRANCH: ${MASTER_BRANCH_MANIFEST}
+  artifacts:
+    when: always
+    paths:
+      - "results/**"
+    reports:
+      junit: results/results-*.xml
+  before_script:
+    - !reference [.setup_ssh]
+  script:
+    # Submit tests to lava server
+    - |-
+      RELEASE=${RELEASE:-$(ls ${DEPLOYPATH_TEST}/)}
+      INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh"
+      FNG_INSTALL_URL="${ARTIFACTS_HOST_URL}/${INSTALLSCRIPT_ABS#/*/}"
+      .gitlab-ci/scripts/submit_test.py \
+          --fng-install "${FNG_INSTALL_URL}" \
+          --name \
+          "Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \
+          --results-path "results" \
+          --test-repo ${TESTS_GIT_URL} \
+          --test-repo-branch ${TEST_REPO_BRANCH} \
+          --test-plan ${CI_PARAM_TEST_SUITE} \
+          ${CI_PARAM_EXTRA} \
+          ${CI_PARAM_PLATFORMS}

build-jobs-ci-test.jinja2

@@ -7,56 +7,17 @@
 include:
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
     ref: {{ GITLAB_CI_REVISION }}
-    file: 'common.yml'
-
-workflow:
-  rules:
-    # This rule is needed, as otherwise the workflow:rules from
-    # the parent job seem to be used and prevent the pipeline generation
-    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+    file:
+      - build-common.yml
+      - common.yml
 
 stages:
-  - infrastructure
   - build
   - test
 
-# --------------------------------------------------------------------------------------
-# Stage: infrastructure
-# --------------------------------------------------------------------------------------
-yamllint:
-  extends: .yamllint
-
 # --------------------------------------------------------------------------------------
 # Stage: build
 # --------------------------------------------------------------------------------------
-.setup_ssh: &setup_ssh
-  # Setup ssh key to access private repos
-  # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor
-  # An SSH keypair has been generated for the manifest's pipeline to be able to access
-  # all private repositories in the manifest. The private key has been stored in the
-  # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has
-  # been added as a deploy key in GitLab's repository settings for all contained
-  # repositories.
-  - eval $(ssh-agent -s)
-  - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-  - mkdir -p ~/.ssh
-  - chmod 700 ~/.ssh
-  # Add GitLab server to known hosts
-  # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys
-  # In order to access the private repositories via SSH, the GitLab server has to be
-  # added to the known_hosts file. The host keys were determined using the command
-  #   ssh-keyscan [-p port] <gitlab-server>
-  # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable.
-  - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts
-  - chmod 644 ~/.ssh/known_hosts
-
-.repo_checkout: &repo_checkout
-  # setup build dir
-  - cd ${CI_PROJECT_DIR}
-  - repo init -u ${CI_REPOSITORY_URL} -b refs/pipelines/${CI_PIPELINE_ID}
-  - repo sync --detach --current-branch --no-tags --force-remove-dirty
-              --optimized-fetch --force-sync
-
 .build: &build
   - cd ${CI_PROJECT_DIR}
   - VERSION=$(cd .repo/manifests && git describe --tags)
@@ -70,30 +31,22 @@ yamllint:
 
 build:files:
   stage: build
+  extends:
+    - .buildbase
   needs: []
   tags:
     - infrastructure
   timeout: 2m
-  image:
-    name: "${CI_IMAGE_YOCTO}"
-    # Override entrypoint so we can pass --id to set the UID and GID for the user that
-    # is created in the container. This is a feature of the crops/poky images.
-    # See poky-entry.py for details.
-    entrypoint:
-      - "/usr/bin/distro-entry.sh"
-      - "/usr/bin/dumb-init"
-      - "--"
-      - "/usr/bin/poky-entry.py"
-      - "--id=118:998"
   variables:
     GIT_STRATEGY: none
+    LOGPREFIX: "CI:build:"
   before_script:
-    - *setup_ssh
+    - !reference [.docker_check]
+    - !reference [.setup_ssh]
+    - !reference [.repo_checkout]
   script:
-    - *repo_checkout
     - *build
   artifacts:
-    expire_in: 7d
     paths:
       - files-*
 
@@ -111,7 +64,7 @@ build:echo:
 # --------------------------------------------------------------------------------------
 # Stage: test
 # --------------------------------------------------------------------------------------
-.test-simple:
+.test_simple:
   stage: test
   rules:
     - when: manual
@@ -124,79 +77,16 @@ build:echo:
 
 test:pass:
   extends:
-    - .test-simple
+    - .test_simple
   variables:
     RETURNCODE: 0
 
 test:fail:
   extends:
-    - .test-simple
+    - .test_simple
   variables:
     RETURNCODE: 1
 
-# ---------------------------------------------------------------------------------------
-# Stage: test
-# ---------------------------------------------------------------------------------------
-.test:
-  extends:
-    - .infrastructure
-  timeout: 1h
-  rules:
-    - when: manual
-      allow_failure: true
-  variables:
-    # Include git submodules
-    GIT_SUBMODULE_STRATEGY: recursive
-    CI_PARAM_TEST_SUITE: '{platform}.jinja2'
-    CI_PARAM_EXTRA: --nop
-    LOGPREFIX: "CI:test:"
-  artifacts:
-    when: always
-    paths:
-      - "results/**"
-    reports:
-      junit: results/results-*.xml
-  before_script:
-    # Setup ssh key to access private repos
-    # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor
-    # An SSH keypair has been generated for the manifest's pipeline to be able to access
-    # all private repositories in the manifest. The private key has been stored in the
-    # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has
-    # been added as a deploy key in GitLab's repository settings for all contained
-    # repositories.
-    - echo "${LOGPREFIX} Setup SSH"
-    - eval $(ssh-agent -s)
-    - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    # Add GitLab server to known hosts
-    # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys
-    # In order to access the private repositories via SSH, the GitLab server has to be
-    # added to the known_hosts file. The host keys were determined using the command
-    #   ssh-keyscan [-p port] <gitlab-server>
-    # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable.
-    - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-
-  script:
-    - |-
-      # Submit tests to lava server
-      # Use existing release as we don't build in CI-test
-      DEPLOYPATH_TEST="http://srv73/SoftwareStore/Linux-Yocto/Releases/"
-      RELEASE="Yocto-dunfell-15.0/"
-      INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh"
-      FNG_INSTALL_URL="${INSTALLSCRIPT_ABS#/*/}"
-      .gitlab-ci/scripts/submit_test.py \
-          --fng-install "${FNG_INSTALL_URL}" \
-          --name \
-          "Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \
-          --results-path "results" \
-          --test-repo ${TESTS_GIT_URL} \
-          --test-repo-branch dunfell \
-          --test-plan ${CI_PARAM_TEST_SUITE} \
-          ${CI_PARAM_EXTRA} \
-          ${CI_PARAM_PLATFORMS}
-
 smoketest:imx6guf:
   extends: .test
   stage: test
@@ -205,3 +95,8 @@ smoketest:imx6guf:
     CI_PARAM_PLATFORMS: imx6guf
     CI_PARAM_TEST_SUITE: boot.jinja2
     CI_PARAM_EXTRA: --all-devices
+    # Use existing hard-coded release as we don't build in CI-test
+    ARTIFACTS_HOST_URL: http://srv73
+    DEPLOYPATH_TEST: SoftwareStore/Linux-Yocto/Releases/
+    RELEASE: Yocto-dunfell-15.0
+    TEST_REPO_BRANCH: dunfell

build-jobs-yocto.jinja2

@@ -7,22 +7,11 @@
 include:
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
     ref: {{ GITLAB_CI_REVISION }}
-    file: 'common.yml'
-  - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_REVISION }}
-    file: 'manifest-build.yml'
-  - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_REVISION }}
-    file: 'manifest-package.yml'
-
-variables:
-  MASTER_BRANCH_MANIFEST: {{ MASTER_BRANCH_MANIFEST }}
-
-workflow:
-  rules:
-    # This rule is needed, as otherwise the workflow:rules from
-    # the parent job seem to be used and prevent the pipeline generation
-    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+    file:
+      - build-common.yml
+      - build-yocto.yml
+      - common.yml
+      - manifest-package.yml
 
 stages:
   - infrastructure
@@ -31,10 +20,12 @@ stages:
   - deploy
   - uploadftp
 
+variables:
+  MASTER_BRANCH_MANIFEST: {{ MASTER_BRANCH_MANIFEST }}
+
 # --------------------------------------------------------------------------------------
 # Stage: infrastructure
 # --------------------------------------------------------------------------------------
-
 changelog:
   extends: .infrastructure
   script: .gitlab-ci/scripts/changelog_generator.py
@@ -49,7 +40,6 @@ changelog:
 # --------------------------------------------------------------------------------------
 # Generated build jobs
 # --------------------------------------------------------------------------------------
-
 {% if CI_PARAM_DISTRO is not defined %}
 {% set CI_PARAM_DISTRO = "guf-wayland" %}
 {% endif %}
@@ -132,7 +122,9 @@ uploadftp-sdk-{{ machine }}:
 {% if platforms %}
 # Run smoketests for this machine which the yocto image
 smoketest:{{ machine }}:
-  extends: .test
+  extends:
+    - .test
+    - .test_override
   stage: test
   needs:
     - job: build-{{ machine }}
@@ -143,7 +135,9 @@ smoketest:{{ machine }}:
     CI_PARAM_EXTRA: --all-devices
 
 platformtest:{{ machine }}:
-  extends: .test
+  extends:
+    - .test
+    - .test_override
   stage: test
   needs:
     - job: build-{{ machine }}

manifest-build.yml → build-yocto.yml

@@ -6,49 +6,6 @@ variables:
   LICENSESPATH: "tmp/deploy/licenses"
   SDKPATH: "tmp/deploy/sdk/"
 
-.docker_check: &docker_check
-  # Check if the build folder is empty. Sometimes the docker volume for the build is not
-  # removed afterwards (e.g. in case of a timeout), then a follow-up build might fail.
-  - |-
-    if [ "$(ls -A)" ]; then
-        echo "ERROR: Build folder is not empty. This might be the case because the" \
-             "docker volume has not been not removed in a previous job. Please check" \
-             "the docker container \"$CI_JOB_IMAGE\" on the GitLab runner" \
-             "\"$CI_RUNNER_SHORT_TOKEN\" ($CI_RUNNER_DESCRIPTION) and remove any" \
-             "attached volumes from it."
-        exit 1
-    fi
-
-.setup_ssh: &setup_ssh
-  # Setup ssh key to access private repos
-  # https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor
-  # An SSH keypair has been generated for the manifest's pipeline to be able to access
-  # all private repositories in the manifest. The private key has been stored in the
-  # CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has
-  # been added as a deploy key in GitLab's repository settings for all contained
-  # repositories.
-  - echo "${LOGPREFIX} Setup SSH"
-  - eval $(ssh-agent -s)
-  - echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-  - mkdir -p ~/.ssh
-  - chmod 700 ~/.ssh
-  # Add GitLab server to known hosts
-  # https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys
-  # In order to access the private repositories via SSH, the GitLab server has to be
-  # added to the known_hosts file. The host keys were determined using the command
-  #   ssh-keyscan [-p port] <gitlab-server>
-  # and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable.
-  - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts
-  - chmod 644 ~/.ssh/known_hosts
-
-.repo_checkout: &repo_checkout
-  - echo "${LOGPREFIX} Perform repo checkout"
-  - cd ${CI_PROJECT_DIR}
-  - repo init --submodules -u ${CI_REPOSITORY_URL}
-      -b refs/pipelines/${CI_PIPELINE_ID}
-  - repo sync --detach --current-branch --no-tags --force-remove-dirty
-      --optimized-fetch --force-sync
-
 .collect_srcrevs: &collect_srcrevs
   # write all package AUTOREVS to file
   - |-
@@ -103,9 +60,9 @@ variables:
     BITBAKE_TASK: "build"
     LOGPREFIX: "CI:build:"
   before_script:
-    - *docker_check
-    - *setup_ssh
-    - *repo_checkout
+    - !reference [.docker_check]
+    - !reference [.setup_ssh]
+    - !reference [.repo_checkout]
   script:
     - *build_script
     - *collect_srcrevs
@@ -114,40 +71,9 @@ variables:
     reports:
       dotenv: build.env
 
-.prepare_test:
-  variables:
-    LOGPREFIX: "CI:test:"
-  before_script:
-    - *setup_ssh
-    - |-
-      # Copy artifacts to local server for automated tests
-      echo "${LOGPREFIX} Copy files for automated tests using package_release"
-      .gitlab-ci/scripts/package_release.py \
-        --images-dir="${BUILDPATH}/${IMAGEPATH}" \
-        --outputdir-local="${DEPLOYPATH_TEST}"
-
 # --------------------------------------------------------------------------------------
 # Stage: build
 # --------------------------------------------------------------------------------------
-.buildbase:
-  tags:
-    - builds
-  timeout: 8h
-  interruptible: true
-  image:
-    name: "${CI_IMAGE_YOCTO}"
-    # Override entrypoint so we can pass --id to set the UID and GID for the
-    # user that is created in the container. This is a feature of the
-    # crops/poky images. See poky-entry.py for details.
-    entrypoint:
-      - "/usr/bin/distro-entry.sh"
-      - "/usr/bin/dumb-init"
-      - "--"
-      - "/usr/bin/poky-entry.py"
-      - "--id=118:998"
-  artifacts:
-    expire_in: 4 weeks
-
 .buildimage:
   extends:
     - .buildbase
@@ -171,48 +97,24 @@ variables:
   variables:
     BITBAKE_TASK: "populate_sdk"
 
-# ---------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 # Stage: test
-# ---------------------------------------------------------------------------------------
-.test:
-  extends:
-    - .infrastructure
-    - .prepare_test
-  timeout: 1h
-  rules:
-    - when: manual
-      allow_failure: true
-  variables:
-    # Include git submodules
-    GIT_SUBMODULE_STRATEGY: recursive
-    CI_PARAM_TEST_SUITE: '{platform}.jinja2'
-    CI_PARAM_EXTRA: --nop
-  artifacts:
-    when: always
-    paths:
-      - "results/**"
-    reports:
-      junit: results/results-*.xml
+# --------------------------------------------------------------------------------------
+.test_override:
+  before_script:
+    - !reference [.setup_ssh]
+    # Copy artifacts to local server for automated tests
+    # This is necessary because the LAVA devices cannot install directly from GitLab
+    # due to missing HTTPS support in FNGSystem.
+    - |-
+      echo "${LOGPREFIX} Copy files for automated tests using package_release"
+      .gitlab-ci/scripts/package_release.py \
+        --images-dir="${BUILDPATH}/${IMAGEPATH}" \
+        --outputdir-local="${DEPLOYPATH_TEST}"
   after_script:
+    # Remove artifacts after testing
     - rm -r "${DEPLOYPATH_TEST}"
 
-  script:
-    - |-
-      # Submit tests to lava server
-      RELEASE=$(ls ${DEPLOYPATH_TEST}/)
-      INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh"
-      FNG_INSTALL_URL="${ARTIFACTS_HOST_URL}/${INSTALLSCRIPT_ABS#/*/}"
-      .gitlab-ci/scripts/submit_test.py \
-          --fng-install "${FNG_INSTALL_URL}" \
-          --name \
-          "Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \
-          --results-path "results" \
-          --test-repo ${TESTS_GIT_URL} \
-          --test-repo-branch ${MASTER_BRANCH_MANIFEST} \
-          --test-plan ${CI_PARAM_TEST_SUITE} \
-          ${CI_PARAM_EXTRA} \
-          ${CI_PARAM_PLATFORMS}
-
 # --------------------------------------------------------------------------------------
 # Stage: deploy
 # --------------------------------------------------------------------------------------

common.yml

@@ -14,12 +14,6 @@ variables:
   # may take too long
   GIT_DEPTH: 1
 
-  DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/"
-  GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
-  GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
-  TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
-
-
 # --------------------------------------------------------------------------------------
 # Common infrastructure settings
 # --------------------------------------------------------------------------------------