YT-65 We perform a repository checkout in the manifest project for each build job.

  Each pipeline builds code for multiple MACHINES.
  This results in multiple downloads from each repository in the manifest.
  Occasionally, certain repositories may fail to fetch data.
  To avoid fetching the same data repeatedly for each build job in the pipeline,
  the `repo_checkout` stage is added.
  In this stage, the repo tool prepares the repository structure once
  and stores it in the cache.
  Subsequent build jobs use the pre-fetched data from the cache.

build-pipeline-yocto.yml.jinja2

@@ -30,6 +30,9 @@ variables:
 # --------------------------------------------------------------------------------------
 # Stage: Infrastructure
 # --------------------------------------------------------------------------------------
+repo_checkout:
+  extends: .repo_checkout
+
 changelog:
   extends: .changelog
 
@@ -39,6 +42,8 @@ build-version:
     # We have to set a machine, even if it is not actually needed for what the job does,
     # so we just choose the first.
     MACHINE: {{ MACHINES.split(' ')[0] }}
+  needs:
+    - repo_checkout
 
 
 # --------------------------------------------------------------------------------------
@@ -67,6 +72,8 @@ build-{{ machine }}:
   variables:
     BITBAKE_TASK: build
     MACHINE: {{ machine }}
+  needs:
+    - repo_checkout
 
 {% else %}
 

build-pipeline.yml

@@ -9,11 +9,12 @@ workflow:
 # Scripts
 # --------------------------------------------------------------------------------------
 .docker_check: &docker_check
-  # Check if the build folder is empty. Sometimes the docker volume for the build is not
-  # removed afterwards (e.g. in case of a timeout), then a follow-up build might fail.
+  # heck if the build folder does not contain outdated data.
+  # Sometimes the docker volume for the build is not removed afterwards
+  # (e.g. in case of a timeout), then a follow-up build might fail.
   - |-
-    if [ "$(ls -A)" ]; then
-        echo "ERROR: Build folder is not empty. This might be the case because the" \
+    if [ "$(ls -A | grep -v repo_checkout)" ]; then
+        echo "ERROR: Build folder is not clear. This might be the case because the" \
              "docker volume has not been removed in a previous job. Please check" \
              "the docker container \"$CI_JOB_IMAGE\" on the GitLab runner" \
              "\"$CI_RUNNER_SHORT_TOKEN\" ($CI_RUNNER_DESCRIPTION) and remove any" \
@@ -95,14 +96,6 @@ workflow:
     done <<< "${GITLAB_PRIVATE_TOKEN}";
     IFS=$original_ifs;
 
-.repo_checkout: &repo_checkout
-  - echo "Perform repo checkout"
-  - cd ${CI_PROJECT_DIR}
-  - repo init --submodules -u ${CI_REPOSITORY_URL}
-      -b refs/pipelines/${CI_PIPELINE_ID} -m ${MANIFEST_FILE}
-  - repo sync --detach --current-branch --force-remove-dirty
-      --optimized-fetch --force-sync
-
 .collect_srcrevs: &collect_srcrevs
   # write all package AUTOREVS to file
   - |-
@@ -146,6 +139,13 @@ workflow:
     fi
 
 .setup_build: &setup_build
+  - cd ${CI_PROJECT_DIR}
+  - |
+    if [ -d repo_checkout ]; then
+      echo "Unpack repo from chached rirectory"
+      find repo_checkout -mindepth 1 -maxdepth 1 -exec mv -t . {} +
+      rm -r repo_checkout
+    fi
   - echo "Build configuration MACHINE=${MACHINE}
       DISTRO=${YOCTO_DISTRO} IMAGE=${YOCTO_IMAGE}"
   - echo "Using build dir ${BUILD_PATH}"
@@ -182,9 +182,53 @@ workflow:
   - echo "MACHINE=${MACHINE}" >> build.env
   - echo "SUPPORTED_HARDWARE='${SUPPORTED_HARDWARE}'" >> build.env
 
+.build_image:
+  image:
+    name: "${CI_IMAGE_YOCTO}"
+    # Override entrypoint so we can pass --id to set the UID and GID for the
+    # user that is created in the container. This is a feature of the
+    # crops/poky images. See poky-entry.py for details.
+    entrypoint:
+      - "/usr/bin/distro-entry.sh"
+      - "/usr/bin/dumb-init"
+      - "--"
+      - "/usr/bin/poky-entry.py"
+      - "--id=118:998"
+
 # --------------------------------------------------------------------------------------
 # Stage: Infrastructure
 # --------------------------------------------------------------------------------------
+
+# Checkout the repository and store it in the cache to use cached data
+# instead of fetching it from Git for each build job.
+.repo_checkout:
+  extends:
+    - .build_image
+  stage: Infrastructure
+  tags:
+    - builds
+  variables:
+    GIT_STRATEGY: none
+    MANIFEST_FILE: "default.xml"
+  before_script:
+    - *docker_check
+    - *setup_ssh
+    - *setup_git_credentials
+  script:
+    - echo "Perform repo checkout"
+    - cd ${CI_PROJECT_DIR}
+    - mkdir repo_checkout
+    - cd repo_checkout
+    - repo init --submodules -u ${CI_REPOSITORY_URL}
+        -b refs/pipelines/${CI_PIPELINE_ID} -m ${MANIFEST_FILE}
+    - repo sync --detach --current-branch --force-remove-dirty
+        --optimized-fetch --force-sync
+  cache:
+    - key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
+      policy: push
+      paths:
+        - ${CI_PROJECT_DIR}/repo_checkout
+
 .changelog:
   extends: .infrastructure
   script:
@@ -208,10 +252,10 @@ workflow:
     - .build_yocto
   stage: Infrastructure
   tags:
-    - infrastructure
+    - builds
   needs: []
   rules:
-    - when: always
+    - when: on_success
   variables:
     # RELEASE_VERSION_EXPRESSION and RELEASE_NAME_EXPRESSION are eval'ed and then saved
     # as RELEASE_VERSION and RELEASE_NAME to version.env, so we can use deferred
@@ -234,32 +278,27 @@ workflow:
     reports:
       dotenv: version.env
     paths: []
-  cache: []
+  cache: &cache
+    # Get prepared repo from cache
+    - key: ${CI_PIPELINE_ID}-repo_checkout
+      policy: pull
+      paths:
+        - ${CI_PROJECT_DIR}/repo_checkout
 
 # --------------------------------------------------------------------------------------
 # Stage: build
 # --------------------------------------------------------------------------------------
 .buildbase:
+  extends:
+    - .build_image
   stage: Build
   tags:
     - ${BUILD_RUNNER_TAG}
   timeout: 8h
   interruptible: true
-  image:
-    name: "${CI_IMAGE_YOCTO}"
-    # Override entrypoint so we can pass --id to set the UID and GID for the
-    # user that is created in the container. This is a feature of the
-    # crops/poky images. See poky-entry.py for details.
-    entrypoint:
-      - "/usr/bin/distro-entry.sh"
-      - "/usr/bin/dumb-init"
-      - "--"
-      - "/usr/bin/poky-entry.py"
-      - "--id=118:998"
   variables:
     BUILD_RUNNER_TAG: "builds"
     MANUAL_BUILD: "false"
-    MANIFEST_FILE: "default.xml"
   rules:
     - if: $MANUAL_BUILD == "true"
       when: manual
@@ -279,6 +318,11 @@ workflow:
       - build-*/tmp/deploy/licenses/**/license.manifest
       - build.env
   cache: &cache
+    # Get prepared repo from cache
+    - key: ${CI_PIPELINE_ID}-repo_checkout
+      policy: pull
+      paths:
+        - ${CI_PROJECT_DIR}/repo_checkout
     # Cache the build artifacts for subsequent stages in the same pipeline
     - key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
       policy: push
@@ -302,7 +346,6 @@ workflow:
     - *docker_check
     - *setup_ssh
     - *setup_git_credentials
-    - *repo_checkout
     - *setup_build
   script:
     - *run_build