build-pipeline-yocto.yml.jinja2

@@ -64,18 +64,11 @@ build-{{ machine }}:
     - .buildbase
     - .simulate_build
   variables:
+    MACHINE: {{ machine }}
     # We have to specify a tag here instead of getting the artifacts from a master
     # branch, because we don't always execute the full pipeline on the master branches.
     # In those cases the actual build job does not exist, which results in a 404 error.
     BUILD_ARTIFACTS: ${BUILD_ARTIFACTS_PREFIX}-{{ machine }}
-  after_script:
-    # FIXME: This is necessary because we changed variable naming, but we still use a
-    # release with the old naming for build simulation. Can be removed as soon as we
-    # switch to a newer release already containing the new naming scheme.
-    - source build.env
-    - echo "IMAGE_PATH=${BUILD_PATH_IMAGE}" >> build.env
-    - echo "SDK_PATH=${BUILD_PATH_SDK}" >> build.env
-    - echo "LICENSES_PATH=${BUILD_PATH_LICENSE}" >> build.env
 
 {% endif %}
 

build-pipeline.yml

@@ -111,12 +111,21 @@ workflow:
   - bitbake "${YOCTO_IMAGE}" -c "${BITBAKE_TASK}"
   - echo -e "section_end:`date +%s`:bitbake_run\r\e[0K"
 
+.source_bitbake_env: &source_bitbake_env
+  # Extract all variable assignments from the BitBake environment and source them.
+  # The rather complicated regex also regards values containing escaped quotes.
+  - source <( ${BITBAKE_ENV_COMMAND} | egrep '^[A-Z]\w*="([^\\"]+|\\.)*"$' )
+
 .save_build_env: &save_build_env
   - cd ${CI_PROJECT_DIR}
   # Artifact paths are needed for packaging
   - echo "IMAGE_PATH=${IMAGE_PATH}" > build.env
   - echo "SDK_PATH=${SDK_PATH}" >> build.env
   - echo "LICENSES_PATH=${LICENSES_PATH}" >> build.env
+  # SDK name, artifacts and images from the BitBake environment are needed for packaging
+  - echo "TOOLCHAIN_OUTPUTNAME=${TOOLCHAIN_OUTPUTNAME}" >> build.env
+  - echo "DISTRO_IMAGES='${DISTRO_IMAGES}'" >> build.env
+  - echo "DISTRO_RELEASE_ARTEFACTS='${DISTRO_RELEASE_ARTEFACTS}'" >> build.env
   # Install script location is needed in test job
   - echo "FNG_INSTALL_URL=${FNG_INSTALL_URL}" >> build.env
 
@@ -151,16 +160,13 @@ workflow:
   rules:
     - when: always
   variables:
-    BITBAKE_ENV_COMMAND: bitbake ${YOCTO_IMAGE} -e
     # RELEASE_VERSION_EXPRESSION and RELEASE_NAME_EXPRESSION are eval'ed and then saved
     # as RELEASE_VERSION and RELEASE_NAME to version.env, so we can use deferred
     # variable expansion or even command execution to construct their values.
     RELEASE_VERSION_EXPRESSION: $${DISTRO_VERSION}
     RELEASE_NAME_EXPRESSION: Yocto-$${RELEASE_VERSION}
   script:
-    # Extract all variable assignments from the BitBake environment and source them.
-    # The rather complicated regex also regards values containing escaped quotes.
-    - source <( ${BITBAKE_ENV_COMMAND} | egrep '^[A-Z]\w*="([^\\"]+|\\.)*"$' )
+    - *source_bitbake_env
     - echo ${RELEASE_VERSION_EXPRESSION}
     - echo ${RELEASE_NAME_EXPRESSION}
     - RELEASE_VERSION=$(eval echo "${RELEASE_VERSION_EXPRESSION}")
@@ -218,26 +224,6 @@ workflow:
       policy: push
       paths: *artifacts_paths
 
-.simulate_build:
-  extends:
-    - .buildbase
-  tags:
-    - misc
-  needs: []
-  timeout: 60m
-  variables:
-    BUILD_ARTIFACTS:
-  script:
-    - if compgen -G build-*; then echo "Using build from cache"; exit 0; fi
-    - echo "Getting Yocto build artifacts"
-    - wget -O artifacts.zip ${BUILD_ARTIFACTS}
-    - unzip artifacts.zip
-  cache:
-    - *cache
-    # Additionally cache the build artifacts for re-runs of this job in other pipelines
-    - key: ${CI_JOB_NAME}-${BUILD_ARTIFACTS}
-      paths: *artifacts_paths
-
 .build_yocto:
   extends:
     - .buildbase
@@ -249,6 +235,7 @@ workflow:
     IMAGE_PATH: "${BUILD_PATH}/tmp/deploy/images/${MACHINE}"
     LICENSES_PATH: "${BUILD_PATH}/tmp/deploy/licenses"
     SDK_PATH: "${BUILD_PATH}/tmp/deploy/sdk"
+    BITBAKE_ENV_COMMAND: bitbake ${YOCTO_IMAGE} -e
     JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}"
     FNG_INSTALL_URL: "${JOB_URL}/artifacts/${IMAGE_PATH}/${INSTALL_SCRIPT}"
   before_script:
@@ -259,9 +246,42 @@ workflow:
   script:
     - *run_build
     - *collect_srcrevs
+    - *source_bitbake_env
     - *save_build_env
     - *dump_install_command
 
+.simulate_build:
+  extends:
+    - .build_yocto
+  tags:
+    - misc
+  needs: []
+  timeout: 60m
+  variables:
+    BUILD_ARTIFACTS:
+  before_script: []
+  script:
+    - if compgen -G build-*; then echo "Using build from cache"; exit 0; fi
+    - echo "Getting Yocto build artifacts"
+    - wget -O artifacts.zip ${BUILD_ARTIFACTS}
+    - unzip artifacts.zip
+  after_script:
+    # FIXME: This is necessary because we're using an old build for the build simulation
+    # which does not export these variables, yet. Can be removed as soon as we switch to
+    # a newer release already containing these variables, then they can be read from the
+    # existing build.env from the cache.
+    # yamllint disable-line rule:line-length
+    - export TOOLCHAIN_OUTPUTNAME="seconorth-wayland-glibc-x86_64-seconorth-image-cortexa9t2hf-neon-$MACHINE-toolchain-kirkstone-7.0"
+    - export DISTRO_IMAGES="seconorth-image-$MACHINE.tar.gz"
+    # yamllint disable-line rule:line-length
+    - export DISTRO_RELEASE_ARTEFACTS="seconorth-image-$MACHINE.tar.gz fng-install.sh pkg.py"
+    - *save_build_env
+  cache:
+    - *cache
+    # Additionally cache the build artifacts for re-runs of this job in other pipelines
+    - key: ${CI_JOB_NAME}-${BUILD_ARTIFACTS}
+      paths: *artifacts_paths
+
 # --------------------------------------------------------------------------------------
 # Stage: test
 # --------------------------------------------------------------------------------------
@@ -338,6 +358,10 @@ workflow:
   script:
     # IMAGE_PATH, SDK_PATH and LICENSES_PATH are available via build.env from build job
     # RELEASE_NAME is available via version.env from build-version job
+    # FIXME: The "--sdk-name" argument shouldn't be necessary for the image package
+    # type. It is used only for the generation of "metainfo.json", which is used for the
+    # overview page on our FTP server, which is going to be replaced by Confluence. As
+    # soon as this is done, remove the "--sdk-name" argument from the image packaging.
     - if [[ "${PACKAGE_TYPE}" == "image" ]]; then
     - .gitlab-ci/scripts/package_release.py
         --images-dir="${IMAGE_PATH}"
@@ -346,12 +370,16 @@ workflow:
         --output-dir=${RELEASE_NAME}
         --release-version=${RELEASE_VERSION}
         --machine=${MACHINE}
+        --sdk-name="${TOOLCHAIN_OUTPUTNAME}"
+        --image-artifacts="${DISTRO_IMAGES}"
+        --artifacts="${DISTRO_RELEASE_ARTEFACTS}"
     - elif [[ "${PACKAGE_TYPE}" == "sdk" ]]; then
     - .gitlab-ci/scripts/package_release.py
         --sdk-dir="${SDK_PATH}"
         --output-dir=${RELEASE_NAME}
         --release-version=${RELEASE_VERSION}
         --machine=${MACHINE}
+        --sdk-name="${TOOLCHAIN_OUTPUTNAME}"
     - fi
   cache:
     - key: ${CI_PIPELINE_ID}-${ASSOCIATED_BUILD_JOB}

scripts/generate_release_metadata.py

@@ -71,4 +71,4 @@ def generate_metadata(
         metadata["files"].append(new_file)
 
     with open(output_file, "w", encoding="utf-8") as file:
-        file.write(json.dumps(metadata))
+        file.write(json.dumps(metadata, indent=4))

scripts/package_release.py

@@ -2,10 +2,8 @@
 import argparse
 import glob
 import hashlib
-import json
 import os
 import shutil
-import sys
 
 from convert_md2html import convertmd2html
 from generate_release_metadata import generate_metadata
@@ -98,37 +96,23 @@ def main():
         dest="machine",
         required=True,
     )
+    parser.add_argument(
+        "--sdk-name",
+        help="""Name of the SDK file""",
+        dest="sdk_name",
+    )
+    parser.add_argument(
+        "--image-artifacts",
+        help="""Space separated list of image artifacts""",
+        dest="image_artifacts",
+    )
+    parser.add_argument(
+        "--artifacts",
+        help="""Space separated list of artifacts""",
+        dest="artifacts",
+    )
     args, _ = parser.parse_known_args()
 
-    # Get bitbake variables from testdata.json file
-    testdata_files = []
-    if args.images_dir is not None:
-        testdata_files += glob.glob(os.path.join(args.images_dir, "*.testdata.json"))
-    if args.sdk_dir is not None:
-        testdata_files += glob.glob(os.path.join(args.sdk_dir, "*.testdata.json"))
-
-    # Debug output if no testdata file found
-    if not testdata_files:
-        if args.images_dir is not None:
-            print(args.images_dir)
-            for f in glob.glob(f"{args.images_dir}/*"):
-                print("-- ", f)
-        if args.sdk_dir is not None:
-            print(args.sdk_dir)
-            for f in glob.glob(f"{args.sdk_dir}/*"):
-                print("-- ", f)
-        sys.exit("ERROR: no *.testdata.json file found in image or sdk dir.")
-
-    # The required build variables from testdata have the same values for image and SDK
-    # builds, so we just read one of them.
-    with open(testdata_files[0], "r", encoding="utf-8") as f:
-        buildvars = json.load(f)
-
-    sdkname = buildvars["TOOLCHAIN_OUTPUTNAME"]
-    image_artifacts = buildvars["DISTRO_IMAGES"].split()
-    artifacts = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
-    artifacts.append("BUILD_SRCREVS.log")
-
     # Create output directory
     os.makedirs(args.output_dir, exist_ok=True)
 
@@ -154,6 +138,10 @@ def main():
 
     # Package image files
     if args.images_dir is not None:
+        image_artifacts = args.image_artifacts.split() if args.image_artifacts else []
+        artifacts = args.artifacts.split() if args.artifacts else []
+        artifacts.append("BUILD_SRCREVS.log")
+
         # Add some additional files to the artifacts
         for artifact in image_artifacts:
             artifacts.append(f"{artifact.split('.')[0]}.manifest")
@@ -180,11 +168,14 @@ def main():
         artifacts.append(image_md5sums_file)
 
         # Generate metadata file
+        # FIXME: This is used for the overview page on our FTP server, which is going to
+        # be replaced by Confluence. As soon as this is done, remove this and the
+        # "--sdk-name" argument from the image packaging call in build-pipeline.yml.
         generate_metadata(
             args.machine,
             args.release_version,
             artifacts,
-            sdkname,
+            args.sdk_name,
             os.path.join(args.images_dir, "metainfo.json"),
         )
         artifacts.append(os.path.join(args.images_dir, "metainfo.json"))
@@ -194,7 +185,7 @@ def main():
 
     # Package SDK
     if args.sdk_dir is not None:
-        sdkfiles = glob.glob(os.path.join(args.sdk_dir, f"{sdkname}*"))
+        sdkfiles = glob.glob(os.path.join(args.sdk_dir, f"{args.sdk_name}*"))
 
         # Generate MD5 sums file
         sdk_md5sums_file = os.path.join(args.machine, "sdk", "md5sums.txt")