Refactoring: remove parsing of testdata.json in package stage

Extract and save these variables in the build stage instead and
explicitly pass them as arguments to the package script.

build-pipeline.yml

@@ -111,12 +111,21 @@ workflow:
   - bitbake "${YOCTO_IMAGE}" -c "${BITBAKE_TASK}"
   - echo -e "section_end:`date +%s`:bitbake_run\r\e[0K"
 
+.source_bitbake_env: &source_bitbake_env
+  # Extract all variable assignments from the BitBake environment and source them.
+  # The rather complicated regex also regards values containing escaped quotes.
+  - source <( ${BITBAKE_ENV_COMMAND} | egrep '^[A-Z]\w*="([^\\"]+|\\.)*"$' )
+
 .save_build_env: &save_build_env
   - cd ${CI_PROJECT_DIR}
   # Artifact paths are needed for packaging
   - echo "IMAGE_PATH=${IMAGE_PATH}" > build.env
   - echo "SDK_PATH=${SDK_PATH}" >> build.env
   - echo "LICENSES_PATH=${LICENSES_PATH}" >> build.env
+  # SDK name, artifacts and images from the BitBake environment are needed for packaging
+  - echo "TOOLCHAIN_OUTPUTNAME=${TOOLCHAIN_OUTPUTNAME}" >> build.env
+  - echo "DISTRO_IMAGES='${DISTRO_IMAGES}'" >> build.env
+  - echo "DISTRO_RELEASE_ARTEFACTS='${DISTRO_RELEASE_ARTEFACTS}'" >> build.env
   # Install script location is needed in test job
   - echo "FNG_INSTALL_URL=${FNG_INSTALL_URL}" >> build.env
 
@@ -151,16 +160,13 @@ workflow:
   rules:
     - when: always
   variables:
-    BITBAKE_ENV_COMMAND: bitbake ${YOCTO_IMAGE} -e
     # RELEASE_VERSION_EXPRESSION and RELEASE_NAME_EXPRESSION are eval'ed and then saved
     # as RELEASE_VERSION and RELEASE_NAME to version.env, so we can use deferred
     # variable expansion or even command execution to construct their values.
     RELEASE_VERSION_EXPRESSION: $${DISTRO_VERSION}
     RELEASE_NAME_EXPRESSION: Yocto-$${RELEASE_VERSION}
   script:
-    # Extract all variable assignments from the BitBake environment and source them.
-    # The rather complicated regex also regards values containing escaped quotes.
-    - source <( ${BITBAKE_ENV_COMMAND} | egrep '^[A-Z]\w*="([^\\"]+|\\.)*"$' )
+    - *source_bitbake_env
     - echo ${RELEASE_VERSION_EXPRESSION}
     - echo ${RELEASE_NAME_EXPRESSION}
     - RELEASE_VERSION=$(eval echo "${RELEASE_VERSION_EXPRESSION}")
@@ -229,6 +235,7 @@ workflow:
     IMAGE_PATH: "${BUILD_PATH}/tmp/deploy/images/${MACHINE}"
     LICENSES_PATH: "${BUILD_PATH}/tmp/deploy/licenses"
     SDK_PATH: "${BUILD_PATH}/tmp/deploy/sdk"
+    BITBAKE_ENV_COMMAND: bitbake ${YOCTO_IMAGE} -e
     JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}"
     FNG_INSTALL_URL: "${JOB_URL}/artifacts/${IMAGE_PATH}/${INSTALL_SCRIPT}"
   before_script:
@@ -239,6 +246,7 @@ workflow:
   script:
     - *run_build
     - *collect_srcrevs
+    - *source_bitbake_env
     - *save_build_env
     - *dump_install_command
 
@@ -258,6 +266,15 @@ workflow:
     - wget -O artifacts.zip ${BUILD_ARTIFACTS}
     - unzip artifacts.zip
   after_script:
+    # FIXME: This is necessary because we're using an old build for the build simulation
+    # which does not export these variables, yet. Can be removed as soon as we switch to
+    # a newer release already containing these variables, then they can be read from the
+    # existing build.env from the cache.
+    # yamllint disable-line rule:line-length
+    - export TOOLCHAIN_OUTPUTNAME="seconorth-wayland-glibc-x86_64-seconorth-image-cortexa9t2hf-neon-$MACHINE-toolchain-kirkstone-7.0"
+    - export DISTRO_IMAGES="seconorth-image-$MACHINE.tar.gz"
+    # yamllint disable-line rule:line-length
+    - export DISTRO_RELEASE_ARTEFACTS="seconorth-image-$MACHINE.tar.gz fng-install.sh pkg.py"
     - *save_build_env
   cache:
     - *cache
@@ -341,6 +358,10 @@ workflow:
   script:
     # IMAGE_PATH, SDK_PATH and LICENSES_PATH are available via build.env from build job
     # RELEASE_NAME is available via version.env from build-version job
+    # FIXME: The "--sdk-name" argument shouldn't be necessary for the image package
+    # type. It is used only for the generation of "metainfo.json", which is used for the
+    # overview page on our FTP server, which is going to be replaced by Confluence. As
+    # soon as this is done, remove the "--sdk-name" argument from the image packaging.
     - if [[ "${PACKAGE_TYPE}" == "image" ]]; then
     - .gitlab-ci/scripts/package_release.py
         --images-dir="${IMAGE_PATH}"
@@ -349,12 +370,16 @@ workflow:
         --output-dir=${RELEASE_NAME}
         --release-version=${RELEASE_VERSION}
         --machine=${MACHINE}
+        --sdk-name="${TOOLCHAIN_OUTPUTNAME}"
+        --image-artifacts="${DISTRO_IMAGES}"
+        --artifacts="${DISTRO_RELEASE_ARTEFACTS}"
     - elif [[ "${PACKAGE_TYPE}" == "sdk" ]]; then
     - .gitlab-ci/scripts/package_release.py
         --sdk-dir="${SDK_PATH}"
         --output-dir=${RELEASE_NAME}
         --release-version=${RELEASE_VERSION}
         --machine=${MACHINE}
+        --sdk-name="${TOOLCHAIN_OUTPUTNAME}"
     - fi
   cache:
     - key: ${CI_PIPELINE_ID}-${ASSOCIATED_BUILD_JOB}

scripts/package_release.py

@@ -2,10 +2,8 @@
 import argparse
 import glob
 import hashlib
-import json
 import os
 import shutil
-import sys
 
 from convert_md2html import convertmd2html
 from generate_release_metadata import generate_metadata
@@ -98,35 +96,28 @@ def main():
         dest="machine",
         required=True,
     )
+    parser.add_argument(
+        "--sdk-name",
+        help="""Name of the SDK file""",
+        dest="sdk_name",
+        required=True,
+    )
+    parser.add_argument(
+        "--image-artifacts",
+        help="""Space separated list of image artifacts""",
+        dest="image_artifacts",
+        required=True,
+    )
+    parser.add_argument(
+        "--artifacts",
+        help="""Space separated list of artifacts""",
+        dest="artifacts",
+        required=True,
+    )
     args, _ = parser.parse_known_args()
 
-    # Get bitbake variables from testdata.json file
-    testdata_files = []
-    if args.images_dir is not None:
-        testdata_files += glob.glob(os.path.join(args.images_dir, "*.testdata.json"))
-    if args.sdk_dir is not None:
-        testdata_files += glob.glob(os.path.join(args.sdk_dir, "*.testdata.json"))
-
-    # Debug output if no testdata file found
-    if not testdata_files:
-        if args.images_dir is not None:
-            print(args.images_dir)
-            for f in glob.glob(f"{args.images_dir}/*"):
-                print("-- ", f)
-        if args.sdk_dir is not None:
-            print(args.sdk_dir)
-            for f in glob.glob(f"{args.sdk_dir}/*"):
-                print("-- ", f)
-        sys.exit("ERROR: no *.testdata.json file found in image or sdk dir.")
-
-    # The required build variables from testdata have the same values for image and SDK
-    # builds, so we just read one of them.
-    with open(testdata_files[0], "r", encoding="utf-8") as f:
-        buildvars = json.load(f)
-
-    sdkname = buildvars["TOOLCHAIN_OUTPUTNAME"]
-    image_artifacts = buildvars["DISTRO_IMAGES"].split()
-    artifacts = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
+    image_artifacts = args.image_artifacts.split()
+    artifacts = args.artifacts.split()
     artifacts.append("BUILD_SRCREVS.log")
 
     # Create output directory
@@ -180,11 +171,14 @@ def main():
         artifacts.append(image_md5sums_file)
 
         # Generate metadata file
+        # FIXME: This is used for the overview page on our FTP server, which is going to
+        # be replaced by Confluence. As soon as this is done, remove this and the
+        # "--sdk-name" argument from the image packaging call in build-pipeline.yml.
         generate_metadata(
             args.machine,
             args.release_version,
             artifacts,
-            sdkname,
+            args.sdk_name,
             os.path.join(args.images_dir, "metainfo.json"),
         )
         artifacts.append(os.path.join(args.images_dir, "metainfo.json"))
@@ -194,7 +188,7 @@ def main():
 
     # Package SDK
     if args.sdk_dir is not None:
-        sdkfiles = glob.glob(os.path.join(args.sdk_dir, f"{sdkname}*"))
+        sdkfiles = glob.glob(os.path.join(args.sdk_dir, f"{args.sdk_name}*"))
 
         # Generate MD5 sums file
         sdk_md5sums_file = os.path.join(args.machine, "sdk", "md5sums.txt")