From 45f2aca73b8c48f0c456030d884c7d4c0c4deda0 Mon Sep 17 00:00:00 2001
From: Tim Jaacks <tim.jaacks@seco.com>
Date: Thu, 14 Sep 2023 11:57:46 +0200
Subject: [PATCH] Refactoring: remove parsing of testdata.json in package stage
Extract and save these variables in the build stage instead and
explicitly pass them as arguments to the package script.
---
build-pipeline.yml | 33 +++++++++++++++++++---
scripts/package_release.py | 57 ++++++++++++++++----------------------
2 files changed, 53 insertions(+), 37 deletions(-)
diff --git a/build-pipeline.yml b/build-pipeline.yml
index 89df3345..f4c09c78 100644
--- a/build-pipeline.yml
+++ b/build-pipeline.yml
@@ -111,12 +111,21 @@ workflow:
- bitbake "${YOCTO_IMAGE}" -c "${BITBAKE_TASK}"
- echo -e "section_end:`date +%s`:bitbake_run\r\e[0K"
+.source_bitbake_env: &source_bitbake_env
+ # Extract all variable assignments from the BitBake environment and source them.
+ # The rather complicated regex also regards values containing escaped quotes.
+ - source <( ${BITBAKE_ENV_COMMAND} | egrep '^[A-Z]\w*="([^\\"]+|\\.)*"$' )
+
.save_build_env: &save_build_env
- cd ${CI_PROJECT_DIR}
# Artifact paths are needed for packaging
- echo "IMAGE_PATH=${IMAGE_PATH}" > build.env
- echo "SDK_PATH=${SDK_PATH}" >> build.env
- echo "LICENSES_PATH=${LICENSES_PATH}" >> build.env
+ # SDK name, artifacts and images from the BitBake environment are needed for packaging
+ - echo "TOOLCHAIN_OUTPUTNAME=${TOOLCHAIN_OUTPUTNAME}" >> build.env
+ - echo "DISTRO_IMAGES='${DISTRO_IMAGES}'" >> build.env
+ - echo "DISTRO_RELEASE_ARTEFACTS='${DISTRO_RELEASE_ARTEFACTS}'" >> build.env
# Install script location is needed in test job
- echo "FNG_INSTALL_URL=${FNG_INSTALL_URL}" >> build.env
@@ -151,16 +160,13 @@ workflow:
rules:
- when: always
variables:
- BITBAKE_ENV_COMMAND: bitbake ${YOCTO_IMAGE} -e
# RELEASE_VERSION_EXPRESSION and RELEASE_NAME_EXPRESSION are eval'ed and then saved
# as RELEASE_VERSION and RELEASE_NAME to version.env, so we can use deferred
# variable expansion or even command execution to construct their values.
RELEASE_VERSION_EXPRESSION: $${DISTRO_VERSION}
RELEASE_NAME_EXPRESSION: Yocto-$${RELEASE_VERSION}
script:
- # Extract all variable assignments from the BitBake environment and source them.
- # The rather complicated regex also regards values containing escaped quotes.
- - source <( ${BITBAKE_ENV_COMMAND} | egrep '^[A-Z]\w*="([^\\"]+|\\.)*"$' )
+ - *source_bitbake_env
- echo ${RELEASE_VERSION_EXPRESSION}
- echo ${RELEASE_NAME_EXPRESSION}
- RELEASE_VERSION=$(eval echo "${RELEASE_VERSION_EXPRESSION}")
@@ -229,6 +235,7 @@ workflow:
IMAGE_PATH: "${BUILD_PATH}/tmp/deploy/images/${MACHINE}"
LICENSES_PATH: "${BUILD_PATH}/tmp/deploy/licenses"
SDK_PATH: "${BUILD_PATH}/tmp/deploy/sdk"
+ BITBAKE_ENV_COMMAND: bitbake ${YOCTO_IMAGE} -e
JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}"
FNG_INSTALL_URL: "${JOB_URL}/artifacts/${IMAGE_PATH}/${INSTALL_SCRIPT}"
before_script:
@@ -239,6 +246,7 @@ workflow:
script:
- *run_build
- *collect_srcrevs
+ - *source_bitbake_env
- *save_build_env
- *dump_install_command
@@ -258,6 +266,15 @@ workflow:
- wget -O artifacts.zip ${BUILD_ARTIFACTS}
- unzip artifacts.zip
after_script:
+ # FIXME: This is necessary because we're using an old build for the build simulation
+ # which does not export these variables, yet. Can be removed as soon as we switch to
+ # a newer release already containing these variables, then they can be read from the
+ # existing build.env from the cache.
+ # yamllint disable-line rule:line-length
+ - export TOOLCHAIN_OUTPUTNAME="seconorth-wayland-glibc-x86_64-seconorth-image-cortexa9t2hf-neon-$MACHINE-toolchain-kirkstone-7.0"
+ - export DISTRO_IMAGES="seconorth-image-$MACHINE.tar.gz"
+ # yamllint disable-line rule:line-length
+ - export DISTRO_RELEASE_ARTEFACTS="seconorth-image-$MACHINE.tar.gz fng-install.sh pkg.py"
- *save_build_env
cache:
- *cache
@@ -341,6 +358,10 @@ workflow:
script:
# IMAGE_PATH, SDK_PATH and LICENSES_PATH are available via build.env from build job
# RELEASE_NAME is available via version.env from build-version job
+ # FIXME: The "--sdk-name" argument shouldn't be necessary for the image package
+ # type. It is used only for the generation of "metainfo.json", which is used for the
+ # overview page on our FTP server, which is going to be replaced by Confluence. As
+ # soon as this is done, remove the "--sdk-name" argument from the image packaging.
- if [[ "${PACKAGE_TYPE}" == "image" ]]; then
- .gitlab-ci/scripts/package_release.py
--images-dir="${IMAGE_PATH}"
@@ -349,12 +370,16 @@ workflow:
--output-dir=${RELEASE_NAME}
--release-version=${RELEASE_VERSION}
--machine=${MACHINE}
+ --sdk-name="${TOOLCHAIN_OUTPUTNAME}"
+ --image-artifacts="${DISTRO_IMAGES}"
+ --artifacts="${DISTRO_RELEASE_ARTEFACTS}"
- elif [[ "${PACKAGE_TYPE}" == "sdk" ]]; then
- .gitlab-ci/scripts/package_release.py
--sdk-dir="${SDK_PATH}"
--output-dir=${RELEASE_NAME}
--release-version=${RELEASE_VERSION}
--machine=${MACHINE}
+ --sdk-name="${TOOLCHAIN_OUTPUTNAME}"
- fi
cache:
- key: ${CI_PIPELINE_ID}-${ASSOCIATED_BUILD_JOB}
diff --git a/scripts/package_release.py b/scripts/package_release.py
index a131d904..0be68004 100755
--- a/scripts/package_release.py
+++ b/scripts/package_release.py
@@ -2,10 +2,8 @@
import argparse
import glob
import hashlib
-import json
import os
import shutil
-import sys
from convert_md2html import convertmd2html
from generate_release_metadata import generate_metadata
@@ -98,37 +96,23 @@ def main():
dest="machine",
required=True,
)
+ parser.add_argument(
+ "--sdk-name",
+ help="""Name of the SDK file""",
+ dest="sdk_name",
+ )
+ parser.add_argument(
+ "--image-artifacts",
+ help="""Space separated list of image artifacts""",
+ dest="image_artifacts",
+ )
+ parser.add_argument(
+ "--artifacts",
+ help="""Space separated list of artifacts""",
+ dest="artifacts",
+ )
args, _ = parser.parse_known_args()
- # Get bitbake variables from testdata.json file
- testdata_files = []
- if args.images_dir is not None:
- testdata_files += glob.glob(os.path.join(args.images_dir, "*.testdata.json"))
- if args.sdk_dir is not None:
- testdata_files += glob.glob(os.path.join(args.sdk_dir, "*.testdata.json"))
-
- # Debug output if no testdata file found
- if not testdata_files:
- if args.images_dir is not None:
- print(args.images_dir)
- for f in glob.glob(f"{args.images_dir}/*"):
- print("-- ", f)
- if args.sdk_dir is not None:
- print(args.sdk_dir)
- for f in glob.glob(f"{args.sdk_dir}/*"):
- print("-- ", f)
- sys.exit("ERROR: no *.testdata.json file found in image or sdk dir.")
-
- # The required build variables from testdata have the same values for image and SDK
- # builds, so we just read one of them.
- with open(testdata_files[0], "r", encoding="utf-8") as f:
- buildvars = json.load(f)
-
- sdkname = buildvars["TOOLCHAIN_OUTPUTNAME"]
- image_artifacts = buildvars["DISTRO_IMAGES"].split()
- artifacts = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
- artifacts.append("BUILD_SRCREVS.log")
-
# Create output directory
os.makedirs(args.output_dir, exist_ok=True)
@@ -154,6 +138,10 @@ def main():
# Package image files
if args.images_dir is not None:
+ image_artifacts = args.image_artifacts.split() if args.image_artifacts else []
+ artifacts = args.artifacts.split() if args.artifacts else []
+ artifacts.append("BUILD_SRCREVS.log")
+
# Add some additional files to the artifacts
for artifact in image_artifacts:
artifacts.append(f"{artifact.split('.')[0]}.manifest")
@@ -180,11 +168,14 @@ def main():
artifacts.append(image_md5sums_file)
# Generate metadata file
+ # FIXME: This is used for the overview page on our FTP server, which is going to
+ # be replaced by Confluence. As soon as this is done, remove this and the
+ # "--sdk-name" argument from the image packaging call in build-pipeline.yml.
generate_metadata(
args.machine,
args.release_version,
artifacts,
- sdkname,
+ args.sdk_name,
os.path.join(args.images_dir, "metainfo.json"),
)
artifacts.append(os.path.join(args.images_dir, "metainfo.json"))
@@ -194,7 +185,7 @@ def main():
# Package SDK
if args.sdk_dir is not None:
- sdkfiles = glob.glob(os.path.join(args.sdk_dir, f"{sdkname}*"))
+ sdkfiles = glob.glob(os.path.join(args.sdk_dir, f"{args.sdk_name}*"))
# Generate MD5 sums file
sdk_md5sums_file = os.path.join(args.machine, "sdk", "md5sums.txt")
--
GitLab