From 51e0cf67dc5961134ef4bd3e7a1b1659ab95dd63 Mon Sep 17 00:00:00 2001
From: Tobias Poganiuch <tobias.poganiuch@seco.com>
Date: Tue, 14 Nov 2023 17:08:14 +0100
Subject: [PATCH] build:stages: Add Firmware Package JSON stage
The old Alphaplan stage is deprecated and was removed in a prior commit.
Add new stage to generate the Firmware Package JSON files that do replace
the old Alphaplan articles.
---
.gitignore | 2 +
build-pipeline-yocto.yml.jinja2 | 31 +++
build-pipeline.yml | 27 +++
scripts/firmware_package_keys.py | 133 ++++++++++++
scripts/generate_firmware_package.py | 291 +++++++++++++++++++++++++++
5 files changed, 484 insertions(+)
create mode 100755 scripts/firmware_package_keys.py
create mode 100755 scripts/generate_firmware_package.py
diff --git a/.gitignore b/.gitignore
index b611c2ec..418ff824 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
.idea/
.vscode/
scripts/__pycache__
+scripts/fng*
+scripts/yocto*
diff --git a/build-pipeline-yocto.yml.jinja2 b/build-pipeline-yocto.yml.jinja2
index 4b338659..6ce2e3f3 100644
--- a/build-pipeline-yocto.yml.jinja2
+++ b/build-pipeline-yocto.yml.jinja2
@@ -20,6 +20,7 @@ stages:
- Deploy SoftwareStore Internal
- Deploy FTP
- Deploy Azure
+ - Firmware Package
- Confluence
variables:
@@ -337,6 +338,36 @@ azure-{{ machine }}:
{% endif %}
+# --------------------------------------------------------------------------------------
+# Stage: Firmware Package
+# --------------------------------------------------------------------------------------
+{% for machine in MACHINES.split(' ') %}
+
+generate-firmware-package-{{ machine }}:
+ extends: .generate_firmware_package
+ variables:
+ MACHINE: {{ machine }}
+ needs:
+ - deploy-{{ machine }}
+ - build-version
+
+{% endfor %}
+
+{% for machine in MACHINES.split(' ') %}
+
+deploy-firmware-package-{{ machine }}:
+ extends: .deploy-software-store
+ stage: Firmware Package
+ variables:
+ ASSOCIATED_PACKAGE_JOB: generate-firmware-package-{{ machine }}
+ needs:
+ - job: generate-firmware-package-{{ machine }}
+ artifacts: false
+ - job: build-version
+
+{% endfor %}
+
+
# --------------------------------------------------------------------------------------
# Stage: Confluence
# --------------------------------------------------------------------------------------
diff --git a/build-pipeline.yml b/build-pipeline.yml
index 47edd008..42e31abf 100644
--- a/build-pipeline.yml
+++ b/build-pipeline.yml
@@ -468,6 +468,33 @@ workflow:
# Get links to uploaded files and remove query string containing the SAS token
- jq -r '.[] | .Blob' result.json | sed "s/?.*//" | tee files.txt
+# --------------------------------------------------------------------------------------
+# Stage: Firmware Package
+# --------------------------------------------------------------------------------------
+.generate_firmware_package:
+ extends:
+ - .infrastructure
+ tags:
+ - misc
+ stage: Firmware Package
+ # build-version is set in the jinja2 template
+ script:
+ # RELEASE_NAME is available from build-version.env
+ - .gitlab-ci/scripts/generate_firmware_package.py
+ --machine="${MACHINE}"
+ --release-name="${RELEASE_NAME}"
+ --files="${FILES}"
+ --md5sums=**/md5sums.txt
+ --output-file="${RELEASE_NAME}/${MACHINE}/firmware-package.json"
+ artifacts:
+ paths:
+ - ${RELEASE_NAME}/${MACHINE}/firmware-package.json
+ cache:
+ - key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
+ policy: push
+ paths:
+ - ${RELEASE_NAME}/${MACHINE}/firmware-package.json
+
# --------------------------------------------------------------------------------------
# Stage: Confluence
# --------------------------------------------------------------------------------------
diff --git a/scripts/firmware_package_keys.py b/scripts/firmware_package_keys.py
new file mode 100755
index 00000000..193486ff
--- /dev/null
+++ b/scripts/firmware_package_keys.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+from enum import Flag, auto
+
+
+class FirmwarePackageKeys(Flag):
+ """Class to specify firmware package keys"""
+
+ YOCTO_PKG_PY = auto()
+ YOCTO_FNG_INSTALL = auto()
+ YOCTO_FS = auto()
+ FNGSYS_INIT = auto()
+ FNGSYS_UPDATE = auto()
+ FNGSYS_FS = auto()
+ FNGSYS_CHECKSUM = auto()
+ FNGSYS_UBOOT_UPDATE = auto()
+ FNGSYS_UBOOT_IMAGE = auto()
+ FNGSYS_UBOOT_CHECKSUM = auto()
+ FNGSYS_UBOOT_IMAGETAR = auto()
+
+
+class FirmwarePackageSubKeys(Flag):
+ """Class to specify firmware package subkeys"""
+
+ MATCH = auto()
+ MATCHCODE = auto()
+ BEZEICHNUNG = auto()
+ LANGTEXT = auto()
+ TYP = auto()
+ ATTRIBUTESET = auto()
+
+
+def get_fwr_pkg_dict(machine_name, machine_name_long, release_name):
+ """Return a customized dict with information for the firmware package"""
+
+ return {
+ FirmwarePackageKeys.YOCTO_PKG_PY: {
+ FirmwarePackageSubKeys.MATCH: "pkg.py",
+ FirmwarePackageSubKeys.MATCHCODE: "FNGUpdate",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} Flash-N-Go Update general pkg.py "
+ "update script for nonverbose fng-install.sh",
+ FirmwarePackageSubKeys.LANGTEXT: "To be used with packages the contain an "
+ "fng-install.sh.\n"
+ "* with --nonverbose mode (new output)\n"
+ "* Able to to local installation with unset TFTP variable\n"
+ "* Handle --Paramfile",
+ FirmwarePackageSubKeys.TYP: "FNGUpdate",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.YOCTO_FNG_INSTALL: {
+ FirmwarePackageSubKeys.MATCH: "fng-install.sh",
+ FirmwarePackageSubKeys.MATCHCODE: "InstallScript",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Install Script",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "US",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.YOCTO_FS: {
+ FirmwarePackageSubKeys.MATCH: f"{machine_name}.tar.gz",
+ FirmwarePackageSubKeys.MATCHCODE: "OS-Filesystem",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Filesystem",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "FS",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_UPDATE: {
+ FirmwarePackageSubKeys.MATCH: "fngsystem-self-update.sh",
+ FirmwarePackageSubKeys.MATCHCODE: "TFTP",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Self Update",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "TFTP",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_INIT: {
+ FirmwarePackageSubKeys.MATCH: "fngsystem-self-init.sh",
+ FirmwarePackageSubKeys.MATCHCODE: "InstallScript",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Init Script",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "Updateskript",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_FS: {
+ FirmwarePackageSubKeys.MATCH: f"{machine_name}.tgz",
+ FirmwarePackageSubKeys.MATCHCODE: "FS",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Filesystem",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "FS",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_CHECKSUM: {
+ FirmwarePackageSubKeys.MATCH: f"{machine_name}.md5",
+ FirmwarePackageSubKeys.MATCHCODE: "TFTP",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Checksum",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "TFTP",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_UBOOT_UPDATE: {
+ FirmwarePackageSubKeys.MATCH: "fng-install-uboot.sh",
+ FirmwarePackageSubKeys.MATCHCODE: "US",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} "
+ "Update script",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "Updateskript",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_UBOOT_IMAGE: {
+ FirmwarePackageSubKeys.MATCH: "imx-boot",
+ FirmwarePackageSubKeys.MATCHCODE: "FS",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} "
+ "Bootloader Image",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "FS",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_UBOOT_IMAGETAR: {
+ FirmwarePackageSubKeys.MATCH: "imx-boot.tar.gz",
+ FirmwarePackageSubKeys.MATCHCODE: "FS",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} "
+ "Bootloader Image",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "FS",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ FirmwarePackageKeys.FNGSYS_UBOOT_CHECKSUM: {
+ FirmwarePackageSubKeys.MATCH: "imx-boot.md5",
+ FirmwarePackageSubKeys.MATCHCODE: "TFTP",
+ FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} Checksum",
+ FirmwarePackageSubKeys.LANGTEXT: "",
+ FirmwarePackageSubKeys.TYP: "TFTP",
+ FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+ },
+ }
diff --git a/scripts/generate_firmware_package.py b/scripts/generate_firmware_package.py
new file mode 100755
index 00000000..872d09d4
--- /dev/null
+++ b/scripts/generate_firmware_package.py
@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+
+import argparse
+import fnmatch
+import glob
+import json
+import os
+import sys
+
+from firmware_package_keys import (
+ FirmwarePackageKeys,
+ FirmwarePackageSubKeys,
+ get_fwr_pkg_dict,
+)
+
+
+def generate_entry(
+ name: str,
+ _type: str,
+ description: str = None,
+ path: str = None,
+ md5sum: str = None,
+ files: dict = None,
+ packages: dict = None,
+):
+ """Create a JSON object for a new firmware package entry"""
+
+ package_entry = {}
+
+ package_entry["name"] = name
+ package_entry["type"] = _type
+
+ if description:
+ package_entry["description"] = description
+
+ if path:
+ package_entry["path"] = path
+
+ if md5sum:
+ package_entry["md5sum"] = md5sum
+
+ if files:
+ package_entry["files"] = files
+
+ if packages:
+ package_entry["packages"] = packages
+
+ return package_entry
+
+
+def generate_subpackage(
+ files: list[str],
+ pkg_key: FirmwarePackageKeys,
+ machine_name: str,
+ machine_name_long: str,
+ release_name: str,
+ md5sums: dict[str, str],
+):
+ """Create a new firmware subpackage"""
+
+ fwr_pkg = None
+
+ # Generate a dictionary for all files that should be included in the firmware package
+ fwr_pkg_dict = get_fwr_pkg_dict(machine_name, machine_name_long, release_name)
+
+ # Match files and get the paths and md5sums
+ for filepath in files:
+ filename = os.path.basename(filepath)
+
+ # Compare the filename with the match from pkg_key
+ if filename.casefold().endswith(
+ fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.MATCH].casefold()
+ ):
+ md5sum = md5sums[filename]
+
+ fwr_pkg = generate_entry(
+ name=fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.BEZEICHNUNG],
+ description=fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.LANGTEXT],
+ _type=fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.TYP],
+ path=filepath,
+ md5sum=md5sum,
+ )
+
+ break
+
+ if fwr_pkg is None:
+ sys.exit(f"ERROR: Can not find key:{pkg_key} in files")
+
+ return fwr_pkg
+
+
+def generate_firmware_package(
+ release_name: str,
+ machine: str,
+ files: list[str],
+ md5sums: dict[str, str],
+ filepath: str,
+ filename: str,
+):
+ """Main function to generate the firmware package for Flash-N-Go System and Yocto"""
+
+ # Modify name strings for firmware package
+ machine_name_long = machine.upper()
+ # Old machine name was something like "imx6guf"
+ machine_name_long = machine_name_long.replace("IMX", "i.MX")
+ machine_name_long = machine_name_long.replace("GUF", "")
+ # New machine name is something like "seco-mx6"
+ machine_name_long = machine_name_long.replace("MX", "i.MX")
+ machine_name_long = machine_name_long.replace("SECO-", "")
+ release_name = release_name.replace("Yocto-", "Yocto ")
+
+ sbom = []
+ sbom_packages = []
+
+ if "fngsystem".casefold() in release_name.casefold():
+ # Flash-N-Go System
+ file_types = [
+ FirmwarePackageKeys.FNGSYS_INIT,
+ FirmwarePackageKeys.FNGSYS_UPDATE,
+ FirmwarePackageKeys.FNGSYS_FS,
+ FirmwarePackageKeys.FNGSYS_CHECKSUM,
+ ]
+
+ sbom_files = []
+
+ for file_type in file_types:
+ sbom_files.append(
+ generate_subpackage(
+ files, file_type, machine, machine_name_long, release_name, md5sums
+ )
+ )
+
+ sbom_packages.append(
+ generate_entry(name="", _type="FNG-SYSTEM", files=sbom_files)
+ )
+
+ # U-Boot (only on i.MX8/Genio)
+
+ sbom_uboot_files = []
+
+ if fnmatch.filter(files, "*/imx-boot.tar.gz"):
+ uboot_file_types = [
+ FirmwarePackageKeys.FNGSYS_UBOOT_UPDATE,
+ FirmwarePackageKeys.FNGSYS_UBOOT_IMAGETAR,
+ ]
+ elif fnmatch.filter(files, "*/imx-boot"):
+ uboot_file_types = [
+ FirmwarePackageKeys.FNGSYS_UBOOT_UPDATE,
+ FirmwarePackageKeys.FNGSYS_UBOOT_IMAGE,
+ FirmwarePackageKeys.FNGSYS_UBOOT_CHECKSUM,
+ ]
+
+ if uboot_file_types:
+ sbom_uboot_files = [
+ generate_subpackage(
+ files,
+ file_type,
+ machine,
+ machine_name_long,
+ release_name,
+ md5sums,
+ )
+ for file_type in uboot_file_types
+ ]
+
+ sbom_packages.append(
+ generate_entry(name="", _type="UBOOT", files=sbom_uboot_files)
+ )
+
+ else:
+ # Yocto OS
+ file_types = [
+ FirmwarePackageKeys.YOCTO_FNG_INSTALL,
+ FirmwarePackageKeys.YOCTO_FS,
+ ]
+
+ sbom_files = []
+
+ for file_type in file_types:
+ sbom_files.append(
+ generate_subpackage(
+ files, file_type, machine, machine_name_long, release_name, md5sums
+ )
+ )
+
+ sbom_packages.append(generate_entry(name="", _type="YOCTO", files=sbom_files))
+
+ sbom = generate_entry(
+ name="", description="", _type="YOCTO", packages=sbom_packages
+ )
+
+ if filepath:
+ if not os.path.exists(filepath):
+ os.makedirs(filepath)
+ else:
+ filepath = "."
+
+ print(
+ f'Saving Firmware Package JSON for "{machine_name_long} {release_name}" '
+ f"to {filepath}/{filename}"
+ )
+ with open(os.path.join(filepath, filename), "w", encoding="utf-8") as jsonfile:
+ json.dump(sbom, jsonfile, indent=2)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--release-name",
+ help="""Name of the release""",
+ dest="release_name",
+ required=True,
+ )
+ parser.add_argument(
+ "--machine",
+ help="""Machine the release it built for""",
+ dest="machine",
+ required=True,
+ )
+ parser.add_argument(
+ "--files",
+ help="""Space-separated list of all released files""",
+ dest="files",
+ required=False,
+ )
+ parser.add_argument(
+ "--files-list",
+ help="""Text file containing a list of all released files""",
+ dest="files_list",
+ required=False,
+ )
+ parser.add_argument(
+ "--md5sums",
+ help="""Text file containing MD5 sums of released files""",
+ dest="md5sums",
+ required=True,
+ )
+ parser.add_argument(
+ "--output-file",
+ help="""Output file name and path""",
+ dest="output_file",
+ default="firmware-package.json",
+ )
+ args, _ = parser.parse_known_args()
+
+ output_filepath, output_filename = os.path.split(args.output_file)
+
+ # Sanity checking
+ if not args.machine:
+ sys.exit("ERROR: --machine requires a non-empty argument")
+ if not args.release_name:
+ sys.exit("ERROR: --release-name requires a non-empty argument")
+ if not output_filename:
+ sys.exit("ERROR: --output-file must at least contain a valid filename")
+
+ # Parse/read file list
+ files = []
+
+ if args.files:
+ files = args.files.split()
+
+ if args.files_list:
+ for files_file in glob.glob(args.files_list, recursive=True):
+ print(f"Reading files from {files_file}")
+ with open(files_file, "r", encoding="utf-8") as f:
+ files = files + f.read().splitlines()
+
+ # Read MD5 sums
+ md5sums = {}
+ for md5sums_file in glob.glob(args.md5sums, recursive=True):
+ print(f"Reading md5sums from {md5sums_file}")
+ with open(md5sums_file, "r", encoding="utf-8") as f:
+ for line in f:
+ # Assuming line format: "<md5sum> <filename>\n"
+ name = line.split(" ")[1].rstrip()
+ md5sum = line.split(" ")[0]
+ md5sums[name] = md5sum
+
+ # Generate firmware package JSON
+ generate_firmware_package(
+ args.release_name,
+ args.machine,
+ files,
+ md5sums,
+ output_filepath,
+ output_filename,
+ )
+
+
+if __name__ == "__main__":
+ main()
--
GitLab