build:stages: Add Firmware Package JSON stage

The old Alphaplan stage is deprecated and was removed in a prior commit.
Add new stage to generate the Firmware Package JSON files that do replace
the old Alphaplan articles.

.gitignore

 .idea/
 .vscode/
 scripts/__pycache__
+scripts/fng*
+scripts/yocto*

build-pipeline-yocto.yml.jinja2

@@ -20,6 +20,7 @@ stages:
   - Deploy SoftwareStore Internal
   - Deploy FTP
   - Deploy Azure
+  - Firmware Package
   - Confluence
 
 variables:
@@ -337,6 +338,36 @@ azure-{{ machine }}:
 {% endif %}
 
 
+# --------------------------------------------------------------------------------------
+# Stage: Firmware Package
+# --------------------------------------------------------------------------------------
+{% for machine in MACHINES.split(' ') %}
+
+generate-firmware-package-{{ machine }}:
+  extends: .generate_firmware_package
+  variables:
+    MACHINE: {{ machine }}
+  needs:
+    - deploy-{{ machine }}
+    - build-version
+
+{% endfor %}
+
+{% for machine in MACHINES.split(' ') %}
+
+deploy-firmware-package-{{ machine }}:
+  extends: .deploy-software-store
+  stage: Firmware Package
+  variables:
+    ASSOCIATED_PACKAGE_JOB: generate-firmware-package-{{ machine }}
+  needs:
+    - job: generate-firmware-package-{{ machine }}
+      artifacts: false
+    - job: build-version
+
+{% endfor %}
+
+
 # --------------------------------------------------------------------------------------
 # Stage: Confluence
 # --------------------------------------------------------------------------------------

build-pipeline.yml

@@ -468,6 +468,33 @@ workflow:
     # Get links to uploaded files and remove query string containing the SAS token
     - jq -r '.[] | .Blob' result.json | sed "s/?.*//" | tee files.txt
 
+# --------------------------------------------------------------------------------------
+# Stage: Firmware Package
+# --------------------------------------------------------------------------------------
+.generate_firmware_package:
+  extends:
+    - .infrastructure
+  tags:
+    - misc
+  stage: Firmware Package
+  # build-version is set in the jinja2 template
+  script:
+    # RELEASE_NAME is available from build-version.env
+    - .gitlab-ci/scripts/generate_firmware_package.py
+        --machine="${MACHINE}"
+        --release-name="${RELEASE_NAME}"
+        --files="${FILES}"
+        --md5sums=**/md5sums.txt
+        --output-file="${RELEASE_NAME}/${MACHINE}/firmware-package.json"
+  artifacts:
+    paths:
+      - ${RELEASE_NAME}/${MACHINE}/firmware-package.json
+  cache:
+    - key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
+      policy: push
+      paths:
+        - ${RELEASE_NAME}/${MACHINE}/firmware-package.json
+
 # --------------------------------------------------------------------------------------
 # Stage: Confluence
 # --------------------------------------------------------------------------------------

scripts/firmware_package_keys.py

+#!/usr/bin/env python3
+
+from enum import Flag, auto
+
+
+class FirmwarePackageKeys(Flag):
+    """Class to specify firmware package keys"""
+
+    YOCTO_PKG_PY = auto()
+    YOCTO_FNG_INSTALL = auto()
+    YOCTO_FS = auto()
+    FNGSYS_INIT = auto()
+    FNGSYS_UPDATE = auto()
+    FNGSYS_FS = auto()
+    FNGSYS_CHECKSUM = auto()
+    FNGSYS_UBOOT_UPDATE = auto()
+    FNGSYS_UBOOT_IMAGE = auto()
+    FNGSYS_UBOOT_CHECKSUM = auto()
+    FNGSYS_UBOOT_IMAGETAR = auto()
+
+
+class FirmwarePackageSubKeys(Flag):
+    """Class to specify firmware package subkeys"""
+
+    MATCH = auto()
+    MATCHCODE = auto()
+    BEZEICHNUNG = auto()
+    LANGTEXT = auto()
+    TYP = auto()
+    ATTRIBUTESET = auto()
+
+
+def get_fwr_pkg_dict(machine_name, machine_name_long, release_name):
+    """Return a customized dict with information for the firmware package"""
+
+    return {
+        FirmwarePackageKeys.YOCTO_PKG_PY: {
+            FirmwarePackageSubKeys.MATCH: "pkg.py",
+            FirmwarePackageSubKeys.MATCHCODE: "FNGUpdate",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} Flash-N-Go Update general pkg.py "
+            "update script for nonverbose fng-install.sh",
+            FirmwarePackageSubKeys.LANGTEXT: "To be used with packages the contain an "
+            "fng-install.sh.\n"
+            "* with --nonverbose mode (new output)\n"
+            "* Able to to local installation with unset TFTP variable\n"
+            "* Handle --Paramfile",
+            FirmwarePackageSubKeys.TYP: "FNGUpdate",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.YOCTO_FNG_INSTALL: {
+            FirmwarePackageSubKeys.MATCH: "fng-install.sh",
+            FirmwarePackageSubKeys.MATCHCODE: "InstallScript",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Install Script",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "US",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.YOCTO_FS: {
+            FirmwarePackageSubKeys.MATCH: f"{machine_name}.tar.gz",
+            FirmwarePackageSubKeys.MATCHCODE: "OS-Filesystem",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Filesystem",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "FS",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_UPDATE: {
+            FirmwarePackageSubKeys.MATCH: "fngsystem-self-update.sh",
+            FirmwarePackageSubKeys.MATCHCODE: "TFTP",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Self Update",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "TFTP",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_INIT: {
+            FirmwarePackageSubKeys.MATCH: "fngsystem-self-init.sh",
+            FirmwarePackageSubKeys.MATCHCODE: "InstallScript",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Init Script",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "Updateskript",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_FS: {
+            FirmwarePackageSubKeys.MATCH: f"{machine_name}.tgz",
+            FirmwarePackageSubKeys.MATCHCODE: "FS",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Filesystem",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "FS",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_CHECKSUM: {
+            FirmwarePackageSubKeys.MATCH: f"{machine_name}.md5",
+            FirmwarePackageSubKeys.MATCHCODE: "TFTP",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} {release_name} Checksum",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "TFTP",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_UBOOT_UPDATE: {
+            FirmwarePackageSubKeys.MATCH: "fng-install-uboot.sh",
+            FirmwarePackageSubKeys.MATCHCODE: "US",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} "
+            "Update script",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "Updateskript",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_UBOOT_IMAGE: {
+            FirmwarePackageSubKeys.MATCH: "imx-boot",
+            FirmwarePackageSubKeys.MATCHCODE: "FS",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} "
+            "Bootloader Image",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "FS",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_UBOOT_IMAGETAR: {
+            FirmwarePackageSubKeys.MATCH: "imx-boot.tar.gz",
+            FirmwarePackageSubKeys.MATCHCODE: "FS",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} "
+            "Bootloader Image",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "FS",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+        FirmwarePackageKeys.FNGSYS_UBOOT_CHECKSUM: {
+            FirmwarePackageSubKeys.MATCH: "imx-boot.md5",
+            FirmwarePackageSubKeys.MATCHCODE: "TFTP",
+            FirmwarePackageSubKeys.BEZEICHNUNG: f"{machine_name_long} U-Boot {release_name} Checksum",
+            FirmwarePackageSubKeys.LANGTEXT: "",
+            FirmwarePackageSubKeys.TYP: "TFTP",
+            FirmwarePackageSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+        },
+    }

scripts/generate_firmware_package.py

+#!/usr/bin/env python3
+
+import argparse
+import fnmatch
+import glob
+import json
+import os
+import sys
+
+from firmware_package_keys import (
+    FirmwarePackageKeys,
+    FirmwarePackageSubKeys,
+    get_fwr_pkg_dict,
+)
+
+
+def generate_entry(
+    name: str,
+    _type: str,
+    description: str = None,
+    path: str = None,
+    md5sum: str = None,
+    files: dict = None,
+    packages: dict = None,
+):
+    """Create a JSON object for a new firmware package entry"""
+
+    package_entry = {}
+
+    package_entry["name"] = name
+    package_entry["type"] = _type
+
+    if description:
+        package_entry["description"] = description
+
+    if path:
+        package_entry["path"] = path
+
+    if md5sum:
+        package_entry["md5sum"] = md5sum
+
+    if files:
+        package_entry["files"] = files
+
+    if packages:
+        package_entry["packages"] = packages
+
+    return package_entry
+
+
+def generate_subpackage(
+    files: list[str],
+    pkg_key: FirmwarePackageKeys,
+    machine_name: str,
+    machine_name_long: str,
+    release_name: str,
+    md5sums: dict[str, str],
+):
+    """Create a new firmware subpackage"""
+
+    fwr_pkg = None
+
+    # Generate a dictionary for all files that should be included in the firmware package
+    fwr_pkg_dict = get_fwr_pkg_dict(machine_name, machine_name_long, release_name)
+
+    # Match files and get the paths and md5sums
+    for filepath in files:
+        filename = os.path.basename(filepath)
+
+        # Compare the filename with the match from pkg_key
+        if filename.casefold().endswith(
+            fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.MATCH].casefold()
+        ):
+            md5sum = md5sums[filename]
+
+            fwr_pkg = generate_entry(
+                name=fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.BEZEICHNUNG],
+                description=fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.LANGTEXT],
+                _type=fwr_pkg_dict[pkg_key][FirmwarePackageSubKeys.TYP],
+                path=filepath,
+                md5sum=md5sum,
+            )
+
+            break
+
+    if fwr_pkg is None:
+        sys.exit(f"ERROR: Can not find key:{pkg_key} in files")
+
+    return fwr_pkg
+
+
+def generate_firmware_package(
+    release_name: str,
+    machine: str,
+    files: list[str],
+    md5sums: dict[str, str],
+    filepath: str,
+    filename: str,
+):
+    """Main function to generate the firmware package for Flash-N-Go System and Yocto"""
+
+    # Modify name strings for firmware package
+    machine_name_long = machine.upper()
+    # Old machine name was something like "imx6guf"
+    machine_name_long = machine_name_long.replace("IMX", "i.MX")
+    machine_name_long = machine_name_long.replace("GUF", "")
+    # New machine name is something like "seco-mx6"
+    machine_name_long = machine_name_long.replace("MX", "i.MX")
+    machine_name_long = machine_name_long.replace("SECO-", "")
+    release_name = release_name.replace("Yocto-", "Yocto ")
+
+    sbom = []
+    sbom_packages = []
+
+    if "fngsystem".casefold() in release_name.casefold():
+        # Flash-N-Go System
+        file_types = [
+            FirmwarePackageKeys.FNGSYS_INIT,
+            FirmwarePackageKeys.FNGSYS_UPDATE,
+            FirmwarePackageKeys.FNGSYS_FS,
+            FirmwarePackageKeys.FNGSYS_CHECKSUM,
+        ]
+
+        sbom_files = []
+
+        for file_type in file_types:
+            sbom_files.append(
+                generate_subpackage(
+                    files, file_type, machine, machine_name_long, release_name, md5sums
+                )
+            )
+
+        sbom_packages.append(
+            generate_entry(name="", _type="FNG-SYSTEM", files=sbom_files)
+        )
+
+        # U-Boot (only on i.MX8/Genio)
+
+        sbom_uboot_files = []
+
+        if fnmatch.filter(files, "*/imx-boot.tar.gz"):
+            uboot_file_types = [
+                FirmwarePackageKeys.FNGSYS_UBOOT_UPDATE,
+                FirmwarePackageKeys.FNGSYS_UBOOT_IMAGETAR,
+            ]
+        elif fnmatch.filter(files, "*/imx-boot"):
+            uboot_file_types = [
+                FirmwarePackageKeys.FNGSYS_UBOOT_UPDATE,
+                FirmwarePackageKeys.FNGSYS_UBOOT_IMAGE,
+                FirmwarePackageKeys.FNGSYS_UBOOT_CHECKSUM,
+            ]
+
+        if uboot_file_types:
+            sbom_uboot_files = [
+                generate_subpackage(
+                    files,
+                    file_type,
+                    machine,
+                    machine_name_long,
+                    release_name,
+                    md5sums,
+                )
+                for file_type in uboot_file_types
+            ]
+
+            sbom_packages.append(
+                generate_entry(name="", _type="UBOOT", files=sbom_uboot_files)
+            )
+
+    else:
+        # Yocto OS
+        file_types = [
+            FirmwarePackageKeys.YOCTO_FNG_INSTALL,
+            FirmwarePackageKeys.YOCTO_FS,
+        ]
+
+        sbom_files = []
+
+        for file_type in file_types:
+            sbom_files.append(
+                generate_subpackage(
+                    files, file_type, machine, machine_name_long, release_name, md5sums
+                )
+            )
+
+        sbom_packages.append(generate_entry(name="", _type="YOCTO", files=sbom_files))
+
+    sbom = generate_entry(
+        name="", description="", _type="YOCTO", packages=sbom_packages
+    )
+
+    if filepath:
+        if not os.path.exists(filepath):
+            os.makedirs(filepath)
+    else:
+        filepath = "."
+
+    print(
+        f'Saving Firmware Package JSON for "{machine_name_long} {release_name}" '
+        f"to {filepath}/{filename}"
+    )
+    with open(os.path.join(filepath, filename), "w", encoding="utf-8") as jsonfile:
+        json.dump(sbom, jsonfile, indent=2)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--release-name",
+        help="""Name of the release""",
+        dest="release_name",
+        required=True,
+    )
+    parser.add_argument(
+        "--machine",
+        help="""Machine the release it built for""",
+        dest="machine",
+        required=True,
+    )
+    parser.add_argument(
+        "--files",
+        help="""Space-separated list of all released files""",
+        dest="files",
+        required=False,
+    )
+    parser.add_argument(
+        "--files-list",
+        help="""Text file containing a list of all released files""",
+        dest="files_list",
+        required=False,
+    )
+    parser.add_argument(
+        "--md5sums",
+        help="""Text file containing MD5 sums of released files""",
+        dest="md5sums",
+        required=True,
+    )
+    parser.add_argument(
+        "--output-file",
+        help="""Output file name and path""",
+        dest="output_file",
+        default="firmware-package.json",
+    )
+    args, _ = parser.parse_known_args()
+
+    output_filepath, output_filename = os.path.split(args.output_file)
+
+    # Sanity checking
+    if not args.machine:
+        sys.exit("ERROR: --machine requires a non-empty argument")
+    if not args.release_name:
+        sys.exit("ERROR: --release-name requires a non-empty argument")
+    if not output_filename:
+        sys.exit("ERROR: --output-file must at least contain a valid filename")
+
+    # Parse/read file list
+    files = []
+
+    if args.files:
+        files = args.files.split()
+
+    if args.files_list:
+        for files_file in glob.glob(args.files_list, recursive=True):
+            print(f"Reading files from {files_file}")
+            with open(files_file, "r", encoding="utf-8") as f:
+                files = files + f.read().splitlines()
+
+    # Read MD5 sums
+    md5sums = {}
+    for md5sums_file in glob.glob(args.md5sums, recursive=True):
+        print(f"Reading md5sums from {md5sums_file}")
+        with open(md5sums_file, "r", encoding="utf-8") as f:
+            for line in f:
+                # Assuming line format: "<md5sum>  <filename>\n"
+                name = line.split("  ")[1].rstrip()
+                md5sum = line.split("  ")[0]
+                md5sums[name] = md5sum
+
+    # Generate firmware package JSON
+    generate_firmware_package(
+        args.release_name,
+        args.machine,
+        files,
+        md5sums,
+        output_filepath,
+        output_filename,
+    )
+
+
+if __name__ == "__main__":
+    main()