Add alphaplan FWR generator

alphaplan_fwr.py: Creates alphaplan software packages for the build
release artifacts. According to the release, all relevant artifacts are
collected in a list structure and the corresponding aphaplan
information are added for each artifact (attributes, md5sum, path, etc.).
The list is attached to major software package structure. This structure
is written in a json file for documentation. Afterwards the structure is
send to an AlphaPlan webservice that creates the FWR articles.
alphaplan_keys.py: Generates a dictionary which contains all AlphaPlan
specific information for the artifacts. Flag classes are used to define
unique keys for the dictionary accesses.

alphaplan_fwr.py

+#!/usr/bin/env python3
+import json
+import os
+import shutil
+import sys
+import requests
+from alphaplan_keys import ApKeys, ApSubKeys, get_ap_dict
+
+
+def ap_send_json(jsonobj):
+    """Sends the generated files to the Alphaplan webservice"""
+    url = os.environ.get("AP_WEBSERVICE_URL")
+    usr = os.environ.get("AP_WEBSERVICE_USR")
+    pw = os.environ.get("AP_WEBSERVICE_PW")
+    cert = os.environ.get("AP_WEBSERVICE_CERT")
+    # Write TLS certificate to file
+    cert_file = "GarzFrickeGmbH-CA.cer"
+    with open(cert_file, "w") as f:
+        f.write(cert)
+    # Send data
+    msg = requests.post(url, json=jsonobj, auth=(usr, pw), verify=cert_file, timeout=10)
+    os.remove(cert_file)
+    msg_json = msg.json()
+    if msg_json["Status"] != "Ok":
+        sys.exit("ERROR: AlphaPlan webservice post request failed")
+    print("AlphaPlan webservice response: {}".format(msg_json["Meldung"]))
+
+
+def ap_id_generator():
+    """Returns a unique ids for each new article"""
+    if not hasattr(ap_id_generator, "value"):
+        ap_id_generator.value = 0
+    ap_id_generator.value += 1
+    return "Neu{}".format(ap_id_generator.value)
+
+
+def new_ap_article(
+    warengruppe,
+    matchcode,
+    bezeichnung,
+    langtext,
+    attributeset,
+    attribute,
+    stueckliste=None,
+):
+    """Creates a dict/list structure for a new AlphaPlan article"""
+    position = {}
+    for idx in range(len(attribute)):
+        if attribute[idx]:
+            position["Attribut{:02d}".format(idx + 1)] = attribute[idx]
+
+    data = {
+        "Artikel": {
+            "ID": ap_id_generator(),
+            "Warengruppe": warengruppe,
+            "MatchCode": matchcode,
+            "Bezeichnung": bezeichnung,
+            "Langtext": langtext,
+            "Attribute": {"AttributeSet": attributeset, "Position": [position]},
+        }
+    }
+
+    if stueckliste:
+        sub_articles = {
+            "Stueckliste": {"Info": "Infofeld in Stueckliste", "Position": []}
+        }
+        for article in stueckliste:
+            sub_articles["Stueckliste"]["Position"].append(article)
+        data["Artikel"].update(sub_articles)
+
+    return data
+
+
+def known_ap_article(artikelnummer):
+    """Creates a dict for a known AlphaPlan FWR article"""
+    data = {"Artikel": {"ID": artikelnummer, "Artikelnummer": artikelnummer}}
+    return data
+
+
+def generate_ap_subarticle(
+    artifacts, ap_key, output_dir, machine, machine_ap, release_name_ap, md5sums
+):
+    """Create an new AlphaPlan FWR subarticle which is added to a part list"""
+    # Generate a dict for all artifacts that should be included in the FWR
+    ap_dict = get_ap_dict(machine, machine_ap, release_name_ap)
+    # Match artifacts and get path and md5sum
+    for artifact in artifacts:
+        if artifact.casefold().endswith(ap_dict[ap_key][ApSubKeys.MATCH].casefold()):
+            path = os.path.join(output_dir, machine, artifact)
+            path = os.path.join(*(path.split(os.path.sep)[2:]))
+            # Distinguish between FNGSystem and Yocto path on drive Z:
+            if "fngsystem".casefold() in output_dir.casefold():
+                path = (
+                    "Z:\\Development\\SoftwareStore\\Flash-N-Go\\"
+                    "FNGSystem\\{}".format(path.replace("/", "\\"))
+                )
+            else:
+                path = "Z:\\Development\\SoftwareStore\\Linux-Yocto\\{}".format(
+                    path.replace("/", "\\")
+                )
+            md5sum = md5sums[artifact]
+    if "path" not in locals():
+        sys.exit("ERROR: Can not find key:{} in artifacts".format(ap_key))
+
+    # AlphaPlan specific entries for attribute set: "Firmware, Bestandteil..."
+    # Field1: type, Field2: path to artifact, Field3: md5sum
+    attribute = [ap_dict[ap_key][ApSubKeys.TYP], path, md5sum]
+
+    return new_ap_article(
+        "FWR",
+        ap_dict[ap_key][ApSubKeys.MATCHCODE],
+        ap_dict[ap_key][ApSubKeys.BEZEICHNUNG],
+        ap_dict[ap_key][ApSubKeys.LANGTEXT],
+        ap_dict[ap_key][ApSubKeys.ATTRIBUTESET],
+        attribute,
+    )
+
+
+def generate_fwr_articles(
+    output_dir, outlocal_dir, machine, release_name_local, artifacts_all, md5sums
+):
+    """Main function to generate the FWR articles for FNGSystem and Yocto"""
+    # Modify name strings for AlphaPlan
+    machine_ap = machine.upper()
+    machine_ap = machine_ap.replace("IMX", "i.MX")
+    machine_ap = machine_ap.replace("GUF", "")
+    release_name_ap = release_name_local.replace("Yocto-", "Yocto ")
+    attb_set_major = "Softwarepaket 32-bit"
+
+    stueckliste = []
+    if "fngsystem".casefold() in release_name_local.casefold():
+        # FNGSystem Release
+        # Set artifact keys
+        subarticles = [
+            ApKeys.FNGSYS_INIT,
+            ApKeys.FNGSYS_UPDATE,
+            ApKeys.FNGSYS_FS,
+            ApKeys.FNGSYS_CHECKSUM,
+        ]
+        for key in subarticles:
+            stueckliste.append(
+                generate_ap_subarticle(
+                    artifacts_all,
+                    key,
+                    outlocal_dir,
+                    machine,
+                    machine_ap,
+                    release_name_ap,
+                    md5sums,
+                )
+            )
+        # Define attribute fields
+        attribute = [None, None, "FNG-SYSTEM"]
+        # Generate dict/list structure
+        data = new_ap_article(
+            "FWR",
+            "SW-Paket",
+            "{} {}".format(machine_ap, release_name_ap),
+            "{}\n{}".format(release_name_ap, machine_ap),
+            attb_set_major,
+            attribute,
+            stueckliste=stueckliste,
+        )
+        # If imx-boot is in artifacts_all (TANARO), add an uboot FWR package
+        if "imx-boot" in artifacts_all:
+            subarticles_uboot = [
+                ApKeys.FNGSYS_UBOOT_UPDATE,
+                ApKeys.FNGSYS_UBOOT_IMAGE,
+                ApKeys.FNGSYS_UBOOT_CHECKSUM,
+            ]
+            stueckliste_uboot = []
+            for key in subarticles_uboot:
+                stueckliste_uboot.append(
+                    generate_ap_subarticle(
+                        artifacts_all,
+                        key,
+                        outlocal_dir,
+                        machine,
+                        machine_ap,
+                        release_name_ap,
+                        md5sums,
+                    )
+                )
+
+            # At the moment there are no attributes specified for uboot FWR
+            attribute_uboot = [None, None, "UBOOT"]
+            data_uboot = new_ap_article(
+                "FWR",
+                "SW-Paket",
+                "{} U-Boot {}".format(machine_ap, release_name_ap),
+                "{}\n{}".format(release_name_ap, machine_ap),
+                attb_set_major,
+                attribute_uboot,
+                stueckliste=stueckliste_uboot,
+            )
+    else:
+        # Same process for a yocto release
+        subarticles = [ApKeys.YOCTO_PKG_PY, ApKeys.YOCTO_FNG_INSTALL, ApKeys.YOCTO_FS]
+        for key in subarticles:
+            stueckliste.append(
+                generate_ap_subarticle(
+                    artifacts_all,
+                    key,
+                    outlocal_dir,
+                    machine,
+                    machine_ap,
+                    release_name_ap,
+                    md5sums,
+                )
+            )
+        attribute = [None, None, None, "YOCTO"]
+        data = new_ap_article(
+            "FWR",
+            "SW-Paket",
+            "{} {}".format(machine_ap, release_name_ap),
+            "{}\n{}".format(release_name_ap, machine_ap),
+            attb_set_major,
+            attribute,
+            stueckliste=stueckliste,
+        )
+
+    # Create an additional import file for uboot
+    if "data_uboot" in locals():
+        print(
+            "Generate {} U-Boot {} AlphaPlan FWR articles".format(
+                machine_ap, release_name_ap
+            )
+        )
+        jsonfile_uboot_name = "alphaplan-import-uboot-{}.json".format(machine)
+        jsonfile_uboot_local = os.path.join(outlocal_dir, machine, jsonfile_uboot_name)
+        jsonfile_uboot_output = os.path.join(output_dir, machine, jsonfile_uboot_name)
+        with open(jsonfile_uboot_local, "w") as jsonfile:
+            json.dump(data_uboot, jsonfile, indent=4)
+
+        shutil.copyfile(
+            jsonfile_uboot_local, jsonfile_uboot_output, follow_symlinks=True
+        )
+        # Send data object to AlphaPlan webservice
+        ap_send_json(data_uboot)
+
+    print("Generate {} {} AlphaPlan FWR articles".format(machine_ap, release_name_ap))
+    # Generate a json file from the dict/list structure for debugging
+    jsonfile_name = "alphaplan-import-{}.json".format(machine)
+    jsonfile_local = os.path.join(outlocal_dir, machine, jsonfile_name)
+    jsonfile_output = os.path.join(output_dir, machine, jsonfile_name)
+    with open(jsonfile_local, "w") as jsonfile:
+        json.dump(data, jsonfile, indent=4)
+    # Copy file to release folder as the atrifacts have already been copied
+    shutil.copyfile(jsonfile_local, jsonfile_output, follow_symlinks=True)
+    # Send data object to AlphaPlan webservice
+    ap_send_json(data)

alphaplan_keys.py

+#!/usr/bin/env python3
+
+from enum import Flag, auto
+
+
+class ApKeys(Flag):
+    """Class to specify article keys"""
+
+    YOCTO_PKG_PY = auto()
+    YOCTO_FNG_INSTALL = auto()
+    YOCTO_FS = auto()
+    FNGSYS_INIT = auto()
+    FNGSYS_UPDATE = auto()
+    FNGSYS_FS = auto()
+    FNGSYS_CHECKSUM = auto()
+    FNGSYS_UBOOT_UPDATE = auto()
+    FNGSYS_UBOOT_IMAGE = auto()
+    FNGSYS_UBOOT_CHECKSUM = auto()
+
+
+class ApSubKeys(Flag):
+    """Class to specify article subkeys"""
+
+    MATCH = auto()
+    MATCHCODE = auto()
+    BEZEICHNUNG = auto()
+    LANGTEXT = auto()
+    TYP = auto()
+    ATTRIBUTESET = auto()
+
+
+def get_ap_dict(machine, machine_ap, release_name_local):
+    """Return a customized dict with information for AlphaPlan FWR articles"""
+    ap_dict = {}
+
+    ap_dict[ApKeys.YOCTO_PKG_PY] = {
+        ApSubKeys.MATCH: "pkg.py",
+        ApSubKeys.MATCHCODE: "FNGUpdate",
+        ApSubKeys.BEZEICHNUNG: "{} Flash-N-Go Update general pkg.py update"
+        "script for nonverbose fng-install.sh".format(machine_ap),
+        ApSubKeys.LANGTEXT: "To be used with packages the contain an "
+        "fng-install.sh.\n"
+        "* with --nonverbose mode (new output)\n"
+        "* Able to to local installation with unset TFTP variable\n"
+        "* Handle --Paramfile",
+        ApSubKeys.TYP: "FNGUpdate",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.YOCTO_FNG_INSTALL] = {
+        ApSubKeys.MATCH: "fng-install.sh",
+        ApSubKeys.MATCHCODE: "InstallScript",
+        ApSubKeys.BEZEICHNUNG: "{} {} Install Script".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "US",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.YOCTO_FS] = {
+        ApSubKeys.MATCH: "{}.tar.gz".format(machine),
+        ApSubKeys.MATCHCODE: "OS-Filesystem",
+        ApSubKeys.BEZEICHNUNG: "{} {} Filesystem".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "FS",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.FNGSYS_UPDATE] = {
+        ApSubKeys.MATCH: "fngsystem-self-update.sh",
+        ApSubKeys.MATCHCODE: "TFTP",
+        ApSubKeys.BEZEICHNUNG: "{} {} Self Update".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "TFTP",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.FNGSYS_INIT] = {
+        ApSubKeys.MATCH: "fngsystem-self-init.sh",
+        ApSubKeys.MATCHCODE: "InstallScript",
+        ApSubKeys.BEZEICHNUNG: "{} {} Init Script".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "Updateskript",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.FNGSYS_FS] = {
+        ApSubKeys.MATCH: "{}.tgz".format(machine),
+        ApSubKeys.MATCHCODE: "FS",
+        ApSubKeys.BEZEICHNUNG: "{} {} Filesystem".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "FS",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.FNGSYS_CHECKSUM] = {
+        ApSubKeys.MATCH: "{}.md5".format(machine),
+        ApSubKeys.MATCHCODE: "TFTP",
+        ApSubKeys.BEZEICHNUNG: "{} {} Checksum".format(machine_ap, release_name_local),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "TFTP",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.FNGSYS_UBOOT_UPDATE] = {
+        ApSubKeys.MATCH: "fng-install-uboot.sh",
+        ApSubKeys.MATCHCODE: "US",
+        ApSubKeys.BEZEICHNUNG: "{} U-Boot {} Update script".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "Updateskript",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.FNGSYS_UBOOT_IMAGE] = {
+        ApSubKeys.MATCH: "imx-boot",
+        ApSubKeys.MATCHCODE: "FS",
+        ApSubKeys.BEZEICHNUNG: "{} U-Boot {} Bootloader Image".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "FS",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    ap_dict[ApKeys.FNGSYS_UBOOT_CHECKSUM] = {
+        ApSubKeys.MATCH: "imx-boot.md5",
+        ApSubKeys.MATCHCODE: "TFTP",
+        ApSubKeys.BEZEICHNUNG: "{} U-Boot {} Checksum".format(
+            machine_ap, release_name_local
+        ),
+        ApSubKeys.LANGTEXT: "",
+        ApSubKeys.TYP: "TFTP",
+        ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
+    }
+
+    return ap_dict

package_release.py

@@ -7,6 +7,7 @@ import sys
 import shutil
 import hashlib
 import tempfile
+import alphaplan_fwr
 
 
 def md5(fname):
@@ -68,6 +69,7 @@ def copy_files(files, input_dir, subdir, output_dir, outlocal_dir):
             )
             print("Copy: %s -> %s" % (source_file, target_file))
             shutil.copyfile(source_file, target_file, follow_symlinks=True)
+    return md5sums
 
 
 def main():
@@ -165,7 +167,25 @@ def main():
             artifacts_all.append(artifact.split(".")[0] + ".manifest")
             artifacts_all.append(artifact.split(".")[0] + ".testdata.json")
 
-        copy_files(artifacts_all, args.images_dir, machine, output_dir, outlocal_dir)
+        md5sums = copy_files(
+            artifacts_all, args.images_dir, machine, output_dir, outlocal_dir
+        )
+
+        # Generate a json file for creating alphaplan FWR articles
+        if (
+            outlocal_dir is not None
+            and output_dir is not None
+            and "ftp" not in outlocal_dir
+            and "ftp" not in output_dir
+        ):
+            alphaplan_fwr.generate_fwr_articles(
+                output_dir,
+                outlocal_dir,
+                machine,
+                release_name_local,
+                artifacts_all,
+                md5sums,
+            )
 
     # Handle SDK if available
     if args.sdk_dir is not None: