#!/usr/bin/env python3 import argparse import glob import hashlib import json import os import shutil import sys from convert_md2html import convertmd2html from generate_release_metadata import generate_metadata def md5(input_file: str): """ Calculate and return the MD5 sum of the given input_file. """ hash_md5 = hashlib.md5() with open(input_file, "rb", encoding=None) as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) return hash_md5.hexdigest() def generate_md5sums_file(input_files: list[str], output_file: str): """ Calculate MD5 sums of all input_files and write them to output_file. """ # This might should be used somewhere before, # but the previous state was not to fail when a file in # the list did not exist, like in copy files md5sums = {os.path.basename(f): md5(f) for f in input_files if os.path.exists(f)} output_dir = os.path.dirname(output_file) if output_dir: os.makedirs(output_dir, exist_ok=True) with open(output_file, "w", encoding="utf-8") as f_md5: for f, h in md5sums.items(): f_md5.write(f"{h} {f}\n") def copy_files(files: list[str], target_dir: str): """ Copy given files to target_dir. Create target_dir, if it does not exist. Subfolder hierarchies on the input files will not be preserved, only plain files are copied. """ if target_dir is None: return os.makedirs(target_dir, exist_ok=True) for source_file in files: if os.path.exists(source_file): target_file = os.path.join(target_dir, os.path.basename(source_file)) print(f"Copy: {source_file} -> {target_file}") shutil.copyfile(source_file, target_file, follow_symlinks=True) else: print(f"Missing: {source_file}") def main(): # FIXME: Remove the sourcery check deactivation below and refactor this method in # order to enhance code quality and make the check pass. # sourcery skip: low-code-quality parser = argparse.ArgumentParser() parser.add_argument( "--images-dir", help="""Yocto images directory""", dest="images_dir", ) parser.add_argument( "--licenses-dir", help="""Yocto licenses directory""", dest="licenses_dir", ) parser.add_argument( "--sdk-dir", help="""Yocto sdk directory""", dest="sdk_dir", ) parser.add_argument( "--doc-dir", help="""Documentation directory""", dest="doc_dir", ) parser.add_argument( "--output-dir", help="""Base directory name for output artifacts (can be specified multiple times)""", dest="output_dir", action="append", required=True, ) parser.add_argument( "--release-suffix", help="""Suffix to append to the release folder""", dest="release_suffix", ) args, _ = parser.parse_known_args() # Get bitbake variables from testdata.json file testdata_files = [] if args.images_dir is not None: testdata_files += glob.glob(os.path.join(args.images_dir, "*.testdata.json")) if args.sdk_dir is not None: testdata_files += glob.glob(os.path.join(args.sdk_dir, "*.testdata.json")) # Debug output if no testdata file found if not testdata_files: if args.images_dir is not None: print(args.images_dir) for f in glob.glob(f"{args.images_dir}/*"): print("-- ", f) if args.sdk_dir is not None: print(args.sdk_dir) for f in glob.glob(f"{args.sdk_dir}/*"): print("-- ", f) sys.exit("ERROR: no *.testdata.json file found in image or sdk dir.") # The required build variables from testdata have the same values for image and SDK # builds, so we just read one of them. with open(testdata_files[0], "r", encoding="utf-8") as f: buildvars = json.load(f) machine = buildvars["MACHINE"] version = buildvars["DISTRO_VERSION"] sdkname = buildvars["TOOLCHAIN_OUTPUTNAME"] image_artifacts = buildvars["DISTRO_IMAGES"].split() artifacts = buildvars["DISTRO_RELEASE_ARTEFACTS"].split() artifacts.append("BUILD_SRCREVS.log") # Set release name if version.startswith("fngsystem"): release_name = version.replace("fngsystem", "FNGSystem") else: release_name = f"Yocto-{version}" # Append release suffix if args.release_suffix is not None: release_name = release_name + args.release_suffix # Create output directories output_dirs = [] for output_dir in args.output_dir: full_output_dir = os.path.join(output_dir, release_name) output_dirs.append(full_output_dir) os.makedirs(full_output_dir, exist_ok=True) # Package documentation files if args.doc_dir is not None: # Convert markdown to html doc_files = glob.glob(os.path.join(args.doc_dir, "*.md")) html_files = [] for f in doc_files: fout = f"{os.path.splitext(f)[0]}.html" convertmd2html(f, fout) html_files.append(fout) files = doc_files + html_files # Generate MD5 sums file doc_md5sums_file = "md5sums.txt" generate_md5sums_file(files, doc_md5sums_file) files.append(doc_md5sums_file) # Copy files for output_dir in output_dirs: copy_files(files, output_dir) # Package image files if args.images_dir is not None: # Add some additional files to the artifacts for artifact in image_artifacts: artifacts.append(f"{artifact.split('.')[0]}.manifest") artifacts.append(f"{artifact.split('.')[0]}.testdata.json") # Prepend path to artifacts artifacts = [os.path.join(args.images_dir, artifact) for artifact in artifacts] # If the path for the licenses is set, we check for the list with all # licenses. If the list is found, we copy it to the output directory # and also add it to the artifacts dictionary. license_manifest = None if args.licenses_dir is not None and os.path.isdir(args.licenses_dir): license_manifest = glob.glob( os.path.join(args.licenses_dir, "**", "license.manifest") ) artifacts.append( os.path.join(os.path.dirname(license_manifest[0]), "license.manifest") ) # Generate MD5 sums file image_md5sums_file = os.path.join(args.images_dir, "md5sums.txt") generate_md5sums_file(artifacts, image_md5sums_file) artifacts.append(image_md5sums_file) # Generate metadata file generate_metadata( machine, version, artifacts, sdkname, os.path.join(args.images_dir, "metainfo.json"), ) artifacts.append(os.path.join(args.images_dir, "metainfo.json")) # Copy files for output_dir in output_dirs: copy_files(artifacts, os.path.join(output_dir, machine)) # Package SDK if args.sdk_dir is not None: sdkfiles = glob.glob(os.path.join(args.sdk_dir, f"{sdkname}*")) # Generate MD5 sums file sdk_md5sums_file = os.path.join(machine, "sdk", "md5sums.txt") generate_md5sums_file(sdkfiles, sdk_md5sums_file) sdkfiles.append(sdk_md5sums_file) # Copy files for output_dir in output_dirs: copy_files(sdkfiles, os.path.join(output_dir, machine, "sdk")) # Store pathes and other stuff in environment variable file with open("package.env", "w", encoding="utf-8") as env_file: env_file.write(f"RELEASE_NAME={release_name}\n") env_file.write(f"VERSION={version}\n") env_file.write(f"MACHINE={machine}\n") if __name__ == "__main__": main()