diff --git a/scripts/generate_release_metadata.py b/scripts/generate_release_metadata.py
index f1f6418bd616d94b04cb9bf51a7cfda0837a0be1..01752a767d9da1cd6f014829458ba6145e3e021f 100755
--- a/scripts/generate_release_metadata.py
+++ b/scripts/generate_release_metadata.py
@@ -7,39 +7,29 @@ from datetime import datetime
def generate_metadata(
machine,
version,
- artifacts_image,
+ image_artifacts,
sdk,
- output_dir,
- outlocal_dir,
+ output_file,
):
- """Generates a metainfo.json for the release"""
+ """Generate a metainfo.json for the release and write it to output_file."""
install_script = None
licenses = None
image_general = None
image_wic = None
- # Join filepath for metadata
-
- if output_dir is not None:
- filepath = os.path.join(output_dir, machine, "metainfo.json")
- elif outlocal_dir is not None:
- filepath = os.path.join(outlocal_dir, machine, "metainfo.json")
- else:
- print("Error: Filepath is empty")
- return -1
-
# Collect metadata and write to metainfo.json
- for artifact in artifacts_image:
- if artifact == "fng-install.sh":
- install_script = artifact
- elif artifact == "license.manifest":
- licenses = artifact
- elif artifact.endswith(machine + ".tar.gz"):
- image_general = artifact
- elif artifact.endswith(machine + ".wic"):
- image_wic = artifact
+ for artifact in image_artifacts:
+ filename = os.path.basename(artifact)
+ if filename == "fng-install.sh":
+ install_script = filename
+ elif filename == "license.manifest":
+ licenses = filename
+ elif filename.endswith(machine + ".tar.gz"):
+ image_general = filename
+ elif filename.endswith(machine + ".wic"):
+ image_wic = filename
metadata = dict()
@@ -78,5 +68,5 @@ def generate_metadata(
new_file["path"] = licenses
metadata["files"].append(new_file)
- with open(filepath, "w", encoding="utf-8") as file:
+ with open(output_file, "w", encoding="utf-8") as file:
file.write(json.dumps(metadata))
diff --git a/scripts/package_release.py b/scripts/package_release.py
index 9e7a414dc21f42cc7a692d840346052300411182..68773cfd5e6f21d403f0ef8690e0fcbd305e491f 100755
--- a/scripts/package_release.py
+++ b/scripts/package_release.py
@@ -6,74 +6,53 @@ import os
import sys
import shutil
import hashlib
-import tempfile
+
from convert_md2html import convertmd2html
from generate_release_metadata import generate_metadata
-def md5(fname):
+def md5(input_file: str):
+ """
+ Calculate and return the MD5 sum of the given input_file.
+ """
hash_md5 = hashlib.md5()
- with open(fname, "rb", encoding=None) as f:
+ with open(input_file, "rb", encoding=None) as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
-def write_md5sums_file(md5sums, subdir, output_dir, outlocal_dir):
- with tempfile.TemporaryDirectory() as tmp:
- source_file = os.path.join(tmp, subdir, "md5sums.txt")
- os.makedirs(os.path.dirname(source_file), exist_ok=True)
-
- with open(source_file, "w", encoding="utf-8") as f_md5:
- for f, h in md5sums.items():
- f_md5.write("{} {}\n".format(h, f))
-
- if output_dir is not None:
- target_file = os.path.join(
- output_dir, subdir, os.path.basename(source_file)
- )
- print("Copy: %s -> %s" % (source_file, target_file))
- shutil.copyfile(source_file, target_file, follow_symlinks=True)
-
- if outlocal_dir is not None:
- target_file = os.path.join(
- outlocal_dir, subdir, os.path.basename(source_file)
- )
- print("Copy: %s -> %s" % (source_file, target_file))
- shutil.copyfile(source_file, target_file, follow_symlinks=True)
-
-
-def copy_files(files, input_dir, subdir, output_dir, outlocal_dir):
+def generate_md5sums_file(input_files: list[str], output_file: str):
+ """
+ Calculate MD5 sums of all input_files and write them to output_file.
+ """
md5sums = {}
- if output_dir is not None:
- os.makedirs(os.path.join(output_dir, subdir), exist_ok=True)
- if outlocal_dir is not None:
- os.makedirs(os.path.join(outlocal_dir, subdir), exist_ok=True)
-
- for f in files:
- source_file = os.path.join(input_dir, f)
+ for f in input_files:
+ md5sums[os.path.basename(f)] = md5(f)
+ output_dir = os.path.dirname(output_file)
+ if output_dir:
+ os.makedirs(output_dir, exist_ok=True)
+ with open(output_file, "w", encoding="utf-8") as f_md5:
+ for f, h in md5sums.items():
+ f_md5.write("{} {}\n".format(h, f))
+
+
+def copy_files(files: list[str], target_dir: str):
+ """
+ Copy given files to target_dir. Create target_dir, if it does not exist. Subfolder
+ hierarchies on the input files will not be preserved, only plain files are copied.
+ """
+ if target_dir is None:
+ return
+ os.makedirs(target_dir, exist_ok=True)
+ for source_file in files:
if os.path.exists(source_file):
-
- if output_dir is not None:
- target_file = os.path.join(
- output_dir, subdir, os.path.basename(source_file)
- )
- print("Copy: %s -> %s" % (source_file, target_file))
- shutil.copyfile(source_file, target_file, follow_symlinks=True)
-
- if outlocal_dir is not None:
- target_file = os.path.join(
- outlocal_dir, subdir, os.path.basename(source_file)
- )
- print("Copy: %s -> %s" % (source_file, target_file))
- shutil.copyfile(source_file, target_file, follow_symlinks=True)
-
- md5sums[os.path.basename(source_file)] = md5(source_file)
+ target_file = os.path.join(target_dir, os.path.basename(source_file))
+ print("Copy: %s -> %s" % (source_file, target_file))
+ shutil.copyfile(source_file, target_file, follow_symlinks=True)
else:
print("Missing: " + source_file)
- return md5sums
-
def main():
parser = argparse.ArgumentParser()
@@ -121,7 +100,7 @@ def main():
if args.sdk_dir is not None:
testdata_files += glob.glob(os.path.join(args.sdk_dir, "*.testdata.json"))
- # Debug stuff
+ # Debug output if no testdata file found
if not testdata_files:
if args.images_dir is not None:
print(args.images_dir)
@@ -133,37 +112,39 @@ def main():
print("-- ", f)
sys.exit("ERROR: no *.testdata.json file found in image or sdk dir.")
+ # The required build variables from testdata have the same values for image and SDK
+ # builds, so we just read one of them.
with open(testdata_files[0], "r", encoding="utf-8") as f:
buildvars = json.load(f)
machine = buildvars["MACHINE"]
version = buildvars["DISTRO_VERSION"]
sdkname = buildvars["TOOLCHAIN_OUTPUTNAME"]
- artifacts_image = buildvars["DISTRO_IMAGES"].split()
- artifacts_all = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
- artifacts_all.append("BUILD_SRCREVS.log")
+ image_artifacts = buildvars["DISTRO_IMAGES"].split()
+ artifacts = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
+ artifacts.append("BUILD_SRCREVS.log")
+ # Set release name
if version.startswith("fngsystem"):
- release_name_local = version.replace("fngsystem", "FNGSystem")
- # outlocal_base = "/artifacts-fngsystem"
+ release_name = version.replace("fngsystem", "FNGSystem")
else:
- release_name_local = "Yocto-%s" % version
- # outlocal_base = "/artifacts-yocto"
+ release_name = "Yocto-%s" % version
# Create output directories
if args.outputdir_upload is not None:
- output_dir = os.path.join(args.outputdir_upload, release_name_local)
- os.makedirs(output_dir, exist_ok=True)
+ outputdir_upload = os.path.join(args.outputdir_upload, release_name)
+ os.makedirs(outputdir_upload, exist_ok=True)
else:
- output_dir = None
+ outputdir_upload = None
if args.outputdir_local is not None:
- outlocal_dir = os.path.join(args.outputdir_local, release_name_local)
- os.makedirs(outlocal_dir, exist_ok=True)
+ outputdir_local = os.path.join(args.outputdir_local, release_name)
+ os.makedirs(outputdir_local, exist_ok=True)
else:
- outlocal_dir = None
+ outputdir_local = None
- # Convert markdown files into html and package them
+ # Package documentation files
if args.doc_dir is not None:
+ # Convert markdown to html
doc_files = glob.glob(os.path.join(args.doc_dir, "*.md"))
html_files = []
for f in doc_files:
@@ -171,20 +152,28 @@ def main():
convertmd2html(f, fout)
html_files.append(fout)
- doc_md5sums = copy_files(
- doc_files + html_files, "", "", output_dir, outlocal_dir
- )
- write_md5sums_file(doc_md5sums, "", output_dir, outlocal_dir)
+ files = doc_files + html_files
+ # Generate MD5 sums file
+ doc_md5sums_file = "md5sums.txt"
+ generate_md5sums_file(files, doc_md5sums_file)
+ files.append(doc_md5sums_file)
+
+ # Copy files
+ if outputdir_upload is not None:
+ copy_files(files, outputdir_upload)
+ if outputdir_local is not None:
+ copy_files(files, outputdir_local)
+
+ # Package image files
if args.images_dir is not None:
# Add some additional files to the artifacts
- for artifact in artifacts_image:
- artifacts_all.append(artifact.split(".")[0] + ".manifest")
- artifacts_all.append(artifact.split(".")[0] + ".testdata.json")
+ for artifact in image_artifacts:
+ artifacts.append(artifact.split(".")[0] + ".manifest")
+ artifacts.append(artifact.split(".")[0] + ".testdata.json")
- md5sums = copy_files(
- artifacts_all, args.images_dir, machine, output_dir, outlocal_dir
- )
+ # Prepend path to artifacts
+ artifacts = [os.path.join(args.images_dir, artifact) for artifact in artifacts]
# If the path for the licenses is set, we check for the list with all
# licenses. If the list is found, we copy it to the output directory
@@ -194,52 +183,49 @@ def main():
license_manifest = glob.glob(
os.path.join(args.licenses_dir, "**", "license.manifest")
)
+ artifacts.append(
+ os.path.join(os.path.dirname(license_manifest[0]), "license.manifest")
+ )
- if license_manifest:
- md5sums.update(
- copy_files(
- ["license.manifest"],
- os.path.dirname(license_manifest[0]),
- machine,
- output_dir,
- outlocal_dir,
- )
- )
- artifacts_all.append("license.manifest")
-
- # Create md5sums file for build artifacts
- write_md5sums_file(md5sums, machine, output_dir, outlocal_dir)
-
- # Generate metadata in case of an image build
- if args.sdk_dir is None:
+ # Generate MD5 sums file
+ image_md5sums_file = os.path.join(args.images_dir, "md5sums.txt")
+ generate_md5sums_file(artifacts, image_md5sums_file)
+ artifacts.append(image_md5sums_file)
+
+ # Generate metadata file
generate_metadata(
machine,
version,
- artifacts_all,
+ artifacts,
sdkname,
- output_dir,
- outlocal_dir,
+ os.path.join(args.images_dir, "metainfo.json"),
)
+ artifacts.append(os.path.join(args.images_dir, "metainfo.json"))
+
+ # Copy files
+ if outputdir_upload is not None:
+ copy_files(artifacts, os.path.join(outputdir_upload, machine))
+ if outputdir_local is not None:
+ copy_files(artifacts, os.path.join(outputdir_local, machine))
- # Handle SDK if available
+ # Package SDK
if args.sdk_dir is not None:
sdkfiles = glob.glob(os.path.join(args.sdk_dir, sdkname + "*"))
- print("package_release.py: Upload the following sdk files:")
- print(sdkfiles)
- sdk_md5sums = copy_files(
- sdkfiles, "", os.path.join(machine, "sdk"), None, outlocal_dir
- )
- write_md5sums_file(
- sdk_md5sums, os.path.join(machine, "sdk"), None, outlocal_dir
- )
- else:
- print("package_release.py: sdk_dir is not specified")
+
+ # Generate MD5 sums file
+ sdk_md5sums_file = os.path.join(machine, "sdk", "md5sums.txt")
+ generate_md5sums_file(sdkfiles, sdk_md5sums_file)
+ sdkfiles.append(sdk_md5sums_file)
+
+ # Copy files
+ if outputdir_local is not None:
+ copy_files(sdkfiles, os.path.join(outputdir_local, machine, "sdk"))
# Store pathes and other stuff in environment variable file
with open("package.env", "w", encoding="utf-8") as env_file:
env_file.write("VERSION={}\n".format(version))
env_file.write("MACHINE={}\n".format(machine))
- env_file.write("LOCALDIR={}\n".format(outlocal_dir))
+ env_file.write("LOCALDIR={}\n".format(outputdir_local))
if __name__ == "__main__":