CI: Adapt package_release to also deploy artifacts locally

To upload the sdk the local deployment on build host is needed as it is
too big for the uploads.

Move copy functionality to extra function. It also generates md5sum.txt
from the source file's md5sums.
Copy locally (host system) and as gitlab artifact.

Add option to copy doc files ( all .md in given folder).

BCS 746-000262

package_release.py

@@ -5,7 +5,69 @@ import json
 import os
 import sys
 import shutil
-import subprocess
+import hashlib
+import tempfile
+
+
+def md5(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def copy_files(files, input_dir, subdir, output_dir, outlocal_dir):
+    md5sums = {}
+    if output_dir is not None:
+        os.makedirs(os.path.join(output_dir, subdir), exist_ok=True)
+    if outlocal_dir is not None:
+        os.makedirs(os.path.join(outlocal_dir, subdir), exist_ok=True)
+
+    for f in files:
+        source_file = os.path.join(input_dir, f)
+        if os.path.exists(source_file):
+
+            if output_dir is not None:
+                target_file = os.path.join(
+                    output_dir, subdir, os.path.basename(source_file)
+                )
+                print("Copy: %s -> %s" % (source_file, target_file))
+                shutil.copyfile(source_file, target_file, follow_symlinks=True)
+
+            if outlocal_dir is not None:
+                target_file = os.path.join(
+                    outlocal_dir, subdir, os.path.basename(source_file)
+                )
+                print("Copy: %s -> %s" % (source_file, target_file))
+                shutil.copyfile(source_file, target_file, follow_symlinks=True)
+
+            md5sums[os.path.basename(source_file)] = md5(source_file)
+        else:
+            print("Missing: " + source_file)
+
+    # Write md5sums file:
+    with tempfile.TemporaryDirectory() as tmp:
+        source_file = os.path.join(tmp, subdir, "md5sums.txt")
+        os.makedirs(os.path.dirname(source_file), exist_ok=True)
+
+        with open(source_file, "w") as f_md5:
+            for f, h in md5sums.items():
+                f_md5.write("{}  {}\n".format(h, f))
+
+        if output_dir is not None:
+            target_file = os.path.join(
+                output_dir, subdir, os.path.basename(source_file)
+            )
+            print("Copy: %s -> %s" % (source_file, target_file))
+            shutil.copyfile(source_file, target_file, follow_symlinks=True)
+
+        if outlocal_dir is not None:
+            target_file = os.path.join(
+                outlocal_dir, subdir, os.path.basename(source_file)
+            )
+            print("Copy: %s -> %s" % (source_file, target_file))
+            shutil.copyfile(source_file, target_file, follow_symlinks=True)
 
 
 def main():
@@ -14,7 +76,6 @@ def main():
         "--images-dir",
         help="""Yocto images directory""",
         dest="images_dir",
-        required=True,
     )
     parser.add_argument(
         "--release-dir",
@@ -27,13 +88,30 @@ def main():
         help="""Yocto sdk directory""",
         dest="sdk_dir",
     )
+    parser.add_argument(
+        "--doc-dir",
+        help="""Documentation directory""",
+        dest="doc_dir",
+    )
 
     args, _ = parser.parse_known_args()
 
     # Get bitbake variables from testdata.json file
-    testdata_files = glob.glob(args.images_dir + "/*.testdata.json")
+    testdata_files = []
+    if args.images_dir is not None:
+        testdata_files += glob.glob(args.images_dir + "/*.testdata.json")
+    if args.sdk_dir is not None:
+        testdata_files += glob.glob(args.sdk_dir + "/*.testdata.json")
+
+    # Debug stuff
     if not testdata_files:
-        sys.exit("ERROR: no *.testdata.json file found in images directory")
+        print(args.images_dir)
+        for f in glob.glob(args.images_dir + "/*"):
+            print("-- ", f)
+        print(args.sdk_dir)
+        for f in glob.glob(args.sdk_dir + "/*"):
+            print("-- ", f)
+        sys.exit("ERROR: no *.testdata.json file found in image or sdk dir.")
 
     with open(testdata_files[0], "r") as f:
         buildvars = json.load(f)
@@ -43,45 +121,41 @@ def main():
     sdkname = buildvars["TOOLCHAIN_OUTPUTNAME"]
     artifacts_image = buildvars["DISTRO_IMAGES"].split()
     artifacts_all = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
+    artifacts_all.append("BUILD_SRCREVS.log")
 
     # Set directories (use relative path)
     release_name = "GUF-Yocto-%s" % version
-    output_dir = "%s/%s/%s" % (args.release_dir, release_name, machine)
-    images_dir_relative = os.path.relpath(args.images_dir, output_dir)
-
+    # output_dir = "%s/%s/%s" % (args.release_dir, release_name, machine)
+    output_dir = os.path.join(args.release_dir, release_name)
     # Create output directory
     os.makedirs(output_dir, exist_ok=True)
 
-    # Create symlinks for all images (to reduce size and network load)
-    for artifact in artifacts_image:
-        source_file = "%s/%s" % (images_dir_relative, artifact)
-        target_file = "%s/%s" % (output_dir, artifact)
-        print("Create link: %s -> %s" % (target_file, source_file))
-        os.symlink(source_file, target_file)
-
-    # Copy all other artifacts
-    for artifact in artifacts_all:
-        if artifact in artifacts_image:
-            continue
-        source_file = "%s/%s" % (args.images_dir, artifact)
-        target_file = "%s/%s" % (output_dir, artifact)
-        print("Copy: %s -> %s" % (target_file, source_file))
-        shutil.copyfile(source_file, target_file, follow_symlinks=True)
+    # local output ( mount on host machine)
+    release_name_local = "Yocto-%s" % version
+    if version.startswith("fngsystem"):
+        outlocal_base = "/artifacts-fngsystem"
+    else:
+        outlocal_base = "/artifacts-yocto"
+    # outlocal_dir = "%s/%s/%s" % (outlocal_base, release_name_local, machine)
+    outlocal_dir = os.path.join(outlocal_base, release_name_local)
+    os.makedirs(outlocal_dir, exist_ok=True)
+
+    if args.doc_dir is not None:
+        doc_files = glob.glob(args.doc_dir + "/*.md")
+        copy_files(doc_files, "", "", output_dir, outlocal_dir)
+
+    if args.images_dir is not None:
+        # Add some additional files to the artifacts
+        for artifact in artifacts_image:
+            artifacts_all.append(artifact.split(".")[0] + ".manifest")
+            artifacts_all.append(artifact.split(".")[0] + ".testdata.json")
+
+        copy_files(artifacts_all, args.images_dir, machine, output_dir, outlocal_dir)
 
     # Handle SDK if available
-    if args.sdk_dir != "":
-        sdkfile = "{}.sh".format(sdkname)
-        sdkfilepath = "{}/{}".format(args.sdk_dir, sdkfile)
-        sdkoutput_dir = "{}/sdk".format(output_dir)
-        sdkfile_parts = "{}/{}.part".format(sdkoutput_dir, sdkfile)
-        if os.path.exists(sdkfilepath):
-            print("Copy and split SDK from ", sdkfilepath)
-            os.makedirs(sdkoutput_dir, exist_ok=True)
-            subprocess.run(
-                ["split", "-b", "512M", sdkfilepath, sdkfile_parts], check=True
-            )
-        else:
-            print("Skipping SDK, not found at ", sdkfilepath)
+    if args.sdk_dir is not None:
+        sdkfiles = glob.glob(args.sdk_dir + sdkname + "*")
+        copy_files(sdkfiles, "", os.path.join(machine, "sdk"), None, outlocal_dir)
 
 
 if __name__ == "__main__":