From de38c9f02270f276cdead60b4704d10482d38ebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20H=C3=B6ppner?= <jonas.hoeppner@garz-fricke.com>
Date: Fri, 6 Aug 2021 13:32:47 +0200
Subject: [PATCH] CI: Adapt package_release to also deploy artifacts locally
To upload the sdk the local deployment on build host is needed as it is
too big for the uploads.
Move copy functionality to extra function. It also generates md5sum.txt
from the source file's md5sums.
Copy locally (host system) and as gitlab artifact.
Add option to copy doc files ( all .md in given folder).
BCS 746-000262
---
package_release.py | 144 ++++++++++++++++++++++++++++++++++-----------
1 file changed, 109 insertions(+), 35 deletions(-)
diff --git a/package_release.py b/package_release.py
index 36dde3c4..b7bc8241 100755
--- a/package_release.py
+++ b/package_release.py
@@ -5,7 +5,69 @@ import json
import os
import sys
import shutil
-import subprocess
+import hashlib
+import tempfile
+
+
+def md5(fname):
+ hash_md5 = hashlib.md5()
+ with open(fname, "rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hash_md5.update(chunk)
+ return hash_md5.hexdigest()
+
+
+def copy_files(files, input_dir, subdir, output_dir, outlocal_dir):
+ md5sums = {}
+ if output_dir is not None:
+ os.makedirs(os.path.join(output_dir, subdir), exist_ok=True)
+ if outlocal_dir is not None:
+ os.makedirs(os.path.join(outlocal_dir, subdir), exist_ok=True)
+
+ for f in files:
+ source_file = os.path.join(input_dir, f)
+ if os.path.exists(source_file):
+
+ if output_dir is not None:
+ target_file = os.path.join(
+ output_dir, subdir, os.path.basename(source_file)
+ )
+ print("Copy: %s -> %s" % (source_file, target_file))
+ shutil.copyfile(source_file, target_file, follow_symlinks=True)
+
+ if outlocal_dir is not None:
+ target_file = os.path.join(
+ outlocal_dir, subdir, os.path.basename(source_file)
+ )
+ print("Copy: %s -> %s" % (source_file, target_file))
+ shutil.copyfile(source_file, target_file, follow_symlinks=True)
+
+ md5sums[os.path.basename(source_file)] = md5(source_file)
+ else:
+ print("Missing: " + source_file)
+
+ # Write md5sums file:
+ with tempfile.TemporaryDirectory() as tmp:
+ source_file = os.path.join(tmp, subdir, "md5sums.txt")
+ os.makedirs(os.path.dirname(source_file), exist_ok=True)
+
+ with open(source_file, "w") as f_md5:
+ for f, h in md5sums.items():
+ f_md5.write("{} {}\n".format(h, f))
+
+ if output_dir is not None:
+ target_file = os.path.join(
+ output_dir, subdir, os.path.basename(source_file)
+ )
+ print("Copy: %s -> %s" % (source_file, target_file))
+ shutil.copyfile(source_file, target_file, follow_symlinks=True)
+
+ if outlocal_dir is not None:
+ target_file = os.path.join(
+ outlocal_dir, subdir, os.path.basename(source_file)
+ )
+ print("Copy: %s -> %s" % (source_file, target_file))
+ shutil.copyfile(source_file, target_file, follow_symlinks=True)
def main():
@@ -14,7 +76,6 @@ def main():
"--images-dir",
help="""Yocto images directory""",
dest="images_dir",
- required=True,
)
parser.add_argument(
"--release-dir",
@@ -27,13 +88,30 @@ def main():
help="""Yocto sdk directory""",
dest="sdk_dir",
)
+ parser.add_argument(
+ "--doc-dir",
+ help="""Documentation directory""",
+ dest="doc_dir",
+ )
args, _ = parser.parse_known_args()
# Get bitbake variables from testdata.json file
- testdata_files = glob.glob(args.images_dir + "/*.testdata.json")
+ testdata_files = []
+ if args.images_dir is not None:
+ testdata_files += glob.glob(args.images_dir + "/*.testdata.json")
+ if args.sdk_dir is not None:
+ testdata_files += glob.glob(args.sdk_dir + "/*.testdata.json")
+
+ # Debug stuff
if not testdata_files:
- sys.exit("ERROR: no *.testdata.json file found in images directory")
+ print(args.images_dir)
+ for f in glob.glob(args.images_dir + "/*"):
+ print("-- ", f)
+ print(args.sdk_dir)
+ for f in glob.glob(args.sdk_dir + "/*"):
+ print("-- ", f)
+ sys.exit("ERROR: no *.testdata.json file found in image or sdk dir.")
with open(testdata_files[0], "r") as f:
buildvars = json.load(f)
@@ -43,45 +121,41 @@ def main():
sdkname = buildvars["TOOLCHAIN_OUTPUTNAME"]
artifacts_image = buildvars["DISTRO_IMAGES"].split()
artifacts_all = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
+ artifacts_all.append("BUILD_SRCREVS.log")
# Set directories (use relative path)
release_name = "GUF-Yocto-%s" % version
- output_dir = "%s/%s/%s" % (args.release_dir, release_name, machine)
- images_dir_relative = os.path.relpath(args.images_dir, output_dir)
-
+ # output_dir = "%s/%s/%s" % (args.release_dir, release_name, machine)
+ output_dir = os.path.join(args.release_dir, release_name)
# Create output directory
os.makedirs(output_dir, exist_ok=True)
- # Create symlinks for all images (to reduce size and network load)
- for artifact in artifacts_image:
- source_file = "%s/%s" % (images_dir_relative, artifact)
- target_file = "%s/%s" % (output_dir, artifact)
- print("Create link: %s -> %s" % (target_file, source_file))
- os.symlink(source_file, target_file)
-
- # Copy all other artifacts
- for artifact in artifacts_all:
- if artifact in artifacts_image:
- continue
- source_file = "%s/%s" % (args.images_dir, artifact)
- target_file = "%s/%s" % (output_dir, artifact)
- print("Copy: %s -> %s" % (target_file, source_file))
- shutil.copyfile(source_file, target_file, follow_symlinks=True)
+ # local output ( mount on host machine)
+ release_name_local = "Yocto-%s" % version
+ if version.startswith("fngsystem"):
+ outlocal_base = "/artifacts-fngsystem"
+ else:
+ outlocal_base = "/artifacts-yocto"
+ # outlocal_dir = "%s/%s/%s" % (outlocal_base, release_name_local, machine)
+ outlocal_dir = os.path.join(outlocal_base, release_name_local)
+ os.makedirs(outlocal_dir, exist_ok=True)
+
+ if args.doc_dir is not None:
+ doc_files = glob.glob(args.doc_dir + "/*.md")
+ copy_files(doc_files, "", "", output_dir, outlocal_dir)
+
+ if args.images_dir is not None:
+ # Add some additional files to the artifacts
+ for artifact in artifacts_image:
+ artifacts_all.append(artifact.split(".")[0] + ".manifest")
+ artifacts_all.append(artifact.split(".")[0] + ".testdata.json")
+
+ copy_files(artifacts_all, args.images_dir, machine, output_dir, outlocal_dir)
# Handle SDK if available
- if args.sdk_dir != "":
- sdkfile = "{}.sh".format(sdkname)
- sdkfilepath = "{}/{}".format(args.sdk_dir, sdkfile)
- sdkoutput_dir = "{}/sdk".format(output_dir)
- sdkfile_parts = "{}/{}.part".format(sdkoutput_dir, sdkfile)
- if os.path.exists(sdkfilepath):
- print("Copy and split SDK from ", sdkfilepath)
- os.makedirs(sdkoutput_dir, exist_ok=True)
- subprocess.run(
- ["split", "-b", "512M", sdkfilepath, sdkfile_parts], check=True
- )
- else:
- print("Skipping SDK, not found at ", sdkfilepath)
+ if args.sdk_dir is not None:
+ sdkfiles = glob.glob(args.sdk_dir + sdkname + "*")
+ copy_files(sdkfiles, "", os.path.join(machine, "sdk"), None, outlocal_dir)
if __name__ == "__main__":
--
GitLab