diff --git a/scripts/report_image_diff.py b/scripts/report_image_diff.py
index aa42ea52edb8ea2ee866a4135e8763b1e386b5b4..35b0f3e6191d3a965686cc20b8f8ed316a948cc2 100755
--- a/scripts/report_image_diff.py
+++ b/scripts/report_image_diff.py
@@ -2,11 +2,15 @@
import argparse
import fnmatch
import logging
+import os
import re
import sys
from difflib import unified_diff
+from platform import machine
from gitlab import Gitlab
+from gitlab import GitlabGetError
+from gitlab.v4.objects import Project
import common
from buildartifacts import BuildArtifacts
@@ -36,6 +40,210 @@ def sizeof_fmt(num: int, p: int = 2) -> str:
return f"{num:.{p}f} YiB"
+class BuildJobArtifactReport:
+ def __init__(self, project: Project, parent_pipeline, _pipeline, _machine, _distro):
+ self.parent_project = project
+ self.parent_pipeline = parent_pipeline
+ self.pipeline = _pipeline
+ self.machine = _machine
+ self.distro = _distro
+
+ self.__installed_packages = None
+ self.__installed_package_info = None
+ self.__installed_package_size = None
+
+ self.artifacts = BuildArtifacts(
+ self.parent_project,
+ self.parent_pipeline.get_jobs(self.pipeline, f"build-{self.machine}")[0],
+ )
+
+ @property
+ def image_name(self):
+ if "YOCTO_IMAGE" in self.artifacts.buildenv.keys():
+ return self.artifacts.buildenv["YOCTO_IMAGE"]
+ if "fngsystem" in self.distro:
+ return "fngsystem-image"
+ return "seconorth-image"
+
+ @property
+ def build_dir(self):
+ return f"build-{self.distro}-{self.machine}"
+
+ @property
+ def buildhistory_dir(self):
+ return os.path.join(
+ self.build_dir,
+ "buildhistory",
+ "images",
+ self.machine.replace("-", "_"),
+ "glibc",
+ self.image_name,
+ )
+
+ def __download_buildhistory(self, filename):
+ file = os.path.join(self.buildhistory_dir, filename)
+ try:
+ data = self.artifacts.get_artifact(file)
+ except GitlabGetError as e:
+ logging.error(f"Failed to download {file}: {e}.")
+ return None
+ return data.decode().splitlines()
+
+ @property
+ def installed_packages(self):
+ if self.__installed_packages is None:
+ self.__installed_packages = self.__download_buildhistory(
+ "installed-packages.txt"
+ )
+ return self.__installed_packages
+
+ @property
+ def installed_package_info(self):
+ if self.__installed_package_info is None:
+ self.__installed_package_info = self.__download_buildhistory(
+ "installed-package-info.txt"
+ )
+ return self.__installed_package_info
+
+ @property
+ def installed_package_size(self):
+ if self.__installed_package_size is None:
+ self.__installed_package_size = self.__download_buildhistory(
+ "installed-package-sizes.txt"
+ )
+ return self.__installed_package_size
+
+ @property
+ def archive_size(self):
+ return self.artifacts.get_archive_size()
+
+ @property
+ def image_size(self):
+ total = 0
+ for line in self.installed_package_size:
+ size, unit, name = line.split()
+ if unit == "MiB":
+ factor = 1024 * 1024
+ elif unit == "KiB":
+ factor = 1024
+ else:
+ factor = 1
+ total += int(size) * factor
+ return total
+
+ def compare_and_summarize(self, other: "BuildJobArtifactReport") -> str:
+ if self.distro != other.distro or self.machine != other.machine:
+ raise Exception("Comparing apples with pears.")
+
+ logging.debug(f"Create report for {self.machine} {self.distro}.")
+
+ # The difference in size of artifacts.zip for main and MR builds
+ zip_size_diff = sizeof_fmt(abs(self.archive_size - other.archive_size))
+ sign = "+" if self.archive_size < other.archive_size else "-"
+ summary = f"**`⮘ {self.distro} | {self.machine} ⮚`**\\\n"
+ summary += f" ├── artifacts.zip size: [ {sizeof_fmt(self.archive_size)} → {sizeof_fmt(other.archive_size)} ] | {sign}{zip_size_diff}\\\n"
+ summary += f" ├── image size: [ {sizeof_fmt(self.image_size)} → {sizeof_fmt(other.image_size)} ]\\\n"
+ summary += " └── installed packages diff:\n"
+ summary += "```diff\n"
+
+ for line in unified_diff(
+ self.installed_packages,
+ other.installed_packages,
+ n=0,
+ lineterm="",
+ ):
+ if fnmatch.fnmatch(line, "@@ * @@"):
+ continue
+ if fnmatch.fnmatch(line, "--- *"):
+ continue
+ if fnmatch.fnmatch(line, "+++ *"):
+ continue
+
+ summary += line + "\n"
+ summary += "```\n"
+ summary += "\n"
+ return summary
+
+
+class BuildArtifactReport:
+ def __init__(self, project: Project, refname: str):
+ self.project = project
+ self.refname = refname
+ self.commit = self.project.commits.list(
+ all=False, ref_name=self.refname, order_by="id", sort="desc"
+ )[0]
+ self.pipeline = FullBuildPipeline(
+ self.project, self.commit.id, ignore_pipeine_status=True
+ )
+
+ self.machines, self.pipelines_and_distros = self.__find_distro_and_machines()
+
+ if not self.machines:
+ raise FileNotFoundError("Failed to extract MACHINES from jobs names")
+ if not self.pipelines_and_distros:
+ raise FileNotFoundError("Failed to extract DISTRO from jobs logs")
+
+ self.buildPipelineArtifacts = self.__get_build_pipelines()
+
+ def __find_distro_and_machines(self):
+ machines = []
+ pipelines_and_distros = {}
+ pattern_distro = r"DISTRO=([\w\d_.-]+)\n"
+ for pipeline, jobs in self.pipeline.build_pipelines.items():
+ for j in (j for j in jobs if j.stage == "Build" and j.status == "success"):
+
+ logging.debug(f"Found job named '{j.name}'.")
+
+ if j.name == "build-documentation":
+ continue
+
+ m = j.name.removeprefix("build-")
+ if m not in machines:
+ machines.append(m)
+ logging.debug(f"Found machine named '{m}'.")
+
+ if pipeline in pipelines_and_distros.keys():
+ continue
+
+ # Job methods (e.g. trace) are not available on ProjectPipelineJob objects.
+ # To use these methods create a ProjectJob object:
+ # pipeline_job = pipeline.jobs.list()[0]
+ # job = project.jobs.get(pipeline_job.id, lazy=True)
+ # job.trace()
+ job_log = self.project.jobs.get(j.id, lazy=True).trace().decode("utf-8")
+
+ search_distro = re.search(pattern_distro, job_log)
+ if search_distro:
+ pipelines_and_distros[pipeline] = search_distro.group(1)
+
+ return machines, pipelines_and_distros
+
+ def __get_build_pipelines(self):
+ buildpipeline = {}
+ for pipeline, distro in self.pipelines_and_distros.items():
+ buildpipeline[distro] = {}
+ for m in sorted(self.machines):
+ try:
+ buildpipeline[distro][m] = BuildJobArtifactReport(
+ self.project, self.pipeline, pipeline, m, distro
+ )
+ except IndexError:
+ logging.warning(
+ f"Failed to find build for {m} in latest pipeline."
+ )
+ continue
+ return buildpipeline
+
+ def compare_and_summarize(self, other: "BuildArtifactReport") -> str:
+ summary = ""
+ for distro, pipelines in self.buildPipelineArtifacts.items():
+ for m in pipelines.keys():
+ summary += pipelines[m].compare_and_summarize(
+ other.buildPipelineArtifacts[distro][m],
+ )
+ return summary
+
+
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -104,159 +312,11 @@ def main():
if comment.body[0:4] == "**`⮘":
comment.delete()
- manifest_commit__main = manifest_project.commits.list(
- all=False, ref_name=args.target_branch, order_by="id", sort="desc"
- )[0]
- build__main = FullBuildPipeline(manifest_project, manifest_commit__main.id)
-
- manifest_commit__mr = manifest_project.commits.list(
- all=False, ref_name=args.source_branch, order_by="id", sort="desc"
- )[0]
- build__mr = FullBuildPipeline(
- manifest_project, manifest_commit__mr.id, ignore_pipeine_status=True
- )
-
- machines = []
- pipelines_and_distros = {}
- pattern_distro = r"DISTRO=([\w\d_.-]+)\n"
- for pipeline, jobs in build__mr.build_pipelines.items():
- for j in (j for j in jobs if j.stage == "Build" and j.status == "success"):
-
- if j.name == "build-documentation":
- continue
-
- machine = j.name.removeprefix("build-")
- if machine not in machines:
- machines.append(machine)
-
- if pipeline in pipelines_and_distros.keys():
- continue
-
- # Job methods (e.g. trace) are not available on ProjectPipelineJob objects.
- # To use these methods create a ProjectJob object:
- # pipeline_job = pipeline.jobs.list()[0]
- # job = project.jobs.get(pipeline_job.id, lazy=True)
- # job.trace()
- job_log = manifest_project.jobs.get(j.id, lazy=True).trace().decode("utf-8")
-
- search_distro = re.search(pattern_distro, job_log)
- if search_distro:
- pipelines_and_distros[pipeline] = search_distro.group(1)
-
- if not machines:
- sys.exit("Failed to extract MACHINES from jobs names")
- if not pipelines_and_distros:
- sys.exit("Failed to extract DISTRO from jobs logs")
-
- summary = ""
-
- for pipeline, distro in pipelines_and_distros.items():
- for machine in sorted(machines):
- summary += f"**`⮘ {distro} | {machine} ⮚`**\\\n"
-
- deploy_img_dir = f"build-{distro}-{machine}/tmp/deploy/images/{machine}/"
-
- artifacts__main = BuildArtifacts(
- manifest_project, build__main.get_jobs(pipeline, f"build-{machine}")[0]
- )
- artifacts__mr = BuildArtifacts(
- manifest_project, build__mr.get_jobs(pipeline, f"build-{machine}")[0]
- )
-
- deploy_files__main = artifacts__main.list_dir(deploy_img_dir)
- deploy_files__mr = artifacts__mr.list_dir(deploy_img_dir)
-
- # The difference in size of artifacts.zip for main and MR builds
- zip_size__main = artifacts__main.get_archive_size()
- zip_size__mr = artifacts__mr.get_archive_size()
-
- zip_size_diff = sizeof_fmt(abs(zip_size__main - zip_size__mr))
- sign = "+" if zip_size__main < zip_size__mr else "-"
-
- zip_size__main = sizeof_fmt(zip_size__main)
- zip_size__mr = sizeof_fmt(zip_size__mr)
-
- summary += f" ├── artifacts.zip size: [ {zip_size__main} → {zip_size__mr} ] | {sign}{zip_size_diff}\\\n"
-
- # The difference in size of image for main and MR build
- image_size__main = 0
- image_size__mr = 0
-
- # The image file format may vary depending on machine and distribution.
- if distro == "seconorth-fngsystem":
- img_pattern = "*.rootfs.cpio.gz"
- else:
- img_pattern = "*.rootfs.tar.gz"
- if "genio" in machine:
- img_pattern = "*.rootfs.wic.img"
-
- for name, size in deploy_files__main.items():
- if fnmatch.fnmatch(name, img_pattern):
- image_size__main = size
- for name, size in deploy_files__mr.items():
- if fnmatch.fnmatch(name, img_pattern):
- image_size__mr = size
- summary += (
- f" ├── image size: [ {image_size__main} → {image_size__mr} ]\\\n"
- )
-
- # Comparison of manifest files for main in MR builds
- manifestfile_lines__main = []
- manifestfile_lines__mr = []
-
- for file in deploy_files__main.keys():
- if fnmatch.fnmatch(file, "*.rootfs.manifest"):
- manifestfile_lines__main = (
- artifacts__main.get_artifact(deploy_img_dir + file)
- .decode("utf-8")
- .splitlines()
- )
- diff_fromfile = file
-
- for file in deploy_files__mr.keys():
- if fnmatch.fnmatch(file, "*.rootfs.manifest"):
- manifestfile_lines__mr = (
- artifacts__mr.get_artifact(deploy_img_dir + file)
- .decode("utf-8")
- .splitlines()
- )
- diff_tofile = file
-
- if not manifestfile_lines__main or not manifestfile_lines__mr:
- continue
-
- summary += " └── manifest diff:\n"
- summary += "```diff\n"
- kernel_diff_lines = 0
- kernel_rev_old = ""
- kernel_rev_new = ""
- for l in unified_diff(
- manifestfile_lines__main,
- manifestfile_lines__mr,
- fromfile=diff_fromfile,
- tofile=diff_tofile,
- n=0,
- lineterm="",
- ):
- if fnmatch.fnmatch(l, "@@ * @@"):
- continue
+ report_main = BuildArtifactReport(manifest_project, args.target_branch)
+ report_mr = BuildArtifactReport(manifest_project, args.source_branch)
+ summary = report_main.compare_and_summarize(report_mr)
- if fnmatch.fnmatch(l, "?kernel-*"):
- kernel_diff_lines += 1
- if l[0] == "-":
- kernel_rev_old = l[-10:]
- if l[0] == "+":
- kernel_rev_new = l[-10:]
- else:
- summary += l + "\n"
- summary += "```\n"
- if kernel_diff_lines:
- summary += f"âš The kernel was updated from `{kernel_rev_old}` to `{kernel_rev_new}` commit.\n"
- summary += f"{kernel_diff_lines} lines were removed from diff.\n"
-
- summary += "\n"
-
- print(summary)
+ logging.info(summary)
mr.notes.create({"body": f"{summary}"})