report-image-diff: Rewrite the report image diff to use buildhistory

The old way used the gitlab UI to load the html overview for the
artefacts, but this didn't worked for private repos, as the login
was not implemented. Loading file from the buildhistory with known
names uses the gitlab API, where the credential handling is provided.

Also replace loops and duplicated code with internal classes, which
massively restructures the code.

scripts/report_image_diff.py

@@ -2,11 +2,15 @@
 import argparse
 import fnmatch
 import logging
+import os
 import re
 import sys
 from difflib import unified_diff
+from platform import machine
 
 from gitlab import Gitlab
+from gitlab import GitlabGetError
+from gitlab.v4.objects import Project
 
 import common
 from buildartifacts import BuildArtifacts
@@ -36,6 +40,210 @@ def sizeof_fmt(num: int, p: int = 2) -> str:
     return f"{num:.{p}f} YiB"
 
 
+class BuildJobArtifactReport:
+    def __init__(self, project: Project, parent_pipeline, _pipeline, _machine, _distro):
+        self.parent_project = project
+        self.parent_pipeline = parent_pipeline
+        self.pipeline = _pipeline
+        self.machine = _machine
+        self.distro = _distro
+
+        self.__installed_packages = None
+        self.__installed_package_info = None
+        self.__installed_package_size = None
+
+        self.artifacts = BuildArtifacts(
+            self.parent_project,
+            self.parent_pipeline.get_jobs(self.pipeline, f"build-{self.machine}")[0],
+        )
+
+    @property
+    def image_name(self):
+        if "YOCTO_IMAGE" in self.artifacts.buildenv.keys():
+            return self.artifacts.buildenv["YOCTO_IMAGE"]
+        if "fngsystem" in self.distro:
+            return "fngsystem-image"
+        return "seconorth-image"
+
+    @property
+    def build_dir(self):
+        return f"build-{self.distro}-{self.machine}"
+
+    @property
+    def buildhistory_dir(self):
+        return os.path.join(
+            self.build_dir,
+            "buildhistory",
+            "images",
+            self.machine.replace("-", "_"),
+            "glibc",
+            self.image_name,
+        )
+
+    def __download_buildhistory(self, filename):
+        file = os.path.join(self.buildhistory_dir, filename)
+        try:
+            data = self.artifacts.get_artifact(file)
+        except GitlabGetError as e:
+            logging.error(f"Failed to download {file}: {e}.")
+            return None
+        return data.decode().splitlines()
+
+    @property
+    def installed_packages(self):
+        if self.__installed_packages is None:
+            self.__installed_packages = self.__download_buildhistory(
+                "installed-packages.txt"
+            )
+        return self.__installed_packages
+
+    @property
+    def installed_package_info(self):
+        if self.__installed_package_info is None:
+            self.__installed_package_info = self.__download_buildhistory(
+                "installed-package-info.txt"
+            )
+        return self.__installed_package_info
+
+    @property
+    def installed_package_size(self):
+        if self.__installed_package_size is None:
+            self.__installed_package_size = self.__download_buildhistory(
+                "installed-package-sizes.txt"
+            )
+        return self.__installed_package_size
+
+    @property
+    def archive_size(self):
+        return self.artifacts.get_archive_size()
+
+    @property
+    def image_size(self):
+        total = 0
+        for line in self.installed_package_size:
+            size, unit, name = line.split()
+            if unit == "MiB":
+                factor = 1024 * 1024
+            elif unit == "KiB":
+                factor = 1024
+            else:
+                factor = 1
+            total += int(size) * factor
+        return total
+
+    def compare_and_summarize(self, other: "BuildJobArtifactReport") -> str:
+        if self.distro != other.distro or self.machine != other.machine:
+            raise Exception("Comparing apples with pears.")
+
+        logging.debug(f"Create report for {self.machine} {self.distro}.")
+
+        # The difference in size of artifacts.zip for main and MR builds
+        zip_size_diff = sizeof_fmt(abs(self.archive_size - other.archive_size))
+        sign = "+" if self.archive_size < other.archive_size else "-"
+        summary = f"**`⮘ {self.distro} | {self.machine} ⮚`**\\\n"
+        summary += f"    ├── artifacts.zip size: [ {sizeof_fmt(self.archive_size)} → {sizeof_fmt(other.archive_size)} ] | {sign}{zip_size_diff}\\\n"
+        summary += f"    ├── image size: [ {sizeof_fmt(self.image_size)} → {sizeof_fmt(other.image_size)} ]\\\n"
+        summary += "    └── installed packages diff:\n"
+        summary += "```diff\n"
+
+        for line in unified_diff(
+            self.installed_packages,
+            other.installed_packages,
+            n=0,
+            lineterm="",
+        ):
+            if fnmatch.fnmatch(line, "@@ * @@"):
+                continue
+            if fnmatch.fnmatch(line, "--- *"):
+                continue
+            if fnmatch.fnmatch(line, "+++ *"):
+                continue
+
+            summary += line + "\n"
+        summary += "```\n"
+        summary += "\n"
+        return summary
+
+
+class BuildArtifactReport:
+    def __init__(self, project: Project, refname: str):
+        self.project = project
+        self.refname = refname
+        self.commit = self.project.commits.list(
+            all=False, ref_name=self.refname, order_by="id", sort="desc"
+        )[0]
+        self.pipeline = FullBuildPipeline(
+            self.project, self.commit.id, ignore_pipeine_status=True
+        )
+
+        self.machines, self.pipelines_and_distros = self.__find_distro_and_machines()
+
+        if not self.machines:
+            raise FileNotFoundError("Failed to extract MACHINES from jobs names")
+        if not self.pipelines_and_distros:
+            raise FileNotFoundError("Failed to extract DISTRO from jobs logs")
+
+        self.buildPipelineArtifacts = self.__get_build_pipelines()
+
+    def __find_distro_and_machines(self):
+        machines = []
+        pipelines_and_distros = {}
+        pattern_distro = r"DISTRO=([\w\d_.-]+)\n"
+        for pipeline, jobs in self.pipeline.build_pipelines.items():
+            for j in (j for j in jobs if j.stage == "Build" and j.status == "success"):
+
+                logging.debug(f"Found job named '{j.name}'.")
+
+                if j.name == "build-documentation":
+                    continue
+
+                m = j.name.removeprefix("build-")
+                if m not in machines:
+                    machines.append(m)
+                    logging.debug(f"Found machine named '{m}'.")
+
+                if pipeline in pipelines_and_distros.keys():
+                    continue
+
+                # Job methods (e.g. trace) are not available on ProjectPipelineJob objects.
+                # To use these methods create a ProjectJob object:
+                #     pipeline_job = pipeline.jobs.list()[0]
+                #     job = project.jobs.get(pipeline_job.id, lazy=True)
+                #     job.trace()
+                job_log = self.project.jobs.get(j.id, lazy=True).trace().decode("utf-8")
+
+                search_distro = re.search(pattern_distro, job_log)
+                if search_distro:
+                    pipelines_and_distros[pipeline] = search_distro.group(1)
+
+        return machines, pipelines_and_distros
+
+    def __get_build_pipelines(self):
+        buildpipeline = {}
+        for pipeline, distro in self.pipelines_and_distros.items():
+            buildpipeline[distro] = {}
+            for m in sorted(self.machines):
+                try:
+                    buildpipeline[distro][m] = BuildJobArtifactReport(
+                        self.project, self.pipeline, pipeline, m, distro
+                    )
+                except IndexError:
+                    logging.warning(
+                        f"Failed to find build for {m} in latest pipeline."
+                    )
+                    continue
+        return buildpipeline
+
+    def compare_and_summarize(self, other: "BuildArtifactReport") -> str:
+        summary = ""
+        for distro, pipelines in self.buildPipelineArtifacts.items():
+            for m in pipelines.keys():
+                summary += pipelines[m].compare_and_summarize(
+                    other.buildPipelineArtifacts[distro][m],
+                )
+        return summary
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -104,159 +312,11 @@ def main():
         if comment.body[0:4] == "**`⮘":
             comment.delete()
 
-    manifest_commit__main = manifest_project.commits.list(
-        all=False, ref_name=args.target_branch, order_by="id", sort="desc"
-    )[0]
-    build__main = FullBuildPipeline(manifest_project, manifest_commit__main.id)
-
-    manifest_commit__mr = manifest_project.commits.list(
-        all=False, ref_name=args.source_branch, order_by="id", sort="desc"
-    )[0]
-    build__mr = FullBuildPipeline(
-        manifest_project, manifest_commit__mr.id, ignore_pipeine_status=True
-    )
-
-    machines = []
-    pipelines_and_distros = {}
-    pattern_distro = r"DISTRO=([\w\d_.-]+)\n"
-    for pipeline, jobs in build__mr.build_pipelines.items():
-        for j in (j for j in jobs if j.stage == "Build" and j.status == "success"):
-
-            if j.name == "build-documentation":
-                continue
-
-            machine = j.name.removeprefix("build-")
-            if machine not in machines:
-                machines.append(machine)
-
-            if pipeline in pipelines_and_distros.keys():
-                continue
-
-            # Job methods (e.g. trace) are not available on ProjectPipelineJob objects.
-            # To use these methods create a ProjectJob object:
-            #     pipeline_job = pipeline.jobs.list()[0]
-            #     job = project.jobs.get(pipeline_job.id, lazy=True)
-            #     job.trace()
-            job_log = manifest_project.jobs.get(j.id, lazy=True).trace().decode("utf-8")
-
-            search_distro = re.search(pattern_distro, job_log)
-            if search_distro:
-                pipelines_and_distros[pipeline] = search_distro.group(1)
-
-    if not machines:
-        sys.exit("Failed to extract MACHINES from jobs names")
-    if not pipelines_and_distros:
-        sys.exit("Failed to extract DISTRO from jobs logs")
-
-    summary = ""
-
-    for pipeline, distro in pipelines_and_distros.items():
-        for machine in sorted(machines):
-            summary += f"**`⮘ {distro} | {machine} ⮚`**\\\n"
-
-            deploy_img_dir = f"build-{distro}-{machine}/tmp/deploy/images/{machine}/"
-
-            artifacts__main = BuildArtifacts(
-                manifest_project, build__main.get_jobs(pipeline, f"build-{machine}")[0]
-            )
-            artifacts__mr = BuildArtifacts(
-                manifest_project, build__mr.get_jobs(pipeline, f"build-{machine}")[0]
-            )
-
-            deploy_files__main = artifacts__main.list_dir(deploy_img_dir)
-            deploy_files__mr = artifacts__mr.list_dir(deploy_img_dir)
-
-            # The difference in size of artifacts.zip for main and MR builds
-            zip_size__main = artifacts__main.get_archive_size()
-            zip_size__mr = artifacts__mr.get_archive_size()
-
-            zip_size_diff = sizeof_fmt(abs(zip_size__main - zip_size__mr))
-            sign = "+" if zip_size__main < zip_size__mr else "-"
-
-            zip_size__main = sizeof_fmt(zip_size__main)
-            zip_size__mr = sizeof_fmt(zip_size__mr)
-
-            summary += f"    ├── artifacts.zip size: [ {zip_size__main} → {zip_size__mr} ] | {sign}{zip_size_diff}\\\n"
-
-            # The difference in size of image for main and MR build
-            image_size__main = 0
-            image_size__mr = 0
-
-            # The image file format may vary depending on machine and distribution.
-            if distro == "seconorth-fngsystem":
-                img_pattern = "*.rootfs.cpio.gz"
-            else:
-                img_pattern = "*.rootfs.tar.gz"
-                if "genio" in machine:
-                    img_pattern = "*.rootfs.wic.img"
-
-            for name, size in deploy_files__main.items():
-                if fnmatch.fnmatch(name, img_pattern):
-                    image_size__main = size
-            for name, size in deploy_files__mr.items():
-                if fnmatch.fnmatch(name, img_pattern):
-                    image_size__mr = size
-            summary += (
-                f"    ├── image size: [ {image_size__main} → {image_size__mr} ]\\\n"
-            )
-
-            # Comparison of manifest files for main in MR builds
-            manifestfile_lines__main = []
-            manifestfile_lines__mr = []
-
-            for file in deploy_files__main.keys():
-                if fnmatch.fnmatch(file, "*.rootfs.manifest"):
-                    manifestfile_lines__main = (
-                        artifacts__main.get_artifact(deploy_img_dir + file)
-                        .decode("utf-8")
-                        .splitlines()
-                    )
-                    diff_fromfile = file
-
-            for file in deploy_files__mr.keys():
-                if fnmatch.fnmatch(file, "*.rootfs.manifest"):
-                    manifestfile_lines__mr = (
-                        artifacts__mr.get_artifact(deploy_img_dir + file)
-                        .decode("utf-8")
-                        .splitlines()
-                    )
-                    diff_tofile = file
-
-            if not manifestfile_lines__main or not manifestfile_lines__mr:
-                continue
-
-            summary += "    └── manifest diff:\n"
-            summary += "```diff\n"
-            kernel_diff_lines = 0
-            kernel_rev_old = ""
-            kernel_rev_new = ""
-            for l in unified_diff(
-                manifestfile_lines__main,
-                manifestfile_lines__mr,
-                fromfile=diff_fromfile,
-                tofile=diff_tofile,
-                n=0,
-                lineterm="",
-            ):
-                if fnmatch.fnmatch(l, "@@ * @@"):
-                    continue
+    report_main = BuildArtifactReport(manifest_project, args.target_branch)
+    report_mr = BuildArtifactReport(manifest_project, args.source_branch)
+    summary = report_main.compare_and_summarize(report_mr)
 
-                if fnmatch.fnmatch(l, "?kernel-*"):
-                    kernel_diff_lines += 1
-                    if l[0] == "-":
-                        kernel_rev_old = l[-10:]
-                    if l[0] == "+":
-                        kernel_rev_new = l[-10:]
-                else:
-                    summary += l + "\n"
-            summary += "```\n"
-            if kernel_diff_lines:
-                summary += f"⚠ The kernel was updated from `{kernel_rev_old}` to `{kernel_rev_new}` commit.\n"
-                summary += f"{kernel_diff_lines} lines were removed from diff.\n"
-
-            summary += "\n"
-
-    print(summary)
+    logging.info(summary)
     mr.notes.create({"body": f"{summary}"})