Confluence: improve variable passing to template

Use a general mechanism of storing all job variables for each build job
separately, so that we have all documentation files in the
FILES_documentation variable. This makes the dedicated HTML_FILES
variable obsolete.
The merged variables are still kept, though, so that we have all
machines in the MACHINE variable to loop over.

confluence-page.xml.jinja2

@@ -18,7 +18,7 @@
                 <tr>
                     <th><b>Architecture / Processor</b></th>
                     <td>
-                        {%- for machine in MACHINE.split('\n') | sort %}
+                        {%- for machine in MACHINE.split(' ') | sort %}
                         <p>{{ machine }}</p>
                         {%- endfor %}
                     </td>
@@ -36,8 +36,10 @@
                 <tr>
                     <th><b>Documentation</b></th>
                     <td>
-                        {%- for html_file in HTML_FILES.split('\n') | sort %}
-                        <p><a href="{{ html_file }}">{{ (html_file | basename | splitext)[0] }}</a></p>
+                        {%- for doc_file in FILES_documentation.split(' ') | sort %}
+                        {%- if doc_file.endswith(".html") %}
+                        <p><a href="{{ doc_file }}">{{ (doc_file | basename | splitext)[0] }}</a></p>
+                        {%- endif %}
                         {%- endfor %}
                     </td>
                 </tr>
@@ -46,12 +48,12 @@
     </ac:rich-text-body>
 </ac:structured-macro>
 
-{%- for machine in MACHINE.split('\n') | sort %}
+{%- for machine in MACHINE.split(' ') | sort %}
 <h1>{{ machine }}</h1>
 <ul>
     {#- see https://stackoverflow.com/questions/72654161/dynamically-referencing-a-variable-in-jinja2 #}
     {%- set FILES_machine = self._TemplateReference__context.resolve('FILES_' ~ machine | replace("-", "_")) -%}
-    {%- for machine_file in FILES_machine.split('\n') | sort %}
+    {%- for machine_file in FILES_machine.split(' ') | sort %}
     <li>
         <p><a href="{{ machine_file }}">{{ (machine_file | basename) }}</a></p>
     </li>

scripts/collect_release_information.py

 #!/usr/bin/env python3
 
 import argparse
-import fnmatch
 import sys
 import tempfile
 
@@ -15,16 +14,14 @@ from get_pipeline_jobs import get_pipeline_jobs
 
 
 def get_job_env_variables(
-    gitlab: Gitlab, project: Project, jobs: list[ProjectPipelineJob], env_file: str
+    gitlab: Gitlab, project: Project, job: ProjectPipelineJob, env_file: str
 ):
-    """Get a list of variables from the env_file files from the given jobs.
-       If a variable is set to different values in different jobs, the values are
-       concatenated and newline-separated.
+    """Get a list of variables from the env_file of the given job.
 
     Args:
         gitlab: Gitlab object
         project: Gitlab project
-        jobs: list of jobs
+        job: GitLab job
         env_file: name of the GitLab dotenv file to read variables from
 
     Returns:
@@ -32,37 +29,42 @@ def get_job_env_variables(
     """
     variables = {}
 
-    for job in jobs:
-
-        with tempfile.NamedTemporaryFile() as target_file:
-            print(f"Downloading file {env_file} from job {job.name}", file=sys.stderr)
-            try:
-                download_job_artifact(
-                    gitlab,
-                    target_file.name,
-                    env_file,
-                    job.id,
-                    project=project,
-                )
-            except GitlabGetError:
-                print(
-                    f"Artifact '{env_file}' not found in {job.web_url}", file=sys.stderr
-                )
-
-            lines = [line.decode("utf-8").rstrip() for line in target_file.readlines()]
-
-            for line in lines:
-                name, value = line.partition("=")[::2]
-                value = value.strip("'").strip('"')
-                variables[name] = (
-                    f"{variables[name]}\n{value}"
-                    if name in variables and variables[name] != value
-                    else value
-                )
+    with tempfile.NamedTemporaryFile() as target_file:
+        print(f"Downloading file {env_file} from job {job.name}", file=sys.stderr)
+        try:
+            download_job_artifact(
+                gitlab,
+                target_file.name,
+                env_file,
+                job.id,
+                project=project,
+            )
+        except GitlabGetError:
+            print(f"Artifact '{env_file}' not found in {job.web_url}", file=sys.stderr)
+
+        lines = [line.decode("utf-8").rstrip() for line in target_file.readlines()]
+
+        for line in lines:
+            name, value = line.partition("=")[::2]
+            value = value.strip("'\" ")
+            variables[name] = value
 
     return variables
 
 
+def add_suffix_to_dict_keys(dictionary: dict, suffix: str):
+    """Add a suffix to every key of a dictionary.
+
+    Args:
+        dictionary: the dictionary
+        suffix: the suffix to add
+
+    Returns:
+        Dictionary with suffixed keys
+    """
+    return {f"{key}{suffix}": dictionary[key] for key in dictionary}
+
+
 def main():
     parser = argparse.ArgumentParser(description=__doc__, usage="%(prog)s [OPTIONS]")
 
@@ -106,38 +108,50 @@ def main():
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
     project = common.get_project(gitlab, args.project)
 
-    jobs = get_pipeline_jobs(gitlab, project, args.pipeline, stage="Build")
-    variables = get_job_env_variables(gitlab, project, jobs, "build.env")
-
-    jobs = get_pipeline_jobs(gitlab, project, args.pipeline, stage=args.deploy_stage)
-    successful_jobs = [job for job in jobs if job.status == "success"]
-    if not successful_jobs:
-        exit(f"ERROR: no successful jobs found in stage '{args.deploy_stage}'")
-
-    variables |= get_job_env_variables(gitlab, project, successful_jobs, "deploy.env")
-
-    files = variables["FILES"].replace("\n", " ").split(" ")
-
-    html_files = set(fnmatch.filter(files, "*.html"))
+    variables = {}
 
-    for html_file in html_files:
-        variables["HTML_FILES"] = (
-            f"{variables['HTML_FILES']}\n{html_file}"
-            if "HTML_FILES" in variables
-            else html_file
+    job_specs = [
+        {
+            "stage": "Build",
+            "env_file": "build.env",
+        },
+        {
+            "stage": args.deploy_stage,
+            "env_file": "deploy.env",
+        },
+    ]
+
+    for job_spec in job_specs:
+        # Get all successful jobs from stage
+        jobs = get_pipeline_jobs(
+            gitlab, project, args.pipeline, stage=job_spec["stage"]
         )
-
-    for machine in variables["MACHINE"].split("\n"):
-        machine_files = fnmatch.filter(files, f"**/{machine}/*")
-        machine_files_variable = f"FILES_{machine.replace('-', '_')}"
-        for machine_file in machine_files:
-            variables[machine_files_variable] = (
-                f"{variables[machine_files_variable]}\n{machine_file}"
-                if machine_files_variable in variables
-                else machine_file
+        successful_jobs = [job for job in jobs if job.status == "success"]
+        if not successful_jobs:
+            exit(f"ERROR: no successful jobs found in stage '{args.deploy_stage}'")
+
+        for job in successful_jobs:
+            # Get variables from each job and prefix them with whatever comes after the
+            # first "-" of the job name.
+            job_env_variables = get_job_env_variables(
+                gitlab, project, job, job_spec["env_file"]
             )
-
-    for k, v in variables.items():
+            suffix = f"{job.name[job.name.find('-'):].replace('-', '_')}"
+            variables |= add_suffix_to_dict_keys(job_env_variables, suffix)
+
+            # Furthermore merge all machine-specific values into one space-separated
+            # variable.
+            for name in job_env_variables:
+                if job_env_variables[name]:
+                    variables[name] = (
+                        f"{variables[name]} {job_env_variables[name]}"
+                        if name in variables
+                        and variables[name] != job_env_variables[name]
+                        else job_env_variables[name]
+                    )
+
+    # Print all variables in a shell compatible form
+    for k, v in sorted(variables.items()):
         print(f'{k}="{v}"')