[BUILD] Define job to filter out unnecessary jobs from build pipeline

* Introduce new job after the one generating build pipeline to
filter out jobs from the generated 'build-pipeline.yml'
file in case the build runs on a MR integration branch.
* The automation script parses the .yml file and remove not necessary
jobs. For instance, if a MR is opened in the 'meta-seco-imx' branch
the filter excludes jobs regarding boards with CPU from other silicon
vendors.
* Introduce 'PROCESSOR' variable in job names in order to easy the
filter out process.

build-pipeline-yocto.yml.jinja2

@@ -252,9 +252,9 @@ release:
           {% set DEFCONFIG = DEFCONFIG + backend %}
           {% set distro = distro + "-" + backend %}
 #----------------------------------------------------
-# {{ SECO_REMOTE }}-build-{{ machine }}-{{ distro }}
+# {{ SECO_REMOTE }}-build-{{PROCESSOR}}-{{ machine }}-{{ distro }}
 #----------------------------------------------------
-build-{{ machine }}-{{ distro }}:
+build-{{PROCESSOR}}-{{ machine }}-{{ distro }}:
   extends: .build
   variables: &build-{{ machine }}-{{ distro }}
      BOARD: {{ machine }}
@@ -274,23 +274,23 @@ build-{{ machine }}-{{ distro }}:
        {{ ca_namespace.CUSTOM_ARTIFACTS }}
      {% endif %}
 #----------------------------------------------------
-# {{ SECO_REMOTE }}-deploy-{{ machine }}-{{ distro }}
+# {{ SECO_REMOTE }}-deploy-{{PROCESSOR}}-{{ machine }}-{{ distro }}
 #----------------------------------------------------
-deploy-{{ machine }}-{{ distro }}:
+deploy-{{PROCESSOR}}-{{ machine }}-{{ distro }}:
   extends: .deploy
   needs:
-   - job: build-{{ machine }}-{{ distro }}
+   - job: build-{{PROCESSOR}}-{{ machine }}-{{ distro }}
      artifacts: true
   variables: &deploy-{{ machine }}-{{ distro }}
       <<: *build-{{ machine }}-{{ distro }}
           {% if testing is defined %}
 #----------------------------------------------------
-# {{ SECO_REMOTE }}-test-{{ machine }}-{{ distro }}
+# {{ SECO_REMOTE }}-test-{{PROCESSOR}}-{{ machine }}-{{ distro }}
 #----------------------------------------------------
-test-{{ machine }}-{{ distro }}:
+test-{{PROCESSOR}}-{{ machine }}-{{ distro }}:
   extends: .test
   needs:
-   - job: deploy-{{ machine }}-{{ distro }}
+   - job: deploy-{{PROCESSOR}}-{{ machine }}-{{ distro }}
      artifacts: true
   variables:
       <<: *build-{{ machine }}-{{ distro }}
@@ -299,9 +299,9 @@ test-{{ machine }}-{{ distro }}:
         {% endfor %}
       {% else %}
 #----------------------------------------------------
-# {{ SECO_REMOTE }}-build-{{ machine }}-{{ distro }}
+# {{ SECO_REMOTE }}-build-{{PROCESSOR}}-{{ machine }}-{{ distro }}
 #----------------------------------------------------
-build-{{ machine }}-{{ distro }}:
+build-{{PROCESSOR}}-{{ machine }}-{{ distro }}:
   extends: .build
   variables: &build-{{ machine }}-{{ distro }}
      BOARD: {{ machine }}
@@ -321,23 +321,23 @@ build-{{ machine }}-{{ distro }}:
        {{ ca_namespace.CUSTOM_ARTIFACTS }}
      {% endif %}
 #----------------------------------------------------
-# {{ SECO_REMOTE }}-deploy-{{ machine }}-{{ distro }}
+# {{ SECO_REMOTE }}-deploy-{{PROCESSOR}}-{{ machine }}-{{ distro }}
 #----------------------------------------------------
-deploy-{{ machine }}-{{ distro }}:
+deploy-{{PROCESSOR}}-{{ machine }}-{{ distro }}:
   extends: .deploy
   needs:
-   - job: build-{{ machine }}-{{ distro }}
+   - job: build-{{PROCESSOR}}-{{ machine }}-{{ distro }}
      artifacts: true
   variables: &deploy-{{ machine }}-{{ distro }}
       <<: *build-{{ machine }}-{{ distro }}
       {% if machine == 'a62-1G-4x256M' or machine == 'c20' or machine == 'd23' or machine == 'intel' %}
 #----------------------------------------------------
-# {{ SECO_REMOTE }}-test-{{ machine }}-{{ distro }}
+# {{ SECO_REMOTE }}-test-{{PROCESSOR}}-{{ machine }}-{{ distro }}
 #----------------------------------------------------
-test-{{ machine }}-{{ distro }}:
+test-{{PROCESSOR}}-{{ machine }}-{{ distro }}:
   extends: .test-things
   needs:
-   - job: deploy-{{ machine }}-{{ distro }}
+   - job: deploy-{{PROCESSOR}}-{{ machine }}-{{ distro }}
      artifacts: true
   variables:
       <<: *build-{{ machine }}-{{ distro }}

manifest-pipeline.yml

@@ -83,17 +83,31 @@ generate-build-pipeline:
     paths:
       - build-pipeline.yml
 
+filter-build-pipeline:
+  extends:
+    - .infrastructure
+    - .full_build_pipeline
+  stage: manifest-pipeline
+  needs: ["generate-build-pipeline"]
+  script:
+    - echo "Filtering jobs from generated build-pipeline.yml"
+    - .gitlab-ci/scripts/filter_jobs.py --file-path build-pipeline.yml
+  artifacts:
+    expire_in: 4 weeks
+    paths:
+      - build-pipeline.yml
+
 .build-pipeline:
   extends:
     - .full_build_pipeline
   stage: trigger
-  needs: ["generate-build-pipeline"]
+  needs: ["filter-build-pipeline"]
   variables:
     MANUAL_BUILD: $MANUAL_BUILD
   trigger:
     include:
       - artifact: build-pipeline.yml
-        job: generate-build-pipeline
+        job: filter-build-pipeline
     strategy: depend
 
 yamllint:

scripts/filter_jobs.py

+#!/usr/bin/env python3
+import os
+import argparse
+
+# GitLab environment variables
+CI_COMMIT_REF_NAME = os.getenv("CI_COMMIT_REF_NAME")
+
+# Filters lists
+CPU_lists = ['imx', 'rk', 'px', 'intel', 'raspberrypi', 'qc', 'genio']
+DISTRO_lists = ['embedded', 'things']
+
+def filter_jobs(file_path, keywords):
+
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+
+    filtered_lines = []
+    skip_mode = False
+
+    for line in lines:
+        # Start of a job block if the line contains a colon and does not start with #
+        if line.startswith((' ', '\t')) == False and ":" in line and not line.strip().startswith("#") and not line.strip().startswith("-"):
+            job_name = line.split(":")[0].strip()
+            skip_mode = any(keyword in job_name for keyword in keywords)
+
+        # If skip_mode is active, skip lines until the next comment block
+        if skip_mode:
+            if line.strip().startswith("#"):  # Reset skip mode at the next comment block
+                skip_mode = False
+                filtered_lines.append(line)  # Keep this comment line
+        else:
+            filtered_lines.append(line)  # Keep lines when not skipping
+
+    # Write the modified content back to the file
+    with open(file_path, 'w') as file:
+        file.writelines(filtered_lines)
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--file-path",
+        help="""The input file, from which there is necessity
+                to filter out some jobs that do not need to be run""",
+        dest="file_path",
+        required=True,
+    )
+
+    # parse known args
+    args, unknown = parser.parse_known_args()
+
+    if "integrate/meta-seco-imx" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for NXP boards
+        CPU_lists.remove("imx")
+        print("Removing all jobs except those regarding NXP devices")
+        filter_jobs(args.file_path, CPU_lists)
+
+    elif "integrate/meta-seco-rk" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for Rockchip boards
+        CPU_lists.remove("rk")
+        CPU_lists.remove("px")
+        print("Removing all jobs except those regarding Rockchip devices")
+        filter_jobs(args.file_path, CPU_lists)
+
+    elif "integrate/meta-seco-intel" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for Intel boards
+        CPU_lists.remove("intel")
+        print("Removing all jobs except those regarding Intel devices")
+        filter_jobs(args.file_path, CPU_lists)
+
+    elif "integrate/meta-seco-rpi" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for Raspberry boards
+        CPU_lists.remove("raspberrypi")
+        print("Removing all jobs except those regarding Raspberry Pi devices")
+        filter_jobs(args.file_path, CPU_lists)
+    
+    elif "integrate/meta-seco-mtk" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for MediaTek boards
+        CPU_lists.remove("genio")
+        print("Removing all jobs except those regarding MediaTek devices")
+        filter_jobs(args.file_path, CPU_lists)
+
+    elif "integrate/meta-seco-qcom" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for Qualcomm boards
+        CPU_lists.remove("qc")
+        print("Removing all jobs except those regarding Qualcomm devices")
+        filter_jobs(args.file_path, CPU_lists)
+
+    elif "integrate/meta-seco-clea-os-things" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for Clea OS Things distro
+        DISTRO_lists.remove("things")
+        print("Removing all jobs except those regarding Clea OS Things distro")
+        filter_jobs(args.file_path, DISTRO_lists)
+
+    elif "integrate/meta-seco-clea-os-embedded" in CI_COMMIT_REF_NAME:
+        # Filter and trigger build jobs for Clea OS Embedded distro
+        DISTRO_lists.remove("embedded")
+        print("Removing all jobs except those regarding Clea OS Embedded distro")
+        filter_jobs(args.file_path, DISTRO_lists)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file