[TEST] Include integration jobs for ci-test repos

Up to now, the ci-test group repositories were used for sporadic
tests but in the optic of a continuous CI/CD improvement they need
to be treated as 'production' repositories. With this approach,
every time a MR is created in the gitlab-ci repo, the modifications
introduced can be tested in the ci-test group repositories.

.gitlab-ci.yml

@@ -149,15 +149,15 @@ executable:
     MANIFEST_BRANCH: ${CUSTOM_5_BRANCH}
 # Jobs
 
-#integrate-ci-test:primary:
-#  extends: .integrate-ci-test
-#  variables:
-#    MANIFEST_BRANCH: primary
+integrate-ci-test:primary:
+  extends: .integrate-ci-test
+  variables:
+    MANIFEST_BRANCH: primary
 
-#integrate-ci-test:secondary:
-#  extends: .integrate-ci-test
-#  variables:
-#    MANIFEST_BRANCH: secondary
+integrate-ci-test:secondary:
+  extends: .integrate-ci-test
+  variables:
+    MANIFEST_BRANCH: secondary
 
 #integrate-yocto:dunfell:
 #  extends: .integrate-yocto
@@ -256,17 +256,17 @@ integrate-yocto:custom-5:
     strategy: depend
 # Jobs
 
-#build-ci-test:primary:
-#  extends: .build-ci-test
-#  needs: ["integrate-ci-test:primary"]
-#  variables:
-#    MANIFEST_BRANCH: primary
+build-ci-test:primary:
+  extends: .build-ci-test
+  needs: ["integrate-ci-test:primary"]
+  variables:
+    MANIFEST_BRANCH: primary
 
-#build-ci-test:secondary:
-#  extends: .build-ci-test
-#  needs: ["integrate-ci-test:secondary"]
-#  variables:
-#    MANIFEST_BRANCH: secondary
+build-ci-test:secondary:
+  extends: .build-ci-test
+  needs: ["integrate-ci-test:secondary"]
+  variables:
+    MANIFEST_BRANCH: secondary
 
 #build-yocto:dunfell:
 #  extends: .build-yocto
@@ -321,12 +321,12 @@ build-yocto:custom-5:
   variables:
     MERGE: --merge --project=${CI_PROJECT_PATH} --branch=${CI_COMMIT_REF_NAME}
 
-#merge-ci-test:
-#  extends:
-#    - .integrate-ci-test
-#    - .merge
-#  variables:
-#    MANIFEST_BRANCH: primary,secondary
+merge-ci-test:
+  extends:
+    - .integrate-ci-test
+    - .merge
+  variables:
+    MANIFEST_BRANCH: primary,secondary
 
 merge-yocto:
   extends:

build-pipeline-ci-test.yml

+---
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+include:
+  - project: '${CI_PROJECT_ROOT_NAMESPACE}/infrastructure/gitlab-ci'
+    ref: ${GITLAB_CI_REVISION}
+    file:
+      - common.yml
+
+stages:
+  - Build
+  - Test
+
+# --------------------------------------------------------------------------------------
+# Stage: Build
+# --------------------------------------------------------------------------------------
+
+build:echo:
+  stage: Build
+  tags:
+    - infrastructure
+  needs: []
+  timeout: 2m
+  image: ${CI_IMAGE_PYTHON}
+  script:
+    - printenv
+    - echo "Build successful"
+
+# --------------------------------------------------------------------------------------
+# Stage: Test
+# --------------------------------------------------------------------------------------
+.test_simple:
+  stage: Test
+  rules:
+    - when: manual
+      allow_failure: true
+  tags:
+    - infrastructure
+  image: ${CI_IMAGE_PYTHON}
+  script:
+    - exit ${RETURNCODE}

manifest-integration-pipelines.yml.jinja2

@@ -3,7 +3,7 @@
 # Global
 # --------------------------------------------------------------------------------------
 include:
-  - project: edgehog/infrastructure/gitlab-ci
+  - project: {{ CI_PROJECT_ROOT_NAMESPACE }}/infrastructure/gitlab-ci
     ref: {{ GITLAB_CI_REVISION }}
     file: common.yml
 
@@ -14,42 +14,6 @@ workflow:
 stages:
   - manifest-integration-pipelines
 
-# --------------------------------------------------------------------------------------
-# Generate job
-# --------------------------------------------------------------------------------------
-# Use one single job to generate multiple yaml files for the downstream pipelines.
-# FIXME: This is only necessary due to a GitLab limitation:
-# https://gitlab.com/gitlab-org/gitlab/-/issues/347469
-# We work around this by generating manifest-integration-jobs.yml from a Jinja2 template
-# and insert the trigger project via a Jinja2 variable.
-# The issue is already fixed and will be released in GitLab 15.3:
-# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92346
-# As soon as we update to this version, we can get rid of the generate job and convert
-# the Jinja2 template to a simple YAML file.
-generate:
-  extends:
-    - .infrastructure
-  stage: manifest-integration-pipelines
-  script:
-    # The job generation script implicitly passes the OS environment to the template, so
-    # that the template has access to all GitLab CI variables. Hence there is no need
-    # to explicitly pass any of them as command line arguments.
-{% for integration in INTEGRATION.split('\n') %}
-{% set SOURCE_BRANCH, TARGET_PROJECT, TARGET_BRANCH = integration.split(':') %}
-{% if (CI_MERGE_REQUEST_TARGET_BRANCH_NAME is defined
-      and SOURCE_BRANCH == CI_MERGE_REQUEST_TARGET_BRANCH_NAME)
-      or SOURCE_BRANCH == CI_COMMIT_REF_NAME %}
-    - TARGET_PROJECT={{ TARGET_PROJECT }}
-        .gitlab-ci/scripts/generate_job_from_template.py
-        --template=.gitlab-ci/manifest-integration-jobs.yml.jinja2
-        > manifest-integration-jobs-{{ loop.index }}.yml
-{% endif %}
-{% endfor %}
-  artifacts:
-    expire_in: 4 weeks
-    paths:
-      - manifest-integration-jobs-*.yml
-
 # --------------------------------------------------------------------------------------
 # Trigger jobs
 # --------------------------------------------------------------------------------------
@@ -61,8 +25,6 @@ generate:
 
 {{ TARGET_PROJECT }}:{{ TARGET_BRANCH }}:
   stage: manifest-integration-pipelines
-  needs:
-    - generate
   variables:
     SOURCE_BRANCH: {{ SOURCE_BRANCH }}
     TARGET_PROJECT: {{ TARGET_PROJECT }}
@@ -72,12 +34,9 @@ generate:
     BB_RECIPE_NAME: {{ BB_RECIPE_NAME }}
   trigger:
     include:
-      # FIXME: Use these settings after switching from jinja2 to yaml (see above)
-      # project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci'
-      # ref: ${GITLAB_CI_REVISION}
-      # file: manifest-integration-jobs.yml
-      artifact: manifest-integration-jobs-{{ loop.index }}.yml
-      job: generate
+      project: '${CI_PROJECT_ROOT_NAMESPACE}/infrastructure/gitlab-ci'
+      ref: ${GITLAB_CI_REVISION}
+      file: manifest-integration-jobs.yml
     strategy: depend
 
 {% endif %}

manifest-pipeline-ci-test.yml

@@ -11,16 +11,31 @@ variables:
   # In the manifest, the remotes are specified by an identifier. This is used to find
   # out included projects for the retrigger job. In custom manifests, the remote may be
   # named differently, so we need a variable that may be overriden.
-  CI_PARAM_SECO_REMOTE: ci-test
+  SECO_REMOTE: ci-test
 
   # GitLab group to search for projects to retrigger
-  RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
+  RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/infrastructure/ci-test
 
   BUILD_TIMEOUT: 2m
 
-  # This is the jinja2 template file used to generate the build jobs
-  BUILD_JOBS_TEMPLATE: build-jobs-ci-test.yml.jinja2
-
   # The master branch is hardcoded here, because it cannot be determined automatically.
   # Has to be modified for new branches, e.g. new Yocto versions or fix releases.
   MASTER_BRANCH: master
+
+  # Projects to include in the changelog in addition to the manifest project
+  CHANGELOG_PROJECTS:
+    edgehog/infrastructure/ci-test/minimal-bsp
+    edgehog/infrastructure/ci-test/minimal-meta-seco
+
+  # List of machines to simulate building images for
+  MACHINES: c12 c20
+
+ci-test-pipeline:
+  stage: trigger
+  needs: []
+  trigger:
+    include:
+      - project: '${CI_PROJECT_ROOT_NAMESPACE}/infrastructure/gitlab-ci'
+        ref: ${GITLAB_CI_REVISION}
+        file: build-pipeline-ci-test.yml
+    strategy: depend
\ No newline at end of file

manifest-pipeline.yml

@@ -7,6 +7,7 @@ include:
 
 stages:
   - manifest-pipeline
+  - trigger
   - retrigger
   - build
 
@@ -16,9 +17,6 @@ workflow:
     - if: $CI_PIPELINE_SOURCE == "api"
     - if: $CI_PIPELINE_SOURCE == "pipeline"
     - if: $CI_PIPELINE_SOURCE == "web"
-    # Do not run pipelines for merge requests
-    - if: $CI_MERGE_REQUEST_IID
-      when: never
     # Do not run pipelines on forked projects
     # (use id instead of name because of rename)
     - if: $CI_PROJECT_ID != $MANIFEST_PROJECT_ID
@@ -26,18 +24,25 @@ workflow:
     # Do not run pipelines on integration branches
     - if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
       when: never
-    # In all other cases, run the pipeline automatically
-    - when: always
+    # Do not run pipeline if the "skip build" label is set on the merge request
+    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
+      when: never
+    # Run pipelines for merge requests
+    - if: $CI_MERGE_REQUEST_IID
+    # Run pipelines on the master branch
+    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH
+    # Run pipelines on tags
+    - if: $CI_COMMIT_TAG
 
 .full_build_pipeline:
   rules:
-    # Run the full build pipeline on non-master branches (i.e. in the merge request)
+    # Run the full build pipeline on merge requests and tags
     # or if explicitly triggered by the API or the web button.
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH
+    - if: $CI_MERGE_REQUEST_IID
+    - if: $CI_COMMIT_TAG
     - if: $CI_PIPELINE_SOURCE == "api"
     - if: $CI_PIPELINE_SOURCE == "pipeline"
     - if: $CI_PIPELINE_SOURCE == "web"
-  stage: manifest-pipeline
 
 .short_master_pipeline:
   rules:
@@ -55,36 +60,43 @@ workflow:
 # --------------------------------------------------------------------------------------
 # Full build pipeline (runs in merge requests, and on master if manually triggered)
 # --------------------------------------------------------------------------------------
-generate-build-jobs:
+generate-build-pipeline:
   extends:
     - .infrastructure
     - .full_build_pipeline
+  stage: manifest-pipeline
+  variables:
+    BUILD_PIPELINE_TEMPLATE: build-pipeline-yocto.yml.jinja2
   script:
-    # The job generation script implicitly passes the OS environment to the template, so
+    - echo "Generating build pipeline from template file '${BUILD_PIPELINE_TEMPLATE}'"
+    # The Jinja2 render script implicitly passes the OS environment to the template, so
     # that the template has access to all GitLab CI variables. Hence there is no need
     # to explicitly pass any of them as command line arguments.
-    - .gitlab-ci/scripts/generate_job_from_template.py
-              --template=.gitlab-ci/${BUILD_JOBS_TEMPLATE}
-              > build-jobs.yml
+    - .gitlab-ci/scripts/render_jinja2_template.py
+              --template=.gitlab-ci/${BUILD_PIPELINE_TEMPLATE}
+              --format=yaml
+              > build-pipeline.yml
   artifacts:
     expire_in: 4 weeks
     paths:
-      - build-jobs.yml
+      - build-pipeline.yml
 
-build-jobs:
+.build-pipeline:
   extends:
     - .full_build_pipeline
-  needs: ["generate-build-jobs"]
+  stage: trigger
+  needs: ["generate-build-pipeline"]
   trigger:
     include:
-      - artifact: build-jobs.yml
-        job: generate-build-jobs
+      - artifact: build-pipeline.yml
+        job: generate-build-pipeline
     strategy: depend
 
 yamllint:
   extends:
     - .yamllint
     - .full_build_pipeline
+  stage: manifest-pipeline
 
 # --------------------------------------------------------------------------------------
 # Short master pipeline (runs on master after merging a merge request)
@@ -113,25 +125,9 @@ build:merge_request:
   timeout: !reference [variables, BUILD_TIMEOUT]
   script:
     - cd ${CI_PROJECT_DIR}
-    # Get pipeline for merge request
-    - MR_PIPELINE=$(.gitlab-ci/scripts/get_pipelines.py
+    - .gitlab-ci/scripts/mirror_mr_pipeline.py
         --gitlab-url=${CI_SERVER_URL}
         --token=${GITBOT_TOKEN}
         --project=${CI_PROJECT_PATH}
         --commit=${CI_COMMIT_SHA}
-        --ref=^${MASTER_BRANCH} || true | head -1)
-    # If pipeline exists, mirror its result
-    - if [ ! -z "${MR_PIPELINE}" ]; then
-        .gitlab-ci/scripts/mirror_pipeline_result.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${CI_PROJECT_PATH}
-          --pipeline=${MR_PIPELINE}
-    # If no pipeline found, trigger a new one on the master
-    - else
-        .gitlab-ci/scripts/trigger_pipeline.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${CI_PROJECT_PATH}
-          --ref=${MASTER_BRANCH}
-    - fi
+        --ref=${MASTER_BRANCH}