Generate integration pipeline based on CI variable

Evaluate the INTEGRATION variable to determine target project and branch
for integration instead of using the hard-coded variables. For now this
works only for a single integration per target project because the
integration branch name does not include the target branch and hence
would be identical when trying to integrate into multiple branches of
the same project.

manifest-integration-jobs.yml.jinja2

+---
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+include:
+  # FIXME: see FIXME comments in manifest-integration-pipelines.yml.jinja2
+  # - project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci'
+  #   ref: ${GITLAB_CI_REVISION}
+  - project: {{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci
+    ref: {{ GITLAB_CI_REVISION }}
+    file: common.yml
+
+workflow:
+  rules:
+    - if: $CI_MERGE_REQUEST_IID
+
+stages:
+  - infrastructure
+  - integrate
+  - merge
+  - build
+  - check
+
+# --------------------------------------------------------------------------------------
+# Stage: infrastructure
+# --------------------------------------------------------------------------------------
+integrate:
+  extends: .infrastructure
+  rules:
+    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+    # The integration is done from the pipeline in gitlab-ci already
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    # We have to make sure that the pipeline runs for the current manifest
+    # master at the time a merge request is created. Otherwise we cannot
+    # guarantee a green master after merging.
+    - if: $CI_MERGE_REQUEST_IID
+  cache:
+    policy: push
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
+    - .gitlab-ci/scripts/integrate_into_manifest.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${MANIFEST_PROJECT}
+        --manifest-file=${MANIFEST_FILE}
+        --integration-base=${MASTER_BRANCH_MANIFEST}
+        --project=${CI_PROJECT_PATH}
+        --merge-request=${MERGE_REQUEST}
+        --save-revision-to=manifest_revision
+        --recipe-name=${BB_RECIPE_NAME}
+  artifacts:
+    paths:
+      - manifest_revision
+
+yamllint:
+  extends: .yamllint
+
+# --------------------------------------------------------------------------------------
+# Stage: merge
+# --------------------------------------------------------------------------------------
+merge:
+  extends: .infrastructure
+  stage: merge
+  rules:
+    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH_PROJECT
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - .gitlab-ci/scripts/merge_into_manifest.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${MANIFEST_PROJECT}
+        --master-branch=${MASTER_BRANCH_MANIFEST}
+        --project=${CI_PROJECT_PATH}
+        --master-branch-project=${MASTER_BRANCH_PROJECT}
+        --commit=${CI_COMMIT_SHA}
+        --save-revision-to=manifest_revision
+        --recipe-name=${BB_RECIPE_NAME}
+  artifacts:
+    paths:
+      - manifest_revision
+
+# --------------------------------------------------------------------------------------
+# Stage: build
+# --------------------------------------------------------------------------------------
+build:
+  stage: build
+  rules:
+    # Do not run build if the "skip build" label is set on the merge request
+    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
+      when: never
+    # execute this in MR only and not for integrate/gitlab-ci/ integrations
+    # branches. These are build after the integration has been done in all
+    # projects
+    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
+  trigger:
+    # FIXME: see FIXME comments in manifest-integration-pipelines.yml.jinja2
+    # project: $MANIFEST_PROJECT
+    project: {{ TARGET_PROJECT }}
+    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+    strategy: depend
+
+# --------------------------------------------------------------------------------------
+# Stage: check
+# --------------------------------------------------------------------------------------
+check:
+  extends: .infrastructure
+  stage: check
+  rules:
+    # Do not run check if the "skip build" label is set on the merge request
+    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
+      when: never
+    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+    # The integration is done from the pipeline in gitlab-ci already
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    - if: $CI_MERGE_REQUEST_IID
+  needs: ["integrate"]
+  allow_failure: true
+  script:
+    - cd ${CI_PROJECT_DIR}
+    # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
+    # but CI_OPEN_MERGE_REQUESTS. We use  the first of this comma separated list
+    # in this case
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
+    # The 'parent_merge_request' is passed from the trigger
+    # in case this check job is part of a gitlab-ci integration
+    # pipeline. It is only used to display the correct MR to run again
+    # in a failed check
+    - if [ -n "${parent_merge_request}" ];then
+        PARENT_MR="--parent-merge-request=${parent_merge_request}";
+      fi
+    - .gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${MANIFEST_PROJECT}
+        --integration-base=${MASTER_BRANCH_MANIFEST}
+        --project=${CI_PROJECT_PATH}
+        --merge-request=${MERGE_REQUEST}
+        --verbose
+        ${PARENT_MR}

manifest-integration-pipelines.yml.jinja2

+---
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+include:
+  - project: {{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci
+    ref: {{ GITLAB_CI_REVISION }}
+    file: common.yml
+
+stages:
+  - infrastructure
+
+workflow:
+  rules:
+    - if: $CI_MERGE_REQUEST_IID
+
+# --------------------------------------------------------------------------------------
+# Generate job
+# --------------------------------------------------------------------------------------
+# Use one single job to generate multiple yaml files for the downstream pipelines.
+# FIXME: This is only necessary due to a GitLab limitation:
+# https://gitlab.com/gitlab-org/gitlab/-/issues/347469
+# We work around this by generating manifest-integration-jobs.yml from a Jinja2 template
+# and insert the trigger project via a Jinja2 variable.
+# The issue is already fixed and will be released in GitLab 15.3:
+# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92346
+# As soon as we update to this version, we can get rid of the generate job and convert
+# the Jinja2 template to a simple YAML file.
+generate:
+  extends:
+    - .infrastructure
+  script:
+    # The job generation script implicitly passes the OS environment to the template, so
+    # that the template has access to all GitLab CI variables. Hence there is no need
+    # to explicitly pass any of them as command line arguments.
+{% for integration in INTEGRATION.split('\n') %}
+{% set SOURCE_BRANCH, TARGET_PROJECT, TARGET_BRANCH = integration.split(':') %}
+    - TARGET_PROJECT={{ TARGET_PROJECT }}
+        .gitlab-ci/scripts/generate_job_from_template.py
+        --template=.gitlab-ci/manifest-integration-jobs.yml.jinja2
+        > manifest-integration-jobs-{{ loop.index }}.yml
+{% endfor %}
+  artifacts:
+    expire_in: 4 weeks
+    paths:
+      - manifest-integration-jobs-*.yml
+
+# --------------------------------------------------------------------------------------
+# Trigger jobs
+# --------------------------------------------------------------------------------------
+{% for integration in INTEGRATION.split('\n') %}
+{% set SOURCE_BRANCH, TARGET_PROJECT, TARGET_BRANCH = integration.split(':') %}
+
+{{ TARGET_PROJECT }}:{{ TARGET_BRANCH }}:
+  stage: infrastructure
+  needs:
+    - generate
+  variables:
+    MASTER_BRANCH_PROJECT: {{ SOURCE_BRANCH }}
+    MANIFEST_PROJECT: {{ TARGET_PROJECT }}
+    MASTER_BRANCH_MANIFEST: {{ TARGET_BRANCH }}
+    GITLAB_CI_REVISION: {{ GITLAB_CI_REVISION }}
+    MANIFEST_FILE: {{ MANIFEST_FILE }}
+    BB_RECIPE_NAME: {{ BB_RECIPE_NAME }}
+  trigger:
+    include:
+      # FIXME: Use these settings after switching from jinja2 to yaml (see above)
+      # project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci'
+      # ref: ${GITLAB_CI_REVISION}
+      # file: manifest-integration-jobs.yml
+      artifact: manifest-integration-jobs-{{ loop.index }}.yml
+      job: generate
+    strategy: depend
+
+{% endfor %}

manifest-integration.yml

@@ -7,10 +7,6 @@ include:
 
 stages:
   - infrastructure
-  - integrate
-  - merge
-  - build
-  - check
 
 variables:
   MANIFEST_FILE: "default.xml"
@@ -43,129 +39,31 @@ workflow:
 # --------------------------------------------------------------------------------------
 # Stage: infrastructure
 # --------------------------------------------------------------------------------------
-integrate:
-  extends: .infrastructure
-  rules:
-    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
-    # The integration is done from the pipeline in gitlab-ci already
-    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
-      when: never
-    # We have to make sure that the pipeline runs for the current manifest
-    # master at the time a merge request is created. Otherwise we cannot
-    # guarantee a green master after merging.
-    - if: $CI_MERGE_REQUEST_IID
-    # Explicitly allow externally triggered pipelines in every case
-    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-  cache:
-    policy: push
-  script:
-    - cd ${CI_PROJECT_DIR}
-    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
-        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-      else
-        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
-      fi
-    - .gitlab-ci/scripts/integrate_into_manifest.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --manifest-project=${MANIFEST_PROJECT}
-        --manifest-file=${MANIFEST_FILE}
-        --integration-base=${MASTER_BRANCH_MANIFEST}
-        --project=${CI_PROJECT_PATH}
-        --merge-request=${MERGE_REQUEST}
-        --save-revision-to=manifest_revision
-        --recipe-name=${BB_RECIPE_NAME}
-  artifacts:
-    paths:
-      - manifest_revision
-
-yamllint:
-  extends: .yamllint
-
-# --------------------------------------------------------------------------------------
-# Stage: merge
-# --------------------------------------------------------------------------------------
-merge:
-  extends: .infrastructure
-  stage: merge
-  rules:
-    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH_PROJECT
+generate-pipelines:
+  extends:
+    - .infrastructure
   script:
-    - cd ${CI_PROJECT_DIR}
-    - .gitlab-ci/scripts/merge_into_manifest.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --manifest-project=${MANIFEST_PROJECT}
-        --master-branch=${MASTER_BRANCH_MANIFEST}
-        --project=${CI_PROJECT_PATH}
-        --master-branch-project=${MASTER_BRANCH_PROJECT}
-        --commit=${CI_COMMIT_SHA}
-        --save-revision-to=manifest_revision
-        --recipe-name=${BB_RECIPE_NAME}
+    # The job generation script implicitly passes the OS environment to the template, so
+    # that the template has access to all GitLab CI variables. Hence there is no need
+    # to explicitly pass any of them as command line arguments.
+    - .gitlab-ci/scripts/generate_job_from_template.py
+        --template=.gitlab-ci/manifest-integration-pipelines.yml.jinja2
+        > manifest-integration-pipelines.yml
   artifacts:
+    expire_in: 4 weeks
     paths:
-      - manifest_revision
+      - manifest-integration-pipelines.yml
 
-# --------------------------------------------------------------------------------------
-# Stage: build
-# --------------------------------------------------------------------------------------
-build:
-  stage: build
-  rules:
-    # Do not run build if the "skip build" label is set on the merge request
-    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
-      when: never
-    # execute this in MR only and not for integrate/gitlab-ci/ integrations
-    # branches. These are build after the integration has been done in all
-    # projects
-    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
+trigger-pipelines:
+  stage: infrastructure
+  needs:
+    - generate-pipelines
   trigger:
-    project: !reference [variables, MANIFEST_PROJECT]
-    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+    include:
+      - artifact: manifest-integration-pipelines.yml
+        job: generate-pipelines
     strategy: depend
 
-# --------------------------------------------------------------------------------------
-# Stage: check
-# --------------------------------------------------------------------------------------
-check:
-  extends: .infrastructure
-  stage: check
-  rules:
-    # Do not run check if the "skip build" label is set on the merge request
-    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
-      when: never
-    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
-    # The integration is done from the pipeline in gitlab-ci already
-    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
-      when: never
-    - if: $CI_MERGE_REQUEST_IID
-    # Explicitly allow externally triggered pipelines in every case
-    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-  needs: ["integrate"]
-  allow_failure: true
-  script:
-    - cd ${CI_PROJECT_DIR}
-    # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
-    # but CI_OPEN_MERGE_REQUESTS. We use  the first of this comma separated list
-    # in this case
-    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
-        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-      else
-        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
-      fi
-    # The 'parent_merge_request' is passed from the trigger
-    # in case this check job is part of a gitlab-ci integration
-    # pipeline. It is only used to display the correct MR to run again
-    # in a failed check
-    - if [ -n "${parent_merge_request}" ];then
-        PARENT_MR="--parent-merge-request=${parent_merge_request}";
-      fi
-    - .gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --manifest-project=${MANIFEST_PROJECT}
-        --integration-base=${MASTER_BRANCH_MANIFEST}
-        --project=${CI_PROJECT_PATH}
-        --merge-request=${MERGE_REQUEST}
-        --verbose
-        ${PARENT_MR}
+yamllint:
+  extends:
+    - .yamllint