Share manifest pipeline for ci-test and yocto

Combine all common yaml parts in manifest-pipeline.yml and add
manifest-pipeline-yocto.yml and manifest-pipeline-ci-test.yml
containing the different variable assignments for each environment.

This change implicitly introduces parent-child build job generation in
the ci-test pipeline, like it is done in the yocto pipeline already.
The ci-test build jobs have been moved to build-jobs-ci-test.jinja2
accordingly.

Furthermore rename GITLAB_CI_CURRENT_REV to GITLAB_CI_REVISION and
remove the run conditions from all generated build jobs, since these are
already present in the upstream trigger job.

The repos including these files have to be updated with the new file
and variable names.

ci-test-manifest.yml → build-jobs-ci-test.jinja2

@@ -2,77 +2,30 @@
 # --------------------------------------------------------------------------------------
 # Global
 # --------------------------------------------------------------------------------------
+# As the trigger job is not executed in a environment with checked out repository, we
+# need to get the includes directly from gitlab
 include:
-  - local: common.yml
+  - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
+    ref: {{ GITLAB_CI_REVISION }}
+    file: 'common.yml'
+
+workflow:
+  rules:
+    # This rule is needed, as otherwise the workflow:rules from
+    # the parent job seem to be used and prevent the pipeline generation
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
 
 stages:
-  - retrigger
   - infrastructure
   - build
   - test
 
-variables:
-  # The master branch is hardcoded here, because it cannot be determined automatically.
-  # Has to be modified for new branches, e.g. new Yocto versions or fix releases.
-  MASTER_BRANCH_MANIFEST: master
-  MASTER_BRANCH_PROJECT: master
-
-workflow:
-  rules:
-    # Explicitly allow externally triggered pipelines in every case
-    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-    # Do not run pipelines for merge requests
-    - if: $CI_MERGE_REQUEST_IID
-      when: never
-    # Do not run pipelines on forked projects
-    # (use id instead of name because of rename)
-    - if: $CI_PROJECT_ID != "1742"
-      when: never
-    # Do not run pipelines on integration branches
-    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
-      when: never
-    # In all other cases, run the pipeline automatically
-    - when: always
-
-
-# --------------------------------------------------------------------------------------
-# Stage: retrigger
-# --------------------------------------------------------------------------------------
-retrigger:
-  extends: .infrastructure
-  stage: retrigger
-  rules:
-    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
-  script:
-    - PROJECTS=$(
-        .gitlab-ci/scripts/get_manifest_projects.py
-        --manifest=default.xml
-        --remote=ci-test
-      )
-    # Add the gitlab-ci project
-    - PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci"
-    # TODO retrigger gitlab-ci integration also
-    # Retrigger also project in SRCREV
-    - echo -e "Projects:\n${PROJECTS}"
-    - for PROJECT in ${PROJECTS}; do
-        .gitlab-ci/scripts/retrigger_mr_pipeline_jobs.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${PROJECT}
-          --state=opened
-          --target-branch=${MASTER_BRANCH_MANIFEST}
-          --job=check
-        ;
-      done
-
-
 # --------------------------------------------------------------------------------------
 # Stage: infrastructure
 # --------------------------------------------------------------------------------------
 yamllint:
   extends: .yamllint
 
-
 # --------------------------------------------------------------------------------------
 # Stage: build
 # --------------------------------------------------------------------------------------
@@ -117,8 +70,6 @@ yamllint:
 
 build:files:
   stage: build
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
   tags:
     - infrastructure
@@ -148,8 +99,6 @@ build:files:
 
 build:echo:
   stage: build
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
   tags:
     - infrastructure
@@ -159,52 +108,13 @@ build:echo:
     - printenv
     - echo "Build successful"
 
-build:merge_request:
-  stage: build
-  rules:
-    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
-  needs: []
-  tags:
-    - infrastructure
-  timeout: 2m
-  image: ${CI_IMAGE_PYTHON}
-  variables:
-    # Include git submodules
-    GIT_SUBMODULE_STRATEGY: recursive
-  script:
-    - cd ${CI_PROJECT_DIR}
-    # Get pipeline for merge request
-    - MR_PIPELINE=$(.gitlab-ci/scripts/get_pipelines.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --project=${CI_PROJECT_PATH}
-        --commit=${CI_COMMIT_SHA}
-        --ref=^${MASTER_BRANCH_MANIFEST} || true | head -1)
-    # If pipeline exists, mirror its result
-    - if [ ! -z "${MR_PIPELINE}" ]; then
-        .gitlab-ci/scripts/mirror_pipeline_result.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${CI_PROJECT_PATH}
-          --pipeline=${MR_PIPELINE}
-    # If no pipeline found, trigger a new one on the master
-    - else
-        .gitlab-ci/scripts/trigger_pipeline.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${CI_PROJECT_PATH}
-          --ref=${MASTER_BRANCH_MANIFEST}
-    - fi
-
-
 # --------------------------------------------------------------------------------------
 # Stage: test
 # --------------------------------------------------------------------------------------
 .test-simple:
   stage: test
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   tags:
     - infrastructure
@@ -232,8 +142,7 @@ test:fail:
     - .infrastructure
   timeout: 1h
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   variables:
     # Include git submodules

build-jobs.jinja2 → build-jobs-yocto.jinja2

 ---
-#======================================================
-# Create build, test and deploy jobs for all machines
-#======================================================
-
-# As the trigger job is not executed in a environment
-# with checked out repository, we need to get the includes
-# directly from gitlab
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+# As the trigger job is not executed in a environment with checked out repository, we
+# need to get the includes directly from gitlab
 include:
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_CURRENT_REV }}
+    ref: {{ GITLAB_CI_REVISION }}
     file: 'common.yml'
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_CURRENT_REV }}
+    ref: {{ GITLAB_CI_REVISION }}
     file: 'manifest-build.yml'
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_CURRENT_REV }}
+    ref: {{ GITLAB_CI_REVISION }}
     file: 'manifest-package.yml'
 
 variables:

manifest-build.yml

@@ -156,8 +156,6 @@ variables:
   extends:
     - .buildbase
     - .build
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
 
 .buildfng:
@@ -173,8 +171,7 @@ variables:
     - .package
   stage: build
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   variables:
     BITBAKE_TASK: "populate_sdk"
@@ -194,8 +191,7 @@ variables:
     - .prepare_test
   timeout: 1h
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   variables:
     # Include git submodules
@@ -236,8 +232,7 @@ variables:
     - .infrastructure
     - .package
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   script:
     # Workaround: We need a command in the script section to be able to run the

manifest-pipeline-ci-test.yml

+---
+include:
+  - local: manifest-pipeline.yml
+
+variables:
+  # The id of the gitlab project used in the rules section to not run pipelines in
+  # forked projects. Using variable here, to allow override in other projects including
+  # this file.
+  MANIFEST_PROJECT_ID: 1725
+
+  # In the manifest, the remotes are specified by an identifier. This is used to find
+  # out included projects for the retrigger job. In custom manifests, the remote may be
+  # named differently, so we need a variable that may be overriden.
+  CI_PARAM_SECO_REMOTE: seco-ne
+
+  BUILD_TIMEOUT: 2m
+
+  # This is the jinja2 template file used to generate the build jobs
+  BUILD_JOBS_TEMPLATE: build-jobs-ci-test.jinja2
+
+  # The master branch is hardcoded here, because it cannot be determined automatically.
+  # Has to be modified for new branches, e.g. new Yocto versions or fix releases.
+  MASTER_BRANCH_MANIFEST: master
+  MASTER_BRANCH_PROJECT: master

manifest-pipeline-yocto.yml

+---
+include:
+  - local: manifest-pipeline.yml
+
+variables:
+  # The id of the gitlab project used in the rules section to not run pipelines in
+  # forked projects. Using variable here, to allow override in other projects including
+  # this file.
+  MANIFEST_PROJECT_ID: 1725
+
+  # In the manifest, the remotes are specified by an identifier. This is used to find
+  # out included projects for the retrigger job. In custom manifests, the remote may be
+  # named differently, so we need a variable that may be overriden.
+  CI_PARAM_SECO_REMOTE: seco-ne
+
+  BUILD_TIMEOUT: 1h
+
+  # This is the jinja2 template file used to generate the build jobs
+  BUILD_JOBS_TEMPLATE: build-jobs-yocto.jinja2
+
+  # Default image and distro
+  CI_PARAM_IMAGE: guf-image
+  CI_PARAM_DISTRO: guf-wayland
+
+  # Flash-N-Go image and distro
+  # In the past, the buildfng job overwrote the image and distro itself. Due to the
+  # transition to the new seconorth names, image and distro for the buildfng must be
+  # settable from outside of the job.
+  CI_PARAM_IMAGE_FNG: fngsystem-image
+  CI_PARAM_DISTRO_FNG: guf-fngsystem
+
+  # List of machines to build images for
+  CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf

manifest.yml → manifest-pipeline.yml

@@ -10,30 +10,6 @@ stages:
   - infrastructure
   - build
 
-variables:
-  # Default image and distro
-  CI_PARAM_IMAGE: guf-image
-  CI_PARAM_DISTRO: guf-wayland
-  # Flash-N-Go image and distro
-  #
-  # In the past, the buildfng job overwrote the image and distro itself.
-  # Because of the transition to the new seconorth names, the image and
-  # distro for the buildfng must be settable from outside of the job.
-  CI_PARAM_IMAGE_FNG: fngsystem-image
-  CI_PARAM_DISTRO_FNG: guf-fngsystem
-  # List of machines to build images for:
-  CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
-  # The id of the gitlab project used in the rules section
-  # to not run pipelines in forked projects
-  # Using variable here, to allow override in other projects
-  # including this file
-  MANIFEST_PROJECT_ID: 1725
-  # In the manifest, the remotes are specified by an identifier
-  # This is used to find out included projects for the retrigger
-  # job. In costum manifests, the remote may be named differently
-  # so we need a variable that may be overriden.
-  CI_PARAM_SECO_REMOTE: seco-ne
-
 workflow:
   rules:
     # Explicitly allow externally triggered pipelines in every case
@@ -94,7 +70,7 @@ generate-build-jobs:
     # that the template has access to all GitLab CI variables. Hence there is no need
     # to explicitly pass any of them as command line arguments.
     - .gitlab-ci/scripts/generate_job_from_template.py
-              --template=.gitlab-ci/build-jobs.jinja2
+              --template=.gitlab-ci/${BUILD_JOBS_TEMPLATE}
               > build-jobs.yml
   artifacts:
     expire_in: 4 weeks
@@ -121,7 +97,10 @@ yamllint:
 build:merge_request:
   extends: .infrastructure
   stage: build
-  timeout: 1h
+  # For some reason we cannot use a variable for the timeout, the CI linter reports
+  # 'jobs:build:merge_request:timeout config should be a duration' then.
+  # Hence we use GitLab's 'reference' syntax instead.
+  timeout: !reference [variables, BUILD_TIMEOUT]
   rules:
     - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
   script: