From 9b8cf51c0c280b8373ae3b809e3a5b24b2d61f30 Mon Sep 17 00:00:00 2001
From: Tim Jaacks <tim.jaacks@garz-fricke.com>
Date: Tue, 21 Jun 2022 17:08:01 +0200
Subject: [PATCH] Share manifest pipeline for ci-test and yocto

Combine all common yaml parts in manifest-pipeline.yml and add
manifest-pipeline-yocto.yml and manifest-pipeline-ci-test.yml
containing the different variable assignments for each environment.

This change implicitly introduces parent-child build job generation in
the ci-test pipeline, like it is done in the yocto pipeline already.
The ci-test build jobs have been moved to build-jobs-ci-test.jinja2
accordingly.

Furthermore rename GITLAB_CI_CURRENT_REV to GITLAB_CI_REVISION and
remove the run conditions from all generated build jobs, since these are
already present in the upstream trigger job.

The repos including these files have to be updated with the new file
and variable names.
---
 ...-manifest.yml => build-jobs-ci-test.jinja2 | 117 ++----------------
 build-jobs.jinja2 => build-jobs-yocto.jinja2  |  18 ++-
 manifest-build.yml                            |  11 +-
 manifest-pipeline-ci-test.yml                 |  24 ++++
 manifest-pipeline-yocto.yml                   |  33 +++++
 manifest.yml => manifest-pipeline.yml         |  31 +----
 6 files changed, 86 insertions(+), 148 deletions(-)
 rename ci-test-manifest.yml => build-jobs-ci-test.jinja2 (64%)
 rename build-jobs.jinja2 => build-jobs-yocto.jinja2 (90%)
 create mode 100644 manifest-pipeline-ci-test.yml
 create mode 100644 manifest-pipeline-yocto.yml
 rename manifest.yml => manifest-pipeline.yml (79%)

diff --git a/ci-test-manifest.yml b/build-jobs-ci-test.jinja2
similarity index 64%
rename from ci-test-manifest.yml
rename to build-jobs-ci-test.jinja2
index 06e1ccb..4aebd92 100644
--- a/ci-test-manifest.yml
+++ b/build-jobs-ci-test.jinja2
@@ -2,77 +2,30 @@
 # --------------------------------------------------------------------------------------
 # Global
 # --------------------------------------------------------------------------------------
+# As the trigger job is not executed in a environment with checked out repository, we
+# need to get the includes directly from gitlab
 include:
-  - local: common.yml
+  - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
+    ref: {{ GITLAB_CI_REVISION }}
+    file: 'common.yml'
+
+workflow:
+  rules:
+    # This rule is needed, as otherwise the workflow:rules from
+    # the parent job seem to be used and prevent the pipeline generation
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
 
 stages:
-  - retrigger
   - infrastructure
   - build
   - test
 
-variables:
-  # The master branch is hardcoded here, because it cannot be determined automatically.
-  # Has to be modified for new branches, e.g. new Yocto versions or fix releases.
-  MASTER_BRANCH_MANIFEST: master
-  MASTER_BRANCH_PROJECT: master
-
-workflow:
-  rules:
-    # Explicitly allow externally triggered pipelines in every case
-    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-    # Do not run pipelines for merge requests
-    - if: $CI_MERGE_REQUEST_IID
-      when: never
-    # Do not run pipelines on forked projects
-    # (use id instead of name because of rename)
-    - if: $CI_PROJECT_ID != "1742"
-      when: never
-    # Do not run pipelines on integration branches
-    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
-      when: never
-    # In all other cases, run the pipeline automatically
-    - when: always
-
-
-# --------------------------------------------------------------------------------------
-# Stage: retrigger
-# --------------------------------------------------------------------------------------
-retrigger:
-  extends: .infrastructure
-  stage: retrigger
-  rules:
-    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
-  script:
-    - PROJECTS=$(
-        .gitlab-ci/scripts/get_manifest_projects.py
-        --manifest=default.xml
-        --remote=ci-test
-      )
-    # Add the gitlab-ci project
-    - PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci"
-    # TODO retrigger gitlab-ci integration also
-    # Retrigger also project in SRCREV
-    - echo -e "Projects:\n${PROJECTS}"
-    - for PROJECT in ${PROJECTS}; do
-        .gitlab-ci/scripts/retrigger_mr_pipeline_jobs.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${PROJECT}
-          --state=opened
-          --target-branch=${MASTER_BRANCH_MANIFEST}
-          --job=check
-        ;
-      done
-
-
 # --------------------------------------------------------------------------------------
 # Stage: infrastructure
 # --------------------------------------------------------------------------------------
 yamllint:
   extends: .yamllint
 
-
 # --------------------------------------------------------------------------------------
 # Stage: build
 # --------------------------------------------------------------------------------------
@@ -117,8 +70,6 @@ yamllint:
 
 build:files:
   stage: build
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
   tags:
     - infrastructure
@@ -148,8 +99,6 @@ build:files:
 
 build:echo:
   stage: build
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
   tags:
     - infrastructure
@@ -159,52 +108,13 @@ build:echo:
     - printenv
     - echo "Build successful"
 
-build:merge_request:
-  stage: build
-  rules:
-    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
-  needs: []
-  tags:
-    - infrastructure
-  timeout: 2m
-  image: ${CI_IMAGE_PYTHON}
-  variables:
-    # Include git submodules
-    GIT_SUBMODULE_STRATEGY: recursive
-  script:
-    - cd ${CI_PROJECT_DIR}
-    # Get pipeline for merge request
-    - MR_PIPELINE=$(.gitlab-ci/scripts/get_pipelines.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --project=${CI_PROJECT_PATH}
-        --commit=${CI_COMMIT_SHA}
-        --ref=^${MASTER_BRANCH_MANIFEST} || true | head -1)
-    # If pipeline exists, mirror its result
-    - if [ ! -z "${MR_PIPELINE}" ]; then
-        .gitlab-ci/scripts/mirror_pipeline_result.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${CI_PROJECT_PATH}
-          --pipeline=${MR_PIPELINE}
-    # If no pipeline found, trigger a new one on the master
-    - else
-        .gitlab-ci/scripts/trigger_pipeline.py
-          --gitlab-url=${CI_SERVER_URL}
-          --token=${GITBOT_TOKEN}
-          --project=${CI_PROJECT_PATH}
-          --ref=${MASTER_BRANCH_MANIFEST}
-    - fi
-
-
 # --------------------------------------------------------------------------------------
 # Stage: test
 # --------------------------------------------------------------------------------------
 .test-simple:
   stage: test
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   tags:
     - infrastructure
@@ -232,8 +142,7 @@ test:fail:
     - .infrastructure
   timeout: 1h
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   variables:
     # Include git submodules
diff --git a/build-jobs.jinja2 b/build-jobs-yocto.jinja2
similarity index 90%
rename from build-jobs.jinja2
rename to build-jobs-yocto.jinja2
index 8d8cbc4..bd91a78 100644
--- a/build-jobs.jinja2
+++ b/build-jobs-yocto.jinja2
@@ -1,20 +1,18 @@
 ---
-#======================================================
-# Create build, test and deploy jobs for all machines
-#======================================================
-
-# As the trigger job is not executed in a environment
-# with checked out repository, we need to get the includes
-# directly from gitlab
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+# As the trigger job is not executed in a environment with checked out repository, we
+# need to get the includes directly from gitlab
 include:
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_CURRENT_REV }}
+    ref: {{ GITLAB_CI_REVISION }}
     file: 'common.yml'
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_CURRENT_REV }}
+    ref: {{ GITLAB_CI_REVISION }}
     file: 'manifest-build.yml'
   - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
-    ref: {{ GITLAB_CI_CURRENT_REV }}
+    ref: {{ GITLAB_CI_REVISION }}
     file: 'manifest-package.yml'
 
 variables:
diff --git a/manifest-build.yml b/manifest-build.yml
index d3d7e7c..7037cc6 100644
--- a/manifest-build.yml
+++ b/manifest-build.yml
@@ -156,8 +156,6 @@ variables:
   extends:
     - .buildbase
     - .build
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
 
 .buildfng:
@@ -173,8 +171,7 @@ variables:
     - .package
   stage: build
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   variables:
     BITBAKE_TASK: "populate_sdk"
@@ -194,8 +191,7 @@ variables:
     - .prepare_test
   timeout: 1h
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   variables:
     # Include git submodules
@@ -236,8 +232,7 @@ variables:
     - .infrastructure
     - .package
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-      when: manual
+    - when: manual
       allow_failure: true
   script:
     # Workaround: We need a command in the script section to be able to run the
diff --git a/manifest-pipeline-ci-test.yml b/manifest-pipeline-ci-test.yml
new file mode 100644
index 0000000..cae23bc
--- /dev/null
+++ b/manifest-pipeline-ci-test.yml
@@ -0,0 +1,24 @@
+---
+include:
+  - local: manifest-pipeline.yml
+
+variables:
+  # The id of the gitlab project used in the rules section to not run pipelines in
+  # forked projects. Using variable here, to allow override in other projects including
+  # this file.
+  MANIFEST_PROJECT_ID: 1725
+
+  # In the manifest, the remotes are specified by an identifier. This is used to find
+  # out included projects for the retrigger job. In custom manifests, the remote may be
+  # named differently, so we need a variable that may be overriden.
+  CI_PARAM_SECO_REMOTE: seco-ne
+
+  BUILD_TIMEOUT: 2m
+
+  # This is the jinja2 template file used to generate the build jobs
+  BUILD_JOBS_TEMPLATE: build-jobs-ci-test.jinja2
+
+  # The master branch is hardcoded here, because it cannot be determined automatically.
+  # Has to be modified for new branches, e.g. new Yocto versions or fix releases.
+  MASTER_BRANCH_MANIFEST: master
+  MASTER_BRANCH_PROJECT: master
diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml
new file mode 100644
index 0000000..30976f6
--- /dev/null
+++ b/manifest-pipeline-yocto.yml
@@ -0,0 +1,33 @@
+---
+include:
+  - local: manifest-pipeline.yml
+
+variables:
+  # The id of the gitlab project used in the rules section to not run pipelines in
+  # forked projects. Using variable here, to allow override in other projects including
+  # this file.
+  MANIFEST_PROJECT_ID: 1725
+
+  # In the manifest, the remotes are specified by an identifier. This is used to find
+  # out included projects for the retrigger job. In custom manifests, the remote may be
+  # named differently, so we need a variable that may be overriden.
+  CI_PARAM_SECO_REMOTE: seco-ne
+
+  BUILD_TIMEOUT: 1h
+
+  # This is the jinja2 template file used to generate the build jobs
+  BUILD_JOBS_TEMPLATE: build-jobs-yocto.jinja2
+
+  # Default image and distro
+  CI_PARAM_IMAGE: guf-image
+  CI_PARAM_DISTRO: guf-wayland
+
+  # Flash-N-Go image and distro
+  # In the past, the buildfng job overwrote the image and distro itself. Due to the
+  # transition to the new seconorth names, image and distro for the buildfng must be
+  # settable from outside of the job.
+  CI_PARAM_IMAGE_FNG: fngsystem-image
+  CI_PARAM_DISTRO_FNG: guf-fngsystem
+
+  # List of machines to build images for
+  CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
diff --git a/manifest.yml b/manifest-pipeline.yml
similarity index 79%
rename from manifest.yml
rename to manifest-pipeline.yml
index 8485d51..94c281a 100644
--- a/manifest.yml
+++ b/manifest-pipeline.yml
@@ -10,30 +10,6 @@ stages:
   - infrastructure
   - build
 
-variables:
-  # Default image and distro
-  CI_PARAM_IMAGE: guf-image
-  CI_PARAM_DISTRO: guf-wayland
-  # Flash-N-Go image and distro
-  #
-  # In the past, the buildfng job overwrote the image and distro itself.
-  # Because of the transition to the new seconorth names, the image and
-  # distro for the buildfng must be settable from outside of the job.
-  CI_PARAM_IMAGE_FNG: fngsystem-image
-  CI_PARAM_DISTRO_FNG: guf-fngsystem
-  # List of machines to build images for:
-  CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf
-  # The id of the gitlab project used in the rules section
-  # to not run pipelines in forked projects
-  # Using variable here, to allow override in other projects
-  # including this file
-  MANIFEST_PROJECT_ID: 1725
-  # In the manifest, the remotes are specified by an identifier
-  # This is used to find out included projects for the retrigger
-  # job. In costum manifests, the remote may be named differently
-  # so we need a variable that may be overriden.
-  CI_PARAM_SECO_REMOTE: seco-ne
-
 workflow:
   rules:
     # Explicitly allow externally triggered pipelines in every case
@@ -94,7 +70,7 @@ generate-build-jobs:
     # that the template has access to all GitLab CI variables. Hence there is no need
     # to explicitly pass any of them as command line arguments.
     - .gitlab-ci/scripts/generate_job_from_template.py
-              --template=.gitlab-ci/build-jobs.jinja2
+              --template=.gitlab-ci/${BUILD_JOBS_TEMPLATE}
               > build-jobs.yml
   artifacts:
     expire_in: 4 weeks
@@ -121,7 +97,10 @@ yamllint:
 build:merge_request:
   extends: .infrastructure
   stage: build
-  timeout: 1h
+  # For some reason we cannot use a variable for the timeout, the CI linter reports
+  # 'jobs:build:merge_request:timeout config should be a duration' then.
+  # Hence we use GitLab's 'reference' syntax instead.
+  timeout: !reference [variables, BUILD_TIMEOUT]
   rules:
     - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
   script:
-- 
GitLab