From 25d7cabdb130955c0754c3742a5be2e4b747c491 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20H=C3=B6ppner?= <jonas.hoeppner@garz-fricke.com>
Date: Thu, 24 Mar 2022 09:47:55 +0100
Subject: [PATCH] CI: Harmonize foobar and yocto CI files

Also set yocto deploy job to manual to test merge.
---
 .gitlab-ci.yml                  |   4 ++
 common.yml                      |  37 +++++++++++
 foobar-manifest-integration.yml | 107 ++++++++++----------------------
 foobar-manifest.yml             |  38 +++++-------
 manifest-integration.yml        |  53 +++-------------
 manifest.yml                    |  10 ++-
 6 files changed, 104 insertions(+), 145 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9edd8f4c..7c570c4c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -89,6 +89,10 @@ yamllint:
 
 deploy:
   extends: .deploy
+  rules:
+    # For now to test the merge step
+    - when: manual
+      allow_failure: true
   variables:
     PROJECT_ROOT:
       ${CI_PROJECT_ROOT_NAMESPACE}
diff --git a/common.yml b/common.yml
index f2f01ae6..6bf9db6c 100644
--- a/common.yml
+++ b/common.yml
@@ -43,3 +43,40 @@ yamllint:
     - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   script:
     - yamllint -c .gitlab-ci/.yamllint.yml .*.yml
+
+# --------------------------------------------------------------------------------------
+# Common check job, used in yocto and foobar manifest-integration projects
+# --------------------------------------------------------------------------------------
+.check:
+  extends: .infrastructure
+  stage: check
+  rules:
+    - if: $CI_MERGE_REQUEST_IID
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+  needs: ["integrate"]
+  script:
+    - cd ${CI_PROJECT_DIR}
+    # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
+    # but CI_OPEN_MERGE_REQUESTS. We use  the first of this comma separated list
+    # in this case
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
+    # The 'parent_merge_request' is passed from the trigger
+    # in case this check job is part of a gitlab-ci integration
+    # pipeline. It is only used to display the correct MR to run again
+    # in a failed check
+    - if [ -n "${parent_merge_request}" ];then
+        PARENT_MR="--parent-merge-request=${parent_merge_request}";
+      fi
+    - .gitlab-ci/check_if_integration_branch_is_up_to_date.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${MANIFEST_PROJECT}
+        --integration-base=${MASTER_BRANCH_MANIFEST}
+        --project=${CI_PROJECT_PATH}
+        --merge-request=${MERGE_REQUEST}
+        ${PARENT_MR}
diff --git a/foobar-manifest-integration.yml b/foobar-manifest-integration.yml
index f29b147b..956cccc9 100644
--- a/foobar-manifest-integration.yml
+++ b/foobar-manifest-integration.yml
@@ -3,14 +3,19 @@
 # CI steps needed to integrate a sub project change into the manifest project
 # foobar variant which is used for testing the CI
 # -------------------------------------------------------------------------------------
+include:
+  - local: common.yml
 
-image: "${CI_IMAGES}/python/3.9:99e363bc5feaa27ff18dbe7731a76ff04d7d0deb"
+image: "${CI_IMAGE_PYTHON}"
+
+stages:
+  - infrastructure
+  - integrate
+  - merge
+  - build
+  - check
 
 variables:
-  # CI_IMAGES_BASEPATH: Environment variable configure in gitlab
-  CI_IMAGES: ${CI_IMAGES_BASEPATH}/ci-images
-  # Include git submodules
-  GIT_SUBMODULE_STRATEGY: recursive
   # FIXME: due to a missing feature in GitLab, we cannot use this variable in the build
   # stage further down this file. If it ever changes, it has to be changed there too.
   # (https://gitlab.com/gitlab-org/gitlab/-/issues/249583)
@@ -18,14 +23,16 @@ variables:
                      ci-test/minimal-manifest"
   # The master branch is hardcoded here, because it cannot be determined automatically.
   # Has to be modified for new branches, e.g. new Yocto versions or fix releases.
-  MASTER_BRANCH: master
+  MASTER_BRANCH_MANIFEST: master
+  MASTER_BRANCH_PROJECT: master
 
-stages:
-  - prepare
-  - integrate
-  - merge
-  - build
-  - check
+  # The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
+  # to match the repository and the bitbake recipe name.
+  # We set it here to none, as every project needing it
+  # has to specify it in its own gitlab-ci.yml file.
+  # The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
+  # used for projects referenced in the manifest file.
+  BB_RECIPE_NAME: none
 
 workflow:
   rules:
@@ -50,38 +57,21 @@ workflow:
         && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID
 
 # ---------------------------------------------------------------------------------------
-# Stage: prepare
-# ---------------------------------------------------------------------------------------
-yamllint:
-  stage: prepare
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH
-  tags:
-    - infrastructure
-  timeout: 2m
-  script:
-    - printenv
-    - yamllint -c .gitlab-ci/.yamllint.yml .*.yml
-
-# ---------------------------------------------------------------------------------------
-# Stage: integrate
+# integrate
+# Create a commit in the manifest to pull this change
 # ---------------------------------------------------------------------------------------
 integrate:
-  stage: integrate
+  extends: .infrastructure
   rules:
-    # We have to make sure that the pipeline runs for the current manifest master at
-    # the time a merge request is created. Otherwise we cannot guarantee a green
-    # master after merging.
+    # We have to make sure that the pipeline runs for the current manifest
+    # master at the time a merge request is created. Otherwise we cannot
+    # guarantee a green master after merging.
     - if: $CI_MERGE_REQUEST_IID
     # Explicitly allow externally triggered pipelines in every case
     - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-  tags:
-    - infrastructure
-  timeout: 2m
   cache:
     policy: push
   script:
-    - printenv
     - cd ${CI_PROJECT_DIR}
     - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
         MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
@@ -92,11 +82,11 @@ integrate:
         --gitlab-url=${CI_SERVER_URL}
         --token=${GITBOT_TOKEN}
         --manifest-project=${MANIFEST_PROJECT}
-        --integration-base=${MASTER_BRANCH}
+        --integration-base=${MASTER_BRANCH_MANIFEST}
         --project=${CI_PROJECT_PATH}
         --merge-request=${MERGE_REQUEST}
         --save-revision-to=manifest_revision
-        --verbose
+        --recipe-name=${BB_RECIPE_NAME}
   artifacts:
     paths:
       - manifest_revision
@@ -105,22 +95,22 @@ integrate:
 # Stage: merge
 # --------------------------------------------------------------------------------------
 merge:
+  extends: .infrastructure
   stage: merge
   rules:
-    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
-  tags:
-    - infrastructure
-  timeout: 3m
+    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH_PROJECT
   script:
     - cd ${CI_PROJECT_DIR}
     - .gitlab-ci/merge_into_manifest.py
         --gitlab-url=${CI_SERVER_URL}
         --token=${GITBOT_TOKEN}
         --manifest-project=${MANIFEST_PROJECT}
-        --master-branch=${MASTER_BRANCH}
+        --master-branch=${MASTER_BRANCH_MANIFEST}
         --project=${CI_PROJECT_PATH}
+        --master-branch-project=${MASTER_BRANCH_PROJECT}
         --commit=${CI_COMMIT_SHA}
         --save-revision-to=manifest_revision
+        --recipe-name=${BB_RECIPE_NAME}
   artifacts:
     paths:
       - manifest_revision
@@ -144,35 +134,4 @@ build:
 # Stage: check
 # --------------------------------------------------------------------------------------
 check:
-  stage: check
-  rules:
-    - if: $CI_MERGE_REQUEST_IID
-    # Explicitly allow externally triggered pipelines in every case
-    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-  needs: ["integrate"]
-  tags:
-    - infrastructure
-  timeout: 2m
-  script:
-    - printenv
-    - cd ${CI_PROJECT_DIR}
-    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
-        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-      else
-        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
-      fi
-    - if [ -n "${parent_merge_request}" ];then
-        PARENT_MR="--parent-merge-request=${parent_merge_request}";
-      fi
-    - echo "${parent_merge_request}"
-    - echo "${PARENT_MR}"
-    - echo "${MERGE_REQUEST}"
-    - .gitlab-ci/check_if_integration_branch_is_up_to_date.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --manifest-project=${MANIFEST_PROJECT}
-        --integration-base=${MASTER_BRANCH}
-        --project=${CI_PROJECT_PATH}
-        --merge-request=${MERGE_REQUEST}
-        --verbose
-        ${PARENT_MR}
+  extends: .check
diff --git a/foobar-manifest.yml b/foobar-manifest.yml
index 2565a4d2..07ec196d 100644
--- a/foobar-manifest.yml
+++ b/foobar-manifest.yml
@@ -1,21 +1,21 @@
+---
 # --------------------------------------------------------------------------------------
 # Global
 # --------------------------------------------------------------------------------------
----
+include:
+  - local: common.yml
+
 stages:
   - retrigger
+  - infrastructure
   - build
   - test
 
 variables:
-  # CI_IMAGES_BASEPATH: Environment variable configure in gitlab
-  CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
-  CI_IMAGES_REV: 835a7096092eef5cecde23fd933209e7a8488637
-  CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REV}"
-  CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REV}"
   # The master branch is hardcoded here, because it cannot be determined automatically.
   # Has to be modified for new branches, e.g. new Yocto versions or fix releases.
-  MASTER_BRANCH: master
+  MASTER_BRANCH_MANIFEST: master
+  MASTER_BRANCH_PROJECT: master
 
 workflow:
   rules:
@@ -39,16 +39,10 @@ workflow:
 # Stage: retrigger
 # --------------------------------------------------------------------------------------
 retrigger:
+  extends: .infrastructure
   stage: retrigger
   rules:
-    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH && $CI_PIPELINE_SOURCE != "api"
-  tags:
-    - infrastructure
-  timeout: 2m
-  image: ${CI_IMAGE_PYTHON}
-  variables:
-    # Include git submodules
-    GIT_SUBMODULE_STRATEGY: recursive
+    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
   script:
     - PROJECTS=$(
         .gitlab-ci/get_manifest_projects.py
@@ -66,7 +60,7 @@ retrigger:
           --token=${GITBOT_TOKEN}
           --project=${PROJECT}
           --state=opened
-          --target-branch=${MASTER_BRANCH}
+          --target-branch=${MASTER_BRANCH_MANIFEST}
           --job=check
         ;
       done
@@ -107,7 +101,7 @@ retrigger:
 build:files:
   stage: build
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH || $CI_PIPELINE_SOURCE == "api"
+    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
   tags:
     - infrastructure
@@ -138,7 +132,7 @@ build:files:
 build:echo:
   stage: build
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH || $CI_PIPELINE_SOURCE == "api"
+    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
   needs: []
   tags:
     - infrastructure
@@ -151,7 +145,7 @@ build:echo:
 build:merge_request:
   stage: build
   rules:
-    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH && $CI_PIPELINE_SOURCE != "api"
+    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
   needs: []
   tags:
     - infrastructure
@@ -168,7 +162,7 @@ build:merge_request:
         --token=${GITBOT_TOKEN}
         --project=${CI_PROJECT_PATH}
         --commit=${CI_COMMIT_SHA}
-        --ref=^${MASTER_BRANCH} || true | head -1)
+        --ref=^${MASTER_BRANCH_MANIFEST} || true | head -1)
     # If pipeline exists, mirror its result
     - if [ ! -z "${MR_PIPELINE}" ]; then
         .gitlab-ci/mirror_pipeline_result.py
@@ -182,7 +176,7 @@ build:merge_request:
           --gitlab-url=${CI_SERVER_URL}
           --token=${GITBOT_TOKEN}
           --project=${CI_PROJECT_PATH}
-          --ref=${MASTER_BRANCH}
+          --ref=${MASTER_BRANCH_MANIFEST}
     - fi
 
 
@@ -192,7 +186,7 @@ build:merge_request:
 .test:
   stage: test
   rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH || $CI_PIPELINE_SOURCE == "api"
+    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
       when: manual
       allow_failure: true
   tags:
diff --git a/manifest-integration.yml b/manifest-integration.yml
index f834401a..18fe3525 100644
--- a/manifest-integration.yml
+++ b/manifest-integration.yml
@@ -3,7 +3,6 @@
 # Global
 # ---------------------------------------------------------------------------------------
 include:
-  - local: manifest-build.yml
   - local: common.yml
 
 stages:
@@ -44,16 +43,6 @@ workflow:
     - if: $CI_PROJECT_ROOT_NAMESPACE == "SECO-Northern-Europe"
         && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID
 
-# ---------------------------------------------------------------------------------------
-# Stage: prepare
-# ---------------------------------------------------------------------------------------
-yamllint:
-  extends: .infrastructure
-  rules:
-    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST
-  script:
-    - yamllint -c .gitlab-ci/.yamllint.yml .*.yml
-
 # ---------------------------------------------------------------------------------------
 # integrate
 # Create a commit in the manifest to pull this change
@@ -85,14 +74,13 @@ integrate:
         --merge-request=${MERGE_REQUEST}
         --save-revision-to=manifest_revision
         --recipe-name=${BB_RECIPE_NAME}
-        --verbose
   artifacts:
     paths:
       - manifest_revision
 
-# ---------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 # Stage: merge
-# ---------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 merge:
   extends: .infrastructure
   stage: merge
@@ -114,9 +102,9 @@ merge:
     paths:
       - manifest_revision
 
-# ---------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 # Stage: build
-# ---------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 build:
   stage: build
   rules:
@@ -129,35 +117,8 @@ build:
     branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
     strategy: depend
 
-# ---------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 # Stage: check
-# ---------------------------------------------------------------------------------------
+# --------------------------------------------------------------------------------------
 check:
-  extends: .infrastructure
-  rules:
-    - if: $CI_MERGE_REQUEST_IID
-    # Explicitly allow externally triggered pipelines in every case
-    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-  needs: ["integrate"]
-  script:
-    - cd ${CI_PROJECT_DIR}
-    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
-        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-      else
-        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
-      fi
-    - if [ -n "${parent_merge_request}" ];then
-        PARENT_MR="--parent-merge-request=${parent_merge_request}";
-      fi
-    - echo "${parent_merge_request}"
-    - echo "${PARENT_MR}"
-    - echo "${MERGE_REQUEST}"
-    - .gitlab-ci/check_if_integration_branch_is_up_to_date.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --manifest-project=${MANIFEST_PROJECT}
-        --integration-base=${MASTER_BRANCH_MANIFEST}
-        --project=${CI_PROJECT_PATH}
-        --merge-request=${MERGE_REQUEST}
-        --verbose
-        ${PARENT_MR}
+  extends: .check
diff --git a/manifest.yml b/manifest.yml
index 31ceb8ca..b62b44db 100644
--- a/manifest.yml
+++ b/manifest.yml
@@ -1,14 +1,14 @@
+---
 # --------------------------------------------------------------------------------------
 # Global
 # --------------------------------------------------------------------------------------
----
 include:
   - local: manifest-build.yml
   - local: common.yml
 
 stages:
   - retrigger
-  - prepare
+  - infrastructure
   - build
   - test
   - deploy
@@ -51,6 +51,10 @@ retrigger:
         --remote=seco-north
         --concat-namespaces
       )
+    # Add the gitlab-ci project
+    - PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci"
+    # TODO retrigger gitlab-ci integration also
+    # Retrigger also project in SRCREV
     - echo -e "Projects:\n${PROJECTS}"
     - for PROJECT in ${PROJECTS}; do
         .gitlab-ci/retrigger_mr_pipeline_jobs.py
@@ -64,7 +68,7 @@ retrigger:
       done
 
 # --------------------------------------------------------------------------------------
-# Stage: prepare
+# Stage: infrastructure
 # --------------------------------------------------------------------------------------
 
 changelog:
-- 
GitLab