From c25eb909992350aba34a7b20024922040fe83b7f Mon Sep 17 00:00:00 2001
From: Lorenzo Pagliai <lorenzo.pagliai@seco.com>
Date: Mon, 14 Nov 2022 16:14:06 +0100
Subject: [PATCH] [CICD] Updating manifest-integration.yml file for edgehog

---
 common.yml                  |   8 +-
 manifest-integration.yml    | 243 ++++++++++++++++++++----------------
 manifest-integration_GF.yml | 144 +++++++++++++++++++++
 3 files changed, 285 insertions(+), 110 deletions(-)
 create mode 100644 manifest-integration_GF.yml

diff --git a/common.yml b/common.yml
index 63dcadc..288d222 100644
--- a/common.yml
+++ b/common.yml
@@ -5,9 +5,9 @@
 variables:
   # CI_IMAGES_BASEPATH: Environment variable configure in gitlab
   CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
-  CI_IMAGES_REV: e6a404c3e1a919f54feb83f1bc061ce171220437
-  CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REV}"
-  CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REV}"
+  CI_IMAGES_REV: test
+  CI_IMAGE_PYTHON: "git.seco.com:5050/edgehog/yocto_ng/seco-base:${CI_IMAGES_REV}"
+  CI_IMAGE_YOCTO: "secodocker/edgehog-builder:latest"
   # Include git submodules
   GIT_SUBMODULE_STRATEGY: recursive
   # Reduced depth as checkout of larger projects (like the kernel)
@@ -22,7 +22,7 @@ variables:
   tags:
     - infrastructure
   timeout: 10m
-  image: secodocker/manifest-test:vtemp
+  image: $CI_IMAGE_PYTHON
   needs: []
   variables:
     # Include git submodules
diff --git a/manifest-integration.yml b/manifest-integration.yml
index 9c6204f..e7e6fcd 100644
--- a/manifest-integration.yml
+++ b/manifest-integration.yml
@@ -2,12 +2,16 @@
 # --------------------------------------------------------------------------------------
 # Global
 # --------------------------------------------------------------------------------------
+
 include:
   - local: common.yml
 
 stages:
-  - testing
-  - manifest-integration
+  - infrastructure
+  - integrate
+  - merge
+  - build
+  - check
 
 variables:
   MANIFEST_FILE: "distro/seco-distro.xml"
@@ -17,128 +21,155 @@ variables:
   # has to specify it in its own gitlab-ci.yml file.
   # The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
   # used for projects referenced in the manifest file.
+  #GITLAB_CI_REVISION: 80380e9065c04e7983fa36d451a9a62d96e5addb
+  # FIXME: This is only necessary due to the following GitLab limitation:
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/209904
+  # As soon as this gets fixed upstream, the hard-coded branch name should be removed.
+  #MASTER_BRANCH: kirkstone/develop
   MANIFEST_PROJECT: seco-manifest
   MASTER_BRANCH_MANIFEST: kirkstone/develop
-  MASTER_BRANCH_PROJECT: main
+  MASTER_BRANCH_PROJECT: $MASTER_BRANCH
   BB_RECIPE_NAME: none
 
-workflow:
+  DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/"
+  GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
+  GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
+  TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
+ 
+# --------------------------------------------------------------------------------------
+# Stage: infrastructure
+# --------------------------------------------------------------------------------------
+integrate:
+  extends: .infrastructure
   rules:
-    # Explicitly allow externally triggered pipelines in every case
-    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
-    # Do not run pipelines on forked projects.
-    # The pipelines would not work anyway because of the users permissions.
-    # There are two cases catched here:
-    # 1. The project is forked into someones gitlab namespace and a MR to
-    #    include a change into this forked project is created. In this case
-    #    is the CI_PROJECT_ROOT_NAMESPACE not seco-ne but the
-    #    namespace the fork lives in.
-    # 2. The MR from the forked project is created to merge the change into this
-    #    the project in the seco-ne namespace (customer sending
-    #    change to us). Here the the IDs used below differ.
-    #
-    - if: $CI_PROJECT_ROOT_NAMESPACE != "seco-ne"
+    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+    # The integration is done from the pipeline in gitlab-ci already
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
       when: never
-    - if: $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != $CI_MERGE_REQUEST_PROJECT_ID
-      when: never
-    # FIXME: Unfortunately we cannot use variables in regular expressions due to a
-    # GitLab limitation: https://gitlab.com/gitlab-org/gitlab/-/issues/209904
-    # As soon as this get fixed, use the regex based rules below instead of checking
-    # against the MASTER_BRANCH variable.
-    # Run pipeline if target branch of the merge request has an integration target, i.e.
-    # INTEGRATION contains a line beginning with the target branch followed by a colon.
-    # This also implies that the pipeline runs in merge request context only.
-    # - if: $INTEGRATION =~ /^$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:/m
-    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $MASTER_BRANCH
-    # Run pipeline on target branch after merging a merge request.
-    # - if: $INTEGRATION =~ /^$CI_COMMIT_BRANCH:/m
-    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
+    # We have to make sure that the pipeline runs for the current manifest
+    # master at the time a merge request is created. Otherwise we cannot
+    # guarantee a green master after merging.
+    #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $CI_MERGE_REQUEST_IID
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: always
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+  cache:
+    policy: push
+  script:
+    - cd ${CI_PROJECT_DIR}
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
+    - python3 scripts/integrate_into_manifest.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${MANIFEST_PROJECT}
+        --manifest-file=${MANIFEST_FILE}
+        --integration-base=${MASTER_BRANCH_MANIFEST}
+        --project=${CI_PROJECT_PATH}
+        --merge-request=${MERGE_REQUEST}
+        --save-revision-to=manifest_revision
+        --recipe-name=${BB_RECIPE_NAME}
+        --verbose
+  artifacts:
+    paths:
+      - manifest_revision
 
-.skip-for-gitlab-ci-integrations:
-  rules:
-    - if: $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
+
+yamllint:
+  extends: .yamllint
 
 # --------------------------------------------------------------------------------------
-# Manifest integration jobs
+# Stage: merge
 # --------------------------------------------------------------------------------------
-testing:
-  tags:
-    - infrastructure
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: always
-  stage: testing
-  timeout: 10 m
-  image: secodocker/manifest-test:vtemp
-  needs: []
-
-generate-pipelines:
+merge:
+  extends: .infrastructure
+  stage: merge
   rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: always
-  extends:
-    - .infrastructure
-    - .skip-for-gitlab-ci-integrations
-  stage: manifest-integration
+     - if: $CI_COMMIT_BRANCH == "$MASTER_BRANCH_PROJECT"
+       when: always
   script:
-    # The job generation script implicitly passes the OS environment to the template, so
-    # that the template has access to all GitLab CI variables. Hence there is no need
-    # to explicitly pass any of them as command line arguments.
-    - .gitlab-ci/scripts/generate_job_from_template.py
-        --template=.gitlab-ci/manifest-integration-pipelines.yml.jinja2
-        > manifest-integration-pipelines.yml
+    - cd ${CI_PROJECT_DIR}
+    - python3 scripts/merge_into_manifest.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${MANIFEST_PROJECT}
+        --master-branch=${MASTER_BRANCH_MANIFEST}
+        --project=${CI_PROJECT_PATH}
+        --master-branch-project=${MASTER_BRANCH_PROJECT}
+        --commit=${CI_COMMIT_SOURCE}
+        --save-revision-to=manifest_revision
+        --recipe-name=${BB_RECIPE_NAME}
   artifacts:
-    expire_in: 4 weeks
     paths:
-      - manifest-integration-pipelines.yml
-
-trigger-pipelines:
+      - manifest_revision
+ 
+ # --------------------------------------------------------------------------------------
+ # Stage: build
+ # --------------------------------------------------------------------------------------
+build:
+  stage: build
+  tags:
+    - infrastructure
   rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: always
-  extends:
-    - .skip-for-gitlab-ci-integrations
-  stage: manifest-integration
-  needs:
-    - generate-pipelines
-  trigger:
-    include:
-      - artifact: manifest-integration-pipelines.yml
-        job: generate-pipelines
-    strategy: depend
-
+    # Do not run build if the "skip build" label is set on the merge request
+    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
+      when: never
+    # execute this in MR only and not for integrate/gitlab-ci/ integrations
+    # branches. These are build after the integration has been done in all
+    # projects
+    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
+  script:
+    - echo "Building"
+  #trigger:
+  #  project: !reference [variables, MANIFEST_PROJECT]
+  #  branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+  #  strategy: depend
+# 
+# --------------------------------------------------------------------------------------
+# Stage: check
+# --------------------------------------------------------------------------------------
 check:
-  extends:
-    - .infrastructure
-    - .skip-for-gitlab-ci-integrations
-  stage: manifest-integration
+  extends: .infrastructure
+  stage: check
   rules:
-    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
-  needs:
-    - trigger-pipelines
+    # Do not run check if the "skip build" label is set on the merge request
+    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/
+      when: never
+    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches
+    # The integration is done from the pipeline in gitlab-ci already
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    - if: $CI_MERGE_REQUEST_IID
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+  needs: ["integrate"]
   allow_failure: true
   script:
     - cd ${CI_PROJECT_DIR}
-    # Loop over all integrations and check each integration branch
-    - while read -r integration; do
-        SOURCE_BRANCH=$(echo $integration | cut -d':' -f1);
-        TARGET_PROJECT=$(echo $integration | cut -d':' -f2);
-        TARGET_BRANCH=$(echo $integration | cut -d':' -f3);
-        if [[ "$SOURCE_BRANCH" == "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]]; then
-          .gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py
-            --gitlab-url=${CI_SERVER_URL}
-            --token=${GITBOT_TOKEN}
-            --target-project=${TARGET_PROJECT}
-            --target-branch=${TARGET_BRANCH}
-            --source-project=${CI_PROJECT_PATH}
-            --merge-request=${CI_MERGE_REQUEST_IID}
-            ;
-        fi;
-      done <<< "$INTEGRATION"
-
-yamllint:
-  extends: .yamllint
-  stage: manifest-integration
+    # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
+    # but CI_OPEN_MERGE_REQUESTS. We use  the first of this comma separated list
+    # in this case
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
+    # The 'parent_merge_request' is passed from the trigger
+    # in case this check job is part of a gitlab-ci integration
+    # pipeline. It is only used to display the correct MR to run again
+    # in a failed check
+    - if [ -n "${parent_merge_request}" ];then
+        PARENT_MR="--parent-merge-request=${parent_merge_request}";
+      fi
+    - python3 scripts/check_if_integration_branch_is_up_to_date.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --manifest-project=${MANIFEST_PROJECT}
+        --integration-base=${MASTER_BRANCH_MANIFEST}
+        --project=${CI_PROJECT_PATH}
+        --merge-request=${MERGE_REQUEST}
+        --verbose
+        ${PARENT_MR}
diff --git a/manifest-integration_GF.yml b/manifest-integration_GF.yml
new file mode 100644
index 0000000..9c6204f
--- /dev/null
+++ b/manifest-integration_GF.yml
@@ -0,0 +1,144 @@
+---
+# --------------------------------------------------------------------------------------
+# Global
+# --------------------------------------------------------------------------------------
+include:
+  - local: common.yml
+
+stages:
+  - testing
+  - manifest-integration
+
+variables:
+  MANIFEST_FILE: "distro/seco-distro.xml"
+  # The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
+  # to match the repository and the bitbake recipe name.
+  # We set it here to none, as every project needing it
+  # has to specify it in its own gitlab-ci.yml file.
+  # The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
+  # used for projects referenced in the manifest file.
+  MANIFEST_PROJECT: seco-manifest
+  MASTER_BRANCH_MANIFEST: kirkstone/develop
+  MASTER_BRANCH_PROJECT: main
+  BB_RECIPE_NAME: none
+
+workflow:
+  rules:
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+    # Do not run pipelines on forked projects.
+    # The pipelines would not work anyway because of the users permissions.
+    # There are two cases catched here:
+    # 1. The project is forked into someones gitlab namespace and a MR to
+    #    include a change into this forked project is created. In this case
+    #    is the CI_PROJECT_ROOT_NAMESPACE not seco-ne but the
+    #    namespace the fork lives in.
+    # 2. The MR from the forked project is created to merge the change into this
+    #    the project in the seco-ne namespace (customer sending
+    #    change to us). Here the the IDs used below differ.
+    #
+    - if: $CI_PROJECT_ROOT_NAMESPACE != "seco-ne"
+      when: never
+    - if: $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != $CI_MERGE_REQUEST_PROJECT_ID
+      when: never
+    # FIXME: Unfortunately we cannot use variables in regular expressions due to a
+    # GitLab limitation: https://gitlab.com/gitlab-org/gitlab/-/issues/209904
+    # As soon as this get fixed, use the regex based rules below instead of checking
+    # against the MASTER_BRANCH variable.
+    # Run pipeline if target branch of the merge request has an integration target, i.e.
+    # INTEGRATION contains a line beginning with the target branch followed by a colon.
+    # This also implies that the pipeline runs in merge request context only.
+    # - if: $INTEGRATION =~ /^$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:/m
+    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $MASTER_BRANCH
+    # Run pipeline on target branch after merging a merge request.
+    # - if: $INTEGRATION =~ /^$CI_COMMIT_BRANCH:/m
+    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH
+    - if: $CI_MERGE_REQUEST_IID
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+
+.skip-for-gitlab-ci-integrations:
+  rules:
+    - if: $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
+
+# --------------------------------------------------------------------------------------
+# Manifest integration jobs
+# --------------------------------------------------------------------------------------
+testing:
+  tags:
+    - infrastructure
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+  stage: testing
+  timeout: 10 m
+  image: secodocker/manifest-test:vtemp
+  needs: []
+
+generate-pipelines:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+  extends:
+    - .infrastructure
+    - .skip-for-gitlab-ci-integrations
+  stage: manifest-integration
+  script:
+    # The job generation script implicitly passes the OS environment to the template, so
+    # that the template has access to all GitLab CI variables. Hence there is no need
+    # to explicitly pass any of them as command line arguments.
+    - .gitlab-ci/scripts/generate_job_from_template.py
+        --template=.gitlab-ci/manifest-integration-pipelines.yml.jinja2
+        > manifest-integration-pipelines.yml
+  artifacts:
+    expire_in: 4 weeks
+    paths:
+      - manifest-integration-pipelines.yml
+
+trigger-pipelines:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+  extends:
+    - .skip-for-gitlab-ci-integrations
+  stage: manifest-integration
+  needs:
+    - generate-pipelines
+  trigger:
+    include:
+      - artifact: manifest-integration-pipelines.yml
+        job: generate-pipelines
+    strategy: depend
+
+check:
+  extends:
+    - .infrastructure
+    - .skip-for-gitlab-ci-integrations
+  stage: manifest-integration
+  rules:
+    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
+  needs:
+    - trigger-pipelines
+  allow_failure: true
+  script:
+    - cd ${CI_PROJECT_DIR}
+    # Loop over all integrations and check each integration branch
+    - while read -r integration; do
+        SOURCE_BRANCH=$(echo $integration | cut -d':' -f1);
+        TARGET_PROJECT=$(echo $integration | cut -d':' -f2);
+        TARGET_BRANCH=$(echo $integration | cut -d':' -f3);
+        if [[ "$SOURCE_BRANCH" == "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]]; then
+          .gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py
+            --gitlab-url=${CI_SERVER_URL}
+            --token=${GITBOT_TOKEN}
+            --target-project=${TARGET_PROJECT}
+            --target-branch=${TARGET_BRANCH}
+            --source-project=${CI_PROJECT_PATH}
+            --merge-request=${CI_MERGE_REQUEST_IID}
+            ;
+        fi;
+      done <<< "$INTEGRATION"
+
+yamllint:
+  extends: .yamllint
+  stage: manifest-integration
-- 
GitLab