CI: deploy_gitlab_ci: Rework the deployment of gitlab-ci into the child projects

The deploy_gitlab_ci now creates the integration commit and branch in each passed subproject and create an integration commit in the manifest containing all these new revisions. A build is then triggered on this commit to test the functionality.

Split the update_submodule functions to reuse them in different ways. Remove some previously used files.

BCS 746-000740

.gitlab-ci.yml

@@ -19,7 +19,7 @@ default:
 stages:
   - analyze
   - integrate
-  - check
+  - build
   - merge
 
 workflow:
@@ -62,7 +62,6 @@ yamllint:
 # ---------------------------------------------------------------------------------------
 # Stage: integrate
 # ---------------------------------------------------------------------------------------
-
 .ci-test-projects:
   variables:
     PROJECT_ROOT:
@@ -73,6 +72,7 @@ yamllint:
     INTEGRATE_INTO:
       ${PROJECT_ROOT}/minimal-foo
       ${PROJECT_ROOT}/minimal-bar
+      ${PROJECT_ROOT}/minimal-srcrev
 
 .yocto-projects:
   variables:
@@ -104,6 +104,8 @@ yamllint:
     - if: $CI_MERGE_REQUEST_IID
       when: manual
       allow_failure: true
+  variables:
+    MERGE: ""
   script:
     - cd ${CI_PROJECT_DIR}
     - ./deploy_gitlab_ci.py
@@ -113,22 +115,9 @@ yamllint:
       --submodule=.gitlab-ci
       --revision=${CI_COMMIT_SHA}
       --verbose
+      ${MERGE}
       ${INTEGRATE_INTO}
 
-    - ./generate_job_from_template.py
-      --template=gitlab-ci-integration.jinja2
-      --image=${CI_IMAGE_PYTHON}
-      --branch="integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
-      --manifest-project=${MANIFEST_PROJECT}
-      --parent_merge_request="${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
-      --verbose
-      ${INTEGRATE_INTO}
-      > integration.yml
-    - cat integration.yml
-  artifacts:
-    paths:
-      - integration.yml
-
 integrate-yocto:
   extends:
     - .integrate
@@ -139,47 +128,47 @@ integrate-ci-test:
     - .integrate
     - .ci-test-projects
 
-trigger-yocto:
-  stage: integrate
+# --------------------------------------------------------------------------------------
+# Stage: build
+# --------------------------------------------------------------------------------------
+build-yocto:
+  stage: build
+  needs: [integrate-yocto]
   rules:
     - if: $CI_MERGE_REQUEST_IID
       allow_failure: true
-  needs: [integrate-yocto]
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: manual
   trigger:
-    include:
-      - artifact: integration.yml
-        job: integrate-yocto
+    project: SECO-Northern-Europe/yocto/manifest
+    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
     strategy: depend
 
-trigger-ci-test:
-  stage: integrate
+build-ci-test:
+  stage: build
+  needs: [integrate-ci-test]
   rules:
     - if: $CI_MERGE_REQUEST_IID
-  needs: [integrate-ci-test]
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: manual
   trigger:
-    include:
-      - artifact: integration.yml
-        job: integrate-ci-test
+    project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
+    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
     strategy: depend
 
 # --------------------------------------------------------------------------------------
 # Stage: merge
 # --------------------------------------------------------------------------------------
 .merge:
+  extends: .integrate
   stage: merge
   rules:
     - if: $CI_COMMIT_BRANCH == "master"
       when: manual
       allow_failure: true
-  script:
-    - cd ${CI_PROJECT_DIR}
-    - ./merge_gitlab_ci.py
-      --gitlab-url=${CI_SERVER_URL}
-      --token=${GITBOT_TOKEN}
-      --manifest-project=${MANIFEST_PROJECT}
-      --submodule=.gitlab-ci
-      --revision=${CI_COMMIT_SHA}
-      ${INTEGRATE_INTO}
+  variables:
+    MERGE: --merge
 
 merge-ci-test:
   extends:
@@ -190,40 +179,3 @@ merge-yocto:
   extends:
     - .merge
     - .yocto-projects
-
-# --------------------------------------------------------------------------------------
-# Stage: check
-# --------------------------------------------------------------------------------------
-check:
-  stage: check
-  needs: [integrate-yocto, integrate-ci-test]
-  rules:
-    # Probably this job gets removed
-    - when: never
-    - if: $CI_MERGE_REQUEST_IID
-  tags:
-    - infrastructure
-  timeout: 2m
-  script:
-    - cd ${CI_PROJECT_DIR}
-    - MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-
-    - MASTER_BRANCH=dunfell
-    - ./check_if_integration_branch_is_up_to_date.py
-      --gitlab-url=${CI_SERVER_URL}
-      --token=${GITBOT_TOKEN}
-      --manifest-project=${CI_PROJECT_ROOT_NAMESPACE}/yocto/manifest
-      --integration-base=${MASTER_BRANCH}
-      --project=${CI_PROJECT_PATH}
-      --merge-request=${CI_MERGE_REQUEST_IID}
-
-    # The check is done for both manifests in one job as the retrigger
-    # looks for jobs named check, though there can only be one
-    - ./check_if_integration_branch_is_up_to_date.py
-        --gitlab-url=${CI_SERVER_URL}
-        --token=${GITBOT_TOKEN}
-        --manifest-project
-          ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test/minimal-manifest
-        --integration-base=master
-        --project=${CI_PROJECT_PATH}
-        --merge-request=${CI_MERGE_REQUEST_IID}