Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • add-lava-testreposrting
  • add-pydoc-config
  • all-src-revs
  • build-documentation-in-dedicated-pipeline
  • build-seco-image-after-fngsystem
  • change-poky-entry-group-id
  • changelog-generator-all-projects
  • debug_credentials
  • enhance-gitlab-backup
  • generate-package-add-name
  • master
  • seco-santvent-fsl
  • skip-deploy-if-target-not-set
  • store-build-variables
  • test_repo_verbose
  • troupbleshoot-qtkuktest-fail-due-privat-key
  • yt-65-proxy-solution-for-repo-checkout
  • yt-65-troubleshoot-meta-freescale
  • yt-93-improve-release-workflow-too-many-buttons-click-mock-test
  • yt-xxx-ci-test-build-fails
  • yt-xxx-ci-test-build-fails-1
21 results

Target

Select target project
No results found
Select Git revision
  • add-lava-testreposrting
  • add-pydoc-config
  • all-src-revs
  • build-documentation-in-dedicated-pipeline
  • build-seco-image-after-fngsystem
  • change-poky-entry-group-id
  • changelog-generator-all-projects
  • debug_credentials
  • enhance-gitlab-backup
  • generate-package-add-name
  • master
  • seco-santvent-fsl
  • skip-deploy-if-target-not-set
  • store-build-variables
  • test_repo_verbose
  • troupbleshoot-qtkuktest-fail-due-privat-key
  • yt-65-proxy-solution-for-repo-checkout
  • yt-65-troubleshoot-meta-freescale
  • yt-93-improve-release-workflow-too-many-buttons-click-mock-test
  • yt-xxx-ci-test-build-fails
  • yt-xxx-ci-test-build-fails-1
21 results
Show changes

Commits on Source 23

26 files
+ 1071
− 560
Compare changes
  • Side-by-side
  • Inline

Files

.gitignore

+1 −1
Original line number Diff line number Diff line 
/__pycache__

scripts/__pycache__

.idea/

.gitlab-ci.yml

+121 −79
Original line number Diff line number Diff line 
---

# ---------------------------------------------------------------------------------------

# Global

# ---------------------------------------------------------------------------------------

---



variables:

  # CI_IMAGES_BASEPATH: Environment variable configure in gitlab

  CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images

  CI_IMAGES_REVISION: 44965ccdd847f1e077670f49d546047f8ad0110c

  CI_IMAGES_REVISION: 5dfddb02d67bbd16beb09ff4e31afcd5380f5788

  CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REVISION}"

  CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REVISION}"
@@ -65,45 +65,6 @@ yamllint: 
# ---------------------------------------------------------------------------------------

# Stage: integrate

# ---------------------------------------------------------------------------------------

.ci-test-projects:

  variables:

    PROJECT_ROOT:

      ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test

    MANIFEST_PROJECT:

      ${PROJECT_ROOT}/minimal-manifest



    INTEGRATE_INTO:

      ${PROJECT_ROOT}/minimal-foo

      ${PROJECT_ROOT}/minimal-bar

      ${PROJECT_ROOT}/minimal-srcrev



.yocto-projects:

  variables:

    PROJECT_ROOT:

      ${CI_PROJECT_ROOT_NAMESPACE}

    MANIFEST_PROJECT:

      ${PROJECT_ROOT}/yocto/manifest

    INTEGRATE_INTO:

      ${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk

      ${PROJECT_ROOT}/kernel/linux-guf

      ${PROJECT_ROOT}/kernel/linux-imx-kuk

      ${PROJECT_ROOT}/kernel/modules/egalaxi2c

      ${PROJECT_ROOT}/kernel/modules/gfplatdetect

      ${PROJECT_ROOT}/tools/gf-emc-test-suite

      ${PROJECT_ROOT}/tools/gf-productiontests

      ${PROJECT_ROOT}/tools/gfeeprom

      ${PROJECT_ROOT}/tools/gfxml2dto

      ${PROJECT_ROOT}/tools/guf-show-demo

      ${PROJECT_ROOT}/tools/libmdb

      ${PROJECT_ROOT}/tools/touchcal-conv

      ${PROJECT_ROOT}/tools/xconfig

      ${PROJECT_ROOT}/tools/qt-multi-screen-compositor

      ${PROJECT_ROOT}/yocto/config

      ${PROJECT_ROOT}/yocto/custom/dual-espresso/meta-seconorth-dual-espresso

      ${PROJECT_ROOT}/yocto/layers/meta-seconorth-distro

      ${PROJECT_ROOT}/yocto/layers/meta-seconorth-machine

      ${PROJECT_ROOT}/yocto/layers/meta-seconorth-nogplv3



.integrate:

  stage: integrate

  rules:
@@ -118,91 +79,172 @@ yamllint: 
      --gitlab-url=${CI_SERVER_URL}

      --token=${GITBOT_TOKEN}

      --manifest-project=${MANIFEST_PROJECT}

      --manifest-branch=${MANIFEST_BRANCH}

      --submodule=.gitlab-ci

      --revision=${CI_COMMIT_SHA}

      --verbose

      --group=${PROJECT_GROUP}

      ${MERGE}

      ${INTEGRATE_INTO}



integrate-yocto:

  extends:

    - .integrate

    - .yocto-projects

# Running multiple integration jobs for the same manifest in parallel can lead to race

# conditions if there are projects integrating a single branch to different manifest

# branches. Use resource groups to force execution one by one.

.integrate-ci-test:

  extends: .integrate

  resource_group: integrate-ci-test

  variables:

    PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test

    MANIFEST_PROJECT: ${PROJECT_GROUP}/minimal-manifest



.integrate-yocto:

  extends: .integrate

  resource_group: integrate-yocto

  variables:

    PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}

    MANIFEST_PROJECT: ${PROJECT_GROUP}/yocto/manifest



# Jobs



integrate-ci-test:primary:

  extends: .integrate-ci-test

  variables:

    MANIFEST_BRANCH: primary



integrate-ci-test:secondary:

  extends: .integrate-ci-test

  variables:

    MANIFEST_BRANCH: secondary



integrate-yocto:dunfell:

  extends: .integrate-yocto

  variables:

    MANIFEST_BRANCH: dunfell



integrate-yocto:kirkstone:

  extends: .integrate-yocto

  variables:

    MANIFEST_BRANCH: kirkstone



integrate-ci-test:

  extends:

    - .integrate

    - .ci-test-projects



# --------------------------------------------------------------------------------------

# Stage: build

# --------------------------------------------------------------------------------------

build-yocto:

.build:

  stage: build

  needs: [integrate-yocto]

  rules:

    - if: $CI_MERGE_REQUEST_IID

      allow_failure: true



.build-ci-test:

  extends: .build

  trigger:

    project: seco-ne/yocto/manifest

    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"

    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest

    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"

    strategy: depend



build-ci-test:

  stage: build

  needs: [integrate-ci-test]

  rules:

    - if: $CI_MERGE_REQUEST_IID

      allow_failure: true

.build-yocto:

  extends: .build

  trigger:

    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest

    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"

    project: seco-ne/yocto/manifest

    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"

    strategy: depend



# Jobs



build-ci-test:primary:

  extends: .build-ci-test

  needs: ["integrate-ci-test:primary"]

  variables:

    MANIFEST_BRANCH: primary



build-ci-test:secondary:

  extends: .build-ci-test

  needs: ["integrate-ci-test:secondary"]

  variables:

    MANIFEST_BRANCH: secondary



build-yocto:dunfell:

  extends: .build-yocto

  needs: ["integrate-yocto:dunfell"]

  variables:

    MANIFEST_BRANCH: dunfell



build-yocto:kirkstone:

  extends: .build-yocto

  needs: ["integrate-yocto:kirkstone"]

  variables:

    MANIFEST_BRANCH: kirkstone





# --------------------------------------------------------------------------------------

# Stage: merge

# --------------------------------------------------------------------------------------

.merge:

  extends: .integrate

  stage: merge

  rules:

    - if: $CI_COMMIT_BRANCH == "master"

      when: manual

      allow_failure: true

  variables:

    MERGE: --merge

    MERGE: --merge --project=${CI_PROJECT_PATH} --branch=${CI_COMMIT_REF_NAME}



merge-ci-test:

  extends:

    - .integrate-ci-test

    - .merge

    - .ci-test-projects

  variables:

    MANIFEST_BRANCH: primary,secondary



merge-yocto:

  extends:

    - .integrate-yocto

    - .merge

    - .yocto-projects

  variables:

    MANIFEST_BRANCH: dunfell,kirkstone





# --------------------------------------------------------------------------------------

# Stage: build

# --------------------------------------------------------------------------------------

build-master-yocto:

.build-master:

  stage: build

  needs: [merge-yocto]

  rules:

    - if: $CI_COMMIT_BRANCH == "master"

      when: manual



.build-master-ci-test:

  extends: .build-master

  needs: ["merge-ci-test"]

  trigger:

    project: seco-ne/yocto/manifest

    branch: "dunfell"

    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest

    branch: "${MANIFEST_BRANCH}"

    strategy: depend



build-master-ci-test:

  stage: build

  needs: [merge-ci-test]

  rules:

    - if: $CI_COMMIT_BRANCH == "master"

      when: manual

.build-master-yocto:

  extends: .build-master

  needs: ["merge-yocto"]

  trigger:

    project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest

    branch: "master"

    project: seco-ne/yocto/manifest

    branch: "${MANIFEST_BRANCH}"

    strategy: depend



# Jobs



build-master-ci-test:primary:

  extends: .build-master-ci-test

  variables:

    MANIFEST_BRANCH: primary



build-master-ci-test:secondary:

  extends: .build-master-ci-test

  variables:

    MANIFEST_BRANCH: secondary



build-master-yocto:dunfell:

  extends: .build-master-yocto

  variables:

    MANIFEST_BRANCH: dunfell



build-master-yocto:kirkstone:

  extends: .build-master-yocto

  variables:

    MANIFEST_BRANCH: kirkstone

build-common.yml

+1 −1
Original line number Diff line number Diff line
@@ -84,7 +84,7 @@ workflow: 
    GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"

    GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"

    TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"

    TEST_REPO_BRANCH: ${MASTER_BRANCH_MANIFEST}

    TEST_REPO_BRANCH: ${MASTER_BRANCH}

  artifacts:

    when: always

    paths:

build-jobs-yocto.jinja2

+2 −2
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ stages: 
  - uploadftp



variables:

  MASTER_BRANCH_MANIFEST: {{ MASTER_BRANCH_MANIFEST }}

  MASTER_BRANCH: {{ MASTER_BRANCH }}



# --------------------------------------------------------------------------------------

# Stage: infrastructure
@@ -30,7 +30,7 @@ changelog: 
  extends: .infrastructure

  script: .gitlab-ci/scripts/changelog_generator.py

              --token=${GITBOT_TOKEN}

              --branch=${MASTER_BRANCH_MANIFEST}

              --branch=${MASTER_BRANCH}

              > changelog.md

  artifacts:

    expire_in: 4 weeks

common.yml

+2 −2
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ 
variables:

  # CI_IMAGES_BASEPATH: Environment variable configure in gitlab

  CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images

  CI_IMAGES_REV: 44965ccdd847f1e077670f49d546047f8ad0110c

  CI_IMAGES_REV: e6a404c3e1a919f54feb83f1bc061ce171220437

  CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REV}"

  CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REV}"

  # Include git submodules
@@ -36,7 +36,7 @@ variables: 
    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"

      when: never

    # Usually run this job only on non-master branches, i.e. in merge requests

    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST

    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH

    # Additionally run this job if pipeline was triggered by the API, which happens e.g.

    # when no pipeline has run in a merge request, so a full build pipeline is needed on

    # the master

docs/automatic-manifest-integration.md

+0 −1
Original line number Diff line number Diff line
@@ -188,7 +188,6 @@ the git revision in the `SRCREV.conf` file on project changes. 


2. Add an approriate `.gitlab-ci.yml` file. For example, copy it from the

   repository [egalxi2c][8]. Modify the following variables in the file:

   * `MASTER_BRANCH_PROJECT`: Set the project master branch

   * `BB_RECIPE_NAME`: Set the name of the bitbake recipe



3. Create a corresponding entry in the `SRCREV.conf` file of the manifest repo:\

manifest-integration-ci-test.yml

deleted100644 → 0
+0 −15
Original line number Diff line number Diff line 
---

include:

  - local: manifest-integration.yml



variables:

  # FIXME: This variable is used in the manifest-integration's build stage for the

  # trigger project. Due to a missing feature in GitLab, we cannot use any variables

  # here and have to hard-code 'seco-ne' instead of using CI_PROJECT_ROOT_NAMESPACE.

  # (https://gitlab.com/gitlab-org/gitlab/-/issues/249583)

  MANIFEST_PROJECT: seco-ne/yocto/infrastructure/ci-test/minimal-manifest



  # The master branch is hardcoded here, because it cannot be determined automatically.

  # Has to be modified for new branches, e.g. new Yocto versions or fix releases.

  MASTER_BRANCH_MANIFEST: master

  MASTER_BRANCH_PROJECT: master

manifest-integration-jobs.yml.jinja2

0 → 100644
+131 −0
Original line number Diff line number Diff line 
---

# --------------------------------------------------------------------------------------

# Global

# --------------------------------------------------------------------------------------

include:

  # FIXME: see FIXME comments in manifest-integration-pipelines.yml.jinja2

  # - project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci'

  #   ref: ${GITLAB_CI_REVISION}

  - project: {{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci

    ref: {{ GITLAB_CI_REVISION }}

    file: common.yml



workflow:

  rules:

    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"



stages:

  - infrastructure



# --------------------------------------------------------------------------------------

# Stage: infrastructure

# --------------------------------------------------------------------------------------

integrate:

  extends: .infrastructure

  rules:

    - if: $CI_MERGE_REQUEST_IID

  cache:

    policy: push

  script:

    - cd ${CI_PROJECT_DIR}

    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then

        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";

      else

        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";

      fi

    - .gitlab-ci/scripts/integrate_into_manifest.py

        --gitlab-url=${CI_SERVER_URL}

        --token=${GITBOT_TOKEN}

        --manifest-project=${TARGET_PROJECT}

        --manifest-file=${MANIFEST_FILE}

        --manifest-branch=${TARGET_BRANCH}

        --project=${CI_PROJECT_PATH}

        --merge-request=${MERGE_REQUEST}

        --save-revision-to=manifest_revision

        --recipe-name=${BB_RECIPE_NAME}

  artifacts:

    paths:

      - manifest_revision



yamllint:

  extends: .yamllint



# --------------------------------------------------------------------------------------

# Stage: merge

# --------------------------------------------------------------------------------------

merge:

  extends: .infrastructure

  rules:

    - if: $CI_COMMIT_BRANCH == $SOURCE_BRANCH

  script:

    - cd ${CI_PROJECT_DIR}

    - .gitlab-ci/scripts/merge_into_manifest.py

        --gitlab-url=${CI_SERVER_URL}

        --token=${GITBOT_TOKEN}

        --manifest-project=${TARGET_PROJECT}

        --manifest-branch=${TARGET_BRANCH}

        --project=${CI_PROJECT_PATH}

        --project-branch=${SOURCE_BRANCH}

        --commit=${CI_COMMIT_SHA}

        --save-revision-to=manifest_revision

        --recipe-name=${BB_RECIPE_NAME}

  artifacts:

    paths:

      - manifest_revision



# --------------------------------------------------------------------------------------

# Stage: build

# --------------------------------------------------------------------------------------

build:

  stage: infrastructure

  needs: ["integrate"]

  rules:

    # Do not run build if the "skip build" label is set on the merge request

    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/

      when: never

    - if: $CI_MERGE_REQUEST_IID

  trigger:

    # FIXME: see FIXME comments in manifest-integration-pipelines.yml.jinja2

    # project: $TARGET_PROJECT

    project: {{ TARGET_PROJECT }}

    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${TARGET_BRANCH}"

    strategy: depend



# --------------------------------------------------------------------------------------

# Stage: check

# --------------------------------------------------------------------------------------

check:

  extends: .infrastructure

  rules:

    # Do not run check if the "skip build" label is set on the merge request

    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/

      when: never

    - if: $CI_MERGE_REQUEST_IID

  needs: ["integrate"]

  allow_failure: true

  script:

    - cd ${CI_PROJECT_DIR}

    # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set

    # but CI_OPEN_MERGE_REQUESTS. We use  the first of this comma separated list

    # in this case

    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then

        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";

      else

        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";

      fi

    # The 'parent_merge_request' is passed from the trigger

    # in case this check job is part of a gitlab-ci integration

    # pipeline. It is only used to display the correct MR to run again

    # in a failed check

    - if [ -n "${parent_merge_request}" ];then

        PARENT_MR="--parent-merge-request=${parent_merge_request}";

      fi

    - .gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py

        --gitlab-url=${CI_SERVER_URL}

        --token=${GITBOT_TOKEN}

        --manifest-project=${TARGET_PROJECT}

        --integration-base=${TARGET_BRANCH}

        --project=${CI_PROJECT_PATH}

        --merge-request=${MERGE_REQUEST}

        --verbose

        ${PARENT_MR}

manifest-integration-pipelines.yml.jinja2

0 → 100644
+75 −0
Original line number Diff line number Diff line 
---

# --------------------------------------------------------------------------------------

# Global

# --------------------------------------------------------------------------------------

include:

  - project: {{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci

    ref: {{ GITLAB_CI_REVISION }}

    file: common.yml



workflow:

  rules:

    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"



stages:

  - infrastructure



# --------------------------------------------------------------------------------------

# Generate job

# --------------------------------------------------------------------------------------

# Use one single job to generate multiple yaml files for the downstream pipelines.

# FIXME: This is only necessary due to a GitLab limitation:

# https://gitlab.com/gitlab-org/gitlab/-/issues/347469

# We work around this by generating manifest-integration-jobs.yml from a Jinja2 template

# and insert the trigger project via a Jinja2 variable.

# The issue is already fixed and will be released in GitLab 15.3:

# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92346

# As soon as we update to this version, we can get rid of the generate job and convert

# the Jinja2 template to a simple YAML file.

generate:

  extends:

    - .infrastructure

  script:

    # The job generation script implicitly passes the OS environment to the template, so

    # that the template has access to all GitLab CI variables. Hence there is no need

    # to explicitly pass any of them as command line arguments.

{% for integration in INTEGRATION.split('\n') %}

{% set SOURCE_BRANCH, TARGET_PROJECT, TARGET_BRANCH = integration.split(':') %}

    - TARGET_PROJECT={{ TARGET_PROJECT }}

        .gitlab-ci/scripts/generate_job_from_template.py

        --template=.gitlab-ci/manifest-integration-jobs.yml.jinja2

        > manifest-integration-jobs-{{ loop.index }}.yml

{% endfor %}

  artifacts:

    expire_in: 4 weeks

    paths:

      - manifest-integration-jobs-*.yml



# --------------------------------------------------------------------------------------

# Trigger jobs

# --------------------------------------------------------------------------------------

{% for integration in INTEGRATION.split('\n') %}

{% set SOURCE_BRANCH, TARGET_PROJECT, TARGET_BRANCH = integration.split(':') %}



{{ TARGET_PROJECT }}:{{ TARGET_BRANCH }}:

  stage: infrastructure

  needs:

    - generate

  variables:

    SOURCE_BRANCH: {{ SOURCE_BRANCH }}

    TARGET_PROJECT: {{ TARGET_PROJECT }}

    TARGET_BRANCH: {{ TARGET_BRANCH }}

    GITLAB_CI_REVISION: {{ GITLAB_CI_REVISION }}

    MANIFEST_FILE: {{ MANIFEST_FILE }}

    BB_RECIPE_NAME: {{ BB_RECIPE_NAME }}

  trigger:

    include:

      # FIXME: Use these settings after switching from jinja2 to yaml (see above)

      # project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci'

      # ref: ${GITLAB_CI_REVISION}

      # file: manifest-integration-jobs.yml

      artifact: manifest-integration-jobs-{{ loop.index }}.yml

      job: generate

    strategy: depend



{% endfor %}

manifest-integration-yocto.yml

deleted100644 → 0
+0 −15
Original line number Diff line number Diff line 
---

include:

  - local: manifest-integration.yml



variables:

  # FIXME: This variable is used in the manifest-integration's build stage for the

  # trigger project. Due to a missing feature in GitLab, we cannot use any variables

  # here and have to hard-code 'seco-ne' instead of using CI_PROJECT_ROOT_NAMESPACE.

  # (https://gitlab.com/gitlab-org/gitlab/-/issues/249583)

  MANIFEST_PROJECT: seco-ne/yocto/manifest



  # The master branch is hardcoded here, because it cannot be determined automatically.

  # Has to be modified for new branches, e.g. new Yocto versions or fix releases.

  MASTER_BRANCH_MANIFEST: dunfell

  MASTER_BRANCH_PROJECT: dunfell

manifest-integration.yml

+43 −123
Original line number Diff line number Diff line
@@ -7,10 +7,6 @@ include: 


stages:

  - infrastructure

  - integrate

  - merge

  - build

  - check



variables:

  MANIFEST_FILE: "default.xml"
@@ -37,135 +33,59 @@ workflow: 
    #    the project in the seco-ne namespace (customer sending

    #    change to us). Here the the IDs used below differ.

    #

    - if: $CI_PROJECT_ROOT_NAMESPACE == "seco-ne"

        && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID



# --------------------------------------------------------------------------------------

# Stage: infrastructure

# --------------------------------------------------------------------------------------

integrate:

  extends: .infrastructure

  rules:

    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches

    # The integration is done from the pipeline in gitlab-ci already

    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/

    - if: $CI_PROJECT_ROOT_NAMESPACE != "seco-ne"

      when: never

    # We have to make sure that the pipeline runs for the current manifest

    # master at the time a merge request is created. Otherwise we cannot

    # guarantee a green master after merging.

    - if: $CI_MERGE_REQUEST_IID

    # Explicitly allow externally triggered pipelines in every case

    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"

  cache:

    policy: push

  script:

    - cd ${CI_PROJECT_DIR}

    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then

        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";

      else

        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";

      fi

    - .gitlab-ci/scripts/integrate_into_manifest.py

        --gitlab-url=${CI_SERVER_URL}

        --token=${GITBOT_TOKEN}

        --manifest-project=${MANIFEST_PROJECT}

        --manifest-file=${MANIFEST_FILE}

        --integration-base=${MASTER_BRANCH_MANIFEST}

        --project=${CI_PROJECT_PATH}

        --merge-request=${MERGE_REQUEST}

        --save-revision-to=manifest_revision

        --recipe-name=${BB_RECIPE_NAME}

  artifacts:

    paths:

      - manifest_revision

    - if: $CI_MERGE_REQUEST_SOURCE_PROJECT_ID != $CI_MERGE_REQUEST_PROJECT_ID

      when: never

    # FIXME: Unfortunately we cannot use variables in regular expressions due to a

    # GitLab limitation: https://gitlab.com/gitlab-org/gitlab/-/issues/209904

    # As soon as this get fixed, use the regex based rules below instead of checking

    # against the MASTER_BRANCH variable.

    # Run pipeline if target branch of the merge request has an integration target, i.e.

    # INTEGRATION contains a line beginning with the target branch followed by a colon.

    # This also implies that the pipeline runs in merge request context only.

    # - if: $INTEGRATION =~ /^$CI_MERGE_REQUEST_TARGET_BRANCH_NAME:/m

    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $MASTER_BRANCH

    # Run pipeline on target branch after merging a merge request.

    # - if: $INTEGRATION =~ /^$CI_COMMIT_BRANCH:/m

    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH



yamllint:

  extends: .yamllint



# --------------------------------------------------------------------------------------

# Stage: merge

# Stage: infrastructure

# --------------------------------------------------------------------------------------

merge:

  extends: .infrastructure

  stage: merge

.skip-for-gitlab-ci-integrations:

  rules:

    - if: $CI_COMMIT_BRANCH == $MASTER_BRANCH_PROJECT

    - if: $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/



generate-pipelines:

  extends:

    - .infrastructure

    - .skip-for-gitlab-ci-integrations

  script:

    - cd ${CI_PROJECT_DIR}

    - .gitlab-ci/scripts/merge_into_manifest.py

        --gitlab-url=${CI_SERVER_URL}

        --token=${GITBOT_TOKEN}

        --manifest-project=${MANIFEST_PROJECT}

        --master-branch=${MASTER_BRANCH_MANIFEST}

        --project=${CI_PROJECT_PATH}

        --master-branch-project=${MASTER_BRANCH_PROJECT}

        --commit=${CI_COMMIT_SHA}

        --save-revision-to=manifest_revision

        --recipe-name=${BB_RECIPE_NAME}

    # The job generation script implicitly passes the OS environment to the template, so

    # that the template has access to all GitLab CI variables. Hence there is no need

    # to explicitly pass any of them as command line arguments.

    - .gitlab-ci/scripts/generate_job_from_template.py

        --template=.gitlab-ci/manifest-integration-pipelines.yml.jinja2

        > manifest-integration-pipelines.yml

  artifacts:

    expire_in: 4 weeks

    paths:

      - manifest_revision

      - manifest-integration-pipelines.yml



# --------------------------------------------------------------------------------------

# Stage: build

# --------------------------------------------------------------------------------------

build:

  stage: build

  rules:

    # Do not run build if the "skip build" label is set on the merge request

    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/

      when: never

    # execute this in MR only and not for integrate/gitlab-ci/ integrations

    # branches. These are build after the integration has been done in all

    # projects

    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/

trigger-pipelines:

  extends:

    - .skip-for-gitlab-ci-integrations

  stage: infrastructure

  needs:

    - generate-pipelines

  trigger:

    project: !reference [variables, MANIFEST_PROJECT]

    branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"

    include:

      - artifact: manifest-integration-pipelines.yml

        job: generate-pipelines

    strategy: depend



# --------------------------------------------------------------------------------------

# Stage: check

# --------------------------------------------------------------------------------------

check:

  extends: .infrastructure

  stage: check

  rules:

    # Do not run check if the "skip build" label is set on the merge request

    - if: $CI_MERGE_REQUEST_LABELS =~ /skip build/

      when: never

    # Do not integration pipeline for merge requests for integrate/gitlab-ci/ branches

    # The integration is done from the pipeline in gitlab-ci already

    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/

      when: never

    - if: $CI_MERGE_REQUEST_IID

    # Explicitly allow externally triggered pipelines in every case

    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"

  needs: ["integrate"]

  allow_failure: true

  script:

    - cd ${CI_PROJECT_DIR}

    # When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set

    # but CI_OPEN_MERGE_REQUESTS. We use  the first of this comma separated list

    # in this case

    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then

        MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";

      else

        MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";

      fi

    # The 'parent_merge_request' is passed from the trigger

    # in case this check job is part of a gitlab-ci integration

    # pipeline. It is only used to display the correct MR to run again

    # in a failed check

    - if [ -n "${parent_merge_request}" ];then

        PARENT_MR="--parent-merge-request=${parent_merge_request}";

      fi

    - .gitlab-ci/scripts/check_if_integration_branch_is_up_to_date.py

        --gitlab-url=${CI_SERVER_URL}

        --token=${GITBOT_TOKEN}

        --manifest-project=${MANIFEST_PROJECT}

        --integration-base=${MASTER_BRANCH_MANIFEST}

        --project=${CI_PROJECT_PATH}

        --merge-request=${MERGE_REQUEST}

        --verbose

        ${PARENT_MR}
 
yamllint:

  extends:

    - .yamllint

manifest-pipeline-ci-test.yml

+4 −2
Original line number Diff line number Diff line
@@ -13,6 +13,9 @@ variables: 
  # named differently, so we need a variable that may be overriden.

  CI_PARAM_SECO_REMOTE: ci-test



  # GitLab group to search for projects to retrigger

  RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test



  BUILD_TIMEOUT: 2m



  # This is the jinja2 template file used to generate the build jobs
@@ -20,5 +23,4 @@ variables: 


  # The master branch is hardcoded here, because it cannot be determined automatically.

  # Has to be modified for new branches, e.g. new Yocto versions or fix releases.

  MASTER_BRANCH_MANIFEST: master

  MASTER_BRANCH_PROJECT: master
 
  MASTER_BRANCH: master

manifest-pipeline-yocto.yml

+3 −0
Original line number Diff line number Diff line
@@ -13,6 +13,9 @@ variables: 
  # named differently, so we need a variable that may be overriden.

  CI_PARAM_SECO_REMOTE: seco-ne



  # GitLab group to search for projects to retrigger

  RETRIGGER_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}



  BUILD_TIMEOUT: 1h



  # This is the jinja2 template file used to generate the build jobs

manifest-pipeline.yml

+10 −25
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ workflow: 
  rules:

    # Run the full build pipeline on non-master branches (i.e. in the merge request)

    # or if explicitly triggered by the API or the web button.

    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST

    - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH

    - if: $CI_PIPELINE_SOURCE == "api"

    - if: $CI_PIPELINE_SOURCE == "pipeline"

    - if: $CI_PIPELINE_SOURCE == "web"
@@ -49,7 +49,7 @@ workflow: 
      when: never

    - if: $CI_PIPELINE_SOURCE == "web"

      when: never

    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST

    - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH



# --------------------------------------------------------------------------------------

# Stage: retrigger
@@ -60,27 +60,12 @@ retrigger: 
    - .short_master_pipeline

  stage: retrigger

  script:

    - PROJECTS=$(

        .gitlab-ci/scripts/get_manifest_projects.py

        --manifest=default.xml

        --remote=${CI_PARAM_SECO_REMOTE}

        --concat-namespaces

      )

    # Add the gitlab-ci project

    - PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci"

    # TODO retrigger gitlab-ci integration also

    # Retrigger also project in SRCREV

    - echo -e "Projects:\n${PROJECTS}"

    - for PROJECT in ${PROJECTS}; do

        .gitlab-ci/scripts/retrigger_mr_pipeline_jobs.py

    - .gitlab-ci/scripts/retrigger_integrating_projects.py

        --gitlab-url=${CI_SERVER_URL}

        --token=${GITBOT_TOKEN}

          --project=${PROJECT}

          --state=opened

          --target-branch=${MASTER_BRANCH_MANIFEST}

          --job=check

        ;

      done

        --manifest-project=${CI_PROJECT_PATH}

        --manifest-branch=${MASTER_BRANCH}

        --group=${RETRIGGER_GROUP}



# --------------------------------------------------------------------------------------

# Stage: infrastructure
@@ -137,7 +122,7 @@ build:merge_request: 
        --token=${GITBOT_TOKEN}

        --project=${CI_PROJECT_PATH}

        --commit=${CI_COMMIT_SHA}

        --ref=^${MASTER_BRANCH_MANIFEST} || true | head -1)

        --ref=^${MASTER_BRANCH} || true | head -1)

    # If pipeline exists, mirror its result

    - if [ ! -z "${MR_PIPELINE}" ]; then

        .gitlab-ci/scripts/mirror_pipeline_result.py
@@ -151,5 +136,5 @@ build:merge_request: 
          --gitlab-url=${CI_SERVER_URL}

          --token=${GITBOT_TOKEN}

          --project=${CI_PROJECT_PATH}

          --ref=${MASTER_BRANCH_MANIFEST}

          --ref=${MASTER_BRANCH}

    - fi

scripts/changelog_generator.py

+1 −1
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ GITLAB_SERVER = "https://git.seco.com" 
GITLAB_GROUP_ID = "556"



DISTRO_PROJECT_ID = "1748"

MACHINE_PROJECT_ID = "1747"

MACHINE_PROJECT_ID = "2074"

MANIFEST_PROJECT_ID = "1725"



DEFAULTBRANCH = "dunfell"

scripts/check_if_integration_branch_is_up_to_date.py

+1 −1
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ def check_if_integration_branch_is_up_to_date( 
            pass

    if integration_branch is None:

        integration_branch_name = common.integration_branch_name(

            project.name, merge_request.source_branch

            project.name, merge_request.source_branch, integration_base

        )

        branch_list.append(integration_branch)

        try:

scripts/colors.py

0 → 100755
+47 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3

# Taken from here: https://stackoverflow.com/a/26445590/3018229

class colors:

    """Colors class:

    Reset all colors with colors.reset

    Two subclasses fg for foreground and bg for background.

    Use as colors.subclass.colorname.

    i.e. colors.fg.red or colors.bg.green

    Also, the generic bold, disable, underline, reverse, strikethrough,

    and invisible work with the main class

    i.e. colors.bold

    """



    reset = "\033[0m"

    bold = "\033[01m"

    dim = "\033[02m"

    underline = "\033[04m"

    reverse = "\033[07m"

    strikethrough = "\033[09m"

    invisible = "\033[08m"



    class fg:

        black = "\033[30m"

        red = "\033[31m"

        green = "\033[32m"

        orange = "\033[33m"

        blue = "\033[34m"

        purple = "\033[35m"

        cyan = "\033[36m"

        lightgrey = "\033[37m"

        darkgrey = "\033[90m"

        lightred = "\033[91m"

        lightgreen = "\033[92m"

        yellow = "\033[93m"

        lightblue = "\033[94m"

        pink = "\033[95m"

        lightcyan = "\033[96m"



    class bg:

        black = "\033[40m"

        red = "\033[41m"

        green = "\033[42m"

        orange = "\033[43m"

        blue = "\033[44m"

        purple = "\033[45m"

        cyan = "\033[46m"

        lightgrey = "\033[47m"

scripts/common.py

+28 −8
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ import logging 
import requests

import sys

import time

from colors import colors

from furl import furl

from git import Actor, GitCommandError

from git.repo.base import Repo
@@ -17,9 +18,16 @@ srcrev_file = "SRCREV.conf" 
pending_states = ["created", "waiting_for_resource", "preparing", "pending", "running"]





def integration_branch_name(project_name, branch_name):

def integration_branch_name(project_name, source_branch_name, target_branch_name):

    """Get integration branch name"""

    return "integrate/" + project_name.lower() + "/" + branch_name

    return (

        "integrate/"

        + project_name.lower()

        + "/"

        + source_branch_name

        + "/into/"

        + target_branch_name

    )





def is_gitlab_ci_integration_branch(branch_name):
@@ -151,27 +159,39 @@ def list_commits(commits): 




def commit_and_push(

    project: Project, repo: Repo, branch, message, name, email, less_verbose=False

    project: Project, repo: Repo, message, name, email, less_verbose=False

):

    """Commit and push to a repo branch"""

    branch = repo.head.reference

    author = Actor(name, email)



    logging.debug("Committing changes:")

    logging.debug(repo.git.diff("--staged"))



    repo.index.commit(message, author=author, committer=author)

    print(repo.git.log("--oneline", "-n", "5"))



    logging.debug("Git log:")

    logging.debug(repo.git.log("--oneline", "-n", "5"))



    # Push commit

    try:

        origin = repo.remote("origin")

        logging.debug("Push branch %s to %s", branch, origin)

        origin.push(branch, force=True)

        origin.push(branch, force=True).raise_if_error()

    except GitCommandError as e:

        sys.exit("ERROR: could not commit changes\n" + str(e))

        sys.exit("ERROR: could not push branch %s to %s\n" % (branch, origin) + str(e))



    # Print commit information

    revision = repo.head.commit.hexsha

    print("Pushed new commit:")

    print(project.web_url + "/-/commit/" + revision)

    if not less_verbose:

        print(repo.git.show("--summary", "--decorate"))

        print(

            colors.fg.lightgrey

            + repo.git.show("--summary", "--decorate")

            + colors.reset

            + "\n"

        )



    return revision
@@ -231,7 +251,7 @@ def clone_project(project: Project, into, branch=None): 
        repo = Repo.clone_from(clone_url.url, into, branch=branch, depth=1)

    except GitCommandError as e:

        raise Exception("could not clone repository\n" + str(e)) from e

    except IndexError:

    except IndexError as e:

        raise Exception("branch '%s' not found" % branch) from e

    return repo

scripts/deploy_gitlab_ci.py

+212 −180
Original line number Diff line number Diff line
@@ -6,14 +6,13 @@ import logging 
import sys

import os

from gitlab import Gitlab

from gitlab.v4.objects import Project, MergeRequest



from accept_merge_request import accept_merge_request

from create_merge_request import create_merge_request

from get_integration_sources import get_integration_sources

from get_merge_requests import get_merge_requests

from update_submodule import (

    update_submodule_and_include_ref,

    get_submodule_project_path_and_revision,

)

from update_submodule import update_submodule_and_include_ref

from integrate_into_manifest import update_manifest, update_srcrev



from ruamel.yaml import YAML
@@ -26,17 +25,12 @@ def read_keys_from_gitlab_ci_yml(gitlab_ci_yml): 
    data = yaml.load(gitlab_ci_yml)

    logging.debug("Yaml: %s", data)



    try:

        masterbranch = data["variables"]["MASTER_BRANCH_PROJECT"]

        logging.debug("Masterbranch %s", masterbranch)

    except KeyError:

        masterbranch = None

    try:

        recipe = data["variables"]["BB_RECIPE_NAME"]

        logging.debug("Recipe %s", recipe)

    except KeyError:

        recipe = None

    return {"recipe": recipe, "masterbranch": masterbranch}

    return {"recipe": recipe}





def integrate_submodule_into(
@@ -53,6 +47,7 @@ def integrate_submodule_into( 


    (

        project_repo,

        project_dir,

        integration_branch_name,

        integration_commit,

        message,
@@ -71,7 +66,9 @@ def integrate_submodule_into( 
    ret = {

        "project": gitlab_project,

        "repo": project_repo,

        "branch": integration_branch_name,

        "dir": project_dir,

        "integration_branch": integration_branch_name,

        "master_branch": branch,

        "commit": integration_commit,

        "message": message,

    }
@@ -83,12 +80,15 @@ def integrate_submodule_into( 
    return ret





def create_integration_merge_request(project, integration_branch_name, source_mr=None):

def create_integration_merge_request(

    project: Project,

    integration_branch: str,

    target_branch: str,

    source_mr: MergeRequest = None,

) -> MergeRequest:

    # Create merge request

    # This should be optional

    mr, created = create_merge_request(

        project, integration_branch_name, project.default_branch

    )

    mr, created = create_merge_request(project, integration_branch, target_branch)

    if created:

        if source_mr is not None:

            common.crosslink_merge_requests(source_mr, mr)
@@ -113,10 +113,16 @@ def main(): 
        required=True,

    )

    parser.add_argument(

        "--project",

        "--manifest-project",

        help="""name of the GitLab project""",

        dest="project",

        help="""name of the manifest project""",

        dest="manifest_project",

        required=True,

    )

    parser.add_argument(

        "--manifest-branch",

        help="""manifest branch to integrate changes into (can be a comma-separated list)""",

        dest="manifest_branch",

        required=True,

    )

    parser.add_argument(

        "--submodule",
@@ -130,13 +136,6 @@ def main(): 
        dest="revision",

        required=True,

    )

    parser.add_argument(

        "--branch",

        help="""project branch (if not default branch)""",

        dest="branch",

        required=False,

        default=None,

    )

    parser.add_argument(

        "--merge",

        help="""if set, perform merge after integration""",
@@ -145,6 +144,20 @@ def main(): 
        required=False,

        default=False,

    )

    parser.add_argument(

        "--project",

        help="""gitlab-ci project path or id""",

        dest="project",

        default=os.environ.get("CI_PROJECT_PATH"),

        required=False,

    )

    parser.add_argument(

        "--branch",

        help="""gitlab-ci branch that we're merging into""",

        dest="branch",

        default="master",

        required=False,

    )

    parser.add_argument(

        "--manifest-file",

        help="""manifest file name (default: 'default.xml')""",
@@ -160,10 +173,10 @@ def main(): 
        required=False,

    )

    parser.add_argument(

        "projects",

        help="""List of projects the change should be deployed to additionally

                to the manifest project given as named parameter.""",

        nargs="*",

        "--group",

        help="""group path or id to limit search scope to""",

        dest="group",

        required=True,

    )

    parser.add_argument(

        "-v",
@@ -180,53 +193,67 @@ def main(): 
            datefmt="%H:%M:%S",

        )



    manifest_branches = args.manifest_branch.split(",")



    gitlab = Gitlab(args.gitlab_url, private_token=args.token)

    group = gitlab.groups.get(args.group)



    # =======================================================

    # Create integration branches and commits with updates

    # submodule in all projects

    # =======================================================

    project_integration = {}

    # Update submodule in all 'child' project

    for p in args.projects:

        print("Create integration commit in", p)

    integration_sources = []

    for manifest_branch in manifest_branches:

        print(

            "Searching for projects in %s that are configured for automatic integration into %s:%s"

            % (args.group, args.manifest_project, manifest_branch)

        )

        for s in get_integration_sources(args.manifest_project, manifest_branch, group):

            if s not in integration_sources:

                integration_sources.append(s)



        res = integrate_submodule_into(

            gitlab, p, args.submodule, args.revision, args.branch

    # Update submodule in all integration sources

    project_integrations = []

    for s in integration_sources:

        print("Create integration commit in %s:%s" % (s["project"], s["branch"]))



        integration = integrate_submodule_into(

            gitlab, s["project"], args.submodule, args.revision, s["branch"]

        )

        # Store in the list if commit is set (meaning there was an update or

        #   an exising integration branch)

        if res["commit"] is not None:

            project_integration[p] = res

        if integration["commit"] is not None:

            project_integrations.append(integration)



    print("Create integration commit in", args.project)

    # Update submodule in manifest project

    manifest_project = integrate_submodule_into(

    # Update submodule in all manifest branches

    manifest_integrations = []

    for manifest_branch in manifest_branches:

        print(

            "Create integration commit in %s:%s"

            % (args.manifest_project, manifest_branch),

        )

        manifest_integrations.append(

            integrate_submodule_into(

                gitlab,

        args.project,

                args.manifest_project,

                args.submodule,

                args.revision,

        args.branch,

                manifest_branch,

                commit_and_push=False,

                force_clone=True,

            )

        )



    branch = args.branch

    if branch is None:

        branch = manifest_project["project"].default_branch

    # =======================================================

    # Create and merge merge_requests if needed

    # =======================================================

    if args.merge:

        # Get source merge request (the one in the gitlab-ci repo)

        submodule_project_path, _ = get_submodule_project_path_and_revision(

            manifest_project["project"], args.submodule, branch

        )

        submodule_project = common.get_project(gitlab, submodule_project_path)

        gitlab_ci_project = common.get_project(gitlab, args.project)

        mrs = get_merge_requests(

            submodule_project,

            # TODO should this be submodule_project's default branch?

            target_branch="master",

            project=gitlab_ci_project,

            target_branch=args.branch,

            state="merged",

            commit=args.revision,

        )

        if not mrs:
@@ -236,67 +263,68 @@ def main(): 
            )

        source_mr = mrs[0]



        for p in project_integration:

            integration = project_integration[p]

            logging.debug("Create MR in %s", integration["project"].name)

        for project_integration in project_integrations:

            logging.debug("Create MR in %s", project_integration["project"].name)

            mr = create_integration_merge_request(

                integration["project"], integration["branch"], source_mr

                project_integration["project"],

                project_integration["integration_branch"],

                project_integration["master_branch"],

                source_mr,

            )

            integration["mr"] = mr

            # Now merge

            logging.debug("Merge %s!%s", p, mr.iid)

            logging.debug("Merge %s!%s", project_integration["project"], mr.iid)



            # Wait until GitLab has checked merge status

            common.wait_until_merge_status_is_set(integration["project"], mr)

            common.wait_until_merge_status_is_set(project_integration["project"], mr)



            # Attempt to merge

            merged, integration_commit = accept_merge_request(

                integration["project"], mr, rebase=True

                project_integration["project"], mr, rebase=True

            )

            # if this has rebased the integration commit needs to be adapted:

            project_integration[p]["commit"] = integration_commit

            project_integration["commit"] = integration_commit

            # Save the target branch here, as the source branch gets deleted

            # during merge

            project_integration[p]["branch"] = mr.target_branch

            project_integration["integration_branch"] = mr.target_branch



            if not merged:

                sys.exit(

                    "Integration MR could not be merged. You have two possibilities to fix "

                    "this:\n"

                    "  1. Checkout the MR and rebase it on the current master manually, or\n"

                    "  2. Delete the MR (Edit -> Delete in the MR UI)\n"

                    "In either case restart this job afterwards in order to get it merged."

                    "Integration MR could not be merged:\n"

                    "%s\n"

                    "This can probably be resolved by creating a new commit in "

                    "gitlab-ci and merging it. The above MR can be closed then."

                    % mr.web_url

                )



        print("Successfully merged")



    # =======================================================

    # Now create the integration commit in the manifest

    # for all subprojects at once

    # =======================================================

    for manifest_integration in manifest_integrations:

        manifest_file_abs = os.path.join(

        manifest_project["repo"].working_tree_dir, args.manifest_file

            manifest_integration["repo"].working_tree_dir, args.manifest_file

        )

        logging.debug("Read manifest from: %s", manifest_file_abs)

        with open(manifest_file_abs, "r", encoding="utf8") as fp:

            manifest = fp.read()

        logging.debug(manifest)

        srcrev_file_abs = os.path.join(

        manifest_project["repo"].working_tree_dir, args.srcrev_file

            manifest_integration["repo"].working_tree_dir, args.srcrev_file

        )

        logging.debug("Read manifest from: %s", srcrev_file_abs)

        with open(srcrev_file_abs, "r", encoding="utf8") as fp:

            srcrev = fp.read()

        logging.debug(srcrev)



    for p in project_integration:

        integration = project_integration[p]

        for project_integration in project_integrations:

            logging.debug(

            "Update %s to %s", integration["project"].name, integration["commit"]

                "Update %s to %s",

                project_integration["project"].name,

                project_integration["commit"],

            )



            new_manifest = update_manifest(

            manifest, integration["project"], integration["commit"]

                manifest, project_integration["project"], project_integration["commit"]

            )

            if new_manifest is not None:

                manifest = new_manifest
@@ -308,88 +336,92 @@ def main(): 
            # the repo if the branch is already up to date



            gitlab_ci_yml = common.get_repository_file_raw(

            integration["project"], ".gitlab-ci.yml", ref=integration["branch"]

                project_integration["project"],

                ".gitlab-ci.yml",

                ref=project_integration["integration_branch"],

            )

            project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)



            new_srcrev = update_srcrev(

            srcrev, project_keys["recipe"], integration["commit"]

                srcrev, project_keys["recipe"], project_integration["commit"]

            )

            if new_srcrev is not None:

                srcrev = new_srcrev

                logging.debug(srcrev)

            else:

            logging.debug("Project %s not found in xml or srcrev file", p)

                logging.debug(

                    "Project %s not found in xml or srcrev file",

                    project_integration["project"],

                )



        # Write manifest

        with open(manifest_file_abs, "w", encoding="utf8") as fp:

            fp.write(manifest)

    manifest_project["repo"].git.add(args.manifest_file)

        manifest_integration["repo"].git.add(args.manifest_file)

        logging.debug(manifest)

        with open(srcrev_file_abs, "w", encoding="utf8") as fp:

            fp.write(srcrev)

    manifest_project["repo"].git.add(args.srcrev_file)

        manifest_integration["repo"].git.add(args.srcrev_file)

        logging.debug(srcrev)



        # ========================================================

        # Squash all commits on the integration branch to one

        # ========================================================

    manifest_project["repo"].remotes.origin.fetch(branch)

    manifest_master = manifest_project["project"].branches.get(branch)

    manifest_project["repo"].git.reset("--soft", manifest_master.commit["id"])

        manifest_integration["repo"].remotes.origin.fetch(

            manifest_integration["master_branch"]

        )

        manifest_master = manifest_integration["project"].branches.get(

            manifest_integration["master_branch"]

        )

        manifest_integration["repo"].git.reset("--soft", manifest_master.commit["id"])



        # ========================================================

        # Now commit and push the changes to the manifest repo

        # ========================================================

        # Make an API request to create the gitlab.user object

    gitlab = manifest_project["project"].manager.gitlab

        gitlab = integration["project"].manager.gitlab

        gitlab.auth()

        integration_commit = common.commit_and_push(

        manifest_project["project"],

        manifest_project["repo"],

        manifest_project["branch"],

        manifest_project["message"],

            manifest_integration["project"],

            manifest_integration["repo"],

            manifest_integration["message"],

            gitlab.user.username,

            gitlab.user.email,

        )



    print(

        "Successfully create integration commit {} in {}".format(

            integration_commit, args.project

        )

    )



    if not args.merge:

        sys.exit(0)



    # ============================================

    # Create merge requests for the manifest

    # ============================================



    logging.debug("Create MR in %s", manifest_project["project"].name)

    manifest_project["mr"] = create_integration_merge_request(

        manifest_project["project"], manifest_project["branch"], source_mr

    for integration in manifest_integrations:

        logging.debug("Create MR in %s", integration["project"].name)

        mr = create_integration_merge_request(

            integration["project"],

            integration["integration_branch"],

            integration["master_branch"],

            source_mr,

        )

        # =================================================

        # Now merge it

        # =================================================

        # The manifest needs to be merged at last

    mr = manifest_project["mr"]

    logging.debug("Merge %s!%s", args.project, mr.iid)

        logging.debug("Merge %s!%s", args.manifest_project, mr.iid)



        # Wait until GitLab has checked merge status

    common.wait_until_merge_status_is_set(manifest_project["project"], mr)

        common.wait_until_merge_status_is_set(integration["project"], mr)



        # Attempt to merge

    merged = accept_merge_request(manifest_project["project"], mr, rebase=True)

        merged = accept_merge_request(integration["project"], mr, rebase=True)



        if not merged:

            sys.exit(

            "Integration MR could not be merged. You have two possibilities to fix "

            "this:\n"

            "  1. Checkout the MR and rebase it on the current master manually, or\n"

            "  2. Delete the MR (Edit -> Delete in the MR UI)\n"

            "In either case restart this job afterwards in order to get it merged."

                "Integration MR could not be merged:\n"

                "%s\n"

                "This can probably be resolved by creating a new commit in "

                "gitlab-ci and merging it. The above MR can be closed then."

                % mr.web_url

            )



        print("Successfully merged")

scripts/get_integration_sources.py

0 → 100755
+107 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3

import argparse

import re

import sys

from gitlab import Gitlab, GitlabGetError

from gitlab.v4.objects import Group





def get_integration_sources(manifest_project: str, manifest_branch: str, group: Group):

    """

    Get a list of projects and branches in the given group which are configured for

    automatic integration into the given branch of the given manifest project.

    """

    integration_sources = []

    gitlab = group.manager.gitlab



    # Recurse into subgroups

    for g in group.subgroups.list():

        subgroup = gitlab.groups.get(g.id)

        integration_sources += get_integration_sources(

            manifest_project, manifest_branch, subgroup

        )



    # Regex to check INTEGRATION variable against

    regex = f":{manifest_project}:{manifest_branch}$"



    for project in group.projects.list():

        try:

            project = gitlab.projects.get(project.id)

            if not project.archived:

                integrations = project.variables.get("INTEGRATION").value

                for integration in integrations.splitlines():

                    if re.search(regex, integration):

                        source_branch = integration.split(":")[0]

                        integration_sources.append(

                            {

                                "project": project.path_with_namespace,

                                "branch": source_branch,

                            }

                        )

        except GitlabGetError as e:

            if e.response_code == 404:  # not found

                pass

            elif e.response_code == 403:  # forbidden

                sys.exit(

                    (

                        "ERROR: could not get INTEGRATION variable of project %s\n"

                        % project.path_with_namespace

                    )

                    + e.error_message

                )

            else:

                raise



    return integration_sources





def main():

    parser = argparse.ArgumentParser()

    parser.add_argument(

        "--gitlab-url",

        help="""URL to the GitLab instance""",

        dest="gitlab_url",

        required=True,

    )

    parser.add_argument(

        "--token",

        help="""GitLab REST API private access token""",

        dest="token",

        required=True,

    )

    parser.add_argument(

        "--manifest-project",

        help="""name of the manifest project""",

        dest="manifest_project",

        required=True,

    )

    parser.add_argument(

        "--manifest-branch",

        help="""manifest branch""",

        dest="manifest_branch",

        required=True,

    )

    parser.add_argument(

        "--group",

        help="""group path or id to limit search scope to""",

        dest="group",

        required=True,

    )



    args, _ = parser.parse_known_args()



    gitlab = Gitlab(args.gitlab_url, private_token=args.token)

    group = gitlab.groups.get(args.group)



    integration_sources = get_integration_sources(

        args.manifest_project,

        args.manifest_branch,

        group,

    )



    for source in integration_sources:

        print("%s:%s" % (source["project"], source["branch"]))





if __name__ == "__main__":

    main()

scripts/get_manifest_projects.py

deleted100755 → 0
+0 −67
Original line number Diff line number Diff line 
#!/usr/bin/env python3

import argparse

import sys

from furl import furl

from lxml import etree





def main():

    parser = argparse.ArgumentParser()

    parser.add_argument(

        "--manifest",

        help="""manifest file to parse projects from""",

        dest="manifest",

        required=True,

    )

    parser.add_argument(

        "--remote",

        help="""get only projects with this remote""",

        dest="remote",

        required=False,

    )

    parser.add_argument(

        "--concat-namespaces",

        help="""parse namespace from fetch URL and prepend it to project names""",

        dest="concat_namespaces",

        action="store_true",

        required=False,

    )



    args, _ = parser.parse_known_args()



    # Parse manifest file

    try:

        manifest = etree.parse(args.manifest)

    except FileNotFoundError:

        sys.exit("ERROR: file '%s' not found" % args.manifest)



    # Get namespace from remote

    # This is needed for cases where the remote URL contains a part of the project

    # namespace (e.g. "ssh://git@gitlab.com/garz-fricke/yocto") and the project name

    # contains another part of it (e.g. "layers/meta-seconorth-machine"). There is no

    # GitLab API call which will find a project given this information.

    # Thus we are adding a possibility to parse the namespace from the remote and pre-

    # pend it to the project name in order to return it including its complete namespace

    # (e.g. "garz-fricke/yocto/layers/meta-seconorth-machine").

    if args.concat_namespaces:

        remote = manifest.find("remote[@name='%s']" % args.remote)

        if remote is None:

            sys.exit("ERROR: remote '%s' not found in manifest" % args.remote)

        path = furl(remote.get("fetch")).path

        prefix = str(path).strip("/") + "/"

    else:

        prefix = ""



    # Find project references in manifest

    if args.remote:

        find_expression = "project[@remote='%s']" % args.remote

    else:

        find_expression = "project"

    projects = manifest.findall(find_expression)



    for project in projects:

        print(prefix + project.get("name"))





if __name__ == "__main__":

    main()

scripts/integrate_into_manifest.py

+13 −12
Original line number Diff line number Diff line
@@ -71,12 +71,12 @@ def update_srcrev(srcrev, recipe_name, new_revision): 




def integrate_into_manifest(

    manifest_project,

    integration_base,

    manifest_project: Project,

    manifest_branch,

    manifest_file,

    srcrev_file,

    recipe_name,

    project,

    project: Project,

    merge_request,

):

    gitlab = manifest_project.manager.gitlab
@@ -92,14 +92,15 @@ def integrate_into_manifest( 


        # Checkout manifest

        # TODO replace checkout with gitlab api access

        print("Cloning manifest repo: %s" % manifest_project.http_url_to_repo)

        try:

            manifest_repo = Repo.clone_from(

                clone_url.url, manifest_dir, branch=integration_base

                clone_url.url, manifest_dir, branch=manifest_branch

            )

        except GitCommandError as e:

            sys.exit("ERROR: could not clone manifest repository\n" + str(e))

        except IndexError:

            sys.exit("ERROR: branch '%s' not found" % integration_base)

            sys.exit("ERROR: branch '%s' not found" % manifest_branch)



        # Special handling for the gitlab-ci integration

        # When the branch 'merge_request.source_branch' already starts with
@@ -115,17 +116,18 @@ def integrate_into_manifest( 
            )

            logging.debug("Heads: %s", manifest_repo.heads)

            manifest_repo.heads[integration_branch].checkout()

            print(manifest_repo.git.log("--oneline", "-n", "5"))

            logging.debug(manifest_repo.git.log("--oneline", "-n", "5"))

            print("Using existing integration branch: %s" % integration_branch)

        else:

            # Create integration branch (delete former one if already exists)

            integration_branch = common.integration_branch_name(

                project.name, merge_request.source_branch

                project.name, merge_request.source_branch, manifest_branch

            )

            for ref in manifest_repo.references:

                if integration_branch == ref.name:

                    manifest_repo.delete_head(ref)



            logging.debug("Integration branch: %s", integration_branch)

            print("Creating integration branch: %s" % integration_branch)

            manifest_repo.head.set_reference(

                manifest_repo.create_head(integration_branch)

            )
@@ -178,7 +180,6 @@ def integrate_into_manifest( 
        manifest_revision = common.commit_and_push(

            manifest_project,

            manifest_repo,

            integration_branch,

            message,

            gitlab.user.username,

            gitlab.user.email,
@@ -209,9 +210,9 @@ def main(): 
        required=True,

    )

    parser.add_argument(

        "--integration-base",

        "--manifest-branch",

        help="""manifest branch to branch off from""",

        dest="integration_base",

        dest="manifest_branch",

        required=True,

    )

    parser.add_argument(
@@ -279,7 +280,7 @@ def main(): 


    manifest_revision = integrate_into_manifest(

        manifest_project=manifest_project,

        integration_base=args.integration_base,

        manifest_branch=args.manifest_branch,

        manifest_file=args.manifest_file,

        srcrev_file=args.srcrev_file,

        recipe_name=args.recipe_name,

scripts/merge_into_manifest.py

+20 −18
Original line number Diff line number Diff line
@@ -14,9 +14,9 @@ from integrate_into_manifest import integrate_into_manifest 


def merge_into_manifest(

    manifest_project,

    master_branch,

    manifest_branch,

    project,

    master_branch_project,

    project_branch,

    srcrev_file,

    recipe_name,

    commit,
@@ -30,7 +30,7 @@ def merge_into_manifest( 
    # Get source merge request

    mrs = get_merge_requests(

        project,

        target_branch=master_branch_project,

        target_branch=project_branch,

        state="merged",

        commit=commit,

    )
@@ -52,8 +52,11 @@ def merge_into_manifest( 
        )

        return ""



    integration_branch = common.integration_branch_name(project.name, original_branch)

    target_branch = master_branch

    target_branch = manifest_branch



    integration_branch = common.integration_branch_name(

        project.name, original_branch, target_branch

    )



    # Create merge request. If there already is a merge request, we assume that it has

    # been opened manually and thus will be merged manually as well, so we fail here.
@@ -92,7 +95,7 @@ def merge_into_manifest( 
            # we want a completely automated process in every case.

            manifest_revision = integrate_into_manifest(

                manifest_project=manifest_project,

                integration_base=target_branch,

                manifest_branch=target_branch,

                manifest_file=common.manifest_file,

                srcrev_file=srcrev_file,

                recipe_name=recipe_name,
@@ -136,9 +139,9 @@ def main(): 
        required=True,

    )

    parser.add_argument(

        "--master-branch",

        help="""master branch to merge changes into""",

        dest="master_branch",

        "--manifest-branch",

        help="""manifest branch to merge changes into""",

        dest="manifest_branch",

        required=True,

    )

    parser.add_argument(
@@ -148,9 +151,9 @@ def main(): 
        required=True,

    )

    parser.add_argument(

        "--master-branch-project",

        help="""master branch to merge changes into (project)""",

        dest="master_branch_project",

        "--project-branch",

        help="""project branch to merge changes into""",

        dest="project_branch",

        default=None,

        required=False,

    )
@@ -195,16 +198,15 @@ def main(): 
    project = common.get_project(gitlab, args.project)

    manifest_project = common.get_project(gitlab, args.manifest_project)



    # If no master_branch_project is set assume manifest and project

    # master branches are named the same

    if not args.master_branch_project:

        args.master_branch_project = args.master_branch

    # If no project_branch is set, assume manifest and project branches are the same

    if not args.project_branch:

        args.project_branch = args.manifest_branch



    manifest_revision = merge_into_manifest(

        manifest_project=manifest_project,

        master_branch=args.master_branch,

        manifest_branch=args.manifest_branch,

        project=project,

        master_branch_project=args.master_branch_project,

        project_branch=args.project_branch,

        srcrev_file=args.srcrev_file,

        recipe_name=args.recipe_name,

        commit=args.commit,

scripts/retrigger_mr_pipeline_jobs.py→scripts/retrigger_integrating_projects.py

+112 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3

import common



import argparse

import sys

from gitlab import Gitlab, GitlabJobRetryError



from get_integration_sources import get_integration_sources

from get_merge_requests import get_merge_requests

from retrigger_mr_pipeline_job import retrigger_mr_pipeline_job

from retrigger_pipeline_jobs import retrigger_pipeline_jobs





def main():

    """

    Retrigger a given job in all open MRs of projects in the given group with a target

    branch that is configured for automatic integration into the given manifest project

    and branch.

    """

    parser = argparse.ArgumentParser()

    parser.add_argument(

        "--gitlab-url",
@@ -24,63 +28,78 @@ def main(): 
        required=True,

    )

    parser.add_argument(

        "--project",

        help="""name of the GitLab project""",

        dest="project",

        "--manifest-project",

        help="""name of the GitLab manifest project""",

        dest="manifest_project",

        required=True,

    )

    parser.add_argument(

        "--state",

        help="""state of the merge request (opened, closed, locked, or merged)""",

        dest="state",

        "--manifest-branch",

        help="""target integration branch""",

        dest="manifest_branch",

        required=True,

    )

    parser.add_argument(

        "--source-branch",

        help="""source branch of the merge request""",

        dest="source_branch",

        required=False,

    )

    parser.add_argument(

        "--target-branch",

        help="""target branch of the merge request""",

        dest="target_branch",

        required=False,

    )

    parser.add_argument(

        "--commit",

        help="""commit sha of the merge request""",

        dest="commit",

        required=False,

        "--group",

        help="""group path or id to limit search scope to""",

        dest="group",

        required=True,

    )

    parser.add_argument(

        "--job",

        help="""job to retrigger""",

        dest="job",

        required=True,

        required=False,

        default="check",

    )



    args, _ = parser.parse_known_args()



    gitlab = Gitlab(args.gitlab_url, private_token=args.token)

    project = common.get_project(gitlab, args.project)

    group = gitlab.groups.get(args.group, retry_transient_errors=True)



    integration_sources = get_integration_sources(

        args.manifest_project, args.manifest_branch, group

    )



    failed = 0



    for source in integration_sources:



        project = gitlab.projects.get(source["project"], retry_transient_errors=True)



        mrs = get_merge_requests(

            project,

        state=args.state,

        source_branch=args.source_branch,

        target_branch=args.target_branch,

        commit=args.commit,

            state="opened",

            target_branch=source["branch"],

        )



    failed = 0

        for mr in mrs:

            # Get pipeline

            if not mr.pipeline:

                print("No pipeline in %s" % mr.web_url)

                continue

            pipeline = project.pipelines.get(

                mr.pipeline.get("id"),

                retry_transient_errors=True,

            )

            try:

            retrigger_mr_pipeline_job(project, mr, args.job, ["success", "running"])

                jobs = retrigger_pipeline_jobs(

                    project,

                    pipeline,

                    args.job,

                    ["success", "running"],

                    include_children=True,

                )

                if not jobs:

                    print(

                        "Could not find any jobs named '%s' in %s"

                        % (args.job, pipeline.web_url)

                    )

            except GitlabJobRetryError as e:

                print(

                "ERROR: Could not retrigger job '%s' of %s!%s: %s"

                % (args.job, args.project, mr.iid, e)

                    "ERROR: Could not retrigger job '%s' in %s: %s"

                    % (args.job, pipeline.web_url, e)

                )

                failed = failed + 1

                continue

scripts/retrigger_mr_pipeline_job.py→scripts/retrigger_pipeline_jobs.py

+122 −0
Original line number Diff line number Diff line
@@ -4,14 +4,34 @@ import common 
import argparse

import sys

from gitlab import Gitlab, GitlabGetError

from gitlab.v4.objects import Project, ProjectPipeline

from typing import List





def retrigger_mr_pipeline_job(project, mr, job_name, status_list):

    # Get pipeline

    if not mr.pipeline:

        print("No pipeline in !%s" % mr.iid)

        return None

    pipeline = project.pipelines.get(mr.pipeline.get("id"), retry_transient_errors=True)

def retrigger_pipeline_jobs(

    project: Project,

    pipeline: ProjectPipeline,

    job_name: str,

    status_list: List[str],

    include_children: bool = False,

):

    jobs = []



    # Recurse to child pipelines

    if include_children:

        for bridge in pipeline.bridges.list():

            if bridge.downstream_pipeline["project_id"] == project.id:

                child_pipeline = project.pipelines.get(

                    bridge.downstream_pipeline["id"],

                    retry_transient_errors=True,

                )

                jobs += retrigger_pipeline_jobs(

                    project,

                    child_pipeline,

                    job_name,

                    status_list,

                    include_children=True,

                )



    # Find job

    job = None
@@ -19,20 +39,20 @@ def retrigger_mr_pipeline_job(project, mr, job_name, status_list): 
        if pipelinejob.name == job_name:

            job = project.jobs.get(pipelinejob.id, retry_transient_errors=True)

    if not job:

        print("Could not find job '%s' in pipeline of !%s" % (job_name, mr.iid))

        return None

        return jobs



    # Only retrigger if job is in certain status

    if job.status not in status_list:

        return None

        return jobs



    # Retrigger job

    job.retry()

    print("Retrigger job '%s' of pipeline #%s:" % (job_name, pipeline.id))

    print("Retrigger job '%s' in %s:" % (job_name, pipeline.web_url))

    job = project.jobs.get(job.id, retry_transient_errors=True)

    print(job.web_url)

    jobs.append(job)



    return job

    return jobs





def main():
@@ -56,9 +76,9 @@ def main(): 
        required=True,

    )

    parser.add_argument(

        "--iid",

        help="""iid of the merge request""",

        dest="iid",

        "--pipeline",

        help="""pipeline id""",

        dest="pipeline",

        required=True,

    )

    parser.add_argument(
@@ -68,21 +88,34 @@ def main(): 
        required=True,

    )



    parser.add_argument(

        "--include-children",

        help="""search for job in child pipelines""",

        dest="include_children",

        action="store_true",

        default=False,

    )



    args, _ = parser.parse_known_args()



    gitlab = Gitlab(args.gitlab_url, private_token=args.token)

    project = common.get_project(gitlab, args.project)

    try:

        mr = project.mergerequests.get(args.iid, retry_transient_errors=True)

        pipeline = project.pipelines.get(args.pipeline, retry_transient_errors=True)

    except GitlabGetError as e:

        sys.exit(

            "ERROR: could not get merge request %s:!%s: %s"

            % (project.name, args.iid, e)

        sys.exit("ERROR: could not get pipeline %s: %s" % (args.pipeline, e))

    if not pipeline:

        sys.exit("ERROR: could not find pipeline %s" % args.pipeline)



    jobs = retrigger_pipeline_jobs(

        project,

        pipeline,

        args.job,

        ["success", "running"],

        args.include_children,

    )

    if not mr:

        sys.exit("ERROR: could not find merge request %s!%s" % (project.name, args.iid))



    retrigger_mr_pipeline_job(project, mr, args.job, ["success", "running"])

    print("Retriggered %d jobs for pipeline #%s" % (len(jobs), args.pipeline))





if __name__ == "__main__":

scripts/update_submodule.py

+15 −7
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ def clone_project_and_submodule(project: Project, submodule_name, branch=None): 


    # Checkout project

    try:

        repo = Repo.clone_from(clone_url.url, project_dir, branch=branch, depth=1)

        repo = Repo.clone_from(clone_url.url, project_dir.name, branch=branch, depth=1)

    except GitCommandError as e:

        sys.exit("ERROR: could not clone repository\n" + str(e))

    except IndexError:
@@ -130,7 +130,9 @@ def clone_project_and_submodule(project: Project, submodule_name, branch=None): 
    with submodule.config_writer() as writer:

        writer.set("url", submodule_relative_url)



    return repo, submodule_project

    # We need to keep the TemporaryDirectory object reference project_dir because we

    # need the cloned repo later, otherwise the directory will be immediately deleted.

    return repo, submodule_project, project_dir





def update_submodule_in_repo(repo: Repo, submodule_project: Project, new_revision):
@@ -209,6 +211,7 @@ def update_submodule_and_include_ref( 


    submodule_update_needed = True

    project_repo = None

    project_dir = None

    integration_commit = None



    if branch is None:
@@ -233,7 +236,7 @@ def update_submodule_and_include_ref( 
            "No commits found in range %s, probably submodule already up-to-date.",

            revision_range,

        )

        return None, None, None, None

        return None, None, None, None, None

    logging.debug("New commits: %s", commits)



    # Find out if top commit is part of a merge request
@@ -248,7 +251,7 @@ def update_submodule_and_include_ref( 


    # Construct integration branch name

    integration_branch_name = common.integration_branch_name(

        submodule_project.name, integration_branch_suffix

        submodule_project.name, integration_branch_suffix, branch

    )



    # Construct commit message
@@ -328,7 +331,7 @@ def update_submodule_and_include_ref( 
            clone_branch = integration_branch_name



        # Actually clone

        project_repo, submodule_project = clone_project_and_submodule(

        project_repo, submodule_project, project_dir = clone_project_and_submodule(

            project, submodule_name, clone_branch

        )
@@ -383,14 +386,19 @@ def update_submodule_and_include_ref( 
            integration_commit = common.commit_and_push(

                project,

                project_repo,

                integration_branch_name,

                message,

                gitlab.user.username,

                gitlab.user.email,

                less_verbose=True,

            )



    return project_repo, integration_branch_name, integration_commit, message

    return (

        project_repo,

        project_dir,

        integration_branch_name,

        integration_commit,

        message,

    )





def main():