CI: Deploy: create integration branches and trigger pipelines in subprojects

Allow deploy_gitlab_ci to change multiple projects at once.
Use it to create integration branches and merge requests in all
projects.

Add a python file to generate a job yml from jinja2 template.
Add a template for the jobs to trigger. These execute the actual
integration in all 'subprojects'.
Create the yml file in the deploy job and trigger it in a new trigger
job.

.gitlab-ci.yml

@@ -19,6 +19,8 @@ default:
 stages:
   - analyze
   - deploy
+  - integrate
+  - test
 
 workflow:
   rules:
@@ -28,66 +30,120 @@ workflow:
 # ---------------------------------------------------------------------------------------
 # Stage: analyze
 # ---------------------------------------------------------------------------------------
-pylint:
-  stage: analyze
-  timeout: 2m
-  script:
-    - pylint --rcfile=pylintrc *.py
-
-black:
-  stage: analyze
-  timeout: 2m
-  script:
-    - black --diff --check *.py
-
-executable:
-  stage: analyze
-  timeout: 2m
-  script:
-    - (! find ! -executable -name "*.py" -exec echo "not executable:"" {}" \; | grep .)
-
-yamllint:
-  stage: analyze
-  timeout: 2m
-  script:
-    - yamllint -c .yamllint.yml *.yml .*.yml
+# pylint:
+#   stage: analyze
+#   timeout: 2m
+#   script:
+#     - pylint --rcfile=pylintrc *.py
+#
+# black:
+#   stage: analyze
+#   timeout: 2m
+#   script:
+#     - black --diff --check *.py
+#
+# executable:
+#   stage: analyze
+#   timeout: 2m
+#   script:
+#     - (! find ! -executable -name "*.py" -exec echo "not executable:"" {}" \; | grep .)
+#
+# yamllint:
+#   stage: analyze
+#   timeout: 2m
+#   script:
+#     - yamllint -c .yamllint.yml *.yml .*.yml
 
 
 # ---------------------------------------------------------------------------------------
-# Stage: deploy
+# Stage: deploy-test
 # ---------------------------------------------------------------------------------------
-.deploy: &deploy
+.deploy:
   stage: deploy
-  when: manual
-  allow_failure: true
   script:
     - cd ${CI_PROJECT_DIR}
-    - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi
+    - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then
+      MERGE="--merge"; else MERGE=""; fi
     - ./deploy_gitlab_ci.py
       --gitlab-url=${CI_SERVER_URL}
       --token=${GITBOT_TOKEN}
-      --project=${CI_PROJECT_ROOT_NAMESPACE}/${CI_JOB_NAME}
+      --manifest-project=${MANIFEST_PROJECT}
       --submodule=.gitlab-ci
       --revision=${CI_COMMIT_SHA}
       ${MERGE}
+      ${DEPLOY_TO}
+
+    - ./generate_job_from_template.py
+      --template=gitlab-ci-integration.jinja2
+      --image=${CI_IMAGE_PYTHON}
+      --branch="integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
+      --manifest-project=${MANIFEST_PROJECT}
+      --verbose
+      ${DEPLOY_TO}
+      > integration.yml
+    - cat integration.yml
+  artifacts:
+    paths:
+      - integration.yml
+
+deploy-foobar:
+  extends: .deploy
+  variables:
+    PROJECT_ROOT:
+      ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
+    MANIFEST_PROJECT:
+      ${PROJECT_ROOT}/minimal-manifest
+
+    DEPLOY_TO:
+      ${PROJECT_ROOT}/minimal-foo
+      ${PROJECT_ROOT}/minimal-bar
+
+trigger-foobar:
+  stage: integrate
+  trigger:
+    include:
+      - artifact: integration.yml
+        job: deploy-foobar
+    strategy: depend
+
+
+# ---------------------------------------------------------------------------------------
+# Stage: deploy
+# ---------------------------------------------------------------------------------------
+# .deploy: &deploy
+#   stage: deploy
+#   when: manual
+#   allow_failure: true
+#   script:
+#     - cd ${CI_PROJECT_DIR}
+#     - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge";
+#          else MERGE=""; fi
+#     - ./deploy_gitlab_ci.py
+#       --gitlab-url=${CI_SERVER_URL}
+#       --token=${GITBOT_TOKEN}
+#       --project=${CI_PROJECT_ROOT_NAMESPACE}/${CI_JOB_NAME}
+#       --submodule=.gitlab-ci
+#       --revision=${CI_COMMIT_SHA}
+#       ${MERGE}
+#
 
-3rd-party/kuk/uboot-imx-kuk: *deploy
-kernel/linux-guf: *deploy
-kernel/linux-imx-kuk: *deploy
-kernel/modules/egalaxi2c: *deploy
-kernel/modules/gfplatdetect: *deploy
-tools/gf-emc-test-suite: *deploy
-tools/gf-productiontests: *deploy
-tools/gfeeprom: *deploy
-tools/gfxml2dto: *deploy
-tools/guf-show-demo: *deploy
-tools/libmdb: *deploy
-tools/touchcal-conv: *deploy
-tools/xconfig: *deploy
-yocto/config: *deploy
-yocto/infrastructure/ci-test/minimal-bar: *deploy
-yocto/infrastructure/ci-test/minimal-foo: *deploy
-yocto/infrastructure/ci-test/minimal-manifest: *deploy
-yocto/layers/meta-guf-distro: *deploy
-yocto/layers/meta-guf-machine: *deploy
-yocto/manifest: *deploy
+# 3rd-party/kuk/uboot-imx-kuk: *deploy
+# kernel/linux-guf: *deploy
+# kernel/linux-imx-kuk: *deploy
+# kernel/modules/egalaxi2c: *deploy
+# kernel/modules/gfplatdetect: *deploy
+# tools/gf-emc-test-suite: *deploy
+# tools/gf-productiontests: *deploy
+# tools/gfeeprom: *deploy
+# tools/gfxml2dto: *deploy
+# tools/guf-show-demo: *deploy
+# tools/libmdb: *deploy
+# tools/touchcal-conv: *deploy
+# tools/xconfig: *deploy
+# yocto/config: *deploy
+# yocto/infrastructure/ci-test/minimal-bar: *deploy
+# yocto/infrastructure/ci-test/minimal-foo: *deploy
+# yocto/infrastructure/ci-test/minimal-manifest: *deploy
+# yocto/layers/meta-guf-distro: *deploy
+# yocto/layers/meta-guf-machine: *deploy
+# yocto/manifest: *deploy

deploy_gitlab_ci.py

@@ -25,6 +25,62 @@ def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
         repo.git.add(gitlab_ci_yml)
 
 
+def deploy_into(project, submodule, revision, branch, replace_exising_branch=False):
+    """Update the submodule and include refs to the submodule in the given project.
+        Create mergerequest if needed.
+
+    Parameters:
+        project ( gitlab project): The project which's submodule should be updated
+        submodule_name (string): The name of the submodule to pull
+        submodule_revision (hex string): The sha hash of the commit to update the submodule to
+        branch (string): branch to update, if None, the projects default branch is used
+        replace_exising_branch: When an existing integration branch is found it is always replaced.
+
+    Returns: tuple of:
+        branch (string): Name of the newly created integration branch
+        merge_request (gitlab mr): Mergerequest for the integration branch
+    """
+    # Update submodule
+    integration_branch, _, submodule_project = update_submodule(
+        project,
+        submodule,
+        revision,
+        branch,
+        pre_commit_hook=update_rev_in_gitlab_ci,
+        replace_exising_branch=replace_exising_branch,
+    )
+
+    logging.debug("Integration branch: %s", integration_branch)
+
+    # If submodule is already at specified revision, return directly
+    if not integration_branch:
+        return None, submodule_project
+
+    # Get source merge request
+    mrs = get_merge_requests(
+        submodule_project,
+        # TODO should this be submodule_project's default branch?
+        target_branch="master",
+        commit=revision,
+    )
+    if not mrs:
+        sys.exit(
+            "ERROR: could not determine source merge request for commit %s" % revision
+        )
+    source_mr = mrs[0]
+
+    # Create merge request
+    mr, created = create_merge_request(
+        project, integration_branch, project.default_branch
+    )
+    if created:
+        common.crosslink_merge_requests(source_mr, mr)
+        print("Created new merge request:\n%s" % mr.web_url)
+    else:
+        print("Existing integration merge request:\n%s" % mr.web_url)
+    return integration_branch, mr
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -41,9 +97,9 @@ def main():
     )
     parser.add_argument(
         "--project",
+        "--manifest-project",
         help="""name of the GitLab project""",
         dest="project",
-        required=True,
     )
     parser.add_argument(
         "--submodule",
@@ -72,6 +128,12 @@ def main():
         required=False,
         default=False,
     )
+    parser.add_argument(
+        "projects",
+        help="""List of projects the change should be deployed to additionally
+                to the manifest project given as named parameter.""",
+        nargs="*",
+    )
     parser.add_argument(
         "-v",
         "--verbose",
@@ -86,42 +148,37 @@ def main():
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
     project = common.get_project(gitlab, args.project)
 
-    # Update submodule
-    integration_branch, _, submodule_project = update_submodule(
+    logging.debug("Integrate into: %s", args.project)
+
+    # Update submodule in this project, create MR
+    integration_branch, mr = deploy_into(
         project,
         args.submodule,
         args.revision,
         args.branch,
-        pre_commit_hook=update_rev_in_gitlab_ci,
+        replace_exising_branch=len(args.projects) > 0,
     )
+    merge_request_manifest = mr
+    merge_requests = []
 
-    logging.debug("Integration branch: %s", integration_branch)
     # If submodule is already at specified revision, exit successfully
+    # TODO: is this correct for multi deploy also?
     if not integration_branch:
         sys.exit(0)
 
-    # Get source merge request
-    mrs = get_merge_requests(
-        submodule_project,
-        target_branch="master",
-        commit=args.revision,
-    )
-    if not mrs:
-        sys.exit(
-            "ERROR: could not determine source merge request for commit %s"
-            % args.revision
+    for p in args.projects:
+        gitlab_p = common.get_project(gitlab, p)
+        logging.debug("Integrate into: %s", p)
+
+        integration_branch, mr = deploy_into(
+            gitlab_p,
+            args.submodule,
+            args.revision,
+            args.branch,
         )
-    source_mr = mrs[0]
+        merge_requests.append(mr)
 
-    # Create merge request
-    mr, created = create_merge_request(
-        project, integration_branch, project.default_branch
-    )
-    if created:
-        common.crosslink_merge_requests(source_mr, mr)
-        print("Created new merge request:\n%s" % mr.web_url)
-    else:
-        print("Existing integration merge request:\n%s" % mr.web_url)
+        logging.debug("Integration branch: %s", integration_branch)
 
     if not args.merge:
         print(
@@ -132,20 +189,24 @@ def main():
         )
         sys.exit(0)
 
-    # Wait until GitLab has checked merge status
-    common.wait_until_merge_status_is_set(project, mr)
-
-    # Attempt to merge
-    merged = accept_merge_request(project, mr, rebase=True)
-
-    if not merged:
-        sys.exit(
-            "Integration MR could not be merged. You have two possibilities to fix "
-            "this:\n"
-            "  1. Checkout the MR and rebase it on the current master manually, or\n"
-            "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
-            "In either case restart this job afterwards in order to get it merged."
-        )
+    # The manifest needs to be merged at last
+    merge_requests.append(merge_request_manifest) 
+    for mr in merge_requests:
+        logging.debug("Merge %s", mr)
+        # Wait until GitLab has checked merge status
+        common.wait_until_merge_status_is_set(project, mr)
+
+        # Attempt to merge
+        merged = accept_merge_request(project, mr, rebase=True)
+
+        if not merged:
+            sys.exit(
+                "Integration MR could not be merged. You have two possibilities to fix "
+                "this:\n"
+                "  1. Checkout the MR and rebase it on the current master manually, or\n"
+                "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
+                "In either case restart this job afterwards in order to get it merged."
+            )
 
     print("Successfully merged")
 

foobar-manifest-integration.yml

@@ -29,7 +29,23 @@ stages:
 
 workflow:
   rules:
-    # Do not run pipelines on forked projects
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
+    # Do not run pipelines for merge requests for integrate/gitlab-ci/ branches
+    # These are trigger explicitly from the integration pipeline in gitlab-ci repo
+    - if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
+      when: never
+    # Do not run pipelines on forked projects.
+    # The pipelines would not work anyway because of the users permissions.
+    # There are two cases catched here:
+    # 1. The project is forked into someones gitlab namespace and a MR to
+    #    include a change into this forked project is created. In this case
+    #    is the CI_PROJECT_ROOT_NAMESPACE not SECO-Northern-Europe but the
+    #    namespace the fork lives in.
+    # 2. The MR from the forked project is created to merge the change into this
+    #    the project in the SECO-Northern-Europe namespace (customer sending
+    #    change to us). Here the the IDs used below differ.
+    #
     - if: $CI_PROJECT_ROOT_NAMESPACE == "SECO-Northern-Europe"
         && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID
 
@@ -44,6 +60,7 @@ yamllint:
     - infrastructure
   timeout: 2m
   script:
+    - printenv
     - yamllint -c .gitlab-ci/.yamllint.yml .*.yml
 
 # ---------------------------------------------------------------------------------------
@@ -56,21 +73,30 @@ integrate:
     # the time a merge request is created. Otherwise we cannot guarantee a green
     # master after merging.
     - if: $CI_MERGE_REQUEST_IID
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
   tags:
     - infrastructure
   timeout: 2m
   cache:
     policy: push
   script:
+    - printenv
     - cd ${CI_PROJECT_DIR}
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+      MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+      MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
     - .gitlab-ci/integrate_into_manifest.py
         --gitlab-url=${CI_SERVER_URL}
         --token=${GITBOT_TOKEN}
         --manifest-project=${MANIFEST_PROJECT}
         --integration-base=${MASTER_BRANCH}
         --project=${CI_PROJECT_PATH}
-        --merge-request=${CI_MERGE_REQUEST_IID}
+        --merge-request=${MERGE_REQUEST}
         --save-revision-to=manifest_revision
+        --verbose
   artifacts:
     paths:
       - manifest_revision
@@ -105,7 +131,10 @@ merge:
 build:
   stage: build
   rules:
-    - if: $CI_MERGE_REQUEST_IID
+    # execute this in MR only and not for integrate/gitlab-ci/ integrations
+    # branches. These are build after the integration has been done in all
+    # projects
+    - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
   trigger:
     project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
     branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
@@ -118,16 +147,23 @@ check:
   stage: check
   rules:
     - if: $CI_MERGE_REQUEST_IID
+    # Explicitly allow externally triggered pipelines in every case
+    - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
   needs: ["integrate"]
   tags:
     - infrastructure
   timeout: 2m
   script:
     - cd ${CI_PROJECT_DIR}
+    - if [ -n "${CI_MERGE_REQUEST_IID}" ];then
+      MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
+      else
+      MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
+      fi
     - .gitlab-ci/check_if_integration_branch_is_up_to_date.py
         --gitlab-url=${CI_SERVER_URL}
         --token=${GITBOT_TOKEN}
         --manifest-project=${MANIFEST_PROJECT}
         --integration-base=${MASTER_BRANCH}
         --project=${CI_PROJECT_PATH}
-        --merge-request=${CI_MERGE_REQUEST_IID}
+        --merge-request=${MERGE_REQUEST}

generate_job_from_template.py

+#!/usr/bin/env python3
+
+from __future__ import print_function
+import argparse
+import io
+import logging
+import os
+import re
+import sys
+from copy import deepcopy
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+from jinja2.exceptions import TemplateNotFound
+from ruamel.yaml import YAML
+
+
+def parse_template(yaml_string):
+    """
+    Round trip lava_job through ruamel to test parsing and improve formatting.
+    Comments are preserved.
+
+    In: yaml-formatted string
+    Out: validated yaml-formatted string
+    """
+    yaml = YAML()
+    yaml.indent(sequence=4, offset=2)
+    # ruamel does not provide a mechanism to dump to string, so use StringIO to catch it
+    output = io.StringIO()
+    yaml.dump(yaml.load(yaml_string), output)
+    return output.getvalue()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--template",
+        help="""A Jinja2 template for the yml file to be genarated.
+                All variables in the templates will be substituted using the
+                Jinja2 engine. Variables can be passed either as environment 
+                variables or as command line arguments
+                (e.g. --my-variable=myvalue). In the latter, hyphens ("-") are 
+                replaced with underscores ("_"), i.e. "--my-variable" 
+                becomes "my_variable".""",
+        required=True,
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
+    parser.add_argument(
+        "arguments",
+        nargs="*",
+        help="""Arguments passed as list to the template.""",
+    )
+
+    # parse known args
+    args, unknown = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    logging.debug("Using template: %s", args.template)
+
+    # parse optional arguments
+    parser = argparse.ArgumentParser()
+    for arg in unknown:
+        if arg.startswith(("-", "--")):
+            parser.add_argument(arg.split("=")[0])
+    optional_args, _ = parser.parse_known_args()
+    logging.debug("Optional args: %s", optional_args)
+
+    # update context with given optional arguments
+    j2_env = Environment(undefined=StrictUndefined, extensions=["jinja2.ext.do"])
+    context = dict(deepcopy(os.environ))
+    for arg in vars(optional_args):
+        context[arg] = vars(optional_args)[arg]
+    if args.arguments:
+        context["optional_arguments"] = args.arguments
+
+    for k, v in context.items():
+        logging.debug("Optional args: %s -- %s", k, v)
+
+    # render the template
+    j2_env.loader = FileSystemLoader(os.path.dirname(args.template))
+
+    try:
+        lava_job = j2_env.get_template(os.path.basename(args.template)).render(context)
+        lava_job = parse_template(lava_job)
+        # Remove duplicate newlines
+        lava_job = re.sub(r"\n+", r"\n", lava_job)
+
+        print(lava_job)
+
+    except TemplateNotFound:
+        print("ERROR: Template %s not found" % args.template, file=sys.stderr)
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

gitlab-ci-integration.jinja2

+---
+#======================================================
+# Trigger the integration pipeline for all repos using 
+# the gitlab-ci repository.
+#======================================================
+
+{% if image is defined %}
+image: {{ image }}
+{% endif %}
+
+workflow:
+  rules:
+    # This rule is needed, as otherwise the workflow:rules from
+    # the parent job seem to be used and prevent the pipeline generation
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+
+stages:
+  - integrate
+  - build
+
+default:
+  tags:
+    # All jobs should use the infrastructure runner
+    - infrastructure
+
+{% set projectneeds = { 'project': "" } %}
+{% if optional_arguments is defined %}
+{% set prev_project = "" %}
+{% for project in optional_arguments %}
+{% set projectshort = project.split('/')[-1] %}
+{{ projectshort }}:
+  stage: integrate
+  needs: [ {{ projectneeds.project }} ]
+  trigger:
+    project: {{ project }}
+    branch: {{ branch }}
+    strategy: depend
+
+{% if projectneeds.update({ 'project': projectshort }) %}{% endif %}
+{% endfor %}
+{% endif %}
+
+{% if manifest_project is defined %}
+{% set manifest_project_short = manifest_project.split('/')[-1] %}
+{{ manifest_project_short }}:
+  stage: build
+  trigger:
+    project: {{ manifest_project }}
+    branch: {{ branch }}
+    strategy: depend
+
+{% endif %}

integrate_into_manifest.py

@@ -2,6 +2,7 @@
 import common
 
 import argparse
+import logging
 import sys
 import tempfile
 import re
@@ -42,14 +43,36 @@ def integrate_into_manifest(
         except IndexError:
             sys.exit("ERROR: branch '%s' not found" % integration_base)
 
-        # Create integration branch (delete former one if already exists)
-        integration_branch = common.integration_branch_name(
-            project.name, merge_request.source_branch
-        )
-        for ref in manifest_repo.references:
-            if integration_branch == ref.name:
-                manifest_repo.delete_head(ref)
-        manifest_repo.head.set_reference(manifest_repo.create_head(integration_branch))
+        # Special handling for the gitlab-ci integration
+        # When the branch 'merge_request.source_branch' already starts with
+        # integrate/gitlab-ci we add our new commit to this branch
+        # Otherwise (normal behaviour) a new integration branch is created
+        integration_branch = None
+        if merge_request.source_branch.startswith("integrate/gitlab-ci"):
+            logging.debug("Integration of gitlab-ci: %s", merge_request.source_branch)
+            for ref in manifest_repo.references:
+                # remove 'origin/' from the ref before compare
+                refname = ref.name.split("/", 1)[1]
+                logging.debug("Found ref: %s", refname)
+                if merge_request.source_branch == ref.name:
+                    logging.debug("Found integration for gitlab-ci")
+                    manifest_repo.head.set_reference(ref)
+                    integration_branch = merge_request.source_branch
+                    break
+
+        if integration_branch is None:
+            # Create integration branch (delete former one if already exists)
+            integration_branch = common.integration_branch_name(
+                project.name, merge_request.source_branch
+            )
+            for ref in manifest_repo.references:
+                if integration_branch == ref.name:
+                    manifest_repo.delete_head(ref)
+            manifest_repo.head.set_reference(
+                manifest_repo.create_head(integration_branch)
+            )
+
+        logging.debug("Integration branch: %s", integration_branch)
 
         # Parse manifest file
         try:
@@ -189,7 +212,8 @@ def main():
     )
     parser.add_argument(
         "--merge-request",
-        help="""project merge request IID containing the changes to be integrated""",
+        help="""project merge request IID or link containing the
+                changes to be integrated""",
         dest="merge_request",
         required=True,
     )
@@ -199,17 +223,49 @@ def main():
         dest="revision_file",
         required=False,
     )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="""Increase verbosity.""",
+    )
 
     args, _ = parser.parse_known_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
 
+    logging.debug(args)
     gitlab = Gitlab(args.gitlab_url, private_token=args.token)
 
     manifest_project = common.get_project(gitlab, args.manifest_project)
     project = common.get_project(gitlab, args.project)
+    logging.debug(project)
+    mr_iid = args.merge_request
+    logging.debug(mr_iid)
+    # MR may also be specified as
+    # SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-bar!115
+    if mr_iid.startswith(args.project):
+        mr_number = int(mr_iid.split("!")[-1])
+        logging.debug("Number of MR: %d", mr_number)
+        logging.debug(project.mergerequests.list())
+        mr_iid = None
+        for mr in project.mergerequests.list():
+            logging.debug(mr)
+            logging.debug(int(mr.iid))
+            logging.debug(mr.id)
+            if int(mr.iid) == mr_number:
+                mr_iid = mr.id
+                break
+
+        logging.debug(mr_iid)
+        if mr_iid is None:
+            sys.exit(
+                "ERROR: could not get %s"
+                % (args.merge_request)
+            )
+
     try:
-        merge_request = project.mergerequests.get(
-            args.merge_request, retry_transient_errors=True
-        )
+        merge_request = project.mergerequests.get(mr_iid, retry_transient_errors=True)
     except GitlabGetError as e:
         sys.exit(
             "ERROR: could not get %s!%s: %s"

update_submodule.py

@@ -12,7 +12,12 @@ from gitlab import Gitlab
 
 
 def update_submodule(
-    project, submodule_name, submodule_revision, branch=None, pre_commit_hook=None
+    project,
+    submodule_name,
+    submodule_revision,
+    branch=None,
+    pre_commit_hook=None,
+    replace_exising_branch=False,
 ):
     """Update submodule of gitlab project to given revision
 
@@ -23,6 +28,7 @@ def update_submodule(
         branch (string): branch to update, if None, the projects default branch is used
         pre_commit_hook: Function to be called before the actual commit is done, to add additional changes.
                          Arguments passed: ( repo, submodule_project, submodule_revision)
+        replace_exising_branch: When an existing integration branch is found it is always replaced.
     Returns: tuple of:
         branch (string): Name of the newly created integration branch
         revision (string): hexsha of the new commit
@@ -116,16 +122,16 @@ def update_submodule(
         if existing_branch:
             repo.head.set_reference(existing_branch)
             submodule = common.get_submodule(repo, submodule_name)
-            if submodule.hexsha == submodule_revision:
+            if replace_exising_branch or submodule.hexsha != submodule_revision:
+                print("Replacing outdated integration branch %s" % integration_branch)
+                repo.head.set_reference(branch)
+                submodule = common.get_submodule(repo, submodule_name)
+            else:
                 print(
                     "Submodule is already at %s on branch %s"
                     % (submodule_revision, integration_branch)
                 )
                 return (integration_branch, existing_branch.commit, submodule_project)
-            else:
-                print("Replacing outdated integration branch %s" % integration_branch)
-                repo.head.set_reference(branch)
-                submodule = common.get_submodule(repo, submodule_name)
         else:
             print("Creating integration branch %s" % integration_branch)