diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b203089976ea38290999c419533854253159a0a..5cc4038a4c58716a5008b03d23cdfe05f0b9bee6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ default: stages: - analyze + - deploy-test - deploy workflow: @@ -53,6 +54,28 @@ yamllint: - yamllint -c .yamllint.yml *.yml .*.yml +# --------------------------------------------------------------------------------------- +# Stage: deploy-test +# --------------------------------------------------------------------------------------- +deploy-foobar-manifest: + stage: deploy-test + # when: manual + allow_failure: true + script: + - cd ${CI_PROJECT_DIR} + - if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi + - PROJECT_ROOT="${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test" + - ./deploy_gitlab_ci.py + --gitlab-url=${CI_SERVER_URL} + --token=${GITBOT_TOKEN} + --submodule=.gitlab-ci + --revision=${CI_COMMIT_SHA} + --verbose + ${MERGE} + --manifest-project=${PROJECT_ROOT}/minimal-manifest + ${PROJECT_ROOT}/minimal-foo + ${PROJECT_ROOT}/minimal-bar + # --------------------------------------------------------------------------------------- # Stage: deploy # --------------------------------------------------------------------------------------- diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py index 106329aed650372e67e4c5af3887d43b50f588a0..31c291e1a422e7e5f18229f103b185d8775efce2 100755 --- a/deploy_gitlab_ci.py +++ b/deploy_gitlab_ci.py @@ -25,6 +25,62 @@ def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision): repo.git.add(gitlab_ci_yml) +def deploy_into(project, submodule, revision, branch, replace_exising_branch=False): + """Update the submodule and include refs to the submodule in the given project. + Create mergerequest if needed. + + Parameters: + project ( gitlab project): The project which's submodule should be updated + submodule_name (string): The name of the submodule to pull + submodule_revision (hex string): The sha hash of the commit to update the submodule to + branch (string): branch to update, if None, the projects default branch is used + replace_exising_branch: When an existing integration branch is found it is always replaced. + + Returns: tuple of: + branch (string): Name of the newly created integration branch + merge_request (gitlab mr): Mergerequest for the integration branch + """ + # Update submodule + integration_branch, _, submodule_project = update_submodule( + project, + submodule, + revision, + branch, + pre_commit_hook=update_rev_in_gitlab_ci, + replace_exising_branch=replace_exising_branch, + ) + + logging.debug("Integration branch: %s", integration_branch) + + # If submodule is already at specified revision, return directly + if not integration_branch: + return None, submodule_project + + # Get source merge request + mrs = get_merge_requests( + submodule_project, + # TODO should this be submodule_project's default branch? + target_branch="master", + commit=revision, + ) + if not mrs: + sys.exit( + "ERROR: could not determine source merge request for commit %s" % revision + ) + source_mr = mrs[0] + + # Create merge request + mr, created = create_merge_request( + project, integration_branch, project.default_branch + ) + if created: + common.crosslink_merge_requests(source_mr, mr) + print("Created new merge request:\n%s" % mr.web_url) + else: + print("Existing integration merge request:\n%s" % mr.web_url) + return integration_branch, mr + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -41,9 +97,9 @@ def main(): ) parser.add_argument( "--project", + "--manifest-project", help="""name of the GitLab project""", dest="project", - required=True, ) parser.add_argument( "--submodule", @@ -72,6 +128,12 @@ def main(): required=False, default=False, ) + parser.add_argument( + "projects", + help="""List of projects the change should be deployed to additionally + to the manifest project given as named parameter.""", + nargs="*", + ) parser.add_argument( "-v", "--verbose", @@ -86,42 +148,37 @@ def main(): gitlab = Gitlab(args.gitlab_url, private_token=args.token) project = common.get_project(gitlab, args.project) - # Update submodule - integration_branch, _, submodule_project = update_submodule( + logging.debug("Integrate into: %s", args.project) + + # Update submodule in this project, create MR + integration_branch, mr = deploy_into( project, args.submodule, args.revision, args.branch, - pre_commit_hook=update_rev_in_gitlab_ci, + replace_exising_branch=len(args.projects) > 0, ) + merge_requests = [] + merge_requests.append(mr) - logging.debug("Integration branch: %s", integration_branch) # If submodule is already at specified revision, exit successfully + # TODO: is this correct for multi deploy also? if not integration_branch: sys.exit(0) - # Get source merge request - mrs = get_merge_requests( - submodule_project, - target_branch="master", - commit=args.revision, - ) - if not mrs: - sys.exit( - "ERROR: could not determine source merge request for commit %s" - % args.revision + for p in args.projects: + gitlab_p = common.get_project(gitlab, p) + logging.debug("Integrate into: %s", p) + + integration_branch, mr = deploy_into( + gitlab_p, + args.submodule, + args.revision, + args.branch, ) - source_mr = mrs[0] + merge_requests.append(mr) - # Create merge request - mr, created = create_merge_request( - project, integration_branch, project.default_branch - ) - if created: - common.crosslink_merge_requests(source_mr, mr) - print("Created new merge request:\n%s" % mr.web_url) - else: - print("Existing integration merge request:\n%s" % mr.web_url) + logging.debug("Integration branch: %s", integration_branch) if not args.merge: print( @@ -132,11 +189,12 @@ def main(): ) sys.exit(0) + # TODO handle all MRs here # Wait until GitLab has checked merge status - common.wait_until_merge_status_is_set(project, mr) + common.wait_until_merge_status_is_set(project, merge_requests[0]) # Attempt to merge - merged = accept_merge_request(project, mr, rebase=True) + merged = accept_merge_request(project, merge_requests[0], rebase=True) if not merged: sys.exit( diff --git a/foobar-manifest-integration.yml b/foobar-manifest-integration.yml index 6ebf3279707beeeed8d2de678ddb8f8d87d41ed6..cbbaf80efa8b34074f5356e5fd72c3bdd6bfd116 100644 --- a/foobar-manifest-integration.yml +++ b/foobar-manifest-integration.yml @@ -29,7 +29,23 @@ stages: workflow: rules: - # Do not run pipelines on forked projects + # Explicitly allow externally triggered pipelines in every case + - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api" + # Do not run pipelines for merge requests for integrate/gitlab-ci/ branches + # These are trigger explicitly from the integration pipeline in gitlab-ci repo + - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/ + when: never + # Do not run pipelines on forked projects. + # The pipelines would not work anyway because of the users permissions. + # There are two cases catched here: + # 1. The project is forked into someones gitlab namespace and a MR to + # include a change into this forked project is created. In this case + # is the CI_PROJECT_ROOT_NAMESPACE not SECO-Northern-Europe but the + # namespace the fork lives in. + # 2. The MR from the forked project is created to merge the change into this + # the project in the SECO-Northern-Europe namespace (customer sending + # change to us). Here the the IDs used below differ. + # - if: $CI_PROJECT_ROOT_NAMESPACE == "SECO-Northern-Europe" && $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID @@ -62,6 +78,7 @@ integrate: cache: policy: push script: + - printenv - cd ${CI_PROJECT_DIR} - .gitlab-ci/integrate_into_manifest.py --gitlab-url=${CI_SERVER_URL} @@ -71,6 +88,7 @@ integrate: --project=${CI_PROJECT_PATH} --merge-request=${CI_MERGE_REQUEST_IID} --save-revision-to=manifest_revision + --verbose artifacts: paths: - manifest_revision @@ -105,7 +123,10 @@ merge: build: stage: build rules: - - if: $CI_MERGE_REQUEST_IID + # execute this in MR only and not for integrate/gitlab-ci/ integrations + # branches. These are build after the integration has been done in all + # projects + - if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/ trigger: project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}" diff --git a/integrate_into_manifest.py b/integrate_into_manifest.py index dec97e8334b80f319ef72cd6b25ef67655957dde..4159c123d8986980490391beb0349dc38906d325 100755 --- a/integrate_into_manifest.py +++ b/integrate_into_manifest.py @@ -2,6 +2,7 @@ import common import argparse +import logging import sys import tempfile import re @@ -42,14 +43,36 @@ def integrate_into_manifest( except IndexError: sys.exit("ERROR: branch '%s' not found" % integration_base) - # Create integration branch (delete former one if already exists) - integration_branch = common.integration_branch_name( - project.name, merge_request.source_branch - ) - for ref in manifest_repo.references: - if integration_branch == ref.name: - manifest_repo.delete_head(ref) - manifest_repo.head.set_reference(manifest_repo.create_head(integration_branch)) + # Special handling for the gitlab-ci integration + # When the branch 'merge_request.source_branch' already starts with + # integrate/gitlab-ci we add our new commit to this branch + # Otherwise (normal behaviour) a new integration branch is created + integration_branch = None + if merge_request.source_branch.startswith("integrate/gitlab-ci"): + logging.debug("Integration of gitlab-ci: %s", merge_request.source_branch) + for ref in manifest_repo.references: + # remove 'origin/' from the ref before compare + refname = ref.name.split("/", 1)[1] + logging.debug("Found ref: %s", refname) + if merge_request.source_branch == ref.name: + logging.debug("Found integration for gitlab-ci") + manifest_repo.head.set_reference(ref) + integration_branch = merge_request.source_branch + break + + if integration_branch is None: + # Create integration branch (delete former one if already exists) + integration_branch = common.integration_branch_name( + project.name, merge_request.source_branch + ) + for ref in manifest_repo.references: + if integration_branch == ref.name: + manifest_repo.delete_head(ref) + manifest_repo.head.set_reference( + manifest_repo.create_head(integration_branch) + ) + + logging.debug("Integration branch: %s", integration_branch) # Parse manifest file try: @@ -199,8 +222,16 @@ def main(): dest="revision_file", required=False, ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="""Increase verbosity.""", + ) args, _ = parser.parse_known_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) gitlab = Gitlab(args.gitlab_url, private_token=args.token) diff --git a/update_submodule.py b/update_submodule.py index 64d19e16f3ca74e08f383722f7146a4ee232547f..b0d5930ef9d92f587a8ea21f4c71deb735b21e41 100755 --- a/update_submodule.py +++ b/update_submodule.py @@ -12,7 +12,12 @@ from gitlab import Gitlab def update_submodule( - project, submodule_name, submodule_revision, branch=None, pre_commit_hook=None + project, + submodule_name, + submodule_revision, + branch=None, + pre_commit_hook=None, + replace_exising_branch=False, ): """Update submodule of gitlab project to given revision @@ -23,6 +28,7 @@ def update_submodule( branch (string): branch to update, if None, the projects default branch is used pre_commit_hook: Function to be called before the actual commit is done, to add additional changes. Arguments passed: ( repo, submodule_project, submodule_revision) + replace_exising_branch: When an existing integration branch is found it is always replaced. Returns: tuple of: branch (string): Name of the newly created integration branch revision (string): hexsha of the new commit @@ -116,16 +122,16 @@ def update_submodule( if existing_branch: repo.head.set_reference(existing_branch) submodule = common.get_submodule(repo, submodule_name) - if submodule.hexsha == submodule_revision: + if replace_exising_branch or submodule.hexsha != submodule_revision: + print("Replacing outdated integration branch %s" % integration_branch) + repo.head.set_reference(branch) + submodule = common.get_submodule(repo, submodule_name) + else: print( "Submodule is already at %s on branch %s" % (submodule_revision, integration_branch) ) return (integration_branch, existing_branch.commit, submodule_project) - else: - print("Replacing outdated integration branch %s" % integration_branch) - repo.head.set_reference(branch) - submodule = common.get_submodule(repo, submodule_name) else: print("Creating integration branch %s" % integration_branch)