deploy_gitlab_ci.py

#!/usr/bin/env python3
import common

import argparse
import logging
import sys
import os
from gitlab import Gitlab
from gitlab.v4.objects import Project, MergeRequest

from accept_merge_request import accept_merge_request
from create_merge_request import create_merge_request
from get_integration_sources import get_integration_sources
from get_merge_requests import get_merge_requests
from update_submodule import update_submodule_and_include_ref
from integrate_into_manifest import update_manifest, update_srcrev

from ruamel.yaml import YAML


def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):

    # Read values from existing file
    yaml = YAML()
    data = yaml.load(gitlab_ci_yml)
    logging.debug("Yaml: %s", data)

    try:
        recipe = data["variables"]["BB_RECIPE_NAME"]
        logging.debug("Recipe %s", recipe)
    except KeyError:
        recipe = None
    return {"recipe": recipe}


def integrate_submodule_into(
    gitlab,
    project_name,
    submodule_name,
    new_revision,
    branch,
    commit_and_push=True,
    force_clone=False,
):

    gitlab_project = common.get_project(gitlab, project_name)

    (
        project_repo,
        project_dir,
        integration_branch_name,
        integration_commit,
        message,
    ) = update_submodule_and_include_ref(
        gitlab_project,
        submodule_name,
        new_revision,
        branch,
        commit_and_push=commit_and_push,
        force_clone=force_clone,
    )
    # ======================================
    # Store the references for creating the integration
    # commit in the manifest later
    # ======================================
    ret = {
        "project": gitlab_project,
        "repo": project_repo,
        "dir": project_dir,
        "integration_branch": integration_branch_name,
        "master_branch": branch,
        "commit": integration_commit,
        "message": message,
    }
    logging.debug(
        "Integration branch: %s (%s)",
        integration_branch_name,
        integration_commit,
    )
    return ret


def create_integration_merge_request(
    project: Project,
    integration_branch: str,
    target_branch: str,
    source_mr: MergeRequest = None,
) -> MergeRequest:
    # Create merge request
    # This should be optional
    mr, created = create_merge_request(project, integration_branch, target_branch)
    if created:
        if source_mr is not None:
            common.crosslink_merge_requests(source_mr, mr)
        print("Created new merge request:\n%s" % mr.web_url)
    else:
        print("Existing integration merge request:\n%s" % mr.web_url)
    return mr


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--gitlab-url",
        help="""URL to the GitLab instance""",
        dest="gitlab_url",
        required=True,
    )
    parser.add_argument(
        "--token",
        help="""GitLab REST API private access token""",
        dest="token",
        required=True,
    )
    parser.add_argument(
        "--manifest-project",
        help="""name of the manifest project""",
        dest="manifest_project",
        required=True,
    )
    parser.add_argument(
        "--manifest-branch",
        help="""manifest branch to integrate changes into (can be a comma-separated list)""",
        dest="manifest_branch",
        required=True,
    )
    parser.add_argument(
        "--submodule",
        help="""submodule to update""",
        dest="submodule",
        required=True,
    )
    parser.add_argument(
        "--revision",
        help="""new revision for submodule""",
        dest="revision",
        required=True,
    )
    parser.add_argument(
        "--merge",
        help="""if set, perform merge after integration""",
        dest="merge",
        action="store_true",
        required=False,
        default=False,
    )
    parser.add_argument(
        "--project",
        help="""gitlab-ci project path or id""",
        dest="project",
        default=os.environ.get("CI_PROJECT_PATH"),
        required=False,
    )
    parser.add_argument(
        "--branch",
        help="""gitlab-ci branch that we're merging into""",
        dest="branch",
        default="master",
        required=False,
    )
    parser.add_argument(
        "--manifest-file",
        help="""manifest file name (default: 'default.xml')""",
        dest="manifest_file",
        default=common.manifest_file,
        required=False,
    )
    parser.add_argument(
        "--srcrev-file",
        help="""source revision file name (default: 'SRCREV.conf')""",
        dest="srcrev_file",
        default=common.srcrev_file,
        required=False,
    )
    parser.add_argument(
        "--group",
        help="""group path or id to limit search scope to""",
        dest="group",
        required=True,
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="""Increase verbosity.""",
    )

    args, _ = parser.parse_known_args()
    if args.verbose:
        logging.basicConfig(
            level=logging.DEBUG,
            format="%(asctime)s %(levelname)-8s %(message)s",
            datefmt="%H:%M:%S",
        )

    manifest_branches = args.manifest_branch.split(",")

    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
    group = gitlab.groups.get(args.group)

    # =======================================================
    # Create integration branches and commits with updates
    # submodule in all projects
    # =======================================================
    integration_sources = {}
    all_integration_sources = []
    for manifest_branch in manifest_branches:
        print(
            "Searching for projects in %s that are configured for automatic integration into %s:%s"
            % (args.group, args.manifest_project, manifest_branch)
        )
        integration_sources[manifest_branch] = get_integration_sources(
            args.manifest_project, manifest_branch, group
        )
        for s in integration_sources[manifest_branch]:
            if s not in all_integration_sources:
                all_integration_sources.append(s)

    # Update submodule in all integration sources
    project_integrations = []
    for s in all_integration_sources:
        print("Create integration commit in %s:%s" % (s["project"], s["branch"]))

        integration = integrate_submodule_into(
            gitlab, s["project"], args.submodule, args.revision, s["branch"]
        )
        # Store in the list if commit is set (meaning there was an update or
        #   an exising integration branch)
        if integration["commit"] is not None:
            project_integrations.append(integration)

    # Update submodule in all manifest branches
    manifest_integrations = []
    for manifest_branch in manifest_branches:
        print(
            "Create integration commit in %s:%s"
            % (args.manifest_project, manifest_branch),
        )
        manifest_integrations.append(
            integrate_submodule_into(
                gitlab,
                args.manifest_project,
                args.submodule,
                args.revision,
                manifest_branch,
                commit_and_push=False,
                force_clone=True,
            )
        )

    # =======================================================
    # Create and merge merge_requests if needed
    # =======================================================
    if args.merge:
        # Get source merge request (the one in the gitlab-ci repo)
        gitlab_ci_project = common.get_project(gitlab, args.project)
        mrs = get_merge_requests(
            project=gitlab_ci_project,
            target_branch=args.branch,
            state="merged",
            commit=args.revision,
        )
        if not mrs:
            sys.exit(
                "ERROR: could not determine source merge request for commit %s"
                % args.revision
            )
        source_mr = mrs[0]

        for project_integration in project_integrations:
            logging.debug("Create MR in %s", project_integration["project"].name)
            mr = create_integration_merge_request(
                project_integration["project"],
                project_integration["integration_branch"],
                project_integration["master_branch"],
                source_mr,
            )
            # Now merge
            logging.debug("Merge %s!%s", project_integration["project"], mr.iid)

            # Wait until GitLab has checked merge status
            common.wait_until_merge_status_is_set(project_integration["project"], mr)

            # Attempt to merge
            merged, integration_commit = accept_merge_request(
                project_integration["project"], mr, rebase=True
            )
            # if this has rebased the integration commit needs to be adapted:
            project_integration["commit"] = integration_commit
            # Save the target branch here, as the source branch gets deleted
            # during merge
            project_integration["integration_branch"] = mr.target_branch

            if not merged:
                sys.exit(
                    "Integration MR could not be merged:\n"
                    "%s\n"
                    "This can probably be resolved by creating a new commit in "
                    "gitlab-ci and merging it. The above MR can be closed then."
                    % mr.web_url
                )

    # =======================================================
    # Now create the integration commit in the manifest
    # for all subprojects at once
    # =======================================================
    for manifest_integration in manifest_integrations:
        manifest_file_abs = os.path.join(
            manifest_integration["repo"].working_tree_dir, args.manifest_file
        )
        logging.debug("Read manifest from: %s", manifest_file_abs)
        with open(manifest_file_abs, "r", encoding="utf8") as fp:
            manifest = fp.read()
        logging.debug(manifest)
        srcrev_file_abs = os.path.join(
            manifest_integration["repo"].working_tree_dir, args.srcrev_file
        )
        logging.debug("Read manifest from: %s", srcrev_file_abs)
        with open(srcrev_file_abs, "r", encoding="utf8") as fp:
            srcrev = fp.read()
        logging.debug(srcrev)

        for project_integration in project_integrations:
            # Check if project integration belongs to this manifest branch
            for source in integration_sources[manifest_integration["master_branch"]]:
                if (
                    source["project"]
                    == project_integration["project"].path_with_namespace
                    and source["branch"] == project_integration["master_branch"]
                ):
                    logging.debug(
                        "Update %s to %s",
                        project_integration["project"].name,
                        project_integration["commit"],
                    )

                    new_manifest = update_manifest(
                        manifest,
                        project_integration["project"],
                        project_integration["commit"],
                    )
                    if new_manifest is not None:
                        manifest = new_manifest
                        logging.debug(manifest)
                        continue

                    # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
                    # Use direct read from gitlab as we have not checked out
                    # the repo if the branch is already up to date

                    gitlab_ci_yml = common.get_repository_file_raw(
                        project_integration["project"],
                        ".gitlab-ci.yml",
                        ref=project_integration["integration_branch"],
                    )
                    project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)

                    new_srcrev = update_srcrev(
                        srcrev, project_keys["recipe"], project_integration["commit"]
                    )
                    if new_srcrev is not None:
                        srcrev = new_srcrev
                        logging.debug(srcrev)
                    else:
                        logging.debug(
                            "Project %s not found in xml or srcrev file",
                            project_integration["project"],
                        )

        # Write manifest
        with open(manifest_file_abs, "w", encoding="utf8") as fp:
            fp.write(manifest)
        manifest_integration["repo"].git.add(args.manifest_file)
        logging.debug(manifest)
        with open(srcrev_file_abs, "w", encoding="utf8") as fp:
            fp.write(srcrev)
        manifest_integration["repo"].git.add(args.srcrev_file)
        logging.debug(srcrev)

        # ========================================================
        # Squash all commits on the integration branch to one
        # ========================================================
        manifest_integration["repo"].remotes.origin.fetch(
            manifest_integration["master_branch"]
        )
        manifest_master = manifest_integration["project"].branches.get(
            manifest_integration["master_branch"]
        )
        manifest_integration["repo"].git.reset("--soft", manifest_master.commit["id"])

        # ========================================================
        # Now commit and push the changes to the manifest repo
        # ========================================================
        # Make an API request to create the gitlab.user object
        gitlab = integration["project"].manager.gitlab
        gitlab.auth()
        integration_commit = common.commit_and_push(
            manifest_integration["project"],
            manifest_integration["repo"],
            manifest_integration["message"],
            gitlab.user.username,
            gitlab.user.email,
        )

    if not args.merge:
        sys.exit(0)

    # ============================================
    # Create merge requests for the manifest
    # ============================================
    for integration in manifest_integrations:
        logging.debug("Create MR in %s", integration["project"].name)
        mr = create_integration_merge_request(
            integration["project"],
            integration["integration_branch"],
            integration["master_branch"],
            source_mr,
        )
        # =================================================
        # Now merge it
        # =================================================
        # The manifest needs to be merged at last
        logging.debug("Merge %s!%s", args.manifest_project, mr.iid)

        # Wait until GitLab has checked merge status
        common.wait_until_merge_status_is_set(integration["project"], mr)

        # Attempt to merge
        merged = accept_merge_request(integration["project"], mr, rebase=True)

        if not merged:
            sys.exit(
                "Integration MR could not be merged:\n"
                "%s\n"
                "This can probably be resolved by creating a new commit in "
                "gitlab-ci and merging it. The above MR can be closed then."
                % mr.web_url
            )

        print("Successfully merged")


if __name__ == "__main__":
    main()