deploy_gitlab_ci.py

#!/usr/bin/env python3
import common

import argparse
import logging
import sys
import os
from gitlab import Gitlab

from accept_merge_request import accept_merge_request
from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
from update_submodule import (
    update_submodule_and_include_ref,
    get_submodule_project_path_and_revision,
)
from integrate_into_manifest import update_manifest, update_srcrev

from ruamel.yaml import YAML


def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):

    # Read values from existing file
    yaml = YAML()
    data = yaml.load(gitlab_ci_yml)
    logging.debug("Yaml: %s", data)

    try:
        masterbranch = data["variables"]["MASTER_BRANCH_PROJECT"]
        logging.debug("Masterbranch %s", masterbranch)
    except KeyError:
        masterbranch = None
    try:
        recipe = data["variables"]["BB_RECIPE_NAME"]
        logging.debug("Recipe %s", recipe)
    except KeyError:
        recipe = None
    return {"recipe": recipe, "masterbranch": masterbranch}


def integrate_submodule_into(
    gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
):

    gitlab_project = common.get_project(gitlab, project_name)

    (
        project_repo,
        integration_branch_name,
        integration_commit,
        message,
    ) = update_submodule_and_include_ref(
        gitlab_project,
        submodule_name,
        new_revision,
        branch,
        commit_and_push=commit_and_push,
    )
    if integration_branch_name is None:
        return None
    # ======================================
    # Store the references for creating the integration
    # commit in the manifest later
    # ======================================
    ret = {
        "project": gitlab_project,
        "repo": project_repo,
        "branch": integration_branch_name,
        "commit": integration_commit,
        "message": message,
    }
    logging.debug(
        "Integration branch: %s (%s)",
        integration_branch_name,
        integration_commit,
    )
    return ret


def create_integration_merge_request(project, integration_branch_name, source_mr=None):
    # Create merge request
    # This should be optional
    mr, created = create_merge_request(
        project, integration_branch_name, project.default_branch
    )
    if created:
        if source_mr is not None:
            common.crosslink_merge_requests(source_mr, mr)
        print("Created new merge request:\n%s" % mr.web_url)
    else:
        print("Existing integration merge request:\n%s" % mr.web_url)
    return mr


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--gitlab-url",
        help="""URL to the GitLab instance""",
        dest="gitlab_url",
        required=True,
    )
    parser.add_argument(
        "--token",
        help="""GitLab REST API private access token""",
        dest="token",
        required=True,
    )
    parser.add_argument(
        "--project",
        "--manifest-project",
        help="""name of the GitLab project""",
        dest="project",
    )
    parser.add_argument(
        "--submodule",
        help="""submodule to update""",
        dest="submodule",
        required=True,
    )
    parser.add_argument(
        "--revision",
        help="""new revision for submodule""",
        dest="revision",
        required=True,
    )
    parser.add_argument(
        "--branch",
        help="""project branch (if not default branch)""",
        dest="branch",
        required=False,
        default=None,
    )
    parser.add_argument(
        "--merge",
        help="""if set, perform merge after integration""",
        dest="merge",
        action="store_true",
        required=False,
        default=False,
    )
    parser.add_argument(
        "--manifest-file",
        help="""manifest file name (default: 'default.xml')""",
        dest="manifest_file",
        default=common.manifest_file,
        required=False,
    )
    parser.add_argument(
        "--srcrev-file",
        help="""source revision file name (default: 'SRCREV.conf')""",
        dest="srcrev_file",
        default=common.srcrev_file,
        required=False,
    )
    parser.add_argument(
        "projects",
        help="""List of projects the change should be deployed to additionally
                to the manifest project given as named parameter.""",
        nargs="*",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="""Increase verbosity.""",
    )

    args, _ = parser.parse_known_args()
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)

    gitlab = Gitlab(args.gitlab_url, private_token=args.token)

    # =======================================================
    # Create integration branches and commits with updates
    # submodule in all projects
    # =======================================================
    project_integration = {}
    # Update submodule in all 'child' project
    for p in args.projects:
        logging.debug("Integrate into: %s", p)

        res = integrate_submodule_into(
            gitlab, p, args.submodule, args.revision, args.branch
        )
        if res is not None:
            project_integration[p] = res

    # Update submodule in manifest project
    manifest_project = integrate_submodule_into(
        gitlab,
        args.project,
        args.submodule,
        args.revision,
        args.branch,
        commit_and_push=False,
    )
    if manifest_project is not None:
        project_integration[args.project] = manifest_project

    # If submodule is already at specified revision in all projects, exit successfully
    if len(project_integration) == 0:
        print("No integration done, changes are already included in all projects.")
        sys.exit(0)

    # =======================================================
    # Create and merge merge_requests if needed
    # =======================================================
    if args.merge:
        # Get source merge request ( the one in the gitlab-ci repo)
        submodule_project_path, _ = get_submodule_project_path_and_revision(
            manifest_project["project"], args.submodule, args.branch
        )
        submodule_project = common.get_project(gitlab, submodule_project_path)
        mrs = get_merge_requests(
            submodule_project,
            # TODO should this be submodule_project's default branch?
            target_branch="master",
            commit=args.revision,
        )
        if not mrs:
            sys.exit(
                "ERROR: could not determine source merge request for commit %s"
                % args.revision
            )
        source_mr = mrs[0]

        for p in args.projects:
            integration = project_integration[p]
            logging.debug("Create MR in %s", integration["project"].name)
            mr = create_integration_merge_request(
                integration["project"], integration["branch"], source_mr
            )
            integration["mr"] = mr
            # Now merge
            logging.debug("Merge %s!%s", p, mr.iid)

            # Wait until GitLab has checked merge status
            common.wait_until_merge_status_is_set(integration["project"], mr)

            # Attempt to merge
            merged, integration_commit = accept_merge_request(
                integration["project"], mr, rebase=True
            )
            # if this has rebased the integration commit needs to be adapted:
            project_integration[p]["commit"] = integration_commit

            if not merged:
                sys.exit(
                    "Integration MR could not be merged. You have two possibilities to fix "
                    "this:\n"
                    "  1. Checkout the MR and rebase it on the current master manually, or\n"
                    "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
                    "In either case restart this job afterwards in order to get it merged."
                )

        print("Successfully merged")

    # =======================================================
    # Now create the integration commit in the manifest
    # for all subprojects at once
    # =======================================================
    manifest_file_abs = os.path.join(
        manifest_project["repo"].working_tree_dir, args.manifest_file
    )
    logging.debug("Read manifest from: %s", manifest_file_abs)
    with open(manifest_file_abs, "r", encoding="utf8") as fp:
        manifest = fp.read()
    logging.debug(manifest)
    srcrev_file_abs = os.path.join(
        manifest_project["repo"].working_tree_dir, args.srcrev_file
    )
    logging.debug("Read manifest from: %s", srcrev_file_abs)
    with open(srcrev_file_abs, "r", encoding="utf8") as fp:
        srcrev = fp.read()
    logging.debug(srcrev)

    for p in args.projects:
        integration = project_integration[p]
        logging.debug(
            "Update %s to %s", integration["project"].name, integration["commit"]
        )

        new_manifest = update_manifest(
            manifest, integration["project"], integration["commit"]
        )
        if new_manifest is not None:
            manifest = new_manifest
            logging.debug(manifest)
            continue

        # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
        gitlab_ci_yml_file = os.path.join(
            integration["repo"].working_tree_dir, ".gitlab-ci.yml"
        )
        logging.debug("Read recipe name from %s", gitlab_ci_yml_file)
        with open(gitlab_ci_yml_file, "r", encoding="utf8") as fp:
            gitlab_ci_yml = fp.read()
        project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)

        new_srcrev = update_srcrev(
            srcrev, project_keys["recipe"], integration["commit"]
        )
        if new_srcrev is not None:
            srcrev = new_srcrev
            logging.debug(srcrev)
        else:
            logging.debug("Project %s not found in xml or srcrev file", p)

    # Write manifest
    with open(manifest_file_abs, "w", encoding="utf8") as fp:
        fp.write(manifest)
    manifest_project["repo"].git.add(args.manifest_file)
    logging.debug(manifest)
    with open(srcrev_file_abs, "w", encoding="utf8") as fp:
        fp.write(srcrev)
    manifest_project["repo"].git.add(args.srcrev_file)
    logging.debug(srcrev)

    # ========================================================
    # Now commit and push the changes to the manifest repo
    # ========================================================
    # Make an API request to create the gitlab.user object
    gitlab = manifest_project["project"].manager.gitlab
    gitlab.auth()
    integration_commit = common.commit_and_push(
        manifest_project["project"],
        manifest_project["repo"],
        manifest_project["branch"],
        manifest_project["message"],
        gitlab.user.username,
        gitlab.user.email,
    )

    print(
        "Successfully create integration commit {} in {}".format(
            integration_commit, args.project
        )
    )

    if not args.merge:
        sys.exit(0)

    # ============================================
    # Create merge requests for the manifest
    # ============================================

    logging.debug("Create MR in %s", manifest_project["project"].name)
    manifest_project["mr"] = create_integration_merge_request(
        manifest_project["project"], manifest_project["branch"], source_mr
    )
    # =================================================
    # Now merge it
    # =================================================
    # The manifest needs to be merged at last
    mr = manifest_project["mr"]
    logging.debug("Merge %s!%s", args.project, mr.iid)

    # Wait until GitLab has checked merge status
    common.wait_until_merge_status_is_set(manifest_project["project"], mr)

    # Attempt to merge
    merged = accept_merge_request(manifest_project["project"], mr, rebase=True)

    if not merged:
        sys.exit(
            "Integration MR could not be merged. You have two possibilities to fix "
            "this:\n"
            "  1. Checkout the MR and rebase it on the current master manually, or\n"
            "  2. Delete the MR (Edit -> Delete in the MR UI)\n"
            "In either case restart this job afterwards in order to get it merged."
        )

    print("Successfully merged")


if __name__ == "__main__":
    main()