common.py

#!/usr/bin/env python3

import logging
import requests
import sys
import time
from furl import furl
from git import Actor, GitCommandError
from git.repo.base import Repo
from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
from gitlab.v4.objects import Project
from gitlab.v4.objects import MergeRequest


manifest_file = "default.xml"
srcrev_file = "SRCREV.conf"
pending_states = ["created", "waiting_for_resource", "preparing", "pending", "running"]


def integration_branch_name(project_name, branch_name):
    """Get integration branch name"""
    return "integrate/" + project_name.lower() + "/" + branch_name


def find_gitlab_ci_integration_branch(repo: Repo, branch_name):
    """
    # Special handling for the gitlab-ci integration
    # When the branch 'merge_request.source_branch' already starts with
    # integrate/gitlab-ci we add our new commit to this branch
    # Otherwise (normal behaviour) a new integration branch is created
    """
    if not branch_name.startswith("integrate/gitlab-ci"):
        return None

    logging.debug("Integration of gitlab-ci: %s", branch_name)
    for ref in repo.references:
        refname = ref.name
        logging.debug("Found ref: %s", refname)
        if not refname.startswith("origin/"):
            continue
        # remove 'origin/' from the ref before compare
        refname = ref.name.split("/", 1)[1]
        logging.debug("Splitted refname: %s", refname)
        if branch_name == refname:
            integration_branch = refname
            logging.debug(
                "Found integration branch for gitlab-ci: %s", integration_branch
            )
            return integration_branch


def get_project(gitlab, project_name):
    """Get a GitLab project by its name (including or excluding namespace)"""
    project = None
    try:
        # First try direct access, assuming name contains full path including namespace
        try:
            project = gitlab.projects.get(project_name, retry_transient_errors=True)
        except GitlabGetError:
            pass
        # If not found, try searching for project, assuming only name given
        if not project:
            for p in gitlab.projects.list(
                search=project_name, retry_transient_errors=True
            ):
                if p.name == project_name:
                    project = p
        if not project:
            sys.exit("ERROR: project '%s' not found" % project_name)
    except requests.ConnectionError:
        sys.exit("ERROR: could not connect to GitLab server")
    except GitlabAuthenticationError:
        sys.exit("ERROR: authentication failed")
    return project


def get_latest_commit(project, branch_name):
    """Get latest commit on a given project branch"""
    try:
        branch = project.branches.get(branch_name, retry_transient_errors=True)
    except GitlabGetError as e:
        sys.exit(
            "ERROR: could not get branch '%s' for project '%s': %s"
            % (branch_name, project.name, e)
        )
    if not branch:
        sys.exit(
            "ERROR: branch '%s' not found in project %s" % (branch_name, project.name)
        )
    return branch.commit


def rebase_merge_request(project, merge_request):
    """Attempt to rebase a merge request and return the updated merge request object"""
    # Rebasing takes more than one API call, see:
    # https://docs.gitlab.com/ce/api/merge_requests.html#rebase-a-merge-request
    try:
        merge_request.rebase()
    except GitlabMRRebaseError as e:
        merge_request.merge_error = "Could not rebase merge request: %s" % e
        return merge_request
    rebase_in_progress = True
    while rebase_in_progress:
        time.sleep(1)
        try:
            updated_merge_request = project.mergerequests.get(
                id=merge_request.iid,
                query_parameters={"include_rebase_in_progress": "True"},
                retry_transient_errors=True,
            )
        except GitlabGetError as e:
            merge_request.merge_error = "Could not get updated merge request: %s" % e
            return merge_request
        rebase_in_progress = updated_merge_request.rebase_in_progress
    return updated_merge_request


def crosslink_merge_requests(source_mr: MergeRequest, integration_mr: MergeRequest):
    """Insert cross-links in merge requests"""
    integration_mr.notes.create(
        {"body": "Source merge request: %s" % source_mr.web_url}
    )
    source_mr.notes.create(
        {"body": "Integration merge request: %s" % integration_mr.web_url}
    )


def wait_until_merge_status_is_set(project: Project, mr: MergeRequest):
    """Periodically query MR until GitLab has checked its merge status"""
    print("Waiting until merge status has been checked", end="", flush=True)
    unchecked_states = ["unchecked", "checking", "cannot_be_merged_recheck"]
    mr = project.mergerequests.get(mr.iid, retry_transient_errors=True)
    while mr.merge_status in unchecked_states:
        print(".", end="", flush=True)
        time.sleep(1)
        mr = project.mergerequests.get(mr.iid, retry_transient_errors=True)
    print(" -> %s" % mr.merge_status)


def list_commits(commits):
    """Create a list of commits along with the commit messages"""
    commit_list = ""
    for commit in commits:
        commit_list += "\n--\n\nCommit: %s\n\n%s" % (commit.web_url, commit.message)
    return commit_list


def commit_and_push(project: Project, repo: Repo, branch, message, name, email):
    """Commit and push to a repo branch"""
    author = Actor(name, email)
    repo.index.commit(message, author=author, committer=author)
    print(repo.git.log("--oneline", "-n", "5"))

    # Push commit
    try:
        origin = repo.remote("origin")
        logging.debug("Push branch %s to %s", branch, origin)
        origin.push(branch, force=True)
    except GitCommandError as e:
        sys.exit("ERROR: could not commit changes\n" + str(e))

    # Print commit information
    revision = repo.head.commit.hexsha
    print("Pushed new commit:")
    print(project.web_url + "/-/commit/" + revision)
    print(repo.git.show("--summary", "--decorate"))

    return revision


def get_submodule(repo: Repo, submodule_name):
    """Find a submodule in a Git repository by its name"""
    submodule = None
    for sm in repo.submodules:
        if sm.name == submodule_name:
            submodule = sm
    if submodule is None:
        sys.exit("ERROR: submodule '%s' not found" % submodule_name)
    return submodule


def extract_message_body(msg):
    """Extract message body out of a commit message"""
    # Remove headline
    msg = msg.split("\n", 1)[-1]

    # Remove all newlines, whitespaces and hyphens from the beginning
    while msg[0] in ["\n", " ", "-"]:
        msg = msg[1:]

    return msg


def get_merge_request(project: Project, merge_request):
    """Return a gitlab mergereqest specified either by id or by link"""

    # MR may also be specified as
    # SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-bar!115
    if "!" in merge_request:
        merge_request = int(merge_request.split("!")[-1])
        logging.debug("Number of MR: %d", merge_request)
    try:
        mr = project.mergerequests.get(merge_request, retry_transient_errors=True)
    except GitlabGetError:
        return None
    return mr


def clone_project(project: Project, into, branch=None):

    gitlab = project.manager.gitlab
    # If no branch is given, use project's default branch
    if branch is None:
        branch = project.default_branch

    # Construct clone url containing access token
    clone_url = furl(project.http_url_to_repo)
    clone_url.username = "gitlab-ci"
    clone_url.password = gitlab.private_token

    # Checkout project
    try:
        repo = Repo.clone_from(clone_url.url, into, branch=branch, depth=1)
    except GitCommandError as e:
        raise Exception("could not clone repository\n" + str(e)) from e
    except IndexError:
        raise Exception("branch '%s' not found" % branch) from e
    return repo


def get_repository_file_raw(project: Project, filename, ref=None):
    # TODO tree objects are not supported
    fileobj = get_repository_file_obj(project, filename, ref)
    return project.repository_raw_blob(
        fileobj["id"], retry_transient_errors=True
    ).decode()


def get_repository_file_obj(project: Project, filename, ref=None):
    # TODO tree objects are not supported
    if ref is None:
        ref = project.default_branch
        logging.debug("Using default branch %s", ref)

    repository_tree = project.repository_tree(
        ref=ref, all=True, retry_transient_errors=True
    )
    logging.debug(repository_tree)
    fileobj = [f for f in repository_tree if f["name"] == filename]

    if len(fileobj) == 0:
        logging.error("Could not find file %s", filename)
        for f in repository_tree:
            logging.debug(f["name"])
        return None
    fileobj = fileobj[0]
    return fileobj


def is_commit_parent_of_project_commit(project: Project, project_commit, commit):
    """Walks through the commits of project, starting with project_commit
    and compares its sha with the given commit.
    Both commits are specified as sha
    """
    try:
        _ = project.commits.get(commit, retry_transient_errors=True)
    except GitlabGetError as e:
        raise Exception(
            "Failed to find commit {} in {}".format(project_commit, project.name)
        ) from e

    # Loop over the parent commits until commit is found
    parent_id = project_commit
    while True:
        try:
            parent = project.commits.get(parent_id, retry_transient_errors=True)
        except GitlabGetError as e:
            raise Exception(
                "Failed to find commit {} in {}".format(parent_id, project.name)
            ) from e

        # The integration branch is up to date if its parent is the integration base
        logging.debug("Compare '%s' and '%s'", parent.id, commit)
        if parent.id == commit:
            return True
        if len(parent.parent_ids) == 0:
            return False
        parent_id = parent.parent_ids[0]  # Assume linear history