integrate_into_manifest.py

#!/usr/bin/env python3
import common

import argparse
import sys
import tempfile
import re
from pathlib import Path
from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab, GitlabGetError
from lxml import etree


def integrate_into_manifest(
    manifest_project,
    integration_base,
    manifest_file,
    srcrev_file,
    recipe_name,
    project,
    merge_request,
):
    gitlab = manifest_project.manager.gitlab

    with tempfile.TemporaryDirectory() as manifest_dir:
        manifest_filepath = Path(manifest_dir) / manifest_file
        srcrev_filepath = Path(manifest_dir) / srcrev_file

        # Construct clone url containing access token
        clone_url = furl(manifest_project.http_url_to_repo)
        clone_url.username = "gitlab-ci"
        clone_url.password = gitlab.private_token

        # Checkout manifest
        try:
            manifest_repo = Repo.clone_from(
                clone_url.url, manifest_dir, branch=integration_base
            )
        except GitCommandError as e:
            sys.exit("ERROR: could not clone manifest repository\n" + str(e))
        except IndexError:
            sys.exit("ERROR: branch '%s' not found" % integration_base)

        # Create integration branch (delete former one if already exists)
        integration_branch = common.integration_branch_name(
            project.name, merge_request.source_branch
        )
        for ref in manifest_repo.references:
            if integration_branch == ref.name:
                manifest_repo.delete_head(ref)
        manifest_repo.head.set_reference(manifest_repo.create_head(integration_branch))

        # Parse manifest file
        try:
            manifest = etree.parse(manifest_filepath.as_posix())
        except FileNotFoundError:
            sys.exit("ERROR: file '%s' not found in manifest repo" % manifest_file)

        # Find project reference in manifest
        # We are using str.endswith() for this in order to support sub-projects as well
        # (e.g."mygroup/myproject", when only "myproject" is given)
        project_node = None
        project_nodes = manifest.findall("project")
        for node in project_nodes:
            name = node.get("name")
            if name is not None and name.endswith(project.path):
                project_node = node
        if project_node is None:
            if recipe_name is None:
                sys.exit(
                    "ERROR: project '%s' not found in manifest and "
                    "no recipe name is specified" % project.path
                )
            # Check if project is referenced in SRCREV.conf
            project_line = None
            content = srcrev_filepath.read_text()
            # Match "...RECIPE_NAME ="
            pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
            for line in content.splitlines():
                if pattern.search(line):
                    project_line = line
            if project_line is None:
                sys.exit(
                    "ERROR: project '%s' not found in manifest and "
                    "SRCREV file" % project.path
                )

            # Get current project revision from SRCREV file
            # Assuming notation: <project> = "<hash>"
            old_revision = project_line.split('"')[1]

            # Get new project revision from merge request
            new_revision = merge_request.sha

            # Update SRCREV file
            content = content.replace(old_revision, new_revision)
            srcrev_filepath.write_text(content)
            manifest_repo.index.add([srcrev_file])
        else:
            # Get current project revision from manifest
            old_revision = project_node.get("revision")

            # Get new project revision from merge request
            new_revision = merge_request.sha

            # Update manifest file
            # We are doing this using a plain text replace action. Unfortunately
            # all python libraries for handling XML data are not able to preserve
            # the file layout, and we want a minimal diff.
            content = manifest_filepath.read_text()
            content = content.replace(old_revision, new_revision)
            manifest_filepath.write_text(content)
            manifest_repo.index.add([manifest_file])

        # Make an API request to create the gitlab.user object
        gitlab.auth()

        # Construct commit message and commit the change
        message = "Integrate %s/%s\n%s" % (
            project.path,
            merge_request.source_branch,
            common.list_commits(merge_request.commits()),
        )
        manifest_revision = common.commit_and_push(
            manifest_project,
            manifest_repo,
            integration_branch,
            message,
            gitlab.user.username,
            gitlab.user.email,
        )

        return manifest_revision


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--gitlab-url",
        help="""URL to the GitLab instance""",
        dest="gitlab_url",
        required=True,
    )
    parser.add_argument(
        "--token",
        help="""GitLab REST API private access token""",
        dest="token",
        required=True,
    )
    parser.add_argument(
        "--manifest-project",
        help="""name of the manifest project""",
        dest="manifest_project",
        required=True,
    )
    parser.add_argument(
        "--integration-base",
        help="""manifest branch to branch off from""",
        dest="integration_base",
        required=True,
    )
    parser.add_argument(
        "--manifest-file",
        help="""manifest file name (default: 'default.xml')""",
        dest="manifest_file",
        default=common.manifest_file,
        required=False,
    )
    parser.add_argument(
        "--srcrev-file",
        help="""source revision file name (default: 'SRCREV.conf')""",
        dest="srcrev_file",
        default=common.srcrev_file,
        required=False,
    )
    parser.add_argument(
        "--recipe-name",
        help="""recipe name to resolve project in 'SRCREV.conf'""",
        dest="recipe_name",
        default=None,
        required=False,
    )
    parser.add_argument(
        "--project",
        help="""name of the project, as specified in the manifest""",
        dest="project",
        required=True,
    )
    parser.add_argument(
        "--merge-request",
        help="""project merge request IID containing the changes to be integrated""",
        dest="merge_request",
        required=True,
    )
    parser.add_argument(
        "--save-revision-to",
        help="""path to a file where the new manifest revision is stored""",
        dest="revision_file",
        required=False,
    )

    args, _ = parser.parse_known_args()

    gitlab = Gitlab(args.gitlab_url, private_token=args.token)

    manifest_project = common.get_project(gitlab, args.manifest_project)
    project = common.get_project(gitlab, args.project)
    try:
        merge_request = project.mergerequests.get(
            args.merge_request, retry_transient_errors=True
        )
    except GitlabGetError as e:
        sys.exit(
            "ERROR: could not get %s!%s: %s"
            % (project.name, args.merge_request, e.error_message)
        )

    manifest_revision = integrate_into_manifest(
        manifest_project=manifest_project,
        integration_base=args.integration_base,
        manifest_file=args.manifest_file,
        srcrev_file=args.srcrev_file,
        recipe_name=args.recipe_name,
        project=project,
        merge_request=merge_request,
    )

    if args.revision_file:
        with open(args.revision_file, "w", encoding="utf-8") as file:
            file.write(manifest_revision)


if __name__ == "__main__":
    main()