#!/usr/bin/env python3 import common import argparse import logging import sys import os from gitlab import Gitlab from accept_merge_request import accept_merge_request from create_merge_request import create_merge_request from get_merge_requests import get_merge_requests from update_submodule import ( update_submodule_and_include_ref, get_submodule_project_path_and_revision, ) from integrate_into_manifest import update_manifest, update_srcrev from ruamel.yaml import YAML def read_keys_from_gitlab_ci_yml(gitlab_ci_yml): # Read values from existing file yaml = YAML() data = yaml.load(gitlab_ci_yml) logging.debug("Yaml: %s", data) try: masterbranch = data["variables"]["MASTER_BRANCH_PROJECT"] logging.debug("Masterbranch %s", masterbranch) except KeyError: masterbranch = None try: recipe = data["variables"]["BB_RECIPE_NAME"] logging.debug("Recipe %s", recipe) except KeyError: recipe = None return {"recipe": recipe, "masterbranch": masterbranch} def integrate_submodule_into( gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True ): gitlab_project = common.get_project(gitlab, project_name) ( project_repo, integration_branch_name, integration_commit, message, ) = update_submodule_and_include_ref( gitlab_project, submodule_name, new_revision, branch, commit_and_push=commit_and_push, ) if integration_branch_name is None: return None # ====================================== # Store the references for creating the integration # commit in the manifest later # ====================================== ret = { "project": gitlab_project, "repo": project_repo, "branch": integration_branch_name, "commit": integration_commit, "message": message, } logging.debug( "Integration branch: %s (%s)", integration_branch_name, integration_commit, ) return ret def create_integration_merge_request(project, integration_branch_name, source_mr=None): # Create merge request # This should be optional mr, created = create_merge_request( project, integration_branch_name, project.default_branch ) if created: if source_mr is not None: common.crosslink_merge_requests(source_mr, mr) print("Created new merge request:\n%s" % mr.web_url) else: print("Existing integration merge request:\n%s" % mr.web_url) return mr def main(): parser = argparse.ArgumentParser() parser.add_argument( "--gitlab-url", help="""URL to the GitLab instance""", dest="gitlab_url", required=True, ) parser.add_argument( "--token", help="""GitLab REST API private access token""", dest="token", required=True, ) parser.add_argument( "--project", "--manifest-project", help="""name of the GitLab project""", dest="project", ) parser.add_argument( "--submodule", help="""submodule to update""", dest="submodule", required=True, ) parser.add_argument( "--revision", help="""new revision for submodule""", dest="revision", required=True, ) parser.add_argument( "--branch", help="""project branch (if not default branch)""", dest="branch", required=False, default=None, ) parser.add_argument( "--merge", help="""if set, perform merge after integration""", dest="merge", action="store_true", required=False, default=False, ) parser.add_argument( "--manifest-file", help="""manifest file name (default: 'default.xml')""", dest="manifest_file", default=common.manifest_file, required=False, ) parser.add_argument( "--srcrev-file", help="""source revision file name (default: 'SRCREV.conf')""", dest="srcrev_file", default=common.srcrev_file, required=False, ) parser.add_argument( "projects", help="""List of projects the change should be deployed to additionally to the manifest project given as named parameter.""", nargs="*", ) parser.add_argument( "-v", "--verbose", action="store_true", help="""Increase verbosity.""", ) args, _ = parser.parse_known_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) gitlab = Gitlab(args.gitlab_url, private_token=args.token) # ======================================================= # Create integration branches and commits with updates # submodule in all projects # ======================================================= project_integration = {} # Update submodule in all 'child' project for p in args.projects: logging.debug("Integrate into: %s", p) res = integrate_submodule_into( gitlab, p, args.submodule, args.revision, args.branch ) if res is not None: project_integration[p] = res # Update submodule in manifest project manifest_project = integrate_submodule_into( gitlab, args.project, args.submodule, args.revision, args.branch, commit_and_push=False, ) if manifest_project is not None: project_integration[args.project] = manifest_project # If submodule is already at specified revision in all projects, exit successfully if len(project_integration) == 0: print("No integration done, changes are already included in all projects.") sys.exit(0) # ======================================================= # Create and merge merge_requests if needed # ======================================================= if args.merge: # Get source merge request ( the one in the gitlab-ci repo) submodule_project_path, _ = get_submodule_project_path_and_revision( manifest_project["project"], args.submodule, args.branch ) submodule_project = common.get_project(gitlab, submodule_project_path) mrs = get_merge_requests( submodule_project, # TODO should this be submodule_project's default branch? target_branch="master", commit=args.revision, ) if not mrs: sys.exit( "ERROR: could not determine source merge request for commit %s" % args.revision ) source_mr = mrs[0] for p in args.projects: integration = project_integration[p] logging.debug("Create MR in %s", integration["project"].name) mr = create_integration_merge_request( integration["project"], integration["branch"], source_mr ) integration["mr"] = mr # Now merge logging.debug("Merge %s!%s", p, mr.iid) # Wait until GitLab has checked merge status common.wait_until_merge_status_is_set(integration["project"], mr) # Attempt to merge merged, integration_commit = accept_merge_request( integration["project"], mr, rebase=True ) # if this has rebased the integration commit needs to be adapted: project_integration[p]["commit"] = integration_commit if not merged: sys.exit( "Integration MR could not be merged. You have two possibilities to fix " "this:\n" " 1. Checkout the MR and rebase it on the current master manually, or\n" " 2. Delete the MR (Edit -> Delete in the MR UI)\n" "In either case restart this job afterwards in order to get it merged." ) print("Successfully merged") # ======================================================= # Now create the integration commit in the manifest # for all subprojects at once # ======================================================= manifest_file_abs = os.path.join( manifest_project["repo"].working_tree_dir, args.manifest_file ) logging.debug("Read manifest from: %s", manifest_file_abs) with open(manifest_file_abs, "r", encoding="utf8") as fp: manifest = fp.read() logging.debug(manifest) srcrev_file_abs = os.path.join( manifest_project["repo"].working_tree_dir, args.srcrev_file ) logging.debug("Read manifest from: %s", srcrev_file_abs) with open(srcrev_file_abs, "r", encoding="utf8") as fp: srcrev = fp.read() logging.debug(srcrev) for p in args.projects: integration = project_integration[p] logging.debug( "Update %s to %s", integration["project"].name, integration["commit"] ) new_manifest = update_manifest( manifest, integration["project"], integration["commit"] ) if new_manifest is not None: manifest = new_manifest logging.debug(manifest) continue # get BB_RECIPE_NAME from the projects .gitlab-ci.yml gitlab_ci_yml_file = os.path.join( integration["repo"].working_tree_dir, ".gitlab-ci.yml" ) logging.debug("Read recipe name from %s", gitlab_ci_yml_file) with open(gitlab_ci_yml_file, "r", encoding="utf8") as fp: gitlab_ci_yml = fp.read() project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml) new_srcrev = update_srcrev( srcrev, project_keys["recipe"], integration["commit"] ) if new_srcrev is not None: srcrev = new_srcrev logging.debug(srcrev) else: logging.debug("Project %s not found in xml or srcrev file", p) # Write manifest with open(manifest_file_abs, "w", encoding="utf8") as fp: fp.write(manifest) manifest_project["repo"].git.add(args.manifest_file) logging.debug(manifest) with open(srcrev_file_abs, "w", encoding="utf8") as fp: fp.write(srcrev) manifest_project["repo"].git.add(args.srcrev_file) logging.debug(srcrev) # ======================================================== # Now commit and push the changes to the manifest repo # ======================================================== # Make an API request to create the gitlab.user object gitlab = manifest_project["project"].manager.gitlab gitlab.auth() integration_commit = common.commit_and_push( manifest_project["project"], manifest_project["repo"], manifest_project["branch"], manifest_project["message"], gitlab.user.username, gitlab.user.email, ) print( "Successfully create integration commit {} in {}".format( integration_commit, args.project ) ) if not args.merge: sys.exit(0) # ============================================ # Create merge requests for the manifest # ============================================ logging.debug("Create MR in %s", manifest_project["project"].name) manifest_project["mr"] = create_integration_merge_request( manifest_project["project"], manifest_project["branch"], source_mr ) # ================================================= # Now merge it # ================================================= # The manifest needs to be merged at last mr = manifest_project["mr"] logging.debug("Merge %s!%s", args.project, mr.iid) # Wait until GitLab has checked merge status common.wait_until_merge_status_is_set(manifest_project["project"], mr) # Attempt to merge merged = accept_merge_request(manifest_project["project"], mr, rebase=True) if not merged: sys.exit( "Integration MR could not be merged. You have two possibilities to fix " "this:\n" " 1. Checkout the MR and rebase it on the current master manually, or\n" " 2. Delete the MR (Edit -> Delete in the MR UI)\n" "In either case restart this job afterwards in order to get it merged." ) print("Successfully merged") if __name__ == "__main__": main()