diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8f48a3415a2e8ca2bdc81bff57ac775d24d4dd16..9272fd2fc8ad33392fb703c8c0213ca2b26663d1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,7 @@ default:
stages:
- analyze
- integrate
- - check
+ - build
- merge
workflow:
@@ -62,7 +62,6 @@ yamllint:
# ---------------------------------------------------------------------------------------
# Stage: integrate
# ---------------------------------------------------------------------------------------
-
.ci-test-projects:
variables:
PROJECT_ROOT:
@@ -73,6 +72,7 @@ yamllint:
INTEGRATE_INTO:
${PROJECT_ROOT}/minimal-foo
${PROJECT_ROOT}/minimal-bar
+ ${PROJECT_ROOT}/minimal-srcrev
.yocto-projects:
variables:
@@ -104,6 +104,8 @@ yamllint:
- if: $CI_MERGE_REQUEST_IID
when: manual
allow_failure: true
+ variables:
+ MERGE: ""
script:
- cd ${CI_PROJECT_DIR}
- ./deploy_gitlab_ci.py
@@ -113,22 +115,9 @@ yamllint:
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
--verbose
+ ${MERGE}
${INTEGRATE_INTO}
- - ./generate_job_from_template.py
- --template=gitlab-ci-integration.jinja2
- --image=${CI_IMAGE_PYTHON}
- --branch="integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
- --manifest-project=${MANIFEST_PROJECT}
- --parent_merge_request="${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
- --verbose
- ${INTEGRATE_INTO}
- > integration.yml
- - cat integration.yml
- artifacts:
- paths:
- - integration.yml
-
integrate-yocto:
extends:
- .integrate
@@ -139,47 +128,47 @@ integrate-ci-test:
- .integrate
- .ci-test-projects
-trigger-yocto:
- stage: integrate
+# --------------------------------------------------------------------------------------
+# Stage: build
+# --------------------------------------------------------------------------------------
+build-yocto:
+ stage: build
+ needs: [integrate-yocto]
rules:
- if: $CI_MERGE_REQUEST_IID
allow_failure: true
- needs: [integrate-yocto]
+ - if: $CI_COMMIT_BRANCH == "master"
+ when: manual
trigger:
- include:
- - artifact: integration.yml
- job: integrate-yocto
+ project: SECO-Northern-Europe/yocto/manifest
+ branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
-trigger-ci-test:
- stage: integrate
+build-ci-test:
+ stage: build
+ needs: [integrate-ci-test]
rules:
- if: $CI_MERGE_REQUEST_IID
- needs: [integrate-ci-test]
+ allow_failure: true
+ - if: $CI_COMMIT_BRANCH == "master"
+ when: manual
trigger:
- include:
- - artifact: integration.yml
- job: integrate-ci-test
+ project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
+ branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
.merge:
+ extends: .integrate
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == "master"
when: manual
allow_failure: true
- script:
- - cd ${CI_PROJECT_DIR}
- - ./merge_gitlab_ci.py
- --gitlab-url=${CI_SERVER_URL}
- --token=${GITBOT_TOKEN}
- --manifest-project=${MANIFEST_PROJECT}
- --submodule=.gitlab-ci
- --revision=${CI_COMMIT_SHA}
- ${INTEGRATE_INTO}
+ variables:
+ MERGE: --merge
merge-ci-test:
extends:
@@ -190,40 +179,3 @@ merge-yocto:
extends:
- .merge
- .yocto-projects
-
-# --------------------------------------------------------------------------------------
-# Stage: check
-# --------------------------------------------------------------------------------------
-check:
- stage: check
- needs: [integrate-yocto, integrate-ci-test]
- rules:
- # Probably this job gets removed
- - when: never
- - if: $CI_MERGE_REQUEST_IID
- tags:
- - infrastructure
- timeout: 2m
- script:
- - cd ${CI_PROJECT_DIR}
- - MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
-
- - MASTER_BRANCH=dunfell
- - ./check_if_integration_branch_is_up_to_date.py
- --gitlab-url=${CI_SERVER_URL}
- --token=${GITBOT_TOKEN}
- --manifest-project=${CI_PROJECT_ROOT_NAMESPACE}/yocto/manifest
- --integration-base=${MASTER_BRANCH}
- --project=${CI_PROJECT_PATH}
- --merge-request=${CI_MERGE_REQUEST_IID}
-
- # The check is done for both manifests in one job as the retrigger
- # looks for jobs named check, though there can only be one
- - ./check_if_integration_branch_is_up_to_date.py
- --gitlab-url=${CI_SERVER_URL}
- --token=${GITBOT_TOKEN}
- --manifest-project
- ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test/minimal-manifest
- --integration-base=master
- --project=${CI_PROJECT_PATH}
- --merge-request=${CI_MERGE_REQUEST_IID}
diff --git a/accept_merge_request.py b/accept_merge_request.py
index 779d6117857b3ed5ea5be77a36267ccf0484fbc8..7500bbe98fc152075fcb66c9c6623223f623090f 100755
--- a/accept_merge_request.py
+++ b/accept_merge_request.py
@@ -47,7 +47,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
print("Merge error: %s" % mr.merge_error)
else:
print("Merge reported success, but MR state is '%s'" % mr.state)
- return False
+ return False, mr.sha
except GitlabMRClosedError as e:
# See HTTP error codes for merge requests here:
@@ -73,7 +73,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
if pipeline_pending:
print("")
print("Merge not possible, has to be rebased manually")
- return False
+ return False, mr.sha
elif e.response_code == 406:
# Merge conflict, automatic rebase is possible
@@ -82,7 +82,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
pipeline_pending = False
print("Merge not possible, but branch can be automatically rebased")
if not rebase:
- return False
+ return False, mr.sha
print("Trying to rebase...")
mr = common.rebase_merge_request(project, mr)
if mr.merge_error:
@@ -96,7 +96,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
print("ERROR: merge not possible: %s" % e)
sys.exit(critical_error)
- return True
+ return True, mr.sha
def main():
diff --git a/common.py b/common.py
index 6da0c1a3074c1002b6640e205339ddd14647ce85..8f77c6d737613d2ec5ffd925d39a073123ae0281 100755
--- a/common.py
+++ b/common.py
@@ -150,7 +150,9 @@ def list_commits(commits):
return commit_list
-def commit_and_push(project: Project, repo: Repo, branch, message, name, email):
+def commit_and_push(
+ project: Project, repo: Repo, branch, message, name, email, less_verbose=False
+):
"""Commit and push to a repo branch"""
author = Actor(name, email)
repo.index.commit(message, author=author, committer=author)
@@ -168,7 +170,8 @@ def commit_and_push(project: Project, repo: Repo, branch, message, name, email):
revision = repo.head.commit.hexsha
print("Pushed new commit:")
print(project.web_url + "/-/commit/" + revision)
- print(repo.git.show("--summary", "--decorate"))
+ if not less_verbose:
+ print(repo.git.show("--summary", "--decorate"))
return revision
diff --git a/deploy_gitlab_ci.py b/deploy_gitlab_ci.py
index a99f2afc0547f109b9e3add863465edadeb47911..2567a11faefff2b0ad6708cd260d1d203d970bd5 100755
--- a/deploy_gitlab_ci.py
+++ b/deploy_gitlab_ci.py
@@ -6,45 +6,24 @@ import logging
import sys
import os
from gitlab import Gitlab
+
from accept_merge_request import accept_merge_request
from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
-from update_submodule import update_submodule
-from update_gitlab_ci import update_gitlab_ci_include
+from update_submodule import (
+ update_submodule_and_include_ref,
+ get_submodule_project_path_and_revision,
+)
+from integrate_into_manifest import update_manifest, update_srcrev
from ruamel.yaml import YAML
-def create_gitlab_ci_yml(repo, gitlab_ci_yml):
- """This code snippet was used to initially populate
- all repos with the minial .gitlab-ci.yml that
- includes the files from this repo.
- Currently it is not used and only the revision is
- changed.
- It has also never been test as this function.
- """
- logging.debug("Origin: %s", repo.remotes.origin.url)
-
- # Contains the base file to be used in the subprojects
- if "ci-test" in repo.remotes.origin.url:
- if "manifest" in repo.remotes.origin.url:
- include_file = "foobar-manifest.yml"
- else:
- include_file = "foobar-manifest-integration.yml"
- else:
- if "manifest" in repo.remotes.origin.url:
- include_file = "manifest.yml"
- else:
- include_file = "manifest-integration.yml"
+def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
- # -----------------------------------
- # Adapt content in the include file
- # -----------------------------------
# Read values from existing file
yaml = YAML()
- with open(gitlab_ci_yml, "r", encoding="utf8") as fp:
- data = yaml.load(fp)
-
+ data = yaml.load(gitlab_ci_yml)
logging.debug("Yaml: %s", data)
try:
@@ -57,103 +36,61 @@ def create_gitlab_ci_yml(repo, gitlab_ci_yml):
logging.debug("Recipe %s", recipe)
except KeyError:
recipe = None
-
- # -----------------------------------
- with open(gitlab_ci_yml, "w", encoding="utf8") as fp:
- fp.write(
- """---
-# ---------------------------------------------------------------------------------------
-# Include the default CI steps from the gitlab-ci repo
-# ---------------------------------------------------------------------------------------
-include:
- - project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci'
- ref: 49cc4204323bca76190e5ffae1a7d5627157c073
-"""
- )
- fp.write(" file: '{}'\n".format(include_file))
- if masterbranch is not None or recipe is not None:
- fp.write("\nvariables:\n")
- if masterbranch is not None:
- fp.write(" MASTER_BRANCH_PROJECT: {}\n".format(masterbranch))
- if recipe is not None:
- fp.write(" BB_RECIPE_NAME: {}\n".format(recipe))
-
- # -----------------------------------
- repo.git.add(gitlab_ci_yml)
-
- # ======================================
-
-
-def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
-
- # Add changed revision also to .gitlab-ci.yml
- gitlab_ci_yml = os.path.join(repo.working_tree_dir, ".gitlab-ci.yml")
-
- if update_gitlab_ci_include(
- gitlab_ci_yml,
- submodule_project.web_url.split("//")[1].split("/", 1)[1],
- submodule_revision,
- ):
- repo.git.add(gitlab_ci_yml)
-
- with open(gitlab_ci_yml, "r", encoding="utf8") as fp:
- logging.debug(fp.read())
+ return {"recipe": recipe, "masterbranch": masterbranch}
-def deploy_into(project, submodule, revision, branch, replace_existing_branch=False):
- """Update the submodule and include refs to the submodule in the given project.
- Create mergerequest if needed.
+def integrate_submodule_into(
+ gitlab, project_name, submodule_name, new_revision, branch, commit_and_push=True
+):
- Parameters:
- project ( gitlab project): The project which's submodule should be updated
- submodule_name (string): The name of the submodule to pull
- submodule_revision (hex string): The sha hash of the commit to update the submodule to
- branch (string): branch to update, if None, the projects default branch is used
- replace_existing_branch: When an existing integration branch is found it is always replaced.
+ gitlab_project = common.get_project(gitlab, project_name)
- Returns: tuple of:
- branch (string): Name of the newly created integration branch
- merge_request (gitlab mr): Mergerequest for the integration branch
- """
- # Update submodule
- integration_branch, _, submodule_project = update_submodule(
- project,
- submodule,
- revision,
+ (
+ project_repo,
+ integration_branch_name,
+ integration_commit,
+ message,
+ ) = update_submodule_and_include_ref(
+ gitlab_project,
+ submodule_name,
+ new_revision,
branch,
- pre_commit_hook=update_rev_in_gitlab_ci,
- replace_existing_branch=replace_existing_branch,
+ commit_and_push=commit_and_push,
)
-
- logging.debug("Integration branch: %s", integration_branch)
-
- # If submodule is already at specified revision, return directly
- if not integration_branch:
- return None, submodule_project
-
- # Get source merge request
- mrs = get_merge_requests(
- submodule_project,
- # TODO should this be submodule_project's default branch?
- target_branch="master",
- commit=revision,
+ if integration_branch_name is None:
+ return None
+ # ======================================
+ # Store the references for creating the integration
+ # commit in the manifest later
+ # ======================================
+ ret = {
+ "project": gitlab_project,
+ "repo": project_repo,
+ "branch": integration_branch_name,
+ "commit": integration_commit,
+ "message": message,
+ }
+ logging.debug(
+ "Integration branch: %s (%s)",
+ integration_branch_name,
+ integration_commit,
)
- if not mrs:
- sys.exit(
- "ERROR: could not determine source merge request for commit %s" % revision
- )
- source_mr = mrs[0]
+ return ret
+
+def create_integration_merge_request(project, integration_branch_name, source_mr=None):
# Create merge request
+ # This should be optional
mr, created = create_merge_request(
- project, integration_branch, project.default_branch
+ project, integration_branch_name, project.default_branch
)
if created:
- common.crosslink_merge_requests(source_mr, mr)
+ if source_mr is not None:
+ common.crosslink_merge_requests(source_mr, mr)
print("Created new merge request:\n%s" % mr.web_url)
else:
print("Existing integration merge request:\n%s" % mr.web_url)
- return integration_branch, mr
+ return mr
def main():
@@ -203,6 +140,20 @@ def main():
required=False,
default=False,
)
+ parser.add_argument(
+ "--manifest-file",
+ help="""manifest file name (default: 'default.xml')""",
+ dest="manifest_file",
+ default=common.manifest_file,
+ required=False,
+ )
+ parser.add_argument(
+ "--srcrev-file",
+ help="""source revision file name (default: 'SRCREV.conf')""",
+ dest="srcrev_file",
+ default=common.srcrev_file,
+ required=False,
+ )
parser.add_argument(
"projects",
help="""List of projects the change should be deployed to additionally
@@ -221,66 +172,217 @@ def main():
logging.basicConfig(level=logging.DEBUG)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
- project = common.get_project(gitlab, args.project)
- logging.debug("Integrate into: %s", args.project)
+ # =======================================================
+ # Create integration branches and commits with updates
+ # submodule in all projects
+ # =======================================================
+ project_integration = {}
+ # Update submodule in all 'child' project
+ for p in args.projects:
+ logging.debug("Integrate into: %s", p)
+
+ res = integrate_submodule_into(
+ gitlab, p, args.submodule, args.revision, args.branch
+ )
+ if res is not None:
+ project_integration[p] = res
- # Update submodule in this project, create MR
- integration_branch, mr = deploy_into(
- project,
+ # Update submodule in manifest project
+ manifest_project = integrate_submodule_into(
+ gitlab,
+ args.project,
args.submodule,
args.revision,
args.branch,
- replace_existing_branch=len(args.projects) > 0,
+ commit_and_push=False,
)
- merge_request_manifest = mr
- merge_requests = []
+ if manifest_project is not None:
+ project_integration[args.project] = manifest_project
- # If submodule is already at specified revision, exit successfully
- if not integration_branch:
+ # If submodule is already at specified revision in all projects, exit successfully
+ if len(project_integration) == 0:
+ print("No integration done, changes are already included in all projects.")
sys.exit(0)
+ # =======================================================
+ # Create and merge merge_requests if needed
+ # =======================================================
+ if args.merge:
+ # Get source merge request ( the one in the gitlab-ci repo)
+ submodule_project_path, _ = get_submodule_project_path_and_revision(
+ manifest_project["project"], args.submodule, args.branch
+ )
+ submodule_project = common.get_project(gitlab, submodule_project_path)
+ mrs = get_merge_requests(
+ submodule_project,
+ # TODO should this be submodule_project's default branch?
+ target_branch="master",
+ commit=args.revision,
+ )
+ if not mrs:
+ sys.exit(
+ "ERROR: could not determine source merge request for commit %s"
+ % args.revision
+ )
+ source_mr = mrs[0]
+
+ for p in args.projects:
+ integration = project_integration[p]
+ logging.debug("Create MR in %s", integration["project"].name)
+ mr = create_integration_merge_request(
+ integration["project"], integration["branch"], source_mr
+ )
+ integration["mr"] = mr
+ # Now merge
+ logging.debug("Merge %s!%s", p, mr.iid)
+
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(integration["project"], mr)
+
+ # Attempt to merge
+ merged, integration_commit = accept_merge_request(
+ integration["project"], mr, rebase=True
+ )
+ # if this has rebased the integration commit needs to be adapted:
+ project_integration[p]["commit"] = integration_commit
+
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged. You have two possibilities to fix "
+ "this:\n"
+ " 1. Checkout the MR and rebase it on the current master manually, or\n"
+ " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
+ "In either case restart this job afterwards in order to get it merged."
+ )
+
+ print("Successfully merged")
+
+ # =======================================================
+ # Now create the integration commit in the manifest
+ # for all subprojects at once
+ # =======================================================
+ manifest_file_abs = os.path.join(
+ manifest_project["repo"].working_tree_dir, args.manifest_file
+ )
+ with open(
+ manifest_file_abs,
+ "r",
+ encoding="utf8",
+ ) as fp:
+ manifest = fp.read()
+ logging.debug(manifest)
+ srcrev_file_abs = os.path.join(
+ manifest_project["repo"].working_tree_dir, args.srcrev_file
+ )
+ with open(
+ srcrev_file_abs,
+ "r",
+ encoding="utf8",
+ ) as fp:
+ srcrev = fp.read()
+ logging.debug(srcrev)
+
for p in args.projects:
- gitlab_p = common.get_project(gitlab, p)
- logging.debug("Integrate into: %s", p)
+ integration = project_integration[p]
+ logging.debug(
+ "Update %s to %s", integration["project"].name, integration["commit"]
+ )
- integration_branch, mr = deploy_into(
- gitlab_p,
- args.submodule,
- args.revision,
- args.branch,
+ new_manifest = update_manifest(
+ manifest, integration["project"], integration["commit"]
)
- merge_requests.append(mr)
+ if new_manifest is not None:
+ manifest = new_manifest
+ logging.debug(manifest)
+ continue
+
+ # get BB_RECIPE_NAME from the projects .gitlab-ci.yml
+ gitlab_ci_yml_file = os.path.join( integration["repo"].working_tree_dir, ".gitlab_ci_yml.yml")
+ with open( gitlab_ci_yml_file, "r",encoding="utf8") as fp:
+ gitlab_ci_yml = fp.read()
+ project_keys = read_keys_from_gitlab_ci_yml(gitlab_ci_yml)
+
+ new_srcrev = update_srcrev(
+ srcrev, project_keys["recipe"], integration["commit"]
+ )
+ if new_srcrev is not None:
+ srcrev = new_srcrev
+ logging.debug(srcrev)
+ else:
+ logging.debug("Project %s not found in xml or srcrev file", p)
+
+ # Write manifest
+ with open(
+ manifest_file_abs,
+ "w",
+ encoding="utf8",
+ ) as fp:
+ fp.write(manifest)
+ manifest_project["repo"].git.add(args.manifest_file)
+ logging.debug(manifest)
+ with open(
+ srcrev_file_abs,
+ "w",
+ encoding="utf8",
+ ) as fp:
+ fp.write(srcrev)
+ manifest_project["repo"].git.add(args.srcrev_file)
+ logging.debug(srcrev)
+
+ # ========================================================
+ # Now commit and push the changes to the manifest repo
+ # ========================================================
+ # Make an API request to create the gitlab.user object
+ gitlab = manifest_project["project"].manager.gitlab
+ gitlab.auth()
+ integration_commit = common.commit_and_push(
+ manifest_project["project"],
+ manifest_project["repo"],
+ manifest_project["branch"],
+ manifest_project["message"],
+ gitlab.user.username,
+ gitlab.user.email,
+ )
- logging.debug("Integration branch: %s", integration_branch)
+ print(
+ "Successfully create integration commit {} in {}".format(
+ integration_commit, args.project
+ )
+ )
if not args.merge:
- print(
- "Skipping automatic merge in MR context. If you like to extend the "
- "integration MR by hand, please do it now. Afterwards you can either merge "
- "it by hand or re-run this job on the master branch after the source MR "
- "has been merged."
- )
sys.exit(0)
+ # ============================================
+ # Create merge requests for the manifest
+ # ============================================
+
+ logging.debug("Create MR in %s", manifest_project["project"].name)
+ manifest_project["mr"] = create_integration_merge_request(
+ manifest_project["project"], manifest_project["branch"], source_mr
+ )
+ # =================================================
+ # Now merge it
+ # =================================================
# The manifest needs to be merged at last
- merge_requests.append(merge_request_manifest)
- for mr in merge_requests:
- logging.debug("Merge %s", mr)
- # Wait until GitLab has checked merge status
- common.wait_until_merge_status_is_set(project, mr)
+ mr = manifest_project["mr"]
+ logging.debug("Merge %s!%s", args.project, mr.iid)
- # Attempt to merge
- merged = accept_merge_request(project, mr, rebase=True)
+ # Wait until GitLab has checked merge status
+ common.wait_until_merge_status_is_set(manifest_project["project"], mr)
- if not merged:
- sys.exit(
- "Integration MR could not be merged. You have two possibilities to fix "
- "this:\n"
- " 1. Checkout the MR and rebase it on the current master manually, or\n"
- " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
- "In either case restart this job afterwards in order to get it merged."
- )
+ # Attempt to merge
+ merged = accept_merge_request(manifest_project["project"], mr, rebase=True)
+
+ if not merged:
+ sys.exit(
+ "Integration MR could not be merged. You have two possibilities to fix "
+ "this:\n"
+ " 1. Checkout the MR and rebase it on the current master manually, or\n"
+ " 2. Delete the MR (Edit -> Delete in the MR UI)\n"
+ "In either case restart this job afterwards in order to get it merged."
+ )
print("Successfully merged")
diff --git a/integrate_into_manifest.py b/integrate_into_manifest.py
index b4c19315c9d45df1cf552d2c5ab5954ac1b0f4b3..fe7d688ff9f00455bf0c9edc7cef9975a408de43 100755
--- a/integrate_into_manifest.py
+++ b/integrate_into_manifest.py
@@ -10,9 +10,66 @@ from pathlib import Path
from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab
+from gitlab.v4.objects import Project
from lxml import etree
+def update_manifest(manifest, project: Project, new_revision):
+ """Returns updated version of the manifest or None id project_name
+ was not found in the manifest
+ """
+ # Parse manifest
+ try:
+ manifestxml = etree.fromstring(manifest.encode())
+ except etree.XMLSyntaxError:
+ sys.exit("ERROR: Failed to parse given manifest")
+
+ # Find project reference in manifest
+ # We are using str.endswith() for this in order to support sub-projects as well
+ # (e.g."mygroup/myproject", when only "myproject" is given)
+ project_node = None
+ project_nodes = manifestxml.findall("project")
+ for node in project_nodes:
+ name = node.get("name")
+ if name is not None and name.endswith(project.path):
+ project_node = node
+ if project_node is None:
+ return None
+
+ # Get current project revision from manifest
+ old_revision = project_node.get("revision")
+ logging.debug("Replace %s with %s", old_revision, new_revision)
+
+ # Update manifest file
+ # We are doing this using a plain text replace action. Unfortunately
+ # all python libraries for handling XML data are not able to preserve
+ # the file layout, and we want a minimal diff.
+ manifest = manifest.replace(old_revision, new_revision)
+ return manifest
+
+
+def update_srcrev(srcrev, recipe_name, new_revision):
+ # Check if project is referenced in SRCREV.conf
+ # Match "...RECIPE_NAME ="
+ pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+
+ project_line = None
+ for line in srcrev.splitlines():
+ if pattern.search(line):
+ project_line = line
+ break
+ if project_line is None:
+ return None
+
+ # Get current project revision from SRCREV file
+ # Assuming notation: <project> = "<hash>"
+ old_revision = project_line.split('"')[1]
+
+ # Update SRCREV file
+ srcrev = srcrev.replace(old_revision, new_revision)
+ return srcrev
+
+
def integrate_into_manifest(
manifest_project,
integration_base,
@@ -34,6 +91,7 @@ def integrate_into_manifest(
clone_url.password = gitlab.private_token
# Checkout manifest
+ # TODO replace checkout with gitlab api access
try:
manifest_repo = Repo.clone_from(
clone_url.url, manifest_dir, branch=integration_base
@@ -71,68 +129,42 @@ def integrate_into_manifest(
manifest_repo.head.set_reference(
manifest_repo.create_head(integration_branch)
)
+ # Get new project revision from merge request
+ new_revision = merge_request.sha
# Parse manifest file
try:
- manifest = etree.parse(manifest_filepath.as_posix())
+ with open(manifest_filepath.as_posix(), "r", encoding="utf8") as fp:
+ manifest = fp.read_text()
except FileNotFoundError:
sys.exit("ERROR: file '%s' not found in manifest repo" % manifest_file)
- # Find project reference in manifest
- # We are using str.endswith() for this in order to support sub-projects as well
- # (e.g."mygroup/myproject", when only "myproject" is given)
- project_node = None
- project_nodes = manifest.findall("project")
- for node in project_nodes:
- name = node.get("name")
- if name is not None and name.endswith(project.path):
- project_node = node
- if project_node is None:
+ new_manifest = update_manifest(manifest, project, new_revision)
+ if new_manifest is not None:
+ # write file
+ with open(manifest_filepath.as_posix(), "w", encoding="utf8") as fp:
+ fp.write_text(new_manifest)
+ manifest_repo.index.add([manifest_file])
+ else:
+ # Look for project in SRCREV as it has not been found in the manifest
if recipe_name is None:
sys.exit(
"ERROR: project '%s' not found in manifest and "
"no recipe name is specified" % project.path
)
- # Check if project is referenced in SRCREV.conf
- project_line = None
- content = srcrev_filepath.read_text()
- # Match "...RECIPE_NAME ="
- pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
- for line in content.splitlines():
- if pattern.search(line):
- project_line = line
- if project_line is None:
+
+ with open(srcrev_filepath, "r", encoding="utf8") as fp:
+ srcrev = fp.read_text()
+ new_srcrev = update_srcrev(srcrev, recipe_name, new_revision)
+ # write file
+ if new_srcrev is None:
sys.exit(
"ERROR: project '%s' not found in manifest and "
- "SRCREV file" % project.path
+ "no recipe name is specified" % project.path
)
-
- # Get current project revision from SRCREV file
- # Assuming notation: <project> = "<hash>"
- old_revision = project_line.split('"')[1]
-
- # Get new project revision from merge request
- new_revision = merge_request.sha
-
- # Update SRCREV file
- content = content.replace(old_revision, new_revision)
- srcrev_filepath.write_text(content)
+ with open(srcrev_filepath.as_posix(), "w", encoding="utf8") as fp:
+ fp.write_text(new_manifest)
manifest_repo.index.add([srcrev_file])
- else:
- # Get current project revision from manifest
- old_revision = project_node.get("revision")
-
- # Get new project revision from merge request
- new_revision = merge_request.sha
-
- # Update manifest file
- # We are doing this using a plain text replace action. Unfortunately
- # all python libraries for handling XML data are not able to preserve
- # the file layout, and we want a minimal diff.
- content = manifest_filepath.read_text()
- content = content.replace(old_revision, new_revision)
- manifest_filepath.write_text(content)
- manifest_repo.index.add([manifest_file])
# Make an API request to create the gitlab.user object
gitlab.auth()
diff --git a/merge_gitlab_ci.py b/merge_gitlab_ci.py
deleted file mode 100755
index 217419dbcb0e23a91a14a20953a69bb12ac0dc89..0000000000000000000000000000000000000000
--- a/merge_gitlab_ci.py
+++ /dev/null
@@ -1,198 +0,0 @@
-#!/usr/bin/env python3
-import common
-
-import argparse
-import logging
-import sys
-from gitlab import Gitlab, GitlabGetError
-from accept_merge_request import accept_merge_request
-import update_submodule
-from gitlab.v4.objects import Project
-
-
-def find_integration_merge_request(
- project: Project, submodule, revision, target_branch
-):
-
- (
- submodule_path,
- submodule_revision,
- ) = update_submodule.get_submodule_project_path_and_revision(
- project, submodule, target_branch
- )
- logging.debug("Module: %s, Revision: %s", submodule_path, submodule_revision)
-
- # Get submodule project
- gitlab = project.manager.gitlab
- submodule_project = common.get_project(gitlab, submodule_path)
-
- # Get the integration branch name
- integration_branch_suffix = (
- update_submodule.get_submodule_integration_branch_suffix(
- submodule_project, revision
- )
- )
- integration_branch_name = common.integration_branch_name(
- submodule_project.name, integration_branch_suffix
- )
- logging.debug(integration_branch_name)
-
- # Get the merge request for the branch TODO catch exception
- try:
- integration_branch = project.branches.get(
- integration_branch_name, retry_transient_errors=True
- )
- except GitlabGetError:
- print("ERROR: Failed to find integration branch for {}".format(submodule))
- return None
-
- logging.debug(integration_branch)
- commit = project.commits.get(
- integration_branch.commit["id"], retry_transient_errors=True
- )
- for mr in commit.merge_requests(retry_transient_errors=True):
- if mr["target_branch"] == target_branch:
- integration_mr = mr
- break
- if integration_mr is None:
- print(
- "ERROR: Failed to find integration merge request for %s",
- integration_branch_name,
- )
- return None
- logging.debug(integration_mr["iid"])
- mr = project.mergerequests.get(integration_mr["iid"], retry_transient_errors=True)
-
- return mr
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "--gitlab-url",
- help="""URL to the GitLab instance""",
- dest="gitlab_url",
- required=True,
- )
- parser.add_argument(
- "--token",
- help="""GitLab REST API private access token""",
- dest="token",
- required=True,
- )
- parser.add_argument(
- "--project",
- "--manifest-project",
- help="""name of the GitLab project""",
- dest="project",
- )
- parser.add_argument(
- "--submodule",
- help="""submodule to update""",
- dest="submodule",
- required=True,
- )
- parser.add_argument(
- "--revision",
- help="""new revision for submodule""",
- dest="revision",
- required=True,
- )
- parser.add_argument(
- "--branch",
- help="""project branch (if not default branch)""",
- dest="branch",
- required=False,
- default=None,
- )
- parser.add_argument(
- "--merge",
- help="""if set, perform merge after integration""",
- dest="merge",
- action="store_true",
- required=False,
- default=False,
- )
- parser.add_argument(
- "projects",
- help="""List of projects the change should be deployed to additionally
- to the manifest project given as named parameter.""",
- nargs="*",
- )
- parser.add_argument(
- "-v",
- "--verbose",
- action="store_true",
- help="""Increase verbosity.""",
- )
-
- args, _ = parser.parse_known_args()
- if args.verbose:
- logging.basicConfig(level=logging.DEBUG)
-
- gitlab = Gitlab(args.gitlab_url, private_token=args.token)
-
- # Start with the manifest here, so the subproject
- # can see that they are already integrated
- for p in args.projects + [args.project]:
- project = common.get_project(gitlab, p)
- branch = args.branch
-
- if branch is None:
- branch = project.default_branch
-
- print(
- "Try to merge {}({}) into {} ({})".format(
- args.submodule, args.revision, project.name, branch
- )
- )
-
- mr = find_integration_merge_request(
- project, args.submodule, args.revision, branch
- )
- if mr is None:
- sys.exit("ERROR: Failed to find the integration MR.")
- # TODO if this ever happens, why ever, we could call
- # deploy_gitlab_ci again to create a new integration
- # commit or make sure the change is already integrated.
-
- print("Merge {}!{}: {}".format(project.name, mr.iid, mr.title))
- # Wait until GitLab has checked merge status
- common.wait_until_merge_status_is_set(project, mr)
-
- # Attempt to merge
- merged = accept_merge_request(
- project,
- mr,
- # Only the file manifest commit may be rebased
- # other rebase would require new integration commits
- # TODO implement rebase
- rebase=(p == args.project),
- )
-
- if not merged:
- print(
- "Integration MR could not be merged."
- "To fix this:\n"
- " 1. Checkout the MR {}!{} and merge it manually.\n"
- " 2. Manually merge the follow up MRs in the following project:".format(
- project.name, mr.iid
- )
- )
- found = False
- for p2 in [args.project] + args.projects:
- if p2 == p:
- found = True
- continue
- if not found:
- continue
- print(" {}".format(p2))
- sys.exit()
-
- print("Successfully merged")
-
- exit()
-
-
-if __name__ == "__main__":
- main()
diff --git a/merge_into_manifest.py b/merge_into_manifest.py
index 603a9ccd6deb2bc1759bf6cf8a2e978b2b84224e..37205cf7f31f3f7364dd1ba5e10d96c11090bb73 100755
--- a/merge_into_manifest.py
+++ b/merge_into_manifest.py
@@ -78,7 +78,7 @@ def merge_into_manifest(
common.wait_until_merge_status_is_set(manifest_project, mr)
# Attempt to merge
- merged = accept_merge_request(manifest_project, mr)
+ merged, _ = accept_merge_request(manifest_project, mr)
if not merged:
# Merge failed, reintegrate the source merge request into the manifest.
diff --git a/update_gitlab_ci.py b/update_gitlab_ci.py
deleted file mode 100755
index c1c1057f03de970b25814c426b553ab9c695505e..0000000000000000000000000000000000000000
--- a/update_gitlab_ci.py
+++ /dev/null
@@ -1,128 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import logging
-import re
-
-
-def update_gitlab_ci_include(filename, include_project, new_revision):
- """Update the include statement in a gitlab-ci yml to a given revision
-
- Parameters:
- filename( string): The path to the file to change.
- include_project( string): The path used to reference the project the include points to.
- new_revision (string): The hex sha to set the include to.
-
- Returns: True if the file was changed.
- """
-
- # Set the possible include in the local .gitlab.yml file
- # to the new revision. The include needs to have the revision
- # specified directly in the file, as it is parsed before the
- # submodule checkout is done
-
- # Use custom read write method as I didn't got ruamel yaml
- # to keep all costum formating (linebreaks ...)
- # This assumes following format of the include block
- # include:
- # - project: 'SECO-Northern-Europe/yocto/infrastructure/gitlab-ci'
- # ref: c5a3793e783fcb364c7f3bda73e8cd7c08a08804
- # file: 'manifest-childs.yml'
-
- # Verify hash format:
- if re.match(r"\A[0-9a-fA-F]{40}\Z", new_revision) is None:
- raise TypeError("Format of specified revision is not correct")
-
- parsestate = 0
- changed = False
-
- # Remove the SECO-Northern-Europe part from the priject filter
- # as it is normally specified by $CI_PROJECT_ROOT_NAMESPACE
- include_project = include_project.split("/", 1)[1]
- logging.debug("Include project: %s", include_project)
- logging.debug("New revision: %s", new_revision)
-
- with open(filename, "r+", encoding="UTF-8") as fp:
- while True:
- linestart = fp.tell()
- line = fp.readline()
- parts = line.partition(":")
- logging.debug("Splitted input line: %s", parts)
- if len(line) == 0:
- break # End of file
- if parsestate == 0:
- if parts[0] == "include":
- # Found include block
- parsestate = 1
- logging.debug("Found 'include' block at %d", linestart)
- elif parsestate == 1:
- if parts[0] == "\n":
- break # End of include block
-
- if (
- parts[0].endswith(" - project")
- and parts[2].find(include_project) >= 0
- ):
- # Found the correct project
- parsestate = 2
- logging.debug("Found 'project' entry at %d", linestart)
- elif parsestate == 2:
- if parts[0].endswith(" ref"):
- # Found the ref: entry, compare the revision
- logging.debug("Found 'ref' entry at %d", linestart)
- parsestate = 1
- if parts[2].find(new_revision) >= 0:
- print(
- "Revision in {} is already set to {}".format(
- filename, new_revision
- )
- )
- else:
- print(
- "Changed revision in {} to {}".format(
- filename, new_revision
- )
- )
- fp.seek(linestart)
- fp.write("{}: {}".format(parts[0], new_revision))
- fp.flush()
- changed = True
- elif parts[0].find("- ") >= 0 or not parts[0].startswith(" "):
- # Format of the line is not 'name: value' assume end of block
- # Block was not found
- break
- return changed
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "--filename",
- help="""File to change""",
- required=True,
- )
- parser.add_argument(
- "--include-project",
- help="""The path to the included project as used in 'filename'""",
- required=True,
- )
- parser.add_argument(
- "--revision",
- help="""new revision for submodule""",
- required=True,
- )
- parser.add_argument(
- "-v",
- "--verbose",
- action="store_true",
- help="""Increase verbosity.""",
- )
-
- args, _ = parser.parse_known_args()
- if args.verbose:
- logging.basicConfig(level=logging.DEBUG)
-
- update_gitlab_ci_include(args.filename, args.include_project, args.revision)
-
-
-if __name__ == "__main__":
- main()
diff --git a/update_submodule.py b/update_submodule.py
index 17167ef9c6972c0d464a7b07585290bb8bc12e19..f74518d00536771cab5a82164b8f74e1073aab4d 100755
--- a/update_submodule.py
+++ b/update_submodule.py
@@ -11,6 +11,7 @@ from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab
from gitlab.v4.objects import Project
+from ruamel.yaml import YAML
def get_submodule_project_path_and_revision(project: Project, submodule, branch=None):
@@ -65,165 +66,235 @@ def get_submodule_integration_branch_suffix(submodule_project: Project, revision
return integration_branch_suffix
-def update_submodule(
+def clone_project_and_submodule(project: Project, submodule_name, branch=None):
+ """Creates a clone of the given project including the submodule
+ return:
+ """
+ gitlab = project.manager.gitlab
+
+ # If no branch is given, use project's default branch
+ if branch is None:
+ branch = project.default_branch
+
+ project_dir = tempfile.TemporaryDirectory()
+
+ # Construct clone url containing access token
+ clone_url = furl(project.http_url_to_repo)
+ clone_url.username = "gitlab-ci"
+ clone_url.password = gitlab.private_token
+
+ # Checkout project
+ try:
+ repo = Repo.clone_from(clone_url.url, project_dir, branch=branch, depth=1)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not clone repository\n" + str(e))
+ except IndexError:
+ sys.exit("ERROR: branch '%s' not found" % branch)
+
+ # Find submodule
+ submodule = common.get_submodule(repo, submodule_name)
+
+ # Check for relative path
+ if not submodule.url.startswith(".."):
+ sys.exit(
+ "ERROR: absolute submodule paths are not supported (%s)" % submodule.url
+ )
+
+ # Get absolute project path
+ # This cannot be done with gitpython directly due to issue:
+ # https://github.com/gitpython-developers/GitPython/issues/730
+ relative_path = os.path.splitext(submodule.url)[0] # remove .git
+ project_path = project.path_with_namespace
+ while relative_path.startswith(".."):
+ relative_path = relative_path[3:] # strip off '../'
+ project_path, _ = os.path.split(project_path) # remove last part
+ submodule_project_path = os.path.join(project_path, relative_path)
+
+ # Get submodule project
+ submodule_project = common.get_project(gitlab, submodule_project_path)
+
+ # Initialize submodule
+ # Hack due to issue above: change to absolute path and switch back afterwards
+ submodule_clone_url = furl(submodule_project.http_url_to_repo)
+ submodule_clone_url.username = "gitlab-ci"
+ submodule_clone_url.password = gitlab.private_token
+ submodule_relative_url = submodule.url
+ with submodule.config_writer() as writer:
+ writer.set("url", submodule_clone_url.url)
+ submodule.update(init=True)
+ with submodule.config_writer() as writer:
+ writer.set("url", submodule_relative_url)
+
+ return repo, submodule_project
+
+
+def update_submodule_in_repo(repo: Repo, submodule_project: Project, new_revision):
+ """Updates the given submodule to the given revision and adds it to the
+ staging of repo
+ """
+
+ # Update submodule
+ try:
+ submodule_project.module().git.checkout(new_revision)
+ except GitCommandError as e:
+ sys.exit("ERROR: could not checkout commit\n" + str(e))
+ repo.git.add(submodule_project.path)
+
+
+def update_gitlab_ci_include(content, include_project, new_revision):
+
+ # Remove the SECO-Northern-Europe part from the project filter
+ # as it is normally specified by $CI_PROJECT_ROOT_NAMESPACE
+ include_project = include_project.split("/", 1)[1]
+
+ yaml = YAML()
+ data = yaml.load(content)
+ logging.debug("Yaml: %s", data)
+ try:
+ includes = data["include"]
+ except KeyError:
+ logging.debug("No include statement found")
+ return None
+ current_revision = None
+ for entry in includes:
+ try:
+ if include_project in entry["project"]:
+ current_revision = entry["ref"]
+ break
+ except KeyError:
+ logging.debug("Failed to parse include statement")
+ return None
+ if current_revision is None:
+ logging.debug("Failed to find %s in include statement", include_project)
+ return None
+
+ # Use plain replacement to keep the content of the file
+ # Yes, this may fail if the 'current_revision' is used multiple
+ # time is this fail. But probably this will not ever happen
+ logging.debug("Replace %s with %s", current_revision, new_revision)
+ return content.replace(current_revision, new_revision)
+
+
+def update_submodule_and_include_ref(
project,
submodule_name,
- submodule_revision,
+ new_revision,
branch=None,
- pre_commit_hook=None,
- replace_existing_branch=False,
+ commit_and_push=True,
):
- """Update submodule of gitlab project to given revision
+ """Update the submodule and include refs to the submodule in the given project.
+ Create mergerequest if needed.
Parameters:
- project (gitlab project): The project which's submodule should be updated
+ project ( gitlab project): The project which's submodule should be updated
submodule_name (string): The name of the submodule to pull
- submodule_revision (hex string): The sha hash of the commit to update the submodule to
+ new_revision (hex string): The sha hash of the commit to update the submodule to
branch (string): branch to update, if None, the projects default branch is used
- pre_commit_hook: Function to be called before the actual commit is done, to add additional changes.
- Arguments passed: ( repo, submodule_project, submodule_revision)
- replace_existing_branch: When an existing integration branch is found it is always replaced.
- Returns: tuple of:
- branch (string): Name of the newly created integration branch
- revision (string): hexsha of the new commit
- submodule_project ( gitlab project): The submodule as gitlab priject instance
+ commit_and_push: Set to false if no commit should be created. Changes are left in staging.
+ Returns: tuple of:
+ project_repo (Repo): GitPython repo with the cloned project
+ integration_branch (string): Name of the newly created integration branch
+ integration_commit (hexsha): Hash of the newly created commit
+ message: Commit message based on the integrated changes.
"""
- gitlab = project.manager.gitlab
-
- # If no branch is given, use project's default branch
if branch is None:
branch = project.default_branch
+ logging.debug("Branch: %s", branch)
- with tempfile.TemporaryDirectory() as project_dir:
+ _, submodule_current_rev = get_submodule_project_path_and_revision(
+ project, submodule_name, branch
+ )
- # Construct clone url containing access token
- clone_url = furl(project.http_url_to_repo)
- clone_url.username = "gitlab-ci"
- clone_url.password = gitlab.private_token
+ # Check if revisions are different
+ if submodule_current_rev == new_revision:
+ print("Submodule is already at %s" % new_revision)
+ return (None, None, None, None)
- # Checkout project
- try:
- repo = Repo.clone_from(clone_url.url, project_dir, branch=branch, depth=1)
- except GitCommandError as e:
- sys.exit("ERROR: could not clone repository\n" + str(e))
- except IndexError:
- sys.exit("ERROR: branch '%s' not found" % branch)
-
- # Find submodule
- submodule = common.get_submodule(repo, submodule_name)
-
- # Check if revisions are different
- if submodule.hexsha == submodule_revision:
- print("Submodule is already at %s" % submodule_revision)
- if not replace_existing_branch:
- # TODO test this
- return (None, None, None)
-
- # Check for relative path
- if not submodule.url.startswith(".."):
- sys.exit(
- "ERROR: absolute submodule paths are not supported (%s)" % submodule.url
- )
-
- # Get absolute project path
- # This cannot be done with gitpython directly due to issue:
- # https://github.com/gitpython-developers/GitPython/issues/730
- relative_path = os.path.splitext(submodule.url)[0] # remove .git
- project_path = project.path_with_namespace
- while relative_path.startswith(".."):
- relative_path = relative_path[3:] # strip off '../'
- project_path, _ = os.path.split(project_path) # remove last part
- submodule_project_path = os.path.join(project_path, relative_path)
-
- # Get submodule project
- submodule_project = common.get_project(gitlab, submodule_project_path)
-
- # Get commits between current and new revision
- revision_range = submodule.hexsha + ".." + submodule_revision
- commits = submodule_project.commits.list(ref_name=revision_range)
- if not commits:
- sys.exit("ERROR: no commits found in range %s" % revision_range)
-
- # Find out if top commit is part of a merge request
- # If so, use source branch of this MR as integration branch name
- # Else use commit sha instead
- integration_branch_suffix = submodule_revision
- for mr in commits[0].merge_requests():
- if mr["target_branch"] == submodule_project.default_branch:
- integration_branch_suffix = mr["source_branch"]
- break
+ project_repo, submodule_project = clone_project_and_submodule(
+ project, submodule_name, branch
+ )
- # Initialize submodule
- # Hack due to issue above: change to absolute path and switch back afterwards
- submodule_clone_url = furl(submodule_project.http_url_to_repo)
- submodule_clone_url.username = "gitlab-ci"
- submodule_clone_url.password = gitlab.private_token
- submodule_relative_url = submodule.url
- with submodule.config_writer() as writer:
- writer.set("url", submodule_clone_url.url)
- submodule.update(init=True)
- with submodule.config_writer() as writer:
- writer.set("url", submodule_relative_url)
-
- # Check if integration branch already exists and if it is up to date
- integration_branch = common.integration_branch_name(
- submodule_project.name, integration_branch_suffix
- )
- existing_branch = None
- for ref in repo.references:
- if "origin/" + integration_branch == ref.name:
- existing_branch = ref
- if existing_branch:
- repo.head.set_reference(existing_branch)
- submodule = common.get_submodule(repo, submodule_name)
- if replace_existing_branch or submodule.hexsha != submodule_revision:
- print("Replacing outdated integration branch %s" % integration_branch)
- repo.head.set_reference(branch)
- submodule = common.get_submodule(repo, submodule_name)
- else:
- print(
- "Submodule is already at %s on branch %s"
- % (submodule_revision, integration_branch)
- )
- return (integration_branch, existing_branch.commit, submodule_project)
- else:
- print("Creating integration branch %s" % integration_branch)
-
- # Create integration branch
- repo.head.set_reference(repo.create_head(integration_branch))
-
- # Update submodule
- try:
- submodule.module().git.checkout(submodule_revision)
- except GitCommandError as e:
- sys.exit("ERROR: could not checkout commit\n" + str(e))
- repo.git.add(submodule.path)
+ # Get commits between current and new revision
+ revision_range = submodule_current_rev + ".." + new_revision
+ commits = submodule_project.commits.list(
+ ref_name=revision_range, retry_transient_errors=True
+ )
+ if not commits:
+ sys.exit("ERROR: no commits found in range %s" % revision_range)
+ logging.debug("New commits: %s", commits)
- if pre_commit_hook is not None:
- pre_commit_hook(repo, submodule_project, submodule_revision)
+ # Find out if top commit is part of a merge request
+ # If so, use source branch of this MR as integration branch name
+ # Else use commit sha instead
+ integration_branch_suffix = new_revision
+ for mr in commits[0].merge_requests():
+ if mr["target_branch"] == submodule_project.default_branch:
+ integration_branch_suffix = mr["source_branch"]
+ break
+ logging.debug("Integration branch suffix: %s", integration_branch_suffix)
+
+ # Setup integration branch
+ integration_branch_name = common.integration_branch_name(
+ submodule_project.name, integration_branch_suffix
+ )
+ # Create integration branch
+ print("Creating/replacing integration branch %s" % integration_branch_name)
+ project_repo.head.set_reference(project_repo.create_head(integration_branch_name))
+
+ # Update submodule to new revision
+ submodule_repo = common.get_submodule(project_repo, submodule_name)
+ update_submodule_in_repo(project_repo, submodule_repo, new_revision)
+
+ # Update the gitlab-ci.yml file to the new revision
+ # Now also update the project '.gitlab-ci.yml' file
+ gitlab_ci_yml_filename = os.path.join(
+ project_repo.working_tree_dir, ".gitlab-ci.yml"
+ )
+ with open(gitlab_ci_yml_filename, "r", encoding="utf8") as fp:
+ gitlab_ci_yml = fp.read()
+ logging.debug(gitlab_ci_yml)
+
+ new_gitlab_ci_yml = update_gitlab_ci_include(
+ gitlab_ci_yml,
+ submodule_project.web_url.split("//")[1].split("/", 1)[1],
+ new_revision,
+ )
+ if new_gitlab_ci_yml is None:
+ print("Failed to update the include revision in '.gitlab-ci.yml'")
+ else:
+ logging.debug(new_gitlab_ci_yml)
+ with open(gitlab_ci_yml_filename, "w", encoding="utf8") as fp:
+ fp.write(new_gitlab_ci_yml)
+ project_repo.git.add(os.path.basename(gitlab_ci_yml_filename))
+
+ # Construct commit message and commit the change
+ message = "Integrate %s/%s%s\n%s" % (
+ submodule_project.name,
+ integration_branch_suffix,
+ " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
+ common.list_commits(commits),
+ )
+ # Commit the changes
+ if commit_and_push:
# Make an API request to create the gitlab.user object
+ gitlab = project.manager.gitlab
gitlab.auth()
-
- # Construct commit message and commit the change
- message = "Integrate %s/%s%s\n%s" % (
- submodule_project.name,
- integration_branch_suffix,
- " and %d more" % (len(commits) - 1) if len(commits) > 1 else "",
- common.list_commits(commits),
- )
- project_revision = common.commit_and_push(
+ integration_commit = common.commit_and_push(
project,
- repo,
- integration_branch,
+ project_repo,
+ integration_branch_name,
message,
gitlab.user.username,
gitlab.user.email,
+ less_verbose=True,
)
+ else:
+ integration_commit = None
- return (integration_branch, project_revision, submodule_project)
+ return project_repo, integration_branch_name, integration_commit, message
def main():
@@ -279,7 +350,9 @@ def main():
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
project = common.get_project(gitlab, args.project)
- update_submodule(project, args.submodule, args.revision, args.branch)
+ update_submodule_and_include_ref(
+ project, args.submodule, args.revision, args.branch
+ )
if __name__ == "__main__":