Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • seco-ne/yocto/infrastructure/gitlab-ci
1 result
Show changes
Commits on Source (26)
Hide whitespace changes
Inline Side-by-side
Showing
with 2571 additions and 141 deletions
.gitlab-ci.yml
View file @ 04cd24f2
......@@ -6,7 +6,7 @@
variables:
# CI_IMAGES_BASEPATH: Environment variable configure in gitlab
CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
CI_IMAGES_REVISION: 835a7096092eef5cecde23fd933209e7a8488637
CI_IMAGES_REVISION: 44965ccdd847f1e077670f49d546047f8ad0110c
CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REVISION}"
CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REVISION}"
......@@ -19,6 +19,8 @@ default:
stages:
- analyze
- deploy
- check
- merge
workflow:
rules:
......@@ -50,44 +52,166 @@ yamllint:
stage: analyze
timeout: 2m
script:
- yamllint -c .yamllint.yml .*.yml
- yamllint -c .yamllint.yml *.yml .*.yml
# ---------------------------------------------------------------------------------------
# Stage: deploy
# Stage: deploy-test
# ---------------------------------------------------------------------------------------
.deploy: &deploy
.foobar-manifest-projects:
variables:
PROJECT_ROOT:
${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
MANIFEST_PROJECT:
${PROJECT_ROOT}/minimal-manifest
DEPLOY_TO:
${PROJECT_ROOT}/minimal-foo
${PROJECT_ROOT}/minimal-bar
.manifest-projects:
variables:
PROJECT_ROOT:
${CI_PROJECT_ROOT_NAMESPACE}
MANIFEST_PROJECT:
${PROJECT_ROOT}/yocto/manifest
DEPLOY_TO:
${PROJECT_ROOT}/3rd-party/kuk/uboot-imx-kuk
${PROJECT_ROOT}/kernel/linux-guf
${PROJECT_ROOT}/kernel/linux-imx-kuk
${PROJECT_ROOT}/kernel/modules/egalaxi2c
${PROJECT_ROOT}/kernel/modules/gfplatdetect
${PROJECT_ROOT}/tools/gf-emc-test-suite
${PROJECT_ROOT}/tools/gf-productiontests
${PROJECT_ROOT}/tools/gfeeprom
${PROJECT_ROOT}/tools/gfxml2dto
${PROJECT_ROOT}/tools/guf-show-demo
${PROJECT_ROOT}/tools/libmdb
${PROJECT_ROOT}/tools/touchcal-conv
${PROJECT_ROOT}/tools/xconfig
${PROJECT_ROOT}/yocto/config
${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-bar
${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-foo
${PROJECT_ROOT}/yocto/infrastructure/ci-test/minimal-manifest
${PROJECT_ROOT}/yocto/layers/meta-guf-distro
${PROJECT_ROOT}/yocto/layers/meta-guf-machine
.deploy:
stage: deploy
when: manual
allow_failure: true
rules:
- if: $CI_MERGE_REQUEST_IID
script:
- cd ${CI_PROJECT_DIR}
- if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi
- ./deploy_gitlab_ci.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_ROOT_NAMESPACE}/${CI_JOB_NAME}
--manifest-project=${MANIFEST_PROJECT}
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
${MERGE}
3rd-party/kuk/uboot-imx-kuk: *deploy
kernel/linux-guf: *deploy
kernel/linux-imx-kuk: *deploy
kernel/modules/egalaxi2c: *deploy
kernel/modules/gfplatdetect: *deploy
tools/gf-emc-test-suite: *deploy
tools/gf-productiontests: *deploy
tools/gfeeprom: *deploy
tools/gfxml2dto: *deploy
tools/guf-show-demo: *deploy
tools/libmdb: *deploy
tools/touchcal-conv: *deploy
tools/xconfig: *deploy
yocto/config: *deploy
yocto/infrastructure/ci-test/minimal-bar: *deploy
yocto/infrastructure/ci-test/minimal-foo: *deploy
yocto/infrastructure/ci-test/minimal-manifest: *deploy
yocto/layers/meta-guf-distro: *deploy
yocto/layers/meta-guf-machine: *deploy
yocto/manifest: *deploy
${DEPLOY_TO}
- ./generate_job_from_template.py
--template=gitlab-ci-integration.jinja2
--image=${CI_IMAGE_PYTHON}
--branch="integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
--manifest-project=${MANIFEST_PROJECT}
--parent_merge_request="${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
${DEPLOY_TO}
> integration.yml
- cat integration.yml
artifacts:
paths:
- integration.yml
deploy:
extends:
- .deploy
- .manifest-projects
deploy-foobar:
extends:
- .deploy
- .foobar-manifest-projects
trigger:
stage: deploy
rules:
- if: $CI_MERGE_REQUEST_IID
needs: [deploy]
trigger:
include:
- artifact: integration.yml
job: deploy
strategy: depend
trigger-foobar:
stage: deploy
rules:
- if: $CI_MERGE_REQUEST_IID
needs: [deploy-foobar]
trigger:
include:
- artifact: integration.yml
job: deploy-foobar
strategy: depend
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
.merge:
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == "master"
script:
- cd ${CI_PROJECT_DIR}
- ./merge_gitlab_ci.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
${DEPLOY_TO}
merge-foobar:
extends:
- .merge
- .foobar-manifest-projects
merge:
extends:
- .merge
- .manifest-projects
# --------------------------------------------------------------------------------------
# Stage: check
# --------------------------------------------------------------------------------------
check:
stage: check
rules:
- if: $CI_MERGE_REQUEST_IID
tags:
- infrastructure
timeout: 2m
script:
- cd ${CI_PROJECT_DIR}
- MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
- MASTER_BRANCH=dunfell
- ./check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${CI_PROJECT_ROOT_NAMESPACE}/yocto/manifest
--integration-base=${MASTER_BRANCH}
--project=${CI_PROJECT_PATH}
--merge-request=${CI_MERGE_REQUEST_IID}
# The check is done for both manifests in one job as the retrigger
# looks for jobs named check, though there can only be one
- ./check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project
${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test/minimal-manifest
--integration-base=master
--project=${CI_PROJECT_PATH}
--merge-request=${CI_MERGE_REQUEST_IID}
README.md
View file @ 04cd24f2
......@@ -22,6 +22,33 @@ via the CI environment variable `GITBOT_TOKEN` (set under project's settings ->
defined.
### Workflow to add a repository to SRCREV.conv for reproducible bitbake builds
Some bitbake recipes are set to `SRCREV = "${AUTOREV}"`. To enable reproducible
builds, the current revision is written to the SRCREV.conf file. This file is
part of the manifest repo, which is used for release tags. This workflow adds
the a gitlab pipeline to the project that automatically sets the current
version in the SRCREV.conf file.
1. Add the gitlab-ci repo as submodule to the project by using the correct relative path\
`git submodule add ../../yocto/infrastructure/gitlab-ci .gitlab-ci`
1. Add an approriate .gitlab-ci.yml file. For example, copy it from the repository
[egalxi2c][4]. Modify the following variables in the file:
* MASTER\_BRANCH\_PROJECT: Set the project master branch
* BB\_RECIPE\_NAME: Set the name of the bitbake recipe
1. Create a corresponding entry in the SRCREV.conf file of the manifest repo\
`SRCREV_pn-<bitbake-recipe> = "<current-commit-hash>"`
1. Add the project into the gitlab-ci project's .gitlab-ci.yml, so changes to the
CI are deployed into the new project.
**Repository settings**
1. Verify that the merge strategy is set to *Fast-forward merge*
(General -> Merge requests -> Merge method)
1. Enable the CI/CD functionality for the repository
(General -> Visibility, project features, permissions -> CI/CD)
1. Check that the default branch is protected and that *Maintainers + Developers*
are allowed to merge (Repository -> Protected branches)
[1]: https://git-scm.com/book/en/v2/Git-Tools-Submodules
[2]: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html
[3]: https://gitlab.com/garz-fricke
[4]: https://gitlab.com/SECO-Northern-Europe/kernel/modules/egalaxi2c
accept_merge_request.py
View file @ 04cd24f2
......@@ -2,6 +2,7 @@
import common
import argparse
import logging
import sys
import time
from gitlab import (
......@@ -51,6 +52,7 @@ def accept_merge_request(project, mr, rebase=False):
except GitlabMRClosedError as e:
# See HTTP error codes for merge requests here:
# https://docs.gitlab.com/ce/api/merge_requests.html#accept-mr
logging.debug("Error from gitlab: %d", e.response_code)
if e.response_code == 405:
# Not allowed (draft, closed, pipeline pending or failed)
......
check_if_integration_branch_is_up_to_date.py
View file @ 04cd24f2
......@@ -3,9 +3,7 @@ import common
import argparse
import sys
import tempfile
from furl import furl
from git import GitCommandError, Repo
import logging
from gitlab import Gitlab, GitlabGetError
......@@ -15,41 +13,53 @@ def check_if_integration_branch_is_up_to_date(
project,
merge_request,
):
gitlab = manifest_project.manager.gitlab
integration_branch = common.integration_branch_name(
project.name, merge_request.source_branch
)
with tempfile.TemporaryDirectory() as manifest_dir:
# Construct clone url containing access token
clone_url = furl(manifest_project.http_url_to_repo)
clone_url.username = "gitlab-ci"
clone_url.password = gitlab.private_token
# Checkout manifest
integration_branch = None
if merge_request.source_branch.startswith("integrate/gitlab-ci"):
try:
manifest_repo = Repo.clone_from(
clone_url.url, manifest_dir, branch=integration_branch
integration_branch = manifest_project.branches.get(
merge_request.source_branch,
retry_transient_errors=True,
)
except GitCommandError as e:
sys.exit("ERROR: could not clone manifest repository\n" + str(e))
# Get branches
try:
integration_branch = manifest_repo.heads[integration_branch]
except IndexError:
sys.exit("ERROR: branch '%s' not found" % integration_branch)
except GitlabGetError:
# Branch not found
pass
if integration_branch is None:
integration_branch_name = common.integration_branch_name(
project.name, merge_request.source_branch
)
try:
integration_base = manifest_repo.remote().refs[integration_base]
except IndexError:
sys.exit("ERROR: branch '%s' not found" % integration_base)
integration_branch = manifest_project.branches.get(
integration_branch_name,
retry_transient_errors=True,
)
except GitlabGetError:
sys.exit("ERROR: could not find integration branch\n")
try:
integration_base_branch = manifest_project.branches.get(
integration_base, retry_transient_errors=True
)
except GitlabGetError:
sys.exit("ERROR: could not find integration base branch\n")
integration_base_id = integration_base_branch.commit["id"]
parent = integration_branch.commit
parent = manifest_project.commits.get(parent["id"], retry_transient_errors=True)
# Loop over the commits until the integration_branch head id is found
while True:
# The integration branch is up to date if its parent is the integration base
up_to_date = integration_branch.commit.parents[0] == integration_base.commit
logging.debug(parent.id)
if parent.id == integration_base_id:
return True
if len(parent.parent_ids) == 0:
break
parent_id = parent.parent_ids[0] # Assume linear history
parent = manifest_project.commits.get(parent_id, retry_transient_errors=True)
return up_to_date
return False
def main():
......@@ -90,22 +100,30 @@ def main():
dest="merge_request",
required=True,
)
parser.add_argument(
"--parent-merge-request",
help="""parent merge requests link, only used for a hint when the check failes""",
dest="parent_merge_request",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
args, _ = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
logging.debug(args)
manifest_project = common.get_project(gitlab, args.manifest_project)
project = common.get_project(gitlab, args.project)
try:
merge_request = project.mergerequests.get(
args.merge_request, retry_transient_errors=True
)
except GitlabGetError as e:
sys.exit(
"ERROR: could not get %s!%s: %s"
% (project.name, args.merge_request, e.error_message)
)
merge_request = common.get_merge_request(project, args.merge_request)
if merge_request is None:
sys.exit("ERROR: could not get %s %s" % (project.name, args.merge_request))
if check_if_integration_branch_is_up_to_date(
manifest_project=manifest_project,
......@@ -115,11 +133,15 @@ def main():
):
print("Integration branch is up to date.")
else:
mr_url = merge_request.web_url + "/pipelines"
if args.parent_merge_request is not None:
mr_url = args.parent_merge_request + "/pipelines"
sys.exit(
"Integration branch is not up to date. Please re-run the MR pipeline:\n"
" 1. Open the MR pipelines page:\n"
" %s\n"
" 2. Click 'Run Pipeline'" % (merge_request.web_url + "/pipelines")
" 2. Click 'Run Pipeline'" % (mr_url)
)
......
common.py
View file @ 04cd24f2
#!/usr/bin/env python3
import logging
import requests
import sys
import time
from furl import furl
from git import Actor, GitCommandError
from git.repo.base import Repo
from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
......@@ -20,6 +22,33 @@ def integration_branch_name(project_name, branch_name):
return "integrate/" + project_name.lower() + "/" + branch_name
def find_gitlab_ci_integration_branch(repo: Repo, branch_name):
"""
# Special handling for the gitlab-ci integration
# When the branch 'merge_request.source_branch' already starts with
# integrate/gitlab-ci we add our new commit to this branch
# Otherwise (normal behaviour) a new integration branch is created
"""
if not branch_name.startswith("integrate/gitlab-ci"):
return None
logging.debug("Integration of gitlab-ci: %s", branch_name)
for ref in repo.references:
refname = ref.name
logging.debug("Found ref: %s", refname)
if not refname.startswith("origin/"):
continue
# remove 'origin/' from the ref before compare
refname = ref.name.split("/", 1)[1]
logging.debug("Splitted refname: %s", refname)
if branch_name == refname:
integration_branch = refname
logging.debug(
"Found integration branch for gitlab-ci: %s", integration_branch
)
return integration_branch
def get_project(gitlab, project_name):
"""Get a GitLab project by its name (including or excluding namespace)"""
project = None
......@@ -120,10 +149,12 @@ def commit_and_push(project: Project, repo: Repo, branch, message, name, email):
"""Commit and push to a repo branch"""
author = Actor(name, email)
repo.index.commit(message, author=author, committer=author)
print(repo.git.log("--oneline", "-n", "5"))
# Push commit
try:
origin = repo.remote("origin")
logging.debug("Push branch %s to %s", branch, origin)
origin.push(branch, force=True)
except GitCommandError as e:
sys.exit("ERROR: could not commit changes\n" + str(e))
......@@ -158,3 +189,69 @@ def extract_message_body(msg):
msg = msg[1:]
return msg
def get_merge_request(project: Project, merge_request):
"""Return a gitlab mergereqest specified either by id or by link"""
# MR may also be specified as
# SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-bar!115
if "!" in merge_request:
merge_request = int(merge_request.split("!")[-1])
logging.debug("Number of MR: %d", merge_request)
try:
mr = project.mergerequests.get(merge_request, retry_transient_errors=True)
except GitlabGetError:
return None
return mr
def clone_project(project: Project, into, branch=None):
gitlab = project.manager.gitlab
# If no branch is given, use project's default branch
if branch is None:
branch = project.default_branch
# Construct clone url containing access token
clone_url = furl(project.http_url_to_repo)
clone_url.username = "gitlab-ci"
clone_url.password = gitlab.private_token
# Checkout project
try:
repo = Repo.clone_from(clone_url.url, into, branch=branch, depth=1)
except GitCommandError as e:
raise Exception("could not clone repository\n" + str(e)) from e
except IndexError:
raise Exception("branch '%s' not found" % branch) from e
return repo
def get_repository_file_raw(project: Project, filename, ref=None):
# TODO tree objects are not supported
fileobj = get_repository_file_obj(project, filename, ref)
return project.repository_raw_blob(
fileobj["id"], retry_transient_errors=True
).decode()
def get_repository_file_obj(project: Project, filename, ref=None):
# TODO tree objects are not supported
if ref is None:
ref = project.default_branch
logging.debug("Using default branch %s", ref)
repository_tree = project.repository_tree(
ref=ref, all=True, retry_transient_errors=True
)
logging.debug(repository_tree)
fileobj = [f for f in repository_tree if f["name"] == filename]
if len(fileobj) == 0:
logging.error("Could not find file %s", filename)
for f in repository_tree:
logging.debug(f["name"])
return None
fileobj = fileobj[0]
return fileobj
common.yml 0 → 100644
View file @ 04cd24f2
# --------------------------------------------------------------------------------------
# Common definitions that may be used in all subprojects
# --------------------------------------------------------------------------------------
---
variables:
# CI_IMAGES_BASEPATH: Environment variable configure in gitlab
CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
CI_IMAGES_REV: 44965ccdd847f1e077670f49d546047f8ad0110c
CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REV}"
CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REV}"
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
# Reduced depth as checkout of larger projects (like the kernel)
# may take too long
GIT_DEPTH: 1
# FIXME: due to a missing feature in GitLab we cannot use this variable
# in the build stage further down this file. If it ever changes, it has
# to be changed there too.
# (https://gitlab.com/gitlab-org/gitlab/-/issues/249583)
MANIFEST_PROJECT: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/manifest
MASTER_BRANCH_MANIFEST: dunfell
MASTER_BRANCH_PROJECT: dunfell
# --------------------------------------------------------------------------------------
# Common infrastructure settings
# --------------------------------------------------------------------------------------
.infrastructure:
stage: infrastructure
tags:
- infrastructure
timeout: 5m
image: "${CI_IMAGE_PYTHON}"
needs: []
variables:
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
yamllint:
extends: .infrastructure
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
script:
- yamllint -c .gitlab-ci/.yamllint.yml .*.yml
# --------------------------------------------------------------------------------------
# Common check job, used in yocto and foobar manifest-integration projects
# --------------------------------------------------------------------------------------
.check:
extends: .infrastructure
stage: check
rules:
- if: $CI_MERGE_REQUEST_IID
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
needs: ["integrate"]
script:
- cd ${CI_PROJECT_DIR}
# When running in a trigger pipeline the CII_MERGE_REQUEST_IID is not set
# but CI_OPEN_MERGE_REQUESTS. We use the first of this comma separated list
# in this case
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
# The 'parent_merge_request' is passed from the trigger
# in case this check job is part of a gitlab-ci integration
# pipeline. It is only used to display the correct MR to run again
# in a failed check
- if [ -n "${parent_merge_request}" ];then
PARENT_MR="--parent-merge-request=${parent_merge_request}";
fi
- .gitlab-ci/check_if_integration_branch_is_up_to_date.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--integration-base=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--merge-request=${MERGE_REQUEST}
${PARENT_MR}
deploy_gitlab_ci.py
View file @ 04cd24f2
......@@ -2,12 +2,152 @@
import common
import argparse
import logging
import sys
import os
from gitlab import Gitlab
from accept_merge_request import accept_merge_request
from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
from update_submodule import update_submodule
from update_gitlab_ci import update_gitlab_ci_include
from ruamel.yaml import YAML
def update_rev_in_gitlab_ci(repo, submodule_project, submodule_revision):
# Add changed revision also to .gitlab-ci.yml
gitlab_ci_yml = os.path.join(repo.working_tree_dir, ".gitlab-ci.yml")
# ======================================
# Hack to deploy the .gitlab-ci.yml which does the include
# to the child project
logging.debug("Origin: %s", repo.remotes.origin.url)
# Contains the base file to be used in the subprojects
if "ci-test" in repo.remotes.origin.url:
if "manifest" in repo.remotes.origin.url:
include_file = "foobar-manifest.yml"
else:
include_file = "foobar-manifest-integration.yml"
else:
if "manifest" in repo.remotes.origin.url:
include_file = "manifest.yml"
else:
include_file = "manifest-integration.yml"
# -----------------------------------
# Adapt content in the include file
# -----------------------------------
# Read values from existing file
yaml = YAML()
with open(gitlab_ci_yml, "r", encoding="utf8") as fp:
data = yaml.load(fp)
logging.debug("Yaml: %s", data)
try:
masterbranch = data["variables"]["MASTER_BRANCH_PROJECT"]
logging.debug("Masterbranch %s", masterbranch)
except KeyError:
masterbranch = None
try:
recipe = data["variables"]["BB_RECIPE_NAME"]
logging.debug("Recipe %s", recipe)
except KeyError:
recipe = None
# -----------------------------------
with open(gitlab_ci_yml, "w", encoding="utf8") as fp:
fp.write(
"""---
# ---------------------------------------------------------------------------------------
# Include the default CI steps from the gitlab-ci repo
# ---------------------------------------------------------------------------------------
include:
- project: '${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci'
ref: 49cc4204323bca76190e5ffae1a7d5627157c073
"""
)
fp.write(" file: '{}'\n".format(include_file))
if masterbranch is not None or recipe is not None:
fp.write("\nvariables:\n")
if masterbranch is not None:
fp.write(" MASTER_BRANCH_PROJECT: {}\n".format(masterbranch))
if recipe is not None:
fp.write(" BB_RECIPE_NAME: {}\n".format(recipe))
# -----------------------------------
repo.git.add(gitlab_ci_yml)
# ======================================
if update_gitlab_ci_include(
gitlab_ci_yml,
submodule_project.web_url.split("//")[1].split("/", 1)[1],
submodule_revision,
):
repo.git.add(gitlab_ci_yml)
with open(gitlab_ci_yml, "r", encoding="utf8") as fp:
logging.debug(fp.read())
def deploy_into(project, submodule, revision, branch, replace_existing_branch=False):
"""Update the submodule and include refs to the submodule in the given project.
Create mergerequest if needed.
Parameters:
project ( gitlab project): The project which's submodule should be updated
submodule_name (string): The name of the submodule to pull
submodule_revision (hex string): The sha hash of the commit to update the submodule to
branch (string): branch to update, if None, the projects default branch is used
replace_existing_branch: When an existing integration branch is found it is always replaced.
Returns: tuple of:
branch (string): Name of the newly created integration branch
merge_request (gitlab mr): Mergerequest for the integration branch
"""
# Update submodule
integration_branch, _, submodule_project = update_submodule(
project,
submodule,
revision,
branch,
pre_commit_hook=update_rev_in_gitlab_ci,
replace_existing_branch=replace_existing_branch,
)
logging.debug("Integration branch: %s", integration_branch)
# If submodule is already at specified revision, return directly
if not integration_branch:
return None, submodule_project
# Get source merge request
mrs = get_merge_requests(
submodule_project,
# TODO should this be submodule_project's default branch?
target_branch="master",
commit=revision,
)
if not mrs:
sys.exit(
"ERROR: could not determine source merge request for commit %s" % revision
)
source_mr = mrs[0]
# Create merge request
mr, created = create_merge_request(
project, integration_branch, project.default_branch
)
if created:
common.crosslink_merge_requests(source_mr, mr)
print("Created new merge request:\n%s" % mr.web_url)
else:
print("Existing integration merge request:\n%s" % mr.web_url)
return integration_branch, mr
def main():
......@@ -26,9 +166,9 @@ def main():
)
parser.add_argument(
"--project",
"--manifest-project",
help="""name of the GitLab project""",
dest="project",
required=True,
)
parser.add_argument(
"--submodule",
......@@ -57,42 +197,57 @@ def main():
required=False,
default=False,
)
parser.add_argument(
"projects",
help="""List of projects the change should be deployed to additionally
to the manifest project given as named parameter.""",
nargs="*",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
args, _ = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
project = common.get_project(gitlab, args.project)
# Update submodule
integration_branch, _, submodule_project = update_submodule(
project, args.submodule, args.revision, args.branch
logging.debug("Integrate into: %s", args.project)
# Update submodule in this project, create MR
integration_branch, mr = deploy_into(
project,
args.submodule,
args.revision,
args.branch,
replace_existing_branch=len(args.projects) > 0,
)
merge_request_manifest = mr
merge_requests = []
# If submodule is already at specified revision, exit successfully
# TODO: is this correct for multi deploy also?
if not integration_branch:
sys.exit(0)
# Get source merge request
mrs = get_merge_requests(
submodule_project,
target_branch="master",
commit=args.revision,
)
if not mrs:
sys.exit(
"ERROR: could not determine source merge request for commit %s"
% args.revision
for p in args.projects:
gitlab_p = common.get_project(gitlab, p)
logging.debug("Integrate into: %s", p)
integration_branch, mr = deploy_into(
gitlab_p,
args.submodule,
args.revision,
args.branch,
)
source_mr = mrs[0]
merge_requests.append(mr)
# Create merge request
mr, created = create_merge_request(
project, integration_branch, project.default_branch
)
if created:
common.crosslink_merge_requests(source_mr, mr)
print("Created new merge request:\n%s" % mr.web_url)
else:
print("Existing integration merge request:\n%s" % mr.web_url)
logging.debug("Integration branch: %s", integration_branch)
if not args.merge:
print(
......@@ -103,20 +258,24 @@ def main():
)
sys.exit(0)
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(project, mr)
# The manifest needs to be merged at last
merge_requests.append(merge_request_manifest)
for mr in merge_requests:
logging.debug("Merge %s", mr)
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(project, mr)
# Attempt to merge
merged = accept_merge_request(project, mr, rebase=True)
# Attempt to merge
merged = accept_merge_request(project, mr, rebase=True)
if not merged:
sys.exit(
"Integration MR could not be merged. You have two possibilities to fix "
"this:\n"
" 1. Checkout the MR and rebase it on the current master manually, or\n"
" 2. Delete the MR (Edit -> Delete in the MR UI)\n"
"In either case restart this job afterwards in order to get it merged."
)
if not merged:
sys.exit(
"Integration MR could not be merged. You have two possibilities to fix "
"this:\n"
" 1. Checkout the MR and rebase it on the current master manually, or\n"
" 2. Delete the MR (Edit -> Delete in the MR UI)\n"
"In either case restart this job afterwards in order to get it merged."
)
print("Successfully merged")
......
docs/setup-k8s-gitlab-runner.md 0 → 100644
View file @ 04cd24f2
# Setup gitlab runner in Azure cloud
## Tools needed locally to control
* kubectl - Comandline tool für k8s
* helm - 'Paketmanager' für k8s
* aks - Azure kubernetes service
* azure-cli - Comandline tools für azure
* unter Manjaro: `yay kubectl helm aks azure-cli`
## Create a kubernetes cluster in azure
* Mostly followed:
`https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal`
* Nodetype: `Standard_D4ds_v5`
*
* Log into Azure Web
* Home -> Create resource
* Create Kubernetes Service -> Create
* Fill in
* Subscription
* Resource group -> Create new -> "gitlab-test-resource-group"
* Reset is default: Region East US ...
* Node size: Standard_D4ds_v5 ( maximum allowed for free substriction)
* Node count 1
## Install gitlab runner in k8s
* Mostly followed:
`https://medium.com/@ruben.laguna/installing-a-gitlab-runner-on-kubernetes-ac386c924bc8`
* Connect to azure:
`az login`
`az aks get-credentials --resource-group gitlab-test-resource-group --name gitlab-test-cluster`
`kubectl cluster-info`
`kubectl get nodes`
NAME STATUS ROLES AGE VERSION
aks-agentpool-94672520-vmss000000 Ready agent 8d v1.21.9
### Create namespace in k8s for the runner
`gitlab-runner-namespace.yaml`
{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": {
"name": "gitlab-runner",
"labels": {
"name": "gitlab-runner"
}
}
}
`kubectl create -f gitlab-runner-namespace.json`
`kubectl get namespace`
### Create a role and set permissions
`gitlab-runner-gitlab-runner-role.yml` from article:
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitlab-runner
namespace: gitlab-runner
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["list", "get", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get"]
Working rules (for me, not nessacarily correct and secure):
`gitlab-runner-gitlab-runner-role.yml`
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitlab-runner
namespace: gitlab-runner
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["list", "get", "watch", "create", "delete", "update"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get"]
- apiGroups: [""]
resources: ["pods/attach"]
verbs: ["list", "get", "create", "delete", "update"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["list", "get", "create", "delete", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["list", "get", "create", "delete", "update"]
`kubectl create -f gitlab-runner-gitlab-runner-role.yaml`
`kubectl replace -f gitlab-runner-gitlab-runner-role.yaml`
`kubectl edit role --namespace gitlab-runner`
`kubectl get --namespace=gitlab-runner role`
Used the following permissions, more then in the article:
`kubectl describe role --namespace gitlab-runner gitlab-runner`
Name: gitlab-runner
Labels: <none>
Annotations: <none>
PolicyRule:
Resources Non-Resource URLs Resource Names Verbs
--------- ----------------- -------------- -----
pods/exec [] [] [create]
pods/log [] [] [get]
configmaps [] [] [list get create delete update]
pods/attach [] [] [list get create delete update]
secrets [] [] [list get create delete update]
pods [] [] [list get watch create delete update]
### Assign the role to the service account
`kubectl create rolebinding --namespace=gitlab-runner gitlab-runner-binding --role=gitlab-runner --serviceaccount=gitlab-runner:default`
`kubectl get --namespace gitlab-runner rolebinding`
### Install gitlab runner on Kubernetes using Helm
Get runner registration token from gitlab:
`https://gitlab.com/groups/SECO-Northern-Europe/-/settings/ci_cd`
The values.yml file contains a set of variables configuring the package installed with helm:
`values.yml` as in article:
gitlabUrl: https://gitlab.com/
runnerRegistrationToken: "ssssssssssss"
`values.yml` actually used:
## The GitLab Server URL (with protocol) that want to register the runner against
## ref: https://docs.gitlab.com/runner/commands/README.html#gitlab-runner-register
##
gitlabUrl: https://gitlab.com/
## The Registration Token for adding new Runners to the GitLab Server. This must
## be retrieved from your GitLab Instance.
## ref: https://docs.gitlab.com/ce/ci/runners/README.html
##
runnerRegistrationToken: "GR1348941XWJHK4__ZszXTLTPda2R"
## Unregister all runners before termination
##
## Updating the runner's chart version or configuration will cause the runner container
## to be terminated and created again. This may cause your Gitlab instance to reference
## non-existant runners. Un-registering the runner before termination mitigates this issue.
## ref: https://docs.gitlab.com/runner/commands/README.html#gitlab-runner-unregister
##
# unregisterRunners: true
## Configuration for the Pods that the runner launches for each new job
##
runners:
## Specify the tags associated with the runner. Comma-separated list of tags.
##
## ref: https://docs.gitlab.com/ee/ci/runners/configure_runners.html#use-tags-to-control-which-jobs-a-runner-can-run
##
tags: "azure"
Get the complete values.yml for the package:
`helm show values gitlab/gitlab-runner`
### Helm
`helm init`
### Gitlab repo zu helm hinzufügen
`helm repo add gitlab https://charts.gitlab.io`
`helm search repo -l gitlab/gitlab-runner`
`helm install --namespace gitlab-runner gitlab-runner -f values.yaml gitlab/gitlab-runner`
`kubectl get --namespace gitlab-runner pod`
NAME READY STATUS RESTARTS AGE
gitlab-runner-gitlab-runner-86f5c5647-qc475 1/1 Running 0 6d20h
runner-m6sb9pz-project-17852514-concurrent-0ggqd8 2/2 Running 0 114m
### Check gitlab for the new registered runner
foobar-manifest-integration.yml 0 → 100644
View file @ 04cd24f2
---
# -------------------------------------------------------------------------------------
# CI steps needed to integrate a sub project change into the manifest project
# foobar variant which is used for testing the CI
# -------------------------------------------------------------------------------------
include:
- local: common.yml
image: "${CI_IMAGE_PYTHON}"
stages:
- infrastructure
- integrate
- merge
- build
- check
variables:
# FIXME: due to a missing feature in GitLab, we cannot use this variable in the build
# stage further down this file. If it ever changes, it has to be changed there too.
# (https://gitlab.com/gitlab-org/gitlab/-/issues/249583)
MANIFEST_PROJECT: "${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/\
ci-test/minimal-manifest"
# The master branch is hardcoded here, because it cannot be determined automatically.
# Has to be modified for new branches, e.g. new Yocto versions or fix releases.
MASTER_BRANCH_MANIFEST: master
MASTER_BRANCH_PROJECT: master
# The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
# to match the repository and the bitbake recipe name.
# We set it here to none, as every project needing it
# has to specify it in its own gitlab-ci.yml file.
# The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
# used for projects referenced in the manifest file.
BB_RECIPE_NAME: none
workflow:
rules:
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
# Do not run pipelines for merge requests for integrate/gitlab-ci/ branches
# These are trigger explicitly from the integration pipeline in gitlab-ci repo
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
# Do not run pipelines on forked projects.
# The pipelines would not work anyway because of the users permissions.
# There are two cases catched here:
# 1. The project is forked into someones gitlab namespace and a MR to
# include a change into this forked project is created. In this case
# is the CI_PROJECT_ROOT_NAMESPACE not SECO-Northern-Europe but the
# namespace the fork lives in.
# 2. The MR from the forked project is created to merge the change into this
# the project in the SECO-Northern-Europe namespace (customer sending
# change to us). Here the the IDs used below differ.
#
- if: $CI_PROJECT_ROOT_NAMESPACE == "SECO-Northern-Europe"
&& $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID
# ---------------------------------------------------------------------------------------
# integrate
# Create a commit in the manifest to pull this change
# ---------------------------------------------------------------------------------------
integrate:
extends: .infrastructure
rules:
# We have to make sure that the pipeline runs for the current manifest
# master at the time a merge request is created. Otherwise we cannot
# guarantee a green master after merging.
- if: $CI_MERGE_REQUEST_IID
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
cache:
policy: push
script:
- cd ${CI_PROJECT_DIR}
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
- .gitlab-ci/integrate_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--integration-base=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--merge-request=${MERGE_REQUEST}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
artifacts:
paths:
- manifest_revision
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
merge:
extends: .infrastructure
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == $MASTER_BRANCH_PROJECT
script:
- cd ${CI_PROJECT_DIR}
- .gitlab-ci/merge_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--master-branch=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--master-branch-project=${MASTER_BRANCH_PROJECT}
--commit=${CI_COMMIT_SHA}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
artifacts:
paths:
- manifest_revision
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
build:
stage: build
rules:
# execute this in MR only and not for integrate/gitlab-ci/ integrations
# branches. These are build after the integration has been done in all
# projects
- if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
trigger:
project: SECO-Northern-Europe/yocto/infrastructure/ci-test/minimal-manifest
branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
# --------------------------------------------------------------------------------------
# Stage: check
# --------------------------------------------------------------------------------------
check:
extends: .check
foobar-manifest.yml 0 → 100644
View file @ 04cd24f2
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- local: common.yml
stages:
- retrigger
- infrastructure
- build
- test
variables:
# The master branch is hardcoded here, because it cannot be determined automatically.
# Has to be modified for new branches, e.g. new Yocto versions or fix releases.
MASTER_BRANCH_MANIFEST: master
MASTER_BRANCH_PROJECT: master
workflow:
rules:
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
# Do not run pipelines for merge requests
- if: $CI_MERGE_REQUEST_IID
when: never
# Do not run pipelines on forked projects
# (use id instead of name because of rename)
- if: $CI_PROJECT_ID != "21745909"
when: never
# Do not run pipelines on integration branches
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
when: never
# In all other cases, run the pipeline automatically
- when: always
# --------------------------------------------------------------------------------------
# Stage: retrigger
# --------------------------------------------------------------------------------------
retrigger:
extends: .infrastructure
stage: retrigger
rules:
- if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
script:
- PROJECTS=$(
.gitlab-ci/get_manifest_projects.py
--manifest=default.xml
--remote=ci-test
)
# Add the gitlab-ci project
- PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci"
# TODO retrigger gitlab-ci integration also
# Retrigger also project in SRCREV
- echo -e "Projects:\n${PROJECTS}"
- for PROJECT in ${PROJECTS}; do
.gitlab-ci/retrigger_mr_pipeline_jobs.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${PROJECT}
--state=opened
--target-branch=${MASTER_BRANCH_MANIFEST}
--job=check
;
done
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
.setup_ssh: &setup_ssh
# setup ssh key to access private repos
# https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
# add content of variable SSH_KNOWN_HOSTS to known hosts
# https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys
- echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
.repo_checkout: &repo_checkout
# setup build dir
- cd ${CI_PROJECT_DIR}
- repo init -u ${CI_REPOSITORY_URL} -b refs/pipelines/${CI_PIPELINE_ID}
- repo sync --detach --current-branch --no-tags --force-remove-dirty
--optimized-fetch --force-sync
.build: &build
- cd ${CI_PROJECT_DIR}
- VERSION=$(cd .repo/manifests && git describe --tags)
- cat .repo/manifests/default.xml
- ls * > files-$VERSION.txt
- ls *
- FOO_FILES=$(cd foo && ls | wc -l)
- BAR_FILES=$(cd bar && ls | wc -l)
- DIFF=$((BAR_FILES-FOO_FILES))
- (($DIFF >= -1 && $DIFF <= 1))
build:files:
stage: build
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
needs: []
tags:
- infrastructure
timeout: 2m
image:
name: "${CI_IMAGE_YOCTO}"
# Override entrypoint so we can pass --id to set the UID and GID for the user that
# is created in the container. This is a feature of the crops/poky images.
# See poky-entry.py for details.
entrypoint:
- "/usr/bin/distro-entry.sh"
- "/usr/bin/dumb-init"
- "--"
- "/usr/bin/poky-entry.py"
- "--id=118:998"
variables:
GIT_STRATEGY: none
before_script:
- *setup_ssh
script:
- *repo_checkout
- *build
artifacts:
expire_in: 7d
paths:
- files-*
build:echo:
stage: build
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
needs: []
tags:
- infrastructure
timeout: 2m
image: ${CI_IMAGE_PYTHON}
script:
- printenv
- echo "Build successful"
build:merge_request:
stage: build
rules:
- if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
needs: []
tags:
- infrastructure
timeout: 2m
image: ${CI_IMAGE_PYTHON}
variables:
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
script:
- cd ${CI_PROJECT_DIR}
# Get pipeline for merge request
- MR_PIPELINE=$(.gitlab-ci/get_pipelines.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--commit=${CI_COMMIT_SHA}
--ref=^${MASTER_BRANCH_MANIFEST} || true | head -1)
# If pipeline exists, mirror its result
- if [ ! -z "${MR_PIPELINE}" ]; then
.gitlab-ci/mirror_pipeline_result.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--pipeline=${MR_PIPELINE}
# If no pipeline found, trigger a new one on the master
- else
.gitlab-ci/trigger_pipeline.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--ref=${MASTER_BRANCH_MANIFEST}
- fi
# --------------------------------------------------------------------------------------
# Stage: test
# --------------------------------------------------------------------------------------
.test:
stage: test
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
when: manual
allow_failure: true
tags:
- infrastructure
image: ${CI_IMAGE_PYTHON}
script:
- exit ${RETURNCODE}
test:pass:
extends:
- .test
variables:
RETURNCODE: 0
test:fail:
extends:
- .test
variables:
RETURNCODE: 1
generate_job_from_template.py 0 → 100755
View file @ 04cd24f2
#!/usr/bin/env python3
from __future__ import print_function
import argparse
import io
import logging
import os
import re
import sys
from copy import deepcopy
from jinja2 import Environment, FileSystemLoader, StrictUndefined
from jinja2.exceptions import TemplateNotFound
from ruamel.yaml import YAML
def parse_template(yaml_string):
"""
Round trip lava_job through ruamel to test parsing and improve formatting.
Comments are preserved.
In: yaml-formatted string
Out: validated yaml-formatted string
"""
yaml = YAML()
yaml.indent(sequence=4, offset=2)
# ruamel does not provide a mechanism to dump to string, so use StringIO to catch it
output = io.StringIO()
yaml.dump(yaml.load(yaml_string), output)
return output.getvalue()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--template",
help="""A Jinja2 template for the yml file to be genarated.
All variables in the templates will be substituted using the
Jinja2 engine. Variables can be passed either as environment
variables or as command line arguments
(e.g. --my-variable=myvalue). In the latter, hyphens ("-") are
replaced with underscores ("_"), i.e. "--my-variable"
becomes "my_variable".""",
required=True,
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
parser.add_argument(
"arguments",
nargs="*",
help="""Arguments passed as list to the template.""",
)
# parse known args
args, unknown = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
logging.debug("Using template: %s", args.template)
# parse optional arguments
parser = argparse.ArgumentParser()
for arg in unknown:
if arg.startswith(("-", "--")):
parser.add_argument(arg.split("=")[0])
optional_args, _ = parser.parse_known_args()
logging.debug("Optional args: %s", optional_args)
# update context with given optional arguments
j2_env = Environment(undefined=StrictUndefined, extensions=["jinja2.ext.do"])
context = dict(deepcopy(os.environ))
for arg in vars(optional_args):
context[arg] = vars(optional_args)[arg]
if args.arguments:
context["optional_arguments"] = args.arguments
for k, v in context.items():
logging.debug("Optional args: %s -- %s", k, v)
# render the template
j2_env.loader = FileSystemLoader(os.path.dirname(args.template))
try:
lava_job = j2_env.get_template(os.path.basename(args.template)).render(context)
lava_job = parse_template(lava_job)
# Remove duplicate newlines
lava_job = re.sub(r"\n+", r"\n", lava_job)
print(lava_job)
except TemplateNotFound:
print("ERROR: Template %s not found" % args.template, file=sys.stderr)
exit(1)
if __name__ == "__main__":
main()
gitlab-ci-integration.jinja2 0 → 100644
View file @ 04cd24f2
---
#======================================================
# Trigger the integration pipeline for all repos using
# the gitlab-ci repository.
#======================================================
{% if image is defined %}
image: {{ image }}
{% endif %}
workflow:
rules:
# This rule is needed, as otherwise the workflow:rules from
# the parent job seem to be used and prevent the pipeline generation
- if: $CI_PIPELINE_SOURCE == "parent_pipeline"
stages:
- integrate
- build
default:
tags:
# All jobs should use the infrastructure runner
- infrastructure
{% set projectneeds = { 'project': "" } %}
{% if optional_arguments is defined %}
{% set prev_project = "" %}
{% for project in optional_arguments %}
{% set projectshort = project.split('/')[-1] %}
{{ projectshort }}:
stage: integrate
needs: [ {{ projectneeds.project }} ]
{% if parent_merge_request is defined %}
variables:
parent_merge_request: {{ parent_merge_request }}
{% endif %}
trigger:
project: {{ project }}
branch: {{ branch }}
strategy: depend
{% if projectneeds.update({ 'project': projectshort }) %}{% endif %}
{% endfor %}
{% endif %}
{% if manifest_project is defined %}
{% set manifest_project_short = manifest_project.split('/')[-1] %}
{{ manifest_project_short }}:
stage: build
trigger:
project: {{ manifest_project }}
branch: {{ branch }}
strategy: depend
{% endif %}
integrate_into_manifest.py
View file @ 04cd24f2
......@@ -2,13 +2,14 @@
import common
import argparse
import logging
import sys
import tempfile
import re
from pathlib import Path
from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab, GitlabGetError
from gitlab import Gitlab
from lxml import etree
......@@ -42,14 +43,34 @@ def integrate_into_manifest(
except IndexError:
sys.exit("ERROR: branch '%s' not found" % integration_base)
# Create integration branch (delete former one if already exists)
integration_branch = common.integration_branch_name(
project.name, merge_request.source_branch
# Special handling for the gitlab-ci integration
# When the branch 'merge_request.source_branch' already starts with
# integrate/gitlab-ci we add our new commit to this branch
# Otherwise (normal behaviour) a new integration branch is created
integration_branch = common.find_gitlab_ci_integration_branch(
manifest_repo, merge_request.source_branch
)
for ref in manifest_repo.references:
if integration_branch == ref.name:
manifest_repo.delete_head(ref)
manifest_repo.head.set_reference(manifest_repo.create_head(integration_branch))
if integration_branch is not None:
manifest_repo.git.checkout(
"-b", integration_branch, "origin/{}".format(integration_branch)
)
logging.debug("Heads: %s", manifest_repo.heads)
manifest_repo.heads[integration_branch].checkout()
print(manifest_repo.git.log("--oneline", "-n", "5"))
else:
# Create integration branch (delete former one if already exists)
integration_branch = common.integration_branch_name(
project.name, merge_request.source_branch
)
for ref in manifest_repo.references:
if integration_branch == ref.name:
manifest_repo.delete_head(ref)
logging.debug("Integration branch: %s", integration_branch)
manifest_repo.head.set_reference(
manifest_repo.create_head(integration_branch)
)
# Parse manifest file
try:
......@@ -130,6 +151,7 @@ def integrate_into_manifest(
gitlab.user.username,
gitlab.user.email,
)
logging.debug("New revision in manifest: %s", manifest_revision)
return manifest_revision
......@@ -189,7 +211,8 @@ def main():
)
parser.add_argument(
"--merge-request",
help="""project merge request IID containing the changes to be integrated""",
help="""project merge request IID or link containing the
changes to be integrated""",
dest="merge_request",
required=True,
)
......@@ -199,22 +222,28 @@ def main():
dest="revision_file",
required=False,
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
args, _ = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
logging.debug(args)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
manifest_project = common.get_project(gitlab, args.manifest_project)
project = common.get_project(gitlab, args.project)
try:
merge_request = project.mergerequests.get(
args.merge_request, retry_transient_errors=True
)
except GitlabGetError as e:
sys.exit(
"ERROR: could not get %s!%s: %s"
% (project.name, args.merge_request, e.error_message)
)
logging.debug("Project: %s", project.name)
logging.debug("Merge Request: %s", args.merge_request)
merge_request = common.get_merge_request(project, args.merge_request)
if merge_request is None:
sys.exit("ERROR: could not get %s %s" % (project.name, args.merge_request))
manifest_revision = integrate_into_manifest(
manifest_project=manifest_project,
......
manifest-build.yml 0 → 100644
View file @ 04cd24f2
---
variables:
BUILDPATH: "build-${CI_PARAM_DISTRO}-${CI_PARAM_MACHINE}"
IMAGEBASEPATH: "tmp/deploy/images/"
IMAGEPATH: "${IMAGEBASEPATH}/${CI_PARAM_MACHINE}"
LICENSESPATH: "tmp/deploy/licenses"
SDKPATH: "tmp/deploy/sdk/"
DEPLOYPATH_TEST: "/artifacts/${CI_JOB_ID}/"
GIT_BASE_URL: "git@${CI_SERVER_HOST}:${CI_PROJECT_ROOT_NAMESPACE}"
TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
.setup_ssh: &setup_ssh
# setup ssh key to access private repos
# https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor
- eval $(ssh-agent -s)
- echo "$JENKINSGUF_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
# add content of variable SSH_KNOWN_HOSTS to known hosts
# https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys
- echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
.repo_checkout: &repo_checkout
- cd ${CI_PROJECT_DIR}
- repo init --submodules -u ${CI_REPOSITORY_URL}
-b refs/pipelines/${CI_PIPELINE_ID}
- repo sync --detach --current-branch --no-tags --force-remove-dirty
--optimized-fetch --force-sync
.collect_srcrevs: &collect_srcrevs
# write all package AUTOREVS to file
- |-
SRCREVS_FILE="${CI_PROJECT_DIR}/${BUILDPATH}/${IMAGEPATH}/BUILD_SRCREVS.log"
if [ -d "$( dirname "${SRCREVS_FILE}" )" ];then
buildhistory-collect-srcrevs > ${SRCREVS_FILE}
cat ${SRCREVS_FILE}
fi
.dump_install_command: &dump_install_command
# print install instructions
- |-
for i in ${INSTALLSCRIPTS};do
SCRIPT="${CI_PROJECT_DIR}/${BUILDPATH}/${IMAGEPATH}/${i}"
if [[ -f "${SCRIPT}" ]]; then
cat <<-EOF
==============================
Install the image:
export GITLAB_TOKEN=<your_access_token>
FNG="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/${BUILDPATH}/${IMAGEPATH}/${i}"
curl --location --header "PRIVATE-TOKEN: \$GITLAB_TOKEN" "\$FNG" \
| sh -s -- --url="\$(dirname "\$FNG")"
==============================
EOF
fi
done
.build_script: &build_script
# setup build environment
- echo "Build configuration MACHINE=${CI_PARAM_MACHINE}
DISTRO=${CI_PARAM_DISTRO} IMAGE=${CI_PARAM_IMAGE}"
- echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env
- echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env
- echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env
- echo "Using build dir ${BUILDPATH}"
- export MACHINE="${CI_PARAM_MACHINE}"
- export DISTRO="${CI_PARAM_DISTRO}"
- export EULA="1"
- source ./"${SETUPSCRIPT}" "${BUILDPATH}"
# start build
- echo -e "section_start:`date +%s`:bitbake_run\r\e[0KBitbake Log"
- echo "bitbake ${CI_PARAM_IMAGE} -c ${BITBAKE_TASK}"
- bitbake "${CI_PARAM_IMAGE}" -c "${BITBAKE_TASK}"
- echo -e "section_end:`date +%s`:bitbake_run\r\e[0K"
.build:
variables:
GIT_STRATEGY: none
SETUPSCRIPT: "setup-environment"
INSTALLSCRIPTS: "fng-install.sh fngsystem-self-update.sh"
BITBAKE_TASK: "build"
before_script:
- *setup_ssh
- *repo_checkout
script:
- *build_script
- *collect_srcrevs
- *dump_install_command
artifacts:
paths:
- "${BUILDPATH}/${IMAGEPATH}/*"
- "${BUILDPATH}/${LICENSESPATH}/**/license.manifest"
reports:
dotenv: build.env
.package:
variables:
CI_PARAM_PACKAGE_FTP: "false"
after_script:
- |-
# Package release files
[ -e build.env ] && source build.env
BUILDPATH="build-${BUILD_DISTRO}-${BUILD_MACHINE}"
[ -z "${ARTIFACTS_IMAGE_PATH}" ] && \
ARTIFACTS_IMAGE_PATH="${BUILDPATH}/${IMAGEBASEPATH}/${BUILD_MACHINE}"
[ -z "${ARTIFACTS_LICENSES_PATH}" ] && \
ARTIFACTS_LICENSES_PATH="${BUILDPATH}/${LICENSESPATH}"
[ -z "${ARTIFACTS_SDK_PATH}" ] && \
ARTIFACTS_SDK_PATH="${BUILDPATH}/${SDKPATH}"
if $CI_PARAM_PACKAGE_FTP; then
UPLOAD_PARAM="" # don't store as gitlab artifact
OUTDIR_BASE="/artifacts-ftp"
else
UPLOAD_PARAM="--outputdir-upload=release"
OUTDIR_BASE="/artifacts"
fi
# If we are on the master branch and a tag is set
# we tread it as release
if [ -n "$CI_COMMIT_TAG" ];then
outdir="${OUTDIR_BASE}-yocto/Releases"
[ "${BUILD_DISTRO}" = "guf-fngsystem" ] && \
outdir="${OUTDIR_BASE}-fngsystem"
else
outdir="${OUTDIR_BASE}-yocto/Interne_Releases"
[ "${BUILD_DISTRO}" = "guf-fngsystem" ] && \
outdir="${OUTDIR_BASE}-fngsystem/CI_Builds"
fi
# Generate AlphaPlan FWR articles if release tag is set
if [ -n "$CI_COMMIT_TAG" ] && \
[[ "${CI_PARAM_PACKAGE_FTP}" == "false" ]];then
UPLOAD_PARAM="${UPLOAD_PARAM} --generate-fwr-articles"
fi
script=".gitlab-ci/package_release.py"
[ ! -x "$script" ] && script=".repo/manifests/$script"
[ ! -x "$script" ] && echo "Failed to find package_release script"
# Image if available
if [ -d "${ARTIFACTS_IMAGE_PATH}" ];then
$script \
--images-dir="${ARTIFACTS_IMAGE_PATH}" \
--licenses-dir="${ARTIFACTS_LICENSES_PATH}" \
--doc-dir=. \
--outputdir-local=${outdir} \
$UPLOAD_PARAM
fi
# SDK if available
if [ -d "${ARTIFACTS_SDK_PATH}" ];then
$script \
--sdk-dir=${ARTIFACTS_SDK_PATH} \
--outputdir-local="${outdir}"
fi
.prepare_test:
before_script:
- *setup_ssh
- |-
# Copy artifacts to local server for automated tests
.gitlab-ci/package_release.py \
--images-dir="${BUILDPATH}/${IMAGEPATH}" \
--outputdir-local="${DEPLOYPATH_TEST}"
manifest-integration.yml 0 → 100644
View file @ 04cd24f2
---
# ---------------------------------------------------------------------------------------
# Global
# ---------------------------------------------------------------------------------------
include:
- local: common.yml
stages:
- infrastructure
- integrate
- merge
- build
- check
variables:
# The BB_RECIPE_NAME is used for projects referenced in the SRCREV file
# to match the repository and the bitbake recipe name.
# We set it here to none, as every project needing it
# has to specify it in its own gitlab-ci.yml file.
# The BB_RECIPE_NAME is passed to the python scripts below anyway, but not
# used for projects referenced in the manifest file.
BB_RECIPE_NAME: none
workflow:
rules:
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
# Do not run pipelines for merge requests for integrate/gitlab-ci/ branches
# These are trigger explicitly from the integration pipeline in gitlab-ci repo
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/gitlab-ci\/.*/
when: never
# Do not run pipelines on forked projects.
# The pipelines would not work anyway because of the users permissions.
# There are two cases catched here:
# 1. The project is forked into someones gitlab namespace and a MR to
# include a change into this forked project is created. In this case
# is the CI_PROJECT_ROOT_NAMESPACE not SECO-Northern-Europe but the
# namespace the fork lives in.
# 2. The MR from the forked project is created to merge the change into this
# the project in the SECO-Northern-Europe namespace (customer sending
# change to us). Here the the IDs used below differ.
#
- if: $CI_PROJECT_ROOT_NAMESPACE == "SECO-Northern-Europe"
&& $CI_MERGE_REQUEST_SOURCE_PROJECT_ID == $CI_MERGE_REQUEST_PROJECT_ID
# ---------------------------------------------------------------------------------------
# integrate
# Create a commit in the manifest to pull this change
# ---------------------------------------------------------------------------------------
integrate:
extends: .infrastructure
rules:
# We have to make sure that the pipeline runs for the current manifest
# master at the time a merge request is created. Otherwise we cannot
# guarantee a green master after merging.
- if: $CI_MERGE_REQUEST_IID
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
cache:
policy: push
script:
- cd ${CI_PROJECT_DIR}
- if [ -n "${CI_MERGE_REQUEST_IID}" ];then
MERGE_REQUEST="${CI_MERGE_REQUEST_IID}";
else
MERGE_REQUEST="${CI_OPEN_MERGE_REQUESTS%%,*}";
fi
- .gitlab-ci/integrate_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--integration-base=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--merge-request=${MERGE_REQUEST}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
artifacts:
paths:
- manifest_revision
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
merge:
extends: .infrastructure
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == $MASTER_BRANCH_PROJECT
script:
- cd ${CI_PROJECT_DIR}
- .gitlab-ci/merge_into_manifest.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--manifest-project=${MANIFEST_PROJECT}
--master-branch=${MASTER_BRANCH_MANIFEST}
--project=${CI_PROJECT_PATH}
--master-branch-project=${MASTER_BRANCH_PROJECT}
--commit=${CI_COMMIT_SHA}
--save-revision-to=manifest_revision
--recipe-name=${BB_RECIPE_NAME}
artifacts:
paths:
- manifest_revision
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
build:
stage: build
rules:
# execute this in MR only and not for integrate/gitlab-ci/ integrations
# branches. These are build after the integration has been done in all
# projects
- if: $CI_MERGE_REQUEST_IID && $CI_COMMIT_REF_NAME !~ /^integrate\/gitlab-ci\/.*/
trigger:
project: SECO-Northern-Europe/yocto/manifest
branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}"
strategy: depend
# --------------------------------------------------------------------------------------
# Stage: check
# --------------------------------------------------------------------------------------
check:
extends: .check
manifest.yml 0 → 100644
View file @ 04cd24f2
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
include:
- local: manifest-build.yml
- local: common.yml
stages:
- retrigger
- infrastructure
- build
- test
- deploy
- uploadftp
variables:
# Default image and distro
CI_PARAM_IMAGE: guf-image
CI_PARAM_DISTRO: guf-wayland
workflow:
rules:
# Explicitly allow externally triggered pipelines in every case
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api"
# Do not run pipelines for merge requests
- if: $CI_MERGE_REQUEST_IID
when: never
# Do not run pipelines on forked projects
# (use id instead of name because of rename)
- if: $CI_PROJECT_ID != "17852514"
when: never
# Do not run pipelines on integration branches
- if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/
when: never
# In all other cases, run the pipeline automatically
- when: always
# --------------------------------------------------------------------------------------
# Stage: retrigger
# --------------------------------------------------------------------------------------
retrigger:
extends: .infrastructure
stage: retrigger
rules:
- if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
script:
- PROJECTS=$(
.gitlab-ci/get_manifest_projects.py
--manifest=default.xml
--remote=seco-north
--concat-namespaces
)
# Add the gitlab-ci project
- PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci"
# TODO retrigger gitlab-ci integration also
# Retrigger also project in SRCREV
- echo -e "Projects:\n${PROJECTS}"
- for PROJECT in ${PROJECTS}; do
.gitlab-ci/retrigger_mr_pipeline_jobs.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${PROJECT}
--state=opened
--target-branch=${MASTER_BRANCH_MANIFEST}
--job=check
;
done
# --------------------------------------------------------------------------------------
# Stage: infrastructure
# --------------------------------------------------------------------------------------
changelog:
extends: .infrastructure
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
variables:
IMAGE_PATH: ${CI_IMAGES_BASEPATH}/changelog-generator
IMAGE_REVISION: 2fd56690a5dabde6eda101182e8359f1c3609664
image:
name: "${IMAGE_PATH}:${IMAGE_REVISION}"
# set entrypoint to noop to be able to run from script
entrypoint: [""]
script: "changelog_generator.py --branch ${MASTER_BRANCH_MANIFEST} > changelog.md"
artifacts:
expire_in: 4 weeks
paths:
- "changelog.md"
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
.buildbase:
tags:
- builds
timeout: 8h
interruptible: true
image:
name: "${CI_IMAGE_YOCTO}"
# Override entrypoint so we can pass --id to set the UID and GID for the
# user that is created in the container. This is a feature of the
# crops/poky images. See poky-entry.py for details.
entrypoint:
- "/usr/bin/distro-entry.sh"
- "/usr/bin/dumb-init"
- "--"
- "/usr/bin/poky-entry.py"
- "--id=118:998"
artifacts:
expire_in: 4 weeks
.buildimage:
extends:
- .buildbase
- .build
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
needs: []
.buildfng:
extends:
- .buildimage
variables:
CI_PARAM_IMAGE: fngsystem-image
CI_PARAM_DISTRO: guf-fngsystem
build:merge_request:
extends: .infrastructure
stage: build
timeout: 1h
rules:
- if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api"
script:
- cd ${CI_PROJECT_DIR}
# Get pipeline for merge request
- MR_PIPELINE=$(.gitlab-ci/get_pipelines.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--commit=${CI_COMMIT_SHA}
--ref=^${MASTER_BRANCH_MANIFEST} || true | head -1)
# If pipeline exists, mirror its result
- if [ ! -z "${MR_PIPELINE}" ]; then
.gitlab-ci/mirror_pipeline_result.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--pipeline=${MR_PIPELINE}
# If no pipeline found, trigger a new one on the master
- else
.gitlab-ci/trigger_pipeline.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--ref=${MASTER_BRANCH_MANIFEST}
- fi
.buildsdk:
extends:
- .buildimage
- .package
stage: build
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
when: manual
allow_failure: true
variables:
BITBAKE_TASK: "populate_sdk"
artifacts:
paths:
- "${BUILDPATH}/${SDKPATH}/*.manifest"
- "${BUILDPATH}/${SDKPATH}/*.json"
reports:
dotenv: package.env
# ---------------------------------------------------------------------------------------
# Stage: test
# ---------------------------------------------------------------------------------------
.test:
extends:
- .infrastructure
- .prepare_test
timeout: 1h
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
when: manual
allow_failure: true
variables:
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
CI_PARAM_TEST_SUITE: '{platform}.jinja2'
CI_PARAM_EXTRA: --nop
artifacts:
when: always
paths:
- "results/**"
reports:
junit: results/results-*.xml
after_script:
- rm -r "${DEPLOYPATH_TEST}"
script:
- |-
# Submit tests to lava server
RELEASE=$(ls ${DEPLOYPATH_TEST}/)
INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh"
FNG_INSTALL_URL="${ARTIFACTS_HOST_URL}/${INSTALLSCRIPT_ABS#/*/}"
.gitlab-ci/submit_test.py \
--fng-install "${FNG_INSTALL_URL}" \
--name \
"Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \
--results-path "results" \
--test-repo ${TESTS_GIT_URL} \
--test-repo-branch ${MASTER_BRANCH_MANIFEST} \
--test-plan ${CI_PARAM_TEST_SUITE} \
${CI_PARAM_EXTRA} \
${CI_PARAM_PLATFORMS}
# --------------------------------------------------------------------------------------
# Stage: deploy
# --------------------------------------------------------------------------------------
.deployimage:
extends:
- .infrastructure
- .package
rules:
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
when: manual
allow_failure: true
script:
- echo
artifacts:
paths:
- release/**/**/*
reports:
dotenv: package.env
# --------------------------------------------------------------------------------------
# Stage: uploadftp
# --------------------------------------------------------------------------------------
.uploadftp:
variables:
CI_PARAM_PACKAGE_FTP: "true"
extends:
- .infrastructure
- .package
rules:
- if: $CI_COMMIT_TAG
when: manual
allow_failure: true
script:
- echo
.uploadsdkftp:
variables:
ARTIFACTS_SDK_PATH: "$LOCALDIR/$BUILD_MACHINE/sdk"
# ---------------------------------------------------------------------------------------
# Actual jobs
# ---------------------------------------------------------------------------------------
build:imx6guf:
extends: .buildimage
stage: build
variables:
CI_PARAM_MACHINE: imx6guf
build:imx6ullguf:
extends: .buildimage
stage: build
variables:
CI_PARAM_MACHINE: imx6ullguf
build:imx8mguf:
extends: .buildimage
stage: build
variables:
CI_PARAM_MACHINE: imx8mguf
build:imx8mpguf:
extends: .buildimage
stage: build
variables:
CI_PARAM_MACHINE: imx8mpguf
build:imx6guf:fngsystem:
extends: .buildfng
stage: build
variables:
CI_PARAM_MACHINE: imx6guf
build:imx6ullguf:fngsystem:
extends: .buildfng
stage: build
variables:
CI_PARAM_MACHINE: imx6ullguf
build:imx8mguf:fngsystem:
extends: .buildfng
stage: build
variables:
CI_PARAM_MACHINE: imx8mguf
build:imx8mpguf:fngsystem:
extends: .buildfng
stage: build
variables:
CI_PARAM_MACHINE: imx8mpguf
# -------------------------------------------------------------------------------------
buildsdk:imx6guf:
extends: .buildsdk
stage: build
variables:
CI_PARAM_MACHINE: imx6guf
buildsdk:imx6ullguf:
extends: .buildsdk
stage: build
variables:
CI_PARAM_MACHINE: imx6ullguf
buildsdk:imx8mguf:
extends: .buildsdk
stage: build
variables:
CI_PARAM_MACHINE: imx8mguf
buildsdk:imx8mpguf:
extends: .buildsdk
stage: build
variables:
CI_PARAM_MACHINE: imx8mpguf
# -------------------------------------------------------------------------------------
deployimage:imx6guf:
extends: .deployimage
stage: deploy
needs: ["build:imx6guf", "changelog"]
deployimage:imx6ullguf:
extends: .deployimage
stage: deploy
needs: ["build:imx6ullguf", "changelog"]
deployimage:imx8mguf:
extends: .deployimage
stage: deploy
needs: ["build:imx8mguf", "changelog"]
deployimage:imx8mpguf:
extends: .deployimage
stage: deploy
needs: ["build:imx8mpguf", "changelog"]
deployimage:imx6guf:fngsystem:
extends: .deployimage
stage: deploy
needs: ["build:imx6guf:fngsystem", "changelog"]
deployimage:imx6ullguf:fngsystem:
extends: .deployimage
stage: deploy
needs: ["build:imx6ullguf:fngsystem", "changelog"]
deployimage:imx8mguf:fngsystem:
extends: .deployimage
stage: deploy
needs: ["build:imx8mguf:fngsystem", "changelog"]
deployimage:imx8mpguf:fngsystem:
extends: .deployimage
stage: deploy
needs: ["build:imx8mpguf:fngsystem", "changelog"]
# -------------------------------------------------------------------------------------
uploadftp:imx6guf:
extends:
- .uploadftp
- .uploadsdkftp
stage: uploadftp
needs: ["build:imx6guf", "buildsdk:imx6guf", "changelog"]
uploadftp:imx6ullguf:
extends:
- .uploadftp
- .uploadsdkftp
stage: uploadftp
needs: ["build:imx6ullguf", "buildsdk:imx6ullguf", "changelog"]
uploadftp:imx8mguf:
extends:
- .uploadftp
- .uploadsdkftp
stage: uploadftp
needs: ["build:imx8mguf", "buildsdk:imx8mguf", "changelog"]
uploadftp:imx8mpguf:
extends:
- .uploadftp
- .uploadsdkftp
stage: uploadftp
needs: ["build:imx8mpguf", "buildsdk:imx8mpguf", "changelog"]
uploadftp:imx6guf:fngsystem:
extends: .uploadftp
stage: uploadftp
needs: ["build:imx6guf:fngsystem", "changelog"]
uploadftp:imx6ullguf:fngsystem:
extends: .uploadftp
stage: uploadftp
needs: ["build:imx6ullguf:fngsystem", "changelog"]
uploadftp:imx8mguf:fngsystem:
extends: .uploadftp
stage: uploadftp
needs: ["build:imx8mguf:fngsystem", "changelog"]
uploadftp:imx8mpguf:fngsystem:
extends: .uploadftp
stage: uploadftp
needs: ["build:imx8mpguf:fngsystem", "changelog"]
# -------------------------------------------------------------------------------------
platformtest:imx6guf:
extends: .test
stage: test
needs:
- job: build:imx6guf
variables:
CI_PARAM_MACHINE: imx6guf
CI_PARAM_PLATFORMS: santaro santoka santino santino-lt
platformtest:imx6ullguf:
extends: .test
stage: test
needs:
- job: build:imx6ullguf
variables:
CI_PARAM_MACHINE: imx6ullguf
CI_PARAM_PLATFORMS: nallino
platformtest:imx8mguf:
extends: .test
stage: test
needs:
- job: build:imx6ullguf
variables:
CI_PARAM_MACHINE: imx8mguf
CI_PARAM_PLATFORMS: tanaro
smoketest:imx6guf:
extends: .test
stage: test
needs:
- job: build:imx6guf
variables:
CI_PARAM_MACHINE: imx6guf
CI_PARAM_PLATFORMS: imx6guf
CI_PARAM_TEST_SUITE: boot.jinja2
CI_PARAM_EXTRA: --all-devices
smoketest:imx6ullguf:
extends: .test
stage: test
needs:
- job: build:imx6ullguf
variables:
CI_PARAM_MACHINE: imx6ullguf
CI_PARAM_PLATFORMS: imx6ullguf
CI_PARAM_TEST_SUITE: boot.jinja2
CI_PARAM_EXTRA: --all-devices
smoketest:imx8mguf:
extends: .test
stage: test
needs:
- job: build:imx8mguf
variables:
CI_PARAM_MACHINE: imx8mguf
CI_PARAM_PLATFORMS: imx8mguf
CI_PARAM_TEST_SUITE: boot.jinja2
CI_PARAM_EXTRA: --all-devices
merge_gitlab_ci.py 0 → 100755
View file @ 04cd24f2
#!/usr/bin/env python3
import common
import argparse
import logging
import sys
from gitlab import Gitlab, GitlabGetError
from accept_merge_request import accept_merge_request
import update_submodule
from gitlab.v4.objects import Project
def find_integration_merge_request(
project: Project, submodule, revision, target_branch
):
(
submodule_path,
submodule_revision,
) = update_submodule.get_submodule_project_path_and_revision(
project, submodule, target_branch
)
logging.debug("Module: %s, Revision: %s", submodule_path, submodule_revision)
# Get submodule project
gitlab = project.manager.gitlab
submodule_project = common.get_project(gitlab, submodule_path)
# Get the integration branch name
integration_branch_suffix = (
update_submodule.get_submodule_integration_branch_suffix(
submodule_project, revision
)
)
integration_branch_name = common.integration_branch_name(
submodule_project.name, integration_branch_suffix
)
logging.debug(integration_branch_name)
# Get the merge request for the branch TODO catch exception
try:
integration_branch = project.branches.get(
integration_branch_name, retry_transient_errors=True
)
except GitlabGetError:
print("ERROR: Failed to find integration branch for {}".format(submodule))
return None
logging.debug(integration_branch)
commit = project.commits.get(
integration_branch.commit["id"], retry_transient_errors=True
)
for mr in commit.merge_requests(retry_transient_errors=True):
if mr["target_branch"] == target_branch:
integration_mr = mr
break
if integration_mr is None:
print(
"ERROR: Failed to find integration merge request for %s",
integration_branch_name,
)
return None
logging.debug(integration_mr["iid"])
mr = project.mergerequests.get(integration_mr["iid"], retry_transient_errors=True)
return mr
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
required=True,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
"--manifest-project",
help="""name of the GitLab project""",
dest="project",
)
parser.add_argument(
"--submodule",
help="""submodule to update""",
dest="submodule",
required=True,
)
parser.add_argument(
"--revision",
help="""new revision for submodule""",
dest="revision",
required=True,
)
parser.add_argument(
"--branch",
help="""project branch (if not default branch)""",
dest="branch",
required=False,
default=None,
)
parser.add_argument(
"--merge",
help="""if set, perform merge after integration""",
dest="merge",
action="store_true",
required=False,
default=False,
)
parser.add_argument(
"projects",
help="""List of projects the change should be deployed to additionally
to the manifest project given as named parameter.""",
nargs="*",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
args, _ = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
for p in args.projects + [args.project]:
project = common.get_project(gitlab, p)
branch = args.branch
if branch is None:
branch = project.default_branch
mr = find_integration_merge_request(
project, args.submodule, args.revision, branch
)
print("Merge {}!{}: {}".format(project.name, mr.iid, mr.title))
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(project, mr)
# Attempt to merge
merged = accept_merge_request(
project,
mr,
# Only the file manifest commit may be rebased
# other rebase would require new integration commits
rebase=(p == args.project),
)
if not merged:
sys.exit(
"Integration MR could not be merged. You have two possibilities to fix "
"this:\n"
" 1. Checkout the MR and rebase it on the current master manually, or\n"
" 2. Delete the MR (Edit -> Delete in the MR UI)\n"
"In either case restart this job afterwards in order to get it merged."
)
print("Successfully merged")
exit()
if __name__ == "__main__":
main()
package_release.py
View file @ 04cd24f2
......@@ -302,14 +302,15 @@ def main():
)
# Generate metadata
generate_metadata(
machine,
version,
artifacts_all,
sdkname,
output_dir,
outlocal_dir,
)
if args.sdk_dir is None:
generate_metadata(
machine,
version,
artifacts_all,
sdkname,
output_dir,
outlocal_dir,
)
# Handle SDK if available
if args.sdk_dir is not None:
......
update_gitlab_ci.py 0 → 100755
View file @ 04cd24f2
#!/usr/bin/env python3
import argparse
import logging
import re
def update_gitlab_ci_include(filename, include_project, new_revision):
"""Update the include statement in a gitlab-ci yml to a given revision
Parameters:
filename( string): The path to the file to change.
include_project( string): The path used to reference the project the include points to.
new_revision (string): The hex sha to set the include to.
Returns: True if the file was changed.
"""
# Set the possible include in the local .gitlab.yml file
# to the new revision. The include needs to have the revision
# specified directly in the file, as it is parsed before the
# submodule checkout is done
# Use custom read write method as I didn't got ruamel yaml
# to keep all costum formating (linebreaks ...)
# This assumes following format of the include block
# include:
# - project: 'SECO-Northern-Europe/yocto/infrastructure/gitlab-ci'
# ref: c5a3793e783fcb364c7f3bda73e8cd7c08a08804
# file: 'manifest-childs.yml'
# Verify hash format:
if re.match(r"\A[0-9a-fA-F]{40}\Z", new_revision) is None:
raise TypeError("Format of specified revision is not correct")
parsestate = 0
changed = False
# Remove the SECO-Northern-Europe part from the priject filter
# as it is normally specified by $CI_PROJECT_ROOT_NAMESPACE
include_project = include_project.split("/", 1)[1]
logging.debug("Include project: %s", include_project)
logging.debug("New revision: %s", new_revision)
with open(filename, "r+", encoding="UTF-8") as fp:
while True:
linestart = fp.tell()
line = fp.readline()
parts = line.partition(":")
logging.debug("Splitted input line: %s", parts)
if len(line) == 0:
break # End of file
if parsestate == 0:
if parts[0] == "include":
# Found include block
parsestate = 1
logging.debug("Found 'include' block at %d", linestart)
elif parsestate == 1:
if parts[0] == "\n":
break # End of include block
if (
parts[0].endswith(" - project")
and parts[2].find(include_project) >= 0
):
# Found the correct project
parsestate = 2
logging.debug("Found 'project' entry at %d", linestart)
elif parsestate == 2:
if parts[0].endswith(" ref"):
# Found the ref: entry, compare the revision
logging.debug("Found 'ref' entry at %d", linestart)
parsestate = 1
if parts[2].find(new_revision) >= 0:
print(
"Revision in {} is already set to {}".format(
filename, new_revision
)
)
else:
print(
"Changed revision in {} to {}".format(
filename, new_revision
)
)
fp.seek(linestart)
fp.write("{}: {}".format(parts[0], new_revision))
fp.flush()
changed = True
elif parts[0].find("- ") >= 0 or not parts[0].startswith(" "):
# Format of the line is not 'name: value' assume end of block
# Block was not found
break
return changed
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--filename",
help="""File to change""",
required=True,
)
parser.add_argument(
"--include-project",
help="""The path to the included project as used in 'filename'""",
required=True,
)
parser.add_argument(
"--revision",
help="""new revision for submodule""",
required=True,
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
args, _ = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
update_gitlab_ci_include(args.filename, args.include_project, args.revision)
if __name__ == "__main__":
main()
update_submodule.py
View file @ 04cd24f2
......@@ -2,16 +2,93 @@
import common
import argparse
import logging
import os
import sys
import tempfile
from configparser import ConfigParser
from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab
from gitlab.v4.objects import Project
def update_submodule(project, submodule_name, submodule_revision, branch=None):
"""Update submodule of gitlab project to given revision"""
def get_submodule_project_path_and_revision(project: Project, submodule, branch=None):
gitmodules = common.get_repository_file_raw(project, ".gitmodules", ref=branch)
if gitmodules is None:
logging.error("Submodule %s not found in %s.", submodule, project.name)
return None, None
logging.debug("Gitmodules: %s", gitmodules)
cfgparse = ConfigParser()
cfgparse.read_string(gitmodules)
try:
section = cfgparse['submodule "{}"'.format(submodule)]
except KeyError:
logging.error("Submodule %s not found in %s.", submodule, project.name)
return None, None
submodule_url = section["url"]
# absolut path to a relative submodule
# Check for relative path
if not submodule_url.startswith(".."):
logging.error("absolute submodule paths are not supported (%s)", submodule_url)
return None, None
# Get absolute project path
# This cannot be done with gitpython directly due to issue:
# https://github.com/gitpython-developers/GitPython/issues/730
relative_path = os.path.splitext(submodule_url)[0] # remove .git
project_path = project.path_with_namespace
while relative_path.startswith(".."):
relative_path = relative_path[3:] # strip off '../'
project_path, _ = os.path.split(project_path) # remove last part
submodule_project_path = os.path.join(project_path, relative_path)
# Get current revision
gitmodule_rev = common.get_repository_file_obj(project, submodule, ref=branch)
return submodule_project_path, gitmodule_rev["id"]
def get_submodule_integration_branch_suffix(submodule_project: Project, revision):
# Find out if top commit is part of a merge request
# If so, use source branch of this MR as integration branch name
# Else use commit sha instead
integration_branch_suffix = revision
for mr in submodule_project.commits.get(revision).merge_requests():
if mr["target_branch"] == submodule_project.default_branch:
integration_branch_suffix = mr["source_branch"]
break
return integration_branch_suffix
def update_submodule(
project,
submodule_name,
submodule_revision,
branch=None,
pre_commit_hook=None,
replace_existing_branch=False,
):
"""Update submodule of gitlab project to given revision
Parameters:
project (gitlab project): The project which's submodule should be updated
submodule_name (string): The name of the submodule to pull
submodule_revision (hex string): The sha hash of the commit to update the submodule to
branch (string): branch to update, if None, the projects default branch is used
pre_commit_hook: Function to be called before the actual commit is done, to add additional changes.
Arguments passed: ( repo, submodule_project, submodule_revision)
replace_existing_branch: When an existing integration branch is found it is always replaced.
Returns: tuple of:
branch (string): Name of the newly created integration branch
revision (string): hexsha of the new commit
submodule_project ( gitlab project): The submodule as gitlab priject instance
"""
gitlab = project.manager.gitlab
......@@ -28,7 +105,7 @@ def update_submodule(project, submodule_name, submodule_revision, branch=None):
# Checkout project
try:
repo = Repo.clone_from(clone_url.url, project_dir, branch=branch)
repo = Repo.clone_from(clone_url.url, project_dir, branch=branch, depth=1)
except GitCommandError as e:
sys.exit("ERROR: could not clone repository\n" + str(e))
except IndexError:
......@@ -40,7 +117,9 @@ def update_submodule(project, submodule_name, submodule_revision, branch=None):
# Check if revisions are different
if submodule.hexsha == submodule_revision:
print("Submodule is already at %s" % submodule_revision)
return (None, None, None)
if not replace_existing_branch:
# TODO test this
return (None, None, None)
# Check for relative path
if not submodule.url.startswith(".."):
......@@ -99,16 +178,16 @@ def update_submodule(project, submodule_name, submodule_revision, branch=None):
if existing_branch:
repo.head.set_reference(existing_branch)
submodule = common.get_submodule(repo, submodule_name)
if submodule.hexsha == submodule_revision:
if replace_existing_branch or submodule.hexsha != submodule_revision:
print("Replacing outdated integration branch %s" % integration_branch)
repo.head.set_reference(branch)
submodule = common.get_submodule(repo, submodule_name)
else:
print(
"Submodule is already at %s on branch %s"
% (submodule_revision, integration_branch)
)
return (integration_branch, existing_branch.commit, submodule_project)
else:
print("Replacing outdated integration branch %s" % integration_branch)
repo.head.set_reference(branch)
submodule = common.get_submodule(repo, submodule_name)
else:
print("Creating integration branch %s" % integration_branch)
......@@ -122,6 +201,9 @@ def update_submodule(project, submodule_name, submodule_revision, branch=None):
sys.exit("ERROR: could not checkout commit\n" + str(e))
repo.git.add(submodule.path)
if pre_commit_hook is not None:
pre_commit_hook(repo, submodule_project, submodule_revision)
# Make an API request to create the gitlab.user object
gitlab.auth()
......@@ -183,8 +265,16 @@ def main():
required=False,
default=None,
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="""Increase verbosity.""",
)
args, _ = parser.parse_known_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
project = common.get_project(gitlab, args.project)
......