Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • seco-ne/yocto/infrastructure/gitlab-ci
1 result
Show changes
Commits on Source (224)
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1699 additions and 293 deletions
.gitignore
View file @ a59a93a0
/__pycache__
scripts/__pycache__
.idea/
.gitlab-ci.yml
View file @ a59a93a0
---
# ---------------------------------------------------------------------------------------
# Global
# ---------------------------------------------------------------------------------------
......@@ -5,7 +6,7 @@
variables:
# CI_IMAGES_BASEPATH: Environment variable configure in gitlab
CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
CI_IMAGES_REVISION: 835a7096092eef5cecde23fd933209e7a8488637
CI_IMAGES_REVISION: 5dfddb02d67bbd16beb09ff4e31afcd5380f5788
CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REVISION}"
CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REVISION}"
......@@ -17,7 +18,9 @@ default:
stages:
- analyze
- deploy
- integrate
- merge
- build
workflow:
rules:
......@@ -27,45 +30,179 @@ workflow:
# ---------------------------------------------------------------------------------------
# Stage: analyze
# ---------------------------------------------------------------------------------------
pylint:
.analyze:
stage: analyze
rules:
- if: $CI_MERGE_REQUEST_IID
- if: $CI_COMMIT_BRANCH == "master"
when: never
timeout: 2m
script:
- pylint --rcfile=pylintrc *.py
black:
stage: analyze
timeout: 2m
extends: .analyze
script:
- cd scripts
- black --diff --check *.py
executable:
stage: analyze
timeout: 2m
extends: .analyze
script:
- cd scripts
- (! find ! -executable -name "*.py" -exec echo "not executable:"" {}" \; | grep .)
isort:
extends: .analyze
script:
- cd scripts
- isort . --check --diff
pylint:
extends: .analyze
script:
- cd scripts
- pylint --rcfile=pylintrc *.py
yamllint:
extends: .analyze
script:
- yamllint -c .yamllint.yml *.yml .*.yml
# ---------------------------------------------------------------------------------------
# Stage: deploy
# Stage: integrate
# ---------------------------------------------------------------------------------------
.deploy: &deploy
stage: deploy
when: manual
allow_failure: true
.integrate:
stage: integrate
rules:
- if: $CI_MERGE_REQUEST_IID
when: manual
allow_failure: true
variables:
MERGE: ""
script:
- cd ${CI_PROJECT_DIR}
- if [[ "$CI_COMMIT_BRANCH" == "master" ]]; then MERGE="--merge"; else MERGE=""; fi
- ./deploy_gitlab_ci.py
- scripts/deploy_gitlab_ci.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_ROOT_NAMESPACE}/${CI_JOB_NAME}
--manifest-project=${MANIFEST_PROJECT}
--manifest-branch=${MANIFEST_BRANCH}
--submodule=.gitlab-ci
--revision=${CI_COMMIT_SHA}
--group=${PROJECT_GROUP}
${MERGE}
yocto/infrastructure/ci-test/minimal-bar: *deploy
yocto/infrastructure/ci-test/minimal-foo: *deploy
yocto/infrastructure/ci-test/minimal-manifest: *deploy
yocto/layers/meta-guf-distro: *deploy
yocto/layers/meta-guf-machine: *deploy
yocto/manifest: *deploy
# Running multiple integration jobs for the same manifest in parallel can lead to race
# conditions if there are projects integrating a single branch to different manifest
# branches. Use resource groups to force execution one by one.
.integrate-ci-test:
extends: .integrate
resource_group: integrate-ci-test
variables:
PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/ci-test
MANIFEST_PROJECT: ${PROJECT_GROUP}/minimal-manifest
.integrate-yocto:
extends: .integrate
resource_group: integrate-yocto
variables:
PROJECT_GROUP: ${CI_PROJECT_ROOT_NAMESPACE}
MANIFEST_PROJECT: ${PROJECT_GROUP}/yocto/manifest
# Jobs
integrate-ci-test:primary:
extends: .integrate-ci-test
variables:
MANIFEST_BRANCH: primary
integrate-ci-test:secondary:
extends: .integrate-ci-test
variables:
MANIFEST_BRANCH: secondary
integrate-yocto:dunfell:
extends: .integrate-yocto
variables:
MANIFEST_BRANCH: dunfell
integrate-yocto:kirkstone:
extends: .integrate-yocto
variables:
MANIFEST_BRANCH: kirkstone
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
.build:
stage: build
rules:
- if: $CI_MERGE_REQUEST_IID
allow_failure: true
.build-ci-test:
extends: .build
trigger:
project: seco-ne/yocto/infrastructure/ci-test/minimal-manifest
branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"
strategy: depend
.build-yocto:
extends: .build
trigger:
project: seco-ne/yocto/manifest
branch: "integrate/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/into/${MANIFEST_BRANCH}"
strategy: depend
# Jobs
build-ci-test:primary:
extends: .build-ci-test
needs: ["integrate-ci-test:primary"]
variables:
MANIFEST_BRANCH: primary
build-ci-test:secondary:
extends: .build-ci-test
needs: ["integrate-ci-test:secondary"]
variables:
MANIFEST_BRANCH: secondary
build-yocto:dunfell:
extends: .build-yocto
needs: ["integrate-yocto:dunfell"]
variables:
MANIFEST_BRANCH: dunfell
build-yocto:kirkstone:
extends: .build-yocto
needs: ["integrate-yocto:kirkstone"]
variables:
MANIFEST_BRANCH: kirkstone
# --------------------------------------------------------------------------------------
# Stage: merge
# --------------------------------------------------------------------------------------
.merge:
stage: merge
rules:
- if: $CI_COMMIT_BRANCH == "master"
when: manual
allow_failure: true
variables:
MERGE: --merge --project=${CI_PROJECT_PATH} --branch=${CI_COMMIT_REF_NAME}
merge-ci-test:
extends:
- .integrate-ci-test
- .merge
variables:
MANIFEST_BRANCH: primary,secondary
merge-yocto:
extends:
- .integrate-yocto
- .merge
variables:
MANIFEST_BRANCH: dunfell,kirkstone
.yamllint.yml 0 → 100644
View file @ a59a93a0
---
extends: default
rules:
line-length:
max: 88
README.md
View file @ a59a93a0
# gitlab-ci
<!----------------------------------------------------------------------------->
# gitlab-ci Repository
<!----------------------------------------------------------------------------->
GitLab CI scripts
This repository contains scripts for Yocto infrastructure tasks, e.g. automatic
integration of project changes into the manifest. It is meant to be included into all
relevant Yocto repositories as a [git submodule][1].
This repository contains **GitLab CI scripts** for Yocto infrastructure tasks,
e.g. automatic integration of project changes into the manifest. It is meant to
be included into all relevant Yocto repositories as a [git submodule][1].
[1]: https://git-scm.com/book/en/v2/Git-Tools-Submodules
### Providing `GITBOT_TOKEN`
The scripts need a [personal GitLab access token][2] of a user with access to all
relevant repositories (manifest and all contained projects) in order to work correctly.
We have created the @guf-gitbot user for this task. Norman and Tim know the login
credentials for it. These should not be needed, though, as the user is member of the
[Garz & Fricke][3] group and thus already has access to all group projects. However,
everyone can add the user as a member to every project he has access to, if needed.
<!----------------------------------------------------------------------------->
## The GitBot user
<!----------------------------------------------------------------------------->
The personal access token of the @guf-gitbot user has to be provided on each repository
via the CI environment variable `GITBOT_TOKEN` (set under project's settings -> CI/CD
-> Variables). Its value can be read on projects which already have this variable
defined.
Most scripts inside this repository need a [personal GitLab access token][2] of
a user with access to all relevant repositories (manifest and all contained
projects) in order to work correctly.
We have created the @gitbot user for this task. The login credentials for it are
stored in our [KeePass][3] safe. These should not be needed, though, as the user
is member of the [`seco-ne`][4] group and thus already has access to all group
projects. However, everyone can add the user as a member to every project he has
access to, if needed.
The personal access token of the @gitbot is provided via the CI environment
variable `GITBOT_TOKEN` for all projects in the `seco-ne` group (set in the
[group's CI/CD settings][5] under "Variables").
[1]: https://git-scm.com/book/en/v2/Git-Tools-Submodules
[2]: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html
[3]: https://gitlab.com/garz-fricke
[3]: https://keepass.info
[4]: https://git.seco.com/seco-ne
[5]: https://git.seco.com/groups/seco-ne/yocto/-/settings/ci_cd
<!----------------------------------------------------------------------------->
## Automatic manifest integration
<!----------------------------------------------------------------------------->
See this chapter for information on how the automatic integration process works:
▶ [Automatic manifest integration][6]
[6]: docs/automatic-manifest-integration.md
<!----------------------------------------------------------------------------->
## Deployment
<!----------------------------------------------------------------------------->
See this chapter for information on how to deploy changes in the `gitlab-ci`
repository to all repositories that are using it.
▶ [gitlab-ci Deployment][7]
[7]: docs/gitlab-ci-deployment.md
<!----------------------------------------------------------------------------->
# Manifest pipeline
<!----------------------------------------------------------------------------->
See this chapter for the pipeline runnning in the manifest repository to build
the images.
▶ [manifest Pipeline][8]
[8]: docs/manifest-pipeline.md
build-common.yml 0 → 100644
View file @ a59a93a0
---
workflow:
rules:
# This rule is needed, as otherwise the workflow:rules from
# the parent job seem to be used and prevent the pipeline generation
- if: $CI_PIPELINE_SOURCE == "parent_pipeline"
# --------------------------------------------------------------------------------------
# Scripts
# --------------------------------------------------------------------------------------
.docker_check:
# Check if the build folder is empty. Sometimes the docker volume for the build is not
# removed afterwards (e.g. in case of a timeout), then a follow-up build might fail.
- |-
if [ "$(ls -A)" ]; then
echo "ERROR: Build folder is not empty. This might be the case because the" \
"docker volume has not been removed in a previous job. Please check" \
"the docker container \"$CI_JOB_IMAGE\" on the GitLab runner" \
"\"$CI_RUNNER_SHORT_TOKEN\" ($CI_RUNNER_DESCRIPTION) and remove any" \
"attached volumes from it."
exit 1
fi
.setup_ssh:
# Setup ssh key to access private repos
# https://docs.gitlab.com/ee/ci/ssh_keys/#ssh-keys-when-using-the-docker-executor
# An SSH keypair has been generated for the manifest's pipeline to be able to access
# all private repositories in the manifest. The private key has been stored in the
# CI/CD variable GITLAB_PRIVATE_KEY of the manifest repository. The public key has
# been added as a deploy key in GitLab's repository settings for all contained
# repositories.
- echo "${LOGPREFIX} Setup SSH"
- eval $(ssh-agent -s)
- echo "$GITLAB_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
# Add GitLab server to known hosts
# https://docs.gitlab.com/ee/ci/ssh_keys/#verifying-the-ssh-host-keys
# In order to access the private repositories via SSH, the GitLab server has to be
# added to the known_hosts file. The host keys were determined using the command
# ssh-keyscan [-p port] <gitlab-server>
# and have been stored in the GITLAB_KNOWN_HOSTS CI/CD variable.
- echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
.repo_checkout:
- echo "${LOGPREFIX} Perform repo checkout"
- cd ${CI_PROJECT_DIR}
- repo init --submodules -u ${CI_REPOSITORY_URL}
-b refs/pipelines/${CI_PIPELINE_ID}
- repo sync --detach --current-branch --force-remove-dirty
--optimized-fetch --force-sync
# --------------------------------------------------------------------------------------
# Stage: Infrastructure
# --------------------------------------------------------------------------------------
.changelog:
extends: .infrastructure
script:
- PROJECT_ARGS=""
- for project in ${CHANGELOG_PROJECTS}; do
PROJECT_ARGS="${PROJECT_ARGS} --project=$project";
done
- .gitlab-ci/scripts/changelog_generator.py
--token=${GITBOT_TOKEN}
--branch=${MASTER_BRANCH}
--project=${CI_PROJECT_PATH}
${PROJECT_ARGS}
> changelog.md
artifacts:
expire_in: 4 weeks
paths:
- "changelog.md"
# --------------------------------------------------------------------------------------
# Stage: build
# --------------------------------------------------------------------------------------
.buildbase:
stage: Build
tags:
- builds
timeout: 8h
interruptible: true
image:
name: "${CI_IMAGE_YOCTO}"
# Override entrypoint so we can pass --id to set the UID and GID for the
# user that is created in the container. This is a feature of the
# crops/poky images. See poky-entry.py for details.
entrypoint:
- "/usr/bin/distro-entry.sh"
- "/usr/bin/dumb-init"
- "--"
- "/usr/bin/poky-entry.py"
- "--id=118:998"
artifacts:
expire_in: 1 week
reports:
dotenv: build.env
.build_image:
artifacts:
paths: &build_image_artifacts_paths
- build-*/tmp/deploy/images/**/*
- build-*/tmp/deploy/licenses/**/license.manifest
- build.env
cache:
# Cache the build artifacts for subsequent stages in the same pipeline
- key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
policy: push
paths: *build_image_artifacts_paths
.build_sdk:
artifacts:
paths: &build_sdk_artifacts_paths
- build-*/tmp/deploy/sdk/*
- build.env
cache:
# Cache the build artifacts for subsequent stages in the same pipeline
- key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
policy: push
paths: *build_sdk_artifacts_paths
# --------------------------------------------------------------------------------------
# Stage: test
# --------------------------------------------------------------------------------------
.test:
extends:
- .infrastructure
timeout: 1h
rules:
- when: manual
allow_failure: true
dependencies: []
variables:
# TODO checkout only gitlab-ci repo to allow running on deleted branches
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
CI_PARAM_TEST_SUITE: '{platform}.jinja2'
CI_PARAM_EXTRA: --nop
GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
TEST_REPO_BRANCH: ${MASTER_BRANCH}
artifacts:
when: always
paths:
- "results/**"
reports:
junit: results/results-*.xml
before_script:
- !reference [.setup_ssh]
script:
- .gitlab-ci/scripts/download_file_from_latest_job.py
--project $CI_PROJECT_ID
--pipeline $CI_PIPELINE_ID
--token=$GITBOT_TOKEN
--name $CI_PARAM_BUILDJOB
--path build.env
- cat build.env
- source build.env
# Submit tests to lava server
- .gitlab-ci/scripts/submit_test.py
--fng-install "$FNG_INSTALL_URL"
--name
"Gitlab $BUILD_MACHINE $BUILD_DISTRO $BUILD_IMAGE ($CI_PIPELINE_ID)"
--results-path "results"
--report-name "testresults-${CI_JOB_NAME}.md"
--test-repo $TESTS_GIT_URL
--test-repo-branch $TEST_REPO_BRANCH
--test-plan $CI_PARAM_TEST_SUITE
$CI_PARAM_EXTRA
$CI_PARAM_PLATFORMS
# --------------------------------------------------------------------------------------
# Stage: package
# --------------------------------------------------------------------------------------
.package:
extends:
- .infrastructure
stage: Package
tags:
# Using misc runner because it runs on the same machine as the build runner. This
# is the only way of being able to use the cache between build and package jobs.
- misc
artifacts:
paths:
- release/**/**/*
reports:
dotenv: package.env
timeout: 60m
before_script:
# We do this manually since we do not use GitLab's default artifact downloader
- source build.env
.pull_build_cache: &pull_build_cache
key: ${CI_PIPELINE_ID}-${ASSOCIATED_BUILD_JOB}
policy: pull
.push_package_cache: &push_package_cache
key: ${CI_PIPELINE_ID}-${CI_JOB_NAME}
policy: push
paths:
- release
- package.env
.package_release:
extends:
- .package
script:
# BUILD_PATH_* variables are available via build.env from build job
- .gitlab-ci/scripts/package_release.py
--images-dir="${BUILD_PATH_IMAGE}"
--licenses-dir="${BUILD_PATH_LICENSE}"
--doc-dir=.
--output-dir=release
--release-suffix="${RELEASE_SUFFIX}"
cache:
- <<: *pull_build_cache
paths: *build_image_artifacts_paths
- *push_package_cache
.package_sdk:
extends:
- .package
script:
# BUILD_PATH_* variables are available via build.env from build job
- .gitlab-ci/scripts/package_release.py
--sdk-dir="${BUILD_PATH_SDK}"
--output-dir=release
--release-suffix="${RELEASE_SUFFIX}"
cache:
- <<: *pull_build_cache
paths: *build_sdk_artifacts_paths
- *push_package_cache
# --------------------------------------------------------------------------------------
# Stage: deploy
# --------------------------------------------------------------------------------------
.deploy:
tags:
- deploy
rules:
- when: manual
allow_failure: true
image: instrumentisto/rsync-ssh:alpine3.16-r4
before_script:
# We do this manually since we do not use GitLab's default artifact downloader
- source package.env
# Save dotenv data for next stage
- echo "RELEASE_NAME=${RELEASE_NAME}" > deploy.env
- echo "VERSION=${VERSION}" >> deploy.env
- echo "MACHINE=${MACHINE}" >> deploy.env
script:
# Expand eventual nested variables contained within the DEPLOY_* variables
- DEPLOY_SOURCE=$(eval echo "${DEPLOY_SOURCE}")
- DEPLOY_TARGET=$(eval echo "${DEPLOY_TARGET}")
- DEPLOY_TARGET_LINK=$(eval echo "${DEPLOY_TARGET_LINK}")
- printf "Syncing %s to %s\n" "${DEPLOY_SOURCE}" "${DEPLOY_TARGET}"
# Capture rsync output, prefix it with the target link and save it to files.txt
- rsync -rh --out-format="%n" --mkpath ${DEPLOY_SOURCE}/ ${DEPLOY_TARGET} |
sed -e "s#^#${DEPLOY_TARGET_LINK}/#" | tee files.txt
- printf "Successfully deployed to:\n%s\n" "${DEPLOY_TARGET_LINK}"
timeout: 60m
cache:
# Get the packaged artifacts from the cache
- key: ${CI_PIPELINE_ID}-${ASSOCIATED_PACKAGE_JOB}
policy: pull
paths:
- release
artifacts:
paths:
- files.txt
- "**/md5sums.txt"
reports:
dotenv: deploy.env
# --------------------------------------------------------------------------------------
# Stage: Alphaplan
# --------------------------------------------------------------------------------------
.generate_alphaplan_data:
extends:
- .infrastructure
stage: Alphaplan
script:
# MACHINE and RELEASE_NAME are available from deploy.env
- .gitlab-ci/scripts/generate_alphaplan_fwr_file.py
--machine="${MACHINE}"
--release-name="${RELEASE_NAME}"
--files-list=files.txt
--md5sums=**/md5sums.txt
artifacts:
paths:
- alphaplan-import*.json
.import_alphaplan_data:
extends:
- .infrastructure
stage: Alphaplan
rules:
- when: manual
allow_failure: true
variables:
# Most AP_WEBSERVICE_* variables are set in GitLab CI variables. We're defining the
# URL here, though, so that it can be overridden in a job's variables block.
AP_WEBSERVICE_URL: >
https://SRV06.hamburg.garz-fricke.de/Alphaplan-API/Artikel/CreateFirmware
script:
- echo "${AP_WEBSERVICE_CERT}" > GarzFrickeGmbH-CA.cer
- .gitlab-ci/scripts/alphaplan_fwr_import.py
--url=${AP_WEBSERVICE_URL}
--user=${AP_WEBSERVICE_USR}
--password=${AP_WEBSERVICE_PW}
--cert-file=GarzFrickeGmbH-CA.cer
--file=alphaplan-import*.json
build-jobs-ci-test.yml.jinja2 0 → 100644
View file @ a59a93a0
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
# As the trigger job is not executed in a environment with checked out repository, we
# need to get the includes directly from gitlab
include:
- project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
ref: {{ GITLAB_CI_REVISION }}
file:
- build-common.yml
- common.yml
stages:
- Infrastructure
- Build
- Test
- Package
- Deploy SoftwareStore
- Deploy FTP
- Alphaplan
# --------------------------------------------------------------------------------------
# Stage: Infrastructure
# --------------------------------------------------------------------------------------
changelog:
extends: .changelog
# --------------------------------------------------------------------------------------
# Stage: Build
# --------------------------------------------------------------------------------------
.build: &build
- cd ${CI_PROJECT_DIR}
- VERSION=$(cd .repo/manifests && git describe --tags)
- cat .repo/manifests/default.xml
- find foo -name "[0-9]*.txt" -printf '%P\n' | sort -V > files-foo-$VERSION.txt
- cat files-foo-$VERSION.txt
- find bar -name "[0-9]*.txt" -printf '%P\n' | sort -V > files-bar-$VERSION.txt
- cat files-bar-$VERSION.txt
- FOO_FILES=$(cat files-foo-$VERSION.txt | wc -l)
- BAR_FILES=$(cat files-bar-$VERSION.txt | wc -l)
- DIFF=$((BAR_FILES-FOO_FILES))
- (($DIFF >= -1 && $DIFF <= 1))
build:files:
stage: Build
extends:
- .buildbase
tags:
- infrastructure
needs: []
timeout: 2m
variables:
GIT_STRATEGY: none
LOGPREFIX: "CI:build:"
before_script:
- !reference [.docker_check]
- !reference [.setup_ssh]
- !reference [.repo_checkout]
script:
- *build
artifacts:
paths:
- files-*
build:echo:
stage: Build
tags:
- infrastructure
needs: []
timeout: 2m
image: ${CI_IMAGE_PYTHON}
script:
- printenv
- echo "Build successful"
build:check-foo-branch:
stage: Build
extends:
- .buildbase
tags:
- infrastructure
needs: []
timeout: 2m
variables:
GIT_STRATEGY: none
before_script:
- !reference [.docker_check]
- !reference [.setup_ssh]
- !reference [.repo_checkout]
script: |
echo "repo branch: $MASTER_BRANCH"
echo "foo branch: $(cat foo/branch.txt)"
if [[ "$MASTER_BRANCH" != $(cat foo/branch.txt) ]]; then
echo "ERROR: Branches do not match!"
exit 1
fi
.simulate_build:
extends:
- .buildbase
tags:
- misc
needs: []
timeout: 60m
variables:
BUILD_ARTIFACTS:
script:
- if compgen -G build-*; then echo "Using build from cache"; exit 0; fi
- echo "Getting Yocto build artifacts"
- wget -O artifacts.zip ${BUILD_ARTIFACTS}
- unzip artifacts.zip
simulate-build-seco-mx6:
extends:
- .simulate_build
- .build_image
variables:
# We have to specify a tag here instead of getting the artifacts from a master
# branch, because we don't always execute the full pipeline on the master branches.
# In those cases the actual build job does not exist, which results in a 404 error.
BUILD_ARTIFACTS: https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/3.0/download?job=build-seco-mx6
cache:
- !reference [.build_image, cache]
# Additionally cache the build artifacts for re-runs of this job in other pipelines
- key: ${CI_JOB_NAME}-${BUILD_ARTIFACTS}
paths: !reference [.build_image, artifacts, paths]
simulate-buildsdk-seco-mx6:
extends:
- .simulate_build
- .build_sdk
rules:
- when: manual
allow_failure: true
variables:
BUILD_ARTIFACTS: https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/3.0/download?job=buildsdk-seco-mx6
cache:
- !reference [.build_sdk, cache]
# Additionally cache the build artifacts for re-runs of this job in other pipelines
- key: ${CI_JOB_NAME}-${BUILD_ARTIFACTS}
paths: !reference [.build_sdk, artifacts, paths]
# --------------------------------------------------------------------------------------
# Stage: Test
# --------------------------------------------------------------------------------------
.test_simple:
stage: Test
rules:
- when: manual
allow_failure: true
tags:
- infrastructure
image: ${CI_IMAGE_PYTHON}
script:
- exit ${RETURNCODE}
test:pass:
extends:
- .test_simple
variables:
RETURNCODE: 0
test:fail:
extends:
- .test_simple
variables:
RETURNCODE: 1
smoketest:seco-mx6:
extends:
- .test
stage: Test
needs: [simulate-build-seco-mx6]
variables:
TEST_REPO_BRANCH: dunfell
CI_PARAM_PLATFORMS: imx6guf
CI_PARAM_EXTRA: --all-devices
CI_PARAM_TEST_SUITE: boot.jinja2
CI_PARAM_BUILDJOB: simulate-build-seco-mx6
# --------------------------------------------------------------------------------------
# Stage: Package
# --------------------------------------------------------------------------------------
package-seco-mx6:
extends: .package_release
variables:
ASSOCIATED_BUILD_JOB: simulate-build-seco-mx6
needs:
- job: simulate-build-seco-mx6
artifacts: false
package-sdk-seco-mx6:
extends: .package_sdk
variables:
ASSOCIATED_BUILD_JOB: simulate-buildsdk-seco-mx6
needs:
- job: simulate-buildsdk-seco-mx6
artifacts: false
# --------------------------------------------------------------------------------------
# Stage: Deploy SoftwareStore
# --------------------------------------------------------------------------------------
.deploy_software_store:
extends: .deploy
stage: Deploy SoftwareStore
variables:
# We can't use the RELEASE_NAME variable from package.env in the variables secion,
# because we're loading the file from the cache. Usually it is downloaded via
# GitLab's standard artifact mechanism, which adds all dotenv variables to the
# pipeline scope. This doesn't work when using the cache, so we have to prevent
# GitLab from expanding these variables early by escaping the $ signs for those
# variables that are not available yet at this point. These will get evaluated
# within the script block of the .deploy class.
DEPLOY_SOURCE: release/$${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-yocto/Test/$${RELEASE_NAME}
DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Linux-Yocto/Test/$${RELEASE_NAME}
deploy-files:
extends: .deploy_software_store
variables:
GIT_STRATEGY: none
DEPLOY_SOURCE: .
DEPLOY_TARGET: /artifacts-yocto/Test/${CI_JOB_ID}
DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Linux-Yocto/Test/${CI_JOB_ID}
needs:
- build:files
# Disable reading package.env and create empty deploy.env (not needed for this job)
before_script:
- touch deploy.env
deploy-seco-mx6:
extends: .deploy_software_store
variables:
ASSOCIATED_PACKAGE_JOB: package-seco-mx6
needs:
- job: package-seco-mx6
artifacts: false
deploy-sdk-seco-mx6:
extends: .deploy_software_store
variables:
ASSOCIATED_PACKAGE_JOB: package-sdk-seco-mx6
needs:
- job: package-sdk-seco-mx6
artifacts: false
# --------------------------------------------------------------------------------------
# Stage: Deploy FTP
# --------------------------------------------------------------------------------------
.deploy_ftp:
extends: .deploy
stage: Deploy FTP
tags:
- ftp
variables:
# env variables are available via package.env from package job
DEPLOY_SOURCE: release/${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-ftp-yocto/Test/${RELEASE_NAME}
DEPLOY_TARGET_LINK: http://support.garz-fricke.com/projects/Linux-Yocto/Test/${RELEASE_NAME}
ftp-files:
extends: .deploy_ftp
variables:
GIT_STRATEGY: none
DEPLOY_SOURCE: .
DEPLOY_TARGET: /artifacts-ftp-yocto/Test/${CI_JOB_ID}
DEPLOY_TARGET_LINK: http://support.garz-fricke.com/projects/Linux-Yocto/Test/${CI_JOB_ID}
needs:
- build:files
ftp-seco-mx6:
extends: .deploy_ftp
variables:
ASSOCIATED_PACKAGE_JOB: package-seco-mx6
needs:
- job: package-seco-mx6
artifacts: false
ftp-sdk-seco-mx6:
extends: .deploy_ftp
variables:
ASSOCIATED_PACKAGE_JOB: package-sdk-seco-mx6
needs:
- job: package-sdk-seco-mx6
artifacts: false
# --------------------------------------------------------------------------------------
# Stage: Alphaplan
# --------------------------------------------------------------------------------------
generate-alphaplan-data-seco-mx6:
extends: .generate_alphaplan_data
needs:
- deploy-seco-mx6
import-alphaplan-data-seco-mx6:
extends: .import_alphaplan_data
variables:
AP_WEBSERVICE_URL: https://SRV06.hamburg.garz-fricke.de/Test/Alphaplan-API/Artikel/CreateFirmware
needs:
- generate-alphaplan-data-seco-mx6
build-jobs-yocto.yml.jinja2 0 → 100644
View file @ a59a93a0
---
# --------------------------------------------------------------------------------------
# Global
# --------------------------------------------------------------------------------------
# As the trigger job is not executed in a environment with checked out repository, we
# need to get the includes directly from gitlab
include:
- project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci'
ref: {{ GITLAB_CI_REVISION }}
file:
- build-common.yml
- build-yocto.yml
- common.yml
stages:
- Infrastructure
- Build
- Test
- Package
- Deploy SoftwareStore
- Deploy SoftwareStore Internal
- Deploy FTP
- Alphaplan
variables:
MASTER_BRANCH: {{ MASTER_BRANCH }}
{% if CI_PARAM_DISTRO is not defined %}
{% set CI_PARAM_DISTRO = "guf-wayland" %}
{% endif %}
{% if CI_PARAM_DISTRO_FNG is not defined %}
{% set CI_PARAM_DISTRO_FNG = "guf-fngsystem" %}
{% endif %}
# --------------------------------------------------------------------------------------
# Stage: Infrastructure
# --------------------------------------------------------------------------------------
changelog:
extends: .changelog
# --------------------------------------------------------------------------------------
# Generated jobs
# --------------------------------------------------------------------------------------
{% if CI_PARAM_MACHINES %}
{% for machine in CI_PARAM_MACHINES.split(' ') %}
# --------------------------------------------------------------------------------------
# Stage: Build
# --------------------------------------------------------------------------------------
{% if CI_PARAM_IMAGE %}
build-{{ machine }}:
extends: .build_yocto_image
variables:
CI_PARAM_MACHINE: {{ machine }}
CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
INSTALLSCRIPT: "fng-install.sh"
# Build jobs for the sdk
buildsdk-{{ machine }}:
extends: .build_yocto_sdk
variables:
CI_PARAM_MACHINE: {{ machine }}
CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
{% endif %}
{% if CI_PARAM_IMAGE_FNG %}
# Build jobs for the fng system image
build-{{ machine }}-fngsystem:
extends: .build_yocto_image
variables:
CI_PARAM_MACHINE: {{ machine }}
CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }}
CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }}
INSTALLSCRIPT: "fngsystem-self-update.sh"
{% endif %}
# --------------------------------------------------------------------------------------
# Stage: Test
# --------------------------------------------------------------------------------------
# Run platform tests for this machine which the yocto image
# This is a little hacky as we need to match the machine name to
# the available platforms
{% if CI_PARAM_IMAGE %}
{% if machine == 'seco-mx6' or machine == 'imx6guf' %}
{% set platforms = "santaro santoka santino santino-lt" %}
{% set lavamachine = "imx6guf" %}
{% elif machine == 'seco-mx6ull' or machine == 'imx6ullguf' %}
{% set platforms = "nallino" %}
{% set lavamachine = "imx6ullguf" %}
{% elif machine == 'seco-mx8mm' or machine == 'imx8mguf' %}
{% set platforms = "tanaro" %}
{% set lavamachine = "imx8mguf" %}
{% else %}
{% set platforms = '' %}
{% set lavamachine = '' %}
{% endif %}
{% if platforms %}
# Run smoketests for this machine which the yocto image
smoketest:{{ machine }}:
extends:
- .test
stage: Test
needs:
- job: build-{{ machine }}
variables:
CI_PARAM_BUILDJOB: build-{{ machine }}
CI_PARAM_MACHINE: {{ lavamachine }}
CI_PARAM_PLATFORMS: {{ lavamachine }}
CI_PARAM_TEST_SUITE: boot.jinja2
CI_PARAM_EXTRA: --all-devices
platformtest:{{ machine }}:
extends:
- .test
stage: Test
needs:
- job: build-{{ machine }}
variables:
CI_PARAM_BUILDJOB: build-{{ machine }}
CI_PARAM_MACHINE: {{ lavamachine }}
CI_PARAM_PLATFORMS: {{ platforms }}
{% endif %}
{% endif %}
# --------------------------------------------------------------------------------------
# Stage: Package
# --------------------------------------------------------------------------------------
{% if CI_PARAM_IMAGE %}
package-{{ machine }}:
extends: .package_release
variables:
ASSOCIATED_BUILD_JOB: build-{{ machine }}
needs:
- job: build-{{ machine }}
artifacts: false
- job: changelog
packagesdk-{{ machine }}:
extends: .package_sdk
variables:
ASSOCIATED_BUILD_JOB: buildsdk-{{ machine }}
needs:
- job: buildsdk-{{ machine }}
artifacts: false
{% endif %}
{% if CI_PARAM_IMAGE_FNG %}
package-{{ machine }}-fngsystem:
extends: .package_release
variables:
ASSOCIATED_BUILD_JOB: build-{{ machine }}-fngsystem
needs:
- job: build-{{ machine }}-fngsystem
artifacts: false
{% endif %}
# --------------------------------------------------------------------------------------
# Stage: Deploy SoftwareStore
# --------------------------------------------------------------------------------------
{% if CI_PARAM_IMAGE %}
{% if CI_COMMIT_TAG is defined %}
{% set deploy_class = ".deploy_software_store_yocto" %}
{% else %}
{% set deploy_class = ".deploy_software_store_yocto_internal" %}
{% endif %}
deploy-{{ machine }}:
extends: {{ deploy_class }}
variables:
ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
needs:
- job: package-{{ machine }}
artifacts: false
- job: changelog
deploysdk-{{ machine }}:
extends: {{ deploy_class }}
variables:
ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
needs:
- job: packagesdk-{{ machine }}
artifacts: false
- job: changelog
{% endif %}
{% if CI_PARAM_IMAGE_FNG %}
{% if CI_COMMIT_TAG is defined %}
{% set deploy_class = ".deploy_software_store_fngsystem" %}
{% else %}
{% set deploy_class = ".deploy_software_store_fngsystem_internal" %}
{% endif %}
deploy-{{ machine }}-fngsystem:
extends: {{ deploy_class }}
variables:
ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
needs:
- job: package-{{ machine }}-fngsystem
artifacts: false
- job: changelog
{% endif %}
# --------------------------------------------------------------------------------------
# Stage: Alphaplan
# --------------------------------------------------------------------------------------
{% if CI_PARAM_IMAGE %}
generate-alphaplan-data-{{ machine }}:
extends: .generate_alphaplan_data
needs:
- deploy-{{ machine }}
import-alphaplan-data-{{ machine }}:
extends: .import_alphaplan_data
needs:
- generate-alphaplan-data-{{ machine }}
{% endif %}
{% if CI_PARAM_IMAGE_FNG %}
generate-alphaplan-data-{{ machine }}-fngsystem:
extends: .generate_alphaplan_data
needs:
- deploy-{{ machine }}-fngsystem
import-alphaplan-data-{{ machine }}-fngsystem:
extends: .import_alphaplan_data
needs:
- generate-alphaplan-data-{{ machine }}-fngsystem
{% endif %}
# --------------------------------------------------------------------------------------
# Stage: Deploy FTP
# --------------------------------------------------------------------------------------
# FIXME: This is a workaround to prevent customer releases being uploaded to our
# public FTP area. It should be removed as soon as we support uploading to different
# FTP target folders.
{% if HIDE_FTP_UPLOAD_STAGE is not defined or not HIDE_FTP_UPLOAD_STAGE %}
{% if CI_COMMIT_TAG is defined %}
{% if CI_PARAM_IMAGE %}
ftp-{{ machine }}:
extends: .deploy_ftp_yocto
variables:
ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
needs:
- job: package-{{ machine }}
artifacts: false
- job: changelog
ftpsdk-{{ machine }}:
extends: .deploy_ftp_yocto
variables:
ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
needs:
- job: packagesdk-{{ machine }}
artifacts: false
- job: changelog
{% endif %}
{% if CI_PARAM_IMAGE_FNG %}
ftp-{{ machine }}-fngsystem:
extends: .deploy_ftp_fngsystem
variables:
ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
needs:
- job: package-{{ machine }}-fngsystem
artifacts: false
- job: changelog
{% endif %}
{% endif %}
{% endif %}
# --------------------------------------------------------------------------------------
# End of generated jobs
# --------------------------------------------------------------------------------------
{% endfor %}
{% endif %} # if CI_PARAM_MACHINES
build-yocto.yml 0 → 100644
View file @ a59a93a0
---
# --------------------------------------------------------------------------------------
# Stage: Build
# --------------------------------------------------------------------------------------
.collect_srcrevs: &collect_srcrevs
# write all package AUTOREVS to file
- |-
SRCREVS_FILE="${CI_PROJECT_DIR}/${BUILDPATH}/${IMAGEPATH}/BUILD_SRCREVS.log"
if [ -d "$( dirname "${SRCREVS_FILE}" )" ];then
buildhistory-collect-srcrevs > ${SRCREVS_FILE}
echo "${LOGPREFIX} buildhistory-collect-srcrevs:"
cat ${SRCREVS_FILE}
fi
.dump_install_command: &dump_install_command
# print install instructions
- |-
SCRIPT="${CI_PROJECT_DIR}/${BUILDPATH}/${IMAGEPATH}/${INSTALLSCRIPT}"
if [ ! -f "${SCRIPT}" ]; then
echo "Install script missing, searched for '$SCRIPT'"
exit 1
fi
if [ "$CI_PROJECT_VISIBILITY" = "public" ];then
cat <<-EOF
==============================
Install the image:
FNG="$FNG_INSTALL_URL"
curl --location "\$FNG" | sh -s -- --url="\$(dirname "\$FNG")"
==============================
EOF
else
cat <<-EOF
==============================
Install the image:
export GITLAB_TOKEN=<your_access_token>
FNG="$FNG_INSTALL_URL"
curl --location --header "PRIVATE-TOKEN: \$GITLAB_TOKEN" "\$FNG" \
| sh -s -- --url="\$(dirname "\$FNG")"
==============================
EOF
fi
.build_script: &build_script
# setup build environment
- echo "${LOGPREFIX} Build configuration MACHINE=${CI_PARAM_MACHINE}
DISTRO=${CI_PARAM_DISTRO} IMAGE=${CI_PARAM_IMAGE}"
- source build.env
- echo "${LOGPREFIX} Using build dir ${BUILDPATH}"
- export MACHINE="${CI_PARAM_MACHINE}"
- export DISTRO="${CI_PARAM_DISTRO}"
- export EULA="1"
- source ./"${SETUPSCRIPT}" "${BUILDPATH}"
# start build
- echo -e "section_start:`date +%s`:bitbake_run\r\e[0KBitbake Log"
- echo "${LOGPREFIX} bitbake ${CI_PARAM_IMAGE} -c ${BITBAKE_TASK}"
- bitbake "${CI_PARAM_IMAGE}" -c "${BITBAKE_TASK}"
- echo -e "section_end:`date +%s`:bitbake_run\r\e[0K"
.save_build_env: &save_build_env
- echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env
- echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env
- echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env
- echo "BUILD_PATH_IMAGE=${BUILDPATH}/${IMAGEPATH}" >> build.env
- echo "BUILD_PATH_SDK=${BUILDPATH}/${SDKPATH}" >> build.env
- echo "BUILD_PATH_LICENSE=${BUILDPATH}/${LICENSESPATH}" >> build.env
- echo "FNG_INSTALL_URL=${JOB_URL}${FNG_INSTALL_PATH}" >> build.env
.build_yocto:
extends:
- .buildbase
needs: []
variables:
GIT_STRATEGY: none
SETUPSCRIPT: "setup-environment"
LOGPREFIX: "CI:build:"
BUILDPATH: "build-${CI_PARAM_DISTRO}-${CI_PARAM_MACHINE}"
IMAGEBASEPATH: "tmp/deploy/images/"
IMAGEPATH: "${IMAGEBASEPATH}/${CI_PARAM_MACHINE}"
LICENSESPATH: "tmp/deploy/licenses"
SDKPATH: "tmp/deploy/sdk/"
JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/"
FNG_INSTALL_PATH: "${BUILDPATH}/${IMAGEPATH}/${INSTALLSCRIPT}"
before_script:
- !reference [.docker_check]
- !reference [.setup_ssh]
- !reference [.repo_checkout]
.build_yocto_image:
extends:
- .build_yocto
- .build_image
variables:
BITBAKE_TASK: "build"
script:
- *save_build_env
- *build_script
- *collect_srcrevs
- *dump_install_command
.build_yocto_sdk:
extends:
- .build_yocto
- .build_sdk
variables:
BITBAKE_TASK: "populate_sdk"
rules:
- when: manual
allow_failure: true
script:
- *save_build_env
- *build_script
# --------------------------------------------------------------------------------------
# Stage: Deploy SoftwareStore
# --------------------------------------------------------------------------------------
.deploy_software_store_yocto:
extends: .deploy
stage: Deploy SoftwareStore
variables:
DEPLOY_SOURCE: release/$${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-yocto/Releases/$${RELEASE_NAME}
DEPLOY_TARGET_LINK: >
Z:/Development/SoftwareStore/Linux-Yocto/Releases/$${RELEASE_NAME}
.deploy_software_store_yocto_internal:
extends: .deploy
stage: Deploy SoftwareStore Internal
variables:
DEPLOY_SOURCE: release/$${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-yocto/Interne_Releases/$${RELEASE_NAME}
DEPLOY_TARGET_LINK: >
Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/$${RELEASE_NAME}
.deploy_software_store_fngsystem:
extends: .deploy
stage: Deploy SoftwareStore
variables:
DEPLOY_SOURCE: release/$${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-fngsystem/$${RELEASE_NAME}
DEPLOY_TARGET_LINK: >
Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
.deploy_software_store_fngsystem_internal:
extends: .deploy
stage: Deploy SoftwareStore Internal
variables:
DEPLOY_SOURCE: release/$${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-fngsystem/CI_Builds/$${RELEASE_NAME}
DEPLOY_TARGET_LINK: >
Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/$${RELEASE_NAME}
# --------------------------------------------------------------------------------------
# Stage: Deploy FTP
# --------------------------------------------------------------------------------------
.deploy_ftp_yocto:
extends: .deploy
stage: Deploy FTP
tags:
- ftp
variables:
DEPLOY_SOURCE: release/$${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-ftp-yocto/Releases/$${RELEASE_NAME}
DEPLOY_TARGET_LINK: >
http://support.garz-fricke.com/projects/Linux-Yocto/Releases/$${RELEASE_NAME}
.deploy_ftp_fngsystem:
extends: .deploy
stage: Deploy FTP
tags:
- ftp
variables:
DEPLOY_SOURCE: release/$${RELEASE_NAME}
DEPLOY_TARGET: /artifacts-ftp-fngsystem/$${RELEASE_NAME}
DEPLOY_TARGET_LINK: >
http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
check_if_integration_branch_is_up_to_date.py deleted 100755 → 0
View file @ 610e4a46
#!/usr/bin/env python3
import common
import argparse
import sys
import tempfile
from furl import furl
from git import GitCommandError, Repo
from gitlab import Gitlab, GitlabGetError
def check_if_integration_branch_is_up_to_date(
manifest_project,
integration_base,
project,
merge_request,
):
gitlab = manifest_project.manager.gitlab
integration_branch = common.integration_branch_name(
project.name, merge_request.source_branch
)
with tempfile.TemporaryDirectory() as manifest_dir:
# Construct clone url containing access token
clone_url = furl(manifest_project.http_url_to_repo)
clone_url.username = "gitlab-ci"
clone_url.password = gitlab.private_token
# Checkout manifest
try:
manifest_repo = Repo.clone_from(
clone_url.url, manifest_dir, branch=integration_branch
)
except GitCommandError as e:
sys.exit("ERROR: could not clone manifest repository\n" + str(e))
# Get branches
try:
integration_branch = manifest_repo.heads[integration_branch]
except IndexError:
sys.exit("ERROR: branch '%s' not found" % integration_branch)
try:
integration_base = manifest_repo.remote().refs[integration_base]
except IndexError:
sys.exit("ERROR: branch '%s' not found" % integration_base)
# The integration branch is up to date if its parent is the integration base
up_to_date = integration_branch.commit.parents[0] == integration_base.commit
return up_to_date
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
required=True,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--manifest-project",
help="""name of the manifest project""",
dest="manifest_project",
required=True,
)
parser.add_argument(
"--integration-base",
help="""manifest branch to branch off from""",
dest="integration_base",
required=True,
)
parser.add_argument(
"--project",
help="""name of the project, as specified in the manifest""",
dest="project",
required=True,
)
parser.add_argument(
"--merge-request",
help="""project merge request IID containing the changes to be integrated""",
dest="merge_request",
required=True,
)
args, _ = parser.parse_known_args()
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
manifest_project = common.get_project(gitlab, args.manifest_project)
project = common.get_project(gitlab, args.project)
try:
merge_request = project.mergerequests.get(
args.merge_request, retry_transient_errors=True
)
except GitlabGetError as e:
sys.exit(
"ERROR: could not get %s!%s: %s"
% (project.name, args.merge_request, e.error_message)
)
if check_if_integration_branch_is_up_to_date(
manifest_project=manifest_project,
integration_base=args.integration_base,
project=project,
merge_request=merge_request,
):
print("Integration branch is up to date.")
else:
sys.exit(
"Integration branch is not up to date. Please re-run the MR pipeline:\n"
" 1. Open the MR pipelines page:\n"
" %s\n"
" 2. Click 'Run Pipeline'" % (merge_request.web_url + "/pipelines")
)
if __name__ == "__main__":
main()
common.yml 0 → 100644
View file @ a59a93a0
# --------------------------------------------------------------------------------------
# Common definitions that may be used in all subprojects
# --------------------------------------------------------------------------------------
---
variables:
# CI_IMAGES_BASEPATH: Environment variable configure in gitlab
CI_IMAGES_PATH: ${CI_IMAGES_BASEPATH}/ci-images
CI_IMAGES_REV: e6a404c3e1a919f54feb83f1bc061ce171220437
CI_IMAGE_PYTHON: "${CI_IMAGES_PATH}/python/3.9:${CI_IMAGES_REV}"
CI_IMAGE_YOCTO: "${CI_IMAGES_PATH}/yocto-build/ubuntu-20.04:${CI_IMAGES_REV}"
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
# Reduced depth as checkout of larger projects (like the kernel)
# may take too long
GIT_DEPTH: 1
# --------------------------------------------------------------------------------------
# Common infrastructure settings
# --------------------------------------------------------------------------------------
.infrastructure:
stage: Infrastructure
tags:
- infrastructure
timeout: 10m
image: "${CI_IMAGE_PYTHON}"
needs: []
variables:
# Include git submodules
GIT_SUBMODULE_STRATEGY: recursive
.yamllint:
extends: .infrastructure
rules:
# Only run this job in the parent pipeline in the manifest, not again in the child
# pipeline
- if: $CI_PIPELINE_SOURCE == "parent_pipeline"
when: never
# Usually run this job only on non-master branches, i.e. in merge requests
- if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH
# Additionally run this job if pipeline was triggered by the API, which happens e.g.
# when no pipeline has run in a merge request, so a full build pipeline is needed on
# the master
- if: $CI_PIPELINE_SOURCE == "api"
script:
- yamllint -c .gitlab-ci/.yamllint.yml .*.yml
deploy_gitlab_ci.py deleted 100755 → 0
View file @ 610e4a46
#!/usr/bin/env python3
import common
import argparse
import sys
from gitlab import Gitlab
from accept_merge_request import accept_merge_request
from create_merge_request import create_merge_request
from get_merge_requests import get_merge_requests
from update_submodule import update_submodule
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
required=True,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
help="""name of the GitLab project""",
dest="project",
required=True,
)
parser.add_argument(
"--submodule",
help="""submodule to update""",
dest="submodule",
required=True,
)
parser.add_argument(
"--revision",
help="""new revision for submodule""",
dest="revision",
required=True,
)
parser.add_argument(
"--branch",
help="""project branch (if not default branch)""",
dest="branch",
required=False,
default=None,
)
parser.add_argument(
"--merge",
help="""if set, perform merge after integration""",
dest="merge",
action="store_true",
required=False,
default=False,
)
args, _ = parser.parse_known_args()
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
project = common.get_project(gitlab, args.project)
# Update submodule
integration_branch, _, submodule_project = update_submodule(
project, args.submodule, args.revision, args.branch
)
# Get source merge request
mrs = get_merge_requests(
submodule_project,
target_branch="master",
commit=args.revision,
)
if not mrs:
sys.exit(
"ERROR: could not determine source merge request for commit %s"
% args.revision
)
source_mr = mrs[0]
# Create merge request
mr, created = create_merge_request(
project, integration_branch, project.default_branch
)
if created:
common.crosslink_merge_requests(source_mr, mr)
print("Created new merge request:\n%s" % mr.web_url)
else:
print("Existing integration merge request:\n%s" % mr.web_url)
if not args.merge:
print(
"Skipping automatic merge in MR context. If you like to extend the "
"integration MR by hand, please do it now. Afterwards you can either merge "
"it by hand or re-run this job on the master branch after the source MR "
"has been merged."
)
sys.exit(0)
# Wait until GitLab has checked merge status
common.wait_until_merge_status_is_set(project, mr)
# Attempt to merge
merged = accept_merge_request(project, mr, rebase=True)
if not merged:
sys.exit(
"Integration MR could not be merged. You have two possibilities to fix "
"this:\n"
" 1. Checkout the MR and rebase it on the current master manually, or\n"
" 2. Delete the MR (Edit -> Delete in the MR UI)\n"
"In either case restart this job afterwards in order to get it merged."
)
print("Successfully merged")
if __name__ == "__main__":
main()
docs/CI_diagram.drawio deleted 100644 → 0
View file @ 610e4a46
<mxfile host="Electron" modified="2021-05-26T13:35:43.217Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/14.6.13 Chrome/89.0.4389.128 Electron/12.0.7 Safari/537.36" etag="EMktwfk9Tznhq6M5jyJl" version="14.6.13" type="device"><diagram name="Page-1" id="6133507b-19e7-1e82-6fc7-422aa6c4b21f">7V1bc6M4Fv41rpp9cIq77cd2LjO7m6nq7Z7a7X1KYSxjpgF5AXfi+fWroxsCge3E16TVlUrQkZCEzneOviMkeuDeZi+/FuFq+Tueo3TgWPOXgXs3cBx74gXkD0g2XOLYXBIXyZzLasHX5C/EhRaXrpM5KhsFK4zTKlk1hRHOcxRVDVlYFPi5WWyB02arqzBGmuBrFKa69D/JvFoyqeO6QZ3xG0ripWg68D2WMwuj73GB1zlvcOC4C/qPZWehqIw/abkM5/hZEbn3A/e2wLhiV9nLLUphdMW4sfseenJlxwuUV/vccP998fBtMv93NP37P/81HM3y4T/w0OfV/AjTNR+RNNyggne52ohxog+KoCp74E6fl0mFvq7CCHKfCTSIbFllKc9e4Lx6CLMkBVT8htIfqEqiEDKSNL3FKS5opa7rPjy4LpGn4Qyln3GZVAnOSV5Enol0wp3+QAXcmj62ClQYmgzTJO4s/olnzHBV4Yz3iIPP9ki6JH1P8vgPqObOqgVTfsPd0PHEbXwQ4MEwlKrgqcZwEx870ih66dWKLXVNrAjhDFXFhhR5EUjj8OAGFPDkcw1GezTiwqUCRH/MhSG3gFjWXYOAXHAcvAYTloaJLMyTBSqr88AiCKbOw8P7hUWchmXJKzgCRAKrCRHb68CI5XRgxAuOgJHReBLYtj9ZWAGajNF4aI80fCRkoOMirJAGkPI5ydIwRzoS1AHcqTGm2GiZpPPHcIPX8IBlRTywSE2XuEj+ItWGog2SXQjlOkGjxFe4kyuoQCUp81low5aix7CseJkIp2m4KpOZ7HAWFnGSS1hQqPAnfWiCmc8JpFZpKrS0mA1Y/VkS8WuK+amcW0RNOaZjWFYF/i5nKrtlO7xQv50pcHd6YNgB1n7nNZrcTLwmOMcd4Jz4OjYliruwyVv8Qmb8MI/JqMsmXb/ZnOtozY07LMEJmo2FKcFZTgA7hUEuNXuQz/o2E3E8zUSiAlH7sHL0TPlMliUV1JFTsiD8q1WgFR44QQqYnpF5OIgr6UYUuyIaqpo2xaDRhoKODmFrKVpUvZbG/d4jLXOnuMYvfEy9HnePSX2LlCJ7mcznKKfIr8IqZMYDEF9h4i+oIvwp+SHqurVu/IFPnuuWpO06TX6geFHd4pw8X5hQqCJimc8wWmJcVBD3u6vdsN40wbILwp53Cu860aCjKT9N2g51L7+wQ/MZ0VeK2rPg0Nbg4OpwcDvU3J69C1a2pf6LaHi0p5Man0DBjj59Upt3P0E4lFRk2IZRorsAolBECwVhBpaWz0r4I6feJ3KFn4QnuVlt9CqKNQmiCtZSki8KMsrFOqrWhT5vG/9yQvSN90Nf4JyI4dv69DRbE2pl2Ns7Y28HxRW2G7QCi5+Ru3UaiKdPwsR3wtrUEp5ng6MKwxJQp9EY16m4zh0I3eKeroKedfcvMBTtLJo+I03r7t42qlbHbHsytRtJ7obUcdxsCDa2cjRarDQO5tywuzg/06efaIkI6TH87GfiZyPnvPws8JvtyVdX10fQHP0dBbcQK1lIniZiY5jciDctwjxagph0yVqvoBiwuHnXurXxsYf62Ml1kzhHf/NpSNwpNH1pEif0+lYSZ1HH8qT4kifmSXYusZnw8CJ4uzR7Ey9/FJ2jeYwEIyNjssQxzsP0vpa2+Etd5hFT7wAw+RNV1YYTjHBNpq4GiNBLUn1Trv8LVRFdsNTdC6+ZJjYikZPn/aYmlLsgWd9GU+K+XgrEHhyedjvnIYOD10WE9pjhCc+MUd/aq9ND5AuUEkv90exFLxn6VBThRinAUV7X/BkECjGbBG2iNFZRs/sG27K23uAcWL7VIXLBnvG4HCwwON+JcxW+W3B+KIR1yPl+c7XVGTk3lvKv5f2YPfJKTsDX9ZWUDJGR0RBkItoPHNHK6JFj0rPMCwduH+M++7B+/8IiWSzDWmW3SEZA17Fb03DMAzkm81dXHL2aXSLn0fSlo1dXX+h63W4Ri/oRsznkPbmXi4ewrvteqH0vd5Gcf2gR3uu1iL/n7KD+NPUZFQkZULAKZTP3UcJbtyM+6NkLePTYQIsW/dY0xTp1smDA1ZdexXbYFuj2OTxwBF6qvfeY6PZndxBFfwsrPcz+3t8Skm5m9o01Gb/FzM6xvCRCgd3x+fhAGzwMCPridXfcfBlL8fwOenReS9EXFqIlCQNRimNtkMziwgdeXLCt9jkps51RGIm+uhAjQvbZgRS2qNBvNYbtH8j23StfTHDNYsJ5NH3pxQRv22KCdABD7htwIRYJyMjnDTgE/1vDsXWYizBfMijiWfgLqR9MjPzuvPobXML4WuD8hws+LdBVCpzjktp3o0hJJwgoYK1e6nbrY3F+++MEPhkPkNKD9TIlxsenI0Qkd3ANHfNhRHwyrLvK2rKswMGbqnHqapgaZA7fLiBy2WjLXMldiISZIWTZNFkzmFrGLA/SnMeAsGl/kAk+GHJUNgNylc/QFmtGU7dRsxqQgR/zdWYDWRbNabCbuhaF4dRlVZZTF20wnbqwznbEsMlPMvg151HuY7xHbRW4T53u5D+QTd0bG4LaCyoKaXREFla4EMhrNiSVKMbSVmHCHNE+QJRZEoO1E/KBuMiSwbgGrbWp5a6tyJ/lY7HawVeJPO6v6kxPuZF4LpkRKz1odE2hVM2+uX5n1yjB6uma+jTtnlFcdnRMI166DdNkYxCbGdzCt1u822vx9EyJzxmYL8+V+HLrUI8voIysww+ovKwJvG481j6Czs/bPYQ6IYPcU6Vf5JAzscLXQEgZG8glZwMpZ23MNsVEXhuf2IbBntqf0l9Uz5SzkRQdjVsqt9ty+kub56EyOdND/pvMTOKw28wUzKrYe401Adfbx5g6wLoXJpXqW5jsBR1lgL2g29MD7oU4zgoVeP3BMAjMsAuKbjcU3V5oNXkiyAtxVyfsrgJLozd65vFpoaS03HZv9FMGPtA3X31r7VNaCSMGb598yuU4ZpAsT8/OgJSenoGL7vMzfZXx11myeeNRP4ZHHSlwfo0VBIpnO8gMqLAZXfByzTBEBE8kUmPxkwhZ9G97mChrH4yYKMsxUZaJsvZzYybKkjkmyvronMBEWW9AnImyTJRloizjUd93lHXfE2apxzf5e60n+V5rv5Ob5sMbZ38/eul9r57f8YK0IrNW3HHIwuwm+sC7iYLWHllzVEnaiP71K8VGLPEZjj/xzDjQoztQ5qCudyuRp2/HNFuJTqHpi28l0rcUvvJcknQaT1nxtEpWKQJgPIHfMOeTrtvNXJyniQMr7+h8xLE+PdD+LyvYQLz2TETvF+t3nodwL3oeQnzUwCj+qN9aGe+pfMb9Lqb8n/eDIydUvuDz1275I6P84yvf3dvyDz2K2qP8AV/lU0LUen3Pvf8/</diagram></mxfile>
\ No newline at end of file
docs/CI_diagram.png deleted 100644 → 0
View file @ 610e4a46
docs/CI_diagram.png

87.3 KiB

docs/automatic-manifest-integration.md 0 → 100644
View file @ a59a93a0
<!----------------------------------------------------------------------------->
# Automatic manifest integration
<!----------------------------------------------------------------------------->
**Contents:**
- [Defined source code state](#defined-source-code-state)
- [Integrating project changes](#integrating-project-changes)
- [Merging project changes](#merging-project-changes)
- [Adding a new project to SRCREV.conf](#adding-a-new-project-to-srcrevconf)
<!----------------------------------------------------------------------------->
## Defined source code state
<!----------------------------------------------------------------------------->
### Manifest
Our [Yocto manifest][1] contains a list of projects which are used to build our
Yocto distribution. A simplified version of the manifest file [`default.xml`][2]
(here containing only two projects) would look something like this:
```xml
<manifest>
<remote
name="seco-ne"
alias="origin"
fetch="ssh://git@git.seco.com:4444/seco-ne/yocto"
/>
<project
name="layers/meta-seconorth-machine"
revision="7aede82170ff7f92de0dcfeca89ed79849869214"
remote="seco-ne"
path="sources/meta-seconorth-machine"
/>
<project
name="layers/meta-seconorth-distro"
revision="c402d855fc8d47e3742ada3b0a3fda4b9649e414"
remote="seco-ne"
path="sources/meta-seconorth-distro"
/>
</manifest>
```
Each project is listed with a **name**, a local **path** where the [repo][3]
tool will clone the source code to, and a fixed git **revision**. We do not use
a branch name here in order to have a defined, taggable manifest state at all
times.
### External source repositories used by BitBake
Some bitbake recipes are set to `SRCREV = "${AUTOREV}"`. To enable reproducible
builds, the revision for these recipes is written to the `SRCREV.conf` file.
This file is also part of the manifest repo and looks something like this:
```ini
SRCREV_pn-libmdb = "1442894df13d9b290cfb5d97183eb9a96c8e4eba"
SRCREV_pn-egalaxi2c = "d2dfb014e8c2ed36801bd020894d8306f02ff146"
SRCREV_pn-xconfig = "0fc1ea45b55e729d551bb7d40dd25fbde02ee1b6"
```
Each project is listed with its bitbake **recipe name** after `SRCREV_PN-` and
the fixed git **revision**.
[1]: https://git.seco.com/seco-ne/yocto/manifest
[2]: https://git.seco.com/seco-ne/yocto/manifest/-/blob/dunfell/default.xml
[3]: https://gerrit.googlesource.com/git-repo
<!----------------------------------------------------------------------------->
## Integrating project changes
<!----------------------------------------------------------------------------->
Everytime a developer pushes a change to one of the projects contained in the
manifest or in the `SRCREV.conf` file, the project gets a new git revision. In
order to have this change integrated into the manifest, the according `revision`
attribute has to be updated. This process has been automated as part of our **CI
pipeline** and runs automatically as soon as a **merge request is created**:
![Project merge request pipeline](project-mr-pipeline.png)
The pipeline consists of three jobs:
1. **Integrate**
Create an integration branch on the manifest and update the project revision
on it.
2. **Build**
Trigger a build on the integration branch, which contains the change.
This uses GitLab's [multi-project pipeline][4] functionality, which adds a
separate *Downstream* pipeline running on the manifest repository (seen on
the right).
3. **Check**
Check if the integration branch is up to date with the current manifest
master (see ["Retrigger" job below][5] for why this is necessary).
See the following diagram for a visualization of this process:
![Create layer merge request](create-layer-mr.svg)
Browse existing project merge requests (e.g. [here][6]) for real world examples
of this pipeline.
The above workflow makes project changes **buildable** and **testable** in the
full manifest environment **before merging them**. Actually we even prevent the
project merge requests from being merged unless the pipeline has run
successfully.
#### Skipping the build
In the case of changes that are not build-relevant or have a rather low
probability of breaking the build (for example documentation), the developer has
the option to skip the build by adding the `skip build` label to a merge request
on its creation. The `build` and `check` stages of the pipeline are left out
then, and the merge request can be merged immediately after the `integrate`
stage has completed.
[4]: https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html
[5]: #merging-project-changes
[6]: https://git.seco.com/seco-ne/yocto/layers/meta-seconorth-distro/-/merge_requests/321
<!----------------------------------------------------------------------------->
## Merging project changes
<!----------------------------------------------------------------------------->
As soon as a pushed project change is ready to be merged (i.e. the pipeline has
successfully run on the merge request), a developer can merge it. This triggers
a **pipeline** with only a **single job**:
![Project master pipeline](project-master-pipeline.png)
1. **Merge**
Create a merge request on the manifest, which merges the project integration
branch into the master branch, and merge it.
This will however automatically trigger **another pipeline** on the **manifest
master branch**, since there is a new commit now:
![Manifest master pipeline](manifest-master-pipeline.png)
This pipeline consists of two jobs:
1. **Retrigger**
Retrigger the *"Check"* job on all open merge requests of all projects
contained in the manifest. This will invalidate the pipeline on these merge
requests. The manifest has a new commit now, so the already completed
pipelines on the open merge requests have run on a now outdated manifest
state.
2. **Build**
Check if a build has already run for this commit. This is usually true,
because every merge request triggers a complete build. If so, just mirror
the build result from there.
There are corner cases however in which no pipeline has run in the merge
request, e.g. when two project merge requests from different projects are
merged simultaneously. In this case, perform a complete build on the commit.
See the following diagram for a visualization of this process:
![Merge layer merge request](merge-layer-mr.svg)
Browse existing manifest merge requests (e.g. [here][7]) for real world examples
of this pipeline.
[7]: https://git.seco.com/seco-ne/yocto/manifest/-/merge_requests/545
<!----------------------------------------------------------------------------->
## Adding a new project to SRCREV.conf
<!----------------------------------------------------------------------------->
This workflow adds a gitlab pipeline to a project which automatically updates
the git revision in the `SRCREV.conf` file on project changes.
#### Source code
1. Add the gitlab-ci repo as submodule to the project by using the correct
relative path:\
`git submodule add ../../yocto/infrastructure/gitlab-ci .gitlab-ci`
2. Add an approriate `.gitlab-ci.yml` file. For example, copy it from the
repository [egalxi2c][8]. Modify the following variables in the file:
* `BB_RECIPE_NAME`: Set the name of the bitbake recipe
3. Create a corresponding entry in the `SRCREV.conf` file of the manifest repo:\
`SRCREV_pn-<bitbake-recipe> = "<current-commit-hash>"`
4. Add the project into the gitlab-ci project's `.gitlab-ci.yml`, so changes to
the CI are deployed into the new project.
#### Repository settings
1. Verify that the merge strategy is set to *Fast-forward merge*
(General -> Merge requests -> Merge method)
2. Enable the CI/CD functionality for the repository
(General -> Visibility, project features, permissions -> CI/CD)
3. Check that the default branch is protected and that *Maintainers +
Developers* are allowed to merge (Repository -> Protected branches)
[8]: https://gitlab.com/seco-ne/kernel/modules/egalaxi2c
docs/create-layer-mr.drawio 0 → 100644
View file @ a59a93a0
<mxfile host="app.diagrams.net" modified="2022-05-10T15:50:28.875Z" agent="5.0 (Windows)" etag="tZCYtCye2vNUvOPvtp5c" version="17.4.6" type="device"><diagram id="fUzyECxa7j4ng1UTdmFO" name="Page-1">7V1bc5s4GP01nm0fksHcjB9jJ253pp3JNNvd9lEG2dYWgxfkxO6vXwkQBkkOlyIHu05mWvgAIXSOvqsgA2O63n2IwGb1OfSgP9A1bzcw7ge6PrQsjfxHJftU4thWKlhGyMtOOgie0E+YCbPrllvkwbh0Ig5DH6NNWeiGQQBdXJKBKApfyqctQr981w1YQkHw5AJflP6DPLxKpSNdO8g/QrRcZXe2tOzAGrBzM0G8Al74UhAZDwNjGoUhTrfWuyn06dixYUmvmx05mvcrggGuc8HnL39P/vx6Ez9/woExdPy7h923m7GdNvMM/G32wFlv8Z6NQBRuAw/SVrSBMXlZIQyfNsClR18I5kS2wmuf7A3J5gL5/jT0wyi51phN6S+RLyPgIdLTexQRlFAYkOMQxJheEga4cEn6Q+QxjsIfsHBk4tBf2pgP4jjrTz6syc4PiN1VtgMiN6PTMG+NQajdWkQmDmE2qs8wwnBXEGVD+gGGa4ijPTllV6bpvrz7UqAKY8SqQJOxkwlBRs9l3vIBQrKRodgA0ZGjGFFrNptNZPA4+tyw7QLW7FgQBvB1CnQAhO6UkRiZugCFKYNiaKmCwr5CkenEmlDk06d7KMYCFPMtIqaAx4M8Ii4P+vHB5eB4mMzuHu6p4vHRko6oS4YQkoMTOnSImJS77MAaeR69oRTvMiNOBIctQ0MVGEMBi4FugzV9+GAeb5Kn5vc1iegePkM/3JAhPj6rho1nlQegs3BlwNuuA+eLbkAxy5iYot0YyjAxVWGiV6sqGHh31KE6qBEPxKt8lIUpw2ytLpCaDFu0/5ab4WT3e3ZtsnO/K555v2fmXfAIkt/XEMEgWkJcTUboldxAEbcCMpYEGCaLoA8wei47jzK0sjs8hoj0+OBO6GVejG0O8DjcRi7Mrip6e1xDJkcw4oaUG0oHRmgoIU/+2O35ZMjmuE9Vq4eeyeaSbpKbgJv13gd7OofTw/OIHWUScv/CNWc413kF/OaT3bw6Jum+ZdR1TMaqoBCQWIMALWCMz5Dolt4zoltdWjUqfwSYuHRBItG1Q5SqwtaBCHM9a2/ljF5ZOcMu82RotLRyxsgsN8QHEIqtXI3sSZ+9pkYcSzF5TZVpNclo9oqMtsaFR3zcU5eMll4mo6DWFJNRDHEvl4w94Q6vyFq764JG5JNSirnD9GZN8rg0AYvcMmWOEaRID62CHr9g4ca9YgYfyA3bahWBGXxDqpkhZgYet/GKSOYRCNxVmppJAjVUiNz+29K6xgR43s0CAryN4EFYCOzQ0bBOTMTVT6tFMEY/wTxpirJqQwcnGS5rMrCSFN0WhzFfHOBikaILXaWPGhQMWP2LqYwsFiv6yDJeqsvGKS4BLRwXutLQY+5YptVRktOwytNEZ09RGFddMqx5dab7cR2du0plzOipSs0h/lWVKjSkWqU289RqMOPf7XrzlF0sc+zgDuGcOGT7e0F+oA3dqWRNZRjQM0Ms2M+2Lppg0fmGFLNGF1NFkpTqNCLWltha7fOXAV3tEHVvqFvmaS/WoBucNmEWpWR5TmnR9WZprx5aHhY590WHjMoQG5rRTofwuQahIdU6RObrHdchTHEcm/ESPUI6DpdRcvmsXN6ZNVYxv6lCMfgqBVu8UFQoQwnb+SxqZwrFqFEvOl+HpWfKRtARZktlI2gtviHVJeAabu75BZa5G1gRWKor3ooJma8bL1XYgmKWVdjLqpbQ9RnFtNJK7hsc1/XFsn1eodRwSP75+HBHeqGFi45dTF7/Cx15l/Q6gC9FJ3cRhWs67vPwGb6Xdee4+3q1QBnnx5zmsMRkynD8igrsnvR9d2l7Yjx4nW+29VStcUVDio2H2ay02i+XoydkEDDsypMQGlJNhhrVqf57EjwcefD3Zp5Eswz1eVeMK8MAO8uvnVtygo8XRo7ebpbzDdlcO4onuSVq/EIyodLzurhl+rLFiSddpm/XWsP7V4SWSxpcaBu0gT4KaBxSzju/UfromqE+klCydInhOWlCyb6687V0O+94WdaonW7nfQ+hIdULJa/ufOfuvO0Mb8fFn3bU4DlW0axiooxkL4d1pBluHb3keWq3pj6u0BDJ3iOMEHkwagF+sbDFXgntCaVs7vVly26pX2yuOG6N6zmPBEewL5yWmc3jHTbkHT7aL/78rF8H1qY96FbZySr1Zx+u5iajIlwdsvJz96+26410w3nHq9WqpN/L6FsvhZbUv2614o99WoMki4KUGSTD7tQe9YUbnJmxtbZmhm/IOI2ZYR2ua2ZYv5SamVENM3NlZkNmmpx2ac3MEdcXVczMOlybmVm/1DJTDO/bfW3jeFqkZ8k7my12YM4SG4IqZ0lVhmUkBtxY9lLxBSFgl4k+NG/fGgNZfUVIoHpw44f71inKc4WLlRAZXJJs92nBElcruSvo/mgMwQWUHmTfzjpp6YH1pwDGNAVDQ3SVUe8qC4U5i2LaQRATgHQtWaDkbqMEHPpRwJjg/Qc9I100xbf3DvgvYE+P42gLWQMLFCWrreAOulv6VYz31xpHRY1jZNVcNMuHqd1xWGEm85JrHI5ezj875QbbVjwqmlWcRWA1+Wv9o7v6h2N3QhSBf683q5ooF5k7dthTvNVSp7HoUFznX1NFza2EH3ejqIVP3jFoTvXZlBp2Ol6BDd3crv07F4dFL+kTmEP/kXg92dfK5iHG4VriRuGQm5nhFtPlMtP8w9XaKxHDm3wvTfhwqDCJZR6VsqiA1RQl8fTBRbcmWapJA2T8F8AlHqol8bR/649QmEb5IxTWSIRWtr5fHbTNqizhBganVc7k5jNEn+ksVLVQu+XL8W2Vc926fmfK+fhL78Upn6Y2yajFW/863yXznfsarC35Tr1z0vnebAHgdb43XBrG10Xbzve6BdbO3vS4yPDHVPemB9k9/AmRFIXD32ExHv4H</diagram></mxfile>
\ No newline at end of file
docs/create-layer-mr.svg 0 → 100644
View file @ a59a93a0
This diff is collapsed.
docs/gitlab-ci-deployment.md 0 → 100644
View file @ a59a93a0
<!----------------------------------------------------------------------------->
# `gitlab-ci` deployment
<!----------------------------------------------------------------------------->
<!----------------------------------------------------------------------------->
## Deploy changes into all projects using 'gitlab-ci' as submodule
<!----------------------------------------------------------------------------->
The gitlab-ci repo is integrated into the other related projects as a submodule
and as reference in their `.gitlab-ci.yml` file. Both references need to be
updated when a change in the repo shall be deployed to the projects using it.
To achieve this the CI pipeline is set up to update all these projects in one
run. The submodule *does not* need to be updated manually, but the CI needs some
manual steps to do the deployment.
As this repo contains all the CI code, some extra care is needed in order not to
break the whole thing. To allow testing the CI before everything is merged into the Yocto pipeline, a sandbox environment in the form of a set of test projects is
available in the [ci-test group][1].
[1]: https://git.seco.com/seco-ne/yocto/infrastructure/ci-test
<!----------------------------------------------------------------------------->
## Workflow to deploy a change
<!----------------------------------------------------------------------------->
* Commit and push change to the gitlab-ci repo, create a merge request.
* The pipeline checks syntax and other stuff in the *Analyze* stage.
* The *Integrate* stage allows testing the change in the complete setup.
There are two possible integrate steps, one to deploy the change into Merge
Request in the *ci-test* group, the other is for the *productive yocto* setup.
* When the integrate stage has run, a build is started in the related manifest
project.
* In all projects there are now integration branches created, named
`integrate/gitlab-ci/<gitlab-ci branch name>`.
These could be used to add additional changes if needed. But that should only
happen in very special cases.
* If everything is OK until here, (build OK, CI works as expected ...)
the change can be merged into the main branch of gitlab-ci.
*This only adds the patch to the gitlab-ci repo's main branch, the reference
in the projects using this are not yet updated*
* After the merge a new pipeline is created. In this pipeline there are two
manual jobs: *'merge-ci-test'* and *'merge-yocto'*
These jobs update the reference in the project groups *'ci-test' *and
*'yocto'*. This is split up to allow more testing in cases where the CI
changes can only be tested when working on the master branch.
* When the behaviour of the CI on the master branch may be effected of the
change only *'merge-ci-test'* should be executed.
After this some tests may be done on the ci-test projects.
Probably you want to create a commit in *minimal-foo*, create a merge-request,
check the running pipeline, and merge it.
After that, check if the merge is succesfull and the manifest is updated as
needed.
* When everything is working the *'merge-yocto'* pipeline can be executed to
deploy the change to the productive projects.
* When the tests on *'ci-test'* have failed, just create a new patch fixing the
issue in the gitlab-ci repo and do the testing again.
docs/manifest-master-pipeline.png 0 → 100644
View file @ a59a93a0
docs/manifest-master-pipeline.png

9.36 KiB

docs/manifest-parent-child.png 0 → 100644
View file @ a59a93a0
docs/manifest-parent-child.png

47.7 KiB

docs/manifest-pipeline.md 0 → 100644
View file @ a59a93a0
<!----------------------------------------------------------------------------->
# Manifest pipeline
<!----------------------------------------------------------------------------->
The pipeline running in the manifest repository is the one that does the real
work, like building images, generating docs and deploying image to download
servers and triggering tests in the lava test rack.
<!----------------------------------------------------------------------------->
## Triggers
<!----------------------------------------------------------------------------->
TODO
<!----------------------------------------------------------------------------->
## Build jobs
<!----------------------------------------------------------------------------->
TODO
<!----------------------------------------------------------------------------->
## Deploy and upload jobs
<!----------------------------------------------------------------------------->
TODO
<!----------------------------------------------------------------------------->
## Test jobs
<!----------------------------------------------------------------------------->
TODO
<!----------------------------------------------------------------------------->
## Job generation, parent child jobs
<!----------------------------------------------------------------------------->
As we support images for multiple machines/architectures we are building a lot
of different image variants, where each needs at least build, deploy and test
job.
To simplify the generation of all these jobs, without a lot of copy and paste
and the possibility to configure the actual build images, we are using gitlab's
dynamic-child-pipeline feature. [See gitlab docs.][1]
[1]: https://docs.gitlab.com/ee/ci/pipelines/parent_child_pipelines.html#dynamic-child-pipeline-example
There is a *'generate-build-jobs'* job, that creates a yaml file containing the
pipeline with all needed jobs.
There are the following CI variables in the 'generate-build-jobs' job controlling
the content (make sure these are not set in a more global scope, as this
would overwrite the settings in the generated yml):
* `CI_PARAM_MACHINES`: Space separated list of machines to build for, like "santaro santoka"
* `CI_PARAM_IMAGE`: The name of the image to build. If set to an empty string,
related jobs are not created.
* `CI_PARAM_DISTRO`: The name of the distro to build
* `CI_PARAM_IMAGE_FNG`: The name of the fngsystem image to build. If set to an
empty string, related jobs are not created.
* `CI_PARAM_DISTRO_FNG`: The name of the fngsystem distro to build
It uses a python script called `generate_job_from_template.py` to convert the
`build-jobs.jinja2` to `build-jobs.yml`. This yml file is then used by the
*'trigger-build-jobs'* job, to setup the pipeline described by it.
![Manifest's parent child pipeline](manifest-parent-child.png)