Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • seco-ne/yocto/infrastructure/gitlab-ci
1 result
Show changes
Commits on Source (1)
  • Jonas Höppner's avatar
    CI: Lava Test: Allow to install test image directly from gitlab · ab09db0b
    Jonas Höppner authored 
    The images from the build job can be directly installed from gitlab.
To achive this some changes in the complete pipeline have been needed.

1. The variables used in the build job, like CI_PARAM_IMAGE, ... and
   related variables like BUILDPATH are only valid in the build job now.
2. The build job writes every variable needed in a follow up job into
   build.env. This also includes the url to the fng-install.sh of the
   final image.
3. The build.env file is used as dotenv artifact, as well as normal
   file artifact.
   The dotenv make the written variables automatically available in
   follow up jobs, that are using the aritfacts, like the deploy job.
   The normal file artifact is available via artifact download.
   (I did't found a way to download the dotenv file instead)
4. Some scripts have been added:
   - Find a job inside the pipeline by name, as the id is not known in
     advance.
   - Download all artifacts or one file of the artifacts from a given
     job
   - Download one file of the latest job by name
5. The scripts are used to download the build.env into the test job
   (where not artifacts are needed anymore)
6. The script is sourced and all variables are available inside the
   script.

Additionally this adds a fake build job to the ci-test pipeline, that
copies an image from srv73 and stores it as artifact in a way that a
test-job can run on it, like in the normal yocto pipeline.
    ab09db0b
Hide whitespace changes
Inline Side-by-side
build-common.yml
View file @ ab09db0b
...@@ -74,13 +74,13 @@ workflow: ...@@ -74,13 +74,13 @@ workflow:
rules: rules:
- when: manual - when: manual
allow_failure: true allow_failure: true
dependencies: []
variables: variables:
LOGPREFIX: "CI:test:" # TODO checkout only gitlab-ci repo to allow running on deleted branches
# Include git submodules # Include git submodules
GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_STRATEGY: recursive
CI_PARAM_TEST_SUITE: '{platform}.jinja2' CI_PARAM_TEST_SUITE: '{platform}.jinja2'
CI_PARAM_EXTRA: --nop CI_PARAM_EXTRA: --nop
DEPLOYPATH_TEST: /artifacts/${CI_JOB_ID}/
GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}" GITLAB_SERVER: "${CI_SERVER_HOST}:${CI_SERVER_SSH_PORT}"
GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}" GIT_BASE_URL: "ssh://git@${GITLAB_SERVER}/${CI_PROJECT_ROOT_NAMESPACE}"
TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git" TESTS_GIT_URL: "${GIT_BASE_URL}/yocto/tests.git"
...@@ -94,18 +94,22 @@ workflow: ...@@ -94,18 +94,22 @@ workflow:
before_script: before_script:
- !reference [.setup_ssh] - !reference [.setup_ssh]
script: script:
- .gitlab-ci/scripts/download_file_from_latest_job.py
--project $CI_PROJECT_ID
--pipeline $CI_PIPELINE_ID
--token=$GITBOT_TOKEN
--name $CI_PARAM_BUILDJOB
--path build.env
- cat build.env
- source build.env
# Submit tests to lava server # Submit tests to lava server
- |- - .gitlab-ci/scripts/submit_test.py
RELEASE=${RELEASE:-$(ls ${DEPLOYPATH_TEST}/)} --fng-install "$FNG_INSTALL_URL"
INSTALLSCRIPT_ABS="$DEPLOYPATH_TEST/$RELEASE/$CI_PARAM_MACHINE/fng-install.sh" --name
FNG_INSTALL_URL="${ARTIFACTS_HOST_URL}/${INSTALLSCRIPT_ABS#/*/}" "Gitlab $BUILD_MACHINE $BUILD_DISTRO $BUILD_IMAGE ($CI_PIPELINE_ID)"
.gitlab-ci/scripts/submit_test.py \ --results-path "results"
--fng-install "${FNG_INSTALL_URL}" \ --test-repo $TESTS_GIT_URL
--name \ --test-repo-branch $TEST_REPO_BRANCH
"Gitlab build test ${CI_PARAM_MACHINE} ${RELEASE} ${CI_PIPELINE_ID}" \ --test-plan $CI_PARAM_TEST_SUITE
--results-path "results" \ $CI_PARAM_EXTRA
--test-repo ${TESTS_GIT_URL} \ $CI_PARAM_PLATFORMS
--test-repo-branch ${TEST_REPO_BRANCH} \
--test-plan ${CI_PARAM_TEST_SUITE} \
${CI_PARAM_EXTRA} \
${CI_PARAM_PLATFORMS}
build-jobs-ci-test.jinja2
View file @ ab09db0b
...@@ -61,6 +61,53 @@ build:echo: ...@@ -61,6 +61,53 @@ build:echo:
- printenv - printenv
- echo "Build successful" - echo "Build successful"
build-imx6guf-fake:
stage: build
needs: []
tags:
- infrastructure
timeout: 20m
image: ${CI_IMAGE_PYTHON}
rules:
- when: manual
allow_failure: true
variables:
MANIFEST_VERSION: Yocto-dunfell-17.0
CI_PARAM_MACHINE: imx6guf
CI_PARAM_DISTRO: guf-wayland
CI_PARAM_IMAGE: guf-image
BUILDPATH: "build-${CI_PARAM_DISTRO}-${CI_PARAM_MACHINE}"
IMAGEBASEPATH: "tmp/deploy/images/"
IMAGEPATH: "${IMAGEBASEPATH}/${CI_PARAM_MACHINE}"
LICENSESPATH: "tmp/deploy/licenses"
SDKPATH: "tmp/deploy/sdk/"
INSTALLSCRIPT: "fng-install.sh"
JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/"
FNG_INSTALL_PATH: "${BUILDPATH}/${IMAGEPATH}/${INSTALLSCRIPT}"
script:
# TODO: this becomes similar to the yocto build and may be merged
- echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env
- echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env
- echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env
- echo "MANIFEST_VERSION=$MANIFEST_VERSION" >> build.env
- echo "BUILD_PATH_IMAGE=${BUILDPATH}/${IMAGEPATH}" >> build.env
- echo "BUILD_PATH_SDK=${BUILDPATH}/${SDKPATH}" >> build.env
- echo "BUILD_PATH_LICENSE=${BUILDPATH}/${LICENSESPATH}" >> build.env
- echo "FNG_INSTALL_URL=${JOB_URL}${FNG_INSTALL_PATH}" >> build.env
- source build.env
- echo "Getting yocto build from Z:"
- mkdir -p $BUILD_PATH_IMAGE
- cp -v /artifacts-yocto/Releases/$MANIFEST_VERSION/$CI_PARAM_MACHINE/*.* $BUILD_PATH_IMAGE
- ls $BUILD_PATH_IMAGE
artifacts:
reports:
dotenv: build.env
paths:
- build-guf-wayland-imx6guf/tmp/deploy/images/imx6guf/*
- build.env
# -------------------------------------------------------------------------------------- # --------------------------------------------------------------------------------------
# Stage: test # Stage: test
# -------------------------------------------------------------------------------------- # --------------------------------------------------------------------------------------
...@@ -87,16 +134,15 @@ test:fail: ...@@ -87,16 +134,15 @@ test:fail:
variables: variables:
RETURNCODE: 1 RETURNCODE: 1
smoketest:imx6guf: smoketest-imx6guf:
extends: .test extends:
- .test
stage: test stage: test
needs: [build-imx6guf-fake]
variables: variables:
CI_PARAM_MACHINE: imx6guf TEST_REPO_BRANCH: dunfell
CI_PARAM_PLATFORMS: imx6guf CI_PARAM_PLATFORMS: imx6guf
CI_PARAM_TEST_SUITE: boot.jinja2
CI_PARAM_EXTRA: --all-devices CI_PARAM_EXTRA: --all-devices
# Use existing hard-coded release as we don't build in CI-test CI_PARAM_TEST_SUITE: boot.jinja2
ARTIFACTS_HOST_URL: http://srv73 CI_PARAM_BUILDJOB: build-imx6guf-fake
DEPLOYPATH_TEST: SoftwareStore/Linux-Yocto/Releases/
RELEASE: Yocto-dunfell-15.0
TEST_REPO_BRANCH: dunfell
build-jobs-yocto.jinja2
View file @ ab09db0b
...@@ -61,11 +61,12 @@ build-{{ machine }}: ...@@ -61,11 +61,12 @@ build-{{ machine }}:
CI_PARAM_MACHINE: {{ machine }} CI_PARAM_MACHINE: {{ machine }}
CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }} CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }} CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
INSTALLSCRIPTS: "fnginstall.sh" INSTALLSCRIPT: "fng-install.sh"
artifacts: artifacts:
paths: paths:
- build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/images/{{ machine }}/* - build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/images/{{ machine }}/*
- build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/licenses/**/license.manifest - build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/licenses/**/license.manifest
- build.env
# Build jobs for the sdk # Build jobs for the sdk
buildsdk-{{ machine }}: buildsdk-{{ machine }}:
...@@ -78,6 +79,7 @@ buildsdk-{{ machine }}: ...@@ -78,6 +79,7 @@ buildsdk-{{ machine }}:
artifacts: artifacts:
paths: paths:
- build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/sdk/* - build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/sdk/*
- build.env
# Deploy jobs for the yocto image # Deploy jobs for the yocto image
deployimage-{{ machine }}: deployimage-{{ machine }}:
...@@ -124,11 +126,11 @@ uploadftp-sdk-{{ machine }}: ...@@ -124,11 +126,11 @@ uploadftp-sdk-{{ machine }}:
smoketest:{{ machine }}: smoketest:{{ machine }}:
extends: extends:
- .test - .test
- .test_override
stage: test stage: test
needs: needs:
- job: build-{{ machine }} - job: build-{{ machine }}
variables: variables:
CI_PARAM_BUILDJOB: build-{{ machine }}
CI_PARAM_MACHINE: {{ machine }} CI_PARAM_MACHINE: {{ machine }}
CI_PARAM_PLATFORMS: {{ machine }} CI_PARAM_PLATFORMS: {{ machine }}
CI_PARAM_TEST_SUITE: boot.jinja2 CI_PARAM_TEST_SUITE: boot.jinja2
...@@ -137,11 +139,11 @@ smoketest:{{ machine }}: ...@@ -137,11 +139,11 @@ smoketest:{{ machine }}:
platformtest:{{ machine }}: platformtest:{{ machine }}:
extends: extends:
- .test - .test
- .test_override
stage: test stage: test
needs: needs:
- job: build-{{ machine }} - job: build-{{ machine }}
variables: variables:
CI_PARAM_BUILDJOB: build-{{ machine }}
CI_PARAM_MACHINE: {{ machine }} CI_PARAM_MACHINE: {{ machine }}
CI_PARAM_PLATFORMS: {{ platforms }} CI_PARAM_PLATFORMS: {{ platforms }}
{% endif %} {% endif %}
...@@ -152,17 +154,18 @@ platformtest:{{ machine }}: ...@@ -152,17 +154,18 @@ platformtest:{{ machine }}:
# Build jobs for the fng system image # Build jobs for the fng system image
build-{{ machine }}-fngsystem: build-{{ machine }}-fngsystem:
extends: .buildfng extends: .buildimage
stage: build stage: build
variables: variables:
CI_PARAM_MACHINE: {{ machine }} CI_PARAM_MACHINE: {{ machine }}
CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }} CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }}
CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }} CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }}
INSTALLSCRIPTS: "fngsystem-self-update.sh" INSTALLSCRIPT: "fngsystem-self-update.sh"
artifacts: artifacts:
paths: paths:
- build-{{ CI_PARAM_DISTRO_FNG }}-{{ machine }}/tmp/deploy/images/{{ machine }}/* - build-{{ CI_PARAM_DISTRO_FNG }}-{{ machine }}/tmp/deploy/images/{{ machine }}/*
- build-{{ CI_PARAM_DISTRO_FNG }}-{{ machine }}/tmp/deploy/licenses/**/license.manifest - build-{{ CI_PARAM_DISTRO_FNG }}-{{ machine }}/tmp/deploy/licenses/**/license.manifest
- build.env
# Deploy jobs for the fngsystem image # Deploy jobs for the fngsystem image
deployimage-{{ machine }}-fngsystem: deployimage-{{ machine }}-fngsystem:
......
build-yocto.yml
View file @ ab09db0b
--- ---
variables:
BUILDPATH: "build-${CI_PARAM_DISTRO}-${CI_PARAM_MACHINE}"
IMAGEBASEPATH: "tmp/deploy/images/"
IMAGEPATH: "${IMAGEBASEPATH}/${CI_PARAM_MACHINE}"
LICENSESPATH: "tmp/deploy/licenses"
SDKPATH: "tmp/deploy/sdk/"
.collect_srcrevs: &collect_srcrevs .collect_srcrevs: &collect_srcrevs
# write all package AUTOREVS to file # write all package AUTOREVS to file
...@@ -19,21 +13,33 @@ variables: ...@@ -19,21 +13,33 @@ variables:
.dump_install_command: &dump_install_command .dump_install_command: &dump_install_command
# print install instructions # print install instructions
- |- - |-
for i in ${INSTALLSCRIPTS};do SCRIPT="${CI_PROJECT_DIR}/${BUILDPATH}/${IMAGEPATH}/${INSTALLSCRIPT}"
SCRIPT="${CI_PROJECT_DIR}/${BUILDPATH}/${IMAGEPATH}/${i}" if [ ! -f "${SCRIPT}" ]; then
if [[ -f "${SCRIPT}" ]]; then echo "Install script missing, searched for '$SCRIPT'"
cat <<-EOF exit 1
fi
if [ "$CI_PROJECT_VISIBILITY" = "public" ];then
cat <<-EOF
==============================
Install the image:
FNG="$FNG_INSTALL_URL"
curl --location "\$FNG" | sh -s -- --url="\$(dirname "\$FNG")"
==============================
EOF
else
cat <<-EOF
============================== ==============================
Install the image: Install the image:
export GITLAB_TOKEN=<your_access_token> export GITLAB_TOKEN=<your_access_token>
FNG="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/${BUILDPATH}/${IMAGEPATH}/${i}" FNG="$FNG_INSTALL_URL"
curl --location --header "PRIVATE-TOKEN: \$GITLAB_TOKEN" "\$FNG" \ curl --location --header "PRIVATE-TOKEN: \$GITLAB_TOKEN" "\$FNG" \
| sh -s -- --url="\$(dirname "\$FNG")" | sh -s -- --url="\$(dirname "\$FNG")"
============================== ==============================
EOF EOF
fi fi
done
.build_script: &build_script .build_script: &build_script
# setup build environment # setup build environment
...@@ -42,6 +48,11 @@ variables: ...@@ -42,6 +48,11 @@ variables:
- echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env - echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env
- echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env - echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env
- echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env - echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env
- echo "BUILD_PATH_IMAGE=${BUILDPATH}/${IMAGEPATH}" >> build.env
- echo "BUILD_PATH_SDK=${BUILDPATH}/${SDKPATH}" >> build.env
- echo "BUILD_PATH_LICENSE=${BUILDPATH}/${LICENSESPATH}" >> build.env
- echo "FNG_INSTALL_URL=${JOB_URL}${FNG_INSTALL_PATH}" >> build.env
- source build.env
- echo "${LOGPREFIX} Using build dir ${BUILDPATH}" - echo "${LOGPREFIX} Using build dir ${BUILDPATH}"
- export MACHINE="${CI_PARAM_MACHINE}" - export MACHINE="${CI_PARAM_MACHINE}"
- export DISTRO="${CI_PARAM_DISTRO}" - export DISTRO="${CI_PARAM_DISTRO}"
...@@ -53,12 +64,22 @@ variables: ...@@ -53,12 +64,22 @@ variables:
- bitbake "${CI_PARAM_IMAGE}" -c "${BITBAKE_TASK}" - bitbake "${CI_PARAM_IMAGE}" -c "${BITBAKE_TASK}"
- echo -e "section_end:`date +%s`:bitbake_run\r\e[0K" - echo -e "section_end:`date +%s`:bitbake_run\r\e[0K"
.build: .buildimage:
extends:
- .buildbase
needs: []
variables: variables:
GIT_STRATEGY: none GIT_STRATEGY: none
SETUPSCRIPT: "setup-environment" SETUPSCRIPT: "setup-environment"
BITBAKE_TASK: "build" BITBAKE_TASK: "build"
LOGPREFIX: "CI:build:" LOGPREFIX: "CI:build:"
BUILDPATH: "build-${CI_PARAM_DISTRO}-${CI_PARAM_MACHINE}"
IMAGEBASEPATH: "tmp/deploy/images/"
IMAGEPATH: "${IMAGEBASEPATH}/${CI_PARAM_MACHINE}"
LICENSESPATH: "tmp/deploy/licenses"
SDKPATH: "tmp/deploy/sdk/"
JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/"
FNG_INSTALL_PATH: "${BUILDPATH}/${IMAGEPATH}/${INSTALLSCRIPT}"
before_script: before_script:
- !reference [.docker_check] - !reference [.docker_check]
- !reference [.setup_ssh] - !reference [.setup_ssh]
...@@ -74,19 +95,6 @@ variables: ...@@ -74,19 +95,6 @@ variables:
# -------------------------------------------------------------------------------------- # --------------------------------------------------------------------------------------
# Stage: build # Stage: build
# -------------------------------------------------------------------------------------- # --------------------------------------------------------------------------------------
.buildimage:
extends:
- .buildbase
- .build
needs: []
.buildfng:
extends:
- .buildimage
variables:
CI_PARAM_IMAGE: ${CI_PARAM_IMAGE_FNG}
CI_PARAM_DISTRO: ${CI_PARAM_DISTRO_FNG}
.buildsdk: .buildsdk:
extends: extends:
- .buildimage - .buildimage
...@@ -97,24 +105,6 @@ variables: ...@@ -97,24 +105,6 @@ variables:
variables: variables:
BITBAKE_TASK: "populate_sdk" BITBAKE_TASK: "populate_sdk"
# --------------------------------------------------------------------------------------
# Stage: test
# --------------------------------------------------------------------------------------
.test_override:
before_script:
- !reference [.setup_ssh]
# Copy artifacts to local server for automated tests
# This is necessary because the LAVA devices cannot install directly from GitLab
# due to missing HTTPS support in FNGSystem.
- |-
echo "${LOGPREFIX} Copy files for automated tests using package_release"
.gitlab-ci/scripts/package_release.py \
--images-dir="${BUILDPATH}/${IMAGEPATH}" \
--outputdir-local="${DEPLOYPATH_TEST}"
after_script:
# Remove artifacts after testing
- rm -r "${DEPLOYPATH_TEST}"
# -------------------------------------------------------------------------------------- # --------------------------------------------------------------------------------------
# Stage: deploy # Stage: deploy
# -------------------------------------------------------------------------------------- # --------------------------------------------------------------------------------------
......
manifest-package.yml
View file @ ab09db0b
...@@ -8,24 +8,17 @@ ...@@ -8,24 +8,17 @@
# Package release files # Package release files
# TODO: At the moment this script is used for # TODO: At the moment this script is used for
# - image deployment, # - image deployment,
# - sdk build, # - sdk deployment,
# - image and sdk ftp upload # - image and sdk ftp upload
# - and artifact deployment for automated tests. # - and artifact deployment for automated tests.
# Some usecases require individual processing, which leads to # Some usecases require individual processing, which leads to
# high script complexity. That should be improved. # high script complexity. That should be improved.
echo "${LOGPREFIX} Assemble build variables" echo "${LOGPREFIX} Assemble build variables"
# We need to source the build.env file which is generated by the # Variables set by build.env from the build artifacts
# previous build step. This is a workaround to get the environment ARTIFACTS_IMAGE_PATH="$BUILD_PATH_IMAGE"
# variables in the after_script step. ARTIFACTS_LICENSES_PATH="$BUILD_PATH_LICENSE"
[ -e build.env ] && source build.env ARTIFACTS_SDK_PATH="$BUILD_PATH_SDK"
BUILDPATH="build-${BUILD_DISTRO}-${BUILD_MACHINE}"
[ -z "${ARTIFACTS_IMAGE_PATH}" ] && \
ARTIFACTS_IMAGE_PATH="${BUILDPATH}/${IMAGEBASEPATH}/${BUILD_MACHINE}"
[ -z "${ARTIFACTS_LICENSES_PATH}" ] && \
ARTIFACTS_LICENSES_PATH="${BUILDPATH}/${LICENSESPATH}"
[ -z "${ARTIFACTS_SDK_PATH}" ] && \
ARTIFACTS_SDK_PATH="${BUILDPATH}/${SDKPATH}"
if ${CI_PARAM_PACKAGE_FTP}; then if ${CI_PARAM_PACKAGE_FTP}; then
UPLOAD_PARAM="" # don't store as gitlab artifact UPLOAD_PARAM="" # don't store as gitlab artifact
...@@ -37,23 +30,22 @@ ...@@ -37,23 +30,22 @@
# If we are on the master branch and a tag is set # If we are on the master branch and a tag is set
# we tread it as release # we tread it as release
if [ -n "${CI_COMMIT_TAG}" ];then if [ -n "$CI_COMMIT_TAG" ];then
outdir="${OUTDIR_BASE}-yocto/Releases" outdir="${OUTDIR_BASE}-yocto/Releases"
[ "${BUILD_DISTRO}" = "guf-fngsystem" ] && \ [ "$BUILD_DISTRO" = "guf-fngsystem" ] && \
outdir="${OUTDIR_BASE}-fngsystem" outdir="${OUTDIR_BASE}-fngsystem"
else else
outdir="${OUTDIR_BASE}-yocto/Interne_Releases" outdir="${OUTDIR_BASE}-yocto/Interne_Releases"
[ "${BUILD_DISTRO}" = "guf-fngsystem" ] && \ [ "$BUILD_DISTRO" = "guf-fngsystem" ] && \
outdir="${OUTDIR_BASE}-fngsystem/CI_Builds" outdir="${OUTDIR_BASE}-fngsystem/CI_Builds"
fi fi
# Print vars for debugging purposes # Print vars for debugging purposes
echo "${LOGPREFIX} BUILDPATH=${BUILDPATH}" echo "${LOGPREFIX} ARTIFACTS_IMAGE_PATH=$ARTIFACTS_IMAGE_PATH"
echo "${LOGPREFIX} ARTIFACTS_IMAGE_PATH=${ARTIFACTS_IMAGE_PATH}" echo "${LOGPREFIX} ARTIFACTS_LICENSES_PATH=$ARTIFACTS_LICENSES_PATH"
echo "${LOGPREFIX} ARTIFACTS_LICENSES_PATH=${ARTIFACTS_LICENSES_PATH}" echo "${LOGPREFIX} ARTIFACTS_SDK_PATH=$ARTIFACTS_SDK_PATH"
echo "${LOGPREFIX} ARTIFACTS_SDK_PATH=${ARTIFACTS_SDK_PATH}" echo "${LOGPREFIX} UPLOAD_PARAM=$UPLOAD_PARAM"
echo "${LOGPREFIX} UPLOAD_PARAM=${UPLOAD_PARAM}" echo "${LOGPREFIX} outdir=$outdir"
echo "${LOGPREFIX} outdir=${outdir}"
# Check if the package_release script is available # Check if the package_release script is available
script=".gitlab-ci/scripts/package_release.py" script=".gitlab-ci/scripts/package_release.py"
......
scripts/download_file_from_latest_job.py 0 → 100755
View file @ ab09db0b
#!/usr/bin/env python3
"""
Downloads given file of the artifacts of a pipeline job.
"""
import argparse
import logging
import os
import sys
import gitlab as gl
__author__ = "Jonas Höppner"
__email__ = "jonas.hoeppner@garz-fricke.com"
from download_job_artifacts import download_job_artifact
from get_pipeline_jobs import get_pipeline_jobs
GITLAB_SERVER = "https://git.seco.com"
verbose = 0
def main(args):
parser = argparse.ArgumentParser(description=__doc__, usage="%(prog)s [OPTIONS]")
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
action="store",
default=GITLAB_SERVER,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
action="store",
dest="project",
help="Specify the project by either by id or by path.",
required=True,
)
parser.add_argument(
"--pipeline",
action="store",
dest="pipeline",
help="Specify the pipeline by id.",
)
parser.add_argument(
"-s",
"--stage",
action="store",
default=None,
help="Filter the jobs by the given stage, if omnitted all jobs are returned.",
)
parser.add_argument(
"-n",
"--name",
action="store",
default=None,
help="Filter the jobs by given name, if omnitted all jobs are returned.",
)
parser.add_argument(
"--path",
action="store",
default=None,
help="Path inside the artifacts, if set only one single file is downloaded instead of the complete artifacts.",
)
parser.add_argument(
"-v",
"--verbose",
action="count",
dest="verbose",
default=0,
help="Increase verbosity.",
)
options = parser.parse_args(args)
if options.verbose:
logging.basicConfig(level=logging.DEBUG)
logging.debug(options)
gitlab = gl.Gitlab(options.gitlab_url, private_token=options.token)
jobs = get_pipeline_jobs(
gitlab, options.project, options.pipeline, options.name, options.stage
)
def sort_by_finish_ts(j):
return j.finished_at
jobs.sort(key=sort_by_finish_ts)
job = jobs[0]
filename = download_job_artifact(
gitlab, dest=os.path.basename(options.path), path=options.path, job=job
)
print("Downloaded {} for job {} to {}".format(options.path, job.name, filename))
if __name__ == "__main__":
main(sys.argv[1:])
scripts/download_job_artifacts.py 0 → 100755
View file @ ab09db0b
#!/usr/bin/env python3
"""
Downloads the job artifacts of a given job
"""
import argparse
import logging
import sys
import os
import zipfile
import tempfile
import gitlab as gl
__author__ = "Jonas Höppner"
__email__ = "jonas.hoeppner@garz-fricke.com"
GITLAB_SERVER = "https://git.seco.com"
verbose = 0
def download_job_artifacts(gitlab, dest, job, project=None, extract=False):
"""Downloads the artifacts and stores them ar dest/job_id
If extract is set, the downloaded zipfile is extracted and removed.
Returns either the dest path or the zipfile name.
"""
# Accept either a gitlab job object, or the project and the job id
if isinstance(job, gl.v4.objects.jobs.ProjectJob):
gl_job = job
else:
# Accept either an gitlab object, or name or id to specify a project
if isinstance(project, gl.v4.objects.projects.Project):
gl_project = project
else:
gl_project = gitlab.projects.get(project)
gl_job = gl_project.jobs.get(job)
dest = os.path.join(dest, str(gl_job.id))
os.makedirs(dest)
file_name = os.path.join(dest, "artifacts.zip")
with open(file_name, "wb") as f:
gl_job.artifacts(streamed=True, action=f.write)
if not extract:
return file_name
zipfile.ZipFile(file_name).extractall(dest)
os.remove(file_name)
return dest
def download_job_artifact(gitlab, dest, path, job, project=None):
"""Downloads one file from artifacts and stores it at dest"""
# Accept either a gitlab job object, or the project and the job id
if isinstance(job, gl.v4.objects.jobs.ProjectJob):
gl_job = job
else:
# Accept either an gitlab object, or name or id to specify a project
if isinstance(project, gl.v4.objects.projects.Project):
gl_project = project
else:
gl_project = gitlab.projects.get(project)
gl_job = gl_project.jobs.get(job)
with open(dest, "wb") as f:
gl_job.artifact(path=path, streamed=True, action=f.write)
return dest
def main(args):
parser = argparse.ArgumentParser(description=__doc__, usage="%(prog)s [OPTIONS]")
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
action="store",
default=GITLAB_SERVER,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
action="store",
dest="project",
help="Specify the project by either by id or by path.",
required=True,
)
parser.add_argument(
"--job",
action="store",
dest="job",
help="Specify the job by id.",
required=True,
)
parser.add_argument(
"--extract",
action="store_true",
dest="job",
default=False,
help="Specify if the artifacts should be extracted after download.",
)
parser.add_argument(
"--dest",
"--destination",
action="store",
dest="destination",
default=None,
help="Folder where the artifacts are stored in, a local tmpfolder is generated if omnitted.",
)
parser.add_argument(
"--path",
action="store",
default=None,
help="Path inside the artifacts, if set only one single file is downloaded instead of the complete artifacts.",
)
parser.add_argument(
"-v",
"--verbose",
action="count",
dest="verbose",
default=0,
help="Increase verbosity.",
)
options = parser.parse_args(args)
if options.verbose:
logging.basicConfig(level=logging.DEBUG)
logging.debug(options)
gitlab = gl.Gitlab(options.gitlab_url, private_token=options.token)
if options.path is None:
if options.destination is None:
destination = tempfile.mkstemp()
else:
destination = options.destination
filename = download_job_artifacts(
gitlab, destination, options.job, options.project, extract=True
)
print("Downloaded artifacts for job {} to {}".format(options.job, filename))
else:
if options.destination is None:
destination = tempfile.mkdtemp()
else:
destination = options.destination
filename = download_job_artifact(
gitlab, destination, options.path, options.job, options.project
)
print(
"Downloaded {} for job {} to {}".format(options.path, options.job, filename)
)
if __name__ == "__main__":
main(sys.argv[1:])
scripts/get_pipeline_jobs.py 0 → 100755
View file @ ab09db0b
#!/usr/bin/env python3
"""
Queries the jobs of a given pipeline, filtered by stage
"""
import argparse
import logging
import sys
import gitlab as gl
__author__ = "Jonas Höppner"
__email__ = "jonas.hoeppner@garz-fricke.com"
GITLAB_SERVER = "https://git.seco.com"
verbose = 0
def get_pipeline_pipelinejobs(gitlab, project, pipeline, name=None, stage=None):
# Accept either an gitlab object, or name or id to specify a project
if isinstance(project, gl.v4.objects.projects.Project):
gl_project = project
else:
gl_project = gitlab.projects.get(project)
gl_pipeline = gl_project.pipelines.get(pipeline)
jobs = gl_pipeline.jobs.list(all=True)
if stage is not None:
jobs = list(filter(lambda j: j.stage == stage, jobs))
if name is not None:
jobs = list(filter(lambda j: j.name == name, jobs))
return jobs
def get_pipeline_jobs(gitlab, project, pipeline, name=None, stage=None):
# Accept either an gitlab object, or name or id to specify a project
if isinstance(project, gl.v4.objects.projects.Project):
gl_project = project
else:
gl_project = gitlab.projects.get(project)
pipeline_jobs = get_pipeline_pipelinejobs(gitlab, project, pipeline, name, stage)
jobs = []
# Project Jobs from Pipeline Job
for job in pipeline_jobs:
jobs.append(gl_project.jobs.get(job.id))
return jobs
def get_pipeline_job_ids(gitlab, project, pipeline, name=None, stage=None):
jobs = get_pipeline_pipelinejobs(gitlab, project, pipeline, name, stage)
return [j.id for j in jobs]
def main(args):
parser = argparse.ArgumentParser(description=__doc__, usage="%(prog)s [OPTIONS]")
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
action="store",
default=GITLAB_SERVER,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
action="store",
dest="project",
help="Specify the project by either by id or by path.",
required=True,
)
parser.add_argument(
"--pipeline",
action="store",
dest="pipeline",
help="Specify the pipeline by id.",
)
parser.add_argument(
"-s",
"--stage",
action="store",
default=None,
help="Filter the jobs by the given stage, if omnitted all jobs are returned.",
)
parser.add_argument(
"-n",
"--name",
action="store",
default=None,
help="Filter the jobs by given name, if omnitted all jobs are returned.",
)
parser.add_argument(
"-v",
"--verbose",
action="count",
dest="verbose",
default=0,
help="Increase verbosity.",
)
options = parser.parse_args(args)
if options.verbose:
logging.basicConfig(level=logging.DEBUG)
logging.debug(options)
gitlab = gl.Gitlab(options.gitlab_url, private_token=options.token)
jobs = get_pipeline_jobs(
gitlab, options.project, options.pipeline, options.name, options.stage
)
for j in jobs:
print(j.name, j.id)
job_ids = get_pipeline_job_ids(
gitlab, options.project, options.pipeline, options.name, options.stage
)
for j in job_ids:
print(j)
if __name__ == "__main__":
main(sys.argv[1:])