diff --git a/ci-test-manifest.yml b/build-jobs-ci-test.jinja2 similarity index 64% rename from ci-test-manifest.yml rename to build-jobs-ci-test.jinja2 index 06e1ccb0b3fbfc55b5a493e13d98f006702d3be6..4aebd92c97568bd5c524ae5c5ced631fdd9dcaa7 100644 --- a/ci-test-manifest.yml +++ b/build-jobs-ci-test.jinja2 @@ -2,77 +2,30 @@ # -------------------------------------------------------------------------------------- # Global # -------------------------------------------------------------------------------------- +# As the trigger job is not executed in a environment with checked out repository, we +# need to get the includes directly from gitlab include: - - local: common.yml + - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' + ref: {{ GITLAB_CI_REVISION }} + file: 'common.yml' + +workflow: + rules: + # This rule is needed, as otherwise the workflow:rules from + # the parent job seem to be used and prevent the pipeline generation + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" stages: - - retrigger - infrastructure - build - test -variables: - # The master branch is hardcoded here, because it cannot be determined automatically. - # Has to be modified for new branches, e.g. new Yocto versions or fix releases. - MASTER_BRANCH_MANIFEST: master - MASTER_BRANCH_PROJECT: master - -workflow: - rules: - # Explicitly allow externally triggered pipelines in every case - - if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "api" - # Do not run pipelines for merge requests - - if: $CI_MERGE_REQUEST_IID - when: never - # Do not run pipelines on forked projects - # (use id instead of name because of rename) - - if: $CI_PROJECT_ID != "1742" - when: never - # Do not run pipelines on integration branches - - if: $CI_COMMIT_REF_NAME =~ /^integrate\/.*/ - when: never - # In all other cases, run the pipeline automatically - - when: always - - -# -------------------------------------------------------------------------------------- -# Stage: retrigger -# -------------------------------------------------------------------------------------- -retrigger: - extends: .infrastructure - stage: retrigger - rules: - - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api" - script: - - PROJECTS=$( - .gitlab-ci/scripts/get_manifest_projects.py - --manifest=default.xml - --remote=ci-test - ) - # Add the gitlab-ci project - - PROJECTS="$PROJECTS ${CI_PROJECT_ROOT_NAMESPACE}/yocto/infrastructure/gitlab-ci" - # TODO retrigger gitlab-ci integration also - # Retrigger also project in SRCREV - - echo -e "Projects:\n${PROJECTS}" - - for PROJECT in ${PROJECTS}; do - .gitlab-ci/scripts/retrigger_mr_pipeline_jobs.py - --gitlab-url=${CI_SERVER_URL} - --token=${GITBOT_TOKEN} - --project=${PROJECT} - --state=opened - --target-branch=${MASTER_BRANCH_MANIFEST} - --job=check - ; - done - - # -------------------------------------------------------------------------------------- # Stage: infrastructure # -------------------------------------------------------------------------------------- yamllint: extends: .yamllint - # -------------------------------------------------------------------------------------- # Stage: build # -------------------------------------------------------------------------------------- @@ -117,8 +70,6 @@ yamllint: build:files: stage: build - rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" needs: [] tags: - infrastructure @@ -148,8 +99,6 @@ build:files: build:echo: stage: build - rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" needs: [] tags: - infrastructure @@ -159,52 +108,13 @@ build:echo: - printenv - echo "Build successful" -build:merge_request: - stage: build - rules: - - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api" - needs: [] - tags: - - infrastructure - timeout: 2m - image: ${CI_IMAGE_PYTHON} - variables: - # Include git submodules - GIT_SUBMODULE_STRATEGY: recursive - script: - - cd ${CI_PROJECT_DIR} - # Get pipeline for merge request - - MR_PIPELINE=$(.gitlab-ci/scripts/get_pipelines.py - --gitlab-url=${CI_SERVER_URL} - --token=${GITBOT_TOKEN} - --project=${CI_PROJECT_PATH} - --commit=${CI_COMMIT_SHA} - --ref=^${MASTER_BRANCH_MANIFEST} || true | head -1) - # If pipeline exists, mirror its result - - if [ ! -z "${MR_PIPELINE}" ]; then - .gitlab-ci/scripts/mirror_pipeline_result.py - --gitlab-url=${CI_SERVER_URL} - --token=${GITBOT_TOKEN} - --project=${CI_PROJECT_PATH} - --pipeline=${MR_PIPELINE} - # If no pipeline found, trigger a new one on the master - - else - .gitlab-ci/scripts/trigger_pipeline.py - --gitlab-url=${CI_SERVER_URL} - --token=${GITBOT_TOKEN} - --project=${CI_PROJECT_PATH} - --ref=${MASTER_BRANCH_MANIFEST} - - fi - - # -------------------------------------------------------------------------------------- # Stage: test # -------------------------------------------------------------------------------------- .test-simple: stage: test rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" - when: manual + - when: manual allow_failure: true tags: - infrastructure @@ -232,8 +142,7 @@ test:fail: - .infrastructure timeout: 1h rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" - when: manual + - when: manual allow_failure: true variables: # Include git submodules diff --git a/build-jobs.jinja2 b/build-jobs-yocto.jinja2 similarity index 90% rename from build-jobs.jinja2 rename to build-jobs-yocto.jinja2 index 8d8cbc452249aba999d246a905846871839cbf32..bd91a7850efb0c243a86bedd59e4c8624ad74bba 100644 --- a/build-jobs.jinja2 +++ b/build-jobs-yocto.jinja2 @@ -1,20 +1,18 @@ --- -#====================================================== -# Create build, test and deploy jobs for all machines -#====================================================== - -# As the trigger job is not executed in a environment -# with checked out repository, we need to get the includes -# directly from gitlab +# -------------------------------------------------------------------------------------- +# Global +# -------------------------------------------------------------------------------------- +# As the trigger job is not executed in a environment with checked out repository, we +# need to get the includes directly from gitlab include: - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' - ref: {{ GITLAB_CI_CURRENT_REV }} + ref: {{ GITLAB_CI_REVISION }} file: 'common.yml' - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' - ref: {{ GITLAB_CI_CURRENT_REV }} + ref: {{ GITLAB_CI_REVISION }} file: 'manifest-build.yml' - project: '{{ CI_PROJECT_ROOT_NAMESPACE }}/yocto/infrastructure/gitlab-ci' - ref: {{ GITLAB_CI_CURRENT_REV }} + ref: {{ GITLAB_CI_REVISION }} file: 'manifest-package.yml' variables: diff --git a/manifest-build.yml b/manifest-build.yml index d3d7e7c8471494195143d84c7b595b958238974b..7037cc6ec430bb75ffa784d710ab12d51013fd43 100644 --- a/manifest-build.yml +++ b/manifest-build.yml @@ -156,8 +156,6 @@ variables: extends: - .buildbase - .build - rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" needs: [] .buildfng: @@ -173,8 +171,7 @@ variables: - .package stage: build rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" - when: manual + - when: manual allow_failure: true variables: BITBAKE_TASK: "populate_sdk" @@ -194,8 +191,7 @@ variables: - .prepare_test timeout: 1h rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" - when: manual + - when: manual allow_failure: true variables: # Include git submodules @@ -236,8 +232,7 @@ variables: - .infrastructure - .package rules: - - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api" - when: manual + - when: manual allow_failure: true script: # Workaround: We need a command in the script section to be able to run the diff --git a/manifest-pipeline-ci-test.yml b/manifest-pipeline-ci-test.yml new file mode 100644 index 0000000000000000000000000000000000000000..cae23bce93fbf1a1a7f490c231a348f474116899 --- /dev/null +++ b/manifest-pipeline-ci-test.yml @@ -0,0 +1,24 @@ +--- +include: + - local: manifest-pipeline.yml + +variables: + # The id of the gitlab project used in the rules section to not run pipelines in + # forked projects. Using variable here, to allow override in other projects including + # this file. + MANIFEST_PROJECT_ID: 1725 + + # In the manifest, the remotes are specified by an identifier. This is used to find + # out included projects for the retrigger job. In custom manifests, the remote may be + # named differently, so we need a variable that may be overriden. + CI_PARAM_SECO_REMOTE: seco-ne + + BUILD_TIMEOUT: 2m + + # This is the jinja2 template file used to generate the build jobs + BUILD_JOBS_TEMPLATE: build-jobs-ci-test.jinja2 + + # The master branch is hardcoded here, because it cannot be determined automatically. + # Has to be modified for new branches, e.g. new Yocto versions or fix releases. + MASTER_BRANCH_MANIFEST: master + MASTER_BRANCH_PROJECT: master diff --git a/manifest-pipeline-yocto.yml b/manifest-pipeline-yocto.yml new file mode 100644 index 0000000000000000000000000000000000000000..30976f68efa44681a77b0269c89496bcd5587aff --- /dev/null +++ b/manifest-pipeline-yocto.yml @@ -0,0 +1,33 @@ +--- +include: + - local: manifest-pipeline.yml + +variables: + # The id of the gitlab project used in the rules section to not run pipelines in + # forked projects. Using variable here, to allow override in other projects including + # this file. + MANIFEST_PROJECT_ID: 1725 + + # In the manifest, the remotes are specified by an identifier. This is used to find + # out included projects for the retrigger job. In custom manifests, the remote may be + # named differently, so we need a variable that may be overriden. + CI_PARAM_SECO_REMOTE: seco-ne + + BUILD_TIMEOUT: 1h + + # This is the jinja2 template file used to generate the build jobs + BUILD_JOBS_TEMPLATE: build-jobs-yocto.jinja2 + + # Default image and distro + CI_PARAM_IMAGE: guf-image + CI_PARAM_DISTRO: guf-wayland + + # Flash-N-Go image and distro + # In the past, the buildfng job overwrote the image and distro itself. Due to the + # transition to the new seconorth names, image and distro for the buildfng must be + # settable from outside of the job. + CI_PARAM_IMAGE_FNG: fngsystem-image + CI_PARAM_DISTRO_FNG: guf-fngsystem + + # List of machines to build images for + CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf diff --git a/manifest.yml b/manifest-pipeline.yml similarity index 79% rename from manifest.yml rename to manifest-pipeline.yml index 8485d516e88a01bab7222cf5a05e2fd273e08fa7..94c281a9d8bee493a65755ccbf2cc69c6fb2664d 100644 --- a/manifest.yml +++ b/manifest-pipeline.yml @@ -10,30 +10,6 @@ stages: - infrastructure - build -variables: - # Default image and distro - CI_PARAM_IMAGE: guf-image - CI_PARAM_DISTRO: guf-wayland - # Flash-N-Go image and distro - # - # In the past, the buildfng job overwrote the image and distro itself. - # Because of the transition to the new seconorth names, the image and - # distro for the buildfng must be settable from outside of the job. - CI_PARAM_IMAGE_FNG: fngsystem-image - CI_PARAM_DISTRO_FNG: guf-fngsystem - # List of machines to build images for: - CI_PARAM_MACHINES: imx6guf imx6ullguf imx8mguf imx8mpguf - # The id of the gitlab project used in the rules section - # to not run pipelines in forked projects - # Using variable here, to allow override in other projects - # including this file - MANIFEST_PROJECT_ID: 1725 - # In the manifest, the remotes are specified by an identifier - # This is used to find out included projects for the retrigger - # job. In costum manifests, the remote may be named differently - # so we need a variable that may be overriden. - CI_PARAM_SECO_REMOTE: seco-ne - workflow: rules: # Explicitly allow externally triggered pipelines in every case @@ -94,7 +70,7 @@ generate-build-jobs: # that the template has access to all GitLab CI variables. Hence there is no need # to explicitly pass any of them as command line arguments. - .gitlab-ci/scripts/generate_job_from_template.py - --template=.gitlab-ci/build-jobs.jinja2 + --template=.gitlab-ci/${BUILD_JOBS_TEMPLATE} > build-jobs.yml artifacts: expire_in: 4 weeks @@ -121,7 +97,10 @@ yamllint: build:merge_request: extends: .infrastructure stage: build - timeout: 1h + # For some reason we cannot use a variable for the timeout, the CI linter reports + # 'jobs:build:merge_request:timeout config should be a duration' then. + # Hence we use GitLab's 'reference' syntax instead. + timeout: !reference [variables, BUILD_TIMEOUT] rules: - if: $CI_COMMIT_REF_NAME == $MASTER_BRANCH_MANIFEST && $CI_PIPELINE_SOURCE != "api" script: