.gitlab-ci.yml

@@ -38,12 +38,6 @@ workflow:
       when: never
   timeout: 2m
 
-pylint:
-  extends: .analyze
-  script:
-    - cd scripts
-    - pylint --rcfile=pylintrc *.py
-
 black:
   extends: .analyze
   script:
@@ -56,6 +50,18 @@ executable:
     - cd scripts
     - (! find ! -executable -name "*.py" -exec echo "not executable:"" {}" \; | grep .)
 
+isort:
+  extends: .analyze
+  script:
+    - cd scripts
+    - isort . --check --diff
+
+pylint:
+  extends: .analyze
+  script:
+    - cd scripts
+    - pylint --rcfile=pylintrc *.py
+
 yamllint:
   extends: .analyze
   script:

build-common.yml

@@ -5,6 +5,9 @@ workflow:
     # the parent job seem to be used and prevent the pipeline generation
     - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
 
+# --------------------------------------------------------------------------------------
+# Scripts
+# --------------------------------------------------------------------------------------
 .docker_check:
   # Check if the build folder is empty. Sometimes the docker volume for the build is not
   # removed afterwards (e.g. in case of a timeout), then a follow-up build might fail.
@@ -40,7 +43,7 @@ workflow:
   - echo "$GITLAB_KNOWN_HOSTS" >> ~/.ssh/known_hosts
   - chmod 644 ~/.ssh/known_hosts
 
-.repo_checkout: &repo_checkout
+.repo_checkout:
   - echo "${LOGPREFIX} Perform repo checkout"
   - cd ${CI_PROJECT_DIR}
   - repo init --submodules -u ${CI_REPOSITORY_URL}
@@ -48,6 +51,9 @@ workflow:
   - repo sync --detach --current-branch --force-remove-dirty
       --optimized-fetch --force-sync
 
+# --------------------------------------------------------------------------------------
+# Stage: build
+# --------------------------------------------------------------------------------------
 .buildbase:
   tags:
     - builds
@@ -67,6 +73,9 @@ workflow:
   artifacts:
     expire_in: 4 weeks
 
+# --------------------------------------------------------------------------------------
+# Stage: test
+# --------------------------------------------------------------------------------------
 .test:
   extends:
     - .infrastructure

build-jobs-ci-test.yml.jinja2

@@ -35,7 +35,6 @@ build:files:
   stage: build
   extends:
     - .buildbase
-  needs: []
   tags:
     - infrastructure
   timeout: 2m
@@ -54,7 +53,6 @@ build:files:
 
 build:echo:
   stage: build
-  needs: []
   tags:
     - infrastructure
   timeout: 2m
@@ -67,7 +65,6 @@ build:check-foo-branch:
   stage: build
   extends:
     - .buildbase
-  needs: []
   tags:
     - infrastructure
   timeout: 2m
@@ -85,51 +82,48 @@ build:check-foo-branch:
       exit 1
     fi
 
-build-imx6guf-fake:
+.simulate-build:
   stage: build
-  needs: []
+  extends:
+    - .buildbase
   tags:
-    - deploy
-  timeout: 20m
-  image: ${CI_IMAGE_PYTHON}
+    - infrastructure
+  timeout: 60m
   rules:
     - when: manual
       allow_failure: true
   variables:
-    MANIFEST_VERSION: Yocto-dunfell-17.0
-    CI_PARAM_MACHINE: imx6guf
-    CI_PARAM_DISTRO: guf-wayland
-    CI_PARAM_IMAGE: guf-image
-    BUILDPATH: "build-${CI_PARAM_DISTRO}-${CI_PARAM_MACHINE}"
-    IMAGEBASEPATH: "tmp/deploy/images/"
-    IMAGEPATH: "${IMAGEBASEPATH}/${CI_PARAM_MACHINE}"
-    LICENSESPATH: "tmp/deploy/licenses"
-    SDKPATH: "tmp/deploy/sdk/"
-    INSTALLSCRIPT: "fng-install.sh"
-    JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/"
-    FNG_INSTALL_PATH: "${BUILDPATH}/${IMAGEPATH}/${INSTALLSCRIPT}"
-
+    BUILD_ARTIFACTS:
   script:
-    # TODO: this becomes similar to the yocto build and may be merged
-    - echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env
-    - echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env
-    - echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env
-    - echo "MANIFEST_VERSION=$MANIFEST_VERSION" >> build.env
-    - echo "BUILD_PATH_IMAGE=${BUILDPATH}/${IMAGEPATH}" >> build.env
-    - echo "BUILD_PATH_SDK=${BUILDPATH}/${SDKPATH}" >> build.env
-    - echo "BUILD_PATH_LICENSE=${BUILDPATH}/${LICENSESPATH}" >> build.env
-    - echo "FNG_INSTALL_URL=${JOB_URL}${FNG_INSTALL_PATH}" >> build.env
-    - source build.env
-
-    - echo "Getting yocto build from Z:"
-    - mkdir -p $BUILD_PATH_IMAGE
-    - cp -v /artifacts-yocto/Releases/$MANIFEST_VERSION/$CI_PARAM_MACHINE/*.* $BUILD_PATH_IMAGE
-    - ls $BUILD_PATH_IMAGE
+    - echo "Getting Yocto build artifacts"
+    - wget -O artifacts.zip ${BUILD_ARTIFACTS}
+    - unzip artifacts.zip
   artifacts:
     reports:
       dotenv: build.env
+
+simulate-build-seco-mx6:
+  extends:
+    - .simulate-build
+  variables:
+    # We have to specify a tag here instead of getting the artifacts from a master
+    # branch, because we don't always execute the full pipeline on the master branches.
+    # In those cases the actual build job does not exist, which results in a 404 error.
+    BUILD_ARTIFACTS: https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/3.0/download?job=build-seco-mx6
+  artifacts:
+    paths:
+      - build-*/tmp/deploy/images/**/*
+      - build-*/tmp/deploy/licenses/**/license.manifest
+      - build.env
+
+simulate-buildsdk-seco-mx6:
+  extends:
+    - .simulate-build
+  variables:
+    BUILD_ARTIFACTS: https://git.seco.com/seco-ne/yocto/manifest/-/jobs/artifacts/kirkstone/3.0/download?job=buildsdk-seco-mx6
+  artifacts:
     paths:
-      - build-guf-wayland-imx6guf/tmp/deploy/images/imx6guf/*
+      - build-*/tmp/deploy/sdk/*
       - build.env
 
 # --------------------------------------------------------------------------------------
@@ -158,15 +152,14 @@ test:fail:
   variables:
     RETURNCODE: 1
 
-smoketest-imx6guf:
+smoketest:seco-mx6:
   extends:
     - .test
   stage: test
-  needs: [build-imx6guf-fake]
+  needs: [simulate-build-seco-mx6]
   variables:
     TEST_REPO_BRANCH: dunfell
     CI_PARAM_PLATFORMS: imx6guf
     CI_PARAM_EXTRA: --all-devices
     CI_PARAM_TEST_SUITE: boot.jinja2
-    CI_PARAM_BUILDJOB: build-imx6guf-fake
-
+    CI_PARAM_BUILDJOB: simulate-build-seco-mx6

build-yocto.yml

 ---
 
+.networkdebug: &networkdebug
+  #
+  - |-
+    set -x
+    ip address show || true
+    ping  -4 -c 2 www.google.com || true
+    ping  -6 -c 2 www.google.com || true
+    ping  -4 -c 1 git.seco.com || true # does not reply but shows the ip
+    set +x
+
 .collect_srcrevs: &collect_srcrevs
   # write all package AUTOREVS to file
   - |-
@@ -45,13 +55,6 @@
   # setup build environment
   - echo "${LOGPREFIX} Build configuration MACHINE=${CI_PARAM_MACHINE}
       DISTRO=${CI_PARAM_DISTRO} IMAGE=${CI_PARAM_IMAGE}"
-  - echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env
-  - echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env
-  - echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env
-  - echo "BUILD_PATH_IMAGE=${BUILDPATH}/${IMAGEPATH}" >> build.env
-  - echo "BUILD_PATH_SDK=${BUILDPATH}/${SDKPATH}" >> build.env
-  - echo "BUILD_PATH_LICENSE=${BUILDPATH}/${LICENSESPATH}" >> build.env
-  - echo "FNG_INSTALL_URL=${JOB_URL}${FNG_INSTALL_PATH}" >> build.env
   - source build.env
   - echo "${LOGPREFIX} Using build dir ${BUILDPATH}"
   - export MACHINE="${CI_PARAM_MACHINE}"
@@ -64,6 +67,15 @@
   - bitbake "${CI_PARAM_IMAGE}" -c "${BITBAKE_TASK}"
   - echo -e "section_end:`date +%s`:bitbake_run\r\e[0K"
 
+.save_build_env: &save_build_env
+  - echo "BUILD_MACHINE=$CI_PARAM_MACHINE" > build.env
+  - echo "BUILD_IMAGE=$CI_PARAM_IMAGE" >> build.env
+  - echo "BUILD_DISTRO=$CI_PARAM_DISTRO" >> build.env
+  - echo "BUILD_PATH_IMAGE=${BUILDPATH}/${IMAGEPATH}" >> build.env
+  - echo "BUILD_PATH_SDK=${BUILDPATH}/${SDKPATH}" >> build.env
+  - echo "BUILD_PATH_LICENSE=${BUILDPATH}/${LICENSESPATH}" >> build.env
+  - echo "FNG_INSTALL_URL=${JOB_URL}${FNG_INSTALL_PATH}" >> build.env
+
 .buildimage:
   extends:
     - .buildbase
@@ -81,10 +93,12 @@
     JOB_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/"
     FNG_INSTALL_PATH: "${BUILDPATH}/${IMAGEPATH}/${INSTALLSCRIPT}"
   before_script:
+    - *networkdebug
     - !reference [.docker_check]
     - !reference [.setup_ssh]
     - !reference [.repo_checkout]
   script:
+    - *save_build_env
     - *build_script
     - *collect_srcrevs
     - *dump_install_command

scripts/.isort.cfg

+[settings]
+include_trailing_comma=true
+line_length=88
+profile=black

scripts/accept_merge_request.py

 #!/usr/bin/env python3
-import common
-
 import argparse
 import logging
 import sys
 import time
-from gitlab import (
-    Gitlab,
-    GitlabGetError,
-    GitlabMRClosedError,
-)
+
+from gitlab import Gitlab, GitlabGetError, GitlabMRClosedError
+
+import common
 
 critical_error = (
     "This is a critical error! Please make sure to:\n"
@@ -72,7 +69,7 @@ def accept_merge_request(project, mr, rebase=False, should_remove_source_branch=
                     # No pipeline created yet
                     print("No pipeline created yet")
                     time.sleep(1)
-                elif mr.head_pipeline["status"] in common.pending_states:
+                elif mr.head_pipeline["status"] in common.PENDING_STATES:
                     # Pipeline pending
                     if not pipeline_pending:
                         print("Waiting for pending pipeline", end="", flush=True)
@@ -116,7 +113,7 @@ def main():
         "--gitlab-url",
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
-        required=True,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",

scripts/alphaplan_fwr.py

 #!/usr/bin/env python3
 import argparse
+import glob
 import json
 import os
 import shutil
 import sys
+
 import requests
-import glob
+
 from alphaplan_keys import ApKeys, ApSubKeys, get_ap_dict
 
 

scripts/analyze_job_logs.py

+#!/usr/bin/env python3
+"""
+
+Query job logs by state, tags and string in the log.
+Output the found logs in csv format to stdout.
+
+"""
+
+
+import argparse
+import logging
+import sys
+
+import gitlab as gl
+
+__author__ = "Jonas Höppner"
+__email__ = "jonas.hoeppner@garz-fricke.com"
+
+GITLAB_SERVER = "https://git.seco.com"
+# ID of the guf_yocto group
+GITLAB_GROUP_ID = "556"
+
+DISTRO_PROJECT_ID = "1748"
+MACHINE_PROJECT_ID = "2074"
+MANIFEST_PROJECT_ID = "1725"
+
+DEFAULTBRANCH = "dunfell"
+
+GITLAB_TIMEFORMAT = "%Y-%m-%dT%H:%M:%S.%f%z"
+TIMEFORMAT = "%Y-%m-%d %H:%M"
+
+verbose = 0
+
+
+def main(args):
+    parser = argparse.ArgumentParser(description=__doc__, usage="%(prog)s [OPTIONS]")
+
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        action="store",
+        default=GITLAB_SERVER,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--project",
+        help="""GitLab project where the jobs are queried from""",
+        dest="project",
+        default="seco-ne/yocto/manifest",
+    )
+
+    parser.add_argument(
+        "--search-pattern",
+        help="""Search the logs for the pattern""",
+        dest="pattern",
+        required=True,
+    )
+    parser.add_argument(
+        "--filter-tag", help="""Only jobs with this gitlab job tag set""", default=None
+    )
+    parser.add_argument(
+        "--filter-status", help="""Only jobs with this this status""", default=None
+    )
+    parser.add_argument(
+        "--limit", help="""Quit after this many found jobs.""", default=None
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        dest="verbose",
+        default=0,
+        help=("Increase verbosity."),
+    )
+
+    options = parser.parse_args(args)
+    if options.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    logging.debug(options)
+    gitlab = gl.Gitlab(options.gitlab_url, private_token=options.token)
+
+    project = gitlab.projects.get(options.project)
+
+    jobs = []
+    job_it = project.jobs.list(iterator=True)
+    for job in job_it:
+        if options.filter_status is not None and job.status != options.filter_status:
+            continue
+        if options.filter_tag is not None:
+            if not options.filter_tag in job.tag_list:
+                continue
+        log = bytes.decode(job.trace())
+        if options.pattern in log:
+            logging.debug(
+                "Found %s: %s %s. Total %d",
+                job.name,
+                job.status,
+                str(job.tag_list),
+                len(jobs),
+            )
+            jobs.append(job)
+            print(
+                "  {}, {:10}, {:40}, {}".format(
+                    job.finished_at, job.id, job.name, job.web_url
+                )
+            )
+        if options.limit is not None and (len(jobs) >= options.limit):
+            break
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

scripts/changelog_generator.py

@@ -18,6 +18,7 @@ import argparse
 import datetime
 import logging
 import sys
+
 import gitlab as gl
 
 __author__ = "Jonas Höppner"

scripts/check_if_integration_branch_is_up_to_date.py

 #!/usr/bin/env python3
-import common
-
 import argparse
-import sys
 import logging
+import sys
+
 from gitlab import Gitlab, GitlabGetError
 from gitlab.v4.objects import Project
 
+import common
+
 
 def check_if_integration_branch_is_up_to_date(
     target_project: Project,
@@ -50,7 +51,7 @@ def main():
         "--gitlab-url",
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
-        required=True,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",

scripts/common.py

 #!/usr/bin/env python3
 
 import logging
-import requests
 import sys
 import time
-from colors import colors
+
+import requests
 from furl import furl
 from git import Actor, GitCommandError
 from git.repo.base import Repo
 from gitlab import GitlabAuthenticationError, GitlabGetError, GitlabMRRebaseError
-from gitlab.v4.objects import Project
-from gitlab.v4.objects import MergeRequest
+from gitlab.v4.objects import MergeRequest, Project
 
+from colors import colors
 
-manifest_file = "default.xml"
-srcrev_file = "SRCREV.conf"
-pending_states = ["created", "waiting_for_resource", "preparing", "pending", "running"]
+GITLAB_URL = "https://git.seco.com"
+MANIFEST_FILE = "default.xml"
+SRCREV_FILE = "SRCREV.conf"
+PENDING_STATES = ["created", "waiting_for_resource", "preparing", "pending", "running"]
 
 
 def integration_branch_name(project_name, source_branch_name, target_branch_name):

scripts/convert_md2html.py

 #!/usr/bin/env python3
 
 import argparse
-from markdown2 import markdown_path
 import codecs
 
+from markdown2 import markdown_path
+
 HEADER = """
 <!doctype html>
 <html lang="en">

scripts/create_merge_request.py

 #!/usr/bin/env python3
-import common
-
 import argparse
 import sys
+
 from gitlab import Gitlab, GitlabGetError
 from gitlab.v4.objects import Project
+
+import common
 from get_merge_requests import get_merge_requests
 
 
@@ -53,7 +54,7 @@ def main():
         "--gitlab-url",
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
-        required=True,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",

scripts/deploy_gitlab_ci.py

 #!/usr/bin/env python3
-import common
-
 import argparse
 import logging
-import sys
 import os
+import sys
+
 from gitlab import Gitlab
-from gitlab.v4.objects import Project, MergeRequest
+from gitlab.v4.objects import MergeRequest, Project
+from ruamel.yaml import YAML
 
+import common
 from accept_merge_request import accept_merge_request
 from create_merge_request import create_merge_request
 from get_integration_sources import get_integration_sources
 from get_merge_requests import get_merge_requests
-from update_submodule import update_submodule_and_include_ref
 from integrate_into_manifest import update_manifest, update_srcrev
-
-from ruamel.yaml import YAML
+from update_submodule import update_submodule_and_include_ref
 
 
 def read_keys_from_gitlab_ci_yml(gitlab_ci_yml):
@@ -104,7 +103,7 @@ def main():
         "--gitlab-url",
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
-        required=True,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",
@@ -162,14 +161,14 @@ def main():
         "--manifest-file",
         help="""manifest file name (default: 'default.xml')""",
         dest="manifest_file",
-        default=common.manifest_file,
+        default=common.MANIFEST_FILE,
         required=False,
     )
     parser.add_argument(
         "--srcrev-file",
         help="""source revision file name (default: 'SRCREV.conf')""",
         dest="srcrev_file",
-        default=common.srcrev_file,
+        default=common.SRCREV_FILE,
         required=False,
     )
     parser.add_argument(

scripts/download_file_from_latest_job.py

@@ -12,14 +12,14 @@ import sys
 
 import gitlab as gl
 
+import common
+
 __author__ = "Jonas Höppner"
 __email__ = "jonas.hoeppner@garz-fricke.com"
 
 from download_job_artifacts import download_job_artifact
 from get_pipeline_jobs import get_pipeline_jobs
 
-GITLAB_SERVER = "https://git.seco.com"
-
 verbose = 0
 
 
@@ -31,7 +31,7 @@ def main(args):
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
         action="store",
-        default=GITLAB_SERVER,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",

scripts/download_job_artifacts.py

@@ -7,17 +7,18 @@ Downloads the job artifacts of a given job
 
 import argparse
 import logging
-import sys
 import os
-import zipfile
+import sys
 import tempfile
+import zipfile
+
 import gitlab as gl
 
+import common
+
 __author__ = "Jonas Höppner"
 __email__ = "jonas.hoeppner@garz-fricke.com"
 
-GITLAB_SERVER = "https://git.seco.com"
-
 verbose = 0
 
 
@@ -77,7 +78,7 @@ def main(args):
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
         action="store",
-        default=GITLAB_SERVER,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",

scripts/generate_job_from_template.py

 #!/usr/bin/env python3
 
 from __future__ import print_function
+
 import argparse
 import io
 import logging
@@ -8,6 +9,7 @@ import os
 import re
 import sys
 from copy import deepcopy
+
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from jinja2.exceptions import TemplateNotFound
 from ruamel.yaml import YAML

scripts/get_current_revision_from_manifest.py

 #!/usr/bin/env python3
-import common
-
 import argparse
 import logging
 import re
+
 from gitlab import Gitlab
 from gitlab.v4.objects import Project
-
 from lxml import etree
 
+import common
+
 
 def get_current_revision_from_manifest(
     manifest_project: Project,
     manifest_branch,
     project: Project,
     recipe_name=None,
-    srcrev_file=common.srcrev_file,
+    srcrev_file=common.SRCREV_FILE,
 ):
     """
     Reads the xml manifest an the SRC_REVC file in the given manifest project
@@ -84,7 +84,7 @@ def main():
         "--gitlab-url",
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
-        required=True,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",
@@ -114,7 +114,7 @@ def main():
         "--srcrev-file",
         help="""source revision file name (default: 'SRCREV.conf')""",
         dest="srcrev_file",
-        default=common.srcrev_file,
+        default=common.SRCREV_FILE,
         required=False,
     )
     parser.add_argument(

scripts/get_integration_sources.py

@@ -2,9 +2,12 @@
 import argparse
 import re
 import sys
+
 from gitlab import Gitlab, GitlabGetError
 from gitlab.v4.objects import Group
 
+import common
+
 
 def get_integration_sources(manifest_project: str, manifest_branch: str, group: Group):
     """
@@ -61,7 +64,7 @@ def main():
         "--gitlab-url",
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
-        required=True,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",

scripts/get_merge_requests.py

 #!/usr/bin/env python3
-import common
-
 import argparse
 import sys
+
 from gitlab import Gitlab, GitlabGetError
 
+import common
+
 
 def get_merge_requests(
     project, state=None, source_branch=None, target_branch=None, commit=None
@@ -46,7 +47,7 @@ def main():
         "--gitlab-url",
         help="""URL to the GitLab instance""",
         dest="gitlab_url",
-        required=True,
+        default=common.GITLAB_URL,
     )
     parser.add_argument(
         "--token",