Skip to content
Snippets Groups Projects
Commit 38c04aef authored by Tim Jaacks's avatar Tim Jaacks
Browse files

Add new deploy pipeline to Yocto

parent e1375513
No related branches found
No related tags found
No related merge requests found
Pipeline #46498 passed with warnings
Stage: analyze
Stage: integrate
Stage: build
Pipeline: manifest

#46501

    Show whitespace changes
    Inline Side-by-side
    build-jobs-yocto.yml.jinja2
    + 219
    − 83
    View file @ 38c04aef
    ......@@ -14,17 +14,28 @@ include:
    - manifest-package.yml
    stages:
    - infrastructure
    - build
    - test
    - deploy
    - uploadftp
    - Infrastructure
    - Build
    - Test
    - Package
    - Deploy SoftwareStore
    - Deploy SoftwareStore Internal
    - Deploy FTP
    - Alphaplan
    variables:
    MASTER_BRANCH: {{ MASTER_BRANCH }}
    {% if CI_PARAM_DISTRO is not defined %}
    {% set CI_PARAM_DISTRO = "guf-wayland" %}
    {% endif %}
    {% if CI_PARAM_DISTRO_FNG is not defined %}
    {% set CI_PARAM_DISTRO_FNG = "guf-fngsystem" %}
    {% endif %}
    # --------------------------------------------------------------------------------------
    # Stage: infrastructure
    # Stage: Infrastructure
    # --------------------------------------------------------------------------------------
    changelog:
    extends: .infrastructure
    ......@@ -37,77 +48,49 @@ changelog:
    paths:
    - "changelog.md"
    # --------------------------------------------------------------------------------------
    # Generated build jobs
    # Generated jobs
    # --------------------------------------------------------------------------------------
    {% if CI_PARAM_DISTRO is not defined %}
    {% set CI_PARAM_DISTRO = "guf-wayland" %}
    {% endif %}
    {% if CI_PARAM_DISTRO_FNG is not defined %}
    {% set CI_PARAM_DISTRO_FNG = "guf-fngsystem" %}
    {% endif %}
    {% if CI_PARAM_MACHINES is defined %}
    {% if CI_PARAM_MACHINES %}
    {% for machine in CI_PARAM_MACHINES.split(' ') %}
    # --------------------------------------------------------------------------------------
    # Stage: Build
    # --------------------------------------------------------------------------------------
    {% if CI_PARAM_IMAGE %}
    # Build jobs for the normal yocto image
    build-{{ machine }}:
    extends: .buildimage
    stage: build
    extends: .build_yocto_image
    variables:
    CI_PARAM_MACHINE: {{ machine }}
    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
    INSTALLSCRIPT: "fng-install.sh"
    artifacts:
    paths:
    - build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/images/{{ machine }}/*
    - build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/licenses/**/license.manifest
    - build.env
    # Build jobs for the sdk
    buildsdk-{{ machine }}:
    extends: .buildsdk
    stage: build
    extends: .build_yocto_sdk
    variables:
    CI_PARAM_MACHINE: {{ machine }}
    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO }}
    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE }}
    artifacts:
    paths:
    - build-{{ CI_PARAM_DISTRO }}-{{ machine }}/tmp/deploy/sdk/*
    - build.env
    # Deploy jobs for the yocto image
    deployimage-{{ machine }}:
    extends: .deployimage
    stage: deploy
    needs: [build-{{ machine }}, changelog]
    # Deploy jobs for the yocto sdk
    deploysdk-{{ machine }}:
    extends: .deployimage
    stage: deploy
    needs: [buildsdk-{{ machine }}]
    # Upload ftp jobs for the yocto image
    uploadftp-{{ machine }}:
    extends:
    - .uploadftp
    stage: uploadftp
    needs: [build-{{ machine }}, changelog]
    {% endif %}
    # Upload ftp jobs for the yocto sdk
    uploadftp-sdk-{{ machine }}:
    extends:
    - .uploadftp
    stage: uploadftp
    needs: [ buildsdk-{{ machine }}]
    {% if CI_PARAM_IMAGE_FNG %}
    # Build jobs for the fng system image
    build-{{ machine }}-fngsystem:
    extends: .build_yocto_image
    variables:
    CI_PARAM_MACHINE: {{ machine }}
    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }}
    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }}
    INSTALLSCRIPT: "fngsystem-self-update.sh"
    {% endif %}
    # --------------------------------------------------------------------------------------
    # Stage: Test
    # --------------------------------------------------------------------------------------
    # Run platform tests for this machine which the yocto image
    # This is a little hacky as we need to match the machine name to
    # the available platforms
    ......@@ -125,7 +108,7 @@ uploadftp-sdk-{{ machine }}:
    {% set lavamachine = '' %}
    {% endif %}
    {% if platforms %}
    {% if CI_PARAM_IMAGE && platforms%}
    # Run smoketests for this machine which the yocto image
    smoketest:{{ machine }}:
    extends:
    ......@@ -152,39 +135,192 @@ platformtest:{{ machine }}:
    CI_PARAM_PLATFORMS: {{ platforms }}
    {% endif %}
    {% endif %} # if CI_PARAM_IMAGE is defined
    # --------------------------------------------------------------------------------------
    # Stage: Package
    # --------------------------------------------------------------------------------------
    {% if CI_PARAM_IMAGE %}
    package-{{ machine }}:
    extends: .package_release
    variables:
    ASSOCIATED_BUILD_JOB: build-{{ machine }}
    needs:
    - job: build-{{ machine }}
    artifacts: false
    packagesdk-{{ machine }}:
    extends: .package_sdk
    variables:
    ASSOCIATED_BUILD_JOB: buildsdk-{{ machine }}
    needs:
    - job: buildsdk-{{ machine }}
    artifacts: false
    {% endif %}
    {% if CI_PARAM_IMAGE_FNG %}
    package-{{ machine }}-fngsystem:
    extends: .package_release
    variables:
    ASSOCIATED_BUILD_JOB: build-{{ machine }}-fngsystem
    needs:
    - job: build-{{ machine }}-fngsystem
    artifacts: false
    {% endif %}
    # Build jobs for the fng system image
    build-{{ machine }}-fngsystem:
    extends: .buildimage
    stage: build
    # --------------------------------------------------------------------------------------
    # Stage: Deploy SoftwareStore
    # --------------------------------------------------------------------------------------
    {% if CI_PARAM_IMAGE %}
    .deploy_software_store:
    extends: .deploy
    {% if CI_COMMIT_TAG %}
    stage: Deploy SoftwareStore
    variables:
    CI_PARAM_MACHINE: {{ machine }}
    CI_PARAM_DISTRO: {{ CI_PARAM_DISTRO_FNG }}
    CI_PARAM_IMAGE: {{ CI_PARAM_IMAGE_FNG }}
    INSTALLSCRIPT: "fngsystem-self-update.sh"
    artifacts:
    paths:
    - build-{{ CI_PARAM_DISTRO_FNG }}-{{ machine }}/tmp/deploy/images/{{ machine }}/*
    - build-{{ CI_PARAM_DISTRO_FNG }}-{{ machine }}/tmp/deploy/licenses/**/license.manifest
    - build.env
    DEPLOY_SOURCE: release/$${RELEASE_NAME}
    DEPLOY_TARGET: /artifacts-yocto/Releases/$${RELEASE_NAME}
    DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Linux-Yocto/Releases/$${RELEASE_NAME}
    {% else %}
    stage: Deploy SoftwareStore Internal
    variables:
    DEPLOY_SOURCE: release/$${RELEASE_NAME}
    DEPLOY_TARGET: /artifacts-yocto/Interne_Releases/$${RELEASE_NAME}
    DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Linux-Yocto/Interne_Releases/$${RELEASE_NAME}
    {% endif %}
    # Deploy jobs for the fngsystem image
    deployimage-{{ machine }}-fngsystem:
    extends: .deployimage
    stage: deploy
    needs: [build-{{ machine }}-fngsystem, changelog]
    deploy-{{ machine }}:
    extends: .deploy_software_store
    variables:
    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
    needs:
    - job: package-{{ machine }}
    artifacts: false
    - job: changelog
    # Upload ftp jobs for the fngsystem image
    uploadftp-{{ machine }}-fngsystem:
    extends: .uploadftp
    stage: uploadftp
    needs: [build-{{ machine }}-fngsystem, changelog]
    deploysdk-{{ machine }}:
    extends: .deploy_software_store
    variables:
    ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
    needs:
    - job: packagesdk-{{ machine }}
    artifacts: false
    - job: changelog
    {% endif %}
    {% endif %} # if CI_PARAM_IMAGE_FNG is defined
    {% if CI_PARAM_IMAGE_FNG %}
    .deploy_software_store_fng_system:
    extends: .deploy
    {% if CI_COMMIT_TAG %}
    stage: Deploy SoftwareStore
    variables:
    DEPLOY_SOURCE: release/$${RELEASE_NAME}
    DEPLOY_TARGET: /artifacts-fngsystem/$${RELEASE_NAME}
    DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
    {% else %}
    stage: Deploy SoftwareStore Internal
    variables:
    DEPLOY_SOURCE: release/$${RELEASE_NAME}
    DEPLOY_TARGET: /artifacts-fngsystem/CI_Builds/$${RELEASE_NAME}
    DEPLOY_TARGET_LINK: Z:/Development/SoftwareStore/Flash-N-Go/FNGSystem/CI_Builds/$${RELEASE_NAME}
    {% endif %}
    {% endfor %}
    deploy-{{ machine }}-fngsystem:
    extends: .deploy_software_store_fng_system
    variables:
    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
    needs:
    - job: package-{{ machine }}-fngsystem
    artifacts: false
    - job: changelog
    {% endif %}
    # --------------------------------------------------------------------------------------
    # Stage: Alphaplan
    # --------------------------------------------------------------------------------------
    {% if CI_PARAM_IMAGE %}
    generate-alphaplan-data-{{ machine }}:
    extends: .generate_alphaplan_data
    needs:
    - deploy-{{ machine }}
    import-alphaplan-data-{{ machine }}:
    extends: .import_alphaplan_data
    needs:
    - generate-alphaplan-data-{{ machine }}
    {% endif %}
    {% if CI_PARAM_IMAGE_FNG %}
    generate-alphaplan-data-{{ machine }}-fngsystem:
    extends: .generate_alphaplan_data
    needs:
    - deploy-{{ machine }}-fngsystem
    import-alphaplan-data-{{ machine }}-fngsystem:
    extends: .import_alphaplan_data
    needs:
    - generate-alphaplan-data-{{ machine }}-fngsystem
    {% endif %}
    # --------------------------------------------------------------------------------------
    # Stage: Deploy FTP
    # --------------------------------------------------------------------------------------
    {% if CI_PARAM_IMAGE %}
    .deploy_ftp:
    extends: .deploy
    stage: Deploy FTP
    variables:
    DEPLOY_SOURCE: release/$${RELEASE_NAME}
    DEPLOY_TARGET: /artifacts-ftp-yocto/Releases/$${RELEASE_NAME}
    DEPLOY_TARGET_LINK: http://support.garz-fricke.com/projects/Linux-Yocto/Releases/$${RELEASE_NAME}
    ftp-{{ machine }}:
    extends: .deploy_ftp
    variables:
    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}
    needs:
    - job: package-{{ machine }}
    artifacts: false
    - job: changelog
    ftpsdk-{{ machine }}:
    extends: .deploy_ftp
    variables:
    ASSOCIATED_PACKAGE_JOB: packagesdk-{{ machine }}
    needs:
    - job: packagesdk-{{ machine }}
    artifacts: false
    - job: changelog
    {% endif %}
    # FIXME: This is a workaround to prevent customer releases being uploaded to our
    # public FTP area. It should be removed as soon as we support uploading to different
    # FTP target folders.
    {% if not HIDE_FTP_UPLOAD_STAGE %}
    {% if CI_PARAM_IMAGE_FNG %}
    .deploy_ftp_fng_system:
    extends: .deploy
    stage: Deploy FTP
    variables:
    DEPLOY_SOURCE: release/$${RELEASE_NAME}
    DEPLOY_TARGET: /artifacts-ftp-fngsystem/$${RELEASE_NAME}
    DEPLOY_TARGET_LINK: http://support.garz-fricke.com/projects/Flash-N-Go/FNGSystem/$${RELEASE_NAME}
    ftp-{{ machine }}-fngsystem:
    extends: .deploy_ftp_fng_system
    variables:
    ASSOCIATED_PACKAGE_JOB: package-{{ machine }}-fngsystem
    needs:
    - job: package-{{ machine }}-fngsystem
    artifacts: false
    - job: changelog
    {% endif %}
    {% endif %}
    # --------------------------------------------------------------------------------------
    # End of generated jobs
    # --------------------------------------------------------------------------------------
    {% endfor %}
    {% endif %} # if CI_PARAM_MACHINES
    build-yocto.yml
    + 16
    − 52
    View file @ 38c04aef
    ---
    # --------------------------------------------------------------------------------------
    # Stage: Build
    # --------------------------------------------------------------------------------------
    .collect_srcrevs: &collect_srcrevs
    # write all package AUTOREVS to file
    - |-
    ......@@ -66,14 +69,13 @@
    - echo "BUILD_PATH_LICENSE=${BUILDPATH}/${LICENSESPATH}" >> build.env
    - echo "FNG_INSTALL_URL=${JOB_URL}${FNG_INSTALL_PATH}" >> build.env
    .buildimage:
    .build_yocto:
    extends:
    - .buildbase
    needs: []
    variables:
    GIT_STRATEGY: none
    SETUPSCRIPT: "setup-environment"
    BITBAKE_TASK: "build"
    LOGPREFIX: "CI:build:"
    BUILDPATH: "build-${CI_PARAM_DISTRO}-${CI_PARAM_MACHINE}"
    IMAGEBASEPATH: "tmp/deploy/images/"
    ......@@ -86,65 +88,27 @@
    - !reference [.docker_check]
    - !reference [.setup_ssh]
    - !reference [.repo_checkout]
    .build_yocto_image:
    extends:
    - .build_yocto
    - .build_image
    variables:
    BITBAKE_TASK: "build"
    script:
    - *save_build_env
    - *build_script
    - *collect_srcrevs
    - *dump_install_command
    artifacts:
    reports:
    dotenv: build.env
    # --------------------------------------------------------------------------------------
    # Stage: build
    # --------------------------------------------------------------------------------------
    .buildsdk:
    .build_yocto_sdk:
    extends:
    - .buildimage
    stage: build
    rules:
    - when: manual
    allow_failure: true
    script:
    - *build_script
    - .build_yocto
    - .build_sdk
    variables:
    BITBAKE_TASK: "populate_sdk"
    # --------------------------------------------------------------------------------------
    # Stage: deploy
    # --------------------------------------------------------------------------------------
    .deployimage:
    extends:
    - .package
    tags:
    - deploy
    rules:
    - when: manual
    allow_failure: true
    artifacts:
    paths:
    - release/**/**/*
    reports:
    dotenv: package.env
    timeout: 30m
    # --------------------------------------------------------------------------------------
    # Stage: uploadftp
    # --------------------------------------------------------------------------------------
    .uploadftp:
    variables:
    CI_PARAM_PACKAGE_FTP: "true"
    extends:
    - .package
    tags:
    - deploy
    rules:
    # FIXME: This is a workaround to prevent customer releases being uploaded to our
    # public FTP area. It should be removed as soon as we support uploading to different
    # FTP target folders.
    - if: $HIDE_FTP_UPLOAD_STAGE
    when: never
    - if: $CI_COMMIT_TAG
    when: manual
    allow_failure: true
    timeout: 30m
    script:
    - *build_script
    common.yml
    + 1
    − 1
    View file @ 38c04aef
    ......@@ -18,7 +18,7 @@ variables:
    # Common infrastructure settings
    # --------------------------------------------------------------------------------------
    .infrastructure:
    stage: infrastructure
    stage: Infrastructure
    tags:
    - infrastructure
    timeout: 10m
    ......
    manifest-package.yml deleted 100644 → 0
    + 0
    − 109
    View file @ e1375513
    ---
    .package:
    variables:
    CI_PARAM_PACKAGE_FTP: "false"
    LOGPREFIX: "CI:package:"
    script:
    - |-
    # Package release files
    # TODO: At the moment this script is used for
    # - image deployment,
    # - sdk deployment,
    # - image and sdk ftp upload
    # - and artifact deployment for automated tests.
    # Some usecases require individual processing, which leads to
    # high script complexity. That should be improved.
    echo "${LOGPREFIX} Assemble build variables"
    # Variables set by build.env from the build artifacts
    ARTIFACTS_IMAGE_PATH="$BUILD_PATH_IMAGE"
    ARTIFACTS_LICENSES_PATH="$BUILD_PATH_LICENSE"
    ARTIFACTS_SDK_PATH="$BUILD_PATH_SDK"
    if ${CI_PARAM_PACKAGE_FTP}; then
    UPLOAD_PARAM="" # don't store as gitlab artifact
    OUTDIR_BASE="/artifacts-ftp"
    else
    UPLOAD_PARAM="--output-dir=release"
    OUTDIR_BASE="/artifacts"
    fi
    # If we are on the master branch and a tag is set
    # we tread it as release
    if [ -n "$CI_COMMIT_TAG" ];then
    outdir="${OUTDIR_BASE}-yocto/Releases"
    [ "$BUILD_DISTRO" = "seconorth-fngsystem" ] && \
    outdir="${OUTDIR_BASE}-fngsystem"
    else
    outdir="${OUTDIR_BASE}-yocto/Interne_Releases"
    [ "$BUILD_DISTRO" = "seconorth-fngsystem" ] && \
    outdir="${OUTDIR_BASE}-fngsystem/CI_Builds"
    fi
    # Print vars for debugging purposes
    echo "${LOGPREFIX} ARTIFACTS_IMAGE_PATH=$ARTIFACTS_IMAGE_PATH"
    echo "${LOGPREFIX} ARTIFACTS_LICENSES_PATH=$ARTIFACTS_LICENSES_PATH"
    echo "${LOGPREFIX} ARTIFACTS_SDK_PATH=$ARTIFACTS_SDK_PATH"
    echo "${LOGPREFIX} UPLOAD_PARAM=$UPLOAD_PARAM"
    echo "${LOGPREFIX} RELEASE_SUFFIX=$RELEASE_SUFFIX"
    echo "${LOGPREFIX} outdir=$outdir"
    # Check if the package_release script is available
    script=".gitlab-ci/scripts/package_release.py"
    # Change script path if ?
    [ ! -x "${script}" ] && script=".repo/manifests/${script}"
    if [ ! -x "${script}" ]; then
    echo "${LOGPREFIX} Failed to find package_release script"
    exit 1
    fi
    # Copy image if available
    echo "${LOGPREFIX} Check if a image build exists"
    if [ -d "${ARTIFACTS_IMAGE_PATH}" ];then
    echo "${LOGPREFIX} Image dir found, execute ${script}"
    ${script} \
    --images-dir="${ARTIFACTS_IMAGE_PATH}" \
    --licenses-dir="${ARTIFACTS_LICENSES_PATH}" \
    --doc-dir=. \
    --output-dir=${outdir} \
    --release-suffix="${RELEASE_SUFFIX}" \
    $UPLOAD_PARAM
    else
    echo "${LOGPREFIX} No image found"
    fi
    # Copy SDK if available
    echo "${LOGPREFIX} Check if a SDK build exists"
    if [ -d "${ARTIFACTS_SDK_PATH}" ];then
    echo "${LOGPREFIX} SDK dir found, execute ${script}"
    ${script} \
    --sdk-dir=${ARTIFACTS_SDK_PATH} \
    --output-dir="${outdir}" \
    --release-suffix="${RELEASE_SUFFIX}"
    else
    echo "${LOGPREFIX} No SDK found"
    fi
    # Generate AlphaPlan FWR articles if
    # - a release tag is set
    # - this is an image build and
    # - the files were deployed to local network (drive Z)
    apscript=".gitlab-ci/scripts/alphaplan_fwr.py"
    echo "${LOGPREFIX} Check if AlphaPlan FWR articles need to be generated"
    if [ -n "$CI_COMMIT_TAG" ] && [ -d "${ARTIFACTS_IMAGE_PATH}" ] && \
    [[ "${CI_PARAM_PACKAGE_FTP}" == "false" ]];then
    echo "${LOGPREFIX} Generate AlphaPlan articles, execute ${apscript}"
    # Check if alphaplan fwr script is available
    if [ ! -x "${apscript}" ]; then
    echo "${LOGPREFIX} Failed to find alphaplan_fwr script"
    exit 1
    fi
    # Call script
    ${apscript} \
    --images-dir="${ARTIFACTS_IMAGE_PATH}" \
    --outputdir-local=${outdir} \
    --outputdir-upload=release \
    --release-suffix="${RELEASE_SUFFIX}"
    else
    echo "${LOGPREFIX} No AlphaPlan FWR articles need to be generated"
    fi
    scripts/alphaplan_fwr.py deleted 100755 → 0
    + 0
    − 351
    View file @ e1375513
    #!/usr/bin/env python3
    import argparse
    import glob
    import json
    import os
    import shutil
    import sys
    import requests
    from alphaplan_keys import ApKeys, ApSubKeys, get_ap_dict
    def ap_send_json(jsonobj):
    """Sends the generated files to the Alphaplan webservice"""
    url = os.environ.get("AP_WEBSERVICE_URL")
    usr = os.environ.get("AP_WEBSERVICE_USR")
    pw = os.environ.get("AP_WEBSERVICE_PW")
    cert = os.environ.get("AP_WEBSERVICE_CERT")
    # Write TLS certificate to file
    cert_file = "GarzFrickeGmbH-CA.cer"
    with open(cert_file, "w", encoding="utf-8") as f:
    f.write(cert)
    # Send data
    msg = requests.post(url, json=jsonobj, auth=(usr, pw), verify=cert_file, timeout=10)
    os.remove(cert_file)
    msg_json = msg.json()
    if msg_json["Status"] != "Ok":
    sys.exit("ERROR: AlphaPlan webservice post request failed")
    print("AlphaPlan webservice response: {}".format(msg_json["Meldung"]))
    def ap_id_generator():
    """Returns a unique ids for each new article"""
    if not hasattr(ap_id_generator, "value"):
    ap_id_generator.value = 0
    ap_id_generator.value += 1
    return "Neu{}".format(ap_id_generator.value)
    def new_ap_article(
    warengruppe,
    matchcode,
    bezeichnung,
    langtext,
    attributeset,
    attribute,
    stueckliste=None,
    ):
    """Creates a dict/list structure for a new AlphaPlan article"""
    position = {}
    for idx in range(len(attribute)):
    if attribute[idx]:
    position["Attribut{:02d}".format(idx + 1)] = attribute[idx]
    data = {
    "Artikel": {
    "ID": ap_id_generator(),
    "Warengruppe": warengruppe,
    "MatchCode": matchcode,
    "Bezeichnung": bezeichnung,
    "Langtext": langtext,
    "Attribute": {"AttributeSet": attributeset, "Position": [position]},
    }
    }
    if stueckliste:
    sub_articles = {
    "Stueckliste": {"Info": "Infofeld in Stueckliste", "Position": []}
    }
    for article in stueckliste:
    sub_articles["Stueckliste"]["Position"].append(article)
    data["Artikel"].update(sub_articles)
    return data
    def known_ap_article(artikelnummer):
    """Creates a dict for a known AlphaPlan FWR article"""
    data = {"Artikel": {"ID": artikelnummer, "Artikelnummer": artikelnummer}}
    return data
    def generate_ap_subarticle(
    artifacts, ap_key, output_dir, machine, machine_ap, release_name_ap, md5sums
    ):
    """Create an new AlphaPlan FWR subarticle which is added to a part list"""
    # Generate a dict for all artifacts that should be included in the FWR
    ap_dict = get_ap_dict(machine, machine_ap, release_name_ap)
    # Match artifacts and get path and md5sum
    for artifact in artifacts:
    # Compare the artifact with the match from ap_key
    if artifact.casefold().endswith(ap_dict[ap_key][ApSubKeys.MATCH].casefold()):
    path = os.path.join(output_dir, machine, artifact)
    path = os.path.join(*(path.split(os.path.sep)[2:]))
    # Distinguish between FNGSystem and Yocto path on drive Z:
    if "fngsystem".casefold() in output_dir.casefold():
    path = (
    "Z:\\Development\\SoftwareStore\\Flash-N-Go\\"
    "FNGSystem\\{}".format(path.replace("/", "\\"))
    )
    else:
    path = "Z:\\Development\\SoftwareStore\\Linux-Yocto\\{}".format(
    path.replace("/", "\\")
    )
    md5sum = md5sums[artifact]
    if "path" not in locals():
    sys.exit("ERROR: Can not find key:{} in artifacts".format(ap_key))
    # AlphaPlan specific entries for attribute set: "Firmware, Bestandteil..."
    # Field1: type, Field2: path to artifact, Field3: md5sum
    attribute = [ap_dict[ap_key][ApSubKeys.TYP], path, md5sum]
    return new_ap_article(
    "FWR",
    ap_dict[ap_key][ApSubKeys.MATCHCODE],
    ap_dict[ap_key][ApSubKeys.BEZEICHNUNG],
    ap_dict[ap_key][ApSubKeys.LANGTEXT],
    ap_dict[ap_key][ApSubKeys.ATTRIBUTESET],
    attribute,
    )
    def generate_fwr_articles(
    output_dir, outlocal_dir, machine, release_name_local, artifacts_all, md5sums
    ):
    """Main function to generate the FWR articles for FNGSystem and Yocto"""
    # Modify name strings for AlphaPlan
    machine_ap = machine.upper()
    machine_ap = machine_ap.replace("IMX", "i.MX")
    machine_ap = machine_ap.replace("GUF", "")
    release_name_ap = release_name_local.replace("Yocto-", "Yocto ")
    attb_set_major = "Softwarepaket 32-bit"
    stueckliste = []
    if "fngsystem".casefold() in release_name_local.casefold():
    # FNGSystem Release
    # Set artifact keys
    subarticles = [
    ApKeys.FNGSYS_INIT,
    ApKeys.FNGSYS_UPDATE,
    ApKeys.FNGSYS_FS,
    ApKeys.FNGSYS_CHECKSUM,
    ]
    for key in subarticles:
    stueckliste.append(
    generate_ap_subarticle(
    artifacts_all,
    key,
    outlocal_dir,
    machine,
    machine_ap,
    release_name_ap,
    md5sums,
    )
    )
    # Define attribute fields
    attribute = [None, None, "FNG-SYSTEM"]
    # Generate dict/list structure
    data = new_ap_article(
    "FWR",
    "SW-Paket",
    "{} {}".format(machine_ap, release_name_ap),
    "{}\n{}".format(release_name_ap, machine_ap),
    attb_set_major,
    attribute,
    stueckliste=stueckliste,
    )
    # If imx-boot.tar.gz or deprecated imx-boot is in artifacts_all (imx8mX)i
    # add an uboot FWR package
    if "imx-boot.tar.gz" in artifacts_all or "imx-boot" in artifacts_all:
    if "imx-boot.tar.gz" in artifacts_all:
    subarticles_uboot = [
    ApKeys.FNGSYS_UBOOT_UPDATE,
    ApKeys.FNGSYS_UBOOT_IMAGETAR,
    ]
    # this is not needed if the above tar.gz is used.
    elif "imx-boot" in artifacts_all:
    subarticles_uboot = [
    ApKeys.FNGSYS_UBOOT_UPDATE,
    ApKeys.FNGSYS_UBOOT_IMAGE,
    ApKeys.FNGSYS_UBOOT_CHECKSUM,
    ]
    stueckliste_uboot = []
    for key in subarticles_uboot:
    stueckliste_uboot.append(
    generate_ap_subarticle(
    artifacts_all,
    key,
    outlocal_dir,
    machine,
    machine_ap,
    release_name_ap,
    md5sums,
    )
    )
    # At the moment there are no attributes specified for uboot FWR
    attribute_uboot = [None, None, "UBOOT"]
    data_uboot = new_ap_article(
    "FWR",
    "SW-Paket",
    "{} U-Boot {}".format(machine_ap, release_name_ap),
    "{}\n{}".format(release_name_ap, machine_ap),
    attb_set_major,
    attribute_uboot,
    stueckliste=stueckliste_uboot,
    )
    else:
    # Same process for a yocto release
    subarticles = [ApKeys.YOCTO_PKG_PY, ApKeys.YOCTO_FNG_INSTALL, ApKeys.YOCTO_FS]
    for key in subarticles:
    stueckliste.append(
    generate_ap_subarticle(
    artifacts_all,
    key,
    outlocal_dir,
    machine,
    machine_ap,
    release_name_ap,
    md5sums,
    )
    )
    attribute = [None, None, None, "YOCTO"]
    data = new_ap_article(
    "FWR",
    "SW-Paket",
    "{} {}".format(machine_ap, release_name_ap),
    "{}\n{}".format(release_name_ap, machine_ap),
    attb_set_major,
    attribute,
    stueckliste=stueckliste,
    )
    # Create an additional import file for uboot
    if "data_uboot" in locals():
    print(
    "Generate {} U-Boot {} AlphaPlan FWR articles".format(
    machine_ap, release_name_ap
    )
    )
    jsonfile_uboot_name = "alphaplan-import-uboot-{}.json".format(machine)
    jsonfile_uboot_local = os.path.join(outlocal_dir, machine, jsonfile_uboot_name)
    with open(jsonfile_uboot_local, "w", encoding="utf-8") as jsonfile:
    json.dump(data_uboot, jsonfile, indent=4)
    if output_dir is not None:
    jsonfile_uboot_output = os.path.join(
    output_dir, machine, jsonfile_uboot_name
    )
    shutil.copyfile(
    jsonfile_uboot_local, jsonfile_uboot_output, follow_symlinks=True
    )
    # Send data object to AlphaPlan webservice
    ap_send_json(data_uboot)
    print("Generate {} {} AlphaPlan FWR articles".format(machine_ap, release_name_ap))
    # Generate a json file from the dict/list structure for debugging
    jsonfile_name = "alphaplan-import-{}.json".format(machine)
    jsonfile_local = os.path.join(outlocal_dir, machine, jsonfile_name)
    with open(jsonfile_local, "w", encoding="utf-8") as jsonfile:
    json.dump(data, jsonfile, indent=4)
    if output_dir is not None:
    # Copy file to release folder as the atrifacts have already been copied
    jsonfile_output = os.path.join(output_dir, machine, jsonfile_name)
    shutil.copyfile(jsonfile_local, jsonfile_output, follow_symlinks=True)
    # Send data object to AlphaPlan webservice
    ap_send_json(data)
    def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
    "--images-dir",
    help="""Yocto images directory""",
    dest="images_dir",
    )
    parser.add_argument(
    "--outputdir-upload",
    help="""Base directory name for uploaded artifacts""",
    dest="outputdir_upload",
    )
    parser.add_argument(
    "--outputdir-local",
    help="""Base directory for locally deployed artifacts, should contain absolut path.""",
    dest="outputdir_local",
    )
    parser.add_argument(
    "--release-suffix",
    help="""Suffix to append to the release folder""",
    dest="release_suffix",
    )
    args, _ = parser.parse_known_args()
    if args.outputdir_upload is None or args.outputdir_local is None:
    sys.exit("ERROR: outputdir-local and outputdir-upload needs to be specified.")
    # Get bitbake variables from testdata.json file
    testdata_files = []
    if args.images_dir is not None:
    testdata_files += glob.glob(os.path.join(args.images_dir, "*.testdata.json"))
    with open(testdata_files[0], "r", encoding="utf-8") as f:
    buildvars = json.load(f)
    machine = buildvars["MACHINE"]
    version = buildvars["DISTRO_VERSION"]
    artifacts_all = buildvars["DISTRO_RELEASE_ARTEFACTS"].split()
    artifacts_all.append("BUILD_SRCREVS.log")
    # Set release name
    if version.startswith("fngsystem"):
    release_name = version.replace("fngsystem", "FNGSystem")
    else:
    release_name = "Yocto-%s" % version
    # Append release suffix
    if args.release_suffix is not None:
    release_name = release_name + args.release_suffix
    output_dir = os.path.join(args.outputdir_upload, release_name)
    outlocal_dir = os.path.join(args.outputdir_local, release_name)
    if not os.path.isdir(outlocal_dir):
    sys.exit("ERROR: ouputdir-local does not exist")
    if not os.path.isdir(output_dir):
    sys.exit("ERROR: ouputdir-upload does not exist")
    # Get md5sums.txt
    md5sums_file = os.path.join(output_dir, machine, "md5sums.txt")
    md5sums = {}
    with open(md5sums_file, encoding="utf-8") as f:
    for line in f:
    # Assuming line format: "<md5sum> <filename>\n"
    name = line.split(" ")[1].rstrip()
    md5sum = line.split(" ")[0]
    md5sums[name] = md5sum
    # Generate alphaplan FWR articles
    generate_fwr_articles(
    output_dir,
    outlocal_dir,
    machine,
    release_name,
    artifacts_all,
    md5sums,
    )
    if __name__ == "__main__":
    main()
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Please to comment