Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • seco-ne/yocto/infrastructure/gitlab-ci
1 result
Show changes
Commits on Source (4)
......@@ -21,7 +21,6 @@ stages:
- Deploy FTP
- Deploy Azure
- Confluence
- Alphaplan
variables:
MASTER_BRANCH: {{ MASTER_BRANCH }}
......@@ -358,24 +357,3 @@ publish-confluence-page:
- .confluence-rules
{% endif %}
# --------------------------------------------------------------------------------------
# Stage: Alphaplan
# --------------------------------------------------------------------------------------
{% for machine in MACHINES.split(' ') %}
generate-alphaplan-data-{{ machine }}:
extends: .generate_alphaplan_data
variables:
MACHINE: {{ machine }}
needs:
- deploy-{{ machine }}
- build-version
import-alphaplan-data-{{ machine }}:
extends: .import_alphaplan_data
needs:
- generate-alphaplan-data-{{ machine }}
{% endfor %}
......@@ -525,46 +525,3 @@ workflow:
--content-file=confluence-page.xml
--label="os-release"
${LABEL_ARGS}
# --------------------------------------------------------------------------------------
# Stage: Alphaplan
# --------------------------------------------------------------------------------------
.generate_alphaplan_data:
extends:
- .infrastructure
stage: Alphaplan
rules:
- if: $ALPHAPLAN_STAGE == "true"
script:
# RELEASE_NAME is available from build-version.env
- .gitlab-ci/scripts/generate_alphaplan_fwr_file.py
--machine="${MACHINE}"
--release-name="${RELEASE_NAME}"
--files-list=files.txt
--md5sums=**/md5sums.txt
artifacts:
paths:
- alphaplan-import*.json
.import_alphaplan_data:
extends:
- .infrastructure
stage: Alphaplan
rules:
- if: $ALPHAPLAN_STAGE == "true"
when: manual
allow_failure: true
variables:
# Most AP_WEBSERVICE_* variables are set in GitLab CI variables. We're defining the
# URL here, though, so that it can be overridden in a job's variables block.
AP_WEBSERVICE_URL: >
https://SRV06.hamburg.garz-fricke.de/Alphaplan-API/Artikel/CreateFirmware
script:
- echo "${AP_WEBSERVICE_CERT}" > GarzFrickeGmbH-CA.cer
- .gitlab-ci/scripts/alphaplan_fwr_import.py
--url=${AP_WEBSERVICE_URL}
--user=${AP_WEBSERVICE_USR}
--password=${AP_WEBSERVICE_PW}
--cert-file=GarzFrickeGmbH-CA.cer
--file=alphaplan-import*.json
{#-
See documentation for storage format:
https://confluence.atlassian.com/doc/confluence-storage-format-790796544.html
-#}
<ac:structured-macro ac:name="details" ac:schema-version="1" data-layout="default" ac:macro-id="4a0ac7ab071a2383a7d0b5134ec5b46b">
<ac:rich-text-body>
<table data-layout="default">
......@@ -66,6 +70,11 @@
<ac:parameter ac:name="title">Downloads</ac:parameter>
<ac:rich-text-body>
<table>
{#- see https://community.atlassian.com/t5/Confluence-questions/How-can-I-set-width-of-Confluence-Cloud-table-columns-using/qaq-p/1327248#M171010 -#}
<colgroup>
<col style="width: 250px;" />
<col style="width: 750px;" />
</colgroup>
<tbody>
<tr>
<th><b>Image</b></th>
......
......@@ -89,8 +89,6 @@ pipelines by overriding these variables in the trigger job:
Type of packaging (supported values: `image`, `sdk`)
* `TEST_STAGE`
Boolean value to set whether to include the Test stage
* `ALPHAPLAN_STAGE`
Boolean value to set whether to include the Alphaplan stage
* `CONFLUENCE_SPACE`
Confluence Space to publish the release overview in
* `CONFLUENCE_PARENT_ID`
......
......@@ -78,9 +78,6 @@ yocto-simulation-pipeline:
TEST_REPO_BRANCH: dunfell
CONFLUENCE_SPACE: RnD
CONFLUENCE_PARENT_ID: 1615560743
ALPHAPLAN_STAGE: "true"
AP_WEBSERVICE_URL: >-
https://SRV06.hamburg.garz-fricke.de/Test/Alphaplan-API/Artikel/CreateFirmware
sdk-simulation-pipeline:
extends:
......
......@@ -66,7 +66,6 @@ yocto-pipeline:
DOCUMENTATION_FILES: "*.md"
PACKAGE_TYPE: image
TEST_STAGE: "true"
ALPHAPLAN_STAGE: "true"
sdk-pipeline:
extends:
......@@ -100,4 +99,3 @@ fngsystem-pipeline:
RELEASE_NAME_EXPRESSION: FNGSystem-${DEFERRED_RELEASE_VERSION}
ARTIFACTS_PATH: build-*/tmp/deploy/images/**/*
PACKAGE_TYPE: image
ALPHAPLAN_STAGE: "true"
#!/usr/bin/env python3
import argparse
import glob
import json
import sys
import requests
def ap_send_json(jsonobj: dict, url: str, user: str, password: str, cert_file: str):
"""Sends the generated files to the Alphaplan webservice"""
try:
msg = requests.post(
url, json=jsonobj, auth=(user, password), verify=cert_file, timeout=10
)
except requests.exceptions.RequestException as e:
sys.exit(f"ERROR: {e}")
try:
msg_json = msg.json()
except json.decoder.JSONDecodeError:
sys.exit("ERROR: Did not receive a valid JSON reply from Alphaplan webservice")
if msg_json["Status"] != "Ok":
sys.exit("ERROR: AlphaPlan webservice post request failed")
print(f'AlphaPlan webservice response: {msg_json["Meldung"]}')
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--file",
help="""JSON file for Alphaplan FWR import""",
dest="file",
required=True,
)
parser.add_argument(
"--url",
help="""URL to the Alphaplan webservice""",
dest="url",
required=True,
)
parser.add_argument(
"--user",
help="""User for the Alphaplan webservice""",
dest="user",
required=True,
)
parser.add_argument(
"--password",
help="""Password for the Alphaplan webservice""",
dest="password",
required=True,
)
parser.add_argument(
"--cert-file",
help="""Certificate file for the Alphaplan webservice""",
dest="cert_file",
required=True,
)
args, _ = parser.parse_known_args()
files = glob.glob(args.file, recursive=True)
if not files:
sys.exit(f"ERROR: no file(s) matching '{args.file}' found")
print(f"Sending data to Alphaplan FWR webservice at {args.url}")
# Get files from passed glob
for filename in files:
print(f"Importing JSON file {filename}")
with open(filename, "r", encoding="utf-8") as f:
try:
json_data = json.load(f)
except json.decoder.JSONDecodeError:
sys.exit(f"ERROR: Could not parse JSON data from {filename}")
ap_send_json(
json_data,
args.url,
args.user,
args.password,
args.cert_file,
)
if __name__ == "__main__":
main()
#!/usr/bin/env python3
from enum import Flag, auto
class ApKeys(Flag):
"""Class to specify article keys"""
YOCTO_PKG_PY = auto()
YOCTO_FNG_INSTALL = auto()
YOCTO_FS = auto()
FNGSYS_INIT = auto()
FNGSYS_UPDATE = auto()
FNGSYS_FS = auto()
FNGSYS_CHECKSUM = auto()
FNGSYS_UBOOT_UPDATE = auto()
FNGSYS_UBOOT_IMAGE = auto()
FNGSYS_UBOOT_CHECKSUM = auto()
FNGSYS_UBOOT_IMAGETAR = auto()
class ApSubKeys(Flag):
"""Class to specify article subkeys"""
MATCH = auto()
MATCHCODE = auto()
BEZEICHNUNG = auto()
LANGTEXT = auto()
TYP = auto()
ATTRIBUTESET = auto()
def get_ap_dict(machine, machine_ap, release_name_local):
"""Return a customized dict with information for AlphaPlan FWR articles"""
return {
ApKeys.YOCTO_PKG_PY: {
ApSubKeys.MATCH: "pkg.py",
ApSubKeys.MATCHCODE: "FNGUpdate",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} Flash-N-Go Update general pkg.py "
"update script for nonverbose fng-install.sh",
ApSubKeys.LANGTEXT: "To be used with packages the contain an "
"fng-install.sh.\n"
"* with --nonverbose mode (new output)\n"
"* Able to to local installation with unset TFTP variable\n"
"* Handle --Paramfile",
ApSubKeys.TYP: "FNGUpdate",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.YOCTO_FNG_INSTALL: {
ApSubKeys.MATCH: "fng-install.sh",
ApSubKeys.MATCHCODE: "InstallScript",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} {release_name_local} Install Script",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "US",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.YOCTO_FS: {
ApSubKeys.MATCH: f"{machine}.tar.gz",
ApSubKeys.MATCHCODE: "OS-Filesystem",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} {release_name_local} Filesystem",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "FS",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_UPDATE: {
ApSubKeys.MATCH: "fngsystem-self-update.sh",
ApSubKeys.MATCHCODE: "TFTP",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} {release_name_local} Self Update",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "TFTP",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_INIT: {
ApSubKeys.MATCH: "fngsystem-self-init.sh",
ApSubKeys.MATCHCODE: "InstallScript",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} {release_name_local} Init Script",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "Updateskript",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_FS: {
ApSubKeys.MATCH: f"{machine}.tgz",
ApSubKeys.MATCHCODE: "FS",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} {release_name_local} Filesystem",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "FS",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_CHECKSUM: {
ApSubKeys.MATCH: f"{machine}.md5",
ApSubKeys.MATCHCODE: "TFTP",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} {release_name_local} Checksum",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "TFTP",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_UBOOT_UPDATE: {
ApSubKeys.MATCH: "fng-install-uboot.sh",
ApSubKeys.MATCHCODE: "US",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} U-Boot {release_name_local} "
"Update script",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "Updateskript",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_UBOOT_IMAGE: {
ApSubKeys.MATCH: "imx-boot",
ApSubKeys.MATCHCODE: "FS",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} U-Boot {release_name_local} "
"Bootloader Image",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "FS",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_UBOOT_IMAGETAR: {
ApSubKeys.MATCH: "imx-boot.tar.gz",
ApSubKeys.MATCHCODE: "FS",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} U-Boot {release_name_local} "
"Bootloader Image",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "FS",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
ApKeys.FNGSYS_UBOOT_CHECKSUM: {
ApSubKeys.MATCH: "imx-boot.md5",
ApSubKeys.MATCHCODE: "TFTP",
ApSubKeys.BEZEICHNUNG: f"{machine_ap} U-Boot {release_name_local} Checksum",
ApSubKeys.LANGTEXT: "",
ApSubKeys.TYP: "TFTP",
ApSubKeys.ATTRIBUTESET: "Firmware, Bestandteil eines SW-Paketes",
},
}
......@@ -92,14 +92,34 @@ def main():
if type(page) is dict:
# Update existing page
page = confluence.update_page(
page["id"], args.page_name, content, args.parent_id
page["id"], args.page_name, content, args.parent_id, full_width=False
)
print("Updated existing Confluence page:")
else:
# Create new page
page = confluence.create_page(
args.space_id, args.page_name, content, args.parent_id
args.space_id,
args.page_name,
content,
args.parent_id,
representation="storage",
type="page",
editor="v2",
full_width=False,
)
# Update it immediately with the same contents because sometimes the
# "full_width=False" argument does not work, see:
# https://community.developer.atlassian.com/t/how-to-set-fixed-width-when-creating-new-page-with-rest-call/53591
page = confluence.update_page(
page["id"],
args.page_name,
content,
args.parent_id,
representation="storage",
type="page",
full_width=False,
minor_edit=True,
)
print("Created new Confluence page:")
......
#!/usr/bin/env python3
import argparse
import requests
from atlassian import Confluence
import common
def main():
# Create parser and add possible command line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
"--confluence-url",
help="""URL to the Confluence instance""",
dest="confluence_url",
default=common.CONFLUENCE_URL,
)
parser.add_argument(
"--username",
help="""Username for the Confluence API""",
dest="username",
required=True,
)
parser.add_argument(
"--token",
help="""Token for the Confluence API""",
dest="token",
required=True,
)
parser.add_argument(
"--space-id",
help="""Confluence space ID or name""",
dest="space_id",
required=True,
)
parser.add_argument(
"--page-name",
help="""Name of the page""",
dest="page_name",
required=True,
)
# Parse command line arguments
args, _ = parser.parse_known_args()
# Connect to confluence
confluence = Confluence(
url=args.confluence_url,
username=args.username,
password=args.token,
)
try:
# Get page including its content. See for reference:
# https://community.atlassian.com/t5/Confluence-questions/how-to-get-full-content-body-html-using-confluence-REST-API/qaq-p/710532
page = confluence.get_page_by_title(
args.space_id, args.page_name, expand="body.storage"
)
except requests.exceptions.HTTPError as e:
exit(f"ERROR: {e}")
print(page["body"]["storage"]["value"])
if __name__ == "__main__":
main()
#!/usr/bin/env python3
import argparse
import fnmatch
import glob
import json
import os
import sys
from alphaplan_keys import ApKeys, ApSubKeys, get_ap_dict
def ap_id_generator():
"""Returns a unique ids for each new article"""
if not hasattr(ap_id_generator, "value"):
ap_id_generator.value = 0
ap_id_generator.value += 1
return f"Neu{ap_id_generator.value}"
def new_ap_article(
warengruppe: str,
matchcode: str,
bezeichnung: str,
langtext: str,
attributeset: str,
attribute: list[str],
stueckliste=None,
):
"""Creates a dict/list structure for a new AlphaPlan article"""
position = {
"Attribut{:02d}".format(idx + 1): attribute[idx]
for idx in range(len(attribute))
if attribute[idx]
}
data = {
"Artikel": {
"ID": ap_id_generator(),
"Warengruppe": warengruppe,
"MatchCode": matchcode,
"Bezeichnung": bezeichnung,
"Langtext": langtext,
"Attribute": {"AttributeSet": attributeset, "Position": [position]},
}
}
if stueckliste:
sub_articles = {
"Stueckliste": {"Info": "Infofeld in Stueckliste", "Position": []}
}
for article in stueckliste:
sub_articles["Stueckliste"]["Position"].append(article)
data["Artikel"].update(sub_articles)
return data
def generate_ap_subarticle(
files: list[str],
ap_key: ApKeys,
machine: str,
machine_ap: str,
release_name_ap: str,
md5sums: dict[str, str],
):
"""Create an new AlphaPlan FWR subarticle which is added to a part list"""
ap_article = None
# Generate a dict for all files that should be included in the FWR
ap_dict = get_ap_dict(machine, machine_ap, release_name_ap)
# Match files and get path and md5sum
for filepath in files:
filename = os.path.basename(filepath)
# Compare the filename with the match from ap_key
if filename.casefold().endswith(ap_dict[ap_key][ApSubKeys.MATCH].casefold()):
md5sum = md5sums[filename]
# AlphaPlan specific entries for attribute set: "Firmware, Bestandteil..."
# Field1: type, Field2: path to artifact, Field3: md5sum
attribute = [ap_dict[ap_key][ApSubKeys.TYP], filepath, md5sum]
ap_article = new_ap_article(
"FWR",
ap_dict[ap_key][ApSubKeys.MATCHCODE],
ap_dict[ap_key][ApSubKeys.BEZEICHNUNG],
ap_dict[ap_key][ApSubKeys.LANGTEXT],
ap_dict[ap_key][ApSubKeys.ATTRIBUTESET],
attribute,
)
break
if ap_article is None:
sys.exit(f"ERROR: Can not find key:{ap_key} in files")
return ap_article
def generate_fwr_articles(
release_name: str, machine: str, files: list[str], md5sums: dict[str, str]
):
"""Main function to generate the FWR articles for FNGSystem and Yocto"""
# Modify name strings for AlphaPlan
machine_ap = machine.upper()
# Old machine name was something like "imx6guf"
machine_ap = machine_ap.replace("IMX", "i.MX")
machine_ap = machine_ap.replace("GUF", "")
# New machine name is something like "seco-mx6"
machine_ap = machine_ap.replace("MX", "i.MX")
machine_ap = machine_ap.replace("SECO-", "")
release_name_ap = release_name.replace("Yocto-", "Yocto ")
attb_set_major = "Softwarepaket 32-bit"
stueckliste = []
if "fngsystem".casefold() in release_name.casefold():
# FNGSystem Release
# Set artifact keys
subarticles = [
ApKeys.FNGSYS_INIT,
ApKeys.FNGSYS_UPDATE,
ApKeys.FNGSYS_FS,
ApKeys.FNGSYS_CHECKSUM,
]
for key in subarticles:
stueckliste.append(
generate_ap_subarticle(
files,
key,
machine,
machine_ap,
release_name_ap,
md5sums,
)
)
# Define attribute fields
attribute = [None, None, "FNG-SYSTEM"]
# Generate dict/list structure
data = new_ap_article(
"FWR",
"SW-Paket",
f"{machine_ap} {release_name_ap}",
f"{release_name_ap}\n{machine_ap}",
attb_set_major,
attribute,
stueckliste=stueckliste,
)
# If imx-boot.tar.gz or deprecated imx-boot is in files (imx8mX)i
# add an uboot FWR package
subarticles_uboot = []
if fnmatch.filter(files, "*/imx-boot.tar.gz"):
subarticles_uboot = [
ApKeys.FNGSYS_UBOOT_UPDATE,
ApKeys.FNGSYS_UBOOT_IMAGETAR,
]
elif fnmatch.filter(files, "*/imx-boot"):
subarticles_uboot = [
ApKeys.FNGSYS_UBOOT_UPDATE,
ApKeys.FNGSYS_UBOOT_IMAGE,
ApKeys.FNGSYS_UBOOT_CHECKSUM,
]
if subarticles_uboot:
stueckliste_uboot = [
generate_ap_subarticle(
files,
key,
machine,
machine_ap,
release_name_ap,
md5sums,
)
for key in subarticles_uboot
]
# At the moment there are no attributes specified for uboot FWR
attribute_uboot = [None, None, "UBOOT"]
data_uboot = new_ap_article(
"FWR",
"SW-Paket",
f"{machine_ap} U-Boot {release_name_ap}",
f"{release_name_ap}\n{machine_ap}",
attb_set_major,
attribute_uboot,
stueckliste=stueckliste_uboot,
)
# Create an additional import file for uboot
jsonfile_uboot_name = f"alphaplan-import-uboot-{machine}.json"
print(
f'Saving Alphaplan data for FWR article "{machine_ap} U-Boot '
f'{release_name_ap}" to {jsonfile_uboot_name}'
)
with open(jsonfile_uboot_name, "w", encoding="utf-8") as jsonfile:
json.dump(data_uboot, jsonfile, indent=2)
else:
# Same process for a yocto release
subarticles = [ApKeys.YOCTO_FNG_INSTALL, ApKeys.YOCTO_FS]
for key in subarticles:
stueckliste.append(
generate_ap_subarticle(
files,
key,
machine,
machine_ap,
release_name_ap,
md5sums,
)
)
attribute = [None, None, None, "YOCTO"]
data = new_ap_article(
"FWR",
"SW-Paket",
f"{machine_ap} {release_name_ap}",
f"{release_name_ap}\n{machine_ap}",
attb_set_major,
attribute,
stueckliste=stueckliste,
)
jsonfile_name = f"alphaplan-import-{machine}.json"
print(
f'Saving Alphaplan data for FWR article "{machine_ap} {release_name_ap}" '
f"to {jsonfile_name}"
)
with open(jsonfile_name, "w", encoding="utf-8") as jsonfile:
json.dump(data, jsonfile, indent=2)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--release-name",
help="""Name of the release""",
dest="release_name",
required=True,
)
parser.add_argument(
"--machine",
help="""Machine the release it built for""",
dest="machine",
required=True,
)
parser.add_argument(
"--files-list",
help="""Text file containing a list of all released files""",
dest="files_list",
required=True,
)
parser.add_argument(
"--md5sums",
help="""Text file containing MD5 sums of released files""",
dest="md5sums",
required=True,
)
args, _ = parser.parse_known_args()
# Sanity checking
if not args.machine:
sys.exit("ERROR: --machine requires a non-empty argument")
if not args.release_name:
sys.exit("ERROR: --release-name requires a non-empty argument")
# Read file list
files = []
for files_file in glob.glob(args.files_list, recursive=True):
print(f"Reading files from {files_file}")
with open(files_file, "r", encoding="utf-8") as f:
files = files + f.read().splitlines()
# Read md5 sums
md5sums = {}
for md5sums_file in glob.glob(args.md5sums, recursive=True):
print(f"Reading md5sums from {md5sums_file}")
with open(md5sums_file, "r", encoding="utf-8") as f:
for line in f:
# Assuming line format: "<md5sum> <filename>\n"
name = line.split(" ")[1].rstrip()
md5sum = line.split(" ")[0]
md5sums[name] = md5sum
# Generate alphaplan FWR articles
generate_fwr_articles(
args.release_name,
args.machine,
files,
md5sums,
)
if __name__ == "__main__":
main()