diff --git a/build-pipeline.yml b/build-pipeline.yml index a3f62820792a5d3e97bde53a7fa6b7ae4049fcc4..38428aa07daf2fc892a18173ce8f53b413d6463a 100644 --- a/build-pipeline.yml +++ b/build-pipeline.yml @@ -19,9 +19,7 @@ workflow: .initDeployENV: &initDeployENV - | echo "Initiating deploy.env" - if [[ "$BITBAKE_TASK" == "populate_sdk" ]]; then echo "SDK_FILE_NAME=$sdk_filename" | tee -a deploy.env - else echo "DEPLOY_DATE=$DEPLOY_DATE" | tee -a deploy.env echo "BOARD=$BOARD" | tee -a deploy.env echo "IMAGE_NAME=$IMAGE_NAME" | tee -a deploy.env @@ -33,7 +31,6 @@ workflow: echo "BMAP_FILE_NAME=$bmap_name" | tee -a deploy.env echo "BUNDLE_FILE_NAME=$bundle_filename" | tee -a deploy.env echo "MD5_SUMS_FILE_NAME=$md5sums_filename" | tee -a deploy.env - fi # ----------------------------------- # Stage: build diff --git a/release-pipeline-clea.yml b/release-pipeline-clea.yml index 37d7120c7f806d6b79d168789a9f4d2c3e3a7176..15c1ba75e823143ddfc4bada64821d27c2d9758c 100644 --- a/release-pipeline-clea.yml +++ b/release-pipeline-clea.yml @@ -3,6 +3,13 @@ # Global # -------------------------------------------------------------------------------------- +default: + tags: + # - release + - infrastructure + timeout: 30m + image: $CI_IMAGE_PYTHON + include: - local: common.yml - local: build-pipeline.yml @@ -12,14 +19,24 @@ workflow: - if: $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG variables: + # Include git submodules + GIT_SUBMODULE_STRATEGY: recursive + CI_COMMIT_SOURCE: ${CI_COMMIT_SHA} + # Release vars RELEASE_VARIABLES_FILE_NAME: release_varibles.env RELEASE_PIPELINE_NAME: release-pipeline RELEASE_TEMPLATES_DIR: release_templates RELEASE_GITLAB_TEMPLATE: release_clea.jinja2 RELEASE_PAGE_FILE_NAME: release-gitlab.md + RELEASE_CONFLUENCE_TECHSRC_TEMPLATE: confluence_secone_tech_page.jinja2 + RELEASE_CONFLUENCE_TECHSRC_FILE_NAME: release-confluence-secone-techresources.xml + RELEASE_CONFLUENCE_TECHSRC_LABELS: confluence_secone_tech_labels.jinja2 + RELEASE_CONFLUENCE_TECHSRC_LABELS_FILE: techsrc-confluence-labels.txt CHANGELOG_FILE: Changelog_${SECO_REMOTE}_${CI_COMMIT_TAG}.md DEPLOY_STAGE: deploy FILTER_LIST: "TOKEN KEY" + CONFLUENCE_TECHSRC_SPACE: SECONorthTech + CONFLUENCE_TECHSRC_PARENT_ID: 2351595548 stages: - collect @@ -68,7 +85,7 @@ stages: # -------------------------------------------------------------------------------------- collect:release-information: stage: collect - extends: .infrastructure + # extends: .infrastructure cache: policy: push script: @@ -104,7 +121,8 @@ changelog:deploy: timeout: 4h tags: - azure_deploy - needs: ["changelog:generate"] + needs: + - changelog:generate script: - *azure_storage_properties - echo "Deploying changelog.." @@ -117,15 +135,15 @@ changelog:deploy: generate:gitlab-release-page: stage: prepare - extends: .infrastructure rules: - if: $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG - needs: ["collect:release-information", "changelog:deploy"] + needs: + - collect:release-information + - changelog:deploy script: - *azure_storage_properties - export CHANGELOG_FILE_NAME="https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_CONTAINER_NAME}/${SECO_REMOTE}/reports/${CHANGELOG_FILE}" - set -o allexport && source ${RELEASE_VARIABLES_FILE_NAME} && set +o allexport - - *printenv - echo "Rendering release page.." - cd .gitlab-ci/${RELEASE_TEMPLATES_DIR} - ../scripts/render_jinja2_template.py @@ -135,10 +153,37 @@ generate:gitlab-release-page: paths: - ${RELEASE_PAGE_FILE_NAME} +generate:confluence-techresources-release-page: + stage: prepare + rules: + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_COMMIT_TAG + needs: + - collect:release-information + - generate:gitlab-release-page + script: + - eval "$(grep '^BOARD=' ${RELEASE_VARIABLES_FILE_NAME})" + - export BOARD + - export RELEASE_DATE=$(date -d "${CI_PIPELINE_CREATED_AT}" +"%Y-%m-%d") + - export RELEASE_NAME="Clea OS ${CI_COMMIT_TAG}" + - export RELEASE_URL="${CI_PROJECT_URL}/-/releases/${CI_COMMIT_TAG}" + - echo "Rendering release page.." + - cd .gitlab-ci/${RELEASE_TEMPLATES_DIR} + - ../scripts/render_jinja2_template.py + --template ${RELEASE_CONFLUENCE_TECHSRC_TEMPLATE} + > ../../${RELEASE_CONFLUENCE_TECHSRC_FILE_NAME} + - ../scripts/render_jinja2_template.py + --template ${RELEASE_CONFLUENCE_TECHSRC_LABELS} + > ../../${RELEASE_CONFLUENCE_TECHSRC_LABELS_FILE} + artifacts: + paths: + - ${RELEASE_CONFLUENCE_TECHSRC_FILE_NAME} + - ${RELEASE_CONFLUENCE_TECHSRC_LABELS_FILE} + allow_failure: true + publish:gitlab-release-page: stage: publish - extends: .infrastructure - needs: ["generate:gitlab-release-page"] + needs: + - generate:gitlab-release-page script: - .gitlab-ci/scripts/create_gitlab_release.py --gitlab-url ${CI_SERVER_URL} @@ -147,3 +192,25 @@ publish:gitlab-release-page: --name "Release-${CI_COMMIT_TAG}" --tag-name ${CI_COMMIT_TAG} --description-file=${RELEASE_PAGE_FILE_NAME} + +publish:confluence-techresources-release-page: + stage: publish + needs: + - collect:release-information + - generate:confluence-techresources-release-page + - publish:gitlab-release-page + script: + - while IFS= read -r line; do + [ -n "$line" ] && LABEL_ARGS="${LABEL_ARGS} --label=$line"; + done < ${RELEASE_CONFLUENCE_TECHSRC_LABELS_FILE} + - .gitlab-ci/scripts/confluence_create_or_update_page.py + --inherit-parent-restrictions + --username="${CONFLUENCE_USERNAME}" + --token="${CONFLUENCE_TOKEN}" + --space-id="${CONFLUENCE_TECHSRC_SPACE}" + --parent-id="${CONFLUENCE_TECHSRC_PARENT_ID}" + --page-name="Clea OS ${CI_COMMIT_TAG}" + --content-file="${RELEASE_CONFLUENCE_TECHSRC_FILE_NAME}" + --label="os-release" + ${LABEL_ARGS} + allow_failure: true diff --git a/release_templates/confluence_secone_tech_labels.jinja2 b/release_templates/confluence_secone_tech_labels.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..8825b31e925db18e2048c9287e46ec9338ff0c49 --- /dev/null +++ b/release_templates/confluence_secone_tech_labels.jinja2 @@ -0,0 +1,21 @@ +{#- Board lables -#} +{%- if BOARD is defined -%} + +{%- set lbs = namespace(labels = []) -%} +{%- for board in BOARD.split(' ') | sort -%} + {%- set label = "" -%} + {%- if "e83-d18" in board -%} + {%- set label = "modular-vision" -%} + {%- elif "tanaro" in board -%} + {%- set label = "tanaro" -%} + {%- endif -%} + {%- if label != "" -%} + {%- set lbs.labels = lbs.labels + [label] -%} + {%- endif -%} +{%- endfor -%} + +{% for label in lbs.labels -%} +{{ label }} +{% endfor -%} + +{%- endif -%} diff --git a/release_templates/confluence_secone_tech_page.jinja2 b/release_templates/confluence_secone_tech_page.jinja2 new file mode 100644 index 0000000000000000000000000000000000000000..a55b34963eaf431544920f2221ddaa420f7b57eb --- /dev/null +++ b/release_templates/confluence_secone_tech_page.jinja2 @@ -0,0 +1,29 @@ +{#- + See documentation for storage format: + https://confluence.atlassian.com/doc/confluence-storage-format-790796544.html +-#} +<ac:structured-macro ac:name="details" ac:schema-version="1" data-layout="default" ac:macro-id="4a0ac7ab071a2383a7d0b5134ec5b46b"> + <ac:rich-text-body> + <table data-layout="default"> + <tbody> + <tr> + {#- + Attention: no line breaks allowed within <th>..</th>, + otherwise the field is not correctly parsed by the Page + Properties Report macro. + #} + <th><b>Version</b></th> + <td>{{ RELEASE_NAME }}</td> + </tr> + <tr> + <th><b>Release Date</b></th> + <td><time datetime="{{ RELEASE_DATE }}"/></td> + </tr> + <tr> + <th><b>Documentation</b></th> + <td><a href="{{ RELEASE_URL }}">{{ RELEASE_URL }}</a></td> + </tr> + </tbody> + </table> + </ac:rich-text-body> +</ac:structured-macro> diff --git a/release_templates/hardware_support.jinja2 b/release_templates/hardware_support.jinja2 index 4ea9d95fa6c9b789a10b7d25b2394a683093bcc3..1a0c2d9d90bd501496ae35be60dad21d165feb7e 100644 --- a/release_templates/hardware_support.jinja2 +++ b/release_templates/hardware_support.jinja2 @@ -2,10 +2,24 @@ This version of Clea OS is compatible with the following hardware: | Board |CPU Architecture |SOC | Board Documentation | | :---------------------- |:--------------- |:--------------- | :-------------------------------------- | +{% set pr = namespace(processed = []) -%} {% for board in boards -%} -{% if board.codename in BOARD and "hw_support_description" in board -%} -{% if board.hw_support_description != "" -%} +{% for machine in BOARD.strip().split(' ') -%} + {% set hyphen_count = machine.count("-") -%} + {% if hyphen_count >= 2 -%} + {% if board.codename in machine and "hw_support_description" in board and board.codename not in pr.processed -%} + {% if board.hw_support_description != "" -%} {{board.hw_support_description}} -{% endif -%} -{% endif -%} +{% set pr.processed = pr.processed + [board.codename] -%} + {% endif -%} + {% endif -%} + {% else -%} + {% if board.codename == machine and "hw_support_description" in board and board.codename not in pr.processed -%} + {% if board.hw_support_description != "" -%} +{{board.hw_support_description}} +{% set pr.processed = pr.processed + [board.codename] -%} + {% endif -%} + {% endif -%} + {% endif -%} +{% endfor -%} {% endfor -%} diff --git a/release_templates/link_references.jinja2 b/release_templates/link_references.jinja2 index c22626a069f48311fa1d9cef36783e72df69c3d7..26b0d9176a531be13f956a33c6f03519c628e196 100644 --- a/release_templates/link_references.jinja2 +++ b/release_templates/link_references.jinja2 @@ -2,7 +2,20 @@ {% set rls = namespace(release_links = []) -%} {% for board in boards -%} -{% if board.codename in BOARD -%} +{% set p = namespace(present =false) -%} +{% for brd in BOARD.strip().split(' ') -%} + {% set hyphen_count = brd.count("-") -%} + {% if hyphen_count >= 2 -%} + {% if board.codename in brd -%} + {% set p.present = true -%} + {% endif -%} + {% else -%} + {% if board.codename == brd -%} + {% set p.present = true -%} + {% endif -%} + {% endif -%} +{% endfor -%} +{% if p.present -%} {% set rls.release_links = [] -%} <details> <summary><strong>{{board.name}}</strong></summary> diff --git a/release_templates/release_clea.jinja2 b/release_templates/release_clea.jinja2 index 7e54c9775c26e63a46654384caa2c115c8d044c3..5b7b0ddeee258493399f1377cb8837bed6043fb9 100644 --- a/release_templates/release_clea.jinja2 +++ b/release_templates/release_clea.jinja2 @@ -43,12 +43,15 @@ {%- macro get_file_link(machine_distro, file_type) -%} {%- set return_link = namespace(rl = "") -%} + {%- set filename = "" -%} {%- set machine_underscore = machine_distro | replace("-", "_") -%} {#- see https://stackoverflow.com/questions/72654161/dynamically-referencing-a-variable-in-jinja2 -#} - {%- set filename = self._TemplateReference__context.resolve(filenames_mapping[file_type] ~ "_FILE_NAME_" ~ machine_underscore) -%} + {%- if self._TemplateReference__context.resolve(filenames_mapping[file_type] ~ "_FILE_NAME_" ~ machine_underscore) is defined -%} + {%- set filename = self._TemplateReference__context.resolve(filenames_mapping[file_type] ~ "_FILE_NAME_" ~ machine_underscore) -%} + {%- endif -%} {%- set link_var = self._TemplateReference__context.resolve("FILE_LINKS_" ~ machine_underscore) -%} {%- for full_link in self._TemplateReference__context.resolve("FILE_LINKS_" ~ machine_underscore).split(" ") -%} - {%- if filename in full_link -%} + {%- if filename != "" and filename in full_link -%} {%- set return_link.rl = full_link -%} {%- endif -%} {%- endfor -%} diff --git a/scripts/common.py b/scripts/common.py index bec4ae711e9c8f0218bd0888affddf75dc39ba75..ee320a48a7f9c8fc875b11e0237dd4930b4d8581 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -13,6 +13,7 @@ from gitlab.v4.objects import Project from gitlab.v4.objects import MergeRequest GITLAB_URL = "https://git.seco.com" +CONFLUENCE_URL = "https://secogroup.atlassian.net" manifest_file = "default.xml" srcrev_file = "SRCREV.conf" pending_states = ["created", "waiting_for_resource", "preparing", "pending", "running"] diff --git a/scripts/confluence_create_or_update_page.py b/scripts/confluence_create_or_update_page.py new file mode 100755 index 0000000000000000000000000000000000000000..f92cc2377acc5b1ab22668dc2b06e356865f0e1b --- /dev/null +++ b/scripts/confluence_create_or_update_page.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +import argparse +import json + +import requests +from atlassian import Confluence +from requests.auth import HTTPBasicAuth + +import common + + +def str_to_bool(argument): # Helper, converter for parsing boolean argument values + if isinstance(argument, bool): + return argument + if argument.lower() in {"true", "yes", "1"}: + return True + elif argument.lower() in {"false", "no", "0"}: + return False + else: + raise argparse.ArgumentTypeError("Boolean value expected.") + + +def main(): + # Create parser and add possible command line arguments + parser = argparse.ArgumentParser() + + parser.add_argument( + "--confluence-url", + help="""URL to the Confluence instance""", + dest="confluence_url", + default=common.CONFLUENCE_URL, + ) + parser.add_argument( + "--username", + help="""Username for the Confluence API""", + dest="username", + required=True, + ) + + parser.add_argument( + "--token", + help="""Token for the Confluence API""", + dest="token", + required=True, + ) + + parser.add_argument( + "--space-id", + help="""Confluence space ID or name""", + dest="space_id", + required=True, + ) + + parser.add_argument( + "--parent-id", + help="""ID of the parent page""", + dest="parent_id", + required=True, + ) + + parser.add_argument( + "--page-name", + help="""Name for the new page""", + dest="page_name", + required=True, + ) + + parser.add_argument( + "--content-file", + help="""File to load the content for the new page from""", + dest="content_file", + required=True, + ) + + parser.add_argument( + "--label", + help="""Label to apply to the new page""", + dest="label", + action="append", + required=False, + ) + + parser.add_argument( + "--inherit-parent-restrictions", + help="Inherit restrictions from parent page (default: false)", + dest="inherit_parent_restrictions", + type=str_to_bool, + nargs="?", + const=True, + default=False, + ) + + # Parse command line arguments + args, _ = parser.parse_known_args() + + # Read content file + try: + with open(args.content_file, "r", encoding="utf8") as content_file: + content = content_file.read() + except FileNotFoundError: + exit(f"ERROR: '{args.content_file}' not found") + + # Connect to confluence + confluence = Confluence( + url=args.confluence_url, + username=args.username, + password=args.token, + ) + + try: + # Check if page already exists + page = confluence.get_page_by_title(args.space_id, args.page_name) + + if type(page) is dict: + # Update existing page + page = confluence.update_page( + page["id"], args.page_name, content, args.parent_id, full_width=False + ) + print("Updated existing Confluence page:") + + else: + # Create new page + page = confluence.create_page( + args.space_id, + args.page_name, + content, + args.parent_id, + representation="storage", + type="page", + editor="v2", + full_width=False, + ) + # Update it immediately with the same contents because sometimes the + # "full_width=False" argument does not work, see: + # https://community.developer.atlassian.com/t/how-to-set-fixed-width-when-creating-new-page-with-rest-call/53591 + page = confluence.update_page( + page["id"], + args.page_name, + content, + args.parent_id, + representation="storage", + type="page", + full_width=False, + minor_edit=True, + ) + print("Created new Confluence page:") + + if type(page) is dict: + # Print absolute page URL + print(page["_links"]["base"] + page["_links"]["webui"]) + + # Set labels for the page + for label in args.label: + confluence.set_page_label(page["id"], label) + + if args.inherit_parent_restrictions: + # Retrieve all restrictions applied to the parent page + parent_restrictions = confluence.get_all_restrictions_for_content( + args.parent_id + ) + # The retrieved data includes restrictions/permissions as well as references to the parent page. + # Before applying it to the new page, we need to remove these old references. + # For more details on the data structure, refer to: + # https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-content-restrictions/#api-wiki-rest-api-content-id-restriction-get + + # Reset '_links' and '_expandable' fields from pointing to the parent page + restrictions_request = {"results": []} + + for key, value in parent_restrictions.items(): + if key == "_links" or key == "_expandable": + continue + if "_links" in value.keys(): + value["_links"] = {} + if "_expandable" in value.keys(): + value["_expandable"] = {} + + restrictions_request["results"].append(value) + + # Directly making this request, as the current Confluence module lacks methods for setting restrictions + + response = requests.request( + "PUT", + f'{args.confluence_url}/wiki/rest/api/content/{page["id"]}/restriction', + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + }, + auth=HTTPBasicAuth(args.username, args.token), + data=json.dumps(restrictions_request), + ) + if response.ok: + print("Page restrictions have been updated.") + else: + print("Could not update page restrictions !") + exit(f"ERROR: {response.text}") + + except requests.exceptions.HTTPError as e: + exit(f"ERROR: {e}") + + +if __name__ == "__main__": + main()