CI: Move changelog_generator to gitlab-ci project

The script can also be run in the normal python container, so this
slightly reduces the complexity.

changelog_generator.py

+#!/usr/bin/env python3
+""" 
+
+Simple changelog generator for Garz&Fricke gitlab projects.
+
+Queries merge request from gitlab and outputs as sorted list in
+markdown format.
+
+Releases are tags on the given branch in the manifest project.
+Changes to list are merged mergerequests with the given branch
+as target. Match with releases is done by the timestamp (merged_at)
+in comparison with the tags timestamp.
+
+"""
+
+
+import sys
+import datetime
+import gitlab as gl
+import argparse
+
+__author__ = "Jonas Höppner"
+__email__ = "jonas.hoeppner@garz-fricke.com"
+
+GITLAB_SERVER = "https://git.seco.com"
+# ID of the guf_yocto group
+GITLAB_GROUP_ID = "556"
+
+DISTRO_PROJECT_ID = "1748"
+MACHINE_PROJECT_ID = "1747"
+MANIFEST_PROJECT_ID = "1725"
+
+DEFAULTBRANCH = "dunfell"
+
+GITLAB_TIMEFORMAT = "%Y-%m-%dT%H:%M:%S.%f%z"
+TIMEFORMAT = "%Y-%m-%d %H:%M"
+TIMEFORMAT_DEBUG = "%Y-%m-%d %H:%M.%f %z"
+
+verbose = 0
+
+
+def print_dbg(*args, **kwargs):
+    if verbose < 1:
+        return
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def decode_timestamp(t):
+    timestamp = datetime.datetime.strptime(t, GITLAB_TIMEFORMAT)
+    return timestamp
+
+
+class Project:
+    def __init__(self, project):
+        self.project = project
+
+    def __str__(self):
+        return "## Project " + self.project.name + "\n"
+
+    def withlink(self):
+        return (
+            "\n\n## Project [" + self.project.name + "](" + self.project.web_url + ")\n"
+        )
+
+    def __eq__(self, p):
+        if not p:
+            return False
+        return self.project.id == p.project.id
+
+
+class Tag:
+    def __init__(self, tag):
+        self.name = tag.name
+        self.message = tag.message
+        self.commit = tag.commit
+        """
+        The tags timestamp is a little more complicated it normally points 
+        to the tagged commit's timestamps. But the merge happens later.
+        To handle this, the relelated mergerequest is found by comparing the
+        sha's and also take the merged_at timestamp.
+        """
+        self.timestamp = decode_timestamp(tag.commit["created_at"])
+
+        """
+        The mr which introduced the taged commit
+        as gitlab-python does not support the V5 API yet
+        this is added later when traversing the mrs anyway
+        with V5 Api: https://docs.gitlab.com/ee/api/commits.html#list-merge-requests-associated-with-a-commit
+        """
+        self.mergerequest = None
+        print_dbg(self.name + "  -- " + self.commit["id"])
+
+    def __str__(self):
+        return self.name + " " + self.timestamp.strftime(TIMEFORMAT)
+
+    def add_mergerequest(self, m):
+        if self.mergerequest:
+            return
+        if m.mr.sha == self.commit["id"]:
+            self.mergerequest = m
+            # Update timestamp
+            # The tag points to the commit, but the merge of the merge request may has happend later
+            # as the commit, so the merged_at date is relevant. Otherwise the tagged commit and may be
+            # more end up in the wrong release
+            new_timestamp = decode_timestamp(self.mergerequest.mr.merged_at)
+            print_dbg("Found matching merge request for ", self)
+            print_dbg(" - " + self.timestamp.strftime(TIMEFORMAT))
+            print_dbg(" - " + new_timestamp.strftime(TIMEFORMAT))
+            self.timestamp = new_timestamp
+
+    def header(self):
+        return (
+            "\n\n\n# Release "
+            + self.name
+            + "\n\nreleased at "
+            + self.timestamp.strftime(TIMEFORMAT)
+            + "\n\n"
+        )
+
+
+class DummyTag:
+    def __init__(
+        self, name, message, date=datetime.datetime.now(tz=datetime.timezone.utc)
+    ):
+        self.name = name
+        self.message = message
+        self.timestamp = date
+
+    def header(self):
+        return "\n\n\n# " + self.name + "\n\n"
+
+    def add_mergerequest(self, m):
+        # Needed as interface but does nothing
+        pass
+
+
+class Release:
+    """Store some release data"""
+
+    def __init__(self, tag):
+        self.tag = tag
+        self.mergerequests = []
+
+    def add_mergerequest(self, m):
+        # Check if this merge_request is related to the tag
+        self.tag.add_mergerequest(m)
+
+        # Adds a mergerequest to the project, but uses some filtering
+        # Ignore automated merge requests
+        if m.mr.author["username"] == "guf-gitbot":
+            return False
+        if m.mr.author["username"] == "gitbot":
+            return False
+        # With the movement to git.seco.com the MRs owned by
+        # the guf-gitbot have been transfered to tobias
+        # As it is not possible to change the owner back
+        # to gitbot we need an extra filter here on the
+        # branch name
+        if m.mr.source_branch.startswith('integrate/'):
+            return False
+
+
+        # Timestamp is not in this release
+        if self.tag.timestamp < m.timestamp:
+            return False
+
+        # Remove duplicates, don't print the same title
+        # twice in the same project and release
+        if any(
+            a.mr.title == m.mr.title and a.project == m.project
+            for a in self.mergerequests
+        ):
+            return True
+
+        self.mergerequests.append(m)
+        return True
+
+    def header(self):
+        return self.tag.header()
+
+    def description(self):
+        m = self.tag.message
+        if not m:
+            return ""
+        return m
+
+    def __str__(self):
+        return self.tag.name
+
+
+class MergeRequest:
+    def __init__(self, mr, p):
+        self.mr = mr
+        self.project = p
+        self.timestamp = decode_timestamp(self.mr.merged_at)
+        print_dbg("\nMergeRequest:")
+        print_dbg(mr)
+
+    def __str__(self):
+        return self.mr.title
+
+    def withlink(self):
+        out = self.mr.title + " [" + self.mr.reference + "](" + self.mr.web_url + ")"
+        if verbose > 1:
+            out += " " + self.timestamp.strftime(TIMEFORMAT)
+        return out
+
+
+def main(args):
+    global verbose
+    global TIMEFORMAT
+
+    parser = argparse.ArgumentParser(description=__doc__, usage="%(prog)s [OPTIONS]")
+
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        action="store",
+        default=GITLAB_SERVER,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "-b",
+        "--branch",
+        action="store",
+        dest="branch",
+        default=DEFAULTBRANCH,
+        help=("Specify the branch to work on, default is dunfell."),
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        dest="verbose",
+        default=0,
+        help=("Increase verbosity."),
+    )
+
+    options = parser.parse_args(args)
+    verbose = options.verbose
+    if verbose > 1:
+        TIMEFORMAT = TIMEFORMAT_DEBUG
+
+    print_dbg(options)
+    gitlab = gl.Gitlab(options.gitlab_url, private_token=options.token)
+
+    # Speed up, complete project lookup takes much longer
+    # then specifying the ID directly
+    distro = Project(gitlab.projects.get(DISTRO_PROJECT_ID))
+    machine = Project(gitlab.projects.get(MACHINE_PROJECT_ID))
+    manifest = Project(gitlab.projects.get(MANIFEST_PROJECT_ID))
+
+    releases = []
+    for t in manifest.project.tags.list(search=options.branch):
+        releases.append(Release(Tag(t)))
+
+    # Add dummy release with date today for new untaged commits
+    releases.append(
+        Release(
+            DummyTag(
+                "Not yet released",
+                "Merged Request already merged into "
+                + options.branch
+                + " but not yet released.",
+            )
+        )
+    )
+
+    # Sort by date, oldest first
+    releases = sorted(releases, key=lambda d: d.tag.timestamp, reverse=False)
+
+    for p in [manifest, distro, machine]:
+        for mr in p.project.mergerequests.list(
+            scope="all", state="merged", target_branch=options.branch, per_page="10000"
+        ):
+            m = MergeRequest(mr, p)
+            for r in releases:
+                if r.add_mergerequest(m):
+                    break
+
+    # Sort by date, newest first
+    releases = sorted(releases, key=lambda d: d.tag.timestamp, reverse=True)
+
+    for r in releases:
+        # Don't show empty releases/tags
+        if not len(r.mergerequests):
+            continue
+
+        print(r.header())
+        print(r.description())
+
+        current_project = None
+        for m in r.mergerequests:
+            if m.project != current_project:
+                current_project = m.project
+                print(current_project.withlink())
+            print(" - ", m.withlink())
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

manifest.yml

@@ -83,14 +83,7 @@ changelog:
   extends: .infrastructure
   rules:
     - if: $CI_COMMIT_REF_NAME != $MASTER_BRANCH_MANIFEST || $CI_PIPELINE_SOURCE == "api"
-  variables:
-    IMAGE_PATH: ${CI_IMAGES_BASEPATH}/changelog-generator
-    IMAGE_REVISION: fe31a5ffe75b8f0dca697dd95ffb3f87af92d6d1
-  image:
-    name: "${IMAGE_PATH}:${IMAGE_REVISION}"
-    # set entrypoint to noop to be able to run from script
-    entrypoint: [""]
-  script: changelog_generator.py
+  script: .gitlab-ci/changelog_generator.py
               --token=${GITBOT_TOKEN}
               --branch ${MASTER_BRANCH_MANIFEST}
               > changelog.md