Add job to cancel outdated pipelines in merge requests

Everytime we push a new commit to a branch, a new pipeline is created.
Often the pipeline for the previous commit has not finished yet, and
GitLab does not automatically cancel it. It consumes valuable build
time, even if we don't need the results anymore.
Adding a MR pipeline job to search for previous pipelines on the same
branch and cancel them explicitly.

manifest-integration.yml

@@ -115,6 +115,20 @@ check:
         fi;
       done <<< "$INTEGRATION"
 
+cancel-previous-pipelines:
+  extends:
+    - .infrastructure
+    - .skip-for-gitlab-ci-integrations
+  stage: manifest-integration
+  allow_failure: true
+  script:
+    - .gitlab-ci/scripts/cancel_pipelines.py
+        --gitlab-url=${CI_SERVER_URL}
+        --token=${GITBOT_TOKEN}
+        --project=${CI_PROJECT_PATH}
+        --ref=${CI_MERGE_REQUEST_REF_PATH}
+        --below-pipeline-id=${CI_PIPELINE_ID}
+
 yamllint:
   extends: .yamllint
   stage: manifest-integration

scripts/cancel_pipelines.py

+#!/usr/bin/env python3
+import argparse
+import sys
+
+from gitlab.client import Gitlab
+from gitlab.v4.objects import Project, ProjectPipeline
+
+import common
+
+
+def cancel_pipelines(
+    project: Project,
+    ref: str = "",
+    below_pipeline_id: int = sys.maxsize,
+) -> list[ProjectPipeline]:
+    """Cancel currently running pipelines.
+
+    Args:
+        project: GitLab project the pipeline belongs to
+        ref: Git ref (branch or tag) the pipeline is running on
+        below_pipeline_id: cancel only pipelines with ID below this
+
+    Returns:
+        List of cancelled pipelines
+    """
+
+    pipelines = project.pipelines.list(
+        ref=ref, status="running", retry_transient_errors=True
+    )
+
+    cancelled_pipelines = []
+    for pipeline in pipelines:
+        if pipeline.id < below_pipeline_id:
+            pipeline.cancel()
+            cancelled_pipelines.append(pipeline)
+
+    return cancelled_pipelines
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gitlab-url",
+        help="""URL to the GitLab instance""",
+        dest="gitlab_url",
+        default=common.GITLAB_URL,
+    )
+    parser.add_argument(
+        "--token",
+        help="""GitLab REST API private access token""",
+        dest="token",
+        required=True,
+    )
+    parser.add_argument(
+        "--project",
+        help="""name of the GitLab project""",
+        dest="project",
+        required=True,
+    )
+    parser.add_argument(
+        "--ref",
+        help="""Git reference (branch or tag)""",
+        dest="ref",
+        default="",
+    )
+    parser.add_argument(
+        "--below-pipeline-id",
+        help="""Cancel only pipelines with IDs lower than this""",
+        dest="below_pipeline_id",
+        type=int,
+        default=sys.maxsize,
+    )
+
+    args, _ = parser.parse_known_args()
+
+    gitlab = Gitlab(args.gitlab_url, private_token=args.token)
+    project = common.get_project(gitlab, args.project)
+
+    print(
+        "Searching for pipelines in project '%s' on ref '%s' with IDs below %d"
+        % (args.project, args.ref, args.below_pipeline_id)
+    )
+
+    cancelled_pipelines = cancel_pipelines(project, args.ref, args.below_pipeline_id)
+
+    if not cancelled_pipelines:
+        print("No running pipelines found.")
+        sys.exit(0)
+
+    print("Cancelled pipelines:")
+    for pipeline in cancelled_pipelines:
+        print(pipeline.web_url)
+
+
+if __name__ == "__main__":
+    main()