Skip to content
Snippets Groups Projects
Commit 0ccb04f6 authored by Andrii Sosiuk's avatar Andrii Sosiuk
Browse files

[BACKUP] Cleanup unused classes and extract common GitLab functions 

  - Removed unused classes from backup.py that were likely
  left over during development.
  - Moved shared GitLab-related functions to common.py for
  better reusability and code organization.
parent 48de066c
No related branches found
No related tags found
1 merge request!448YT-380: Use Batch Processing in gitlab_backup.py
Hide whitespace changes
Inline Side-by-side
scripts/common.py
+ 17
− 0
View file @ 0ccb04f6
......@@ -82,6 +82,23 @@ def get_project(gitlab, project_name):
return project
def get_projects_from_group(gitlab, base_group):
"""Recurse through all subgroups and create a flat list of all projects"""
p_list = [
gitlab.projects.get(group_project.id)
for group_project in base_group.projects.list(
as_list=False, retry_transient_errors=True
)
]
for subgroup in base_group.subgroups.list(
as_list=False, retry_transient_errors=True
):
group = gitlab.groups.get(subgroup.id)
p_list += get_projects_from_group(gitlab, group)
return p_list
def get_latest_commit(project, branch_name):
"""Get latest commit on a given project branch"""
try:
......
scripts/gitlab_backup.py
+ 2
− 174
View file @ 0ccb04f6
......@@ -5,9 +5,7 @@ Backup using gitlab project exports for all projects in a given group
"""
import argparse
import datetime
import logging
import os
import sys
......@@ -15,7 +13,7 @@ import time
import gitlab as gl
import common
from common import GITLAB_URL, get_projects_from_group
__author__ = "Jonas Höppner"
__email__ = "jonas.hoeppner@garz-fricke.com"
......@@ -23,175 +21,6 @@ __email__ = "jonas.hoeppner@garz-fricke.com"
# ID of the Seco Northern Europe
GITLAB_GROUP_ID = "556"
GITLAB_TIMEFORMAT = "%Y-%m-%dT%H:%M:%S.%f%z"
TIMEFORMAT = "%Y-%m-%d %H:%M"
verbose = 0
def decode_timestamp(t):
return datetime.datetime.strptime(t, GITLAB_TIMEFORMAT)
class Project:
def __init__(self, project):
self.project = project
def __str__(self):
return f"## Project {self.project.name}\n"
def withlink(self):
return "\n\n## Project [{self.project.name}]({self.project.web_url})\n"
def __eq__(self, p):
return self.project.id == p.project.id if p else False
class Tag:
def __init__(self, tag):
self.name = tag.name
self.message = tag.message
self.commit = tag.commit
"""
The tags timestamp is a little more complicated it normally points
to the tagged commit's timestamps. But the merge happens later.
To handle this, the relelated mergerequest is found by comparing the
sha's and also take the merged_at timestamp.
"""
self.timestamp = decode_timestamp(tag.commit["created_at"])
"""
The mr which introduced the taged commit
as gitlab-python does not support the V5 API yet
this is added later when traversing the mrs anyway
with V5 Api: https://docs.gitlab.com/ee/api/commits.html#list-merge-requests-associated-with-a-commit
"""
self.mergerequest = None
logging.debug(f"{self.name} -- {self.commit['id']}")
def __str__(self):
return f"{self.name} {self.timestamp.strftime(TIMEFORMAT)}"
def add_mergerequest(self, m):
if self.mergerequest:
return
if m.mr.sha == self.commit["id"]:
self.mergerequest = m
# Update timestamp
# The tag points to the commit, but the merge of the merge request may has happend later
# as the commit, so the merged_at date is relevant. Otherwise the tagged commit and may be
# more end up in the wrong release
new_timestamp = decode_timestamp(self.mergerequest.mr.merged_at)
logging.debug(f"Found matching merge request for {self}")
logging.debug(f" - {self.timestamp.strftime(TIMEFORMAT)}")
logging.debug(f" - {new_timestamp.strftime(TIMEFORMAT)}")
self.timestamp = new_timestamp
def header(self):
return (
f"\n\n\n# Release {self.name}"
f"\n\nreleased at {self.timestamp.strftime(TIMEFORMAT)}\n\n"
)
class DummyTag:
def __init__(
self, name, message, date=datetime.datetime.now(tz=datetime.timezone.utc)
):
self.name = name
self.message = message
self.timestamp = date
def header(self):
return f"\n\n\n# {self.name}\n\n"
def add_mergerequest(self, m):
# Needed as interface but does nothing
pass
class Release:
"""Store some release data"""
def __init__(self, tag):
self.tag = tag
self.mergerequests = []
def add_mergerequest(self, m):
# Check if this merge_request is related to the tag
self.tag.add_mergerequest(m)
# Adds a mergerequest to the project, but uses some filtering
# Ignore automated merge requests
if m.mr.author["username"] == "guf-gitbot":
return False
if m.mr.author["username"] == "gitbot":
return False
# With the movement to git.seco.com the MRs owned by
# the guf-gitbot have been transfered to tobias
# As it is not possible to change the owner back
# to gitbot we need an extra filter here on the
# branch name
if m.mr.source_branch.startswith("integrate/"):
return False
# Timestamp is not in this release
if self.tag.timestamp < m.timestamp:
return False
# Remove duplicates, don't print the same title
# twice in the same project and release
if any(
a.mr.title == m.mr.title and a.project == m.project
for a in self.mergerequests
):
return True
self.mergerequests.append(m)
return True
def header(self):
return self.tag.header()
def description(self):
m = self.tag.message
return m or ""
def __str__(self):
return self.tag.name
class MergeRequest:
def __init__(self, mr, p):
self.mr = mr
self.project = p
self.timestamp = decode_timestamp(self.mr.merged_at)
logging.debug("\nMergeRequest:")
logging.debug(mr)
def __str__(self):
return self.mr.title
def withlink(self):
return f"{self.mr.title} [{self.mr.reference}]({self.mr.web_url})"
def get_projects_from_group(gitlab, base_group):
"""Recurse through all subgroups and create a flat list of all projects"""
p_list = [
gitlab.projects.get(group_project.id)
for group_project in base_group.projects.list(
as_list=False, retry_transient_errors=True
)
]
for subgroup in base_group.subgroups.list(
as_list=False, retry_transient_errors=True
):
group = gitlab.groups.get(subgroup.id)
p_list += get_projects_from_group(gitlab, group)
return p_list
def check_project_is_shared(project, base_group):
"""Check if project is shared with base_group"""
......@@ -212,7 +41,7 @@ def main(args):
help="""URL to the GitLab instance""",
dest="gitlab_url",
action="store",
default=common.GITLAB_URL,
default=GITLAB_URL,
)
parser.add_argument(
"--token",
......@@ -236,7 +65,6 @@ def main(args):
default="./export",
help=("Specify the path where the exported projects are stored."),
)
parser.add_argument(
"-v",
"--verbose",
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment