Skip to content
Snippets Groups Projects
Commit 169b2fa7 authored by Tim Jaacks's avatar Tim Jaacks
Browse files

Improve mirroring MR pipeline 

Refactor three scripts into one in order to support triggering a new
pipeline in the case of a skipped MR pipeline. This was not possible to
implement with the previous three-script design. The single scripts
were not used in other places anyway.
parent 897ba059
No related branches found
No related tags found
1 merge request!238Improve mirroring MR pipeline
Pipeline #48843 passed with stage
in 2 minutes and 18 seconds
Hide whitespace changes
Inline Side-by-side
manifest-pipeline.yml
+ 2
− 18
View file @ 169b2fa7
...@@ -113,25 +113,9 @@ build:merge_request: ...@@ -113,25 +113,9 @@ build:merge_request:
timeout: !reference [variables, BUILD_TIMEOUT] timeout: !reference [variables, BUILD_TIMEOUT]
script: script:
- cd ${CI_PROJECT_DIR} - cd ${CI_PROJECT_DIR}
# Get pipeline for merge request - .gitlab-ci/scripts/mirror_mr_pipeline.py
- MR_PIPELINE=$(.gitlab-ci/scripts/get_pipelines.py
--gitlab-url=${CI_SERVER_URL} --gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN} --token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH} --project=${CI_PROJECT_PATH}
--commit=${CI_COMMIT_SHA} --commit=${CI_COMMIT_SHA}
--ref=^${MASTER_BRANCH} || true | head -1) --ref=${MASTER_BRANCH}
# If pipeline exists, mirror its result
- if [ ! -z "${MR_PIPELINE}" ]; then
.gitlab-ci/scripts/mirror_pipeline_result.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--pipeline=${MR_PIPELINE}
# If no pipeline found, trigger a new one on the master
- else
.gitlab-ci/scripts/trigger_pipeline.py
--gitlab-url=${CI_SERVER_URL}
--token=${GITBOT_TOKEN}
--project=${CI_PROJECT_PATH}
--ref=${MASTER_BRANCH}
- fi
scripts/get_pipelines.py → scripts/mirror_mr_pipeline.py
+ 116
− 92
View file @ 169b2fa7
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse import argparse
import sys import sys
import time
from gitlab import Gitlab
from gitlab.v4.objects import Project from gitlab import GitlabCreateError
from gitlab.client import Gitlab
import common from gitlab.v4.objects import Project
import common
def get_pipelines(project: Project, commit, ref: str):
""" TERMINATED_STATES = ["success", "failed", "canceled", "skipped"]
Get all pipelines for a given commit and ref.
The ref can be negated with a preceding '^', e.g. '^master' finds pipelines on any
ref different from 'master'. def get_pipelines(project: Project, commit: str, ref: str):
""" """
Get all pipelines for a given commit and ref.
# Find pipelines for commit The ref can be negated with a preceding '^', e.g. '^master' finds pipelines on any
pipelines = project.pipelines.list(sha=commit, retry_transient_errors=True) ref different from 'master'.
"""
# Remove pipelines not matching the ref condition
if ref: # Find pipelines for commit
for p in pipelines[:]: pipelines = project.pipelines.list(sha=commit, retry_transient_errors=True)
if (
ref.startswith("^") # Remove pipelines not matching the ref condition
and p.ref == ref[1:] if ref:
or not ref.startswith("^") for p in pipelines[:]:
and p.ref != ref if (
): ref.startswith("^")
pipelines.remove(p) and p.ref == ref[1:]
or not ref.startswith("^")
if not pipelines: and p.ref != ref
raise LookupError( ):
"No pipeline for commit '%s'%s found" pipelines.remove(p)
% (commit, " on ref '%s'" % ref if ref else "")
) if not pipelines:
raise LookupError(
return pipelines "No pipeline for commit '%s'%s found"
% (commit, " on ref '%s'" % ref if ref else "")
)
def main():
parser = argparse.ArgumentParser() return pipelines
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""", def main():
dest="gitlab_url", parser = argparse.ArgumentParser()
default=common.GITLAB_URL, parser.add_argument(
) "--gitlab-url",
parser.add_argument( help="""URL to the GitLab instance""",
"--token", dest="gitlab_url",
help="""GitLab REST API private access token""", default=common.GITLAB_URL,
dest="token", )
required=True, parser.add_argument(
) "--token",
parser.add_argument( help="""GitLab REST API private access token""",
"--project", dest="token",
help="""name of the GitLab project""", required=True,
dest="project", )
required=True, parser.add_argument(
) "--project",
parser.add_argument( help="""name of the GitLab project""",
"--commit", dest="project",
help="""sha of the project commit to check""", required=True,
dest="commit", )
required=True, parser.add_argument(
) "--commit",
parser.add_argument( help="""sha of the project commit to check""",
"--ref", dest="commit",
help="""ref the pipeline ran on (can be negated with preceding '^')""", required=True,
dest="ref", )
default="", parser.add_argument(
required=False, "--ref",
) help="""ref the pipeline shall be mirrored to""",
dest="ref",
args, _ = parser.parse_known_args() default="",
required=False,
gitlab = Gitlab(args.gitlab_url, private_token=args.token) )
project = common.get_project(gitlab, args.project)
args, _ = parser.parse_known_args()
try:
pipelines = get_pipelines(project, args.commit, args.ref) gitlab = Gitlab(args.gitlab_url, private_token=args.token)
except LookupError as e: project = common.get_project(gitlab, args.project)
sys.exit(e)
# Get MR pipeline
for p in pipelines: try:
print(p.id) pipelines = get_pipelines(project, args.commit, "^" + args.ref)
pipeline = pipelines[0]
sys.exit(0) except LookupError as e:
sys.exit(str(e))
if __name__ == "__main__": # Wait for pipeline termination
main() print("Waiting for pipeline %s" % pipeline.web_url)
while pipeline.status not in TERMINATED_STATES:
print(".", end="", flush=True)
time.sleep(1)
pipeline = project.pipelines.get(pipeline.id, retry_transient_errors=True)
print("")
print("Result: %s" % pipeline.status, flush=True)
# Mirror result in sucess/failure case
if pipeline.status == "success":
sys.exit(0)
elif pipeline.status == "failed":
sys.exit(1)
# Else start a new pipeline on given ref
try:
pipeline = project.pipelines.create({"ref": args.ref})
except GitlabCreateError as e:
sys.exit("ERROR: %s" % e)
print("Created new pipeline for %s:" % args.ref)
print(pipeline.web_url)
sys.exit(0)
scripts/mirror_pipeline_result.py deleted 100755 → 0
+ 0
− 94
View file @ 897ba059
#!/usr/bin/env python3
import argparse
import os
import sys
import time
from gitlab import Gitlab, GitlabGetError
from gitlab.v4.objects import Project
import common
def mirror_pipeline_result(project: Project, pipeline_id, job=None):
"""
Get latest pipeline status for a given commit and ref.
The ref can be negated with a preceding '^', e.g. '^master' finds a pipeline on any
ref different from 'master'.
"""
pipeline = project.pipelines.get(pipeline_id, retry_transient_errors=True)
# Wait for pipeline termination
print("Waiting for pipeline %s" % pipeline.web_url)
terminated_states = ["success", "failed", "canceled", "skipped"]
while pipeline.status not in terminated_states:
print(".", end="", flush=True)
time.sleep(1)
pipeline = project.pipelines.get(pipeline.id, retry_transient_errors=True)
print("")
print("Result: %s" % pipeline.status, flush=True)
# If we are running in a job environment and the upstream status is canceled,
# explicitly cancel this job as well
if job and pipeline.status == "canceled":
time.sleep(5)
job.cancel()
# Else just reflect job status in return value
return pipeline.status
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
default=common.GITLAB_URL,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
help="""name of the GitLab project""",
dest="project",
required=True,
)
parser.add_argument(
"--pipeline",
help="""id of the pipeline to mirror the result of""",
dest="pipeline",
required=True,
)
args, _ = parser.parse_known_args()
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
project = common.get_project(gitlab, args.project)
# Check if we are running in a GitLab job context
job_project_path = os.environ.get("CI_PROJECT_PATH")
job_id = os.environ.get("CI_JOB_ID")
if job_project_path and job_id:
try:
job_project = common.get_project(gitlab, job_project_path)
job = job_project.jobs.get(id=job_id, retry_transient_errors=True)
except GitlabGetError:
print("WARNING: job %s not found in project %s" % (args.job, args.project))
job = None
status = mirror_pipeline_result(project, args.pipeline, job)
if status != "success":
sys.exit(1)
sys.exit(0)
if __name__ == "__main__":
main()
scripts/trigger_pipeline.py deleted 100755 → 0
+ 0
− 54
View file @ 897ba059
#!/usr/bin/env python3
import argparse
import sys
from gitlab import Gitlab, GitlabCreateError
import common
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--gitlab-url",
help="""URL to the GitLab instance""",
dest="gitlab_url",
default=common.GITLAB_URL,
)
parser.add_argument(
"--token",
help="""GitLab REST API private access token""",
dest="token",
required=True,
)
parser.add_argument(
"--project",
help="""name of the GitLab project""",
dest="project",
required=True,
)
parser.add_argument(
"--ref",
help="""branch or tag ref to trigger the pipeline for""",
dest="ref",
required=True,
)
args, _ = parser.parse_known_args()
gitlab = Gitlab(args.gitlab_url, private_token=args.token)
project = common.get_project(gitlab, args.project)
try:
pipeline = project.pipelines.create({"ref": args.ref})
except GitlabCreateError as e:
sys.exit("ERROR: %s" % e)
print("Created new pipeline:")
print(pipeline.web_url)
sys.exit(0)
if __name__ == "__main__":
main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment