From 36cb477a73f980a8e0c71f7c4c089a8655d51244 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20H=C3=B6ppner?= <jonas.hoeppner@garz-fricke.com>
Date: Thu, 16 Sep 2021 16:19:31 +0200
Subject: [PATCH] submit_test.py: Add option to submit tests for all devices of
a type
Uses submit_all instead of submit.
Some code cleanup also.
---
submit_test.py | 181 ++++++++++++++++++++++++++++++++-----------------
1 file changed, 120 insertions(+), 61 deletions(-)
diff --git a/submit_test.py b/submit_test.py
index d6ea30aa..e723b805 100755
--- a/submit_test.py
+++ b/submit_test.py
@@ -13,6 +13,30 @@ import junitparser
TESTS_GIT_URL = "git@gitlab.com:garz-fricke/tests/development-tests.git"
+def call(cmd, stdout=None):
+ logging.debug("Call: %s", cmd)
+ try:
+ if stdout is None:
+ result = subprocess.run(cmd, capture_output=True, check=True)
+ else:
+ result = subprocess.run(cmd, stdout=stdout, check=True)
+ except subprocess.CalledProcessError as e:
+ out = ""
+ err = ""
+ if e.stdout is not None:
+ out = e.stdout.decode()
+ if e.stderr is not None:
+ err = e.stderr.decode()
+ logging.error("Command failed %s: %s %s", cmd, out, err)
+ exit(1)
+ if result is not None and result.stdout is not None:
+ res = result.stdout.decode().strip()
+ logging.debug("Command returned: %s", res)
+ else:
+ res = ""
+ return res
+
+
def main():
parser = argparse.ArgumentParser()
@@ -48,8 +72,8 @@ def main():
)
parser.add_argument(
"--checkout-path",
- help="""Subfolder where the test repository is checked out into.""",
- default="tests",
+ help="""When set the script expects the tests to be checkout out in the given directory.""",
+ default=None,
dest="checkout_path",
)
parser.add_argument(
@@ -59,8 +83,15 @@ def main():
dest="results_path",
)
parser.add_argument(
- "--test-suite",
- help="""Name pattern of the test suite to use, file needs to be
+ "--all-devices",
+ help="""Submit the test to all devices of a given device type, platforms parameter is treated as devicetype in this case.""",
+ default=False,
+ action="store_true",
+ dest="all_devices",
+ )
+ parser.add_argument(
+ "--test-plan",
+ help="""Name pattern of the test plan to use, file needs to be
found in the 'tests' subfolder of the test repo and may use
{platform} to be replaced by the actual platform name.""",
default="{platform}.jinja2",
@@ -72,7 +103,7 @@ def main():
parser.add_argument(
"platforms",
help="""Platforms to submit the tests for.""",
- nargs="*",
+ nargs="+",
)
args, _ = parser.parse_known_args()
@@ -82,50 +113,77 @@ def main():
else:
logging.basicConfig(level=40)
- if os.path.exists(args.checkout_path):
- shutil.rmtree(args.checkout_path)
+ if args.checkout_path is None:
+ checkout_path = "test"
+ if os.path.exists(checkout_path):
+ shutil.rmtree(checkout_path)
+
+ logging.debug("Cloning %s", args.testrepo)
+ try:
+ git.Repo.clone_from(args.testrepo, checkout_path)
+ except git.GitError as error:
+ print("Failed to clone the test repo:", error)
+ sys.exit(1)
+ else:
+ checkout_path = args.checkout_path
+ logging.debug("Using test repo at %s", checkout_path)
+
if os.path.exists(args.results_path):
shutil.rmtree(args.results_path)
os.mkdir(args.results_path)
- logging.debug("Cloning %s", args.testrepo)
- try:
- git.Repo.clone_from(args.testrepo, args.checkout_path)
- except git.GitError as error:
- print("Failed to clone the test repo:", error)
- sys.exit(1)
-
jobs = {}
+ cmd_submit = os.path.join(checkout_path, "scripts", "submit.py")
+ cmd_query = os.path.join(checkout_path, "scripts", "query.py")
+ cmd_submitall = os.path.join(checkout_path, "scripts", "submit_all.py")
+ cmd_generate = os.path.join(checkout_path, "scripts", "generate_lava_job.py")
+
+ logging.debug("Test suite %s", args.test_suite)
+
for platform in args.platforms:
- cmd = [
- os.path.join(args.checkout_path, "scripts", "generate_lava_job.py"),
- "--test-plan",
- os.path.join(
- args.checkout_path, "tests", args.test_suite.format(platform=platform)
- ),
- "--test-repo",
- args.testrepo,
- "--fng-install",
- args.fnginstall,
- "--name",
- args.name.format(platform=platform),
- ]
- logging.debug(cmd)
- jobfile = "results/{}.yaml".format(platform)
- with open(jobfile, "w") as jobfile_handle:
- subprocess.run(cmd, stdout=jobfile_handle, check=True)
- cmd = ["tests/scripts/submit.py", jobfile]
- logging.debug(cmd)
+
+ test_suite = os.path.join(
+ checkout_path, "tests", args.test_suite.format(platform=platform)
+ )
+ logging.debug("Test suite %s", test_suite)
+
+ if os.path.splitext(test_suite)[1] == ".jinja2":
+ cmd = [
+ cmd_generate,
+ "--test-plan",
+ test_suite,
+ "--test-repo",
+ args.testrepo,
+ "--fng-install",
+ args.fnginstall,
+ "--name",
+ args.name.format(platform=platform),
+ ]
+ logging.debug("Generate job: %s", cmd)
+ jobfile = os.path.join(args.results_path, "{}.yaml".format(platform))
+ with open(jobfile, "w") as jobfile_handle:
+ call(cmd, stdout=jobfile_handle)
+ else:
+ jobfile = test_suite
+
+ if args.all_devices:
+ cmd = [cmd_submitall, "--device-type", platform, jobfile]
+ else:
+ cmd = [cmd_submit, jobfile]
+
if args.dry:
- print("Skipping submit because of dry run")
+ print("Skipping submit because of dry run: %s", cmd)
continue
- result = subprocess.run(cmd, capture_output=True, check=True)
- url = result.stdout.decode().strip().replace("\n", "")
- print("Started testjob {}: {}".format(platform, url))
- jobid = url.split("/")[-1]
- jobs[jobid] = [platform, ""]
- logging.debug(jobs)
+ result = call(cmd)
+
+ for line in result.splitlines():
+ url = line.strip().replace("\n", "")
+ print("Started testjob {}: {}".format(platform, url))
+ jobid = url.split("/")[-1]
+ jobs[jobid] = [platform, ""]
+
+ logging.debug("Queued jobs: %s", jobs)
# Wait for the results
pending = True
@@ -134,13 +192,8 @@ def main():
print(".", end="", flush=True)
pending = False
for jobid in jobs:
- cmd = ["tests/scripts/query.py", jobid]
- logging.debug(cmd)
- result = subprocess.run(cmd, capture_output=True, check=True)
- result = result.stdout.decode().strip().replace("\n", "")
- logging.debug(result)
+ result = call([cmd_query, jobid]).replace("\n", "")
jobs[jobid][1] = result
- logging.debug(jobs)
if result == "Pending":
pending = True
@@ -151,24 +204,30 @@ def main():
jobplatform = jobs[jobid][0]
jobresult = jobs[jobid][1]
- cmd = ["tests/scripts/query.py", "--get-results", jobid]
- logging.debug(cmd)
- resultfile = "results/results-{}-{}.yaml".format(jobid, jobplatform)
+ # Get results as yaml
+ resultfile = os.path.join(
+ args.results_path, "results-{}-{}.yaml".format(jobid, jobplatform)
+ )
with open(resultfile, "w") as resultfile_handle:
- subprocess.run(cmd, stdout=resultfile_handle, check=True)
-
- cmd = [
- "tests/scripts/query.py",
- "--get-results",
- "--result-format",
- "junit",
- jobid,
- ]
- logging.debug(cmd)
- resultfile = "results/results-{}-{}.xml".format(jobid, jobplatform)
+ call([cmd_query, "--get-results", jobid], stdout=resultfile_handle)
+
+ # Get results as junit xml
+ resultfile = os.path.join(
+ args.results_path, "results-{}-{}.xml".format(jobid, jobplatform)
+ )
with open(resultfile, "w") as resultfile_handle:
- subprocess.run(cmd, stdout=resultfile_handle, check=True)
+ call(
+ [
+ cmd_query,
+ "--get-results",
+ "--result-format",
+ "junit",
+ jobid,
+ ],
+ stdout=resultfile_handle,
+ )
+ # Read the number of errors from the results
resultsxml = junitparser.JUnitXml.fromfile(resultfile)
joberrors = resultsxml.errors
--
GitLab