submit_test.py: Add option to submit tests for all devices of a type

Uses submit_all instead of submit.
Some code cleanup also.

submit_test.py

@@ -13,6 +13,30 @@ import junitparser
 TESTS_GIT_URL = "git@gitlab.com:garz-fricke/tests/development-tests.git"
 
 
+def call(cmd, stdout=None):
+    logging.debug("Call: %s", cmd)
+    try:
+        if stdout is None:
+            result = subprocess.run(cmd, capture_output=True, check=True)
+        else:
+            result = subprocess.run(cmd, stdout=stdout, check=True)
+    except subprocess.CalledProcessError as e:
+        out = ""
+        err = ""
+        if e.stdout is not None:
+            out = e.stdout.decode()
+        if e.stderr is not None:
+            err = e.stderr.decode()
+        logging.error("Command failed %s: %s %s", cmd, out, err)
+        exit(1)
+    if result is not None and result.stdout is not None:
+        res = result.stdout.decode().strip()
+        logging.debug("Command returned: %s", res)
+    else:
+        res = ""
+    return res
+
+
 def main():
 
     parser = argparse.ArgumentParser()
@@ -48,8 +72,8 @@ def main():
     )
     parser.add_argument(
         "--checkout-path",
-        help="""Subfolder where the test repository is checked out into.""",
-        default="tests",
+        help="""When set the script expects the tests to be checkout out in the given directory.""",
+        default=None,
         dest="checkout_path",
     )
     parser.add_argument(
@@ -59,8 +83,15 @@ def main():
         dest="results_path",
     )
     parser.add_argument(
-        "--test-suite",
-        help="""Name pattern of the test suite to use, file needs to be 
+        "--all-devices",
+        help="""Submit the test to all devices of a given device type, platforms parameter is treated as devicetype in this case.""",
+        default=False,
+        action="store_true",
+        dest="all_devices",
+    )
+    parser.add_argument(
+        "--test-plan",
+        help="""Name pattern of the test plan to use, file needs to be 
                 found in the 'tests' subfolder of the test repo and may use
                 {platform} to be replaced by the actual platform name.""",
         default="{platform}.jinja2",
@@ -72,7 +103,7 @@ def main():
     parser.add_argument(
         "platforms",
         help="""Platforms to submit the tests for.""",
-        nargs="*",
+        nargs="+",
     )
 
     args, _ = parser.parse_known_args()
@@ -82,50 +113,77 @@ def main():
     else:
         logging.basicConfig(level=40)
 
-    if os.path.exists(args.checkout_path):
-        shutil.rmtree(args.checkout_path)
+    if args.checkout_path is None:
+        checkout_path = "test"
+        if os.path.exists(checkout_path):
+            shutil.rmtree(checkout_path)
+
+        logging.debug("Cloning %s", args.testrepo)
+        try:
+            git.Repo.clone_from(args.testrepo, checkout_path)
+        except git.GitError as error:
+            print("Failed to clone the test repo:", error)
+            sys.exit(1)
+    else:
+        checkout_path = args.checkout_path
+        logging.debug("Using test repo at %s", checkout_path)
+
     if os.path.exists(args.results_path):
         shutil.rmtree(args.results_path)
     os.mkdir(args.results_path)
 
-    logging.debug("Cloning %s", args.testrepo)
-    try:
-        git.Repo.clone_from(args.testrepo, args.checkout_path)
-    except git.GitError as error:
-        print("Failed to clone the test repo:", error)
-        sys.exit(1)
-
     jobs = {}
 
+    cmd_submit = os.path.join(checkout_path, "scripts", "submit.py")
+    cmd_query = os.path.join(checkout_path, "scripts", "query.py")
+    cmd_submitall = os.path.join(checkout_path, "scripts", "submit_all.py")
+    cmd_generate = os.path.join(checkout_path, "scripts", "generate_lava_job.py")
+
+    logging.debug("Test suite %s", args.test_suite)
+
     for platform in args.platforms:
-        cmd = [
-            os.path.join(args.checkout_path, "scripts", "generate_lava_job.py"),
-            "--test-plan",
-            os.path.join(
-                args.checkout_path, "tests", args.test_suite.format(platform=platform)
-            ),
-            "--test-repo",
-            args.testrepo,
-            "--fng-install",
-            args.fnginstall,
-            "--name",
-            args.name.format(platform=platform),
-        ]
-        logging.debug(cmd)
-        jobfile = "results/{}.yaml".format(platform)
-        with open(jobfile, "w") as jobfile_handle:
-            subprocess.run(cmd, stdout=jobfile_handle, check=True)
-        cmd = ["tests/scripts/submit.py", jobfile]
-        logging.debug(cmd)
+
+        test_suite = os.path.join(
+            checkout_path, "tests", args.test_suite.format(platform=platform)
+        )
+        logging.debug("Test suite %s", test_suite)
+
+        if os.path.splitext(test_suite)[1] == ".jinja2":
+            cmd = [
+                cmd_generate,
+                "--test-plan",
+                test_suite,
+                "--test-repo",
+                args.testrepo,
+                "--fng-install",
+                args.fnginstall,
+                "--name",
+                args.name.format(platform=platform),
+            ]
+            logging.debug("Generate job: %s", cmd)
+            jobfile = os.path.join(args.results_path, "{}.yaml".format(platform))
+            with open(jobfile, "w") as jobfile_handle:
+                call(cmd, stdout=jobfile_handle)
+        else:
+            jobfile = test_suite
+
+        if args.all_devices:
+            cmd = [cmd_submitall, "--device-type", platform, jobfile]
+        else:
+            cmd = [cmd_submit, jobfile]
+
         if args.dry:
-            print("Skipping submit because of dry run")
+            print("Skipping submit because of dry run: %s", cmd)
             continue
-        result = subprocess.run(cmd, capture_output=True, check=True)
-        url = result.stdout.decode().strip().replace("\n", "")
-        print("Started testjob {}: {}".format(platform, url))
-        jobid = url.split("/")[-1]
-        jobs[jobid] = [platform, ""]
-        logging.debug(jobs)
+        result = call(cmd)
+
+        for line in result.splitlines():
+            url = line.strip().replace("\n", "")
+            print("Started testjob {}: {}".format(platform, url))
+            jobid = url.split("/")[-1]
+            jobs[jobid] = [platform, ""]
+
+    logging.debug("Queued jobs: %s", jobs)
 
     # Wait for the results
     pending = True
@@ -134,13 +192,8 @@ def main():
         print(".", end="", flush=True)
         pending = False
         for jobid in jobs:
-            cmd = ["tests/scripts/query.py", jobid]
-            logging.debug(cmd)
-            result = subprocess.run(cmd, capture_output=True, check=True)
-            result = result.stdout.decode().strip().replace("\n", "")
-            logging.debug(result)
+            result = call([cmd_query, jobid]).replace("\n", "")
             jobs[jobid][1] = result
-            logging.debug(jobs)
             if result == "Pending":
                 pending = True
 
@@ -151,24 +204,30 @@ def main():
         jobplatform = jobs[jobid][0]
         jobresult = jobs[jobid][1]
 
-        cmd = ["tests/scripts/query.py", "--get-results", jobid]
-        logging.debug(cmd)
-        resultfile = "results/results-{}-{}.yaml".format(jobid, jobplatform)
+        # Get results as yaml
+        resultfile = os.path.join(
+            args.results_path, "results-{}-{}.yaml".format(jobid, jobplatform)
+        )
         with open(resultfile, "w") as resultfile_handle:
-            subprocess.run(cmd, stdout=resultfile_handle, check=True)
-
-        cmd = [
-            "tests/scripts/query.py",
-            "--get-results",
-            "--result-format",
-            "junit",
-            jobid,
-        ]
-        logging.debug(cmd)
-        resultfile = "results/results-{}-{}.xml".format(jobid, jobplatform)
+            call([cmd_query, "--get-results", jobid], stdout=resultfile_handle)
+
+        # Get results as junit xml
+        resultfile = os.path.join(
+            args.results_path, "results-{}-{}.xml".format(jobid, jobplatform)
+        )
         with open(resultfile, "w") as resultfile_handle:
-            subprocess.run(cmd, stdout=resultfile_handle, check=True)
+            call(
+                [
+                    cmd_query,
+                    "--get-results",
+                    "--result-format",
+                    "junit",
+                    jobid,
+                ],
+                stdout=resultfile_handle,
+            )
 
+        # Read the number of errors from the results
         resultsxml = junitparser.JUnitXml.fromfile(resultfile)
         joberrors = resultsxml.errors