Revise network.py to support per-run and overall test criteria (New)

providers/base/bin/network.py

@@ -61,10 +61,12 @@ def __init__(
         interface,
         target,
         fail_threshold,
+        overall_fail_threshold,
         cpu_load_fail_threshold,
         iperf3,
         num_threads,
         reverse,
+        percent_total=0,
         protocol="tcp",
         data_size="1",
         run_time=None,
@@ -75,8 +77,10 @@ def __init__(
         self.iface = Interface(interface)
         self.interface = interface
         self.target = target
+        self.percent_total = percent_total
         self.protocol = protocol
         self.fail_threshold = fail_threshold
+        self.overall_fail_threshold = overall_fail_threshold
         self.cpu_load_fail_threshold = cpu_load_fail_threshold
         self.iperf3 = iperf3
         self.num_threads = num_threads
@@ -210,7 +214,7 @@ def extract_core_list(self, line):
         """Extract a list of CPU cores from a line of the form:
         NUMA node# CPU(s):    a-b[,c-d[,...]]"""
         colon = line.find(":")
-        cpu_list = line[colon + 1 :]
+        cpu_list = line[colon + 1 :]  # noqa: E203
         core_list = []
         for core_range in cpu_list.split(","):
             # Skip it if the CPU list for the NUMA node is empty....
@@ -353,6 +357,7 @@ def run(self):
             # it's up to the reviewer to pass or fail.
             percent = 0
             invalid_speed = True
+        self.percent_total += percent
         logging.info("Avg Transfer speed: {} Mb/s".format(throughput))
         if invalid_speed:
             # If we have no link_speed (e.g. wireless interfaces don't
@@ -673,12 +678,15 @@ def run_test(args, test_target):
         return 1
 
     # Execute requested networking test
+    if args.overall_fail_threshold is None:
+        args.overall_fail_threshold = args.fail_threshold
     if args.test_type.lower() == "iperf":
         error_number = 0
         iperf_benchmark = IPerfPerformanceTest(
             args.interface,
             test_target,
             args.fail_threshold,
+            args.overall_fail_threshold,
             args.cpu_load_fail_threshold,
             args.iperf3,
             args.num_threads,
@@ -698,11 +706,24 @@ def run_test(args, test_target):
             )
             if iperf_benchmark.num_threads > 2:
                 iperf_benchmark.optimize_num_threads()
+        iperf_benchmark.percent_total = 0
         while not error_number and run_num < args.num_runs:
             run_num += 1
             logging.info(" Test Run Number %s ".center(60, "-"), run_num)
             error_number = iperf_benchmark.run()
             logging.info("")
+        overall_avg = iperf_benchmark.percent_total / run_num
+        logging.info("Overall run results: {}".format(overall_avg))
+        if overall_avg < iperf_benchmark.overall_fail_threshold:
+            logging.warning(
+                "Test failed because overall results are less than the "
+            )
+            logging.warning(
+                "minimum ({} percent).".format(
+                    iperf_benchmark.overall_fail_threshold
+                )
+            )
+            error_number = 1
     elif args.test_type.lower() == "stress":
         stress_benchmark = StressPerformanceTest(
             args.interface, test_target, args.iperf3
@@ -1213,6 +1234,16 @@ def main():
             "%(default)s)"
         ),
     )
+    test_parser.add_argument(
+        "--overall-fail-threshold",
+        type=int,
+        default=None,
+        help=(
+            "IPERF Test ONLY. Set the overall failure threshold (percent of "
+            "maximum theoretical bandwidth, averaged across all runs). "
+            "(Default is the same as specified by --fail-threshold.)"
+        ),
+    )
     test_parser.add_argument(
         "--cpu-load-fail-threshold",
         type=int,

providers/base/tests/test_network.py

@@ -607,6 +607,28 @@ def test_interface_test_no_test_type(self):
         args = Namespace()
         self.assertIsNone(network.interface_test(args))
 
+    # test_run_test_with_overall_fail_threshold
+    @patch("network.run_test")
+    def test_run_test_with_overall_fail_threshold(self):
+        # Setup for testing overall_fail_threshold
+        overall_fail_threshold = 5
+        result = run_test(overall_fail_threshold=overall_fail_threshold)
+        self.assertLessEqual(
+            result.fail_count,
+            overall_fail_threshold,
+            "Expected fail count to be <= the overall fail threshold.",
+        )
+
+    # test_run_test_with_no_overall_fail_threshold
+    @patch("network.run_test")
+    def test_run_test_with_no_overall_fail_threshold(self):
+        # Setup for testing when no overall_fail_threshold is specified
+        result = run_test()
+        self.assertIsNotNone(
+            result,
+            "Expected result should != None without overall_fail_threshold.",
+        )
+
     @patch("logging.error")
     @patch("network.make_target_list")
     @patch("network.get_test_parameters")

providers/base/units/ethernet/jobs.pxu

@@ -89,7 +89,7 @@ requires:
  executable.name == 'nmap'
 user: root
 environ: TEST_TARGET_IPERF
-command: network.py test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --cpu-load-fail-threshold 90 --runtime 900 --num_runs 4
+command: network.py test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --overall-fail-threshold 90 --cpu-load-fail-threshold 90 --runtime 900 --num_runs 4
 _purpose: This test uses iperf3 to ensure network devices pass data at an acceptable
  minimum percentage of advertised speed.
 
@@ -108,7 +108,7 @@ requires:
  executable.name == 'nmap'
 user: root
 environ: TEST_TARGET_IPERF
-command: network.py test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --cpu-load-fail-threshold 90 --runtime 1200
+command: network.py test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --overall-fail-threshold 90 --cpu-load-fail-threshold 90 --runtime 1200
 _purpose:
  This is a shorter SRU version of the standard iperf3 based network test for
  servers. It is intended to ONLY be used for SRU and is not valid for official
@@ -129,7 +129,7 @@ requires:
  executable.name == 'nmap'
 user: root
 environ: TEST_TARGET_IPERF
-command: network.py test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --cpu-load-fail-threshold 90 --runtime 900 --num_runs 4 --underspeed-ok
+command: network.py test -i {interface} -t iperf --iperf3 --scan-timeout 3600 --fail-threshold 80 --overall-fail-threshold 90 --cpu-load-fail-threshold 90 --runtime 900 --num_runs 4 --underspeed-ok
 _purpose:
  This is the standard Multi-NIC Iperf3 test with the speed check disabled
  for retesting systems that are incorrectly reporting supported speeds.