Enhance test deduplication in PyTorch test parsing

easybuild/easyblocks/p/pytorch.py

@@ -1048,12 +1048,6 @@ def extract_path(classname: str) -> str:
             raise ValueError("Could not infer test suite name from class names for {xml_file}.")
         # We can remove possible class names by only using the common part
         suite_name = os.path.commonpath(possible_paths)
-        # Strip of common prefix to all classes, but keep the last part for uniqueness
-        non_classname_prefix = 'test.' + os.path.dirname(suite_name).replace(os.path.sep, '.') + '.'
-        for testcase in test_cases:
-            classname = testcase.attrib["classname"]
-            if classname.startswith(non_classname_prefix):
-                testcase.attrib["classname"] = classname[len(non_classname_prefix):]
     else:
         # Pytest reports, have the name in folder and file e.g.:
         # distributed.pipeline.sync.skip.test_stash_pop/distributed.pipeline.sync.skip.test_stash_pop-052ae03efad18.xml
@@ -1062,13 +1056,34 @@ def extract_path(classname: str) -> str:
         if test_file_path != suite_name:
             raise ValueError(f"Path from folder and filename should be equal. "
                              f"Got: '{test_file_path}' != '{suite_name}'")
+    # Strip of common prefix to all classes
+    non_classname_prefix = 'test.' + suite_name.replace(os.path.sep, '.') + '.'
+    for testcase in test_cases:
+        try:
+            classname = testcase.attrib["classname"]
+        except KeyError:
+            continue
+        if classname.startswith(non_classname_prefix):
+            testcase.attrib["classname"] = classname[len(non_classname_prefix):]
     # Variant might be dist-gloo, dist-mpi or similar which is the same test code ran in different configurations!
     variant = xml_file.parent.parent.name
     if variant not in ('python-unittest', 'python-pytest'):
         suite_name = os.path.join(variant, suite_name)
     return suite_name
 
 
+def handle_xfail_results(test_suite_el: ET.Element) -> None:
+    """Transform elements such that xfails are counted as success"""
+    numXFail = 0
+    for testcase in test_suite_el.iterfind("testcase"):
+        skipped = testcase.find('skipped')
+        if skipped is not None and skipped.get('type') == 'pytest.xfail':
+            numXFail += 1
+            testcase.remove(skipped)
+    if numXFail:
+        test_suite_el.attrib["skipped"] = int(test_suite_el.attrib["skipped"]) - numXFail
+
+
 def parse_test_result_file(xml_file: Path) -> List[TestSuite]:
     """
     Parses the given XML file into TestSuite and TestCase objects.
@@ -1100,6 +1115,7 @@ def parse_test_result_file(xml_file: Path) -> List[TestSuite]:
         test_suites: List[TestSuite] = []
 
         for test_suite in test_suite_xml:
+            handle_xfail_results(test_suite)
             # Those are based on the number of the corresponding elements in all <testcase>-elements.
             # This means e.g. that a test with multiple <skipped> will be counted as multiple skipped tests.
             errors = int(test_suite.attrib["errors"])

test/easyblocks/easyblock_specific.py

@@ -732,7 +732,7 @@ def test_pytorch_test_log_parsing(self):
             self.assertEqual((name, suite.summary), (name, results2[name].summary))
         del results2
 
-        self.assertEqual(len(results), 15)
+        self.assertEqual(len(results), 16)
 
         # 2 small test suites used as a smoke test using a most features
         self.assertIn('backends/xeon/test_launch', results)
@@ -762,9 +762,10 @@ def test_pytorch_test_log_parsing(self):
             dist-nccl-init-env/distr/algorithms/quantization/test_quantization: 0 failed, 1 passed, 0 skipped, 0 errors
             dist-nccl-init-file/distr/algorithms/quantization/test_quantization: 0 failed, 1 passed, 0 skipped, 0 errors
             dist/foo/bar: 0 failed, 4 passed, 0 skipped, 0 errors
+            distributed/_composable/test_composability/test_pp_composability: 0 failed, 2 passed, 0 skipped, 0 errors
             distributed/tensor/test_dtensor_ops: 0 failed, 2 passed, 2 skipped, 0 errors
             dynamo/test_dynamic_shapes: 3 failed, 14 passed, 0 skipped, 0 errors
-            dynamo/test_misc: 1 failed, 9 passed, 0 skipped, 0 errors
+            dynamo/test_misc: 3 failed, 11 passed, 0 skipped, 0 errors
             inductor/test_aot_inductor_arrayref: 2 failed, 0 passed, 0 skipped, 0 errors
             inductor/test_cudagraph_trees: 1 failed, 0 passed, 0 skipped, 0 errors
             jit/test_builtins: 0 failed, 1 passed, 0 skipped, 0 errors
@@ -778,6 +779,8 @@ def test_pytorch_test_log_parsing(self):
         self.assertEqual(tests, textwrap.dedent("""
             AOTInductorTestABICompatibleCpuWithStackAllocation.test_fail_and_skip: failure
             AOTInductorTestABICompatibleCpuWithStackAllocation.test_skip_and_fail: failure
+            ComposabilityTest.test_pass_on_rerun_different_classname: success
+            ComposabilityTest.test_pp_and_dcp: success
             CudaGraphTreeTests.test_workspace_allocation_error: failure
             DistQuantizationTests.test_all_gather_fp16: success
             DistQuantizationTests.test_all_gather_fp16: success
@@ -800,6 +803,10 @@ def test_pytorch_test_log_parsing(self):
             DynamicShapesMiscTests.test_python_slice_dynamic_shapes: success
             DynamicShapesMiscTests.test_pytree_tree_flatten_unflatten_dynamic_shapes: success
             DynamicShapesMiscTests.test_pytree_tree_leaves_dynamic_shapes: failure
+            MiscTests.test_fail_then_skip2: failure
+            MiscTests.test_fail_then_skip: failure
+            MiscTests.test_fail_then_xfail2: success
+            MiscTests.test_fail_then_xfail: success
             MiscTests.test_packaging_version_parse: success
             MiscTests.test_pair: success
             MiscTests.test_param_shape_binops: success
@@ -852,9 +859,9 @@ def test_pytorch_test_log_parsing(self):
             TestTorchrun.test_multi_threads: success
             TestTorchrun.test_reshape_cpu_float64: failure
             TestTracer.test_jit_save: success
-            bar.test_2.test_func3: success
-            bar.test_foo.TestBar.test_func2: success
-            bar.test_foo.TestName.test_func1: success
+            test_2.test_func3: success
+            test_foo.TestBar.test_func2: success
+            test_foo.TestName.test_func1: success
         """).strip())
 
         #  Some error cases

test/pytorch_test_logs/cleanup_files.py

@@ -37,8 +37,6 @@ def shorten_content(path: Path):
     # Ignore timestamp & hostname
     content = re.sub(r'timestamp="[^"]+"', '', content)
     content = re.sub(r'hostname="[^"]+"', '', content)
-    # Remove type attribute from <skipped> tags
-    content = re.sub(r'(<skipped)\s+type="[^"]+"', r'\1', content)
 
     # Remove stdout/stderr from about half of the files.
     # For the other half just shorten it.

test/pytorch_test_logs/test-reports/python-pytest/distributed._composable.test_composability.test_pp_composability/distributed._composable.test_composability.test_pp_composability-4d421.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0"?>
+<testsuites name="pytest tests">
+  <testsuite name="pytest" errors="0" failures="0" skipped="0" tests="1" time="4.2">
+    <testcase classname="ComposabilityTest" name="test_pass_on_rerun_different_classname" time="4.2" file="distributed/_composable/test_composability/test_pp_composability.py"/>
+  </testsuite>
+</testsuites>

test/pytorch_test_logs/test-reports/python-pytest/distributed._composable.test_composability.test_pp_composability/distributed._composable.test_composability.test_pp_composability-912a8.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites name="pytest tests">
+  <testsuite name="pytest" errors="0" failures="1" skipped="0" tests="2" time="4.2">
+    <testcase classname="test.distributed._composable.test_composability.test_pp_composability.ComposabilityTest" name="test_pp_and_dcp" time="4.2"/>
+    <testcase classname="test.distributed._composable.test_composability.test_pp_composability.ComposabilityTest" name="test_pass_on_rerun_different_classname" time="4.2">
+      <failure message="...">[snip]</failure>
+    </testcase>
+  </testsuite>
+</testsuites>

test/pytorch_test_logs/test-reports/python-pytest/dynamo.test_misc/dynamo.test_misc-18930.xml

@@ -1,11 +1,23 @@
 <?xml version="1.0"?>
 <testsuites>
-  <testsuite name="pytest" errors="0" failures="1" skipped="0" tests="1" time="4.2">
+  <testsuite name="pytest" errors="0" failures="3" skipped="2" tests="5" time="4.2">
     <testcase classname="MiscTests" name="test_parameter_free" time="4.2" file="dynamo/test_misc.py">
       <rerun message="...">[snip]</rerun>
       <rerun message="...">[snip]</rerun>
       <failure message="...">[snip]</failure>
       <system-out>[snip]</system-out>
     </testcase>
+    <testcase classname="MiscTests" name="test_fail_then_xfail" time="4.2" file="dynamo/test_misc.py">
+      <failure message="...">[snip]</failure>
+    </testcase>
+    <testcase classname="MiscTests" name="test_fail_then_skip" time="4.2" file="dynamo/test_misc.py">
+      <failure message="...">[snip]</failure>
+    </testcase>
+    <testcase classname="MiscTests" name="test_fail_then_xfail2" time="4.2" file="dynamo/test_misc.py">
+      <skipped type="pytest.xfail" message=""/>
+    </testcase>
+    <testcase classname="MiscTests" name="test_fail_then_skip2" time="4.2" file="dynamo/test_misc.py">
+      <skipped type="pytest.skip" message="..."/>
+    </testcase>
   </testsuite>
 </testsuites>

test/pytorch_test_logs/test-reports/python-pytest/dynamo.test_misc/dynamo.test_misc-86d5b.xml

@@ -1,11 +1,23 @@
 <?xml version="1.0"?>
 <testsuites>
-  <testsuite name="pytest" errors="0" failures="1" skipped="2" tests="34" time="4.2">
+  <testsuite name="pytest" errors="0" failures="3" skipped="2" tests="34" time="4.2">
     <testcase classname="MiscTests" name="test_pytree_tree_map" time="4.2" file="dynamo/test_misc.py"/>
     <testcase classname="MiscTests" name="test_shape_env_no_recording" time="4.2" file="dynamo/test_misc.py"/>
     <testcase classname="MiscTests" name="test_shape_env_recorded_function_fallback" time="4.2" file="dynamo/test_misc.py"/>
     <testcase classname="MiscTests" name="test_param_shape_binops" time="4.2" file="dynamo/test_misc.py"/>
     <testcase classname="MiscTests" name="test_yield_from_in_a_loop" time="4.2" file="dynamo/test_misc.py"/>
+    <testcase classname="MiscTests" name="test_fail_then_xfail" time="4.2" file="dynamo/test_misc.py">
+      <skipped type="pytest.xfail" message=""/>
+    </testcase>
+    <testcase classname="MiscTests" name="test_fail_then_skip" time="4.2" file="dynamo/test_misc.py">
+      <skipped type="pytest.skip" message="..."/>
+    </testcase>
+    <testcase classname="MiscTests" name="test_fail_then_xfail2" time="4.2" file="dynamo/test_misc.py">
+      <failure message="...">[snip]</failure>
+    </testcase>
+    <testcase classname="MiscTests" name="test_fail_then_skip2" time="4.2" file="dynamo/test_misc.py">
+      <failure message="...">[snip]</failure>
+    </testcase>
     <testcase classname="MiscTests" name="test_parameter_free" time="4.2" file="dynamo/test_misc.py">
       <rerun message="...">[snip]</rerun>
       <failure message="...">[snip]</failure>

test/pytorch_test_logs/test-reports/python-unittest/-c/TEST-TestFooCPU-20251.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="TestFooCPU-20251030150550" tests="1" file=".py" time="4.2" failures="0" errors="0" skipped="0">
+  <testcase classname="TestFooCPU" name="test_bar_cpu" time="4.2">
+    <system-out>[snip]</system-out>
+  </testcase>
+</testsuite>

test/pytorch_test_logs/test-reports/python-unittest/-c/TEST-unittest.suite._ErrorHolder-20251.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="unittest.suite._ErrorHolder-20251030150550" tests="1" file="unittest/suite.py" time="4.2" failures="0" errors="1" skipped="0">
+  <testcase classname="" name="tearDownClass (__main__.TestFooCPU)" time="4.2">
+    <error type="RuntimeError" message="...">[snip]</error>
+  </testcase>
+</testsuite>