[lit] Support GoogleTest test discovery through prefixes, too (llvm#137423)

Author: jroelofs

llvm/utils/lit/lit/formats/googletest.py

@@ -15,7 +15,7 @@
 
 
 class GoogleTest(TestFormat):
-    def __init__(self, test_sub_dirs, test_suffix, run_under=[]):
+    def __init__(self, test_sub_dirs, test_suffix, run_under=[], test_prefix=None):
         self.seen_executables = set()
         self.test_sub_dirs = str(test_sub_dirs).split(";")
 
@@ -26,6 +26,7 @@ def __init__(self, test_sub_dirs, test_suffix, run_under=[]):
 
         # Also check for .py files for testing purposes.
         self.test_suffixes = {exe_suffix, test_suffix + ".py"}
+        self.test_prefixes = {test_prefix} if test_prefix else None
         self.run_under = run_under
 
     def get_num_tests(self, path, litConfig, localConfig):
@@ -55,7 +56,9 @@ def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
             dir_path = os.path.join(source_path, subdir)
             if not os.path.isdir(dir_path):
                 continue
-            for fn in lit.util.listdir_files(dir_path, suffixes=self.test_suffixes):
+            for fn in lit.util.listdir_files(
+                dir_path, suffixes=self.test_suffixes, prefixes=self.test_prefixes
+            ):
                 # Discover the tests in this executable.
                 execpath = os.path.join(source_path, subdir, fn)
                 if execpath in self.seen_executables:

llvm/utils/lit/lit/util.py

@@ -182,7 +182,7 @@ def mkdir_p(path):
     mkdir(path)
 
 
-def listdir_files(dirname, suffixes=None, exclude_filenames=None):
+def listdir_files(dirname, suffixes=None, exclude_filenames=None, prefixes=None):
     """Yields files in a directory.
 
     Filenames that are not excluded by rules below are yielded one at a time, as
@@ -214,12 +214,15 @@ def listdir_files(dirname, suffixes=None, exclude_filenames=None):
         exclude_filenames = set()
     if suffixes is None:
         suffixes = {""}
+    if prefixes is None:
+        prefixes = {""}
     for filename in os.listdir(dirname):
         if (
             os.path.isdir(os.path.join(dirname, filename))
             or filename.startswith(".")
             or filename in exclude_filenames
             or not any(filename.endswith(sfx) for sfx in suffixes)
+            or not any(filename.startswith(pfx) for pfx in prefixes)
         ):
             continue
         yield filename

llvm/utils/lit/tests/Inputs/googletest-prefix/DummySubDir/test_one.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests":
+    if sys.argv[2] != "--gtest_filter=-*DISABLED_*":
+        raise ValueError("unexpected argument: %s" % (sys.argv[2]))
+    print(
+        """\
+FirstTest.
+  subTestA
+  subTestB
+  subTestC
+  subTestD
+ParameterizedTest/0.
+  subTest
+ParameterizedTest/1.
+  subTest"""
+    )
+    sys.exit(0)
+elif len(sys.argv) != 1:
+    # sharding and json output are specified using environment variables
+    raise ValueError("unexpected argument: %r" % (" ".join(sys.argv[1:])))
+
+for e in ["GTEST_TOTAL_SHARDS", "GTEST_SHARD_INDEX", "GTEST_OUTPUT"]:
+    if e not in os.environ:
+        raise ValueError("missing environment variables: " + e)
+
+if not os.environ["GTEST_OUTPUT"].startswith("json:"):
+    raise ValueError("must emit json output: " + os.environ["GTEST_OUTPUT"])
+
+output = """\
+{
+"random_seed": 123,
+"testsuites": [
+    {
+        "name": "FirstTest",
+        "testsuite": [
+            {
+                "name": "subTestA",
+                "result": "COMPLETED",
+                "time": "0.001s"
+            },
+            {
+                "name": "subTestB",
+                "result": "COMPLETED",
+                "time": "0.001s",
+                "failures": [
+                    {
+                        "failure": "I am subTest B, I FAIL\\nAnd I have two lines of output",
+                        "type": ""
+                    }
+                ]
+            },
+            {
+                "name": "subTestC",
+                "result": "SKIPPED",
+                "time": "0.001s"
+            },
+            {
+                "name": "subTestD",
+                "result": "UNRESOLVED",
+                "time": "0.001s"
+            }
+        ]
+    },
+    {
+        "name": "ParameterizedTest/0",
+        "testsuite": [
+            {
+                "name": "subTest",
+                "result": "COMPLETED",
+                "time": "0.001s"
+            }
+        ]
+    },
+    {
+        "name": "ParameterizedTest/1",
+        "testsuite": [
+            {
+                "name": "subTest",
+                "result": "COMPLETED",
+                "time": "0.001s"
+            }
+        ]
+    }
+]
+}"""
+
+dummy_output = """\
+{
+"testsuites": [
+]
+}"""
+
+json_filename = os.environ["GTEST_OUTPUT"].split(":", 1)[1]
+with open(json_filename, "w", encoding="utf-8") as f:
+    if os.environ["GTEST_TOTAL_SHARDS"] == "1":
+        print("[ RUN      ] FirstTest.subTestB", flush=True)
+        print("I am subTest B output", file=sys.stderr, flush=True)
+        print("[  FAILED  ] FirstTest.subTestB (8 ms)", flush=True)
+
+        f.write(output)
+        exit_code = 1
+    else:
+        f.write(dummy_output)
+        exit_code = 0
+
+sys.exit(exit_code)

llvm/utils/lit/tests/Inputs/googletest-prefix/lit.cfg

@@ -0,0 +1,4 @@
+import lit.formats
+
+config.name = "googletest-format"
+config.test_format = lit.formats.GoogleTest("DummySubDir", test_suffix="", test_prefix="test_")

llvm/utils/lit/tests/googletest-prefix.py

@@ -0,0 +1,11 @@
+# RUN: env GTEST_TOTAL_SHARDS=1 GTEST_SHARD_INDEX=0 \
+# RUN: %{lit} -v --order=random --no-gtest-sharding %{inputs}/googletest-prefix --show-tests > %t.out
+# FIXME: Temporarily dump test output so we can debug failing tests on
+# buildbots.
+# RUN: cat %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK:      -- Available Tests --
+# CHECK-NEXT:   googletest-format :: DummySubDir/test_one.py