[libc++] Optionally support filecheck and split-file (llvm#165769)

This patch adds optional support for FileCheck and split-file in
libc++'s test suite. Whether FileCheck / split-file is available
can be queried using Lit features.

This should make it possible to test several things that were
previously impossible to test, especially for specific code
generation.

Supersedes llvm#65917 and llvm#188283

Author: ldionne

.github/workflows/libcxx-build-and-test.yaml

@@ -58,7 +58,11 @@ jobs:
         with:
           persist-credentials: false
       - name: ${{ matrix.config }}.${{ matrix.cxx }}
-        run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+        run: |
+          python3 -m venv --system-site-packages .venv
+          source .venv/bin/activate
+          pip install -r libcxx/test/requirements.txt
+          libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
           CC: ${{ matrix.cc }}
           CXX: ${{ matrix.cxx }}
@@ -105,7 +109,11 @@ jobs:
         with:
           persist-credentials: false
       - name: ${{ matrix.config }}
-        run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+        run: |
+          python3 -m venv --system-site-packages .venv
+          source .venv/bin/activate
+          pip install -r libcxx/test/requirements.txt
+          libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
           CC: ${{ matrix.cc }}
           CXX: ${{ matrix.cxx }}
@@ -162,7 +170,11 @@ jobs:
         with:
           persist-credentials: false
       - name: ${{ matrix.config }}
-        run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
+        run: |
+          python3 -m venv --system-site-packages .venv
+          source .venv/bin/activate
+          pip install -r libcxx/test/requirements.txt
+          libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
           CC: clang-22
           CXX: clang++-22
@@ -220,7 +232,7 @@ jobs:
         run: |
           python3 -m venv .venv
           source .venv/bin/activate
-          python -m pip install psutil
+          pip install -r libcxx/test/requirements.txt
           xcrun bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
           CC: clang
@@ -260,7 +272,7 @@ jobs:
           persist-credentials: false
       - name: Install dependencies
         run: |
-          pip install psutil
+          pip install -r libcxx/test/requirements.txt
       - name: Install a current LLVM
         if: ${{ matrix.mingw != true }}
         run: |

libcxx/docs/TestingLibcxx.rst

@@ -23,6 +23,13 @@ Please see the `Lit Command Guide`_ for more information about LIT.
 
 .. _LIT Command Guide: https://llvm.org/docs/CommandGuide/lit.html
 
+Dependencies
+------------
+
+The libc++ test suite has a few optional dependencies. These can be installed
+with ``pip install -r libcxx/test/requirements.txt``. Installing these dependencies
+will ensure that the maximum number of tests can be run.
+
 Usage
 -----
 

libcxx/test/requirements.txt

@@ -0,0 +1,5 @@
+#
+# This file defines Python requirements to run the libc++ test suite.
+#
+llvm-testing-tools==23.0.0
+psutil==5.9.8

libcxx/test/selftest/filecheck.negative.sh.cpp

@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-filecheck
+
+// Make sure that FileCheck fails when it should fail. This ensure that FileCheck
+// actually checks the content of the file.
+
+// RUN: echo "hello world" | not FileCheck %s
+// CHECK: foobar

libcxx/test/selftest/filecheck.sh.cpp

@@ -0,0 +1,15 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-filecheck
+
+// Make sure that we can use FileCheck to write tests when the `has-filecheck`
+// Lit feature is defined.
+
+// RUN: echo "hello world" | FileCheck %s
+// CHECK: hello world

libcxx/test/selftest/splitfile.sh.cpp

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-splitfile
+
+// Make sure that we can use split-file to write tests when the `has-splitfile`
+// Lit feature is defined.
+
+// RUN: split-file %s %{temp}
+
+// RUN: grep 'int main' %{temp}/main.cpp
+// RUN: grep 'return 0' %{temp}/main.cpp
+// RUN: not grep -c 'Pre-delimiter' %{temp}/main.cpp
+// RUN: not grep -c 'foo' %{temp}/main.cpp
+// RUN: not grep -c '//---' %{temp}/main.cpp
+
+// RUN: grep foo %{temp}/input.txt
+// RUN: grep bar %{temp}/input.txt
+// RUN: not grep -c 'Pre-delimiter' %{temp}/input.txt
+// RUN: not grep -c 'int main' %{temp}/input.txt
+// RUN: not grep -c '//---' %{temp}/input.txt
+
+// Pre-delimiter comment.
+
+//--- main.cpp
+
+int main() {
+  return 0;
+}
+
+//--- input.txt
+
+foo
+bar

libcxx/utils/libcxx/test/features/misc.py

@@ -296,4 +296,13 @@ def _mingwSupportsModules(cfg):
             """,
         ),
     ),
+    # Whether `FileCheck` and `split-file` executables are available.
+    Feature(
+        name="has-filecheck",
+        when=lambda cfg: runScriptExitCode(cfg, ["FileCheck --version"]) == 0,
+    ),
+    Feature(
+        name="has-splitfile",
+        when=lambda cfg: runScriptExitCode(cfg, ["split-file --version"]) == 0,
+    ),
 ]