shell: support extra output files in experimental_test_shell_command

docs/notes/2.26.x.md

@@ -82,9 +82,11 @@ The Pants repo now uses Ruff format in lieu of Black. This was not a drop-in rep
 
 #### Shell
 
-The `experiemental_test_shell_command` target type may now be used with the `test` goal's `--debug` flag to execute the test interactively.
+The `experiemental_test_shell_command` target type learned several new features:
 
-The `experiemental_test_shell_command` target type now supports the same `runnable_dependencies` field already supported by the `shell_command` target type.
+- It now supports the `test` goal's `--debug` flag to execute tests interactively.
+- Outputs may now be captured as test "extra outputs" as specified by the new `output_files`, `output_directories`, `root_output_directory`, and `outputs_match_mode` fields. (These fields operate in the same manner as for the `shell_command` target type.) Captured output will be written in a directory under `dist/` based on the target name when a test completes.
+- It now supports the `runnable_dependencies` field already supported by the `shell_command` target type.
 
 #### Terraform
 

src/python/pants/backend/shell/goals/test.py

@@ -33,7 +33,9 @@
 from pants.core.util_rules.adhoc_process_support import rules as adhoc_process_support_rules
 from pants.core.util_rules.adhoc_process_support import run_prepared_adhoc_process
 from pants.core.util_rules.environments import EnvironmentField
+from pants.engine.fs import EMPTY_DIGEST, Snapshot
 from pants.engine.internals.graph import resolve_target
+from pants.engine.intrinsics import digest_to_snapshot
 from pants.engine.process import InteractiveProcess, ProcessCacheScope
 from pants.engine.rules import collect_rules, implicitly, rule
 from pants.engine.target import Target, WrappedTargetRequest
@@ -99,10 +101,16 @@ async def test_shell_command(
         if result.process_result.exit_code == 0:
             break
 
+    extra_output: Snapshot | None = None
+    if results[-1].adjusted_digest != EMPTY_DIGEST:
+        extra_output = await digest_to_snapshot(results[-1].adjusted_digest)
+
     return TestResult.from_fallible_process_result(
         process_results=tuple(r.process_result for r in results),
         address=field_set.address,
         output_setting=test_subsystem.output,
+        extra_output=extra_output,
+        log_extra_output=extra_output is not None,
     )
 
 

src/python/pants/backend/shell/goals/test_test.py

@@ -21,6 +21,8 @@
 from pants.core.goals import package
 from pants.core.goals.test import TestDebugRequest, TestResult, get_filtered_environment
 from pants.core.util_rules import archive, source_files, system_binaries
+from pants.engine.fs import DigestContents, FileContent
+from pants.engine.internals.scheduler import ExecutionError
 from pants.engine.rules import QueryRule
 from pants.engine.target import Target
 from pants.testutil.rule_runner import RuleRunner, mock_console
@@ -56,7 +58,7 @@ def rule_runner() -> RuleRunner:
 
 
 @pytest.mark.platform_specific_behavior
-def test_shell_command_as_test(rule_runner: RuleRunner) -> None:
+def test_basic_usage_of_test_shell_command(rule_runner: RuleRunner) -> None:
     rule_runner.write_files(
         {
             "BUILD": dedent(
@@ -152,3 +154,133 @@ def run_test(test_target: Target) -> TestResult:
     )
     pass_for_runnable_dependency_result = run_test(pass_for_runnable_dependency_target)
     assert pass_for_runnable_dependency_result.exit_code == 0
+
+
+@pytest.mark.platform_specific_behavior
+def test_extra_outputs_support(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """\
+                shell_sources(name="src")
+
+                experimental_test_shell_command(
+                  name="test",
+                  execution_dependencies=[":src"],
+                  tools=["echo", "mkdir"],
+                  command="./test.sh msg.txt message",
+                  output_files=["world.txt"],
+                  output_directories=["some-dir"],
+                )
+                """
+            ),
+            "test.sh": dedent(
+                """\
+                mkdir -p some-dir
+                echo "xyzzy" > some-dir/foo.txt
+                echo "hello" > world.txt
+                """
+            ),
+        }
+    )
+    (Path(rule_runner.build_root) / "test.sh").chmod(0o555)
+
+    def test_batch_for_target(test_target: Target) -> ShellTestRequest.Batch:
+        return ShellTestRequest.Batch("", (TestShellCommandFieldSet.create(test_target),), None)
+
+    def run_test(test_target: Target) -> TestResult:
+        return rule_runner.request(TestResult, [test_batch_for_target(test_target)])
+
+    result = run_test(rule_runner.get_target(Address("", target_name="test")))
+    assert result.extra_output is not None
+    digest_contents = rule_runner.request(DigestContents, [result.extra_output.digest])
+    digest_contents_sorted = sorted(digest_contents, key=lambda x: x.path)
+    assert len(digest_contents_sorted) == 2
+    assert digest_contents_sorted[0] == FileContent("some-dir/foo.txt", b"xyzzy\n")
+    assert digest_contents_sorted[1] == FileContent("world.txt", b"hello\n")
+
+
+def test_outputs_match_mode_support(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """\
+            experimental_test_shell_command(
+                name="allow_empty",
+                command="true",
+                output_files=["non-existent-file"],
+                output_directories=["non-existent-dir"],
+                outputs_match_mode="allow_empty",
+            )
+            experimental_test_shell_command(
+                name="all_with_present_file",
+                command="touch some-file",
+                tools=["touch"],
+                output_files=["some-file"],
+                output_directories=["some-directory"],
+                outputs_match_mode="all",
+            )
+            experimental_test_shell_command(
+                name="all_with_present_directory",
+                command="mkdir some-directory",
+                tools=["mkdir"],
+                output_files=["some-file"],
+                output_directories=["some-directory"],
+                outputs_match_mode="all",
+            )
+            experimental_test_shell_command(
+                name="at_least_one_with_present_file",
+                command="touch some-file",
+                tools=["touch"],
+                output_files=["some-file"],
+                output_directories=["some-directory"],
+                outputs_match_mode="at_least_one",
+            )
+            experimental_test_shell_command(
+                name="at_least_one_with_present_directory",
+                command="mkdir some-directory && touch some-directory/foo.txt",
+                tools=["mkdir", "touch"],
+                output_files=["some-file"],
+                output_directories=["some-directory"],
+                outputs_match_mode="at_least_one",
+            )
+            """
+            )
+        }
+    )
+
+    def test_batch_for_target(test_target: Target) -> ShellTestRequest.Batch:
+        return ShellTestRequest.Batch("", (TestShellCommandFieldSet.create(test_target),), None)
+
+    def run_test(address: Address) -> TestResult:
+        test_target = rule_runner.get_target(address)
+        return rule_runner.request(TestResult, [test_batch_for_target(test_target)])
+
+    def assert_result(
+        address: Address,
+        expected_contents: dict[str, str],
+    ) -> None:
+        result = run_test(address)
+        if expected_contents:
+            assert result.extra_output
+            assert result.extra_output.files == tuple(expected_contents)
+
+            contents = rule_runner.request(DigestContents, [result.extra_output.digest])
+            for fc in contents:
+                assert fc.content == expected_contents[fc.path].encode()
+
+    assert_result(Address("", target_name="allow_empty"), {})
+
+    with pytest.raises(ExecutionError) as exc_info:
+        run_test(Address("", target_name="all_with_present_file"))
+    assert "some-directory" in str(exc_info)
+
+    with pytest.raises(ExecutionError) as exc_info:
+        run_test(Address("", target_name="all_with_present_directory"))
+    assert "some-file" in str(exc_info)
+
+    assert_result(Address("", target_name="at_least_one_with_present_file"), {"some-file": ""})
+    assert_result(
+        Address("", target_name="at_least_one_with_present_directory"),
+        {"some-directory/foo.txt": ""},
+    )

src/python/pants/backend/shell/target_types.py

@@ -501,6 +501,10 @@ class ShellCommandTestTarget(Target):
         EnvironmentField,
         SkipShellCommandTestsField,
         ShellCommandWorkdirField,
+        ShellCommandOutputFilesField,
+        ShellCommandOutputDirectoriesField,
+        ShellCommandOutputRootDirField,
+        ShellCommandOutputsMatchMode,
     )
     help = help_text(
         """

src/python/pants/core/goals/test.py

@@ -624,7 +624,7 @@ class EnvironmentAware:
         advanced=True,
         help="The maximum timeout (in seconds) that may be used on a test target.",
     )
-    attempts_default = IntOption(
+    _attempts_default = IntOption(
         default=1,
         help=softwrap(
             """
@@ -701,6 +701,15 @@ class EnvironmentAware:
     def report_dir(self, distdir: DistDir) -> PurePath:
         return PurePath(self._report_dir.format(distdir=distdir.relpath))
 
+    @property
+    def attempts_default(self):
+        if self._attempts_default < 1:
+            raise ValueError(
+                "The `--test-attempts-default` option must have a value equal or greater than 1. "
+                f"Instead, it was set to {self._attempts_default}."
+            )
+        return self._attempts_default
+
 
 class Test(Goal):
     subsystem_cls = TestSubsystem