Upload undeclared outputs of failed/flaky/timed out tests (#2169)

Test failures can require information present in undeclared outputs to
diagnose, which if not uploaded means needing to first reproduce the
failure locally.

e.g.
https://buildkite.com/bazel/bazel-bazel-github-presubmit/builds/25623#0194aa2c-7bd9-466b-bd15-d7eec8ef58dc
(failure in `test.log` references a file in `test.outputs`) from
bazelbuild/bazel#25050

This PR adds uploading of `test.outputs/*` files for tests that
fail/flake/timeout.

It also;
* Simplifies `local_exec_root.as_ref().map(|p| p.as_path()),` to
`local_exec_root.as_deref(),` (clippy suggestion)
* Lifts the upload decision (dubbed `include_output` in this PR) up a
level so it can be used in the new code branch. I'm fairly `test.log` is
a mandatory test output, so this change should not affect performance.

Author: Silic0nS0ldier

Diff for: agent/src/artifact/upload.rs

@@ -126,16 +126,16 @@ fn watch_bep_json_file(
                             .map(|str| Path::new(str).to_path_buf());
                     } else if build_event.is_test_result() {
                         let test_result = build_event.test_result();
+                        let include_output = ["FAILED", "TIMEOUT", "FLAKY"]
+                            .contains(&test_result.status.as_str());
                         for output in test_result.test_action_outputs.iter() {
                             match output.name.as_str() {
                                 "test.log" => {
-                                    if ["FAILED", "TIMEOUT", "FLAKY"]
-                                        .contains(&test_result.status.as_str())
-                                    {
+                                    if include_output {
                                         if let Err(error) = upload_test_log(
                                             &mut uploader,
                                             dry,
-                                            local_exec_root.as_ref().map(|p| p.as_path()),
+                                            local_exec_root.as_deref(),
                                             &output.uri,
                                             mode,
                                         ) {
@@ -148,7 +148,7 @@ fn watch_bep_json_file(
                                         if let Err(error) = upload_test_xml(
                                             &mut uploader,
                                             dry,
-                                            local_exec_root.as_ref().map(|p| p.as_path()),
+                                            local_exec_root.as_deref(),
                                             &output.uri,
                                             test_result.label.as_ref(),
                                             mode,
@@ -157,6 +157,19 @@ fn watch_bep_json_file(
                                         }
                                     }
                                 }
+                                n if n.starts_with("test.outputs/") => {
+                                    if include_output {
+                                        if let Err(error) = upload_test_output(
+                                            &mut uploader,
+                                            dry,
+                                            local_exec_root.as_deref(),
+                                            &output.uri,
+                                            mode,
+                                        ) {
+                                            error!("{:?}", error);
+                                        }
+                                    }
+                                }
                                 _ => {}
                             }
                         }
@@ -655,6 +668,23 @@ fn upload_test_xml(
     return uploader.upload_test_xml(dry, cwd.as_ref().map(|pb| pb.as_path()), &artifact, label);
 }
 
+fn upload_test_output(
+    uploader: &mut Uploader,
+    dry: bool,
+    local_exec_root: Option<&Path>,
+    test_output: &str,
+    mode: Mode,
+) -> Result<()> {
+    let (cwd, artifact) = resolve_artifact(test_output, local_exec_root)?;
+    return uploader.upload_artifact(
+        dry,
+        cwd.as_ref().map(|pb| pb.as_path()),
+        &artifact,
+        mode,
+        None,
+    );
+}
+
 #[derive(Debug)]
 pub struct TestActionOutput {
     pub name: String,

Diff for: agent/tests/artifact/upload.rs

@@ -42,6 +42,24 @@ fn test_logs_uploaded_to_buildkite() -> Result<()> {
     Ok(())
 }
 
+#[cfg(not(target_os = "windows"))]
+#[test]
+fn test_undeclared_output_uploaded_to_buildkite() -> Result<()> {
+    let mut cmd = Command::cargo_bin("bazelci-agent")?;
+    cmd.args([
+        "artifact",
+        "upload",
+        "--dry",
+        "--mode=buildkite",
+        "--build_event_json_file=tests/data/test_bep_undeclared_output.json",
+    ]);
+    cmd.assert()
+        .success()
+        .stdout(predicates::str::contains("buildkite-agent artifact upload src/test/shell/bazel/tags_propagation_native_test/test_attempts/attempt_1.outputs/test_cc_library_tags_not_propagated_when_incompatible_flag_off/741B943E-B846-4453-A3AF-0D9010431815/output1\n"));
+
+    Ok(())
+}
+
 fn with_tmpfile<F>(f: F) -> Result<()>
 where
     F: Fn(File, &Path) -> Result<()>,

Diff for: agent/tests/data/test_bep_undeclared_output.json

@@ -0,0 +1,4 @@
+{"id":{"workspace":{}},"workspaceInfo":{"localExecRoot":"/private/var/tmp/_bazel_buildkite/39eb3a0c7f548273ec33e4a77eb36882/execroot/_main"}}
+{"id":{"testResult":{"label":"//src/test/shell/bazel:tags_propagation_native_test","run":1,"shard":1,"attempt":1,"configuration":{"id":"65571a9c3d56c8ba36ef27c072426c3096c8925f94bc60c163904cd36caf68b9"}}},"children":[{"testResult":{"label":"//src/test/shell/bazel:tags_propagation_native_test","run":1,"shard":1,"attempt":2,"configuration":{"id":"65571a9c3d56c8ba36ef27c072426c3096c8925f94bc60c163904cd36caf68b9"}}}],"testResult":{"testActionOutput":[{"name":"test.log","uri":"file:///private/var/tmp/_bazel_buildkite/39eb3a0c7f548273ec33e4a77eb36882/execroot/_main/bazel-out/darwin_arm64-fastbuild/testlogs/src/test/shell/bazel/tags_propagation_native_test/test_attempts/attempt_1.log"},{"name":"test.xml","uri":"file:///private/var/tmp/_bazel_buildkite/39eb3a0c7f548273ec33e4a77eb36882/execroot/_main/bazel-out/darwin_arm64-fastbuild/testlogs/src/test/shell/bazel/tags_propagation_native_test/test_attempts/attempt_1.xml"},{"name":"test.outputs/test_cc_library_tags_not_propagated_when_incompatible_flag_off/741B943E-B846-4453-A3AF-0D9010431815/output1","uri":"file:///private/var/tmp/_bazel_buildkite/39eb3a0c7f548273ec33e4a77eb36882/execroot/_main/bazel-out/darwin_arm64-fastbuild/testlogs/src/test/shell/bazel/tags_propagation_native_test/test_attempts/attempt_1.outputs/test_cc_library_tags_not_propagated_when_incompatible_flag_off/741B943E-B846-4453-A3AF-0D9010431815/output1"},{"name":"test.outputs_manifest__MANIFEST","uri":"file:///private/var/tmp/_bazel_buildkite/39eb3a0c7f548273ec33e4a77eb36882/execroot/_main/bazel-out/darwin_arm64-fastbuild/testlogs/src/test/shell/bazel/tags_propagation_native_test/test_attempts/attempt_1.outputs_manifest/MANIFEST"}],"testAttemptDurationMillis":"38056","status":"FAILED","testAttemptStartMillisEpoch":"1738022034867","executionInfo":{"strategy":"darwin-sandbox","timingBreakdown":{"child":[{"name":"parseTime","time":"0s"},{"name":"fetchTime","time":"0s"},{"name":"queueTime","time":"0s"},{"name":"uploadTime","time":"0s"},{"name":"setupTime","time":"0s"},{"name":"executionWallTime","time":"38.056s"},{"name":"processOutputsTime","time":"0s"},{"name":"networkTime","time":"0s"}],"name":"totalTime","time":"38.056s"}},"testAttemptStart":"2025-01-27T23:53:54.867Z","testAttemptDuration":"38.056s"}}
+{"id":{"testSummary":{"label":"//src/test/shell/bazel:tags_propagation_native_test","configuration":{"id":"65571a9c3d56c8ba36ef27c072426c3096c8925f94bc60c163904cd36caf68b9"}}},"testSummary":{"totalRunCount":3,"failed":[{"uri":"file:///private/var/tmp/_bazel_buildkite/39eb3a0c7f548273ec33e4a77eb36882/execroot/_main/bazel-out/darwin_arm64-fastbuild/testlogs/src/test/shell/bazel/tags_propagation_native_test/test_attempts/attempt_1.log"}],"overallStatus":"FAILED","firstStartTimeMillis":"1738022104632","lastStopTimeMillis":"1738022135626","totalRunDurationMillis":"30994","runCount":1,"totalRunDuration":"30.994s","firstStartTime":"2025-01-27T23:55:04.632Z","lastStopTime":"2025-01-27T23:55:35.626Z","attemptCount":3}}
+{"id":{"progress":{}},"progress":{},"lastMessage":true}