[testing] expose the xcresult bundle to post action binaries for processing (#2711)

Allows for customization of xcresult handling outside of the test
runner.

---------

Co-authored-by: Luis Padron <[email protected]>

Author: aaronsky

apple/testing/default_runner/ios_test_runner.template.sh

@@ -269,6 +269,7 @@ else
   )
 fi
 
+# Run a pre-action binary, if provided.
 pre_action_binary=%(pre_action_binary)s
 "$pre_action_binary"
 
@@ -279,6 +280,7 @@ cmd=("%(testrunner_binary)s"
   "${custom_xctestrunner_args[@]}")
 "${cmd[@]}" 2>&1 || test_exit_code=$?
 
+# Run a post-action binary, if provided.
 post_action_binary=%(post_action_binary)s
 TEST_EXIT_CODE=$test_exit_code \
   "$post_action_binary"

apple/testing/default_runner/ios_xctestrun_runner.bzl

@@ -190,7 +190,7 @@ A binary to run prior to test execution. Runs after simulator creation. Sets the
             executable = True,
             cfg = "exec",
             doc = """
-A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, in addition to any other variables available to the test runner.
+A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, the `$TEST_XCRESULT_BUNDLE_PATH` environment variable if the test run produces an XCResult bundle, and any other variables available to the test runner.
 """,
         ),
         "_simulator_creator": attr.label(

apple/testing/default_runner/ios_xctestrun_runner.template.sh

@@ -446,6 +446,7 @@ if (( ${#custom_xcodebuild_args[@]} )); then
   should_use_xcodebuild=true
 fi
 
+# Run a pre-action binary, if provided.
 pre_action_binary=%(pre_action_binary)s
 SIMULATOR_UDID="$simulator_id" \
   "$pre_action_binary"
@@ -547,11 +548,20 @@ else
     || test_exit_code=$?
 fi
 
+# Run a post-action binary, if provided.
 post_action_binary=%(post_action_binary)s
-TEST_EXIT_CODE=$test_exit_code \
-  TEST_LOG_FILE="$testlog" \
-  SIMULATOR_UDID="$simulator_id" \
-  "$post_action_binary"
+if [[ -n "${result_bundle_path:-}" ]]; then
+  TEST_EXIT_CODE=$test_exit_code \
+    TEST_LOG_FILE="$testlog" \
+    SIMULATOR_UDID="$simulator_id" \
+    TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \
+    "$post_action_binary"
+else
+  TEST_EXIT_CODE=$test_exit_code \
+    TEST_LOG_FILE="$testlog" \
+    SIMULATOR_UDID="$simulator_id" \
+    "$post_action_binary"
+fi
 
 if [[
   "$test_exit_code" -eq 0 &&

apple/testing/default_runner/macos_test_runner.bzl

@@ -138,7 +138,7 @@ A binary to run prior to test execution. Sets any environment variables availabl
             executable = True,
             cfg = "exec",
             doc = """
-A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE` environment variable, in addition to any other variables available to the test runner.
+A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$TEST_XCRESULT_BUNDLE_PATH` environment variables, in addition to any other variables available to the test runner.
 """,
         ),
         "_test_template": attr.label(

apple/testing/default_runner/macos_test_runner.template.sh

@@ -134,22 +134,32 @@ if [[ "$XML_OUTPUT_FILE" != /* ]]; then
   export XML_OUTPUT_FILE="$PWD/$XML_OUTPUT_FILE"
 fi
 
+
+# Run a pre-action binary, if provided.
 pre_action_binary=%(pre_action_binary)s
 "$pre_action_binary"
 
+readonly result_bundle_path="$TEST_UNDECLARED_OUTPUTS_DIR/tests.xcresult"
+# TEST_UNDECLARED_OUTPUTS_DIR isn't cleaned up with multiple retries of flaky tests
+rm -rf "$result_bundle_path"
+
 test_exit_code=0
+readonly testlog="$TEST_TMP_DIR/test.log"
 # Run xcodebuild with the xctestrun file just created. If the test failed, this
 # command will return non-zero, which is enough to tell bazel that the test
 # failed.
-rm -rf "$TEST_UNDECLARED_OUTPUTS_DIR/tests.xcresult"
 xcodebuild test-without-building \
     -destination "platform=macOS" \
-    -resultBundlePath "$TEST_UNDECLARED_OUTPUTS_DIR/tests.xcresult" \
+    -resultBundlePath "$result_bundle_path" \
     -xctestrun "$XCTESTRUN" \
+    2>&1 | tee -i "$testlog" \
     || test_exit_code=$?
 
+# Run a post-action binary, if provided.
 post_action_binary=%(post_action_binary)s
 TEST_EXIT_CODE=$test_exit_code \
+  TEST_LOG_FILE="$testlog" \
+  TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \
   "$post_action_binary"
 
 if [[ "$test_exit_code" -ne 0 ]]; then

doc/rules-ios.md

@@ -718,7 +718,7 @@ in Xcode.
 | <a id="ios_xctestrun_runner-destination_timeout"></a>destination_timeout |  Use the specified timeout when searching for a destination device. The default is 30 seconds.   | Integer | optional |  `0`  |
 | <a id="ios_xctestrun_runner-device_type"></a>device_type |  The device type of the iOS simulator to run test. The supported types correspond to the output of `xcrun simctl list devicetypes`. E.g., iPhone X, iPad Air. By default, it reads from --ios_simulator_device or falls back to some device.   | String | optional |  `""`  |
 | <a id="ios_xctestrun_runner-os_version"></a>os_version |  The os version of the iOS simulator to run test. The supported os versions correspond to the output of `xcrun simctl list runtimes`. E.g., 15.5. By default, it reads --ios_simulator_version and then falls back to the latest supported version.   | String | optional |  `""`  |
-| <a id="ios_xctestrun_runner-post_action"></a>post_action |  A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, in addition to any other variables available to the test runner.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional |  `None`  |
+| <a id="ios_xctestrun_runner-post_action"></a>post_action |  A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, the `$TEST_XCRESULT_BUNDLE_PATH` environment variable if the test run produces an XCResult bundle, and any other variables available to the test runner.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional |  `None`  |
 | <a id="ios_xctestrun_runner-pre_action"></a>pre_action |  A binary to run prior to test execution. Runs after simulator creation. Sets the `$SIMULATOR_UDID` environment variable, in addition to any other variables available to the test runner.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional |  `None`  |
 | <a id="ios_xctestrun_runner-random"></a>random |  Whether to run the tests in random order to identify unintended state dependencies.   | Boolean | optional |  `False`  |
 | <a id="ios_xctestrun_runner-reuse_simulator"></a>reuse_simulator |  Toggle simulator reuse. The default behavior is to reuse an existing device of the same type and OS version. When disabled, a new simulator is created before testing starts and shutdown when the runner completes.   | Boolean | optional |  `True`  |