Tests: Allow post_action to determine exit status (#2713)

Add new attr `post_action_determines_exit_code` to let the test runner
post action determine the exit status. Our use case is allow soft
failing of certain test targets.

```starlark
ios_xctestrun_runner(
    name = "ios_sim_runner_with_soft_fail",
    device_type = "iPhone Xs",
    post_action = ":post_action_soft_fail",
    post_action_determines_exit_code = True,
)
```

Author: kapoorlakshya

apple/testing/default_runner/ios_test_runner.bzl

@@ -27,7 +27,8 @@ def _get_template_substitutions(
         simulator_creator,
         testrunner,
         pre_action_binary,
-        post_action_binary):
+        post_action_binary,
+        post_action_determines_exit_code):
     """Returns the template substitutions for this runner."""
     subs = {
         "device_type": device_type,
@@ -36,6 +37,7 @@ def _get_template_substitutions(
         "testrunner_binary": testrunner,
         "pre_action_binary": pre_action_binary,
         "post_action_binary": post_action_binary,
+        "post_action_determines_exit_code": post_action_determines_exit_code,
     }
     return {"%(" + k + ")s": subs[k] for k in subs}
 
@@ -66,8 +68,10 @@ def _ios_test_runner_impl(ctx):
         pre_action_binary = ctx.executable.pre_action.short_path
         runfiles = runfiles.merge(ctx.attr.pre_action[DefaultInfo].default_runfiles)
 
+    post_action_determines_exit_code = False
     if ctx.executable.post_action:
         post_action_binary = ctx.executable.post_action.short_path
+        post_action_determines_exit_code = ctx.attr.post_action_determines_exit_code
         runfiles = runfiles.merge(ctx.attr.post_action[DefaultInfo].default_runfiles)
 
     ctx.actions.expand_template(
@@ -80,6 +84,7 @@ def _ios_test_runner_impl(ctx):
             testrunner = ctx.executable._testrunner.short_path,
             pre_action_binary = pre_action_binary,
             post_action_binary = post_action_binary,
+            post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
         ),
     )
     return [
@@ -143,6 +148,12 @@ A binary to run prior to test execution. Runs after simulator creation. Sets any
             cfg = "exec",
             doc = """
 A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE` environment variable, in addition to any other variables available to the test runner.
+""",
+        ),
+        "post_action_determines_exit_code": attr.bool(
+            default = False,
+            doc = """
+When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.
 """,
         ),
         "_test_template": attr.label(

apple/testing/default_runner/ios_test_runner.template.sh

@@ -282,12 +282,21 @@ cmd=("%(testrunner_binary)s"
 
 # Run a post-action binary, if provided.
 post_action_binary=%(post_action_binary)s
+post_action_determines_exit_code="%(post_action_determines_exit_code)s"
+post_action_exit_code=0
 TEST_EXIT_CODE=$test_exit_code \
-  "$post_action_binary"
+  "$post_action_binary" || post_action_exit_code=$?
 
-if [[ "$test_exit_code" -ne 0 ]]; then
-  echo "error: tests exited with '$test_exit_code'" >&2
-  exit "$test_exit_code"
+if [[ "$post_action_determines_exit_code" == true ]]; then
+  if [[ "$post_action_exit_code" -ne 0 ]]; then
+    echo "error: post_action exited with '$post_action_exit_code'" >&2
+    exit "$post_action_exit_code"
+  fi
+else
+  if [[ "$test_exit_code" -ne 0 ]]; then
+    echo "error: tests exited with '$test_exit_code'" >&2
+    exit "$test_exit_code"
+  fi
 fi
 
 if [[ "${COVERAGE:-}" -ne 1 || "${APPLE_COVERAGE:-}" -ne 1 ]]; then

apple/testing/default_runner/ios_xctestrun_runner.bzl

@@ -24,7 +24,8 @@ def _get_template_substitutions(
         reuse_simulator,
         xctrunner_entitlements_template,
         pre_action_binary,
-        post_action_binary):
+        post_action_binary,
+        post_action_determines_exit_code):
     substitutions = {
         "device_type": device_type,
         "os_version": os_version,
@@ -41,6 +42,7 @@ def _get_template_substitutions(
         "xctrunner_entitlements_template": xctrunner_entitlements_template,
         "pre_action_binary": pre_action_binary,
         "post_action_binary": post_action_binary,
+        "post_action_determines_exit_code": post_action_determines_exit_code,
     }
 
     return {"%({})s".format(key): value for key, value in substitutions.items()}
@@ -78,8 +80,10 @@ def _ios_xctestrun_runner_impl(ctx):
         pre_action_binary = ctx.executable.pre_action.short_path
         runfiles = runfiles.merge(ctx.attr.pre_action[DefaultInfo].default_runfiles)
 
+    post_action_determines_exit_code = False
     if ctx.executable.post_action:
         post_action_binary = ctx.executable.post_action.short_path
+        post_action_determines_exit_code = ctx.attr.post_action_determines_exit_code
         runfiles = runfiles.merge(ctx.attr.post_action[DefaultInfo].default_runfiles)
 
     ctx.actions.expand_template(
@@ -100,6 +104,7 @@ def _ios_xctestrun_runner_impl(ctx):
             xctrunner_entitlements_template = ctx.file._xctrunner_entitlements_template.short_path,
             pre_action_binary = pre_action_binary,
             post_action_binary = post_action_binary,
+            post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
         ),
     )
 
@@ -191,6 +196,12 @@ A binary to run prior to test execution. Runs after simulator creation. Sets the
             cfg = "exec",
             doc = """
 A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, the `$TEST_XCRESULT_BUNDLE_PATH` environment variable if the test run produces an XCResult bundle, and any other variables available to the test runner.
+""",
+        ),
+        "post_action_determines_exit_code": attr.bool(
+            default = False,
+            doc = """
+When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.
 """,
         ),
         "_simulator_creator": attr.label(

apple/testing/default_runner/ios_xctestrun_runner.template.sh

@@ -550,17 +550,26 @@ fi
 
 # Run a post-action binary, if provided.
 post_action_binary=%(post_action_binary)s
+post_action_determines_exit_code="%(post_action_determines_exit_code)s"
+post_action_exit_code=0
 if [[ -n "${result_bundle_path:-}" ]]; then
   TEST_EXIT_CODE=$test_exit_code \
     TEST_LOG_FILE="$testlog" \
     SIMULATOR_UDID="$simulator_id" \
     TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \
-    "$post_action_binary"
+    "$post_action_binary" || post_action_exit_code=$?
 else
   TEST_EXIT_CODE=$test_exit_code \
     TEST_LOG_FILE="$testlog" \
     SIMULATOR_UDID="$simulator_id" \
-    "$post_action_binary"
+    "$post_action_binary" || post_action_exit_code=$?
+fi
+
+if [[ "$post_action_determines_exit_code" == true ]]; then
+  if [[ "$post_action_exit_code" -ne 0 ]]; then
+    echo "error: post_action exited with '$post_action_exit_code'" >&2
+    exit "$post_action_exit_code"
+  fi
 fi
 
 if [[
@@ -586,9 +595,16 @@ if [[ "${COLLECT_PROFDATA:-0}" == "1" && -f "$profdata" ]]; then
   cp -R "$profdata" "$TEST_UNDECLARED_OUTPUTS_DIR"
 fi
 
-if [[ "$test_exit_code" -ne 0 ]]; then
-  echo "error: tests exited with '$test_exit_code'" >&2
-  exit "$test_exit_code"
+if [[ "$post_action_determines_exit_code" == true ]]; then
+  if [[ "$post_action_exit_code" -ne 0 ]]; then
+    echo "error: post_action exited with '$post_action_exit_code'" >&2
+    exit "$post_action_exit_code"
+  fi
+else
+  if [[ "$test_exit_code" -ne 0 ]]; then
+    echo "error: tests exited with '$test_exit_code'" >&2
+    exit "$test_exit_code"
+  fi
 fi
 
 if [[ "${ERROR_ON_NO_TESTS_RAN:-1}" == "1" ]]; then

apple/testing/default_runner/macos_test_runner.bzl

@@ -57,12 +57,14 @@ def _get_template_substitutions(
         *,
         xctestrun_template,
         pre_action_binary,
-        post_action_binary):
+        post_action_binary,
+        post_action_determines_exit_code):
     """Returns the template substitutions for this runner."""
     subs = {
         "xctestrun_template": xctestrun_template.short_path,
         "pre_action_binary": pre_action_binary,
         "post_action_binary": post_action_binary,
+        "post_action_determines_exit_code": post_action_determines_exit_code,
     }
 
     return {"%(" + k + ")s": subs[k] for k in subs}
@@ -101,8 +103,10 @@ def _macos_test_runner_impl(ctx):
         pre_action_binary = ctx.executable.pre_action.short_path
         runfiles = runfiles.merge(ctx.attr.pre_action[DefaultInfo].default_runfiles)
 
+    post_action_determines_exit_code = False
     if ctx.executable.post_action:
         post_action_binary = ctx.executable.post_action.short_path
+        post_action_determines_exit_code = ctx.attr.post_action_determines_exit_code
         runfiles = runfiles.merge(ctx.attr.post_action[DefaultInfo].default_runfiles)
 
     ctx.actions.expand_template(
@@ -112,6 +116,7 @@ def _macos_test_runner_impl(ctx):
             xctestrun_template = preprocessed_xctestrun_template,
             pre_action_binary = pre_action_binary,
             post_action_binary = post_action_binary,
+            post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
         ),
     )
 
@@ -139,6 +144,12 @@ A binary to run prior to test execution. Sets any environment variables availabl
             cfg = "exec",
             doc = """
 A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$TEST_XCRESULT_BUNDLE_PATH` environment variables, in addition to any other variables available to the test runner.
+""",
+        ),
+        "post_action_determines_exit_code": attr.bool(
+            default = False,
+            doc = """
+When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.
 """,
         ),
         "_test_template": attr.label(

apple/testing/default_runner/macos_test_runner.template.sh

@@ -157,14 +157,23 @@ xcodebuild test-without-building \
 
 # Run a post-action binary, if provided.
 post_action_binary=%(post_action_binary)s
+post_action_determines_exit_code="%(post_action_determines_exit_code)s"
+post_action_exit_code=0
 TEST_EXIT_CODE=$test_exit_code \
   TEST_LOG_FILE="$testlog" \
   TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \
-  "$post_action_binary"
+  "$post_action_binary" || post_action_exit_code=$?
 
-if [[ "$test_exit_code" -ne 0 ]]; then
-  echo "error: tests exited with '$test_exit_code'" >&2
-  exit "$test_exit_code"
+if [[ "$post_action_determines_exit_code" == true ]]; then
+  if [[ "$post_action_exit_code" -ne 0 ]]; then
+    echo "error: post_action exited with '$post_action_exit_code'" >&2
+    exit "$post_action_exit_code"
+  fi
+else
+  if [[ "$test_exit_code" -ne 0 ]]; then
+    echo "error: tests exited with '$test_exit_code'" >&2
+    exit "$test_exit_code"
+  fi
 fi
 
 if [[ "${COVERAGE:-}" -ne 1 ]]; then

doc/rules-ios.md

@@ -529,8 +529,8 @@ Builds and bundles an iOS Sticker Pack Extension.
 ## ios_test_runner
 
 <pre>
-ios_test_runner(<a href="#ios_test_runner-name">name</a>, <a href="#ios_test_runner-device_type">device_type</a>, <a href="#ios_test_runner-execution_requirements">execution_requirements</a>, <a href="#ios_test_runner-os_version">os_version</a>, <a href="#ios_test_runner-post_action">post_action</a>, <a href="#ios_test_runner-pre_action">pre_action</a>,
-                <a href="#ios_test_runner-test_environment">test_environment</a>)
+ios_test_runner(<a href="#ios_test_runner-name">name</a>, <a href="#ios_test_runner-device_type">device_type</a>, <a href="#ios_test_runner-execution_requirements">execution_requirements</a>, <a href="#ios_test_runner-os_version">os_version</a>, <a href="#ios_test_runner-post_action">post_action</a>,
+                <a href="#ios_test_runner-post_action_determines_exit_code">post_action_determines_exit_code</a>, <a href="#ios_test_runner-pre_action">pre_action</a>, <a href="#ios_test_runner-test_environment">test_environment</a>)
 </pre>
 
 Rule to identify an iOS runner that runs tests for iOS.
@@ -557,6 +557,7 @@ Outputs:
 | <a id="ios_test_runner-execution_requirements"></a>execution_requirements |  Dictionary of strings to strings which specifies the execution requirements for the runner. In most common cases, this should not be used.   | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | optional |  `{"requires-darwin": ""}`  |
 | <a id="ios_test_runner-os_version"></a>os_version |  The os version of the iOS simulator to run test. The supported os versions correspond to the output of `xcrun simctl list runtimes`. ' 'E.g., 11.2, 9.3. By default, it is the latest supported version of the device type.'   | String | optional |  `""`  |
 | <a id="ios_test_runner-post_action"></a>post_action |  A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE` environment variable, in addition to any other variables available to the test runner.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional |  `None`  |
+| <a id="ios_test_runner-post_action_determines_exit_code"></a>post_action_determines_exit_code |  When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.   | Boolean | optional |  `False`  |
 | <a id="ios_test_runner-pre_action"></a>pre_action |  A binary to run prior to test execution. Runs after simulator creation. Sets any environment variables available to the test runner.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional |  `None`  |
 | <a id="ios_test_runner-test_environment"></a>test_environment |  Optional dictionary with the environment variables that are to be propagated into the XCTest invocation.   | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | optional |  `{}`  |
 
@@ -665,8 +666,9 @@ of the attributes inherited by all test rules, please check the
 
 <pre>
 ios_xctestrun_runner(<a href="#ios_xctestrun_runner-name">name</a>, <a href="#ios_xctestrun_runner-attachment_lifetime">attachment_lifetime</a>, <a href="#ios_xctestrun_runner-command_line_args">command_line_args</a>, <a href="#ios_xctestrun_runner-create_xcresult_bundle">create_xcresult_bundle</a>,
-                     <a href="#ios_xctestrun_runner-destination_timeout">destination_timeout</a>, <a href="#ios_xctestrun_runner-device_type">device_type</a>, <a href="#ios_xctestrun_runner-os_version">os_version</a>, <a href="#ios_xctestrun_runner-post_action">post_action</a>, <a href="#ios_xctestrun_runner-pre_action">pre_action</a>, <a href="#ios_xctestrun_runner-random">random</a>,
-                     <a href="#ios_xctestrun_runner-reuse_simulator">reuse_simulator</a>, <a href="#ios_xctestrun_runner-xcodebuild_args">xcodebuild_args</a>)
+                     <a href="#ios_xctestrun_runner-destination_timeout">destination_timeout</a>, <a href="#ios_xctestrun_runner-device_type">device_type</a>, <a href="#ios_xctestrun_runner-os_version">os_version</a>, <a href="#ios_xctestrun_runner-post_action">post_action</a>,
+                     <a href="#ios_xctestrun_runner-post_action_determines_exit_code">post_action_determines_exit_code</a>, <a href="#ios_xctestrun_runner-pre_action">pre_action</a>, <a href="#ios_xctestrun_runner-random">random</a>, <a href="#ios_xctestrun_runner-reuse_simulator">reuse_simulator</a>,
+                     <a href="#ios_xctestrun_runner-xcodebuild_args">xcodebuild_args</a>)
 </pre>
 
 This rule creates a test runner for iOS tests that uses xctestrun files to run
@@ -719,6 +721,7 @@ in Xcode.
 | <a id="ios_xctestrun_runner-device_type"></a>device_type |  The device type of the iOS simulator to run test. The supported types correspond to the output of `xcrun simctl list devicetypes`. E.g., iPhone X, iPad Air. By default, it reads from --ios_simulator_device or falls back to some device.   | String | optional |  `""`  |
 | <a id="ios_xctestrun_runner-os_version"></a>os_version |  The os version of the iOS simulator to run test. The supported os versions correspond to the output of `xcrun simctl list runtimes`. E.g., 15.5. By default, it reads --ios_simulator_version and then falls back to the latest supported version.   | String | optional |  `""`  |
 | <a id="ios_xctestrun_runner-post_action"></a>post_action |  A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, the `$TEST_XCRESULT_BUNDLE_PATH` environment variable if the test run produces an XCResult bundle, and any other variables available to the test runner.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional |  `None`  |
+| <a id="ios_xctestrun_runner-post_action_determines_exit_code"></a>post_action_determines_exit_code |  When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.   | Boolean | optional |  `False`  |
 | <a id="ios_xctestrun_runner-pre_action"></a>pre_action |  A binary to run prior to test execution. Runs after simulator creation. Sets the `$SIMULATOR_UDID` environment variable, in addition to any other variables available to the test runner.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional |  `None`  |
 | <a id="ios_xctestrun_runner-random"></a>random |  Whether to run the tests in random order to identify unintended state dependencies.   | Boolean | optional |  `False`  |
 | <a id="ios_xctestrun_runner-reuse_simulator"></a>reuse_simulator |  Toggle simulator reuse. The default behavior is to reuse an existing device of the same type and OS version. When disabled, a new simulator is created before testing starts and shutdown when the runner completes.   | Boolean | optional |  `True`  |

test/bazel_testrunner.sh

@@ -79,6 +79,7 @@ module(name = "build_bazel_rules_apple_integration_tests", version = "0")
 bazel_dep(name = "apple_support", version = "0.11.0", repo_name = "build_bazel_apple_support")
 bazel_dep(name = "rules_swift", version = "2.0.0", repo_name = "build_bazel_rules_swift")
 bazel_dep(name = "rules_apple", version = "0", repo_name = "build_bazel_rules_apple")
+bazel_dep(name = "rules_shell", version = "0.3.0")
 
 xcode_configure = use_extension("@bazel_tools//tools/osx:xcode_configure.bzl", "xcode_configure_extension")
 use_repo(xcode_configure, "local_config_xcode")

test/ios_test_runner_unit_test.sh

@@ -80,6 +80,27 @@ ios_test_runner(
     post_action = ":post_action",
 )
 
+genrule(
+  name = "post_action_soft_fail_gen",
+  executable = True,
+  outs = ["post_action_soft_fail.bash"],
+  cmd = """
+echo 'echo "POST-ACTION: Soft failing." && exit 0' > \$@
+""",
+)
+
+sh_binary(
+  name = "post_action_soft_fail",
+  srcs = [":post_action_soft_fail_gen"],
+)
+
+ios_test_runner(
+    name = "ios_x86_64_sim_runner_with_soft_fail",
+    device_type = "iPhone Xs",
+    post_action = ":post_action_soft_fail",
+    post_action_determines_exit_code = True,
+)
+
 EOF
 }
 
@@ -300,6 +321,14 @@ ios_unit_test(
     runner = ":ios_x86_64_sim_runner",
 )
 
+ios_unit_test(
+    name = 'SoftFailingUnitTest',
+    infoplists = ["FailUnitTest-Info.plist"],
+    deps = [":fail_unit_test_lib"],
+    minimum_os_version = "${MIN_OS_IOS}",
+    runner = ":ios_x86_64_sim_runner_with_soft_fail",
+)
+
 ios_unit_test(
     name = 'FailingWithHost',
     infoplists = ["FailUnitTest-Info.plist"],
@@ -727,6 +756,17 @@ function test_ios_unit_test_fail() {
   expect_log "Executed 1 test, with 1 failure"
 }
 
+function test_ios_unit_test_soft_fail() {
+  create_sim_runners
+  create_ios_unit_tests
+  do_ios_test //ios:SoftFailingUnitTest || fail "should pass"
+
+  expect_log "Test Suite 'FailingUnitTest' failed"
+  expect_log "Test Suite 'SoftFailingUnitTest.xctest' failed"
+  expect_log "Executed 1 test, with 1 failure"
+  expect_log "POST-ACTION: Soft failing."
+}
+
 function test_ios_unit_test_with_host_fail() {
   create_sim_runners
   create_test_host_app