Skip to content

Commit 281186e

Browse files
Tests: Allow post_action to determine exit status (#2713)
Add new attr `post_action_determines_exit_code` to let the test runner post action determine the exit status. Our use case is allow soft failing of certain test targets. ```starlark ios_xctestrun_runner( name = "ios_sim_runner_with_soft_fail", device_type = "iPhone Xs", post_action = ":post_action_soft_fail", post_action_determines_exit_code = True, ) ```
1 parent 5013033 commit 281186e

11 files changed

+211
-20
lines changed

apple/testing/default_runner/ios_test_runner.bzl

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ def _get_template_substitutions(
2727
simulator_creator,
2828
testrunner,
2929
pre_action_binary,
30-
post_action_binary):
30+
post_action_binary,
31+
post_action_determines_exit_code):
3132
"""Returns the template substitutions for this runner."""
3233
subs = {
3334
"device_type": device_type,
@@ -36,6 +37,7 @@ def _get_template_substitutions(
3637
"testrunner_binary": testrunner,
3738
"pre_action_binary": pre_action_binary,
3839
"post_action_binary": post_action_binary,
40+
"post_action_determines_exit_code": post_action_determines_exit_code,
3941
}
4042
return {"%(" + k + ")s": subs[k] for k in subs}
4143

@@ -66,8 +68,10 @@ def _ios_test_runner_impl(ctx):
6668
pre_action_binary = ctx.executable.pre_action.short_path
6769
runfiles = runfiles.merge(ctx.attr.pre_action[DefaultInfo].default_runfiles)
6870

71+
post_action_determines_exit_code = False
6972
if ctx.executable.post_action:
7073
post_action_binary = ctx.executable.post_action.short_path
74+
post_action_determines_exit_code = ctx.attr.post_action_determines_exit_code
7175
runfiles = runfiles.merge(ctx.attr.post_action[DefaultInfo].default_runfiles)
7276

7377
ctx.actions.expand_template(
@@ -80,6 +84,7 @@ def _ios_test_runner_impl(ctx):
8084
testrunner = ctx.executable._testrunner.short_path,
8185
pre_action_binary = pre_action_binary,
8286
post_action_binary = post_action_binary,
87+
post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
8388
),
8489
)
8590
return [
@@ -143,6 +148,12 @@ A binary to run prior to test execution. Runs after simulator creation. Sets any
143148
cfg = "exec",
144149
doc = """
145150
A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE` environment variable, in addition to any other variables available to the test runner.
151+
""",
152+
),
153+
"post_action_determines_exit_code": attr.bool(
154+
default = False,
155+
doc = """
156+
When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.
146157
""",
147158
),
148159
"_test_template": attr.label(

apple/testing/default_runner/ios_test_runner.template.sh

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -282,12 +282,21 @@ cmd=("%(testrunner_binary)s"
282282

283283
# Run a post-action binary, if provided.
284284
post_action_binary=%(post_action_binary)s
285+
post_action_determines_exit_code="%(post_action_determines_exit_code)s"
286+
post_action_exit_code=0
285287
TEST_EXIT_CODE=$test_exit_code \
286-
"$post_action_binary"
288+
"$post_action_binary" || post_action_exit_code=$?
287289

288-
if [[ "$test_exit_code" -ne 0 ]]; then
289-
echo "error: tests exited with '$test_exit_code'" >&2
290-
exit "$test_exit_code"
290+
if [[ "$post_action_determines_exit_code" == true ]]; then
291+
if [[ "$post_action_exit_code" -ne 0 ]]; then
292+
echo "error: post_action exited with '$post_action_exit_code'" >&2
293+
exit "$post_action_exit_code"
294+
fi
295+
else
296+
if [[ "$test_exit_code" -ne 0 ]]; then
297+
echo "error: tests exited with '$test_exit_code'" >&2
298+
exit "$test_exit_code"
299+
fi
291300
fi
292301

293302
if [[ "${COVERAGE:-}" -ne 1 || "${APPLE_COVERAGE:-}" -ne 1 ]]; then

apple/testing/default_runner/ios_xctestrun_runner.bzl

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ def _get_template_substitutions(
2424
reuse_simulator,
2525
xctrunner_entitlements_template,
2626
pre_action_binary,
27-
post_action_binary):
27+
post_action_binary,
28+
post_action_determines_exit_code):
2829
substitutions = {
2930
"device_type": device_type,
3031
"os_version": os_version,
@@ -41,6 +42,7 @@ def _get_template_substitutions(
4142
"xctrunner_entitlements_template": xctrunner_entitlements_template,
4243
"pre_action_binary": pre_action_binary,
4344
"post_action_binary": post_action_binary,
45+
"post_action_determines_exit_code": post_action_determines_exit_code,
4446
}
4547

4648
return {"%({})s".format(key): value for key, value in substitutions.items()}
@@ -78,8 +80,10 @@ def _ios_xctestrun_runner_impl(ctx):
7880
pre_action_binary = ctx.executable.pre_action.short_path
7981
runfiles = runfiles.merge(ctx.attr.pre_action[DefaultInfo].default_runfiles)
8082

83+
post_action_determines_exit_code = False
8184
if ctx.executable.post_action:
8285
post_action_binary = ctx.executable.post_action.short_path
86+
post_action_determines_exit_code = ctx.attr.post_action_determines_exit_code
8387
runfiles = runfiles.merge(ctx.attr.post_action[DefaultInfo].default_runfiles)
8488

8589
ctx.actions.expand_template(
@@ -100,6 +104,7 @@ def _ios_xctestrun_runner_impl(ctx):
100104
xctrunner_entitlements_template = ctx.file._xctrunner_entitlements_template.short_path,
101105
pre_action_binary = pre_action_binary,
102106
post_action_binary = post_action_binary,
107+
post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
103108
),
104109
)
105110

@@ -191,6 +196,12 @@ A binary to run prior to test execution. Runs after simulator creation. Sets the
191196
cfg = "exec",
192197
doc = """
193198
A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, the `$TEST_XCRESULT_BUNDLE_PATH` environment variable if the test run produces an XCResult bundle, and any other variables available to the test runner.
199+
""",
200+
),
201+
"post_action_determines_exit_code": attr.bool(
202+
default = False,
203+
doc = """
204+
When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.
194205
""",
195206
),
196207
"_simulator_creator": attr.label(

apple/testing/default_runner/ios_xctestrun_runner.template.sh

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -550,17 +550,26 @@ fi
550550

551551
# Run a post-action binary, if provided.
552552
post_action_binary=%(post_action_binary)s
553+
post_action_determines_exit_code="%(post_action_determines_exit_code)s"
554+
post_action_exit_code=0
553555
if [[ -n "${result_bundle_path:-}" ]]; then
554556
TEST_EXIT_CODE=$test_exit_code \
555557
TEST_LOG_FILE="$testlog" \
556558
SIMULATOR_UDID="$simulator_id" \
557559
TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \
558-
"$post_action_binary"
560+
"$post_action_binary" || post_action_exit_code=$?
559561
else
560562
TEST_EXIT_CODE=$test_exit_code \
561563
TEST_LOG_FILE="$testlog" \
562564
SIMULATOR_UDID="$simulator_id" \
563-
"$post_action_binary"
565+
"$post_action_binary" || post_action_exit_code=$?
566+
fi
567+
568+
if [[ "$post_action_determines_exit_code" == true ]]; then
569+
if [[ "$post_action_exit_code" -ne 0 ]]; then
570+
echo "error: post_action exited with '$post_action_exit_code'" >&2
571+
exit "$post_action_exit_code"
572+
fi
564573
fi
565574

566575
if [[
@@ -586,9 +595,16 @@ if [[ "${COLLECT_PROFDATA:-0}" == "1" && -f "$profdata" ]]; then
586595
cp -R "$profdata" "$TEST_UNDECLARED_OUTPUTS_DIR"
587596
fi
588597

589-
if [[ "$test_exit_code" -ne 0 ]]; then
590-
echo "error: tests exited with '$test_exit_code'" >&2
591-
exit "$test_exit_code"
598+
if [[ "$post_action_determines_exit_code" == true ]]; then
599+
if [[ "$post_action_exit_code" -ne 0 ]]; then
600+
echo "error: post_action exited with '$post_action_exit_code'" >&2
601+
exit "$post_action_exit_code"
602+
fi
603+
else
604+
if [[ "$test_exit_code" -ne 0 ]]; then
605+
echo "error: tests exited with '$test_exit_code'" >&2
606+
exit "$test_exit_code"
607+
fi
592608
fi
593609

594610
if [[ "${ERROR_ON_NO_TESTS_RAN:-1}" == "1" ]]; then

apple/testing/default_runner/macos_test_runner.bzl

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,14 @@ def _get_template_substitutions(
5757
*,
5858
xctestrun_template,
5959
pre_action_binary,
60-
post_action_binary):
60+
post_action_binary,
61+
post_action_determines_exit_code):
6162
"""Returns the template substitutions for this runner."""
6263
subs = {
6364
"xctestrun_template": xctestrun_template.short_path,
6465
"pre_action_binary": pre_action_binary,
6566
"post_action_binary": post_action_binary,
67+
"post_action_determines_exit_code": post_action_determines_exit_code,
6668
}
6769

6870
return {"%(" + k + ")s": subs[k] for k in subs}
@@ -101,8 +103,10 @@ def _macos_test_runner_impl(ctx):
101103
pre_action_binary = ctx.executable.pre_action.short_path
102104
runfiles = runfiles.merge(ctx.attr.pre_action[DefaultInfo].default_runfiles)
103105

106+
post_action_determines_exit_code = False
104107
if ctx.executable.post_action:
105108
post_action_binary = ctx.executable.post_action.short_path
109+
post_action_determines_exit_code = ctx.attr.post_action_determines_exit_code
106110
runfiles = runfiles.merge(ctx.attr.post_action[DefaultInfo].default_runfiles)
107111

108112
ctx.actions.expand_template(
@@ -112,6 +116,7 @@ def _macos_test_runner_impl(ctx):
112116
xctestrun_template = preprocessed_xctestrun_template,
113117
pre_action_binary = pre_action_binary,
114118
post_action_binary = post_action_binary,
119+
post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
115120
),
116121
)
117122

@@ -139,6 +144,12 @@ A binary to run prior to test execution. Sets any environment variables availabl
139144
cfg = "exec",
140145
doc = """
141146
A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$TEST_XCRESULT_BUNDLE_PATH` environment variables, in addition to any other variables available to the test runner.
147+
""",
148+
),
149+
"post_action_determines_exit_code": attr.bool(
150+
default = False,
151+
doc = """
152+
When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria.
142153
""",
143154
),
144155
"_test_template": attr.label(

apple/testing/default_runner/macos_test_runner.template.sh

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,14 +157,23 @@ xcodebuild test-without-building \
157157

158158
# Run a post-action binary, if provided.
159159
post_action_binary=%(post_action_binary)s
160+
post_action_determines_exit_code="%(post_action_determines_exit_code)s"
161+
post_action_exit_code=0
160162
TEST_EXIT_CODE=$test_exit_code \
161163
TEST_LOG_FILE="$testlog" \
162164
TEST_XCRESULT_BUNDLE_PATH="$result_bundle_path" \
163-
"$post_action_binary"
165+
"$post_action_binary" || post_action_exit_code=$?
164166

165-
if [[ "$test_exit_code" -ne 0 ]]; then
166-
echo "error: tests exited with '$test_exit_code'" >&2
167-
exit "$test_exit_code"
167+
if [[ "$post_action_determines_exit_code" == true ]]; then
168+
if [[ "$post_action_exit_code" -ne 0 ]]; then
169+
echo "error: post_action exited with '$post_action_exit_code'" >&2
170+
exit "$post_action_exit_code"
171+
fi
172+
else
173+
if [[ "$test_exit_code" -ne 0 ]]; then
174+
echo "error: tests exited with '$test_exit_code'" >&2
175+
exit "$test_exit_code"
176+
fi
168177
fi
169178

170179
if [[ "${COVERAGE:-}" -ne 1 ]]; then

doc/rules-ios.md

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -529,8 +529,8 @@ Builds and bundles an iOS Sticker Pack Extension.
529529
## ios_test_runner
530530

531531
<pre>
532-
ios_test_runner(<a href="#ios_test_runner-name">name</a>, <a href="#ios_test_runner-device_type">device_type</a>, <a href="#ios_test_runner-execution_requirements">execution_requirements</a>, <a href="#ios_test_runner-os_version">os_version</a>, <a href="#ios_test_runner-post_action">post_action</a>, <a href="#ios_test_runner-pre_action">pre_action</a>,
533-
<a href="#ios_test_runner-test_environment">test_environment</a>)
532+
ios_test_runner(<a href="#ios_test_runner-name">name</a>, <a href="#ios_test_runner-device_type">device_type</a>, <a href="#ios_test_runner-execution_requirements">execution_requirements</a>, <a href="#ios_test_runner-os_version">os_version</a>, <a href="#ios_test_runner-post_action">post_action</a>,
533+
<a href="#ios_test_runner-post_action_determines_exit_code">post_action_determines_exit_code</a>, <a href="#ios_test_runner-pre_action">pre_action</a>, <a href="#ios_test_runner-test_environment">test_environment</a>)
534534
</pre>
535535

536536
Rule to identify an iOS runner that runs tests for iOS.
@@ -557,6 +557,7 @@ Outputs:
557557
| <a id="ios_test_runner-execution_requirements"></a>execution_requirements | Dictionary of strings to strings which specifies the execution requirements for the runner. In most common cases, this should not be used. | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | optional | `{"requires-darwin": ""}` |
558558
| <a id="ios_test_runner-os_version"></a>os_version | The os version of the iOS simulator to run test. The supported os versions correspond to the output of `xcrun simctl list runtimes`. ' 'E.g., 11.2, 9.3. By default, it is the latest supported version of the device type.' | String | optional | `""` |
559559
| <a id="ios_test_runner-post_action"></a>post_action | A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE` environment variable, in addition to any other variables available to the test runner. | <a href="https://bazel.build/concepts/labels">Label</a> | optional | `None` |
560+
| <a id="ios_test_runner-post_action_determines_exit_code"></a>post_action_determines_exit_code | When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria. | Boolean | optional | `False` |
560561
| <a id="ios_test_runner-pre_action"></a>pre_action | A binary to run prior to test execution. Runs after simulator creation. Sets any environment variables available to the test runner. | <a href="https://bazel.build/concepts/labels">Label</a> | optional | `None` |
561562
| <a id="ios_test_runner-test_environment"></a>test_environment | Optional dictionary with the environment variables that are to be propagated into the XCTest invocation. | <a href="https://bazel.build/rules/lib/dict">Dictionary: String -> String</a> | optional | `{}` |
562563

@@ -665,8 +666,9 @@ of the attributes inherited by all test rules, please check the
665666

666667
<pre>
667668
ios_xctestrun_runner(<a href="#ios_xctestrun_runner-name">name</a>, <a href="#ios_xctestrun_runner-attachment_lifetime">attachment_lifetime</a>, <a href="#ios_xctestrun_runner-command_line_args">command_line_args</a>, <a href="#ios_xctestrun_runner-create_xcresult_bundle">create_xcresult_bundle</a>,
668-
<a href="#ios_xctestrun_runner-destination_timeout">destination_timeout</a>, <a href="#ios_xctestrun_runner-device_type">device_type</a>, <a href="#ios_xctestrun_runner-os_version">os_version</a>, <a href="#ios_xctestrun_runner-post_action">post_action</a>, <a href="#ios_xctestrun_runner-pre_action">pre_action</a>, <a href="#ios_xctestrun_runner-random">random</a>,
669-
<a href="#ios_xctestrun_runner-reuse_simulator">reuse_simulator</a>, <a href="#ios_xctestrun_runner-xcodebuild_args">xcodebuild_args</a>)
669+
<a href="#ios_xctestrun_runner-destination_timeout">destination_timeout</a>, <a href="#ios_xctestrun_runner-device_type">device_type</a>, <a href="#ios_xctestrun_runner-os_version">os_version</a>, <a href="#ios_xctestrun_runner-post_action">post_action</a>,
670+
<a href="#ios_xctestrun_runner-post_action_determines_exit_code">post_action_determines_exit_code</a>, <a href="#ios_xctestrun_runner-pre_action">pre_action</a>, <a href="#ios_xctestrun_runner-random">random</a>, <a href="#ios_xctestrun_runner-reuse_simulator">reuse_simulator</a>,
671+
<a href="#ios_xctestrun_runner-xcodebuild_args">xcodebuild_args</a>)
670672
</pre>
671673

672674
This rule creates a test runner for iOS tests that uses xctestrun files to run
@@ -719,6 +721,7 @@ in Xcode.
719721
| <a id="ios_xctestrun_runner-device_type"></a>device_type | The device type of the iOS simulator to run test. The supported types correspond to the output of `xcrun simctl list devicetypes`. E.g., iPhone X, iPad Air. By default, it reads from --ios_simulator_device or falls back to some device. | String | optional | `""` |
720722
| <a id="ios_xctestrun_runner-os_version"></a>os_version | The os version of the iOS simulator to run test. The supported os versions correspond to the output of `xcrun simctl list runtimes`. E.g., 15.5. By default, it reads --ios_simulator_version and then falls back to the latest supported version. | String | optional | `""` |
721723
| <a id="ios_xctestrun_runner-post_action"></a>post_action | A binary to run following test execution. Runs after testing but before test result handling and coverage processing. Sets the `$TEST_EXIT_CODE`, `$TEST_LOG_FILE`, and `$SIMULATOR_UDID` environment variables, the `$TEST_XCRESULT_BUNDLE_PATH` environment variable if the test run produces an XCResult bundle, and any other variables available to the test runner. | <a href="https://bazel.build/concepts/labels">Label</a> | optional | `None` |
724+
| <a id="ios_xctestrun_runner-post_action_determines_exit_code"></a>post_action_determines_exit_code | When true, the exit code of the test run will be set to the exit code of the post action. This is useful for tests that need to fail the test run based on their own criteria. | Boolean | optional | `False` |
722725
| <a id="ios_xctestrun_runner-pre_action"></a>pre_action | A binary to run prior to test execution. Runs after simulator creation. Sets the `$SIMULATOR_UDID` environment variable, in addition to any other variables available to the test runner. | <a href="https://bazel.build/concepts/labels">Label</a> | optional | `None` |
723726
| <a id="ios_xctestrun_runner-random"></a>random | Whether to run the tests in random order to identify unintended state dependencies. | Boolean | optional | `False` |
724727
| <a id="ios_xctestrun_runner-reuse_simulator"></a>reuse_simulator | Toggle simulator reuse. The default behavior is to reuse an existing device of the same type and OS version. When disabled, a new simulator is created before testing starts and shutdown when the runner completes. | Boolean | optional | `True` |

test/bazel_testrunner.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ module(name = "build_bazel_rules_apple_integration_tests", version = "0")
7979
bazel_dep(name = "apple_support", version = "0.11.0", repo_name = "build_bazel_apple_support")
8080
bazel_dep(name = "rules_swift", version = "2.0.0", repo_name = "build_bazel_rules_swift")
8181
bazel_dep(name = "rules_apple", version = "0", repo_name = "build_bazel_rules_apple")
82+
bazel_dep(name = "rules_shell", version = "0.3.0")
8283
8384
xcode_configure = use_extension("@bazel_tools//tools/osx:xcode_configure.bzl", "xcode_configure_extension")
8485
use_repo(xcode_configure, "local_config_xcode")

test/ios_test_runner_unit_test.sh

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,27 @@ ios_test_runner(
8080
post_action = ":post_action",
8181
)
8282
83+
genrule(
84+
name = "post_action_soft_fail_gen",
85+
executable = True,
86+
outs = ["post_action_soft_fail.bash"],
87+
cmd = """
88+
echo 'echo "POST-ACTION: Soft failing." && exit 0' > \$@
89+
""",
90+
)
91+
92+
sh_binary(
93+
name = "post_action_soft_fail",
94+
srcs = [":post_action_soft_fail_gen"],
95+
)
96+
97+
ios_test_runner(
98+
name = "ios_x86_64_sim_runner_with_soft_fail",
99+
device_type = "iPhone Xs",
100+
post_action = ":post_action_soft_fail",
101+
post_action_determines_exit_code = True,
102+
)
103+
83104
EOF
84105
}
85106

@@ -300,6 +321,14 @@ ios_unit_test(
300321
runner = ":ios_x86_64_sim_runner",
301322
)
302323
324+
ios_unit_test(
325+
name = 'SoftFailingUnitTest',
326+
infoplists = ["FailUnitTest-Info.plist"],
327+
deps = [":fail_unit_test_lib"],
328+
minimum_os_version = "${MIN_OS_IOS}",
329+
runner = ":ios_x86_64_sim_runner_with_soft_fail",
330+
)
331+
303332
ios_unit_test(
304333
name = 'FailingWithHost',
305334
infoplists = ["FailUnitTest-Info.plist"],
@@ -727,6 +756,17 @@ function test_ios_unit_test_fail() {
727756
expect_log "Executed 1 test, with 1 failure"
728757
}
729758

759+
function test_ios_unit_test_soft_fail() {
760+
create_sim_runners
761+
create_ios_unit_tests
762+
do_ios_test //ios:SoftFailingUnitTest || fail "should pass"
763+
764+
expect_log "Test Suite 'FailingUnitTest' failed"
765+
expect_log "Test Suite 'SoftFailingUnitTest.xctest' failed"
766+
expect_log "Executed 1 test, with 1 failure"
767+
expect_log "POST-ACTION: Soft failing."
768+
}
769+
730770
function test_ios_unit_test_with_host_fail() {
731771
create_sim_runners
732772
create_test_host_app

0 commit comments

Comments
 (0)