From facb2ee9673f30131f381bb007b52d848f77d5a1 Mon Sep 17 00:00:00 2001 From: Eduardo Nunes Date: Mon, 9 Nov 2026 16:56:31 -0300 Subject: [PATCH] feat: report the tests the skip list actually skipped Each test process that reads a skip list writes skipped.tsv beside its output (`framework\tmodule\tsuite\tname`): the XCTest tests it removed, once the removal took effect, and the Swift Testing tests the trait cancelled. A reader can then tell a listed test that was skipped from one that ran anyway: a suite without the trait, Swift before 6.3, or an XCTest without -setTests:. The file is created empty as soon as the list is read, and removed when a line can't be written, so a missing file never reads as "nothing skipped". It's only written with TEST_COVERAGE_ATTRIBUTION_DIR. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 7 +- README.md | 10 ++- .../CoverageAttributionTrait.swift | 3 +- .../TestCoverageAttributionObserver.m | 77 ++++++++++++++++++- .../include/TestCoverageAttributionObserver.h | 5 ++ .../AttributionTests.swift | 35 +++++---- .../SkipListTests.swift | 63 +++++++++++++-- .../TestCoverageAttributionTests/Skips.swift | 34 ++++++++ 8 files changed, 200 insertions(+), 34 deletions(-) create mode 100644 Tests/TestCoverageAttributionTests/Skips.swift diff --git a/AGENTS.md b/AGENTS.md index d08db4a..28a1a1a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,13 +3,13 @@ A Swift package that test targets link to record which coverage counters each test moves (per-test coverage attribution). Tuist's CLI reduces the output to per-test evidence; the record layout is a contract with it (`TuistKit.CoverageObserverOutput` in tuist/tuist). ## Layout -- `Sources/TestCoverageAttributionObserver` - Objective-C, Foundation only. XCTest is looked up at run time (class, protocol and selectors by name), so nothing links XCTest. It reads each instrumented image's `__llvm_prf_cnts` directly and never calls the LLVM profile runtime. Registers its XCTest observer from a constructor when the test bundle loads, and hooks `-[XCUIApplication launch]`/`-activate` once the bundle starts, to list the app processes UI tests launch. Its constructor also reads the skip list (`TEST_COVERAGE_ATTRIBUTION_SKIP_FILE`) and hooks `+[XCTestCase defaultTestSuite]` to remove the listed tests; it lives in this file because `-ObjC` loads only archive members that define a class. -- `Sources/TestCoverageAttribution` - the Swift Testing trait (`.coverageAttribution`) that calls `test_coverage_attribution_scope_begin/end` around each test, and cancels the tests `test_coverage_attribution_is_skipped` finds in the skip list. Cancelling needs `Test.cancel`, so it's compiled only with Swift 6.3 or later; before, listed tests run, since a test a trait doesn't run is reported as passed. +- `Sources/TestCoverageAttributionObserver` - Objective-C, Foundation only. XCTest is looked up at run time (class, protocol and selectors by name), so nothing links XCTest. It reads each instrumented image's `__llvm_prf_cnts` directly and never calls the LLVM profile runtime. Registers its XCTest observer from a constructor when the test bundle loads, and hooks `-[XCUIApplication launch]`/`-activate` once the bundle starts, to list the app processes UI tests launch. Its constructor also reads the skip list (`TEST_COVERAGE_ATTRIBUTION_SKIP_FILE`) and hooks `+[XCTestCase defaultTestSuite]` to remove the listed tests, reporting the ones it removed in `skipped.tsv`; it lives in this file because `-ObjC` loads only archive members that define a class. +- `Sources/TestCoverageAttribution` - the Swift Testing trait (`.coverageAttribution`) that calls `test_coverage_attribution_scope_begin/end` around each test, and cancels the tests `test_coverage_attribution_is_skipped` finds in the skip list, reporting them with `test_coverage_attribution_report_skipped`. Cancelling needs `Test.cancel`, so it's compiled only with Swift 6.3 or later; before, listed tests run, since a test a trait doesn't run is reported as passed. - `Tests/TestCoverageAttributionTests` - runs the fixtures with attribution on and parses what the observer wrote: `Records.swift` mirrors the `records.bin` layout and `FunctionTable.swift` maps counters to function names as Tuist's reader does. - `Fixtures/Example` - with SwiftPM, xcodebuild on macOS and xcodebuild on an iOS simulator. - `Fixtures/Concurrency` - the overlapping, nested-trait and failure cases, in a package of their own since Swift Testing runs suites in parallel across the process. - `Fixtures/StaticArchive` - links the observer from a static archive, with and without `-ObjC`. - - `Fixtures/UITests` - an app, its UI tests and its hosted unit tests. It's a Tuist project (`Project.swift`), because a UI test needs an app target and SwiftPM can't build apps; `Launches.swift` reads `launches.tsv`, and `SkipListTests.swift` runs the `SkipList` scheme serially, in parallel and as a shard and reads the result bundle. + - `Fixtures/UITests` - an app, its UI tests and its hosted unit tests. It's a Tuist project (`Project.swift`), because a UI test needs an app target and SwiftPM can't build apps; `Launches.swift` reads `launches.tsv`, and `SkipListTests.swift` runs the `SkipList` scheme serially, in parallel and as a shard and reads the result bundle and `skipped.tsv` (`Skips.swift`). ## Rules - Never crash or block the test process: every failure path returns and the tests run as if nothing were linked. @@ -17,6 +17,7 @@ A Swift package that test targets link to record which coverage counters each te - Changing `records.bin`: update `write_record()`, `Records.swift`, the README and Tuist's reader together, and bump `kRecordVersion` when a record gains a field. - Changing `launches.tsv`: update `write_launch()`, `Launches.swift`, the README and Tuist's reader (`CoverageObserverOutput.parseLaunches`) together. - Changing the skip list format: update `load_skip_list()`, `SkipListTests.swift`, the README and Tuist's writer together. +- Changing `skipped.tsv`: update `report_skipped()`, `Skips.swift`, the README and Tuist's reader together. - No `unsafeFlags` in `Package.swift`: they make the package unusable as a versioned dependency. ## Commands diff --git a/README.md b/README.md index 86f37a8..0596015 100644 --- a/README.md +++ b/README.md @@ -94,10 +94,14 @@ TEST_RUNNER_TEST_COVERAGE_ATTRIBUTION_SKIP_FILE="$PWD/skip.txt" xcodebuild test - **Names.** Each line is `module/suite/name`, the way `-skip-testing` names a test and the records name it: the test bundle, the class without its Swift module, and the test as the result bundle names it (`testAppliesDiscount()`). The selector works too (`testAppliesDiscount`), including the bridged selectors of throwing and asynchronous Swift tests (`…AndReturnError:`, `…WithCompletionHandler:`). - **Only tests.** A line without exactly three components is ignored. Skip a whole target with `-skip-testing:CheckoutTests` instead. A suite is never skipped by naming it, because it can gain tests the list doesn't know about. -- **Forgiving.** Blank lines and tests that don't exist are ignored. An unreadable file skips nothing. Skipping doesn't depend on `TEST_COVERAGE_ATTRIBUTION_DIR`. +- **Forgiving.** Blank lines and tests that don't exist are ignored. An unreadable file skips nothing. Skipping doesn't depend on `TEST_COVERAGE_ATTRIBUTION_DIR`, but reporting what was skipped does. - **Where it applies.** `xcodebuild` passes the variable to the test process with the `TEST_RUNNER_` prefix: the app that hosts unit tests, and the runner of UI tests. It works with `test` and `test-without-building`, with `-only-testing`, and with parallel testing, where Xcode lists the tests, without the removed ones, before it hands them out to its clones. `swift test` reads it from the environment. - **What a skipped test looks like.** It's removed from the suite XCTest builds for its class, so it doesn't run and isn't in the result bundle. A class whose tests are all removed doesn't run its class `setUp` and `tearDown`. A class with tests left runs them as usual, class `setUp` included. - **Swift Testing.** In a suite with the `.coverageAttribution` trait, a listed test is cancelled before it starts, every argument of a parameterized test included, and reported as skipped, whether tests run in parallel or not. The line names it as the records do: the Swift module, the innermost suite, and the test's display name when it declares one (`Applies a discount`), its function otherwise (`appliesDiscount()`). A display name with a `/` can't be listed. This needs Swift 6.3 (Xcode 26.4) or later: before, a trait can skip a test only by not running it, which reports it as passed, so the listed tests run. Listed tests in suites without the trait run too. +- **What was skipped.** A listed test can still run: a suite without the trait, Swift before 6.3, an XCTest without the private method partial skipping uses (see [Limitations](#limitations)). So with `TEST_COVERAGE_ATTRIBUTION_DIR` set, each process that read a skip list writes [`skipped.tsv`](#output), the tests it actually removed or cancelled. A listed test that no process reports ran, or was never part of the run. A test is reported once the removal took effect, by the process that removed it: + - **Serially**, the test process reports every listed test of the classes XCTest builds suites for and every Swift Testing test the trait cancels. A hosted unit test target and a UI test target run in processes of their own, so each reports its own tests. + - **With parallel testing**, the tests are spread over clones, and each reports the listed tests of the classes it builds suites for, a class whose tests are all listed included. A test can be reported by more than one process, such as one that only lists the tests, so read the union across processes. + - **With `-only-testing`**, as in a shard, XCTest builds suites only for the selected classes, so a listed test outside them isn't reported: it isn't part of the run. ## Output @@ -109,6 +113,7 @@ The output is written under `$TEST_COVERAGE_ATTRIBUTION_DIR//`, one directo | `.data`, `.names` | Image `n`'s raw `__llvm_prf_data` and `__llvm_prf_names` sections: what maps a counter to its function. | | `records.bin` | One record per scope. | | `launches.tsv` | One line per app process a UI test launched, written when `XCUIApplication.launch()` or `activate()` returns: the process's pid, the test bundle, the suite and the test, separated by tabs. A launch outside a test, such as in a class `setUp`, has an empty test name. A pid is listed once, with the scope that launched it, even when later tests activate the same process. Only written by a process that launches an app. | +| `skipped.tsv` | One line per test the [skip list](#skipping-tests) had removed (XCTest) or cancelled (Swift Testing) in this process: the framework (`xctest` or `swift-testing`), the test bundle or Swift module, the suite and the test, separated by tabs, named as the records name them (an XCTest test as XCTest names it, `testAppliesDiscount` for `testAppliesDiscount() throws`). A test is listed once. Created, empty, as soon as the process reads a skip list, so an empty file means nothing was skipped. If a line can't be written, or a name holds a tab or a line break, the file is removed: a missing file means the process didn't report, not that it skipped nothing. Only written by a process with a skip list. | A scope is one of the following: - an XCTest test; @@ -127,12 +132,13 @@ The deltas are what turns counters into lines. In the image's coverage mapping ( - It never resets or writes the coverage counters: Xcode's and SwiftPM's own coverage reports are unchanged. - It never crashes or blocks the tests. Any failure stops recording, removes what it wrote for the process so a reader never gets partial output, and lets the tests run. +- `skipped.tsv` is never partial: a test that was skipped is either listed, or the file is gone. Each copy of the package in the process appends its own lines to it, one write per line. - Only one copy records per process. If the package ends up linked more than once into the same process, for example into two test bundles loaded by the same host, the first copy to load does the recording and the others forward to it. The Objective-C runtime then logs `Class TestCoverageAttributionObserver is implemented in both …` once, which is harmless. ## Limitations - **Swift Testing needs the trait.** Tests in suites without it aren't attributed or skipped, tests that run at the same time get no evidence, and skipping them needs Swift 6.3. See [Usage](#usage). -- **Skipping some of a class's tests uses a private method.** XCTest has no public way to remove tests from a class's suite, so the package uses `XCTestSuite`'s private `-setTests:`, which keeps the suite and the class `setUp` it runs. If a future Xcode removes it, a class with some tests listed runs all of them; a class with all of its tests listed is still skipped. A class that overrides `defaultTestSuite` without calling `super` isn't filtered. +- **Skipping some of a class's tests uses a private method.** XCTest has no public way to remove tests from a class's suite, so the package uses `XCTestSuite`'s private `-setTests:`, which keeps the suite and the class `setUp` it runs. If a future Xcode removes it, a class with some tests listed runs all of them; a class with all of its tests listed is still skipped. Either way, `skipped.tsv` lists only the tests that were removed. A class that overrides `defaultTestSuite` without calling `super` isn't filtered. - **UI tests record which app process each test launched, not the app's counters.** The app's coverage comes from the profile Xcode writes for each app process (`-.profraw` beside the run's `Coverage.profdata`). The pid is read from `XCUIApplication`'s private `processID` property; if a future Xcode removes it, nothing is written. Processes the app starts itself, such as extensions, aren't listed. - **Images loaded late aren't observed.** An image loaded after the first test starts, such as a framework the tests `dlopen`, is missing from the records. - **The package's own code shows up in the records.** The package is instrumented like the code under test, so its functions appear in the records, and a reader should ignore them. diff --git a/Sources/TestCoverageAttribution/CoverageAttributionTrait.swift b/Sources/TestCoverageAttribution/CoverageAttributionTrait.swift index f8a3950..83afe72 100644 --- a/Sources/TestCoverageAttribution/CoverageAttributionTrait.swift +++ b/Sources/TestCoverageAttribution/CoverageAttributionTrait.swift @@ -5,7 +5,7 @@ import Testing /// Attributes the code each Swift Testing test executes to that test, by marking where each test /// starts and ends: Swift Testing has no observation center a library can join. It also skips the -/// tests the skip list names, by cancelling them before they run. +/// tests the skip list names, by cancelling them before they run, and reports the ones it cancels. /// /// The trait has to be added to every suite, and Swift Testing runs tests in parallel within a /// process by default, which leaves overlapping tests without evidence. @@ -38,6 +38,7 @@ public struct CoverageAttributionTrait: TestTrait, SuiteTrait, TestScoping { // `Test.cancel` reports the test as skipped. Before Swift 6.3 a trait can only // skip a test by not running it, which reports it as passed, so it runs instead. if test_coverage_attribution_is_skipped(module, suite, name) { + test_coverage_attribution_report_skipped(module, suite, name) try Test.cancel("Listed in TEST_COVERAGE_ATTRIBUTION_SKIP_FILE", sourceLocation: test.sourceLocation) } #endif diff --git a/Sources/TestCoverageAttributionObserver/TestCoverageAttributionObserver.m b/Sources/TestCoverageAttributionObserver/TestCoverageAttributionObserver.m index 116d61e..889f22e 100644 --- a/Sources/TestCoverageAttributionObserver/TestCoverageAttributionObserver.m +++ b/Sources/TestCoverageAttributionObserver/TestCoverageAttributionObserver.m @@ -24,14 +24,17 @@ // .names the image's raw __llvm_prf_names section // records.bin one record per scope, see write_record() // launches.tsv one line per process a UI test launched, see record_launch() +// skipped.tsv one line per test the skip list had removed or cancelled, see report_skipped() // // Skipping: $TEST_COVERAGE_ATTRIBUTION_SKIP_FILE (through xcodebuild, // TEST_RUNNER_TEST_COVERAGE_ATTRIBUTION_SKIP_FILE), an absolute path, names a file of tests not to // run: XCTest tests are removed before XCTest runs them, and the trait cancels Swift Testing tests. -// See load_skip_list(). It works with or without recording. +// See load_skip_list(). It works with or without recording; what it skipped is reported only with +// $TEST_COVERAGE_ATTRIBUTION_DIR. #import #import "TestCoverageAttributionObserver.h" #include +#include #include #include #include @@ -411,6 +414,9 @@ static bool is_class_suite(NSString *name) { // the listed ones. static NSSet *skipped_tests; static IMP original_default_test_suite; +static int skip_report = -1; +static char skip_report_path[PATH_MAX]; +static NSMutableSet *reported_skips; static NSString *skip_identifier(NSString *module, NSString *suite, NSString *name) { for (NSString *suffix in @[@"()", @"AndReturnError:", @"WithCompletionHandler:"]) { @@ -445,21 +451,79 @@ bool test_coverage_attribution_is_skipped(const char *module, const char *suite, return identifier && [skipped_tests containsObject:identifier]; } +// skipped.tsv, beside the process's output: one line per test this process removed or cancelled, +// `framework\tmodule\tsuite\tname`, framework `xctest` or `swift-testing`, named as the records +// name it. A listed test that isn't in it ran here or wasn't part of the run, unless the file is +// missing. The file is created as soon as the list is read, so an empty one says nothing was +// skipped; a write that fails, or a name a line can't hold, removes it, and a reader then can't +// tell what the process skipped rather than believing it skipped less. Each line is one write to a +// file opened for appending, so a second copy of the library in the process, which opens it too, +// never interleaves with this one. A test is written once per copy: XCTest builds a class's suite +// more than once. +static void open_skip_report(const char *dir) { + if (!skipped_tests || !dir || !*dir) return; + char file[PATH_MAX]; + mkdir(dir, 0755); + snprintf(file, sizeof file, "%s/%d", dir, getpid()); + mkdir(file, 0755); + snprintf(skip_report_path, sizeof skip_report_path, "%s/%d/skipped.tsv", dir, getpid()); + skip_report = open(skip_report_path, O_WRONLY | O_CREAT | O_APPEND | O_CLOEXEC, 0644); +} + +static void report_skipped(const char *framework, NSString *module, NSString *suite, NSString *name) { + NSCharacterSet *separators = [NSCharacterSet characterSetWithCharactersInString:@"\t\r\n"]; + bool fits = true; + for (NSString *field in @[module, suite, name]) { + if ([field rangeOfCharacterFromSet:separators].location != NSNotFound) fits = false; + } + NSString *line = [NSString stringWithFormat:@"%s\t%@\t%@\t%@\n", framework, module, suite, name]; + const char *bytes = line.UTF8String; + size_t length = bytes ? strlen(bytes) : 0; + pthread_mutex_lock(&lock); + if (skip_report >= 0) { + if (!reported_skips) reported_skips = [NSMutableSet set]; + if (![reported_skips containsObject:line]) { + if (fits && bytes && write(skip_report, bytes, length) == (ssize_t)length) { + [reported_skips addObject:line]; + } else { + close(skip_report); + skip_report = -1; + unlink(skip_report_path); + } + } + } + pthread_mutex_unlock(&lock); +} + +void test_coverage_attribution_report_skipped(const char *module, const char *suite, const char *name) { + if (!module || !suite || !name) return; + report_skipped("swift-testing", @(module), @(suite), @(name)); +} + // A class's suite holds its tests. When some are listed, the suite is kept and its tests replaced // through XCTestSuite's private -setTests:, since the suite is what runs the class setUp and // tearDown; an XCTest without it runs every test. When all are listed, an empty suite takes its -// place, so the class setUp doesn't run. +// place, so the class setUp doesn't run. The removed tests are reported once the suite without +// them is in place: an XCTest without -setTests: runs them, and reports none. +static void report_removed(NSString *module, NSArray *> *removed) { + for (NSArray *test in removed) report_skipped("xctest", module, test[0], test[1]); +} + static id suite_without_skipped_tests(Class test_class, id suite) { Class test_case = NSClassFromString(@"XCTestCase"); NSArray *tests = [suite valueForKey:@"tests"]; if (!test_case || ![tests isKindOfClass:[NSArray class]]) return suite; NSString *module = [NSBundle bundleForClass:test_class].bundleURL.lastPathComponent.stringByDeletingPathExtension ?: @""; NSMutableArray *kept = [NSMutableArray arrayWithCapacity:tests.count]; + NSMutableArray *> *removed = [NSMutableArray array]; for (id test in tests) { if ([test isKindOfClass:test_case]) { NSString *test_suite, *method; split_xctest_name([test name], &test_suite, &method); - if ([skipped_tests containsObject:skip_identifier(module, test_suite, method)]) continue; + if ([skipped_tests containsObject:skip_identifier(module, test_suite, method)]) { + [removed addObject:@[test_suite, method]]; + continue; + } } [kept addObject:test]; } @@ -468,11 +532,15 @@ static id suite_without_skipped_tests(Class test_class, id suite) { Class xctest_suite = NSClassFromString(@"XCTestSuite"); SEL with_name = NSSelectorFromString(@"testSuiteWithName:"); if (![xctest_suite respondsToSelector:with_name]) return suite; - return ((id (*)(Class, SEL, NSString *))objc_msgSend)(xctest_suite, with_name, [suite name]); + id empty = ((id (*)(Class, SEL, NSString *))objc_msgSend)(xctest_suite, with_name, [suite name]); + if (!empty) return suite; + report_removed(module, removed); + return empty; } SEL set_tests = NSSelectorFromString(@"setTests:"); if (![suite respondsToSelector:set_tests]) return suite; ((void (*)(id, SEL, NSArray *))objc_msgSend)(suite, set_tests, kept); + report_removed(module, removed); return suite; } @@ -587,6 +655,7 @@ static void register_observer(void) { static void start_skipping(void) { load_skip_list(); if (!skipped_tests) return; + open_skip_report(getenv("TEST_COVERAGE_ATTRIBUTION_DIR")); if (NSClassFromString(@"XCTestCase")) { skip_listed_tests(); } else { diff --git a/Sources/TestCoverageAttributionObserver/include/TestCoverageAttributionObserver.h b/Sources/TestCoverageAttributionObserver/include/TestCoverageAttributionObserver.h index 2aede35..71ee5d2 100644 --- a/Sources/TestCoverageAttributionObserver/include/TestCoverageAttributionObserver.h +++ b/Sources/TestCoverageAttributionObserver/include/TestCoverageAttributionObserver.h @@ -19,6 +19,11 @@ void test_coverage_attribution_scope_end(const char *module, const char *suite, /// XCTest tests are. bool test_coverage_attribution_is_skipped(const char *module, const char *suite, const char *name); +/// Reports that the test the skip list names won't run, in `skipped.tsv` beside the process's +/// output: call it once the skip is certain, as right before cancelling the test. Does nothing +/// unless the process has a skip list and `TEST_COVERAGE_ATTRIBUTION_DIR`. +void test_coverage_attribution_report_skipped(const char *module, const char *suite, const char *name); + #ifdef __cplusplus } #endif diff --git a/Tests/TestCoverageAttributionTests/AttributionTests.swift b/Tests/TestCoverageAttributionTests/AttributionTests.swift index 27612a7..f388361 100644 --- a/Tests/TestCoverageAttributionTests/AttributionTests.swift +++ b/Tests/TestCoverageAttributionTests/AttributionTests.swift @@ -67,7 +67,7 @@ struct AttributionTests { /// Linked from a static archive, as Tuist links packages by default, an XCTest-only target /// references nothing in the package, so the linker drops the observer unless the target links /// with `-ObjC`, which loads every archive member that defines an Objective-C class. The skip - /// list is read by the same member: with `-ObjC`, the listed test doesn't run. + /// list is read by the same member: with `-ObjC`, the listed test doesn't run, and is reported. @Test(arguments: [false, true]) func needsObjCToLinkTheObserverFromAStaticArchive(objC: Bool) throws { let scratch = FileManager.default.temporaryDirectory.appending(path: "TestCoverageAttribution-\(UUID().uuidString)") @@ -93,6 +93,7 @@ struct AttributionTests { #expect((test != nil) == objC) #expect(test?.ran("3sum") ?? !objC) #expect(!records.contains { $0.name == "testSkipped" }) + #expect(try Skip.all(in: output) == (objC ? ["xctest ArchiveTests/ArchiveTests/testSkipped"] : [])) } /// Two tests that run at the same time are both marked overlapped, including the one that @@ -189,21 +190,6 @@ struct AttributionTests { return try Record.all(in: output) } - /// The coverage report of the last SwiftPM run of `Fixtures/Example`. - private func swiftPMCoverageReport() throws -> Data { - let output = try Self.execute( - URL(filePath: "/usr/bin/xcrun"), - [ - "swift", "test", "--package-path", Self.packageRoot.appending(path: "Fixtures/Example").path(), - "--scratch-path", Self.packageRoot.appending(path: ".build/fixtures/Example/swiftpm").path(), - "--show-codecov-path", - ], - environment: Self.cleanEnvironment, - ) - let path = try #require(output.split(separator: "\n").last.map(String.init)) - return try Data(contentsOf: URL(filePath: path)) - } - /// This process's environment without the observer's variables, which the package's own test /// run may have set. static var cleanEnvironment: [String: String] { @@ -325,3 +311,20 @@ struct AttributionTests { ) } } + +extension AttributionTests { + /// The coverage report of the last SwiftPM run of `Fixtures/Example`. + private func swiftPMCoverageReport() throws -> Data { + let output = try Self.execute( + URL(filePath: "/usr/bin/xcrun"), + [ + "swift", "test", "--package-path", Self.packageRoot.appending(path: "Fixtures/Example").path(), + "--scratch-path", Self.packageRoot.appending(path: ".build/fixtures/Example/swiftpm").path(), + "--show-codecov-path", + ], + environment: Self.cleanEnvironment, + ) + let path = try #require(output.split(separator: "\n").last.map(String.init)) + return try Data(contentsOf: URL(filePath: path)) + } +} diff --git a/Tests/TestCoverageAttributionTests/SkipListTests.swift b/Tests/TestCoverageAttributionTests/SkipListTests.swift index c0ed63d..3eb7166 100644 --- a/Tests/TestCoverageAttributionTests/SkipListTests.swift +++ b/Tests/TestCoverageAttributionTests/SkipListTests.swift @@ -15,7 +15,8 @@ extension AttributionTests { /// run its class `setUp`, which crashes. The tests left run, after their class `setUp`. Built /// with Swift 6.3 or later, the listed Swift Testing tests of a suite with the trait are /// skipped, every argument of a parameterized one included; before, they run. A listed Swift - /// Testing test without the trait runs. + /// Testing test without the trait runs. `skipped.tsv` lists the tests each process removed or + /// cancelled, and no other. @Test(arguments: SkipListRun.allCases) func skipsTheListedTests(run: SkipListRun) throws { let scratch = FileManager.default.temporaryDirectory.appending(path: "TestCoverageAttribution-\(UUID().uuidString)") @@ -40,7 +41,9 @@ extension AttributionTests { SkipUITests/SkippedUITests/testSkipped() """.write(to: skipList, atomically: true, encoding: .utf8) let resultBundle = scratch.appending(path: "Tests.xcresult") - try runSkipListScheme(run, skipList: skipList, resultBundle: resultBundle) + let output = scratch.appending(path: "output") + try FileManager.default.createDirectory(at: output, withIntermediateDirectories: true) + try runSkipListScheme(run, skipList: skipList, resultBundle: resultBundle, output: output) #if compiler(>=6.3) let listedSwiftTesting = "Skipped" @@ -60,23 +63,60 @@ extension AttributionTests { "AppTests/UntraitedSwiftTestingTests/runs() Passed", "SkipUITests/PartlySkippedUITests/testLaunch() Passed", ])) + + let reports = try Skip.reports(in: output) + #expect(try Skip.all(in: output) == Self.expectedSkips(run), "\(reports)") + } + + /// What `skipsTheListedTests` expects the processes to report skipping, named as the records + /// name the tests: XCTest tests as XCTest names them, without `()` or a bridged suffix. A shard + /// builds the suites of its `-only-testing` classes alone, and runs no Swift Testing test. + private static func expectedSkips(_ run: SkipListRun) -> [String] { + var skips = [ + "xctest AppTests/PartlySkippedTests/testSkipped", + "xctest AppTests/PartlySkippedTests/testSkippedAsync", + "xctest AppTests/PartlySkippedTests/testSkippedThrowing", + "xctest AppTests/SkippedTests/testFirst", + "xctest AppTests/SkippedTests/testSecond", + "xctest SkipUITests/PartlySkippedUITests/testSkipped", + ] + if run != .shard { + skips.append("xctest SkipUITests/SkippedUITests/testSkipped") + #if compiler(>=6.3) + skips += [ + "swift-testing AppTests/SwiftTestingTests/skipped()", + "swift-testing AppTests/SwiftTestingTests/skippedWithArguments(value:)", + ] + #endif + } + return skips.sorted() } /// Built with Swift 6.3 or later, the trait cancels a listed Swift Testing test before it - /// starts, so it leaves no record; before, it runs. The list names it by its display name. + /// starts, so it leaves no record, and reports it in `skipped.tsv`; before, it runs and the + /// process reports skipping nothing. The list names it by its display name. @Test func skipsTheListedSwiftTestingTests() throws { + var skips: [String] = [] + var reports = 0 let records = try runFixture( .swiftPM, attributing: true, skipList: "ExampleTests/MultiplicationTests/Multiplying by zero\n", - ) + ) { + skips = try Skip.all(in: $0) + reports = try Skip.reports(in: $0).count + } let names = records.filter { $0.kind == .swiftTesting }.map(\.name) #expect(names.contains("multiply()")) + // The file is there even when nothing was skipped. + #expect(reports > 0) #if compiler(>=6.3) #expect(!names.contains("Multiplying by zero")) + #expect(skips == ["swift-testing ExampleTests/MultiplicationTests/Multiplying by zero"]) #else #expect(names.contains("Multiplying by zero")) + #expect(skips.isEmpty) #endif } @@ -113,15 +153,14 @@ extension AttributionTests { } /// Generates `Fixtures/UITests` and runs its `SkipList` scheme on an iPhone simulator, the way - /// `run` says, with the skip list. - private func runSkipListScheme(_ run: SkipListRun, skipList: URL, resultBundle: URL) throws { + /// `run` says, with the skip list, writing its output to `output`. + private func runSkipListScheme(_ run: SkipListRun, skipList: URL, resultBundle: URL, output: URL) throws { let fixture = Self.packageRoot.appending(path: "Fixtures/UITests") let derivedData = Self.packageRoot.appending(path: ".build/fixtures/UITests/skip-list") let simulator = try Self.iPhoneSimulator(fixture: Self.packageRoot.appending(path: "Fixtures/Example")) try Self.execute(URL(filePath: "/usr/bin/xcrun"), ["simctl", "bootstatus", simulator, "-b"]) try Self.execute(URL(filePath: "/usr/bin/env"), ["tuist", "generate", "--no-open"], directory: fixture) - var environment = Self.cleanEnvironment - environment["TEST_RUNNER_TEST_COVERAGE_ATTRIBUTION_SKIP_FILE"] = skipList.path() + let environment = Self.skipListEnvironment(skipList: skipList, output: output) let destination = ["-destination", "id=\(simulator)"] let parallel = ["-parallel-testing-enabled", "YES", "-parallel-testing-worker-count", "2"] @@ -169,6 +208,14 @@ extension AttributionTests { } } + /// The environment xcodebuild passes the skip list and the output directory to the test processes in. + private static func skipListEnvironment(skipList: URL, output: URL) -> [String: String] { + cleanEnvironment.merging([ + "TEST_RUNNER_TEST_COVERAGE_ATTRIBUTION_SKIP_FILE": skipList.path(), + "TEST_RUNNER_TEST_COVERAGE_ATTRIBUTION_DIR": output.path(), + ]) { $1 } + } + /// How long one xcodebuild invocation may take before it is stopped and its output reported, /// well under the suite's time limit. static let xcodebuildTimeout: TimeInterval = 360 diff --git a/Tests/TestCoverageAttributionTests/Skips.swift b/Tests/TestCoverageAttributionTests/Skips.swift new file mode 100644 index 0000000..78d8caa --- /dev/null +++ b/Tests/TestCoverageAttributionTests/Skips.swift @@ -0,0 +1,34 @@ +import Foundation + +/// A line of `skipped.tsv`: a test the skip list had removed (XCTest) or cancelled (Swift Testing). +struct Skip: Hashable, CustomStringConvertible { + let framework: String + let module: String + let suite: String + let name: String + + var description: String { + "\(framework) \(module)/\(suite)/\(name)" + } + + /// What each process under `directory` reported skipping, keyed by its pid; a process without + /// `skipped.tsv` is left out. + static func reports(in directory: URL) throws -> [String: [Skip]] { + var reports: [String: [Skip]] = [:] + for process in try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) { + let file = process.appending(path: "skipped.tsv") + guard FileManager.default.fileExists(atPath: file.path()) else { continue } + reports[process.lastPathComponent] = try String(contentsOf: file, encoding: .utf8) + .split(separator: "\n").map { line in + let fields = line.split(separator: "\t", omittingEmptySubsequences: false).map(String.init) + return Skip(framework: fields[0], module: fields[1], suite: fields[2], name: fields[3]) + } + } + return reports + } + + /// The tests any process under `directory` reported skipping, as sorted descriptions. + static func all(in directory: URL) throws -> [String] { + Set(try reports(in: directory).values.joined()).map(\.description).sorted() + } +}