Reimplement polling expectations to make them more reliable under heavy system loads (#1199)

* Reimplement polling expectations to make them more reliable under heavy system loads

This trades wall-clock time accuracy for reliability. It's nowhere near as important for a polling expectation to finish within the specified duration as it is to actually poll as expected.

Basically, under heavy system loads, the actual gap between polling attempts increases. Under sufficiently heavy loads, with a sufficiently low timeout, we might never have polled in the first place.

Instead, this calculate how many times we should poll by dividing the timeout by the interval. We then poll exactly that many times, sleeping for however long the polling interval is.

If polling interval is ever 0, then we set it to 1 nanosecond.

WaitUntil is also re-implemented in both async and sync, but that's because the older infrastructure for doing polling was replaced. Starting in Swift 6.3, we will emit a warning when WaitUntil is used in Swift Testing.

This change also removes the ability to detect a blocked runloop, so Nimble will no longer detect for that. In my opinion, this is a worthwhile tradeoff, as blocked runloop was a source of test flakiness.

* Bump xcode version, drop < swift 6.0

* Update workflows to specify xcode versions including swift 6

Fix build errors on older xcodes/linux

* Fix swiftlint errors

* Fix async waitUntil waiting until whenever the closure finishes before returning

* Remove unnecessary CoreFoundation import

* Update the test watch simulator

* Check for more CF imports, and guard it where necessary

* Mitigate AsyncAwaitTests.testWaitUntilDoesNotCompleteBeforeRunLoopIsWaiting flakes some more

Author: younata

.github/workflows/carthage.yml

@@ -13,7 +13,7 @@ on:
 jobs:
   carthage:
     name: Carthage Build
-    runs-on: macos-14
+    runs-on: macos-15
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1

.github/workflows/ci-swiftpm.yml

@@ -11,24 +11,24 @@ on:
       - "*"
 
 jobs:
-  swiftpm_darwin_ventura:
+  swiftpm_darwin_sonoma:
     name: SwiftPM, Darwin, Xcode ${{ matrix.xcode }}
-    runs-on: macos-13
+    runs-on: macos-14
     strategy:
       matrix:
-        xcode: ["14.3.1"]
+        xcode: ["16.1"]
     env:
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode }}.app"
     steps:
       - uses: actions/checkout@v4
       - run: ./test swiftpm
 
-  swiftpm_darwin_sonoma:
+  swiftpm_darwin_sequoia:
     name: SwiftPM, Darwin, Xcode ${{ matrix.xcode }}
-    runs-on: macos-14
+    runs-on: macos-15
     strategy:
       matrix:
-        xcode: ["15.3", "16.1"]
+        xcode: ["16.3"]
     env:
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode }}.app"
     steps:
@@ -41,10 +41,9 @@ jobs:
     strategy:
       matrix:
         container:
-          - swift:5.7
-          - swift:5.8
-          - swift:5.9
           - swift:6.0
+          - swift:6.1
+          - swift:6.2
           # - swiftlang/swift:nightly
       fail-fast: false
     container: ${{ matrix.container }}
@@ -61,7 +60,7 @@ jobs:
       - name: Install Swift
         uses: compnerd/gha-setup-swift@main
         with:
-          branch: swift-5.9-release
-          tag: 5.9-RELEASE
+          branch: swift-6.2-release
+          tag: 6.2-RELEASE
       - name: Test Windows
         run: swift test -Xswiftc -suppress-warnings

.github/workflows/ci-xcode.yml

@@ -11,12 +11,12 @@ on:
       - "*"
 
 jobs:
-  xcode_ventura:
+  xcode_sonoma:
     name: Xcode ${{ matrix.xcode }} (Xcode Project)
-    runs-on: macos-13
+    runs-on: macos-14
     strategy:
       matrix:
-        xcode: ["14.3.1"]
+        xcode: ["16.1"]
       fail-fast: false
     env:
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode }}.app"
@@ -27,12 +27,12 @@ jobs:
       - run: ./test tvos
       - run: ./test watchos
 
-  xcode_sonoma:
+  xcode_sequoia:
     name: Xcode ${{ matrix.xcode }} (Xcode Project)
-    runs-on: macos-14
+    runs-on: macos-15
     strategy:
       matrix:
-        xcode: ["15.4", "16.1"]
+        xcode: ["16.3"]
       fail-fast: false
     env:
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode }}.app"
@@ -45,10 +45,10 @@ jobs:
 
   xcode_spm:
     name: Xcode ${{ matrix.xcode }} (Swift Package)
-    runs-on: macos-14
+    runs-on: macos-15
     strategy:
       matrix:
-        xcode: ["16.1"]
+        xcode: ["16.3"]
       fail-fast: false
     env:
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode }}.app"

.github/workflows/cocoapods.yml

@@ -13,7 +13,7 @@ on:
 jobs:
   cocoapods:
     name: CocoaPods Lint
-    runs-on: macos-14
+    runs-on: macos-15
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1

.github/workflows/release.yml

@@ -6,11 +6,11 @@ on:
 
 jobs:
   carthage_archive:
-    name: Darwin, Xcode 14.0
-    runs-on: macos-14
+    name: Darwin, Xcode 16.x
+    runs-on: macos-15
     strategy:
       matrix:
-        xcode: ["16.1"]
+        xcode: ["16.3"]
     env:
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode }}.app"
     steps:

Nimble.xcodeproj/project.pbxproj

@@ -142,10 +142,10 @@
 		898F28B025D9F4C30052B8D0 /* AlwaysFailMatcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 898F28AF25D9F4C30052B8D0 /* AlwaysFailMatcher.swift */; };
 		899441EF2902EE4B00C1FAF9 /* AsyncAwaitTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 899441EE2902EE4B00C1FAF9 /* AsyncAwaitTest.swift */; };
 		899441F82902EF2500C1FAF9 /* DSL+AsyncAwait.swift in Sources */ = {isa = PBXBuildFile; fileRef = 899441F32902EF0900C1FAF9 /* DSL+AsyncAwait.swift */; };
+		89A5126C2E74790600423EDF /* NimbleTimeIntervalTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 89A5126B2E74790200423EDF /* NimbleTimeIntervalTest.swift */; };
 		89B8C60F2C6476A6001F12D3 /* Negation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 89B8C60E2C6476A6001F12D3 /* Negation.swift */; };
 		89B8C6112C6478F2001F12D3 /* NegationTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 89B8C6102C6478F2001F12D3 /* NegationTest.swift */; };
 		89C297CC2A911CDA002A143F /* AsyncTimerSequenceTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 89C297CB2A911CDA002A143F /* AsyncTimerSequenceTest.swift */; };
-		89C297CE2A92AB34002A143F /* AsyncPromiseTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 89C297CD2A92AB34002A143F /* AsyncPromiseTest.swift */; };
 		89D8AC852B3211C600410644 /* CwlCatchException in Frameworks */ = {isa = PBXBuildFile; productRef = 89D8AC842B3211C600410644 /* CwlCatchException */; };
 		89D8AC872B3211EA00410644 /* CwlPosixPreconditionTesting in Frameworks */ = {isa = PBXBuildFile; platformFilters = (tvos, watchos, ); productRef = 89D8AC862B3211EA00410644 /* CwlPosixPreconditionTesting */; };
 		89D8AC892B3211EA00410644 /* CwlPreconditionTesting in Frameworks */ = {isa = PBXBuildFile; platformFilters = (driverkit, ios, maccatalyst, macos, xros, ); productRef = 89D8AC882B3211EA00410644 /* CwlPreconditionTesting */; };
@@ -333,10 +333,10 @@
 		898F28AF25D9F4C30052B8D0 /* AlwaysFailMatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AlwaysFailMatcher.swift; sourceTree = "<group>"; };
 		899441EE2902EE4B00C1FAF9 /* AsyncAwaitTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncAwaitTest.swift; sourceTree = "<group>"; };
 		899441F32902EF0900C1FAF9 /* DSL+AsyncAwait.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "DSL+AsyncAwait.swift"; sourceTree = "<group>"; };
+		89A5126B2E74790200423EDF /* NimbleTimeIntervalTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NimbleTimeIntervalTest.swift; sourceTree = "<group>"; };
 		89B8C60E2C6476A6001F12D3 /* Negation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Negation.swift; sourceTree = "<group>"; };
 		89B8C6102C6478F2001F12D3 /* NegationTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NegationTest.swift; sourceTree = "<group>"; };
 		89C297CB2A911CDA002A143F /* AsyncTimerSequenceTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncTimerSequenceTest.swift; sourceTree = "<group>"; };
-		89C297CD2A92AB34002A143F /* AsyncPromiseTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncPromiseTest.swift; sourceTree = "<group>"; };
 		89EEF5A42A03293100988224 /* AsyncMatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncMatcher.swift; sourceTree = "<group>"; };
 		89EEF5B22A032C2500988224 /* AsyncPredicateTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncPredicateTest.swift; sourceTree = "<group>"; };
 		89EEF5BB2A06210D00988224 /* AsyncHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AsyncHelpers.swift; sourceTree = "<group>"; };
@@ -484,6 +484,7 @@
 			children = (
 				CDC157902511957100EAA480 /* DSLTest.swift */,
 				89F5E096290C37B8001F9377 /* OnFailureThrowsTest.swift */,
+				89A5126B2E74790200423EDF /* NimbleTimeIntervalTest.swift */,
 				89F5E06C290765BB001F9377 /* PollingTest.swift */,
 				892282892B2833B7002DA355 /* PollingTest+Require.swift */,
 				CDBC39B82462EA7D00069677 /* PredicateTest.swift */,
@@ -492,7 +493,6 @@
 				899441EE2902EE4B00C1FAF9 /* AsyncAwaitTest.swift */,
 				8922828E2B283956002DA355 /* AsyncAwaitTest+Require.swift */,
 				89C297CB2A911CDA002A143F /* AsyncTimerSequenceTest.swift */,
-				89C297CD2A92AB34002A143F /* AsyncPromiseTest.swift */,
 				965B0D0B1B62C06D0005AE66 /* UserDescriptionTest.swift */,
 				895644DE2C1B71DE0006EC12 /* SwiftTestingSupportTest.swift */,
 				6CAEDD091CAEA86F003F1584 /* LinuxSupport.swift */,
@@ -939,7 +939,6 @@
 				8969624A2A5FAD5F00A7929D /* AsyncAllPassTest.swift in Sources */,
 				1F4A56671A3B305F009E1637 /* ObjCAsyncTest.m in Sources */,
 				1F925EFD195C186800ED456B /* BeginWithTest.swift in Sources */,
-				89C297CE2A92AB34002A143F /* AsyncPromiseTest.swift in Sources */,
 				89F5E06D290765BB001F9377 /* PollingTest.swift in Sources */,
 				DDB4D5F119FE442800E9D9FE /* MatchTest.swift in Sources */,
 				1F4A56741A3B3210009E1637 /* ObjCBeginWithTest.m in Sources */,
@@ -1004,6 +1003,7 @@
 				DD72EC651A93874A002F7651 /* AllPassTest.swift in Sources */,
 				1F4A569E1A3B3565009E1637 /* ObjCMatchTest.m in Sources */,
 				1F925EEA195C124400ED456B /* BeAnInstanceOfTest.swift in Sources */,
+				89A5126C2E74790600423EDF /* NimbleTimeIntervalTest.swift in Sources */,
 				8923E6102B47D08300F3961A /* MapTest.swift in Sources */,
 				29EA59641B551ED2002D767E /* ThrowErrorTest.swift in Sources */,
 				6CAEDD0B1CAEA86F003F1584 /* LinuxSupport.swift in Sources */,

Package.swift

@@ -1,10 +1,10 @@
-// swift-tools-version:5.7
+// swift-tools-version:6.0
 import PackageDescription
 
 let package = Package(
     name: "Nimble",
     platforms: [
-      .macOS(.v10_15), .iOS(.v13), .tvOS(.v13), .watchOS(.v6)
+      .macOS(.v10_15), .iOS(.v13), .tvOS(.v13), .watchOS(.v6), .visionOS(.v1)
     ],
     products: [
         .library(
@@ -19,7 +19,7 @@ let package = Package(
         ),
     ],
     dependencies: [
-        .package(url: "https://github.com/mattgallagher/CwlPreconditionTesting.git", .upToNextMajor(from: "2.1.0")),
+        .package(url: "https://github.com/mattgallagher/CwlPreconditionTesting.git", .upToNextMajor(from: "2.2.0")),
         .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"),
     ],
     targets: {
@@ -32,14 +32,11 @@ let package = Package(
                 name: "Nimble",
                 dependencies: [
                     .product(name: "CwlPreconditionTesting", package: "CwlPreconditionTesting",
-                             condition: .when(platforms: [.macOS, .iOS, .macCatalyst])),
+                             condition: .when(platforms: [.macOS, .iOS, .macCatalyst, .visionOS])),
                     .product(name: "CwlPosixPreconditionTesting", package: "CwlPreconditionTesting",
                              condition: .when(platforms: [.tvOS, .watchOS]))
                 ],
-                exclude: ["Info.plist"],
-                resources: [
-                    .copy("PrivacyInfo.xcprivacy")
-                ]
+                exclude: ["Info.plist"]
             ),
             .target(
                 name: "NimbleSharedTestHelpers",

Sources/Nimble/Adapters/NimbleEnvironment.swift

@@ -36,20 +36,4 @@ internal class NimbleEnvironment: NSObject {
 
     var suppressTVOSAssertionWarning: Bool = false
     var suppressWatchOSAssertionWarning: Bool = false
-    #if !os(WASI)
-    var awaiter: Awaiter
-    #endif
-
-    override init() {
-        #if !os(WASI)
-        let timeoutQueue = DispatchQueue.global(qos: .userInitiated)
-        awaiter = Awaiter(
-            waitLock: AssertionWaitLock(),
-            asyncQueue: .main,
-            timeoutQueue: timeoutQueue
-        )
-        #endif
-
-        super.init()
-    }
 }

Sources/Nimble/DSL+AsyncAwait.swift

@@ -134,31 +134,17 @@ private enum ErrorResult {
 private func throwableUntil(
     timeout: NimbleTimeInterval,
     sourceLocation: SourceLocation,
-    action: @escaping (@escaping @Sendable () -> Void) async throws -> Void) async {
+    action: @escaping @Sendable (@escaping @Sendable () -> Void) async throws -> Void) async {
         let leeway = timeout.divided
         let result = await performBlock(
-            timeoutInterval: timeout,
+            timeout: timeout,
             leeway: leeway,
-            sourceLocation: sourceLocation) { @MainActor (done: @escaping (ErrorResult) -> Void) async throws -> Void in
-                do {
-                    try await action {
-                        done(.none)
-                    }
-                } catch let e {
-                    done(.error(e))
-                }
-            }
+            sourceLocation: sourceLocation,
+            closure: action
+        )
 
         switch result {
         case .incomplete: internalError("Reached .incomplete state for waitUntil(...).")
-        case .blockedRunLoop:
-            fail(
-                blockedRunLoopErrorMessageFor("-waitUntil()", leeway: leeway),
-                fileID: sourceLocation.fileID,
-                file: sourceLocation.filePath,
-                line: sourceLocation.line,
-                column: sourceLocation.column
-            )
         case .timedOut:
             fail(
                 "Waited more than \(timeout.description)",
@@ -175,15 +161,7 @@ private func throwableUntil(
                 line: sourceLocation.line,
                 column: sourceLocation.column
             )
-        case .completed(.error(let error)):
-            fail(
-                "Unexpected error thrown: \(error)",
-                fileID: sourceLocation.fileID,
-                file: sourceLocation.filePath,
-                line: sourceLocation.line,
-                column: sourceLocation.column
-            )
-        case .completed(.none): // success
+        case .completed: // success
             break
         }
 }