Add support for handling Issues with lower severity (#21)

Author: x-sheep

Sources/PropertyBased/FixedSeedTrait.swift

@@ -41,7 +41,7 @@ public struct FixedSeedTrait: TestTrait, TestScoping {
             }
         }
 
-        if foundIssues == 0 {
+        if foundIssues.errors == 0 {
             Issue.record(
                 "A fixed seed was used, but no expectation failure occured. The property was not tested fully.",
                 sourceLocation: sourceLocation)

Sources/PropertyBased/IssueCounting.swift

@@ -7,17 +7,34 @@
 
 import Testing
 
+struct CountResult {
+    var errors: Int = 0
+    var warnings: Int = 0
+}
+
 func countIssues(isolation: isolated (any Actor)? = #isolation, suppress: Bool, perform: () async throws -> Void) async
-    -> Int
+    -> CountResult
 {
-    let found = Mutex(0)
+    let found = Mutex(CountResult())
 
     #if swift(>=6.2)
     nonisolated(unsafe) let closure = perform
 
-    let handler = IssueHandlingTrait.filterIssues { _ in
-        found.withLock { $0 += 1 }
-        return !suppress
+    let handler = IssueHandlingTrait.compactMapIssues { input in
+        var issue = input
+        if FixedSeedTrait.fixedRandom != nil {
+            issue.isWarning = false
+        }
+
+        found.withLock {
+            if issue.isWarning {
+                $0.warnings += 1
+            } else {
+                $0.errors += 1
+            }
+        }
+        guard !suppress else { return nil }
+        return issue
     }
 
     try? await handler.provideScope(for: Test.current!, testCase: Test.Case.current) {
@@ -27,14 +44,30 @@ func countIssues(isolation: isolated (any Actor)? = #isolation, suppress: Bool,
     try? await withKnownIssue(isIntermittent: true, isolation: isolation) {
         try await perform()
     } matching: { _ in
-        found.withLock { $0 += 1 }
+        found.withLock { $0.errors += 1 }
         return suppress
     }
     #endif
 
     return found.withLock { $0 }
 }
 
+extension Issue {
+    #if swift(>=6.3)
+    var isWarning: Bool {
+        get { severity == .warning }
+        set {
+            severity = newValue ? .warning : .error
+        }
+    }
+    #else
+    var isWarning: Bool {
+        get { false }
+        set {}
+    }
+    #endif
+}
+
 @inlinable
 func run(_ closure: () async throws -> Void, in isolation: isolated (any Actor)?) async throws {
     try await closure()

Sources/PropertyBased/PropertyCheck.swift

@@ -115,6 +115,7 @@ public func propertyCheck<InputValue, ResultValue>(
     guard count > 0 else { return }
 
     let fixedRng = FixedSeedTrait.fixedRandom
+    var rngWithIssues: (rng: Xoshiro, value: InputValue, isError: Bool)?
 
     let actualCount = fixedRng != nil ? 1 : count
 
@@ -130,62 +131,88 @@ public func propertyCheck<InputValue, ResultValue>(
             try await body(resultValue)
         }
 
-        if foundIssues > 0 {
-            let seed = rngCopy.traitHint
+        if foundIssues.errors > 0 {
+            rngWithIssues = (rngCopy, inputValue, isError: true)
+            break
+        } else if rngWithIssues == nil, foundIssues.warnings > 0 {
+            rngWithIssues = (rngCopy, inputValue, isError: false)
+        }
+    }
 
-            var shrunkenInput = inputValue
+    if let rngWithIssues {
+        let seed = rngWithIssues.rng.traitHint
 
-            var didShrink = EnableShrinkTrait.isEnabled
-            var shrinkCount = 0
-            while didShrink {
-                didShrink = false
-                let candidates = input._shrinker(shrunkenInput)
+        var shrunkenInput = rngWithIssues.value
+        var isErrorLevel = rngWithIssues.isError
 
-                for c in candidates {
-                    guard !Task.isCancelled else { return }
+        var didShrink = EnableShrinkTrait.isEnabled
+        var shrinkCount = 0
+        while didShrink {
+            didShrink = false
+            let candidates = input._shrinker(shrunkenInput)
 
-                    guard let mappedShrunk = input._mapFilter(c) else { continue }
+            for c in candidates {
+                guard !Task.isCancelled else { return }
 
-                    let shrunkIssues = await countIssues(isolation: isolation, suppress: true) {
-                        try await body(mappedShrunk)
-                    }
+                guard let mappedShrunk = input._mapFilter(c) else { continue }
 
-                    if shrunkIssues > 0 {
-                        didShrink = true
-                        shrinkCount += 1
-                        shrunkenInput = c
-                        break
-                    }
+                let shrunkIssues = await countIssues(isolation: isolation, suppress: true) {
+                    try await body(mappedShrunk)
                 }
-            }
 
-            // If previous inputs were suppressed, run the block one more time to fully record all issues.
-            if EnableShrinkTrait.isEnabled {
-                _ = await countIssues(isolation: isolation, suppress: false) {
-                    try await body(input._mapFilter(shrunkenInput)!)
+                if shrunkIssues.errors > 0 || (shrunkIssues.warnings > 0 && !isErrorLevel) {
+                    didShrink = true
+                    shrinkCount += 1
+                    shrunkenInput = c
+                    break
                 }
             }
+        }
+
+        // If previous inputs were suppressed, run the block one more time to fully record all issues.
+        if EnableShrinkTrait.isEnabled {
+            let finalCount = await countIssues(isolation: isolation, suppress: false) {
+                try await body(input._mapFilter(shrunkenInput)!)
+            }
+            if finalCount.errors > 0 {
+                isErrorLevel = true
+            }
+        }
 
-            let originalParamLabel = String(describingForTest: resultValue)
+        let resultValue = input._mapFilter(rngWithIssues.value)!
+        let originalParamLabel = String(describingForTest: resultValue)
 
-            let failureMessage: String
+        let failureMessage: String
 
-            if shrinkCount == 0 {
-                failureMessage = "Failure occured with input \(originalParamLabel)."
-            } else {
-                let shrunkParamLabel = String(describingForTest: input._mapFilter(shrunkenInput)!)
-                failureMessage =
-                    "Failure occured with input \(shrunkParamLabel).\n(shrunk down from \(originalParamLabel) after \(shrinkCount) iteration\(shrinkCount != 1 ? "s" : ""))"
+        let issueTypeLabel = isErrorLevel ? "Failure" : "Warning"
+
+        if shrinkCount == 0 {
+            failureMessage = "\(issueTypeLabel) occured with input \(originalParamLabel)."
+        } else {
+            let shrunkParamLabel = String(describingForTest: input._mapFilter(shrunkenInput)!)
+            failureMessage =
+                "\(issueTypeLabel) occured with input \(shrunkParamLabel).\n(shrunk down from \(originalParamLabel) after \(shrinkCount) iteration\(shrinkCount != 1 ? "s" : ""))"
+        }
+
+        if fixedRng == nil {
+            if isErrorLevel {
+                Issue.record(
+                    "\(failureMessage)\n\nAdd `.fixedSeed\(seed)` to the Test to reproduce this issue.",
+                    sourceLocation: sourceLocation)
             }
 
-            if fixedRng == nil {
+            #if swift(>=6.3)
+            if !isErrorLevel {
                 Issue.record(
                     "\(failureMessage)\n\nAdd `.fixedSeed\(seed)` to the Test to reproduce this issue.",
+                    severity: .warning,
                     sourceLocation: sourceLocation)
-            } else {
-                Issue.record("\(failureMessage)", sourceLocation: sourceLocation)
             }
-            return
+            #endif
+
+        } else {
+            Issue.record("\(failureMessage)", sourceLocation: sourceLocation)
         }
+        return
     }
 }

Tests/PropertyBasedTests/ActorTests.swift

@@ -24,7 +24,7 @@ import Testing
                 #expect(i < 500)
             }
         }
-        #expect(issues > 0)
+        #expect(issues.errors > 0)
     }
 
     @OtherActor @Test func testOnOtherActor() async {
@@ -35,6 +35,6 @@ import Testing
                 #expect(i < 500)
             }
         }
-        #expect(issues > 0)
+        #expect(issues.errors > 0)
     }
 }

Tests/PropertyBasedTests/WarningTests.swift

@@ -0,0 +1,84 @@
+//
+//  WarningTests.swift
+//  PropertyBased
+//
+//  Created by Lennard Sprong on 13/04/2026.
+//
+
+#if swift(>=6.3)
+import Testing
+
+@testable import PropertyBased
+
+@Suite struct WarningTests {
+    @Test func testWarningIssues() async {
+        let count = await countIssues(suppress: true) {
+            await propertyCheck(input: Gen.int(in: 0...100)) { i in
+                if i > 10 {
+                    Issue.record("Above 10", severity: .warning)
+                }
+            }
+        }
+
+        #expect(count.errors == 0)
+        #expect(count.warnings > 0)
+
+        let texts = await gatherIssues {
+            await propertyCheck(input: Gen.int(in: 0...100)) { i in
+                if i > 10 {
+                    Issue.record("Above 10", severity: .warning)
+                }
+            }
+        }
+
+        #expect(
+            texts.contains(where: {
+                $0.contains("Warning")
+            }))
+    }
+
+    @Test func testErrorSupersedesWarning() async {
+        // Don't fail the rare case where the very first iteration is zero
+        let mutex = Mutex(false)
+
+        let count = await countIssues(suppress: true) {
+            await propertyCheck(input: Gen.int(in: 0...1_000_000)) { i in
+                let wasAboveTen = mutex.withLock {
+                    if i > 10 {
+                        $0 = true
+                    }
+                    return $0
+                }
+
+                if wasAboveTen {
+                    #expect(i > 0)
+                }
+                if i > 10 {
+                    Issue.record("Above 10", severity: .warning)
+                }
+            }
+        }
+
+        #expect(count.errors > 0)
+        #expect(count.warnings == 0)
+    }
+
+    @Test func testFixedSeedUpgradesWarnings() async {
+        let count = await countIssues(suppress: true) {
+            await FixedSeedTrait.fixedSeed(
+                16_227_743_274_320_082_944, 3_104_485_330_390_707_246, 16_893_853_035_950_354_083,
+                9_960_939_931_137_502_728
+            ).provideScope(for: .current!, testCase: .current) {
+                await propertyCheck(input: Gen.int(in: 0...100)) { i in
+                    if i > 10 {
+                        Issue.record("Above 10", severity: .warning)
+                    }
+                }
+            }
+        }
+
+        #expect(count.errors > 0)
+        #expect(count.warnings == 0)
+    }
+}
+#endif