diff --git a/README.md b/README.md index c13664f..636639a 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Property-Based Testing can be used as an alternative for (or in addition to) tes Add the following line to the dependencies array in your `Package.swift` file: ```swift -.package(url: "https://github.com/x-sheep/swift-property-based.git", from: "1.0.0") +.package(url: "https://github.com/x-sheep/swift-property-based.git", from: "2.0.0") ``` Then add the following to your test target: diff --git a/Sources/PropertyBased/Documentation.docc/Generator.md b/Sources/PropertyBased/Documentation.docc/Generator.md index a46c23d..e04603e 100644 --- a/Sources/PropertyBased/Documentation.docc/Generator.md +++ b/Sources/PropertyBased/Documentation.docc/Generator.md @@ -10,6 +10,7 @@ ### Testing a generator - ``run(using:)`` +- ``run(using:limit:)`` ### Grouping generated values diff --git a/Sources/PropertyBased/Gen+Collection.swift b/Sources/PropertyBased/Gen+Collection.swift index f79f576..ce88c9d 100644 --- a/Sources/PropertyBased/Gen+Collection.swift +++ b/Sources/PropertyBased/Gen+Collection.swift @@ -79,14 +79,14 @@ extension Generator { @inlinable public func array(of count: ClosedRange) -> Generator<[ResultValue], ArrayShrink> { return .init( - run: { rng in + run: { rng, limit in let itemCount = Int.random(in: count, using: &rng) var collection: [InputValue] = [] collection.reserveCapacity(itemCount) for _ in 0.. preferredDistanceFromNow @@ -240,7 +240,7 @@ extension Gen where Value == Date { let end = Int(Date().timeIntervalSinceReferenceDate / secondsPerDay) return .init( - run: { rng in Int.random(in: interval, using: &rng) }, + run: { rng, _ in Int.random(in: interval, using: &rng) }, shrink: { $0.shrink(within: interval, towards: end) }, finalResult: { Date(timeIntervalSinceReferenceDate: TimeInterval($0) * secondsPerDay) } ) diff --git a/Sources/PropertyBased/Gen+Frequency.swift b/Sources/PropertyBased/Gen+Frequency.swift index edfbca9..710dbda 100644 --- a/Sources/PropertyBased/Gen+Frequency.swift +++ b/Sources/PropertyBased/Gen+Frequency.swift @@ -83,11 +83,11 @@ extension Gen { precondition(total > 0, "At least one generator with a weight above 0 must be specified.") return Generator( - run: { [total] rng in + run: { [total] rng, limit in let pick = FloatLiteralType.random(in: 0.. pick }! as Int - return (index: index, value: options[index].gen.runFull(&rng).input) + return try (index: index, value: options[index].gen.runFull(&rng, limit).input) }, shrink: { pair in let opt = options[pair.index] diff --git a/Sources/PropertyBased/Generator.swift b/Sources/PropertyBased/Generator.swift index c331b89..b93f3e8 100644 --- a/Sources/PropertyBased/Generator.swift +++ b/Sources/PropertyBased/Generator.swift @@ -23,7 +23,7 @@ public struct Generator: Send /// Generate a single result, before mapping or filtering. @usableFromInline - internal var _runIntermediate: @Sendable (inout any SeededRandomNumberGenerator) -> sending InputValue + internal var _runIntermediate: @Sendable (inout any SeededRandomNumberGenerator, Int) throws -> sending InputValue /// Map an intermediate result to its final value, or return `nil` if the value should be filtered. @usableFromInline @@ -33,8 +33,9 @@ public struct Generator: Send internal var _shrinker: @Sendable (InputValue) -> ShrinkSequence /// Run the generator until a single unfiltered value is found. - @inlinable - internal func runFull(_ rng: inout G) + @usableFromInline + internal func runFull(_ rng: inout G, _ limit: Int) + throws -> sending ( input: InputValue, result: ResultValue ) @@ -42,13 +43,17 @@ public struct Generator: Send var arng: any SeededRandomNumberGenerator = rng defer { rng = arng as! G } - while true { - let run = _runIntermediate(&arng) + var attempts = 0 + + while attempts <= limit { + let run = try _runIntermediate(&arng, limit) if let ret = _mapFilter(run) { return (run, ret) } + attempts += 1 } + throw GeneratorError.runLimitExceeded(limit) } } @@ -57,7 +62,22 @@ extension Generator { /// - Parameter rng: The random number generator to use. /// - Returns: A randomly generated value. public func run(using rng: inout G) -> sending ResultValue { - runFull(&rng).result + try! runFull(&rng, Int.max).result + } + + /// Generate a single value within a certain amount of attempts. + /// - Parameter rng: The random number generator to use. + /// - Parameter limit: The maximum amount of attempts before the generator stops. + /// - Returns: A randomly generated value. + /// - Throws: When the limit is reached. + public func run(using rng: inout G, limit: Int) + throws(GeneratorError) -> sending ResultValue + { + do { + return try runFull(&rng, limit).result + } catch { + throw error as! GeneratorError + } } /// Remove the shrinker for this generator. @@ -100,7 +120,7 @@ extension Generator where InputValue == ResultValue { run: @Sendable @escaping (inout any SeededRandomNumberGenerator) -> sending ResultValue, shrink: @Sendable @escaping (InputValue) -> sending ShrinkSequence, ) { - self._runIntermediate = run + _runIntermediate = { rng, _ in run(&rng) } self._shrinker = shrink self._mapFilter = { $0 } } @@ -109,7 +129,7 @@ extension Generator where InputValue == ResultValue { extension Generator { @inlinable internal init( - run: @Sendable @escaping (inout any SeededRandomNumberGenerator) -> sending InputValue, + run: @Sendable @escaping (inout any SeededRandomNumberGenerator, Int) throws -> sending InputValue, shrink: @Sendable @escaping (InputValue) -> ShrinkSequence, finalResult: @Sendable @escaping (InputValue) -> ResultValue? ) { @@ -128,7 +148,7 @@ extension Generator where ShrinkSequence == Shrink.None { public init( run: @Sendable @escaping (inout any SeededRandomNumberGenerator) -> sending ResultValue ) { - _runIntermediate = run + _runIntermediate = { rng, _ in run(&rng) } _shrinker = { _ in .init() } _mapFilter = { $0 } } @@ -220,11 +240,11 @@ extension Generator { /// - Returns: A generator of optional values. public func optional(valueRate: Float = 0.75) -> Generator> { return .init( - run: { rng in + run: { rng, limit in if Float.random(in: 0..<1, using: &rng) >= valueRate { return nil as InputValue? } - return self._runIntermediate(&rng) + return try self._runIntermediate(&rng, limit) }, shrink: { value in if let value { @@ -283,8 +303,8 @@ extension Generator { /// - Returns: A copy of this generator. @inlinable public func eraseToAny() -> Generator> { return .init( - run: { rng in - self._runIntermediate(&rng) as Any + run: { rng, limit in + try self._runIntermediate(&rng, limit) as Any }, shrink: { AnySequence(_shrinker($0 as! InputValue).lazy.map { $0 as Any }) diff --git a/Sources/PropertyBased/GeneratorError.swift b/Sources/PropertyBased/GeneratorError.swift new file mode 100644 index 0000000..fd1507b --- /dev/null +++ b/Sources/PropertyBased/GeneratorError.swift @@ -0,0 +1,29 @@ +// +// GeneratorError.swift +// PropertyBased +// +// Created by Lennard Sprong on 13/08/2026. +// + +#if canImport(Foundation) +import Foundation +#endif + +/// Errors that may be thrown by a generator. +public enum GeneratorError: Equatable, Error, CustomStringConvertible { + /// A generator failed to generate a valid value within the specified amount of attempts. + case runLimitExceeded(Int) + + public var description: String { + switch self { + case .runLimitExceeded(let count): + "Failed to generate a valid input after \(count) attempts. Check if the Generator is filtering too many values." + } + } +} + +#if canImport(Foundation) +extension GeneratorError: LocalizedError { + public var localizedDescription: String { description } +} +#endif diff --git a/Sources/PropertyBased/MaximumAttemptsTrait.swift b/Sources/PropertyBased/MaximumAttemptsTrait.swift new file mode 100644 index 0000000..82d7a59 --- /dev/null +++ b/Sources/PropertyBased/MaximumAttemptsTrait.swift @@ -0,0 +1,39 @@ +// +// MaximumAttemptsTrait.swift +// PropertyBased +// +// Created by Lennard Sprong on 10/08/2026. +// + +import Testing + +/// A trait that changes how often a generator can reject values before stopping. +/// +/// Use ``maximumAttempts(_:)`` to construct an instance of this trait. +public struct MaximumAttemptsTrait: TestTrait, SuiteTrait, TestScoping { + @_documentation(visibility: internal) + public var isRecursive: Bool { false } + + public func provideScope( + for test: Test, testCase: Test.Case?, performing function: @Sendable () async throws -> Void + ) async throws { + try await Self.$_maxAttempts.withValue(value) { + try await function() + } + } + + @TaskLocal static var _maxAttempts: Int? + + var value: Int +} + +extension Trait where Self == MaximumAttemptsTrait { + /// Change how often a generator can run before stopping. + /// - Parameter limit: The new limit. + /// - Returns: An instance of ``MaximumAttemptsTrait``. + /// - Precondition: Limit must have a positive value. + public static func maximumAttempts(_ limit: Int) -> Self { + precondition(limit >= 0, "Limit must have a positive value.") + return Self(value: limit) + } +} diff --git a/Sources/PropertyBased/PropertyCheck.swift b/Sources/PropertyBased/PropertyCheck.swift index fbea1b7..fded4e3 100644 --- a/Sources/PropertyBased/PropertyCheck.swift +++ b/Sources/PropertyBased/PropertyCheck.swift @@ -118,6 +118,7 @@ public func propertyCheck( var rngWithIssues: (rng: Xoshiro, value: InputValue, isError: Bool)? let actualCount = fixedRng != nil ? 1 : count + let runLimit = MaximumAttemptsTrait._maxAttempts for _ in 0..( var rng = fixedRng?.rng ?? Xoshiro() let rngCopy = rng - let (inputValue, resultValue) = input.runFull(&rng) + let inputValue: InputValue + let resultValue: ResultValue + do { + (inputValue, resultValue) = try input.runFull(&rng, runLimit ?? 10000) + } catch { + var failureMessage = String(describing: error) + if runLimit == nil, let genError = error as? GeneratorError, case .runLimitExceeded = genError { + failureMessage += "\n\nYou can add `.maximumAttempts()` to the Test or Suite to increase the limit." + } + + if fixedRng == nil { + let seed = rngCopy.traitHint + failureMessage += "\n\nAdd `.fixedSeed\(seed)` to the Test to reproduce this issue." + } + + Issue.record("\(failureMessage)", sourceLocation: sourceLocation) + return + } let foundIssues = await countIssues(isolation: isolation, suppress: EnableShrinkTrait.isEnabled) { try await body(resultValue) diff --git a/Sources/PropertyBased/Zip.swift b/Sources/PropertyBased/Zip.swift index cfd3df2..8d8772d 100644 --- a/Sources/PropertyBased/Zip.swift +++ b/Sources/PropertyBased/Zip.swift @@ -22,10 +22,10 @@ public func zip( Shrink.Tuple<(InA, InB)> > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -59,11 +59,11 @@ public func zip( Shrink.Tuple<(InA, InB, InC)> > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -100,12 +100,12 @@ public func zip( Shrink.Tuple<(InA, InB, InC, InD)> > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, - p3.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, + p3.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -145,13 +145,13 @@ public func zip( Shrink.Tuple<(InA, InB, InC, InD, InE)> > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, - p3.runFull(&rng).input, - p4.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, + p3.runFull(&rng, limit).input, + p4.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -194,14 +194,14 @@ public func zip > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, - p3.runFull(&rng).input, - p4.runFull(&rng).input, - p5.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, + p3.runFull(&rng, limit).input, + p4.runFull(&rng, limit).input, + p5.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -248,15 +248,15 @@ public func zip > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, - p3.runFull(&rng).input, - p4.runFull(&rng).input, - p5.runFull(&rng).input, - p6.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, + p3.runFull(&rng, limit).input, + p4.runFull(&rng, limit).input, + p5.runFull(&rng, limit).input, + p6.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -307,16 +307,16 @@ public func zip > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, - p3.runFull(&rng).input, - p4.runFull(&rng).input, - p5.runFull(&rng).input, - p6.runFull(&rng).input, - p7.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, + p3.runFull(&rng, limit).input, + p4.runFull(&rng, limit).input, + p5.runFull(&rng, limit).input, + p6.runFull(&rng, limit).input, + p7.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -370,17 +370,17 @@ public func zip > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, - p3.runFull(&rng).input, - p4.runFull(&rng).input, - p5.runFull(&rng).input, - p6.runFull(&rng).input, - p7.runFull(&rng).input, - p8.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, + p3.runFull(&rng, limit).input, + p4.runFull(&rng, limit).input, + p5.runFull(&rng, limit).input, + p6.runFull(&rng, limit).input, + p7.runFull(&rng, limit).input, + p8.runFull(&rng, limit).input, ) }, shrink: { tuple in @@ -439,18 +439,18 @@ public func zip< Shrink.Tuple<(InA, InB, InC, InD, InE, InF, InG, InH, InI, InJ)> > { return .init( - run: { rng in - ( - p0.runFull(&rng).input, - p1.runFull(&rng).input, - p2.runFull(&rng).input, - p3.runFull(&rng).input, - p4.runFull(&rng).input, - p5.runFull(&rng).input, - p6.runFull(&rng).input, - p7.runFull(&rng).input, - p8.runFull(&rng).input, - p9.runFull(&rng).input, + run: { rng, limit in + try ( + p0.runFull(&rng, limit).input, + p1.runFull(&rng, limit).input, + p2.runFull(&rng, limit).input, + p3.runFull(&rng, limit).input, + p4.runFull(&rng, limit).input, + p5.runFull(&rng, limit).input, + p6.runFull(&rng, limit).input, + p7.runFull(&rng, limit).input, + p8.runFull(&rng, limit).input, + p9.runFull(&rng, limit).input, ) }, shrink: { tuple in diff --git a/Sources/PropertyBased/Zip.swift.gyb b/Sources/PropertyBased/Zip.swift.gyb index fc47ef8..9896ee4 100644 --- a/Sources/PropertyBased/Zip.swift.gyb +++ b/Sources/PropertyBased/Zip.swift.gyb @@ -31,9 +31,9 @@ public func zip<${inTupleType}, ${outTupleType}>( Shrink.Tuple<(${inTupleType})> > { return .init( - run: { rng in ( + run: { rng, limit in try ( % for n in range(size): - p${n}.runFull(&rng).input, + p${n}.runFull(&rng, limit).input, % end )}, shrink: { tuple in diff --git a/Tests/PropertyBasedTests/MaximumAttemptsTraitTest.swift b/Tests/PropertyBasedTests/MaximumAttemptsTraitTest.swift new file mode 100644 index 0000000..5f3b27c --- /dev/null +++ b/Tests/PropertyBasedTests/MaximumAttemptsTraitTest.swift @@ -0,0 +1,56 @@ +// +// MaximumAttemptsTraitTest.swift +// PropertyBased +// +// Created by Lennard Sprong on 10/08/2026. +// + +import Testing + +@testable import PropertyBased + +@Suite struct MaximumAttemptsTraitTest { + @Test func testRunUsesLimit() { + let useless = Gen.always(false).filter { $0 } + + #expect(throws: GeneratorError.runLimitExceeded(25)) { + var rng = Xoshiro() + _ = try useless.run(using: &rng, limit: 25) + } + } + + @Test func testTraitCanModifyCount() async throws { + let useless = Gen.always(false).filter { $0 } + + let trait = MaximumAttemptsTrait.maximumAttempts(20) + let scope = try #require(trait.scopeProvider(for: Test.current!, testCase: Test.Case.current)) + + let issues = await gatherIssues { + try await scope.provideScope(for: Test.current!, testCase: Test.Case.current) { + await propertyCheck(input: useless) { _ in + try #require(Bool(false), "block must not be called") + } + } + } + #expect(issues.count == 1) + #expect( + issues.contains(where: { + $0.contains("20 attempts") && !$0.contains("maximumAttempts()") + })) + } + + @Test func testTraitSuggestion() async throws { + let useless = Gen.always(false).filter { $0 } + let issues = await gatherIssues { + await propertyCheck(input: useless) { _ in + try #require(Bool(false), "block must not be called") + } + } + + #expect(issues.count == 1) + #expect( + issues.contains(where: { + $0.contains("maximumAttempts()") + })) + } +} diff --git a/Tests/PropertyBasedTests/Utils.swift b/Tests/PropertyBasedTests/Utils.swift index adfd47f..77c1471 100644 --- a/Tests/PropertyBasedTests/Utils.swift +++ b/Tests/PropertyBasedTests/Utils.swift @@ -79,6 +79,6 @@ func testGen(_ gen: Generator) async { #expect(count > 50) var rng = Xoshiro() as any SeededRandomNumberGenerator - let value = gen._runIntermediate(&rng) + let value = try! gen._runIntermediate(&rng, Int.max) gen._shrinker(value).reduce(into: ()) { _, _ in } } diff --git a/Tests/PropertyBasedTests/ZipTests.swift b/Tests/PropertyBasedTests/ZipTests.swift index 6821833..369b0a8 100644 --- a/Tests/PropertyBasedTests/ZipTests.swift +++ b/Tests/PropertyBasedTests/ZipTests.swift @@ -67,56 +67,56 @@ import Testing repeat { let gen = zip(Gen.bool(1), Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip(Gen.bool(1), Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip(Gen.bool(1), Gen.bool, Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip(Gen.bool(1), Gen.bool, Gen.bool, Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip(Gen.bool(1), Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip(Gen.bool(1), Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip(Gen.bool(1), Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip(Gen.bool(1), Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false repeat { let gen = zip( Gen.bool(1), Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool, Gen.bool) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false diff --git a/Tests/PropertyBasedTests/ZipTests.swift.gyb b/Tests/PropertyBasedTests/ZipTests.swift.gyb index 2aa72c3..3e284d5 100644 --- a/Tests/PropertyBasedTests/ZipTests.swift.gyb +++ b/Tests/PropertyBasedTests/ZipTests.swift.gyb @@ -21,7 +21,7 @@ import Testing % for size in range(2, 11): repeat { let gen = zip(Gen.bool(1), ${', '.join([f"Gen.bool" for n in range(size-1)])}) - let result = gen._runIntermediate(&rng) + let result = try! gen._runIntermediate(&rng, Int.max) let shrunk = gen._shrinker(result).makeIterator().next() #expect(shrunk != nil) } while false