From 36ee3aa8676cd401eec97bd0ca8b2a7bc6612ab0 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 15 Jan 2025 13:08:28 -0500 Subject: [PATCH 1/3] [WIP] Add an experimental `.serialized(.globally)` trait. Known issue: suites don't report they've ended until after all tests have run. --- Sources/Testing/Running/Runner.Plan.swift | 84 ++++++++++++++++++- Sources/Testing/Running/Runner.swift | 47 +++++------ Sources/Testing/Testing.docc/Traits.md | 1 + Sources/Testing/Testing.docc/Traits/Trait.md | 1 + .../Testing/Traits/ParallelizationTrait.swift | 56 ++++++++++++- 5 files changed, 158 insertions(+), 31 deletions(-) diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index 26cb00d14..ec567c6db 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -73,6 +73,24 @@ extension Runner { } } + /// Stages of a test run. + /// + /// This enumeration conforms to `CaseIterable`, so callers can iterate over + /// all stages by looping over `Stage.allCases`. `Stage.allCases.first` and + /// `Stage.allCases.last` are respectively the first and last stages to run. + enum Stage: Sendable, CaseIterable { + /// Tests that might run in parallel (globally or locally) are being run. + case parallelizationAllowed + + /// Tests that are globally serialized are being run. + case globallySerialized + + /// The default stage to run tests in. + static var `default`: Self { + .parallelizationAllowed + } + } + /// A type describing a step in a runner plan. /// /// An instance of this type contains a test and the corresponding action an @@ -83,6 +101,49 @@ extension Runner { /// The action to perform with ``test``. public var action: Action + + /// The stage at which this step should be performed. + var stage: Stage = .default + + /// Whether or not this step may perform work over multiple stages of a + /// test run. + var isMultistaged: Bool { + test.isSuite + } + + /// Whether or not this step performs its first work in the given test + /// run stage. + /// + /// - Parameters: + /// - stage: The stage of interest. + /// + /// - Returns: Whether or not `stage` is the first stage in which this + /// step performs some work. + func starts(in stage: Stage) -> Bool { + let firstStage = if isMultistaged { + Stage.allCases.first + } else { + self.stage + } + return stage == firstStage + } + + /// Whether or not this step performs its final work in the given test + /// run stage. + /// + /// - Parameters: + /// - stage: The stage of interest. + /// + /// - Returns: Whether or not `stage` is the last stage in which this + /// step performs some work. + func ends(in stage: Stage) -> Bool { + let lastStage = if isMultistaged { + Stage.allCases.last + } else { + self.stage + } + return stage == lastStage + } } /// The graph of the steps in the runner plan. @@ -318,9 +379,26 @@ extension Runner.Plan { (action, recursivelyApply: action.isRecursive) } - // Zip the tests and actions together and return them. - return zip(testGraph, actionGraph).mapValues { _, pair in - pair.0.map { Step(test: $0, action: pair.1) } + // Figure out what stage each test should operate in. + let stageGraph: Graph = testGraph.mapValues { _, test in + switch test?.isGloballySerialized { + case nil: + .default + case .some(false): + .parallelizationAllowed + case .some(true): + .globallySerialized + } + } + + // Zip the tests, actions, and stages together and return them. + return zip(zip(testGraph, actionGraph), stageGraph).mapValues { _, tuple in + let test = tuple.0.0 + let action = tuple.0.1 + let stage = tuple.1 + return test.map { test in + Step(test: test, action: action, stage: stage) + } } } diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 16eff103f..c6a9070d7 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -113,11 +113,13 @@ extension Runner { /// /// - Parameters: /// - sequence: The sequence to enumerate. + /// - stage: The stage at which the runner is operating. /// - body: The function to invoke. /// /// - Throws: Whatever is thrown by `body`. private static func _forEach( in sequence: some Sequence, + stage: Plan.Stage, _ body: @Sendable @escaping (E) async throws -> Void ) async throws where E: Sendable { try await withThrowingTaskGroup(of: Void.self) { taskGroup in @@ -128,7 +130,7 @@ extension Runner { } // If not parallelizing, wait after each task. - if !_configuration.isParallelizationEnabled { + if stage == .globallySerialized || !_configuration.isParallelizationEnabled { try await taskGroup.waitForAll() } } @@ -139,6 +141,7 @@ extension Runner { /// /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, is to be run. + /// - stage: The stage at which the runner is operating. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. @@ -153,63 +156,53 @@ extension Runner { /// ## See Also /// /// - ``Runner/run()`` - private static func _runStep(atRootOf stepGraph: Graph) async throws { + private static func _runStep(atRootOf stepGraph: Graph, stage: Plan.Stage) async throws { // Exit early if the task has already been cancelled. try Task.checkCancellation() - // Whether to send a `.testEnded` event at the end of running this step. - // Some steps' actions may not require a final event to be sent — for - // example, a skip event only sends `.testSkipped`. - let shouldSendTestEnded: Bool - let configuration = _configuration // Determine what action to take for this step. - if let step = stepGraph.value { + if let step = stepGraph.value, step.starts(in: stage) { Event.post(.planStepStarted(step), for: (step.test, nil), configuration: configuration) // Determine what kind of event to send for this step based on its action. switch step.action { case .run: Event.post(.testStarted, for: (step.test, nil), configuration: configuration) - shouldSendTestEnded = true case let .skip(skipInfo): Event.post(.testSkipped(skipInfo), for: (step.test, nil), configuration: configuration) - shouldSendTestEnded = false case let .recordIssue(issue): Event.post(.issueRecorded(issue), for: (step.test, nil), configuration: configuration) - shouldSendTestEnded = false } - } else { - shouldSendTestEnded = false } defer { - if let step = stepGraph.value { - if shouldSendTestEnded { + if let step = stepGraph.value, step.ends(in: stage) { + if case .run = step.action { Event.post(.testEnded, for: (step.test, nil), configuration: configuration) } Event.post(.planStepEnded(step), for: (step.test, nil), configuration: configuration) } } - if let step = stepGraph.value, case .run = step.action { + if let step = stepGraph.value, case .run = step.action, step.stage == stage { await Test.withCurrent(step.test) { _ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) { try await _applyScopingTraits(for: step.test, testCase: nil) { // Run the test function at this step (if one is present.) if let testCases = step.test.testCases { - try await _runTestCases(testCases, within: step) + try await _runTestCases(testCases, within: step, stage: stage) } // Run the children of this test (i.e. the tests in this suite.) - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, stage: stage) } } } } else { // There is no test at this node in the graph, so just skip down to the // child nodes. - try await _runChildren(of: stepGraph) + try await _runChildren(of: stepGraph, stage: stage) } } @@ -234,10 +227,11 @@ extension Runner { /// - Parameters: /// - stepGraph: The subgraph whose root value, a step, will be used to /// find children to run. + /// - stage: The stage at which the runner is operating. /// /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. - private static func _runChildren(of stepGraph: Graph) async throws { + private static func _runChildren(of stepGraph: Graph, stage: Plan.Stage) async throws { let childGraphs = if _configuration.isParallelizationEnabled { // Explicitly shuffle the steps to help detect accidental dependencies // between tests due to their ordering. @@ -267,8 +261,8 @@ extension Runner { } // Run the child nodes. - try await _forEach(in: childGraphs) { _, childGraph in - try await _runStep(atRootOf: childGraph) + try await _forEach(in: childGraphs, stage: stage) { _, childGraph in + try await _runStep(atRootOf: childGraph, stage: stage) } } @@ -277,20 +271,21 @@ extension Runner { /// - Parameters: /// - testCases: The test cases to be run. /// - step: The runner plan step associated with this test case. + /// - stage: The stage at which the runner is operating. /// /// - Throws: Whatever is thrown from a test case's body. Thrown errors are /// normally reported as test failures. /// /// If parallelization is supported and enabled, the generated test cases will /// be run in parallel using a task group. - private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step) async throws { + private static func _runTestCases(_ testCases: some Sequence, within step: Plan.Step, stage: Plan.Stage) async throws { // Apply the configuration's test case filter. let testCaseFilter = _configuration.testCaseFilter let testCases = testCases.lazy.filter { testCase in testCaseFilter(testCase, step.test) } - try await _forEach(in: testCases) { testCase in + try await _forEach(in: testCases, stage: stage) { testCase in try await _runTestCase(testCase, within: step) } } @@ -384,7 +379,9 @@ extension Runner { await withTaskGroup(of: Void.self) { [runner] taskGroup in _ = taskGroup.addTaskUnlessCancelled { - try? await _runStep(atRootOf: runner.plan.stepGraph) + for stage in Plan.Stage.allCases { + try? await _runStep(atRootOf: runner.plan.stepGraph, stage: stage) + } } await taskGroup.waitForAll() } diff --git a/Sources/Testing/Testing.docc/Traits.md b/Sources/Testing/Testing.docc/Traits.md index 5fadb2bdc..af40b60ab 100644 --- a/Sources/Testing/Testing.docc/Traits.md +++ b/Sources/Testing/Testing.docc/Traits.md @@ -36,6 +36,7 @@ behavior of test functions. - - ``Trait/serialized`` + ### Annotating tests diff --git a/Sources/Testing/Testing.docc/Traits/Trait.md b/Sources/Testing/Testing.docc/Traits/Trait.md index f0e84aaeb..7aba5b391 100644 --- a/Sources/Testing/Testing.docc/Traits/Trait.md +++ b/Sources/Testing/Testing.docc/Traits/Trait.md @@ -27,6 +27,7 @@ See https://swift.org/CONTRIBUTORS.txt for Swift project authors ### Running tests serially or in parallel - ``Trait/serialized`` + ### Categorizing tests diff --git a/Sources/Testing/Traits/ParallelizationTrait.swift b/Sources/Testing/Traits/ParallelizationTrait.swift index df34f4d63..a0bb5e272 100644 --- a/Sources/Testing/Traits/ParallelizationTrait.swift +++ b/Sources/Testing/Traits/ParallelizationTrait.swift @@ -24,8 +24,30 @@ /// globally disabled (by, for example, passing `--no-parallel` to the /// `swift test` command.) /// -/// To add this trait to a test, use ``Trait/serialized``. -public struct ParallelizationTrait: TestTrait, SuiteTrait {} +/// To add this trait to a test, use ``Trait/serialized`` or +/// ``Trait/serialized(_:)``. +public struct ParallelizationTrait: TestTrait, SuiteTrait { + /// Scopes in which suites and test functions can be serialized using the + /// ``serialized(_:)`` trait. + @_spi(Experimental) + public enum Scope: Sendable { + /// Parallelization is applied locally. + /// + /// TODO: More blurb. + case locally + + /// Parallelization is applied globally. + /// + /// TODO: More blurb. + case globally + } + + var scope: Scope + + public var isRecursive: Bool { + scope == .globally + } +} // MARK: - TestScoping @@ -45,10 +67,38 @@ extension ParallelizationTrait: TestScoping { extension Trait where Self == ParallelizationTrait { /// A trait that serializes the test to which it is applied. /// + /// This value is equivalent to ``serialized(_:)`` with the argument + /// ``ParallelizationTrait/Scope/locally``. + /// /// ## See Also /// /// - ``ParallelizationTrait`` public static var serialized: Self { - Self() + Self(scope: .locally) + } + + /// A trait that serializes the test to which it is applied. + /// + /// - Parameters: + /// - scope: The scope in which parallelization is enforced. + /// + /// ## See Also + /// + /// - ``ParallelizationTrait`` + @_spi(Experimental) + public static func serialized(_ scope: ParallelizationTrait.Scope) -> Self { + Self(scope: scope) + } +} + +// MARK: - + +extension Test { + /// Whether or not this test has been globally serialized. + var isGloballySerialized: Bool { + traits.lazy + .compactMap { $0 as? ParallelizationTrait } + .map(\.scope) + .contains(.globally) } } From bd04d59b2a417e6ac2276bf3a8429cc028156389 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Wed, 15 Jan 2025 14:50:52 -0500 Subject: [PATCH 2/3] Add test, ensure suites start/end appropriately --- Sources/Testing/Running/Runner.Plan.swift | 80 ++++++++----------- Sources/Testing/Running/Runner.swift | 12 +-- .../Traits/ParallelizationTraitTests.swift | 40 ++++++++++ 3 files changed, 78 insertions(+), 54 deletions(-) diff --git a/Sources/Testing/Running/Runner.Plan.swift b/Sources/Testing/Running/Runner.Plan.swift index ec567c6db..e040e7043 100644 --- a/Sources/Testing/Running/Runner.Plan.swift +++ b/Sources/Testing/Running/Runner.Plan.swift @@ -78,7 +78,12 @@ extension Runner { /// This enumeration conforms to `CaseIterable`, so callers can iterate over /// all stages by looping over `Stage.allCases`. `Stage.allCases.first` and /// `Stage.allCases.last` are respectively the first and last stages to run. - enum Stage: Sendable, CaseIterable { + /// + /// The names of cases are meant to describe what happens during them (so as + /// to aid debugging.) Most code that uses test run stages doesn't need to + /// care about them in isolation, but rather looks at `Stage.allCases`, + /// ranges of stages, etc. + enum Stage: Sendable, Comparable, CaseIterable { /// Tests that might run in parallel (globally or locally) are being run. case parallelizationAllowed @@ -102,48 +107,8 @@ extension Runner { /// The action to perform with ``test``. public var action: Action - /// The stage at which this step should be performed. - var stage: Stage = .default - - /// Whether or not this step may perform work over multiple stages of a - /// test run. - var isMultistaged: Bool { - test.isSuite - } - - /// Whether or not this step performs its first work in the given test - /// run stage. - /// - /// - Parameters: - /// - stage: The stage of interest. - /// - /// - Returns: Whether or not `stage` is the first stage in which this - /// step performs some work. - func starts(in stage: Stage) -> Bool { - let firstStage = if isMultistaged { - Stage.allCases.first - } else { - self.stage - } - return stage == firstStage - } - - /// Whether or not this step performs its final work in the given test - /// run stage. - /// - /// - Parameters: - /// - stage: The stage of interest. - /// - /// - Returns: Whether or not `stage` is the last stage in which this - /// step performs some work. - func ends(in stage: Stage) -> Bool { - let lastStage = if isMultistaged { - Stage.allCases.last - } else { - self.stage - } - return stage == lastStage - } + /// The stages at which this step operates. + var stages: ClosedRange = .default ... .default } /// The graph of the steps in the runner plan. @@ -254,6 +219,23 @@ extension Runner.Plan { synthesizeSuites(in: &graph, sourceLocation: &sourceLocation) } + /// Recursively widen the range of test run stages each (yet-to-be-created) + /// step in the specified graph will operate in. + /// + /// - Parameters: + /// - graph: The graph in which test run stage ranges should be computed. + private static func _recursivelyComputeStageRanges(in graph: inout Graph>) { + var minStage = graph.value.lowerBound + var maxStage = graph.value.upperBound + for (key, var childGraph) in graph.children { + _recursivelyComputeStageRanges(in: &childGraph) + graph.children[key] = childGraph + minStage = min(minStage, childGraph.value.lowerBound) + maxStage = max(maxStage, childGraph.value.upperBound) + } + graph.value = minStage ... maxStage + } + /// Construct a graph of runner plan steps for the specified tests. /// /// - Parameters: @@ -379,9 +361,9 @@ extension Runner.Plan { (action, recursivelyApply: action.isRecursive) } - // Figure out what stage each test should operate in. - let stageGraph: Graph = testGraph.mapValues { _, test in - switch test?.isGloballySerialized { + // Figure out what stages each step should operate in. + var stageGraph: Graph> = testGraph.mapValues { _, test in + let bound: Stage = switch test?.isGloballySerialized { case nil: .default case .some(false): @@ -389,15 +371,17 @@ extension Runner.Plan { case .some(true): .globallySerialized } + return bound ... bound } + _recursivelyComputeStageRanges(in: &stageGraph) // Zip the tests, actions, and stages together and return them. return zip(zip(testGraph, actionGraph), stageGraph).mapValues { _, tuple in let test = tuple.0.0 let action = tuple.0.1 - let stage = tuple.1 + let stages = tuple.1 return test.map { test in - Step(test: test, action: action, stage: stage) + Step(test: test, action: action, stages: stages) } } } diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index c6a9070d7..d0d70767c 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -163,7 +163,7 @@ extension Runner { let configuration = _configuration // Determine what action to take for this step. - if let step = stepGraph.value, step.starts(in: stage) { + if let step = stepGraph.value, step.stages.lowerBound == stage { Event.post(.planStepStarted(step), for: (step.test, nil), configuration: configuration) // Determine what kind of event to send for this step based on its action. @@ -177,7 +177,7 @@ extension Runner { } } defer { - if let step = stepGraph.value, step.ends(in: stage) { + if let step = stepGraph.value, step.stages.upperBound == stage { if case .run = step.action { Event.post(.testEnded, for: (step.test, nil), configuration: configuration) } @@ -185,7 +185,7 @@ extension Runner { } } - if let step = stepGraph.value, case .run = step.action, step.stage == stage { + if let step = stepGraph.value, case .run = step.action, step.stages.lowerBound == stage { await Test.withCurrent(step.test) { _ = await Issue.withErrorRecording(at: step.test.sourceLocation, configuration: configuration) { try await _applyScopingTraits(for: step.test, testCase: nil) { @@ -200,8 +200,8 @@ extension Runner { } } } else { - // There is no test at this node in the graph, so just skip down to the - // child nodes. + // There is no test at this node in the graph, or the test at this node + // runs in another stage, so just skip down to the child nodes. try await _runChildren(of: stepGraph, stage: stage) } } @@ -232,7 +232,7 @@ extension Runner { /// - Throws: Whatever is thrown from the test body. Thrown errors are /// normally reported as test failures. private static func _runChildren(of stepGraph: Graph, stage: Plan.Stage) async throws { - let childGraphs = if _configuration.isParallelizationEnabled { + let childGraphs = if stage != .globallySerialized && _configuration.isParallelizationEnabled { // Explicitly shuffle the steps to help detect accidental dependencies // between tests due to their ordering. Array(stepGraph.children) diff --git a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift index e43ca50b7..44eeddb4d 100644 --- a/Tests/TestingTests/Traits/ParallelizationTraitTests.swift +++ b/Tests/TestingTests/Traits/ParallelizationTraitTests.swift @@ -9,6 +9,9 @@ // @testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +#if canImport(XCTest) +import XCTest +#endif @Suite("Parallelization Trait Tests", .tags(.traitRelated)) struct ParallelizationTraitTests { @@ -44,6 +47,43 @@ struct ParallelizationTraitTests { } } +#if canImport(XCTest) +final class ParallelizationTraitXCTests: XCTestCase { + // Implemented in XCTest so we can use enforceOrder (see #297) + func testSerializedGlobally() async { + var expectations = [XCTestExpectation]() + + var sourceLocation = #_sourceLocation + let parallelizedRanFirst = expectation(description: "Parallelized tests ran first") + var tests: [Test] = [ + Test(sourceLocation: sourceLocation) { + if #available(_clockAPI, *) { + try await Test.Clock.sleep(for: .nanoseconds(50_000_000)) + } + parallelizedRanFirst.fulfill() + }, + ] + expectations.append(parallelizedRanFirst) + + for i in 0 ..< 100 { + sourceLocation.line += 1 + let serializedTestRan = expectation(description: "Globally serialized test #\(i) ran") + tests.append( + Test(.serialized(.globally), sourceLocation: sourceLocation) { + serializedTestRan.fulfill() + } + ) + expectations.append(serializedTestRan) + } + let plan = await Runner.Plan(tests: tests, configuration: .init()) + let runner = Runner(plan: plan, configuration: .init()) + await runner.run() + await fulfillment(of: expectations, enforceOrder: true) + } + +} +#endif + // MARK: - Fixtures @Suite(.hidden, .serialized) From d5b624b8b5b4604bbb80c05747676be893b5aa47 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Fri, 17 Jan 2025 12:33:24 -0500 Subject: [PATCH 3/3] Placeholder for 'parallelization groups' case --- Sources/Testing/Traits/ParallelizationTrait.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Sources/Testing/Traits/ParallelizationTrait.swift b/Sources/Testing/Traits/ParallelizationTrait.swift index a0bb5e272..7f451275a 100644 --- a/Sources/Testing/Traits/ParallelizationTrait.swift +++ b/Sources/Testing/Traits/ParallelizationTrait.swift @@ -30,12 +30,19 @@ public struct ParallelizationTrait: TestTrait, SuiteTrait { /// Scopes in which suites and test functions can be serialized using the /// ``serialized(_:)`` trait. @_spi(Experimental) - public enum Scope: Sendable { + public enum Scope: Sendable, Equatable { /// Parallelization is applied locally. /// /// TODO: More blurb. case locally + /// Parallelization is applied across all suites and test functions in the + /// given group. + /// + /// TODO: More blurb. + @available(*, unavailable, message: "Unimplemented") + case withinGroup(_ groupName: String) + /// Parallelization is applied globally. /// /// TODO: More blurb.