diff --git a/Fixtures/Miscellaneous/TestDebugging/Package.swift b/Fixtures/Miscellaneous/TestDebugging/Package.swift new file mode 100644 index 00000000000..96687b0b207 --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebugging/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "TestDebugging", + targets: [ + .target(name: "TestDebugging"), + .testTarget(name: "TestDebuggingTests", dependencies: ["TestDebugging"]), + ] +) \ No newline at end of file diff --git a/Fixtures/Miscellaneous/TestDebugging/Sources/TestDebugging/TestDebugging.swift b/Fixtures/Miscellaneous/TestDebugging/Sources/TestDebugging/TestDebugging.swift new file mode 100644 index 00000000000..8a3b22fdebd --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebugging/Sources/TestDebugging/TestDebugging.swift @@ -0,0 +1,23 @@ +public struct Calculator { + public init() {} + + public func add(_ a: Int, _ b: Int) -> Int { + return a + b + } + + public func subtract(_ a: Int, _ b: Int) -> Int { + return a - b + } + + public func multiply(_ a: Int, _ b: Int) -> Int { + return a * b + } + + public func divide(_ a: Int, _ b: Int) -> Int { + return a / b + } + + public func purposelyFail() -> Bool { + return false + } +} \ No newline at end of file diff --git a/Fixtures/Miscellaneous/TestDebugging/Tests/TestDebuggingTests/TestDebuggingTests.swift b/Fixtures/Miscellaneous/TestDebugging/Tests/TestDebuggingTests/TestDebuggingTests.swift new file mode 100644 index 00000000000..c5b3aca2fd3 --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebugging/Tests/TestDebuggingTests/TestDebuggingTests.swift @@ -0,0 +1,34 @@ +import XCTest +import Testing +@testable import TestDebugging + +// MARK: - XCTest Suite +final class XCTestCalculatorTests: XCTestCase { + + func testAdditionPasses() { + let calculator = Calculator() + let result = calculator.add(2, 3) + XCTAssertEqual(result, 5, "Addition should return 5 for 2 + 3") + } + + func testSubtractionFails() { + let calculator = Calculator() + let result = calculator.subtract(5, 3) + XCTAssertEqual(result, 3, "This test is designed to fail - subtraction 5 - 3 should equal 2, not 3") + } +} + +// MARK: - Swift Testing Suite +@Test("Calculator Addition Works Correctly") +func calculatorAdditionPasses() { + let calculator = Calculator() + let result = calculator.add(4, 6) + #expect(result == 10, "Addition should return 10 for 4 + 6") +} + +@Test("Calculator Boolean Check Fails") +func calculatorBooleanFails() { + let calculator = Calculator() + let result = calculator.purposelyFail() + #expect(result == true, "This test is designed to fail - purposelyFail() should return false, not true") +} \ No newline at end of file diff --git a/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Package.swift b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Package.swift new file mode 100644 index 00000000000..b7de5cb1fcf --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Package.swift @@ -0,0 +1,12 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "TestDebuggingMultiProduct", + targets: [ + .target(name: "LibA"), + .target(name: "LibB"), + .testTarget(name: "LibATests", dependencies: ["LibA"]), + .testTarget(name: "LibBTests", dependencies: ["LibB"]), + ] +) diff --git a/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Sources/LibA/LibA.swift b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Sources/LibA/LibA.swift new file mode 100644 index 00000000000..9c5d598df37 --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Sources/LibA/LibA.swift @@ -0,0 +1,7 @@ +public struct LibA { + public init() {} + + public func greet() -> String { + return "Hello from LibA" + } +} diff --git a/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Sources/LibB/LibB.swift b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Sources/LibB/LibB.swift new file mode 100644 index 00000000000..2073dc27549 --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Sources/LibB/LibB.swift @@ -0,0 +1,7 @@ +public struct LibB { + public init() {} + + public func greet() -> String { + return "Hello from LibB" + } +} diff --git a/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Tests/LibATests/LibATests.swift b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Tests/LibATests/LibATests.swift new file mode 100644 index 00000000000..1b411d17861 --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Tests/LibATests/LibATests.swift @@ -0,0 +1,16 @@ +import XCTest +import Testing +@testable import LibA + +final class LibAXCTests: XCTestCase { + func testGreet() { + let lib = LibA() + XCTAssertEqual(lib.greet(), "Hello from LibA") + } +} + +@Test("LibA greeting works") +func libAGreeting() { + let lib = LibA() + #expect(lib.greet() == "Hello from LibA") +} diff --git a/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Tests/LibBTests/LibBTests.swift b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Tests/LibBTests/LibBTests.swift new file mode 100644 index 00000000000..3dc72a6edda --- /dev/null +++ b/Fixtures/Miscellaneous/TestDebuggingMultiProduct/Tests/LibBTests/LibBTests.swift @@ -0,0 +1,16 @@ +import XCTest +import Testing +@testable import LibB + +final class LibBXCTests: XCTestCase { + func testGreet() { + let lib = LibB() + XCTAssertEqual(lib.greet(), "Hello from LibB") + } +} + +@Test("LibB greeting works") +func libBGreeting() { + let lib = LibB() + #expect(lib.greet() == "Hello from LibB") +} diff --git a/Sources/Commands/CMakeLists.txt b/Sources/Commands/CMakeLists.txt index 2d73346a0b2..cc083ba274e 100644 --- a/Sources/Commands/CMakeLists.txt +++ b/Sources/Commands/CMakeLists.txt @@ -54,6 +54,7 @@ add_library(Commands Utilities/DOTManifestSerializer.swift Utilities/MermaidPackageSerializer.swift Utilities/MultiRootSupport.swift + Utilities/NonEmpty.swift Utilities/PlainTextEncoder.swift Utilities/PluginDelegate.swift Utilities/RefactoringSupport.swift diff --git a/Sources/Commands/SwiftRunCommand.swift b/Sources/Commands/SwiftRunCommand.swift index 1b6d7448214..cf9e8063267 100644 --- a/Sources/Commands/SwiftRunCommand.swift +++ b/Sources/Commands/SwiftRunCommand.swift @@ -19,7 +19,6 @@ import PackageModel import SPMBuildCore import enum TSCBasic.ProcessEnv -import func TSCBasic.exec import enum TSCUtility.Diagnostics @@ -153,7 +152,8 @@ public struct SwiftRunCommand: AsyncSwiftCommand { fileSystem: swiftCommandState.fileSystem, executablePath: interpreterPath, originalWorkingDirectory: swiftCommandState.originalWorkingDirectory, - arguments: arguments + arguments: arguments, + observabilityScope: swiftCommandState.observabilityScope ) case .debugger: @@ -183,12 +183,17 @@ public struct SwiftRunCommand: AsyncSwiftCommand { fileSystem: swiftCommandState.fileSystem, executablePath: debuggerPath, originalWorkingDirectory: swiftCommandState.originalWorkingDirectory, - arguments: debugger.extraCLIOptions + [productAbsolutePath.pathString] + options.arguments + arguments: debugger.extraCLIOptions + [productAbsolutePath.pathString] + options.arguments, + observabilityScope: swiftCommandState.observabilityScope ) } else { let pathRelativeToWorkingDirectory = productAbsolutePath.relative(to: swiftCommandState.originalWorkingDirectory) let lldbPath = try swiftCommandState.getTargetToolchain().getLLDB() - try exec(path: lldbPath.pathString, args: ["--", pathRelativeToWorkingDirectory.pathString] + options.arguments) + try safeExec( + path: lldbPath.pathString, + args: ["--", pathRelativeToWorkingDirectory.pathString] + options.arguments, + observabilityScope: swiftCommandState.observabilityScope + ) } } catch let error as RunError { swiftCommandState.observabilityScope.emit(error) @@ -207,7 +212,8 @@ public struct SwiftRunCommand: AsyncSwiftCommand { fileSystem: swiftCommandState.fileSystem, executablePath: swiftInterpreterPath, originalWorkingDirectory: swiftCommandState.originalWorkingDirectory, - arguments: arguments + arguments: arguments, + observabilityScope: swiftCommandState.observabilityScope ) return } @@ -245,7 +251,8 @@ public struct SwiftRunCommand: AsyncSwiftCommand { fileSystem: swiftCommandState.fileSystem, executablePath: runnerPath, originalWorkingDirectory: swiftCommandState.originalWorkingDirectory, - arguments: arguments + arguments: arguments, + observabilityScope: swiftCommandState.observabilityScope ) } catch Diagnostics.fatalError { throw ExitCode.failure @@ -294,7 +301,8 @@ public struct SwiftRunCommand: AsyncSwiftCommand { fileSystem: FileSystem, executablePath: AbsolutePath, originalWorkingDirectory: AbsolutePath, - arguments: [String] + arguments: [String], + observabilityScope: ObservabilityScope ) throws { // Make sure we are running from the original working directory. let cwd: AbsolutePath? = fileSystem.currentWorkingDirectory @@ -303,7 +311,7 @@ public struct SwiftRunCommand: AsyncSwiftCommand { } let pathRelativeToWorkingDirectory = executablePath.relative(to: originalWorkingDirectory) - try execute(path: executablePath.pathString, args: [pathRelativeToWorkingDirectory.pathString] + arguments) + try safeExec(path: executablePath.pathString, args: [pathRelativeToWorkingDirectory.pathString] + arguments, observabilityScope: observabilityScope) } /// Determines if a path points to a valid swift file. @@ -326,41 +334,6 @@ public struct SwiftRunCommand: AsyncSwiftCommand { return fileSystem.isFile(absolutePath) } - /// A safe wrapper of TSCBasic.exec. - private func execute(path: String, args: [String]) throws -> Never { - #if !os(Windows) - // Dispatch will disable almost all asynchronous signals on its worker threads, and this is called from `async` - // context. To correctly `exec` a freshly built binary, we will need to: - // 1. reset the signal masks - for i in 1..() + if xctestEnabled { + let xctestSuites = try TestingSupport.getTestSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + enableCodeCoverage: options.enableCodeCoverage, + shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, + experimentalTestOutput: options.enableExperimentalTestOutput, + sanitizers: globalOptions.build.sanitizers + ) + for (product, suites) in xctestSuites where !suites.isEmpty { + productsWithXCTests.insert(product.bundlePath) + } + } + + var productsWithSwiftTests = Set() + if swiftTestingEnabled { + let swiftTestingSuites = try TestingSupport.getSwiftTestingSuites( + in: testProducts, + swiftCommandState: swiftCommandState, + shouldSkipBuilding: options.sharedOptions.shouldSkipBuilding, + sanitizers: globalOptions.build.sanitizers + ) + for (binaryPath, tests) in swiftTestingSuites where !tests.isEmpty { + productsWithSwiftTests.insert(binaryPath) + } + } + + var targets = [DebuggableTestSession.Target]() + for testProduct in testProducts { + if productsWithXCTests.contains(testProduct.bundlePath) { + targets.append(DebuggableTestSession.Target( + productName: testProduct.productName, + kind: .xctest(bundlePath: testProduct.bundlePath), + additionalArgs: try additionalLLDBArguments( + for: .xctest, + testProduct: testProduct, + swiftCommandState: swiftCommandState + ) + )) + } + if productsWithSwiftTests.contains(testProduct.binaryPath) { + targets.append(DebuggableTestSession.Target( + productName: testProduct.productName, + kind: .swiftTesting(binaryPath: testProduct.binaryPath), + additionalArgs: try additionalLLDBArguments( + for: .swiftTesting, + testProduct: testProduct, + swiftCommandState: swiftCommandState + ) + )) + } + } + + guard let sessionTargets = NonEmpty(targets) else { + throw DebuggerError.noEnabledTestingLibraries + } + + try await runTestLibrariesWithLLDB( + target: DebuggableTestSession(targets: sessionTargets), + testProducts: testProducts, + productsBuildParameters: productsBuildParameters, + swiftCommandState: swiftCommandState, + toolchain: toolchain + ) + } + + private func additionalLLDBArguments(for library: TestingLibrary, testProduct: BuiltTestProduct, swiftCommandState: SwiftCommandState) throws -> [String] { + switch library { + case .xctest: + let (xctestArgs, _, _) = try xctestArgs(for: [testProduct], swiftCommandState: swiftCommandState) + return xctestArgs + + case .swiftTesting: + let commandLineArguments = CommandLine.arguments.dropFirst() + var swiftTestingArgs = ["--testing-library", "swift-testing", "--enable-swift-testing"] + + if let separatorIndex = commandLineArguments.firstIndex(of: "--") { + let offset = commandLineArguments.distance(from: commandLineArguments.startIndex, to: separatorIndex) + swiftTestingArgs += Array(commandLineArguments.dropFirst(offset + 1)) + } + return swiftTestingArgs + } + } + + private func runTestLibrariesWithLLDB( + target: DebuggableTestSession, + testProducts: [BuiltTestProduct], + productsBuildParameters: BuildParameters, + swiftCommandState: SwiftCommandState, + toolchain: UserToolchain + ) async throws { + let debugRunner = DebugTestRunner( + target: target, + buildParameters: productsBuildParameters, + toolchain: toolchain, + testEnv: try TestingSupport.constructTestEnvironment( + toolchain: toolchain, + destinationBuildParameters: productsBuildParameters, + sanitizers: globalOptions.build.sanitizers, + library: .swiftTesting, // This is ignored by the DebugTestRunner, so we just hardcode it + testProductPaths: Array(Set(testProducts.flatMap { [$0.bundlePath, $0.binaryPath] })), + interopMode: nil + ), + fileSystem: swiftCommandState.fileSystem, + observabilityScope: swiftCommandState.observabilityScope, + verbose: globalOptions.logging.verbose + ) + + try debugRunner.run() + } + private func runTestProducts( _ testProducts: [BuiltTestProduct], additionalArguments: [String], @@ -757,6 +904,17 @@ public struct SwiftTestCommand: AsyncSwiftCommand { /// /// - Throws: if a command argument is invalid private func validateArguments(swiftCommandState: SwiftCommandState) throws { + // Validation for --debugger first, since it affects other validations. + if options.shouldLaunchInLLDB { + try Self.validateLLDBCompatibility( + configuration: options.globalOptions.build.configuration ?? swiftCommandState.preferredBuildConfiguration, + shouldRunInParallel: options.shouldRunInParallel, + numberOfWorkers: options.numberOfWorkers, + shouldListTests: options._deprecated_shouldListTests, + shouldPrintCodeCovPath: options.shouldPrintCodeCovPath + ) + } + // Validation for --num-workers. if let workers = options.numberOfWorkers { // The --num-worker option should be called with --parallel. Since @@ -780,6 +938,40 @@ public struct SwiftTestCommand: AsyncSwiftCommand { } } + /// Validates that --debugger is compatible with other provided arguments. + /// + /// Extracted as a static function so the validation logic can be tested + /// directly without invoking the full command pipeline. + /// + /// - Throws: if --debugger is used with incompatible flags. + static func validateLLDBCompatibility( + configuration: BuildConfiguration, + shouldRunInParallel: Bool, + numberOfWorkers: Int?, + shouldListTests: Bool, + shouldPrintCodeCovPath: Bool + ) throws { + if configuration == .release { + throw StringError("--debugger cannot be used with release configuration (debugging requires debug symbols)") + } + + if shouldRunInParallel { + throw StringError("--debugger cannot be used with --parallel (debugging requires sequential execution)") + } + + if numberOfWorkers != nil { + throw StringError("--debugger cannot be used with --num-workers (debugging requires sequential execution)") + } + + if shouldListTests { + throw StringError("--debugger cannot be used with --list-tests (use 'swift test list' for listing tests)") + } + + if shouldPrintCodeCovPath { + throw StringError("--debugger cannot be used with --show-codecov-path (debugging session cannot show paths)") + } + } + public init() {} } diff --git a/Sources/Commands/Utilities/NonEmpty.swift b/Sources/Commands/Utilities/NonEmpty.swift new file mode 100644 index 00000000000..a6a31464b8a --- /dev/null +++ b/Sources/Commands/Utilities/NonEmpty.swift @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// A collection guaranteed to contain at least one element. +struct NonEmpty: Sequence { + let first: Element + let rest: [Element] + + init(_ first: Element, _ rest: [Element] = []) { + self.first = first + self.rest = rest + } + + init?(_ elements: [Element]) { + guard let first = elements.first else { return nil } + self.first = first + self.rest = Array(elements.dropFirst()) + } + + func makeIterator() -> IndexingIterator<[Element]> { + ([first] + rest).makeIterator() + } +} diff --git a/Sources/Commands/Utilities/TestingSupport.swift b/Sources/Commands/Utilities/TestingSupport.swift index b173ca63825..d354c143427 100644 --- a/Sources/Commands/Utilities/TestingSupport.swift +++ b/Sources/Commands/Utilities/TestingSupport.swift @@ -12,11 +12,20 @@ import Basics import CoreCommands +import Foundation import PackageModel import SPMBuildCore import TSCUtility import Workspace +#if canImport(WinSDK) +import WinSDK +#elseif canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + import struct TSCBasic.FileSystemError import class Basics.AsyncProcess import struct Basics.AsyncProcessResult @@ -24,7 +33,54 @@ import TSCLibc import var TSCBasic.stderrStream import var TSCBasic.stdoutStream import func TSCBasic.withTemporaryFile -import Foundation +import func TSCBasic.withTemporaryDirectory +import func TSCBasic.exec + +struct DebuggableTestSession { + struct Target { + enum Kind { + case xctest(bundlePath: AbsolutePath) + case swiftTesting(binaryPath: AbsolutePath) + } + + let productName: String + let kind: Kind + let additionalArgs: [String] + + var library: TestingLibrary { + switch kind { + case .xctest: .xctest + case .swiftTesting: .swiftTesting + } + } + } + + let targets: NonEmpty + + /// Whether this is part of a multi-session sequence + var isMultiSession: Bool { + !targets.rest.isEmpty + } +} + +enum DebuggerError: Swift.Error { + case noTestProducts + case noEnabledTestingLibraries + case xctestNotFoundInToolchain +} + +extension DebuggerError: CustomStringConvertible { + var description: String { + switch self { + case .noTestProducts: + return "No test products found for debugging" + case .noEnabledTestingLibraries: + return "No testing libraries are enabled for debugging" + case .xctestNotFoundInToolchain: + return "XCTest not found in toolchain" + } + } +} /// Internal helper functionality for the SwiftTestTool command and for the /// plugin support. @@ -418,6 +474,575 @@ extension ToolsVersion { } } +/// A safe wrapper around TSCBasic.exec that resets signal masks and +/// configures file descriptors before replacing the current process. +/// +/// Dispatch disables almost all asynchronous signals on its worker threads. +/// When called from async context, we must reset signal handlers and unblock +/// all signals before exec so the child process starts with clean state. +/// We also mark all non-standard file descriptors as close-on-exec to avoid +/// leaking them to the new process. +func safeExec(path: String, args: [String], observabilityScope: ObservabilityScope) throws -> Never { + #if !os(Windows) + for i in 1.. String { + s.replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + } + + /// Builds an inline LLDB `script` command that registers an + /// `SBDebugger.SetDestroyCallback` to delete the given directory + /// when LLDB tears down. Fires on `quit`, `exit`, EOF, and most + /// normal shutdown paths; hard crashes / SIGKILL still bypass it. + static func atexitCleanupCommand(tempDirectory: AbsolutePath) -> String { + let escaped = escapeForQuotedLLDBArgument(tempDirectory.pathString) + return "script import shutil; lldb.debugger.SetDestroyCallback(lambda _id, d=\"\(escaped)\": shutil.rmtree(d, ignore_errors=True))" + } + + /// Creates an instance of debug test runner. + init( + target: DebuggableTestSession, + buildParameters: BuildParameters, + toolchain: UserToolchain, + testEnv: Environment, + fileSystem: FileSystem, + observabilityScope: ObservabilityScope, + verbose: Bool = false + ) { + self.target = target + self.buildParameters = buildParameters + self.toolchain = toolchain + self.testEnv = testEnv + self.fileSystem = fileSystem + self.observabilityScope = observabilityScope + self.verbose = verbose + } + + /// Launches the test binary under LLDB for interactive debugging. + /// + /// Uses `exec()` so LLDB inherits the controlling terminal directly. + func run() throws { + let lldbPath: AbsolutePath + do { + lldbPath = try toolchain.getLLDB() + } catch { + observabilityScope.emit(error: "Unable to get LLDB: \(error)") + throw error + } + + let lldbArgs = try prepareLLDBArguments(for: target) + observabilityScope.emit(info: "LLDB will run: \(lldbPath.pathString) \(lldbArgs.joined(separator: " "))") + + // Set on the current process so the exec'd LLDB inherits them. + for (key, value) in testEnv { + try Environment.set(key: key, value: value) + } + + try safeExec(path: lldbPath.pathString, args: [lldbPath.pathString] + lldbArgs, observabilityScope: observabilityScope) + } + + /// Prepares LLDB arguments for debugging based on the testing library. + /// + /// This method creates a temporary LLDB command file with the necessary setup commands + /// for debugging tests, including target creation, argument configuration, and symbol loading. + /// + /// - Parameter target: The testing target being used (XCTest or Swift Testing) + /// - Returns: Array of LLDB command line arguments + /// - Throws: Various errors if required tools are not found or file operations fail + private func prepareLLDBArguments(for target: DebuggableTestSession) throws -> [String] { + // One unique scratch directory per invocation so concurrent runs don't collide on fixed filenames + let scratchDir = try withTemporaryDirectory( + dir: try fileSystem.tempDirectory, + prefix: "swiftpm-lldb-", + removeTreeOnDeinit: false + ) { $0 } + + // On success, cleanup is handed off to LLDB's SetDestroyCallback via + // atexitCleanupCommand. If we throw before exec, that callback never + // runs, so we must clean up ourselves or leak /tmp/swiftpm-lldb-*. + do { + var lldbCommands: [String] = [] + try setupTargets(&lldbCommands, scratchDir: scratchDir) + + lldbCommands.append(Self.atexitCleanupCommand(tempDirectory: scratchDir)) + + // Clear the screen of all the previous commands to unclutter the users initial state. + // Skip clearing in verbose mode so startup commands remain visible + if !verbose { + lldbCommands.append("script print(\"\\033[H\\033[J\", end=\"\")") + } + + if Environment.current["SWIFTPM_TESTS_MODULECACHE"] != nil { + if Environment.current["SWIFTPM_TESTS_LLDB_RUN"] != nil { + lldbCommands.append("run") + } + lldbCommands.append("quit") + } + + let commandScript = lldbCommands.joined(separator: "\n") + let lldbCommandFile = scratchDir.appending("lldb-commands.txt") + try fileSystem.writeFileContents(lldbCommandFile, string: commandScript) + + // Return script file arguments without batch mode to allow interactive debugging + return ["-s", lldbCommandFile.pathString] + } catch { + try? fileSystem.removeFileTree(scratchDir) + throw error + } + } + + private func setupTargets(_ lldbCommands: inout [String], scratchDir: AbsolutePath) throws { + var hasSwiftTesting = false + var hasXCTest = false + + for testingLibrary in target.targets { + let (executable, args) = try getExecutableAndArgs(for: testingLibrary) + let escapedExecutable = Self.escapeForQuotedLLDBArgument(executable.pathString) + let escapedProductName = Self.escapeForQuotedLLDBArgument(testingLibrary.productName) + // Smoke-test CI doesn't ship the toolchain's lldb alongside the in-development + // swiftc; it falls back to an older system lldb on PATH that doesn't support + // `target create -l`. Skip the label there so the command parses. + if Environment.current["SWIFTCI_USE_LOCAL_DEPS"] != nil { + lldbCommands.append("target create \"\(escapedExecutable)\"") + } else { + lldbCommands.append("target create -l \"\(escapedProductName) (\(testingLibrary.library))\" \"\(escapedExecutable)\"") + } + lldbCommands.append("settings clear target.run-args") + + for arg in args { + lldbCommands.append("settings append target.run-args \"\(Self.escapeForQuotedLLDBArgument(arg))\"") + } + + let modulePath = try getModulePath(for: testingLibrary) + lldbCommands.append("target modules add \"\(Self.escapeForQuotedLLDBArgument(modulePath.pathString))\"") + + switch testingLibrary.kind { + case .xctest: + hasXCTest = true + case .swiftTesting: + hasSwiftTesting = true + } + } + + setupCommandAliases(&lldbCommands, hasSwiftTesting: hasSwiftTesting, hasXCTest: hasXCTest) + + if target.isMultiSession { + let scriptPath = try createTargetSwitchingScript(in: scratchDir) + lldbCommands.append("command script import \"\(Self.escapeForQuotedLLDBArgument(scriptPath.pathString))\"") + lldbCommands.append("target select 0") + } + } + + private func setupCommandAliases(_ lldbCommands: inout [String], hasSwiftTesting: Bool, hasXCTest: Bool) { + #if os(macOS) + let swiftTestingFailureBreakpoint = "-s Testing -n \"failureBreakpoint()\"" + let xctestFailureBreakpoint = "-n \"_XCTFailureBreakpoint\"" + #elseif os(Windows) + let swiftTestingFailureBreakpoint = "-s Testing.dll -n \"failureBreakpoint()\"" + let xctestFailureBreakpoint = "-s XCTest.dll -n \"XCTest.XCTestCase.recordFailure\"" + #else + let swiftTestingFailureBreakpoint = "-s libTesting.so -n \"Testing.failureBreakpoint\"" + let xctestFailureBreakpoint = "-s libXCTest.so -n \"XCTest.XCTestCase.recordFailure\"" + #endif + + // Add failure breakpoint commands based on available libraries + if hasSwiftTesting && hasXCTest { + lldbCommands.append("command alias failbreak script lldb.debugger.HandleCommand('breakpoint set \(swiftTestingFailureBreakpoint)'); lldb.debugger.HandleCommand('breakpoint set \(xctestFailureBreakpoint)')") + } else if hasSwiftTesting { + lldbCommands.append("command alias failbreak breakpoint set \(swiftTestingFailureBreakpoint)") + } else if hasXCTest { + lldbCommands.append("command alias failbreak breakpoint set \(xctestFailureBreakpoint)") + } + } + + /// Gets the executable path and arguments for a given testing library + private func getExecutableAndArgs(for target: DebuggableTestSession.Target) throws -> (AbsolutePath, [String]) { + switch target.kind { + case .xctest(let bundlePath): + #if os(macOS) + guard let xctestPath = toolchain.xctestPath else { + throw DebuggerError.xctestNotFoundInToolchain + } + return (xctestPath, target.additionalArgs + [bundlePath.pathString]) + #else + return (bundlePath, target.additionalArgs) + #endif + case .swiftTesting(let binaryPath): + #if os(macOS) + let executable = try toolchain.getSwiftTestingHelper() + let args = ["--test-bundle-path", binaryPath.pathString] + target.additionalArgs + #else + let executable = binaryPath + let args = target.additionalArgs + #endif + return (executable, args) + } + } + + /// Gets the module path for symbol loading + private func getModulePath(for target: DebuggableTestSession.Target) throws -> AbsolutePath { + switch target.kind { + case .xctest(let bundlePath): + guard buildParameters.triple.isDarwin() else { + return bundlePath + } + guard let name = bundlePath.components.last?.replacing(".xctest", with: "") else { + return bundlePath + } + let relativePath = try RelativePath(validating: "Contents/MacOS/\(name)") + return bundlePath.appending(relativePath) + case .swiftTesting(let binaryPath): + return binaryPath + } + } + + /// Creates a Python script that handles automatic target switching + private func createTargetSwitchingScript(in scratchDir: AbsolutePath) throws -> AbsolutePath { + let scriptPath = scratchDir.appending("target_switcher.py") + + let pythonScript = """ +# target_switcher.py +import lldb +import re +import threading +import time +import sys + +current_target_index = 0 +max_targets = 0 +debugger_ref = None +sequence_active = True # Start active by default + +def _log_error(message): + \"\"\"Emit a diagnostic to stderr; never raise.\"\"\" + try: + sys.stderr.write("[swift test] " + str(message) + "\\n") + sys.stderr.flush() + except Exception: + pass + +def _run_lldb_command(command): + \"\"\"Run an LLDB command and surface failures via stderr.\"\"\" + if not debugger_ref: + _log_error("skipping command, no debugger: " + command) + return False + return_obj = lldb.SBCommandReturnObject() + debugger_ref.GetCommandInterpreter().HandleCommand(command, return_obj) + if not return_obj.Succeeded(): + _log_error("command failed: " + command + " -- " + (return_obj.GetError() or "")) + return False + return True + +def sync_breakpoints_to_target(source_target, dest_target): + \"\"\"Synchronize breakpoints from source target to destination target.\"\"\" + if not source_target or not dest_target: + return + + def breakpoint_exists_in_target_by_spec(target, file_name, line_number, function_name): + \"\"\"Check if a breakpoint already exists in the target by specification.\"\"\" + for i in range(target.GetNumBreakpoints()): + existing_bp = target.GetBreakpointAtIndex(i) + if not existing_bp.IsValid(): + continue + + # Check function name breakpoints + if function_name: + # Get the breakpoint's function name specifications + names = lldb.SBStringList() + existing_bp.GetNames(names) + + # Check names from GetNames() + for j in range(names.GetSize()): + if names.GetStringAtIndex(j) == function_name: + return True + + # If no names found, check the description for pending breakpoints + if names.GetSize() == 0: + bp_desc = str(existing_bp).strip() + match = re.search(r"name = '([^']+)'", bp_desc) + if match and match.group(1) == function_name: + return True + + # Check file/line breakpoints (only if resolved) + if file_name and line_number: + for j in range(existing_bp.GetNumLocations()): + location = existing_bp.GetLocationAtIndex(j) + if location.IsValid(): + addr = location.GetAddress() + line_entry = addr.GetLineEntry() + if line_entry.IsValid(): + existing_file_spec = line_entry.GetFileSpec() + existing_line_number = line_entry.GetLine() + if (existing_file_spec.GetFilename() == file_name and + existing_line_number == line_number): + return True + return False + + # Get all breakpoints from source target + for i in range(source_target.GetNumBreakpoints()): + bp = source_target.GetBreakpointAtIndex(i) + if not bp.IsValid(): + continue + + # Handle breakpoints by their specifications, not just resolved locations + # First check if this is a function name breakpoint + names = lldb.SBStringList() + bp.GetNames(names) + + # For pending breakpoints, GetNames() might be empty, so also check the description + bp_desc = str(bp).strip() + + # Extract function name from description if names is empty + function_names_to_sync = [] + if names.GetSize() > 0: + # Use the names from GetNames() + for j in range(names.GetSize()): + function_name = names.GetStringAtIndex(j) + if function_name: + function_names_to_sync.append(function_name) + else: + # Parse function name from description for pending breakpoints + match = re.search(r"name = '([^']+)'", bp_desc) + if match: + function_name = match.group(1) + function_names_to_sync.append(function_name) + + # Sync the function name breakpoints + for function_name in function_names_to_sync: + if not breakpoint_exists_in_target_by_spec(dest_target, None, None, function_name): + new_bp = dest_target.BreakpointCreateByName(function_name) + if new_bp.IsValid(): + new_bp.SetEnabled(bp.IsEnabled()) + new_bp.SetCondition(bp.GetCondition()) + new_bp.SetIgnoreCount(bp.GetIgnoreCount()) + else: + _log_error("failed to create breakpoint by name: " + function_name) + + # Handle resolved location-based breakpoints (file/line) + # Only process if the breakpoint has resolved locations + if bp.GetNumLocations() > 0: + for j in range(bp.GetNumLocations()): + location = bp.GetLocationAtIndex(j) + if not location.IsValid(): + continue + + addr = location.GetAddress() + line_entry = addr.GetLineEntry() + + if line_entry.IsValid(): + file_spec = line_entry.GetFileSpec() + line_number = line_entry.GetLine() + file_name = file_spec.GetFilename() + + # Check if this breakpoint already exists in destination target + if breakpoint_exists_in_target_by_spec(dest_target, file_name, line_number, None): + continue + + # Create the same breakpoint in the destination target + new_bp = dest_target.BreakpointCreateByLocation(file_spec, line_number) + if new_bp.IsValid(): + # Copy breakpoint properties + new_bp.SetEnabled(bp.IsEnabled()) + new_bp.SetCondition(bp.GetCondition()) + new_bp.SetIgnoreCount(bp.GetIgnoreCount()) + else: + _log_error("failed to create breakpoint at " + str(file_name) + ":" + str(line_number)) + +def sync_breakpoints_to_all_targets(): + \"\"\"Synchronize breakpoints from current target to all other targets.\"\"\" + global debugger_ref, max_targets + + if not debugger_ref or max_targets <= 1: + return + + current_target = debugger_ref.GetSelectedTarget() + if not current_target: + return + + # Sync to all other targets + for i in range(max_targets): + target = debugger_ref.GetTargetAtIndex(i) + if target and target != current_target: + sync_breakpoints_to_target(current_target, target) + +def monitor_breakpoints(): + \"\"\"Monitor breakpoint changes and sync them across targets.\"\"\" + global debugger_ref, max_targets + + if max_targets <= 1: + return + + last_breakpoint_count = 0 + + while True: # Keep running forever, not just while current_target_index < max_targets + try: + if debugger_ref: + current_target = debugger_ref.GetSelectedTarget() + if current_target: + current_bp_count = current_target.GetNumBreakpoints() + + # If breakpoint count changed, sync to all targets + if current_bp_count != last_breakpoint_count: + sync_breakpoints_to_all_targets() + last_breakpoint_count = current_bp_count + except Exception as e: + _log_error("monitor_breakpoints iteration failed: " + repr(e)) + + time.sleep(0.5) # Check every 500ms + +def check_process_status(): + \"\"\"Periodically check if the current process has exited.\"\"\" + global current_target_index, max_targets, debugger_ref, sequence_active + + while True: # Keep running forever, don't exit + try: + if debugger_ref: + target = debugger_ref.GetSelectedTarget() + if target: + process = target.GetProcess() + if process and process.GetState() == lldb.eStateExited: + # Process has exited + if sequence_active and current_target_index < max_targets: + # We're in an active sequence, trigger switch + current_target_index += 1 + + if current_target_index < max_targets: + # Switch to next target and launch immediately + print("\\n") + _run_lldb_command(f'target select {current_target_index}') + print(" ") + + # Get target name for user feedback + new_target = debugger_ref.GetSelectedTarget() + target_name = new_target.GetExecutable().GetFilename() if new_target else "Unknown" + + # Launch the next target immediately + _run_lldb_command('process launch') + else: + # Reset to first target and deactivate sequence until user runs again + current_target_index = 0 + sequence_active = False # Pause automatic switching + + print("\\n") + _run_lldb_command('target select 0') + print("\\nAll testing targets completed.") + print("Type 'run' to restart the entire test sequence from the beginning.\\n") + + # Clear the current line and move cursor to start + sys.stdout.write("\\033[2K\\r") + # Reprint a fake prompt + sys.stdout.write("(lldb) ") + sys.stdout.flush() + elif process and process.GetState() in [lldb.eStateRunning, lldb.eStateLaunching]: + # Process is running - if sequence was inactive, reactivate it + if not sequence_active: + sequence_active = True + # Find which target is currently selected to set the correct index + selected_target = debugger_ref.GetSelectedTarget() + if selected_target: + for i in range(max_targets): + if debugger_ref.GetTargetAtIndex(i) == selected_target: + current_target_index = i + break + except Exception as e: + _log_error("check_process_status iteration failed: " + repr(e)) + + time.sleep(0.1) # Check every 100ms + +def __lldb_init_module(debugger, internal_dict): + global max_targets, debugger_ref + + debugger_ref = debugger + + # Count the number of targets + max_targets = debugger.GetNumTargets() + + if max_targets > 1: + # Start the process status checker + status_thread = threading.Thread(target=check_process_status, daemon=True) + status_thread.start() + + # Start the breakpoint monitor + bp_thread = threading.Thread(target=monitor_breakpoints, daemon=True) + bp_thread.start() +""" + + try fileSystem.writeFileContents(scriptPath, string: pythonScript) + return scriptPath + } +} + extension SwiftCommandState { func buildParametersForTest( enableCodeCoverage: Bool, diff --git a/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift b/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift index c2333414075..bd58e8e5bbf 100644 --- a/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift +++ b/Sources/_InternalTestSupport/SwiftTesting+Helpers.swift @@ -12,6 +12,8 @@ import Basics import Testing +import Foundation +import class TSCBasic.BufferedOutputByteStream fileprivate func fileErrorMessage( @@ -255,3 +257,28 @@ private func _expectThrowsCommandExecutionError( } return try errorHandler(CommandExecutionError(result: processResult, stdout: stdout, stderr: stderr)) } + +/// Checks if an output stream contains a specific string, with retry logic for asynchronous writes. +/// - Parameters: +/// - outputStream: The output stream to check +/// - needle: The string to search for in the output stream +/// - timeout: Maximum time to wait for the string to appear (default: 3 seconds) +/// - retryInterval: Time to wait between checks (default: 50 milliseconds) +/// - Returns: True if the string was found within the timeout period +public func waitForOutputStreamToContain( + _ outputStream: BufferedOutputByteStream, + _ needle: String, + timeout: TimeInterval = 3.0, + retryInterval: TimeInterval = 0.05 +) async throws -> Bool { + let startTime = Date() + while Date().timeIntervalSince(startTime) < timeout { + if outputStream.bytes.description.contains(needle) { + return true + } + + try await Task.sleep(nanoseconds: UInt64(retryInterval * 1_000_000_000)) + } + + return outputStream.bytes.description.contains(needle) +} diff --git a/Sources/_InternalTestSupport/misc.swift b/Sources/_InternalTestSupport/misc.swift index 66efaea877b..d743dba2ee9 100644 --- a/Sources/_InternalTestSupport/misc.swift +++ b/Sources/_InternalTestSupport/misc.swift @@ -594,14 +594,7 @@ private func swiftArgs( Xswiftc: [String], buildSystem: BuildSystemProvider.Kind? ) -> [String] { - var args = ["--configuration"] - switch configuration { - case .debug: - args.append("debug") - case .release: - args.append("release") - } - + var args = configuration.buildArgs args += Xcc.flatMap { ["-Xcc", $0] } args += Xld.flatMap { ["-Xlinker", $0] } args += Xswiftc.flatMap { ["-Xswiftc", $0] } @@ -643,6 +636,19 @@ public func loadPackageGraph( ) } +extension BuildConfiguration { + public var buildArgs: [String] { + var args = ["--configuration"] + switch self { + case .debug: + args.append("debug") + case .release: + args.append("release") + } + return args + } +} + public let emptyZipFile = ByteString([0x80, 0x75, 0x05, 0x06] + [UInt8](repeating: 0x00, count: 18)) extension FileSystem { diff --git a/Tests/CommandsTests/TestCommandTests.swift b/Tests/CommandsTests/TestCommandTests.swift index e03ce0bc845..d510d93e244 100644 --- a/Tests/CommandsTests/TestCommandTests.swift +++ b/Tests/CommandsTests/TestCommandTests.swift @@ -10,8 +10,10 @@ // //===----------------------------------------------------------------------===// -import Foundation +@testable import Commands +@testable import CoreCommands +import Foundation import Basics import Commands import struct SPMBuildCore.BuildSystemProvider @@ -21,6 +23,10 @@ import _InternalTestSupport import TSCTestSupport import Testing +import struct ArgumentParser.ExitCode +import protocol ArgumentParser.AsyncParsableCommand +import class TSCBasic.BufferedOutputByteStream + @Suite( .serialized, // to limit the number of swift executable running. .tags( @@ -303,7 +309,7 @@ struct TestCommandTests { let configuration = BuildConfiguration.debug try await withKnownIssue(isIntermittent: true) { try await fixture(name: "Miscellaneous/TestableExeWithDifferentProductName") { fixturePath in - let result = try await execute( + _ = try await execute( ["--vv"], packagePath: fixturePath, configuration: configuration, @@ -1566,7 +1572,7 @@ struct TestCommandTests { try await fixture(name: "Miscellaneous/Errors/FatalErrorInSingleXCTest/TypeLibrary") { fixturePath in // WHEN swift-test is executed let error = await #expect(throws: SwiftPMError.self) { - try await self.execute( + try await execute( [], packagePath: fixturePath, configuration: configuration, @@ -1602,7 +1608,7 @@ struct TestCommandTests { } @Test( - .IssueWindowsLongPath, + .IssueWindowsLongPath, .tags( .Feature.TargetType.Executable, ), @@ -1614,7 +1620,7 @@ struct TestCommandTests { let configuration = BuildConfiguration.debug try await withKnownIssue(isIntermittent: true) { try await fixture(name: "Miscellaneous/TestableExeWithResources") { fixturePath in - let result = try await execute( + _ = try await execute( ["--vv"], packagePath: fixturePath, configuration: configuration, @@ -1625,6 +1631,498 @@ struct TestCommandTests { .windows == ProcessInfo.hostOperatingSystem || ProcessInfo.processInfo.environment["SWIFTCI_EXHIBITS_GH_9524"] != nil } - } + } + + // MARK: - LLDB Flag Validation Tests + + @Suite + struct LLDBTests { + private func execute( + _ args: [String], + packagePath: AbsolutePath? = nil, + configuration: BuildConfiguration = .debug, + buildSystem: BuildSystemProvider.Kind, + throwIfCommandFails: Bool = true + ) async throws -> (stdout: String, stderr: String) { + try await executeSwiftTest( + packagePath, + configuration: configuration, + extraArgs: args, + buildSystem: buildSystem, + throwIfCommandFails: throwIfCommandFails + ) + } + + /// Smoke test that verifies `validateLLDBCompatibility` is wired into + /// the command pipeline. The individual incompatibility rules are + /// covered directly by `ValidationTests` below. + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbValidationIsWiredIntoCommandPipeline(buildSystem: BuildSystemProvider.Kind) async throws { + let args = args(["--debugger", "--parallel"], for: buildSystem) + let command = try #require(SwiftTestCommand.parseAsRoot(args) as? SwiftTestCommand) + let (state, outputStream) = try commandState() + + let error = await #expect(throws: ExitCode.self) { + try await command.run(state) + } + + #expect(error == ExitCode.failure, "Expected ExitCode.failure, got \(String(describing: error))") + + // The output stream is written to asynchronously on a DispatchQueue and can + // receive output after the command has thrown. + let found = try await waitForOutputStreamToContain(outputStream, "--debugger") + #expect( + found, + "Expected validation error to surface via the command pipeline, got: \(outputStream.bytes.description)" + ) + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithAllTestingLibrariesDisabledThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (_, stderr) = try await execute( + ["--debugger", "--disable-xctest", "--disable-swift-testing"], + packagePath: fixturePath, + buildSystem: buildSystem, + throwIfCommandFails: false + ) + + #expect( + stderr.contains("No testing libraries are enabled for debugging"), + "Expected error about no testing libraries, got stderr: \(stderr)" + ) + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbWithNoTestTargetsThrowsError(buildSystem: BuildSystemProvider.Kind) async throws { + try await fixture(name: "Miscellaneous/AtMainSupport") { fixturePath in + let (_, stderr) = try await execute( + ["--debugger"], + packagePath: fixturePath, + buildSystem: buildSystem, + throwIfCommandFails: false + ) + + #expect( + stderr.contains("no tests found"), + "Expected error about no tests found, got stderr: \(stderr)" + ) + } + } + @Test( + arguments: SupportedBuildSystemOnAllPlatforms, + ) + func debuggerFlagWithXCTestSuite(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-swift-testing", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + + #if os(macOS) + let targetName = "xctest" + #else + let targetName = buildSystem == .swiftbuild ? "test-runner" : "xctest" + #endif + + #expect( + stdout.contains("target create") && stdout.contains(targetName), + "Expected LLDB to target xctest binary, got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("failbreak breakpoint set"), + "Expected a failure breakpoint to be setup, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test( + arguments: SupportedBuildSystemOnAllPlatforms + ) + func debuggerFlagWithSwiftTestingSuite(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-xctest", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + + #if os(macOS) + let targetName = "swiftpm-testing-helper" + #else + let targetName = buildSystem == .native ? "TestDebuggingPackageTests.xctest" : "TestDebuggingTests-test-runner" + #endif + + #expect( + stdout.contains("target create") && stdout.contains(targetName), + "Expected LLDB to target swiftpm-testing-helper binary, got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("failbreak breakpoint set"), + "Expected Swift Testing failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test( + arguments: SupportedBuildSystemOnAllPlatforms + ) + func debuggerFlagWithBothTestingSuites(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + #expect( + !stderr.contains("error: --debugger cannot be used with"), + "got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("target create"), + "Expected LLDB to create targets, got stdout: \(stdout), stderr: \(stderr)", + ) + + let productName = buildSystem == .native ? "TestDebuggingPackageTests" : "TestDebuggingTests" + withKnownIssue { + #expect( + stdout.contains("\(productName) (XCTest)") && stdout.contains("\(productName) (Swift Testing)"), + "Expected labeled LLDB targets, got stdout: \(stdout), stderr: \(stderr)", + ) + } when: { + // Smoke-test CI's lldb is too old to support `target create -l`, + // so the production code skips emitting labels there. + CiEnvironment.runningInSmokeTestPipeline + } + + #expect( + getNumberOfMatches(of: "breakpoint set", in: stdout) == 2, + "Expected combined failure breakpoint setup, got stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("command script import"), + "Expected Python script import for multi-target switching, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } + + @Test( + arguments: SupportedBuildSystemOnAllPlatforms + ) + func debuggerFlagWithMultipleTestProducts(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await withKnownIssue { + try await fixture(name: "Miscellaneous/TestDebuggingMultiProduct") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + withKnownIssue { + #expect( + !stderr.contains("error:"), + "Expected no errors, got stdout: \(stdout), stderr: \(stderr)", + ) + } when: { + // Smoke-test CI's lldb lacks Python bindings, so `command script import` + // and `script print(...)` emit Python errors to stderr. + CiEnvironment.runningInSmokeTestPipeline + } + + let targetCreateCount = getNumberOfMatches(of: "target create", in: stdout) + // Native build system produces a single umbrella product (2 targets: xctest + swift-testing). + // Swiftbuild produces one product per test target (4 targets: 2 products x 2 libraries). + let expectedMinTargets = buildSystem == .native ? 2 : 4 + #expect( + targetCreateCount >= expectedMinTargets, + "Expected at least \(expectedMinTargets) LLDB targets, got \(targetCreateCount). stdout: \(stdout), stderr: \(stderr)", + ) + + #expect( + stdout.contains("command script import"), + "Expected Python script import for multi-target switching, got stdout: \(stdout), stderr: \(stderr)", + ) + } + } when: { + // swift-build on Windows fails to emit the per-target *.LinkFileList + // for the second test runner in a multi-test-target package, so the + // build itself fails before lldb is ever invoked. Same root cause as + // noteTestFailures with TestMixedFailuresAcrossTargets. + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .windows + } + } + + @Test( + arguments: SupportedBuildSystemOnAllPlatforms + ) + func debuggerFlagWithMultipleTestProductsXCTestOnly(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await withKnownIssue { + try await fixture(name: "Miscellaneous/TestDebuggingMultiProduct") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-swift-testing", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + withKnownIssue { + #expect( + !stderr.contains("error:"), + "Expected no errors, got stdout: \(stdout), stderr: \(stderr)", + ) + } when: { + // Smoke-test CI's lldb is too old and lacks Python bindings. + CiEnvironment.runningInSmokeTestPipeline + } + + let targetCreateCount = getNumberOfMatches(of: "target create", in: stdout) + // Native: 1 umbrella product → 1 xctest target. + // Swiftbuild: 2 products → 2 xctest targets. + let expectedMinTargets = buildSystem == .native ? 1 : 2 + #expect( + targetCreateCount >= expectedMinTargets, + "Expected at least \(expectedMinTargets) LLDB targets, got \(targetCreateCount). stdout: \(stdout), stderr: \(stderr)", + ) + } + } when: { + // swift-build on Windows fails to emit the per-target *.LinkFileList + // for the second test runner in a multi-test-target package, so the + // build itself fails before lldb is ever invoked. Same root cause as + // noteTestFailures with TestMixedFailuresAcrossTargets. + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .windows + } + } + + @Test( + arguments: SupportedBuildSystemOnAllPlatforms + ) + func debuggerFlagWithMultipleTestProductsSwiftTestingOnly(buildSystem: BuildSystemProvider.Kind) async throws { + let configuration = BuildConfiguration.debug + try await withKnownIssue { + try await fixture(name: "Miscellaneous/TestDebuggingMultiProduct") { fixturePath in + let (stdout, stderr) = try await execute( + ["--debugger", "--disable-xctest", "--verbose"], + packagePath: fixturePath, + configuration: configuration, + buildSystem: buildSystem, + ) + + withKnownIssue { + #expect( + !stderr.contains("error:"), + "Expected no errors, got stdout: \(stdout), stderr: \(stderr)", + ) + } when: { + // Smoke-test CI's lldb is too old and lacks Python bindings. + CiEnvironment.runningInSmokeTestPipeline + } + + let targetCreateCount = getNumberOfMatches(of: "target create", in: stdout) + // Native: 1 umbrella product → 1 swift-testing target. + // Swiftbuild: 2 products → 2 swift-testing targets. + let expectedMinTargets = buildSystem == .native ? 1 : 2 + #expect( + targetCreateCount >= expectedMinTargets, + "Expected at least \(expectedMinTargets) LLDB targets, got \(targetCreateCount). stdout: \(stdout), stderr: \(stderr)", + ) + } + } when: { + // swift-build on Windows fails to emit the per-target *.LinkFileList + // for the second test runner in a multi-test-target package, so the + // build itself fails before lldb is ever invoked. Same root cause as + // noteTestFailures with TestMixedFailuresAcrossTargets. + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .windows + } + } + + @Test(arguments: SupportedBuildSystemOnAllPlatforms) + func lldbRunExecutesTestsSuccessfully(buildSystem: BuildSystemProvider.Kind) async throws { + try await fixture(name: "Miscellaneous/TestDebugging") { fixturePath in + let (stdout, stderr) = try await executeSwiftTest( + fixturePath, + configuration: .debug, + extraArgs: [ + "--debugger", + "--disable-swift-testing", + "--verbose", + "--filter", "XCTestCalculatorTests/testAdditionPasses", + ] + getBuildSystemArgs(for: buildSystem), + env: ["SWIFTPM_TESTS_LLDB_RUN": "1"], + buildSystem: buildSystem, + throwIfCommandFails: false + ) + + withKnownIssue { + #expect( + stdout.contains("Process") && stdout.contains("launched"), + "Expected LLDB to launch the process, got stdout: \(stdout), stderr: \(stderr)" + ) + + #expect( + stdout.contains("exited with status = 0"), + "Expected process to exit with status 0, got stdout: \(stdout), stderr: \(stderr)" + ) + } when: { + // Smoke-test CI runs an old /opt/swift/5.9.2/usr/bin/lldb that was built + // without Python bindings. + CiEnvironment.runningInSmokeTestPipeline + } + } + } + + /// Direct unit tests for the validation logic, exercising + /// `validateLLDBCompatibility` without going through the full + /// command pipeline. + @Suite + struct ValidationTests { + private func expectValidationError( + configuration: BuildConfiguration = .debug, + shouldRunInParallel: Bool = false, + numberOfWorkers: Int? = nil, + shouldListTests: Bool = false, + shouldPrintCodeCovPath: Bool = false, + containing substrings: [String] + ) { + let error = #expect(throws: StringError.self) { + try SwiftTestCommand.validateLLDBCompatibility( + configuration: configuration, + shouldRunInParallel: shouldRunInParallel, + numberOfWorkers: numberOfWorkers, + shouldListTests: shouldListTests, + shouldPrintCodeCovPath: shouldPrintCodeCovPath + ) + } + let message = error?.description ?? "" + for substring in substrings { + #expect( + message.contains(substring), + "Expected error message to contain '\(substring)', got: \(message)" + ) + } + } + + @Test + func releaseConfigurationIsRejected() { + expectValidationError( + configuration: .release, + containing: ["--debugger", "release configuration"] + ) + } + + @Test + func parallelFlagIsRejected() { + expectValidationError( + shouldRunInParallel: true, + containing: ["--debugger", "--parallel"] + ) + } + + @Test + func numWorkersIsRejected() { + expectValidationError( + numberOfWorkers: 2, + containing: ["--debugger", "--num-workers"] + ) + } + + @Test + func listTestsIsRejected() { + expectValidationError( + shouldListTests: true, + containing: ["--debugger", "--list-tests"] + ) + } + + @Test + func showCodeCovPathIsRejected() { + expectValidationError( + shouldPrintCodeCovPath: true, + containing: ["--debugger", "--show-codecov-path"] + ) + } + + @Test + func compatibleOptionsPassValidation() throws { + try SwiftTestCommand.validateLLDBCompatibility( + configuration: .debug, + shouldRunInParallel: false, + numberOfWorkers: nil, + shouldListTests: false, + shouldPrintCodeCovPath: false + ) + } + + @Test + func configurationIsCheckedBeforeOtherFlags() { + // When multiple incompatible flags are set, the release-configuration + // check fires first so callers see a single, deterministic error. + expectValidationError( + configuration: .release, + shouldRunInParallel: true, + numberOfWorkers: 2, + shouldListTests: true, + shouldPrintCodeCovPath: true, + containing: ["release configuration"] + ) + } + } + + func args(_ args: [String], for buildSystem: BuildSystemProvider.Kind, buildConfiguration: BuildConfiguration = .debug) -> [String] { + return args + buildConfiguration.buildArgs + getBuildSystemArgs(for: buildSystem) + } + + func commandState() throws -> (SwiftCommandState, BufferedOutputByteStream) { + let outputStream = BufferedOutputByteStream() + + let state = try SwiftCommandState( + outputStream: outputStream, + options: try GlobalOptions.parse([]), + toolWorkspaceConfiguration: .init(shouldInstallSignalHandlers: false), + workspaceDelegateProvider: { + CommandWorkspaceDelegate( + observabilityScope: $0, + outputHandler: $1, + progressHandler: $2, + inputHandler: $3 + ) + }, + workspaceLoaderProvider: { + XcodeWorkspaceLoader( + fileSystem: $0, + observabilityScope: $1 + ) + }, + createPackagePath: false + ) + return (state, outputStream) + } + } } +