Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Rethink how we capture expectation conditions and their subexpressions. #840

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,13 @@ let package = Package(
.enableExperimentalFeature("SymbolLinkageMarkers"),
]
),
.testTarget(
name: "SubexpressionShowcase",
dependencies: [
"Testing",
],
swiftSettings: .packageSettings
),

.macro(
name: "TestingMacros",
Expand Down Expand Up @@ -152,6 +159,7 @@ extension Array where Element == PackageDescription.SwiftSetting {
.unsafeFlags(["-require-explicit-sendable"]),
.enableUpcomingFeature("ExistentialAny"),
.enableExperimentalFeature("SuppressedAssociatedTypes"),
.enableExperimentalFeature("NonescapableTypes"),

.enableExperimentalFeature("AccessLevelOnImport"),
.enableUpcomingFeature("InternalImportsByDefault"),
Expand Down
34 changes: 34 additions & 0 deletions Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedExpectation.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2025 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

extension ABIv0 {
/// A type implementing the JSON encoding of ``Expectation`` for the ABI entry
/// point and event stream output.
///
/// This type is not part of the public interface of the testing library. It
/// assists in converting values to JSON; clients that consume this JSON are
/// expected to write their own decoders.
///
/// - Warning: Expectations are not yet part of the JSON schema.
struct EncodedExpectation: Sendable {
/// The expression evaluated by this expectation.
///
/// - Warning: Expressions are not yet part of the JSON schema.
var _expression: EncodedExpression

init(encoding expectation: borrowing Expectation, in eventContext: borrowing Event.Context) {
_expression = EncodedExpression(encoding: expectation.evaluatedExpression, in: eventContext)
}
}
}

// MARK: - Codable

extension ABIv0.EncodedExpectation: Codable {}
52 changes: 52 additions & 0 deletions Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedExpression.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2025 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

extension ABIv0 {
/// A type implementing the JSON encoding of ``Expression`` for the ABI entry
/// point and event stream output.
///
/// This type is not part of the public interface of the testing library. It
/// assists in converting values to JSON; clients that consume this JSON are
/// expected to write their own decoders.
///
/// - Warning: Expressions are not yet part of the JSON schema.
struct EncodedExpression: Sendable {
/// The source code of the original captured expression.
var sourceCode: String

/// A string representation of the runtime value of this expression.
///
/// If the runtime value of this expression has not been evaluated, the
/// value of this property is `nil`.
var runtimeValue: String?

/// The fully-qualified name of the type of value represented by
/// `runtimeValue`, or `nil` if that value has not been captured.
var runtimeTypeName: String?

/// Any child expressions within this expression.
var children: [EncodedExpression]?

init(encoding expression: borrowing __Expression, in eventContext: borrowing Event.Context) {
sourceCode = expression.sourceCode
runtimeValue = expression.runtimeValue.map(String.init(describingForTest:))
runtimeTypeName = expression.runtimeValue.map(\.typeInfo.fullyQualifiedName)
if !expression.subexpressions.isEmpty {
children = expression.subexpressions.map { [eventContext = copy eventContext] subexpression in
Self(encoding: subexpression, in: eventContext)
}
}
}
}
}

// MARK: - Codable

extension ABIv0.EncodedExpression: Codable {}
8 changes: 8 additions & 0 deletions Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ extension ABIv0 {
/// - Warning: Errors are not yet part of the JSON schema.
var _error: EncodedError?

/// The expectation associated with this issue, if applicable.
///
/// - Warning: Expectations are not yet part of the JSON schema.
var _expectation: EncodedExpectation?

init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) {
isKnown = issue.isKnown
sourceLocation = issue.sourceLocation
Expand All @@ -41,6 +46,9 @@ extension ABIv0 {
if let error = issue.error {
_error = EncodedError(encoding: error, in: eventContext)
}
if case let .expectationFailed(expectation) = issue.kind {
_expectation = EncodedExpectation(encoding: expectation, in: eventContext)
}
}
}
}
Expand Down
8 changes: 8 additions & 0 deletions Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,19 @@ extension ABIv0 {
/// The symbol associated with this message.
var symbol: Symbol

/// How much to indent this message when presenting it.
///
/// - Warning: This property is not yet part of the JSON schema.
var _indentation: Int?

/// The human-readable, unformatted text associated with this message.
var text: String

init(encoding message: borrowing Event.HumanReadableOutputRecorder.Message) {
symbol = Symbol(encoding: message.symbol ?? .default)
if message.indentation > 0 {
_indentation = message.indentation
}
text = message.conciseStringValue ?? message.stringValue
}
}
Expand Down
6 changes: 5 additions & 1 deletion Sources/Testing/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ add_library(Testing
ABI/v0/Encoded/ABIv0.EncodedBacktrace.swift
ABI/v0/Encoded/ABIv0.EncodedError.swift
ABI/v0/Encoded/ABIv0.EncodedEvent.swift
ABI/v0/Encoded/ABIv0.EncodedExpectation.swift
ABI/v0/Encoded/ABIv0.EncodedExpression.swift
ABI/v0/Encoded/ABIv0.EncodedInstant.swift
ABI/v0/Encoded/ABIv0.EncodedIssue.swift
ABI/v0/Encoded/ABIv0.EncodedMessage.swift
Expand All @@ -39,6 +41,8 @@ add_library(Testing
Expectations/Expectation.swift
Expectations/Expectation+Macro.swift
Expectations/ExpectationChecking+Macro.swift
Expectations/ExpectationContext.swift
Expectations/ExpectationContext+Pointers.swift
Issues/Confirmation.swift
Issues/ErrorSnapshot.swift
Issues/Issue.swift
Expand All @@ -61,7 +65,7 @@ add_library(Testing
SourceAttribution/Backtrace+Symbolication.swift
SourceAttribution/CustomTestStringConvertible.swift
SourceAttribution/Expression.swift
SourceAttribution/Expression+Macro.swift
SourceAttribution/ExpressionID.swift
SourceAttribution/SourceContext.swift
SourceAttribution/SourceLocation.swift
SourceAttribution/SourceLocation+Macro.swift
Expand Down
40 changes: 25 additions & 15 deletions Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,22 @@ private let _ansiEscapeCodePrefix = "\u{001B}["
private let _resetANSIEscapeCode = "\(_ansiEscapeCodePrefix)0m"

extension Event.Symbol {
/// Get the string value to use for a message with no associated symbol.
///
/// - Parameters:
/// - options: Options to use when writing the symbol.
///
/// - Returns: A string representation of "no symbol" appropriate for writing
/// to a stream.
fileprivate static func placeholderStringValue(options: Event.ConsoleOutputRecorder.Options) -> String {
#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst))
if options.useSFSymbols {
return " "
}
#endif
return " "
}

/// Get the string value for this symbol with the given write options.
///
/// - Parameters:
Expand Down Expand Up @@ -169,7 +185,7 @@ extension Event.Symbol {
case .attachment:
return "\(_ansiEscapeCodePrefix)94m\(symbolCharacter)\(_resetANSIEscapeCode)"
case .details:
return symbolCharacter
return "\(symbolCharacter)"
}
}
return "\(symbolCharacter)"
Expand Down Expand Up @@ -303,18 +319,12 @@ extension Event.ConsoleOutputRecorder {
/// - Returns: Whether any output was produced and written to this instance's
/// destination.
@discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool {
let messages = _humanReadableOutputRecorder.record(event, in: context)

// Padding to use in place of a symbol for messages that don't have one.
var padding = " "
#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst))
if options.useSFSymbols {
padding = " "
}
#endif
let symbolPlaceholder = Event.Symbol.placeholderStringValue(options: options)

let messages = _humanReadableOutputRecorder.record(event, in: context)
let lines = messages.lazy.map { [test = context.test] message in
let symbol = message.symbol?.stringValue(options: options) ?? padding
let symbol = message.symbol?.stringValue(options: options) ?? symbolPlaceholder
let indentation = String(repeating: " ", count: message.indentation)

if case .details = message.symbol {
// Special-case the detail symbol to apply grey to the entire line of
Expand All @@ -323,17 +333,17 @@ extension Event.ConsoleOutputRecorder {
// to the indentation provided by the symbol.
var lines = message.stringValue.split(whereSeparator: \.isNewline)
lines = CollectionOfOne(lines[0]) + lines.dropFirst().map { line in
"\(padding) \(line)"
"\(indentation)\(symbolPlaceholder) \(line)"
}
let stringValue = lines.joined(separator: "\n")
if options.useANSIEscapeCodes, options.ansiColorBitDepth > 1 {
return "\(_ansiEscapeCodePrefix)90m\(symbol) \(stringValue)\(_resetANSIEscapeCode)\n"
return "\(_ansiEscapeCodePrefix)90m\(symbol) \(indentation)\(stringValue)\(_resetANSIEscapeCode)\n"
} else {
return "\(symbol) \(stringValue)\n"
return "\(symbol) \(indentation)\(stringValue)\n"
}
} else {
let colorDots = test.map { self.colorDots(for: $0.tags) } ?? ""
return "\(symbol) \(colorDots)\(message.stringValue)\n"
return "\(symbol) \(indentation)\(colorDots)\(message.stringValue)\n"
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,15 @@ extension Event {
/// The symbol associated with this message, if any.
var symbol: Symbol?

/// How much to indent this message when presenting it.
///
/// The way in which this additional indentation is rendered is
/// implementation-defined. Typically, the greater the value of this
/// property, the more whitespace characters are inserted.
///
/// Rendering of indentation is optional.
var indentation = 0

/// The human-readable message.
var stringValue: String

Expand Down Expand Up @@ -415,20 +424,18 @@ extension Event.HumanReadableOutputRecorder {
}
additionalMessages += _formattedComments(issue.comments)

if verbosity > 0, case let .expectationFailed(expectation) = issue.kind {
if verbosity >= 0, case let .expectationFailed(expectation) = issue.kind {
let expression = expectation.evaluatedExpression
func addMessage(about expression: __Expression) {
let description = expression.expandedDebugDescription()
additionalMessages.append(Message(symbol: .details, stringValue: description))
}
let subexpressions = expression.subexpressions
if subexpressions.isEmpty {
addMessage(about: expression)
} else {
for subexpression in subexpressions {
addMessage(about: subexpression)
func addMessage(about expression: __Expression, depth: Int) {
let description = expression.expandedDescription(verbose: verbosity > 0)
if description != expression.sourceCode {
additionalMessages.append(Message(symbol: .details, indentation: depth, stringValue: description))
}
for subexpression in expression.subexpressions {
addMessage(about: subexpression, depth: depth + 1)
}
}
addMessage(about: expression, depth: 0)
}

let atSourceLocation = issue.sourceLocation.map { " at \($0)" } ?? ""
Expand Down
12 changes: 8 additions & 4 deletions Sources/Testing/ExitTests/ExitTest.swift
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ func callExitTest(
identifiedBy exitTestID: ExitTest.ID,
exitsWith expectedExitCondition: ExitCondition,
observing observedValues: [any PartialKeyPath<ExitTestArtifacts> & Sendable],
expression: __Expression,
sourceCode: @escaping @autoclosure @Sendable () -> [__ExpressionID: String],
comments: @autoclosure () -> [Comment],
isRequired: Bool,
isolation: isolated (any Actor)? = #isolation,
Expand Down Expand Up @@ -335,10 +335,14 @@ func callExitTest(
let actualExitCondition = result.exitCondition

// Plumb the exit test's result through the general expectation machinery.
return __checkValue(
let expectationContext = __ExpectationContext(
sourceCode: sourceCode(),
runtimeValues: [.root: { Expression.Value(reflecting: actualExitCondition) }]
)
return check(
expectedExitCondition == actualExitCondition,
expression: expression,
expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualExitCondition),
expectationContext: expectationContext,
mismatchedErrorDescription: nil,
mismatchedExitConditionDescription: String(describingForTest: expectedExitCondition),
comments: comments(),
isRequired: isRequired,
Expand Down
8 changes: 4 additions & 4 deletions Sources/Testing/Expectations/Expectation+Macro.swift
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,10 @@
/// running in the current task and an instance of ``ExpectationFailedError`` is
/// thrown.
@freestanding(expression) public macro require<T>(
_ optionalValue: T?,
_ optionalValue: consuming T?,
_ comment: @autoclosure () -> Comment? = nil,
sourceLocation: SourceLocation = #_sourceLocation
) -> T = #externalMacro(module: "TestingMacros", type: "RequireMacro")
) -> T = #externalMacro(module: "TestingMacros", type: "UnwrapMacro") where T: ~Copyable

/// Unwrap an optional boolean value or, if it is `nil`, fail and throw an
/// error.
Expand Down Expand Up @@ -124,10 +124,10 @@ public macro require(
@freestanding(expression)
@_documentation(visibility: private)
public macro require<T>(
_ optionalValue: T,
_ optionalValue: consuming T,
_ comment: @autoclosure () -> Comment? = nil,
sourceLocation: SourceLocation = #_sourceLocation
) -> T = #externalMacro(module: "TestingMacros", type: "NonOptionalRequireMacro")
) -> T = #externalMacro(module: "TestingMacros", type: "NonOptionalRequireMacro") where T: ~Copyable

// MARK: - Matching errors by type

Expand Down
8 changes: 5 additions & 3 deletions Sources/Testing/Expectations/Expectation.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@
public struct Expectation: Sendable {
/// The expression evaluated by this expectation.
@_spi(ForToolsIntegrationOnly)
public var evaluatedExpression: Expression
public internal(set) var evaluatedExpression: Expression

/// A description of the error mismatch that occurred, if any.
///
/// If this expectation passed, the value of this property is `nil` because no
/// error mismatch occurred.
@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
public var mismatchedErrorDescription: String?
public internal(set) var mismatchedErrorDescription: String?

/// A description of the difference between the operands in the expression
/// evaluated by this expectation, if the difference could be determined.
Expand All @@ -28,7 +28,9 @@ public struct Expectation: Sendable {
/// the difference is only computed when necessary to assist with diagnosing
/// test failures.
@_spi(Experimental) @_spi(ForToolsIntegrationOnly)
public var differenceDescription: String?
public var differenceDescription: String? {
evaluatedExpression.differenceDescription
}

/// A description of the exit condition that was expected to be matched.
///
Expand Down
Loading