From 1de520549271de747f09436f2ebcdbeda1a00640 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:07:48 +0100 Subject: [PATCH 01/11] fix(agent): harden model execution continuity --- .../Agent/AgentCompatibilityTypes.swift | 87 +++- .../PeekabooAgentService+Enhancements.swift | 4 + .../PeekabooAgentService+Execution.swift | 178 ++++--- .../PeekabooAgentService+ModelIdentity.swift | 186 +++++++ ...eekabooAgentService+SessionLifecycle.swift | 111 +++- .../Agent/PeekabooAgentService+Sessions.swift | 98 +++- ...eekabooAgentService+StreamProcessing.swift | 38 +- .../PeekabooAgentService+Streaming.swift | 474 ++++++++++++++---- .../Agent/PeekabooAgentService.swift | 37 +- .../Agent/QueueMode.swift | 18 + .../MCP/Tools/MCPAgentTool.swift | 10 +- .../Protocols/AgentServiceProtocol.swift | 14 + .../Services/AI/PeekabooAIService.swift | 43 +- Tachikoma | 2 +- 14 files changed, 1079 insertions(+), 221 deletions(-) create mode 100644 Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+ModelIdentity.swift diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/AgentCompatibilityTypes.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/AgentCompatibilityTypes.swift index 2fa342568..fd8f4bf3c 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/AgentCompatibilityTypes.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/AgentCompatibilityTypes.swift @@ -174,6 +174,15 @@ public struct AgentSession: Sendable, Codable { /// Model name used in this session public let modelName: String + /// Credential-free provider-qualified model selection used to resume this session. + public let modelSelection: String? + + /// Hash of the canonical provider endpoint used by this session. + public let modelEndpointIdentity: String? + + /// Credential-free provider kind used to prevent namespace or protocol drift. + public let modelProviderIdentity: String? + /// Complete conversation history public let messages: [ModelMessage] @@ -189,6 +198,9 @@ public struct AgentSession: Sendable, Codable { public init( id: String, modelName: String, + modelSelection: String? = nil, + modelEndpointIdentity: String? = nil, + modelProviderIdentity: String? = nil, messages: [ModelMessage], metadata: SessionMetadata, createdAt: Date, @@ -196,6 +208,9 @@ public struct AgentSession: Sendable, Codable { { self.id = id self.modelName = modelName + self.modelSelection = modelSelection + self.modelEndpointIdentity = modelEndpointIdentity + self.modelProviderIdentity = modelProviderIdentity self.messages = messages self.metadata = metadata self.createdAt = createdAt @@ -235,6 +250,18 @@ public struct SessionMetadata: Sendable, Codable { } } +/// Errors raised while resolving session persistence paths. +public enum AgentSessionManagerError: Error, LocalizedError, Equatable, Sendable { + case invalidSessionID + + public var errorDescription: String? { + switch self { + case .invalidSessionID: + "Invalid session ID." + } + } +} + /// Manages agent conversation sessions with persistence and caching @MainActor public final class AgentSessionManager: @unchecked Sendable { @@ -250,11 +277,11 @@ public final class AgentSessionManager: @unchecked Sendable { public init(sessionDirectory: URL? = nil) throws { if let sessionDirectory { - self.sessionDirectory = sessionDirectory + self.sessionDirectory = sessionDirectory.standardizedFileURL } else { // Default to ~/.peekaboo/sessions/ let homeDir = self.fileManager.homeDirectoryForCurrentUser - self.sessionDirectory = homeDir.appendingPathComponent(".peekaboo/sessions") + self.sessionDirectory = homeDir.appendingPathComponent(".peekaboo/sessions").standardizedFileURL } // Create session directory if it doesn't exist @@ -275,6 +302,9 @@ public final class AgentSessionManager: @unchecked Sendable { do { let data = try Data(contentsOf: url) let session = try JSONDecoder().decode(AgentSession.self, from: data) + guard try self.sessionFileURL(for: session.id) == url.standardizedFileURL else { + return nil + } let resourceValues = try url.resourceValues(forKeys: [ .creationDateKey, @@ -289,7 +319,7 @@ public final class AgentSessionManager: @unchecked Sendable { createdAt: createdAt, lastAccessedAt: lastAccessedAt, messageCount: session.messages.count, - status: self.isSessionExpired(lastAccessedAt) ? .expired : .active, + status: self.sessionStatus(for: session, lastAccessedAt: lastAccessedAt), summary: self.generateSessionSummary(from: session.messages)) } catch { return nil @@ -303,9 +333,9 @@ public final class AgentSessionManager: @unchecked Sendable { /// Save a session to persistent storage public func saveSession(_ session: AgentSession) throws { // Save a session to persistent storage - let sessionFile = self.sessionDirectory.appendingPathComponent("\(session.id).json") + let sessionFile = try self.sessionFileURL(for: session.id) let data = try JSONEncoder().encode(session) - try data.write(to: sessionFile) + try data.write(to: sessionFile, options: .atomic) self.sessionCache[session.id] = session self.evictOldCacheEntries() @@ -313,18 +343,21 @@ public final class AgentSessionManager: @unchecked Sendable { /// Load a session from storage public func loadSession(id: String) async throws -> AgentSession? { + let sessionFile = try self.sessionFileURL(for: id) if let cachedSession = self.sessionCache[id] { return cachedSession } // Load from disk - let sessionFile = self.sessionDirectory.appendingPathComponent("\(id).json") guard self.fileManager.fileExists(atPath: sessionFile.path) else { return nil } let data = try Data(contentsOf: sessionFile) let session = try JSONDecoder().decode(AgentSession.self, from: data) + guard session.id == id else { + throw AgentSessionManagerError.invalidSessionID + } self.sessionCache[id] = session self.evictOldCacheEntries() @@ -335,7 +368,7 @@ public final class AgentSessionManager: @unchecked Sendable { /// Delete a session public func deleteSession(id: String) async throws { // Delete a session - let sessionFile = self.sessionDirectory.appendingPathComponent("\(id).json") + let sessionFile = try self.sessionFileURL(for: id) try self.fileManager.removeItem(at: sessionFile) self.sessionCache.removeValue(forKey: id) @@ -354,6 +387,46 @@ public final class AgentSessionManager: @unchecked Sendable { // MARK: - Private Methods + private func sessionFileURL(for id: String) throws -> URL { + guard !id.isEmpty, + id != ".", + id != "..", + !id.contains("/"), + !id.contains("\\"), + !id.utf8.contains(0) + else { + throw AgentSessionManagerError.invalidSessionID + } + + let fileName = "\(id).json" + let sessionFile = self.sessionDirectory + .appendingPathComponent(fileName, isDirectory: false) + .standardizedFileURL + guard sessionFile.deletingLastPathComponent().standardizedFileURL == self.sessionDirectory, + sessionFile.lastPathComponent == fileName + else { + throw AgentSessionManagerError.invalidSessionID + } + return sessionFile + } + + private func sessionStatus(for session: AgentSession, lastAccessedAt: Date) -> SessionStatus { + if self.isSessionExpired(lastAccessedAt) { + return .expired + } + + switch session.metadata.customData["status"] { + case SessionStatus.completed.rawValue: + return .completed + case SessionStatus.failed.rawValue, "cancelled": + return .failed + case SessionStatus.expired.rawValue: + return .expired + default: + return .active + } + } + private func isSessionExpired(_ lastAccessed: Date) -> Bool { Date().timeIntervalSince(lastAccessed) > Self.maxSessionAge } diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Enhancements.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Enhancements.swift index 5705e3b97..d9d1ff263 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Enhancements.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Enhancements.swift @@ -427,6 +427,7 @@ extension PeekabooAgentService { /// Configuration for streaming loop with enhancements. struct EnhancedStreamingConfiguration { let model: LanguageModel + let provider: any ModelProvider let tools: [AgentTool] let sessionId: String let eventHandler: EventHandler? @@ -434,12 +435,14 @@ extension PeekabooAgentService { init( model: LanguageModel, + provider: any ModelProvider, tools: [AgentTool], sessionId: String, eventHandler: EventHandler?, enhancementOptions: AgentEnhancementOptions = .default) { self.model = model + self.provider = provider self.tools = tools self.sessionId = sessionId self.eventHandler = eventHandler @@ -458,6 +461,7 @@ extension PeekabooAgentService { // Convert to standard configuration, passing through enhancement options let standardConfig = StreamingLoopConfiguration( model: configuration.model, + provider: configuration.provider, tools: configuration.tools, sessionId: configuration.sessionId, eventHandler: configuration.eventHandler, diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift index 2186c8a8b..830c91b7d 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift @@ -5,6 +5,7 @@ import Foundation import PeekabooAutomation +import PeekabooFoundation import Tachikoma @available(macOS 14.0, *) @@ -235,9 +236,14 @@ extension PeekabooAgentService { eventContinuation.finish() await eventTask.value return result + } catch let error as CancellationError { + eventContinuation.finish() + await eventTask.value + throw error } catch { + await eventHandler.send(.error(message: error.localizedDescription)) eventContinuation.finish() - eventTask.cancel() + await eventTask.value throw error } } @@ -288,34 +294,62 @@ extension PeekabooAgentService { eventHandler: EventHandler? = nil, enhancementOptions: AgentEnhancementOptions? = nil) async throws -> AgentExecutionResult { + let maxSteps = try AgentStepBudget.validate(maxSteps) _ = streamingDelegate let tools = await self.buildToolset(for: model) self.logModelUsage(model, prefix: "Streaming ") + guard let provider = context.provider else { + throw PeekabooError.invalidInput("The session has no verified model provider; refusing to execute it.") + } let configuration = StreamingLoopConfiguration( model: model, + provider: provider, tools: tools, sessionId: context.id, eventHandler: eventHandler, enhancementOptions: enhancementOptions) - let outcome = try await self.runStreamingLoop( - configuration: configuration, - maxSteps: maxSteps, - initialMessages: context.messages, - queueMode: queueMode) + var latestCheckpoint = self.makeLoopOutcome( + state: StreamingLoopState(messages: context.messages), + reachedStepLimit: false) + let outcome: StreamingLoopOutcome + do { + outcome = try await self.runStreamingLoop( + configuration: configuration, + maxSteps: maxSteps, + initialMessages: context.messages, + queueMode: queueMode) + { latestCheckpoint = $0 } + } catch { + let wasCancelled = self.isAgentCancellation(error) + self.preserveExecutionCheckpoint( + context: context, + model: model, + checkpoint: latestCheckpoint, + status: wasCancelled ? "cancelled" : "failed") + if wasCancelled { + throw CancellationError() + } + throw error + } let endTime = Date() let executionTime = endTime.timeIntervalSince(context.executionStart) let toolCallCount = outcome.toolCallCount - try self.saveCompletedSession( + try self.saveExecutionSession( context: context, model: model, finalMessages: outcome.messages, endTime: endTime, toolCallCount: toolCallCount, - usage: outcome.usage) + usage: outcome.usage, + status: outcome.reachedStepLimit ? "max_steps_exhausted" : "completed") + + if outcome.reachedStepLimit { + throw AgentStepLimitExceededError(maxSteps: maxSteps, sessionId: context.id) + } return AgentExecutionResult( content: outcome.content, @@ -338,31 +372,59 @@ extension PeekabooAgentService { eventHandler: EventHandler? = nil, enhancementOptions: AgentEnhancementOptions? = nil) async throws -> AgentExecutionResult { + let maxSteps = try AgentStepBudget.validate(maxSteps) let tools = await self.buildToolset(for: model) self.logModelUsage(model, prefix: "") + guard let provider = context.provider else { + throw PeekabooError.invalidInput("The session has no verified model provider; refusing to execute it.") + } let configuration = StreamingLoopConfiguration( model: model, + provider: provider, tools: tools, sessionId: context.id, eventHandler: eventHandler, enhancementOptions: enhancementOptions) - let outcome = try await self.runGenerationLoop( - configuration: configuration, - maxSteps: maxSteps, - initialMessages: context.messages) + var latestCheckpoint = self.makeLoopOutcome( + state: StreamingLoopState(messages: context.messages), + reachedStepLimit: false) + let outcome: StreamingLoopOutcome + do { + outcome = try await self.runGenerationLoop( + configuration: configuration, + maxSteps: maxSteps, + initialMessages: context.messages) + { latestCheckpoint = $0 } + } catch { + let wasCancelled = self.isAgentCancellation(error) + self.preserveExecutionCheckpoint( + context: context, + model: model, + checkpoint: latestCheckpoint, + status: wasCancelled ? "cancelled" : "failed") + if wasCancelled { + throw CancellationError() + } + throw error + } let endTime = Date() let executionTime = endTime.timeIntervalSince(context.executionStart) - try self.saveCompletedSession( + try self.saveExecutionSession( context: context, model: model, finalMessages: outcome.messages, endTime: endTime, toolCallCount: outcome.toolCallCount, - usage: outcome.usage) + usage: outcome.usage, + status: outcome.reachedStepLimit ? "max_steps_exhausted" : "completed") + + if outcome.reachedStepLimit { + throw AgentStepLimitExceededError(maxSteps: maxSteps, sessionId: context.id) + } return AgentExecutionResult( content: outcome.content, @@ -380,7 +442,8 @@ extension PeekabooAgentService { func runGenerationLoop( configuration: StreamingLoopConfiguration, maxSteps: Int, - initialMessages: [ModelMessage]) async throws -> StreamingLoopOutcome + initialMessages: [ModelMessage], + onCheckpoint: ((StreamingLoopOutcome) -> Void)? = nil) async throws -> StreamingLoopOutcome { var state = StreamingLoopState(messages: initialMessages) let toolContext = ToolHandlingContext( @@ -391,14 +454,12 @@ extension PeekabooAgentService { enhancementOptions: configuration.enhancementOptions) let resolvedConfiguration = TachikomaConfiguration.resolve(.current) - let provider = try resolvedConfiguration.makeProvider(for: configuration.model) - var totalInputTokens = 0 - var totalOutputTokens = 0 - var totalInputCost = 0.0 - var totalOutputCost = 0.0 - var hasUsage = false + let provider = configuration.provider + var usageAccumulator = AgentUsageAccumulator() + var reachedStepLimit = false for stepIndex in 0.. 0 || totalOutputCost > 0 - ? Usage.Cost(input: totalInputCost, output: totalOutputCost) - : nil - state.usage = Usage(inputTokens: totalInputTokens, outputTokens: totalOutputTokens, cost: totalCost) - } + try Task.checkCancellation() let recordedAssistantTurn = self.appendResponseHistory( from: response, @@ -448,7 +508,6 @@ extension PeekabooAgentService { peekabooConfiguration: self.services.configuration, to: &state.messages) - let toolCalls = response.toolCalls ?? [] if toolCalls.isEmpty { self.appendFinalStep( text: response.text, @@ -459,16 +518,30 @@ extension PeekabooAgentService { break } - let step = try await self.handleToolCalls( - stepText: response.text, - toolCalls: toolCalls, - context: toolContext, - currentMessages: &state.messages, - stepIndex: stepIndex, - appendAssistantMessage: !recordedAssistantTurn, - emitToolStartEvents: true) + var cancellationStep: GenerationStep? + let step: GenerationStep + do { + step = try await self.handleToolCalls( + stepText: response.text, + toolCalls: toolCalls, + context: toolContext, + currentMessages: &state.messages, + stepIndex: stepIndex, + appendAssistantMessage: !recordedAssistantTurn, + emitToolStartEvents: true, + onCancellationCheckpoint: { cancellationStep = $0 }) + } catch { + if self.isAgentCancellation(error), let cancellationStep { + state.steps.append(cancellationStep) + state.toolCallCount += cancellationStep.toolResults.count + onCheckpoint?(self.makeLoopOutcome(state: state, reachedStepLimit: false)) + throw CancellationError() + } + throw error + } state.steps.append(step) state.toolCallCount += step.toolResults.count + onCheckpoint?(self.makeLoopOutcome(state: state, reachedStepLimit: false)) if let stopReason = self.turnBoundaryStopReason(from: step.toolResults) { if state.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { @@ -477,20 +550,11 @@ extension PeekabooAgentService { break } - if response.finishReason != .toolCalls, response.finishReason != .stop { - break + if stepIndex == maxSteps - 1 { + reachedStepLimit = true } } - if !hasUsage { - state.usage = nil - } - - return StreamingLoopOutcome( - content: state.content, - messages: state.messages, - steps: state.steps, - usage: state.usage, - toolCallCount: state.toolCallCount) + return self.makeLoopOutcome(state: state, reachedStepLimit: reachedStepLimit) } } diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+ModelIdentity.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+ModelIdentity.swift new file mode 100644 index 000000000..1f1c214af --- /dev/null +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+ModelIdentity.swift @@ -0,0 +1,186 @@ +import CryptoKit +import Foundation +import PeekabooAutomation +import PeekabooFoundation +import Tachikoma + +@available(macOS 14.0, *) +extension PeekabooAgentService { + struct PersistedModelIdentity: Equatable { + let displayName: String + let selection: String? + let endpointIdentity: String? + let providerIdentity: String? + } + + func persistedModelIdentity(for model: LanguageModel) -> PersistedModelIdentity { + let configuration = TachikomaConfiguration.resolve(.current) + guard let provider = try? configuration.makeProvider(for: model) else { + return PersistedModelIdentity( + displayName: self.safeModelDisplayName(for: model), + selection: nil, + endpointIdentity: nil, + providerIdentity: nil) + } + return self.persistedModelIdentity(for: model, provider: provider) + } + + func persistedModelIdentity( + for model: LanguageModel, + provider: any ModelProvider) -> PersistedModelIdentity + { + let displayName = self.safeModelDisplayName(for: model) + let endpointIdentity = Self.canonicalEndpointIdentity(provider.baseURL) + let providerIdentity = self.modelProviderIdentity(for: model, provider: provider) + guard let selection = self.persistedModelSelection(for: model), + endpointIdentity != nil, + providerIdentity != nil + else { + return PersistedModelIdentity( + displayName: displayName, + selection: nil, + endpointIdentity: endpointIdentity, + providerIdentity: providerIdentity) + } + + return PersistedModelIdentity( + displayName: displayName, + selection: selection, + endpointIdentity: endpointIdentity, + providerIdentity: providerIdentity) + } + + func currentEndpointIdentity(for model: LanguageModel) -> String? { + let configuration = TachikomaConfiguration.resolve(.current) + guard let provider = try? configuration.makeProvider(for: model) else { + return nil + } + return Self.canonicalEndpointIdentity(provider.baseURL) + } + + func safeModelDisplayName(for model: LanguageModel) -> String { + switch model { + case let .azureOpenAI(deployment, _, _, _): + "AzureOpenAI/\(deployment)" + case let .openaiCompatible(modelID, _): + "OpenAI-Compatible/\(modelID)" + case let .anthropicCompatible(modelID, _): + "Anthropic-Compatible/\(modelID)" + default: + model.description + } + } + + func modelProviderIdentity(for model: LanguageModel) -> String? { + let configuration = TachikomaConfiguration.resolve(.current) + guard let provider = try? configuration.makeProvider(for: model) else { return nil } + return self.modelProviderIdentity(for: model, provider: provider) + } + + func modelProviderIdentity( + for model: LanguageModel, + provider: any ModelProvider) -> String? + { + guard self.providerMatchesModel(model, provider: provider) else { return nil } + + if case let .custom(configuredModel) = model { + guard let configuredProvider = configuredModel as? any PeekabooCustomProviderIdentityProviding + else { + return nil + } + return "custom:\(configuredProvider.providerTypeIdentity)" + } + + return self.builtInProviderIdentity(for: model, provider: provider) + } + + private func builtInProviderIdentity( + for model: LanguageModel, + provider: any ModelProvider) -> String? + { + switch model { + case .openai where provider is OpenAIResponsesProvider: "openai:responses" + case .openai: "openai:chat" + case .anthropic: "anthropic" + case .google: "google" + case .mistral: "mistral" + case .groq: "groq" + case .grok: "grok" + case .ollama: "ollama" + case .lmstudio: "lmstudio" + case .minimax: "minimax" + case .minimaxCN: "minimax-cn" + case .kimi: "kimi" + case .azureOpenAI: "azure-openai" + case .openRouter: "openrouter" + case .together: "together" + case .replicate: "replicate" + case .openaiCompatible: "openai-compatible" + case .anthropicCompatible: "anthropic-compatible" + case .custom: nil + } + } + + private func providerMatchesModel( + _ model: LanguageModel, + provider: any ModelProvider) -> Bool + { + switch model { + case .openai: + return provider is OpenAIProvider || provider is OpenAIResponsesProvider + case .anthropic: return provider is AnthropicProvider + case .google: return provider is GoogleProvider + case .mistral: return provider is MistralProvider + case .groq: return provider is GroqProvider + case .grok: return provider is GrokProvider + case .ollama: return provider is OllamaProvider + case .lmstudio: return provider is LMStudioProvider + case .minimax, .minimaxCN: return provider is AnthropicCompatibleProvider + case .kimi: return provider is KimiProvider + case .azureOpenAI: return provider is AzureOpenAIProvider + case .openRouter: return provider is OpenRouterProvider + case .together: return provider is TogetherProvider + case .replicate: return provider is ReplicateProvider + case .openaiCompatible: return provider is OpenAICompatibleProvider + case .anthropicCompatible: return provider is AnthropicCompatibleProvider + case let .custom(configuredModel): + guard + let expected = configuredModel as? any PeekabooCustomProviderIdentityProviding, + let actual = provider as? any PeekabooCustomProviderIdentityProviding + else { + return false + } + return expected.providerID == actual.providerID && + expected.resolvedModelID == actual.resolvedModelID && + expected.providerTypeIdentity == actual.providerTypeIdentity + } + } + + nonisolated static func canonicalEndpointIdentity(_ rawValue: String?) -> String? { + guard + let trimmed = rawValue?.trimmingCharacters(in: .whitespacesAndNewlines), + !trimmed.isEmpty, + var components = URLComponents(string: trimmed), + let scheme = components.scheme?.lowercased(), + let host = components.host?.lowercased() + else { + return nil + } + + components.scheme = scheme + components.host = host + components.password = nil + components.fragment = nil + while components.path.count > 1, components.path.hasSuffix("/") { + components.path.removeLast() + } + + guard let value = components.string, let data = value.data(using: .utf8) else { + return nil + } + let digest = SHA256.hash(data: data) + .map { String(format: "%02x", $0) } + .joined() + return "sha256:\(digest)" + } +} diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+SessionLifecycle.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+SessionLifecycle.swift index ce306b0be..b0c0761fc 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+SessionLifecycle.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+SessionLifecycle.swift @@ -9,6 +9,12 @@ import Tachikoma @available(macOS 14.0, *) extension PeekabooAgentService { + struct ResolvedContinuationModel { + let model: LanguageModel + let provider: any ModelProvider + let identity: PersistedModelIdentity + } + public func continueSession( sessionId: String, userMessage: String, @@ -20,30 +26,65 @@ extension PeekabooAgentService { verbose: Bool = false, enhancementOptions: AgentEnhancementOptions? = .default) async throws -> AgentExecutionResult { + try await self.continueSessionInternal( + sessionId: sessionId, + userMessage: userMessage, + model: model, + maxSteps: maxSteps, + dryRun: dryRun, + queueMode: queueMode, + eventDelegate: eventDelegate, + verbose: verbose, + enhancementOptions: enhancementOptions) + } + + // swiftlint:disable:next function_parameter_count + private func continueSessionInternal( + sessionId: String, + userMessage: String?, + model: LanguageModel?, + maxSteps: Int, + dryRun: Bool, + queueMode: QueueMode, + eventDelegate: (any AgentEventDelegate)?, + verbose: Bool, + enhancementOptions: AgentEnhancementOptions?) async throws -> AgentExecutionResult + { + let maxSteps = try AgentStepBudget.validate(maxSteps) self.isVerbose = verbose TachikomaConfiguration.current.setVerbose(verbose) guard let existingSession = try await self.sessionManager.loadSession(id: sessionId) else { throw PeekabooError.sessionNotFound(sessionId) } + let resolvedModel = try self.resolveContinuationModelContext( + explicitModel: model, + session: existingSession) + let selectedModel = resolvedModel.model + let taskDescription = userMessage ?? "Resume session \(sessionId)" if dryRun { let now = Date() return AgentExecutionResult( - content: "Dry run completed. Session \(sessionId) would receive: \(userMessage)", + content: userMessage.map { "Dry run completed. Session \(sessionId) would receive: \($0)" } ?? + "Dry run completed. Session \(sessionId) would resume from its saved turn boundary.", messages: existingSession.messages, sessionId: sessionId, usage: nil, metadata: AgentMetadata( executionTime: 0, toolCallCount: 0, - modelName: existingSession.modelName, + modelName: self.safeModelDisplayName(for: selectedModel), startTime: now, endTime: now)) } - let selectedModel = self.resolveModel(model) - let sessionContext = self.makeContinuationContext(from: existingSession, userMessage: userMessage) + let sessionContext = self.makeContinuationContext( + from: existingSession, + userMessage: userMessage, + model: selectedModel, + provider: resolvedModel.provider, + modelIdentity: resolvedModel.identity) if let eventDelegate { let unsafeDelegate = UnsafeTransfer(eventDelegate) @@ -51,7 +92,7 @@ extension PeekabooAgentService { let eventTask = Task { @MainActor in let delegate = unsafeDelegate.wrappedValue - delegate.agentDidEmitEvent(.started(task: userMessage)) + delegate.agentDidEmitEvent(.started(task: taskDescription)) for await event in eventStream { delegate.agentDidEmitEvent(event) } @@ -88,9 +129,14 @@ extension PeekabooAgentService { eventContinuation.finish() await eventTask.value return result + } catch let error as CancellationError { + eventContinuation.finish() + await eventTask.value + throw error } catch { + await eventHandler.send(.error(message: error.localizedDescription)) eventContinuation.finish() - eventTask.cancel() + await eventTask.value throw error } } else { @@ -110,13 +156,13 @@ extension PeekabooAgentService { eventDelegate: (any AgentEventDelegate)? = nil, enhancementOptions: AgentEnhancementOptions? = .default) async throws -> AgentExecutionResult { - let continuationPrompt = "Continue from where we left off." - return try await self.continueSession( + try await self.continueSessionInternal( sessionId: sessionId, - userMessage: continuationPrompt, + userMessage: nil, model: model, maxSteps: maxSteps, dryRun: false, + queueMode: .oneAtATime, eventDelegate: eventDelegate, verbose: self.isVerbose, enhancementOptions: enhancementOptions) @@ -137,6 +183,53 @@ extension PeekabooAgentService { try await self.sessionManager.loadSession(id: sessionId) } + func resolveContinuationModel( + explicitModel: LanguageModel?, + session: AgentSession) throws -> LanguageModel + { + try self.resolveContinuationModelContext( + explicitModel: explicitModel, + session: session).model + } + + func resolveContinuationModelContext( + explicitModel: LanguageModel?, + session: AgentSession) throws -> ResolvedContinuationModel + { + if let explicitModel { + let provider = try TachikomaConfiguration.resolve(.current).makeProvider(for: explicitModel) + return ResolvedContinuationModel( + model: explicitModel, + provider: provider, + identity: self.persistedModelIdentity(for: explicitModel, provider: provider)) + } + + guard let selection = session.modelSelection, + let endpointIdentity = session.modelEndpointIdentity, + let providerIdentity = session.modelProviderIdentity, + let persistedModel = self.resolveConfiguredModel(selection), + persistedModel.supportsTools + else { + throw PeekabooError.invalidInput( + "Session \(session.id) was created with model '\(session.modelName)', but its original " + + "provider, model, and endpoint can no longer be verified safely. Pass an explicit model " + + "to resume this session.") + } + let provider = try TachikomaConfiguration.resolve(.current).makeProvider(for: persistedModel) + let identity = self.persistedModelIdentity(for: persistedModel, provider: provider) + guard identity.displayName == session.modelName, + identity.selection == selection, + identity.endpointIdentity == endpointIdentity, + identity.providerIdentity == providerIdentity + else { + throw PeekabooError.invalidInput( + "Session \(session.id) was created with model '\(session.modelName)', but its original " + + "provider, model, and endpoint can no longer be verified safely. Pass an explicit model " + + "to resume this session.") + } + return ResolvedContinuationModel(model: persistedModel, provider: provider, identity: identity) + } + /// Delete a specific session public func deleteSession(id: String) async throws { // Delete a specific session diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Sessions.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Sessions.swift index ffe3ff0a9..03de82fb0 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Sessions.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Sessions.swift @@ -8,12 +8,17 @@ import Tachikoma @available(macOS 14.0, *) extension PeekabooAgentService { + private static let usageObservedMetadataKey = "agent_usage_observed" + private static let usageCostCompleteMetadataKey = "agent_usage_cost_complete" + struct SessionContext { let id: String let messages: [ModelMessage] let createdAt: Date let executionStart: Date let metadata: SessionMetadata + let modelIdentity: PersistedModelIdentity + let provider: (any ModelProvider)? } enum SessionLogBehavior { @@ -34,10 +39,16 @@ extension PeekabooAgentService { ModelMessage.system(AgentSystemPrompt.generate(for: model)), ModelMessage.user(task), ] + let configuration = TachikomaConfiguration.resolve(.current) + let provider = try configuration.makeProvider(for: model) + let modelIdentity = self.persistedModelIdentity(for: model, provider: provider) let session = AgentSession( id: sessionId, - modelName: model.description, + modelName: modelIdentity.displayName, + modelSelection: modelIdentity.selection, + modelEndpointIdentity: modelIdentity.endpointIdentity, + modelProviderIdentity: modelIdentity.providerIdentity, messages: messages, metadata: SessionMetadata(), createdAt: startTime, @@ -60,36 +71,57 @@ extension PeekabooAgentService { messages: messages, createdAt: startTime, executionStart: startTime, - metadata: SessionMetadata()) + metadata: SessionMetadata(), + modelIdentity: modelIdentity, + provider: provider) } // swiftlint:disable:next function_parameter_count - func saveCompletedSession( + func saveExecutionSession( context: SessionContext, model: LanguageModel, finalMessages: [ModelMessage], endTime: Date, toolCallCount: Int, - usage: Usage?) throws + usage: Usage?, + status: String) throws { let executionTime = endTime.timeIntervalSince(context.executionStart) let totalTokens = context.metadata.totalTokens + (usage?.totalTokens ?? 0) - let additionalCost = usage?.cost?.total - let accumulatedCost: Double? = if additionalCost == nil, context.metadata.totalCost == nil { - nil + let hadPreviousUsage = context.metadata.customData[Self.usageObservedMetadataKey] + .flatMap(Bool.init) ?? (context.metadata.totalTokens > 0 || context.metadata.totalCost != nil) + let previousCostWasComplete = context.metadata.customData[Self.usageCostCompleteMetadataKey] + .flatMap(Bool.init) ?? (context.metadata.totalCost != nil) + let hasAdditionalUsage = usage != nil + let additionalCostIsComplete = usage?.cost != nil + let hasAccumulatedUsage = hadPreviousUsage || hasAdditionalUsage + let accumulatedCostIsComplete = (!hadPreviousUsage || previousCostWasComplete) && + (!hasAdditionalUsage || additionalCostIsComplete) + let accumulatedCost: Double? = if hasAccumulatedUsage, accumulatedCostIsComplete { + (context.metadata.totalCost ?? 0) + (usage?.cost?.total ?? 0) } else { - (context.metadata.totalCost ?? 0) + (additionalCost ?? 0) + nil } + let customData = context.metadata.customData.merging([ + "status": status, + Self.usageObservedMetadataKey: String(hasAccumulatedUsage), + Self.usageCostCompleteMetadataKey: String(accumulatedCostIsComplete), + ]) { _, new in new } + let updatedMetadata = SessionMetadata( totalTokens: totalTokens, totalCost: accumulatedCost, toolCallCount: context.metadata.toolCallCount + toolCallCount, totalExecutionTime: context.metadata.totalExecutionTime + executionTime, - customData: context.metadata.customData.merging(["status": "completed"]) { _, new in new }) + customData: customData) + let modelIdentity = context.modelIdentity let updatedSession = AgentSession( id: context.id, - modelName: model.description, + modelName: modelIdentity.displayName, + modelSelection: modelIdentity.selection, + modelEndpointIdentity: modelIdentity.endpointIdentity, + modelProviderIdentity: modelIdentity.providerIdentity, messages: finalMessages, metadata: updatedMetadata, createdAt: context.createdAt, @@ -97,6 +129,27 @@ extension PeekabooAgentService { try self.sessionManager.saveSession(updatedSession) } + func preserveExecutionCheckpoint( + context: SessionContext, + model: LanguageModel, + checkpoint: StreamingLoopOutcome, + status: String) + { + do { + try self.saveExecutionSession( + context: context, + model: model, + finalMessages: checkpoint.messages, + endTime: Date(), + toolCallCount: checkpoint.toolCallCount, + usage: checkpoint.usage, + status: status) + } catch { + let message = "Failed to preserve \(status) agent session \(context.id): \(error.localizedDescription)" + self.logger.error("\(message, privacy: .public)") + } + } + func makeExecutionMetadata( model: LanguageModel, executionTime: TimeInterval, @@ -107,15 +160,15 @@ extension PeekabooAgentService { AgentMetadata( executionTime: executionTime, toolCallCount: toolCallCount, - modelName: model.description, + modelName: self.safeModelDisplayName(for: model), startTime: startTime, endTime: endTime) } func logModelUsage(_ model: LanguageModel, prefix: String) { guard self.isVerbose else { return } - self.logger.debug("\(prefix)Using model: \(model)") - self.logger.debug("\(prefix)Model description: \(model.description)") + let displayName = self.safeModelDisplayName(for: model) + self.logger.debug("\(prefix)Using model: \(displayName, privacy: .public)") } private func logSession(_ message: String, force: Bool) { @@ -124,14 +177,27 @@ extension PeekabooAgentService { } } - func makeContinuationContext(from session: AgentSession, userMessage: String) -> SessionContext { + func makeContinuationContext( + from session: AgentSession, + userMessage: String?, + model: LanguageModel, + provider: (any ModelProvider)? = nil, + modelIdentity: PersistedModelIdentity? = nil) -> SessionContext + { var updatedMessages = session.messages - updatedMessages.append(.user(userMessage)) + if let userMessage { + updatedMessages.append(.user(userMessage)) + } + let provider = provider ?? (try? TachikomaConfiguration.resolve(.current).makeProvider(for: model)) + let modelIdentity = modelIdentity ?? provider.map { self.persistedModelIdentity(for: model, provider: $0) } ?? + self.persistedModelIdentity(for: model) return SessionContext( id: session.id, messages: updatedMessages, createdAt: session.createdAt, executionStart: Date(), - metadata: session.metadata) + metadata: session.metadata, + modelIdentity: modelIdentity, + provider: provider) } } diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+StreamProcessing.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+StreamProcessing.swift index ff367b959..4950b5276 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+StreamProcessing.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+StreamProcessing.swift @@ -31,7 +31,8 @@ extension PeekabooAgentService { from streamResult: StreamTextResult, model: LanguageModel, eventHandler: EventHandler?, - stepIndex: Int) async throws -> StreamProcessingOutput + stepIndex: Int, + onTerminalUsage: ((Usage?) -> Void)? = nil) async throws -> StreamProcessingOutput { var stepText = "" var reasoningBlocks: [ReasoningBlock] = [] @@ -52,7 +53,11 @@ extension PeekabooAgentService { self.logger.debug("Starting to process stream for step \(stepIndex)") } + try Task.checkCancellation() for try await delta in streamResult.stream { + if delta.type != .done { + try Task.checkCancellation() + } if self.isVerbose { self.logger.debug("Received delta type: \(String(describing: delta.type))") } @@ -85,7 +90,6 @@ extension PeekabooAgentService { stepToolCalls: &stepToolCalls, seenToolCallIds: &seenToolCallIds, bufferedEvents: &bufferedEvents, - buffersEventsUntilDone: buffersAssistantTextUntilDone, eventHandler: eventHandler) } @@ -135,6 +139,7 @@ extension PeekabooAgentService { eventHandler: eventHandler) case .done: + let isFirstDone = !didReceiveDone didReceiveDone = true self.flushPendingReasoningText( &pendingReasoningText, @@ -142,23 +147,30 @@ extension PeekabooAgentService { type: pendingReasoningType, reasoningBlocks: &reasoningBlocks) usage = delta.usage + if isFirstDone { + onTerminalUsage?(usage) + } finishReason = delta.finishReason + try Task.checkCancellation() default: break } } + try Task.checkCancellation() if buffersAssistantTextUntilDone, !didReceiveDone { throw TachikomaError.apiError("Provider stream ended without a terminal event") } - if buffersAssistantTextUntilDone, finishReason != .contentFilter { - for event in bufferedEvents { - switch event { - case let .event(agentEvent): - await eventHandler?.send(agentEvent) - } + try self.validateToolContinuationFinishReason( + finishReason, + hasToolCalls: !stepToolCalls.isEmpty) + + for event in bufferedEvents { + switch event { + case let .event(agentEvent): + await eventHandler?.send(agentEvent) } } @@ -188,6 +200,7 @@ extension PeekabooAgentService { pendingReasoningText = "" } + // swiftlint:disable:next function_parameter_count private func handleTextDelta( _ content: String, stepText: inout String, @@ -229,7 +242,6 @@ extension PeekabooAgentService { stepToolCalls: inout [AgentToolCall], seenToolCallIds: inout Set, bufferedEvents: inout [BufferedStreamEvent], - buffersEventsUntilDone: Bool, eventHandler: EventHandler?) async throws { if self.isVerbose { @@ -244,7 +256,7 @@ extension PeekabooAgentService { stepToolCalls.append(toolCall) } - guard let eventHandler else { return } + guard eventHandler != nil else { return } let argumentsData = try JSONEncoder().encode(toolCall.arguments) let argumentsJSON = AgentToolCallArgumentPreview.redacted(from: argumentsData) @@ -255,11 +267,7 @@ extension PeekabooAgentService { .toolCallUpdated(name: toolCall.name, arguments: argumentsJSON) } - if buffersEventsUntilDone { - bufferedEvents.append(.event(event)) - } else { - await eventHandler.send(event) - } + bufferedEvents.append(.event(event)) } private func handleReasoningDelta( diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift index 31489b919..c1a4ad98c 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift @@ -3,7 +3,6 @@ // PeekabooCore // -import CryptoKit import Foundation import PeekabooAutomation import Tachikoma @@ -16,14 +15,53 @@ extension PeekabooAgentService { let steps: [GenerationStep] let usage: Usage? let toolCallCount: Int + let reachedStepLimit: Bool + } + + public struct AgentStepLimitExceededError: LocalizedError, Sendable { + public let maxSteps: Int + public let sessionId: String + + public init(maxSteps: Int, sessionId: String) { + self.maxSteps = maxSteps + self.sessionId = sessionId + } + + public var errorDescription: String? { + let resumeGuidance = "Session \(self.sessionId) was saved and can be resumed to continue." + guard self.maxSteps < AgentStepBudget.supportedRange.upperBound else { + return "Agent reached the \(self.maxSteps)-step limit after executing tools whose results still " + + "require model review. \(resumeGuidance)" + } + return "Agent reached the \(self.maxSteps)-step limit after executing tools whose results still " + + "require model review. \(resumeGuidance) You can also retry with a larger --max-steps value " + + "(maximum \(AgentStepBudget.supportedRange.upperBound))." + } } struct StreamingLoopConfiguration { let model: LanguageModel + let provider: any ModelProvider let tools: [AgentTool] let sessionId: String let eventHandler: EventHandler? let enhancementOptions: AgentEnhancementOptions? + + init( + model: LanguageModel, + provider: any ModelProvider, + tools: [AgentTool], + sessionId: String, + eventHandler: EventHandler?, + enhancementOptions: AgentEnhancementOptions?) + { + self.model = model + self.provider = provider + self.tools = tools + self.sessionId = sessionId + self.eventHandler = eventHandler + self.enhancementOptions = enhancementOptions + } } struct ToolHandlingContext { @@ -53,6 +91,11 @@ extension PeekabooAgentService { } } + private struct ToolCallExecutionOptions { + let stepIndex: Int + let allowSuccessfulToolBoundary: Bool + } + struct StreamingLoopState { var messages: [ModelMessage] var content: String = "" @@ -62,12 +105,81 @@ extension PeekabooAgentService { var desktopContextState = DesktopContextRefreshState() } + struct AgentUsageAccumulator { + private var inputTokens = 0 + private var outputTokens = 0 + private var inputCost = 0.0 + private var outputCost = 0.0 + private var allCostsKnown = true + + mutating func record(_ usage: Usage) -> Usage { + self.inputTokens += usage.inputTokens + self.outputTokens += usage.outputTokens + if let cost = usage.cost { + self.inputCost += cost.input + self.outputCost += cost.output + } else { + self.allCostsKnown = false + } + + return Usage( + inputTokens: self.inputTokens, + outputTokens: self.outputTokens, + cost: self.allCostsKnown ? Usage.Cost(input: self.inputCost, output: self.outputCost) : nil) + } + } + + func isAgentCancellation(_ error: any Error) -> Bool { + if Task.isCancelled || error is CancellationError { + return true + } + if (error as? URLError)?.code == .cancelled { + return true + } + + let nsError = error as NSError + if nsError.domain == NSURLErrorDomain, nsError.code == NSURLErrorCancelled { + return true + } + + if let tachikomaError = error as? TachikomaError { + switch tachikomaError { + case let .networkError(underlyingError): + return self.isAgentCancellation(underlyingError) + case let .retryError(retryError): + if let lastError = retryError.lastError, + self.isAgentCancellation(lastError) + { + return true + } + return retryError.errors.contains { self.isAgentCancellation($0) } + default: + break + } + } + + if let unifiedError = error as? TachikomaUnifiedError, + let underlyingError = unifiedError.underlyingError + { + return self.isAgentCancellation(underlyingError) + } + + if let modelError = error as? ModelError, + case let .networkError(underlyingError) = modelError + { + return self.isAgentCancellation(underlyingError) + } + + return false + } + func runStreamingLoop( configuration: StreamingLoopConfiguration, maxSteps: Int, initialMessages: [ModelMessage], queueMode: QueueMode = .oneAtATime, - pendingUserMessages: [ModelMessage] = []) async throws -> StreamingLoopOutcome + pendingUserMessages: [ModelMessage] = [], + onCheckpoint: ((StreamingLoopOutcome) -> Void)? = nil) async throws -> StreamingLoopOutcome { var state = StreamingLoopState(messages: initialMessages) let resolvedConfiguration = TachikomaConfiguration.resolve(.current) @@ -81,8 +193,11 @@ extension PeekabooAgentService { // Queue of pending user messages (set by caller). For now, this is empty // and will be injected by higher-level chat loop when we add that support. var queuedMessages: [ModelMessage] = pendingUserMessages + var reachedStepLimit = false + var usageAccumulator = AgentUsageAccumulator() for stepIndex in 0.. StreamingLoopOutcome + { + StreamingLoopOutcome( content: state.content, messages: state.messages, steps: state.steps, usage: state.usage, - toolCallCount: totalToolCalls) + toolCallCount: state.toolCallCount, + reachedStepLimit: reachedStepLimit) } func logStreamingStepStart(_ stepIndex: Int, tools: [AgentTool]) { @@ -198,6 +350,43 @@ extension PeekabooAgentService { self.logger.debug("Available tools: \(toolNames)") } + func validateToolContinuationFinishReason( + _ finishReason: FinishReason?, + hasToolCalls: Bool) throws + { + if finishReason == .contentFilter { + throw TachikomaError.apiError("Model refused to answer") + } + + if finishReason == .toolCalls, !hasToolCalls { + throw TachikomaError.apiError( + "Model reported a tool-call finish reason, but no tool calls were decoded; " + + "refusing to treat the response as complete.") + } + + switch finishReason { + case nil, .toolCalls, .stop: + return + case .contentFilter: + preconditionFailure("Content-filter responses are rejected before tool-finish validation") + case let finishReason?: + if hasToolCalls { + throw TachikomaError.apiError( + "Model returned tool calls with finish reason '\(finishReason.rawValue)'; " + + "refusing to execute incomplete tool calls.") + } + throw TachikomaError.apiError( + "Model returned a terminal response with finish reason '\(finishReason.rawValue)'; " + + "refusing to mark the task complete.") + } + } + + func validateTerminalResponse(text: String) throws { + guard text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return } + throw TachikomaError.apiError( + "Model returned an empty terminal response; refusing to mark the task complete.") + } + func appendFinalStep( text: String, to messages: inout [ModelMessage], @@ -216,13 +405,6 @@ extension PeekabooAgentService { toolResults: [])) } - func appendReasoningOnlyBoundary(to messages: inout [ModelMessage]) { - messages.append(ModelMessage( - role: .assistant, - content: [.text("")], - metadata: .init(customData: ["tachikoma.internal.boundary": "reasoning_only"]))) - } - func appendAnthropicReasoningBlock( text: String, signature: String?, @@ -369,6 +551,25 @@ extension PeekabooAgentService { return metadata } + if block.type == "ollama_thinking", + let target = ReasoningReplayTarget( + model: model, + configuration: configuration, + peekabooConfiguration: peekabooConfiguration), + target.provider == "ollama" + { + var metadata = [ + "ollama.thinking": block.text, + "tachikoma.reasoning.type": block.type, + "tachikoma.reasoning.provider": target.provider, + "tachikoma.reasoning.model": target.modelId, + ] + if let endpointIdentity = target.endpointIdentity { + metadata["tachikoma.reasoning.base_url"] = endpointIdentity + } + return metadata + } + var customData: [String: String] = if let target = ReasoningReplayTarget( model: model, configuration: configuration, @@ -414,39 +615,12 @@ extension PeekabooAgentService { "tachikoma.reasoning.provider": "openrouter", "tachikoma.reasoning.model": modelId, ] - if let endpointIdentity = self.canonicalReasoningEndpointIdentity(baseURL) { + if let endpointIdentity = Self.canonicalEndpointIdentity(baseURL) { metadata["tachikoma.reasoning.base_url"] = endpointIdentity } return metadata } - private func canonicalReasoningEndpointIdentity(_ rawValue: String?) -> String? { - guard - let trimmed = rawValue?.trimmingCharacters(in: .whitespacesAndNewlines), - !trimmed.isEmpty, - var components = URLComponents(string: trimmed), - let scheme = components.scheme?.lowercased(), - let host = components.host?.lowercased() - else { - return nil - } - - components.scheme = scheme - components.host = host - components.user = nil - components.password = nil - components.fragment = nil - while components.path.count > 1, components.path.hasSuffix("/") { - components.path.removeLast() - } - - guard let value = components.string, let data = value.data(using: .utf8) else { return nil } - let digest = SHA256.hash(data: data) - .map { String(format: "%02x", $0) } - .joined() - return "sha256:\(digest)" - } - func handleToolCalls( stepText: String, toolCalls: [AgentToolCall], @@ -454,7 +628,8 @@ extension PeekabooAgentService { currentMessages: inout [ModelMessage], stepIndex: Int, appendAssistantMessage: Bool = true, - emitToolStartEvents: Bool = false) async throws -> GenerationStep + emitToolStartEvents: Bool = false, + onCancellationCheckpoint: ((GenerationStep) -> Void)? = nil) async throws -> GenerationStep { if appendAssistantMessage { self.appendAssistantMessage( @@ -466,6 +641,25 @@ extension PeekabooAgentService { var toolResults: [AgentToolResult] = [] for (index, toolCall) in toolCalls.enumerated() { + do { + try Task.checkCancellation() + } catch { + await self.appendCancelledToolResults( + toolCalls: toolCalls, + startingAt: index, + activeToolCallId: nil, + context: context, + currentMessages: ¤tMessages, + toolResults: &toolResults, + emitToolStartEvents: emitToolStartEvents) + onCancellationCheckpoint?(GenerationStep( + stepIndex: stepIndex, + text: stepText, + toolCalls: toolCalls, + toolResults: toolResults)) + throw CancellationError() + } + if emitToolStartEvents { try await self.sendToolStartEvent(toolCall, eventHandler: context.eventHandler) } @@ -478,15 +672,74 @@ extension PeekabooAgentService { eventHandler: context.eventHandler) currentMessages.append(ModelMessage(role: .tool, content: [.toolResult(unavailableResult)])) toolResults.append(unavailableResult) + do { + try Task.checkCancellation() + } catch { + await self.appendCancelledToolResults( + toolCalls: toolCalls, + startingAt: index + 1, + activeToolCallId: nil, + context: context, + currentMessages: ¤tMessages, + toolResults: &toolResults, + emitToolStartEvents: emitToolStartEvents) + onCancellationCheckpoint?(GenerationStep( + stepIndex: stepIndex, + text: stepText, + toolCalls: toolCalls, + toolResults: toolResults)) + throw CancellationError() + } continue } - let result = try await self.executeToolCall( - toolCall, - tool: tool, - context: context, - currentMessages: ¤tMessages, - stepIndex: stepIndex) + let result: AgentToolResult + do { + result = try await self.executeToolCall( + toolCall, + tool: tool, + context: context, + currentMessages: ¤tMessages, + options: ToolCallExecutionOptions( + stepIndex: stepIndex, + allowSuccessfulToolBoundary: !toolResults.contains { $0.isError })) + } catch { + guard self.isAgentCancellation(error) else { throw error } + await self.appendCancelledToolResults( + toolCalls: toolCalls, + startingAt: index, + activeToolCallId: toolCall.id, + context: context, + currentMessages: ¤tMessages, + toolResults: &toolResults, + emitToolStartEvents: emitToolStartEvents) + onCancellationCheckpoint?(GenerationStep( + stepIndex: stepIndex, + text: stepText, + toolCalls: toolCalls, + toolResults: toolResults)) + throw CancellationError() + } toolResults.append(result) + + do { + try Task.checkCancellation() + } catch { + await self.appendCancelledToolResults( + toolCalls: toolCalls, + startingAt: index + 1, + activeToolCallId: nil, + context: context, + currentMessages: ¤tMessages, + toolResults: &toolResults, + emitToolStartEvents: emitToolStartEvents) + onCancellationCheckpoint?(GenerationStep( + stepIndex: stepIndex, + text: stepText, + toolCalls: toolCalls, + toolResults: toolResults)) + throw CancellationError() + } + if let stopReason = self.turnBoundaryStopReason(from: result) { let remainingToolCalls = toolCalls.dropFirst(index + 1) for skippedToolCall in remainingToolCalls { @@ -509,6 +762,44 @@ extension PeekabooAgentService { toolResults: toolResults) } + // swiftlint:disable:next function_parameter_count + private func appendCancelledToolResults( + toolCalls: [AgentToolCall], + startingAt startIndex: Int, + activeToolCallId: String?, + context: ToolHandlingContext, + currentMessages: inout [ModelMessage], + toolResults: inout [AgentToolResult], + emitToolStartEvents: Bool) async + { + guard startIndex < toolCalls.count else { return } + + for toolCall in toolCalls[startIndex...] { + let wasActive = toolCall.id == activeToolCallId + if emitToolStartEvents, !wasActive { + try? await self.sendToolStartEvent(toolCall, eventHandler: context.eventHandler) + } + + var payload = [ + "cancelled": AnyAgentToolValue(bool: true), + "reason": AnyAgentToolValue(string: "Agent execution was cancelled"), + ] + if !wasActive { + payload["skipped"] = AnyAgentToolValue(bool: true) + } + let result = AgentToolResult( + toolCallId: toolCall.id, + result: AnyAgentToolValue(object: payload), + isError: true) + await self.sendToolCompletionEvent( + name: toolCall.name, + payload: self.toolResultPayload(from: result.result, toolName: toolCall.name), + eventHandler: context.eventHandler) + currentMessages.append(ModelMessage(role: .tool, content: [.toolResult(result)])) + toolResults.append(result) + } + } + private func appendAssistantMessage( stepText: String, toolCalls: [AgentToolCall], @@ -554,7 +845,7 @@ extension PeekabooAgentService { tool: AgentTool, context: ToolHandlingContext, currentMessages: inout [ModelMessage], - stepIndex: Int) async throws -> AgentToolResult + options: ToolCallExecutionOptions) async throws -> AgentToolResult { let boundaryDecision = context.turnBoundary.record(toolName: toolCall.name, arguments: toolCall.arguments) @@ -564,7 +855,7 @@ extension PeekabooAgentService { model: context.model, settings: self.generationSettings(for: context.model), sessionId: context.sessionId, - stepIndex: stepIndex) + stepIndex: options.stepIndex) let toolArguments = AgentToolArguments(toolCall.arguments) let execution = try await self.executeTool( tool, @@ -577,8 +868,15 @@ extension PeekabooAgentService { toolValue = self.addVerification(verification, to: toolValue) await context.eventHandler?.send(.verificationCompleted(toolName: toolCall.name, result: verification)) } - if case let .stopAfterCurrentStep(reason) = boundaryDecision { + switch boundaryDecision { + case let .stopAfterCurrentStep(reason): + toolValue = self.addTurnBoundaryStopReason(reason, to: toolValue) + case let .stopAfterSuccessfulTool(reason) where options.allowSuccessfulToolBoundary: toolValue = self.addTurnBoundaryStopReason(reason, to: toolValue) + case .stopAfterSuccessfulTool: + break + case .continueTurn: + break } let toolResult = AgentToolResult.success(toolCallId: toolCall.id, result: toolValue) await self.sendToolCompletionEvent( @@ -587,9 +885,10 @@ extension PeekabooAgentService { eventHandler: context.eventHandler) currentMessages.append(ModelMessage(role: .tool, content: [.toolResult(toolResult)])) return toolResult - } catch let error as CancellationError { - throw error } catch { + if self.isAgentCancellation(error) { + throw CancellationError() + } var errorValue = AnyAgentToolValue(string: error.localizedDescription) if case let .stopAfterCurrentStep(reason) = boundaryDecision { errorValue = self.addTurnBoundaryStopReason(reason, to: errorValue) @@ -920,6 +1219,12 @@ private struct ReasoningReplayTarget { self.baseURL = configuration.getBaseURL(for: .kimi) ?? Provider.kimi.defaultBaseURL self.allowsReasoningBoundaries = true self.allowsLegacyUnknown = false + case let .ollama(model): + self.provider = "ollama" + self.modelId = model.modelId + self.baseURL = (try? configuration.makeProvider(for: .ollama(model)))?.baseURL + self.allowsReasoningBoundaries = true + self.allowsLegacyUnknown = false case let .custom(provider): if let anthropicProvider = provider as? AnthropicProvider { self.provider = "anthropic" @@ -984,29 +1289,6 @@ private struct ReasoningReplayTarget { } var endpointIdentity: String? { - guard - let trimmed = self.baseURL?.trimmingCharacters(in: .whitespacesAndNewlines), - !trimmed.isEmpty, - var components = URLComponents(string: trimmed), - let scheme = components.scheme?.lowercased(), - let host = components.host?.lowercased() - else { - return nil - } - - components.scheme = scheme - components.host = host - components.user = nil - components.password = nil - components.fragment = nil - while components.path.count > 1, components.path.hasSuffix("/") { - components.path.removeLast() - } - - guard let value = components.string, let data = value.data(using: .utf8) else { return nil } - let digest = SHA256.hash(data: data) - .map { String(format: "%02x", $0) } - .joined() - return "sha256:\(digest)" + PeekabooAgentService.canonicalEndpointIdentity(self.baseURL) } } diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService.swift index 332990685..64fae851a 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService.swift @@ -58,7 +58,31 @@ public final class PeekabooAgentService: AgentServiceProtocol { /// Credential-free provider-qualified reference for callers that need to resolve the default model. public var defaultModelSelection: String { - switch self.defaultLanguageModel { + self.modelSelectionReference(for: self.defaultLanguageModel) + } + + /// Credential-free provider-qualified reference suitable for session persistence. + func persistedModelSelection(for model: LanguageModel) -> String? { + let selection: String + switch model { + case .azureOpenAI, .openaiCompatible, .anthropicCompatible, .together, .replicate: + return nil + case let .custom(provider): + selection = provider.modelId + default: + selection = self.modelSelectionReference(for: model) + } + + guard let resolved = self.resolveConfiguredModel(selection), + resolved == model + else { + return nil + } + return selection + } + + private func modelSelectionReference(for model: LanguageModel) -> String { + switch model { case let .openai(model): "openai/\(model.modelId)" case let .anthropic(model): "anthropic/\(model.modelId)" case let .google(model): "google/\(model.userFacingModelId)" @@ -75,7 +99,7 @@ public final class PeekabooAgentService: AgentServiceProtocol { case let .replicate(modelID): "replicate/\(modelID)" case let .custom(provider): provider.modelId case .azureOpenAI, .openaiCompatible, .anthropicCompatible: - self.defaultLanguageModel.description + model.description } } @@ -199,6 +223,7 @@ public final class PeekabooAgentService: AgentServiceProtocol { queueMode: QueueMode = .oneAtATime, eventDelegate: (any AgentEventDelegate)? = nil) async throws -> AgentExecutionResult { + let maxSteps = try AgentStepBudget.validate(maxSteps) if dryRun { let transcript = audioContent.transcript let durationSeconds = Int(audioContent.duration ?? 0) @@ -253,6 +278,7 @@ public final class PeekabooAgentService: AgentServiceProtocol { verbose: Bool = false, enhancementOptions: AgentEnhancementOptions? = .default) async throws -> AgentExecutionResult { + let maxSteps = try AgentStepBudget.validate(maxSteps) // Store the verbose flag for this execution self.isVerbose = verbose if verbose { @@ -339,9 +365,14 @@ public final class PeekabooAgentService: AgentServiceProtocol { eventContinuation.finish() await eventTask.value return result + } catch let error as CancellationError { + eventContinuation.finish() + await eventTask.value + throw error } catch { + await eventHandler.send(.error(message: error.localizedDescription)) eventContinuation.finish() - eventTask.cancel() + await eventTask.value throw error } } else { diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/QueueMode.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/QueueMode.swift index 6f432ae82..f3e1d322f 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/QueueMode.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/QueueMode.swift @@ -12,6 +12,7 @@ final class AgentTurnBoundary { enum Decision: Equatable { case continueTurn case stopAfterCurrentStep(reason: String) + case stopAfterSuccessfulTool(reason: String) } private static let perceiveTools: Set = [ @@ -57,6 +58,23 @@ final class AgentTurnBoundary { { let normalizedName = Self.normalized(toolName) + if normalizedName == "done" { + let message = arguments["message"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines) + let reason = if let message, !message.isEmpty { + message + } else { + "Task completed successfully." + } + return .stopAfterSuccessfulTool(reason: reason) + } + + if normalizedName == "need_info", + let question = arguments["question"]?.stringValue?.trimmingCharacters(in: .whitespacesAndNewlines), + !question.isEmpty + { + return .stopAfterSuccessfulTool(reason: "Need more information: \(question)") + } + if Self.perceiveTools.contains(normalizedName) { self.hasPerceived = true return .continueTurn diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/MCP/Tools/MCPAgentTool.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/MCP/Tools/MCPAgentTool.swift index 6479450f7..f3ef95f35 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/MCP/Tools/MCPAgentTool.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/MCP/Tools/MCPAgentTool.swift @@ -65,7 +65,9 @@ public struct MCPAgentTool: MCPTool { description: "Dry run - show planned steps without executing", default: false), "max_steps": SchemaBuilder.integer( - description: "Maximum number of steps the agent can take (1-100)"), + description: "Maximum model/tool-loop turns before failing " + + "(\(AgentStepBudget.supportedRange.lowerBound)-" + + "\(AgentStepBudget.supportedRange.upperBound), default 20)"), "resume": SchemaBuilder.boolean( description: "Resume the most recent session", default: false), @@ -205,8 +207,10 @@ public struct MCPAgentTool: MCPTool { static func validatedMaxSteps(_ maxSteps: Int?) throws -> Int { let resolved = maxSteps ?? 20 - guard (1...100).contains(resolved) else { - throw AgentToolError("max_steps must be between 1 and 100") + guard AgentStepBudget.supportedRange.contains(resolved) else { + throw AgentToolError( + "max_steps must be between \(AgentStepBudget.supportedRange.lowerBound) and " + + "\(AgentStepBudget.supportedRange.upperBound)") } return resolved } diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Protocols/AgentServiceProtocol.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Protocols/AgentServiceProtocol.swift index 49b1cf9af..b8df0a096 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Protocols/AgentServiceProtocol.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Protocols/AgentServiceProtocol.swift @@ -1,7 +1,21 @@ import Foundation import PeekabooAutomation +import PeekabooFoundation import Tachikoma +public enum AgentStepBudget { + public static let supportedRange = 1...100 + + public static func validate(_ maxSteps: Int) throws -> Int { + guard self.supportedRange.contains(maxSteps) else { + throw PeekabooError.invalidInput( + "Maximum agent steps must be between \(self.supportedRange.lowerBound) and " + + "\(self.supportedRange.upperBound); received \(maxSteps).") + } + return maxSteps + } +} + /// Protocol defining the agent service interface @available(macOS 14.0, *) @MainActor diff --git a/Core/PeekabooCore/Sources/PeekabooAutomation/Services/AI/PeekabooAIService.swift b/Core/PeekabooCore/Sources/PeekabooAutomation/Services/AI/PeekabooAIService.swift index ac9ed3a08..c2813e5cd 100644 --- a/Core/PeekabooCore/Sources/PeekabooAutomation/Services/AI/PeekabooAIService.swift +++ b/Core/PeekabooCore/Sources/PeekabooAutomation/Services/AI/PeekabooAIService.swift @@ -44,7 +44,13 @@ public enum AnthropicModelCapabilityInference { } } -private final class PeekabooCustomProviderModel: ModelProvider, @unchecked Sendable { +public protocol PeekabooCustomProviderIdentityProviding: ModelProvider { + var providerID: String { get } + var resolvedModelID: String { get } + var providerTypeIdentity: String { get } +} + +private final class PeekabooCustomProviderModel: PeekabooCustomProviderIdentityProviding, @unchecked Sendable { enum Kind { case openai case anthropic @@ -59,6 +65,13 @@ private final class PeekabooCustomProviderModel: ModelProvider, @unchecked Senda let additionalHeaders: [String: String] let capabilities: ModelCapabilities + var providerTypeIdentity: String { + switch self.kind { + case .openai: "openai" + case .anthropic: "anthropic" + } + } + init( providerID: String, resolvedModelID: String, @@ -406,15 +419,9 @@ public final class PeekabooAIService { case "openrouter": return .openRouter(modelId: modelString) case "mistral": - if case .mistral = loose { - return loose - } - return nil + return LanguageModel.Mistral(rawValue: modelString).map(LanguageModel.mistral) case "groq": - if case .groq = loose { - return loose - } - return nil + return LanguageModel.Groq(rawValue: modelString).map(LanguageModel.groq) case "grok", "xai": guard !self.isUnsupportedGrokModel(modelString) else { return nil } if case .grok = loose { @@ -439,13 +446,21 @@ public final class PeekabooAIService { private static func parseLocalProviderEntry(provider: String, modelString: String) -> LanguageModel? { switch provider { case "ollama": - // For Ollama, prefer preserving the exact model id string. - // Heuristics for custom model capabilities live in Tachikoma (LanguageModel.Ollama). - .ollama(.custom(modelString)) + if case let .ollama(model) = LanguageModel.parse(from: "ollama/\(modelString)"), + model.modelId == modelString + { + return .ollama(model) + } + return .ollama(.custom(modelString)) case "lmstudio", "lm-studio": - .lmstudio(.custom(modelString)) + if case let .lmstudio(model) = LanguageModel.parse(from: "lmstudio/\(modelString)"), + model.modelId == modelString + { + return .lmstudio(model) + } + return .lmstudio(.custom(modelString)) default: - nil + return nil } } diff --git a/Tachikoma b/Tachikoma index 0ece63ecb..06385196f 160000 --- a/Tachikoma +++ b/Tachikoma @@ -1 +1 @@ -Subproject commit 0ece63ecb74902be96867faa8237bbe7df785273 +Subproject commit 06385196fc5c5f3f9834d3324c32ec40ed69e486 From d6cbdc5650d9f6d32f709d4d259c0db41e548441 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:07:55 +0100 Subject: [PATCH 02/11] test(agent): cover step and event boundaries --- .../AgentTurnBoundaryTranscriptTests.swift | 169 ++ .../PeekabooAgentEventLifecycleTests.swift | 162 ++ .../PeekabooAgentStepLimitTests.swift | 1613 +++++++++++++++++ 3 files changed, 1944 insertions(+) create mode 100644 Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentEventLifecycleTests.swift create mode 100644 Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStepLimitTests.swift diff --git a/Core/PeekabooCore/Tests/PeekabooTests/AgentTurnBoundaryTranscriptTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/AgentTurnBoundaryTranscriptTests.swift index edff68c30..36a7f1b61 100644 --- a/Core/PeekabooCore/Tests/PeekabooTests/AgentTurnBoundaryTranscriptTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooTests/AgentTurnBoundaryTranscriptTests.swift @@ -116,4 +116,173 @@ struct AgentTurnBoundaryTranscriptTests { #expect(cancelled) } + + @Test + func `parent cancellation checkpoints completed tools and skips remaining dispatch`() async throws { + let service = try PeekabooAgentService(services: PeekabooServices()) + let probe = ToolCancellationProbe() + let transcriptStore = CancellationTranscriptStore() + let toolCalls = [ + AgentToolCall(id: "first-call", name: "first", arguments: [:]), + AgentToolCall(id: "second-call", name: "second", arguments: [:]), + AgentToolCall(id: "third-call", name: "third", arguments: [:]), + ] + let tools = [ + AgentTool( + name: "first", + description: "first", + parameters: AgentToolParameters(properties: [:], required: []), + execute: { _ in AnyAgentToolValue(string: "first-ok") }), + AgentTool( + name: "second", + description: "second", + parameters: AgentToolParameters(properties: [:], required: []), + execute: { _ in + await probe.markSecondStarted() + do { + try await Task.sleep(for: .seconds(30)) + } catch is CancellationError { + // Simulate a side effect that completed while ignoring cooperative cancellation. + } + return AnyAgentToolValue(string: "second-ok") + }), + AgentTool( + name: "third", + description: "third", + parameters: AgentToolParameters(properties: [:], required: []), + execute: { _ in + await probe.markThirdExecuted() + return AnyAgentToolValue(string: "third-ok") + }), + ] + let context = PeekabooAgentService.ToolHandlingContext( + model: .anthropic(.sonnet45), + tools: tools, + eventHandler: nil, + sessionId: "test-session") + + let task = Task { @MainActor in + var messages: [ModelMessage] = [] + var checkpoint: GenerationStep? + do { + _ = try await service.handleToolCalls( + stepText: "", + toolCalls: toolCalls, + context: context, + currentMessages: &messages, + stepIndex: 0, + onCancellationCheckpoint: { checkpoint = $0 }) + } catch { + await transcriptStore.capture(messages: messages, checkpoint: checkpoint) + throw error + } + } + + await probe.waitForSecondStart() + task.cancel() + await #expect(throws: CancellationError.self) { + try await task.value + } + + let snapshot = await transcriptStore.snapshot() + let checkpoint = try #require(snapshot.checkpoint) + #expect(checkpoint.toolResults.map(\.toolCallId) == ["first-call", "second-call", "third-call"]) + #expect(checkpoint.toolResults.map(\.isError) == [false, false, true]) + #expect(snapshot.messages.count(where: { $0.role == .tool }) == toolCalls.count) + #expect(await probe.thirdExecutionCount == 0) + + let skippedPayload = try #require(try checkpoint.toolResults[2].result.toJSON() as? [String: Any]) + #expect(skippedPayload["cancelled"] as? Bool == true) + #expect(skippedPayload["skipped"] as? Bool == true) + } + + @Test + func `URL cancellation is rethrown and remaining tool calls are not dispatched`() async throws { + let service = try PeekabooAgentService(services: PeekabooServices()) + let probe = ToolCancellationProbe() + var messages: [ModelMessage] = [] + var checkpoint: GenerationStep? + let toolCalls = [ + AgentToolCall(id: "cancelled-call", name: "cancelled", arguments: [:]), + AgentToolCall(id: "never-call", name: "never", arguments: [:]), + ] + let tools = [ + AgentTool( + name: "cancelled", + description: "cancelled", + parameters: AgentToolParameters(properties: [:], required: []), + execute: { _ in throw URLError(.cancelled) }), + AgentTool( + name: "never", + description: "never", + parameters: AgentToolParameters(properties: [:], required: []), + execute: { _ in + await probe.markThirdExecuted() + return AnyAgentToolValue(string: "unexpected") + }), + ] + let context = PeekabooAgentService.ToolHandlingContext( + model: .anthropic(.sonnet45), + tools: tools, + eventHandler: nil, + sessionId: "test-session") + + await #expect(throws: CancellationError.self) { + _ = try await service.handleToolCalls( + stepText: "", + toolCalls: toolCalls, + context: context, + currentMessages: &messages, + stepIndex: 0, + onCancellationCheckpoint: { checkpoint = $0 }) + } + + let captured = try #require(checkpoint) + #expect(captured.toolResults.map(\.toolCallId) == ["cancelled-call", "never-call"]) + #expect(captured.toolResults.map(\.isError) == [true, true]) + #expect(messages.count(where: { $0.role == .tool }) == toolCalls.count) + #expect(await probe.thirdExecutionCount == 0) + } +} + +private actor ToolCancellationProbe { + private var secondStarted = false + private var secondStartWaiters: [CheckedContinuation] = [] + private(set) var thirdExecutionCount = 0 + + func markSecondStarted() { + self.secondStarted = true + let waiters = self.secondStartWaiters + self.secondStartWaiters.removeAll() + for waiter in waiters { + waiter.resume() + } + } + + func waitForSecondStart() async { + if self.secondStarted { + return + } + await withCheckedContinuation { continuation in + self.secondStartWaiters.append(continuation) + } + } + + func markThirdExecuted() { + self.thirdExecutionCount += 1 + } +} + +private actor CancellationTranscriptStore { + private var messages: [ModelMessage] = [] + private var checkpoint: GenerationStep? + + func capture(messages: [ModelMessage], checkpoint: GenerationStep?) { + self.messages = messages + self.checkpoint = checkpoint + } + + func snapshot() -> (messages: [ModelMessage], checkpoint: GenerationStep?) { + (self.messages, self.checkpoint) + } } diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentEventLifecycleTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentEventLifecycleTests.swift new file mode 100644 index 000000000..dd1173389 --- /dev/null +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentEventLifecycleTests.swift @@ -0,0 +1,162 @@ +import Foundation +import Tachikoma +import Testing +@testable import PeekabooAgentRuntime +@testable import PeekabooCore + +@Suite(.serialized) +struct PeekabooAgentEventLifecycleTests { + @Test + @MainActor + func `Failure drains tool completion before terminal error`() async throws { + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in EventLifecycleToolProvider() } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = EventLifecycleDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: PeekabooAgentService.AgentStepLimitExceededError.self) { + _ = try await agentService.executeTask( + "Use the unavailable tool.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + let completionIndex = try #require(delegate.events.firstIndex { event in + if case .toolCallCompleted = event { + true + } else { + false + } + }) + let errorIndex = try #require(delegate.events.firstIndex { event in + if case .error = event { + true + } else { + false + } + }) + #expect(completionIndex < errorIndex) + #expect(delegate.events.contains { event in + if case let .error(message) = event { + message.contains("1-step limit") + } else { + false + } + }) + #expect(!delegate.events.contains { event in + if case .completed = event { + true + } else { + false + } + }) + + try await agentService.deleteSession(id: error.sessionId) + } + + @Test + @MainActor + func `Cancellation drains events without emitting a failure`() async throws { + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in EventLifecycleCancellationProvider() } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = EventLifecycleDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + await #expect(throws: CancellationError.self) { + _ = try await agentService.executeTask( + "Cancel this run.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + } + + #expect(delegate.events.contains { event in + if case .started = event { + true + } else { + false + } + }) + #expect(!delegate.events.contains { event in + if case .error = event { + true + } else { + false + } + }) + #expect(!delegate.events.contains { event in + if case .completed = event { + true + } else { + false + } + }) + } +} + +private final class EventLifecycleToolProvider: ModelProvider, @unchecked Sendable { + let modelId = "event-lifecycle-tool-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + ProviderResponse(text: "", finishReason: .toolCalls, toolCalls: [self.toolCall]) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.yield(.tool(self.toolCall)) + continuation.yield(.done(finishReason: .toolCalls)) + continuation.finish() + } + } + + private var toolCall: AgentToolCall { + AgentToolCall(id: "missing-tool-call", name: "missing_event_lifecycle_tool", arguments: [:]) + } +} + +private final class EventLifecycleCancellationProvider: ModelProvider, @unchecked Sendable { + let modelId = "event-lifecycle-cancellation-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + throw CancellationError() + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.finish(throwing: CancellationError()) + } + } +} + +@MainActor +private final class EventLifecycleDelegate: AgentEventDelegate { + private(set) var events: [AgentEvent] = [] + + func agentDidEmitEvent(_ event: AgentEvent) { + self.events.append(event) + } +} diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStepLimitTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStepLimitTests.swift new file mode 100644 index 000000000..364ce3e42 --- /dev/null +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStepLimitTests.swift @@ -0,0 +1,1613 @@ +import Foundation +import Tachikoma +import Testing +@testable import PeekabooAgentRuntime +@testable import PeekabooCore + +@Suite(.serialized) +struct PeekabooAgentStepLimitTests { + @Test + @MainActor + func `Nonstreaming step exhaustion saves resumable tool history and throws`() async throws { + let provider = PerpetualToolProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: PeekabooAgentService.AgentStepLimitExceededError.self) { + _ = try await agentService.executeTask( + "Keep using tools.", + maxSteps: 1, + model: .openai(.gpt55), + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(provider.requestCount == 1) + #expect(error.maxSteps == 1) + #expect(error.localizedDescription.contains("can be resumed")) + + let loadedSession = try await agentService.getSessionInfo(sessionId: error.sessionId) + let session = try #require(loadedSession) + #expect(session.metadata.customData["status"] == "max_steps_exhausted") + #expect(session.messages.containsToolCall(id: "tool-call-1")) + #expect(session.messages.containsToolResult(id: "tool-call-1")) + + try await agentService.deleteSession(id: error.sessionId) + } + + @Test + @MainActor + func `Streaming step exhaustion saves resumable tool history without completion event`() async throws { + let provider = PerpetualToolProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: PeekabooAgentService.AgentStepLimitExceededError.self) { + _ = try await agentService.executeTask( + "Keep using tools.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(provider.requestCount == 1) + #expect(!delegate.events.contains { event in + if case .completed = event { + true + } else { + false + } + }) + + let loadedSession = try await agentService.getSessionInfo(sessionId: error.sessionId) + let session = try #require(loadedSession) + #expect(session.metadata.customData["status"] == "max_steps_exhausted") + #expect(session.messages.containsToolCall(id: "tool-call-1")) + #expect(session.messages.containsToolResult(id: "tool-call-1")) + + try await agentService.deleteSession(id: error.sessionId) + } + + @Test(arguments: [TerminalTool.done, .needInfo]) + @MainActor + func `Terminal tool succeeds on final nonstreaming step`(_ terminalTool: TerminalTool) async throws { + let provider = TerminalToolProvider(terminalTool: terminalTool) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let result = try await agentService.executeTask( + "Finish this turn.", + maxSteps: 1, + model: .openai(.gpt55), + enhancementOptions: nil) + + #expect(result.content == terminalTool.expectedReason) + let sessionId = try #require(result.sessionId) + let loadedSession = try await agentService.getSessionInfo(sessionId: sessionId) + #expect(loadedSession?.metadata.customData["status"] == "completed") + try await agentService.deleteSession(id: sessionId) + } + + @Test(arguments: [TerminalTool.done, .needInfo]) + @MainActor + func `Terminal tool succeeds on final streaming step`(_ terminalTool: TerminalTool) async throws { + let provider = TerminalToolProvider(terminalTool: terminalTool) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let result = try await agentService.executeTask( + "Finish this turn.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + + #expect(result.content == terminalTool.expectedReason) + #expect(delegate.events.contains { event in + if case let .completed(summary, _) = event { + summary == terminalTool.expectedReason + } else { + false + } + }) + let sessionId = try #require(result.sessionId) + let loadedSession = try await agentService.getSessionInfo(sessionId: sessionId) + #expect(loadedSession?.metadata.customData["status"] == "completed") + try await agentService.deleteSession(id: sessionId) + } + + @Test(arguments: [FinishReason.length, .error, .cancelled, .other]) + @MainActor + func `Incomplete tool responses are rejected before execution`(_ finishReason: FinishReason) throws { + let agentService = try PeekabooAgentService(services: PeekabooServices()) + + let thrownError = #expect(throws: TachikomaError.self) { + try agentService.validateToolContinuationFinishReason(finishReason, hasToolCalls: true) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains(finishReason.rawValue)) + #expect(error.localizedDescription.contains("refusing to execute incomplete tool calls")) + } + + @Test + @MainActor + func `Ollama compatible tool finish reasons remain accepted`() throws { + let agentService = try PeekabooAgentService(services: PeekabooServices()) + + #expect(throws: Never.self) { + try agentService.validateToolContinuationFinishReason(nil, hasToolCalls: true) + } + #expect(throws: Never.self) { + try agentService.validateToolContinuationFinishReason(.toolCalls, hasToolCalls: true) + } + #expect(throws: Never.self) { + try agentService.validateToolContinuationFinishReason(.stop, hasToolCalls: true) + } + } + + @Test + @MainActor + func `Tool-call finish without decoded calls is rejected`() throws { + let agentService = try PeekabooAgentService(services: PeekabooServices()) + + let thrownError = #expect(throws: TachikomaError.self) { + try agentService.validateToolContinuationFinishReason(.toolCalls, hasToolCalls: false) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains("no tool calls were decoded")) + #expect(error.localizedDescription.contains("refusing to treat the response as complete")) + } + + @Test + @MainActor + func `Nonstreaming tool-call finish without decoded calls fails before completion`() async throws { + let provider = MalformedToolResponseProvider(finishReason: .toolCalls, toolCalls: []) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .anthropic(.fable5)) + + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.executeTask( + "Return a malformed response.", + maxSteps: 1, + model: .anthropic(.fable5), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains("no tool calls were decoded")) + #expect(provider.generateRequestCount == 1) + #expect(provider.streamRequestCount == 0) + #expect(!delegate.events.containsCompletedEvent) + } + + @Test + @MainActor + func `Streaming tool-call finish without decoded calls fails before completion`() async throws { + let provider = MalformedToolResponseProvider(finishReason: .toolCalls, toolCalls: []) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.executeTask( + "Return a malformed response.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains("no tool calls were decoded")) + #expect(provider.generateRequestCount == 0) + #expect(provider.streamRequestCount == 1) + #expect(!delegate.events.containsCompletedEvent) + } + + @Test(arguments: [false, true]) + @MainActor + func `Empty terminal response after tool use fails instead of completing`(_ streaming: Bool) async throws { + let provider = EmptyTerminalAfterToolProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.executeTask( + "Use a tool, then return an empty response.", + maxSteps: 2, + model: .openai(.gpt55), + eventDelegate: streaming ? delegate : nil, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains("empty terminal response")) + #expect(provider.generateRequestCount == (streaming ? 0 : 2)) + #expect(provider.streamRequestCount == (streaming ? 2 : 0)) + #expect(!delegate.events.containsCompletedEvent) + } + + @Test(arguments: [FinishReason.length, .error, .cancelled, .other]) + @MainActor + func `Nonstreaming incomplete terminal response fails instead of completing`( + _ finishReason: FinishReason) async throws + { + let provider = MalformedToolResponseProvider( + finishReason: finishReason, + toolCalls: [], + text: "Partial response") + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.executeTask( + "Return a truncated terminal response.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: nil, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains(finishReason.rawValue)) + #expect(error.localizedDescription.contains("refusing to mark the task complete")) + #expect(provider.generateRequestCount == 1) + #expect(provider.streamRequestCount == 0) + } + + @Test(arguments: [FinishReason.length, .error, .cancelled, .other]) + @MainActor + func `Streaming incomplete terminal response fails instead of completing`( + _ finishReason: FinishReason) async throws + { + let provider = MalformedToolResponseProvider( + finishReason: finishReason, + toolCalls: [], + text: "Partial response") + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.executeTask( + "Return a truncated terminal response.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains(finishReason.rawValue)) + #expect(error.localizedDescription.contains("refusing to mark the task complete")) + #expect(provider.generateRequestCount == 0) + #expect(provider.streamRequestCount == 1) + #expect(!delegate.events.containsCompletedEvent) + } + + @Test(arguments: [false, true]) + @MainActor + func `Provider failure preserves last complete tool batch`(_ streaming: Bool) async throws { + try await self.assertCheckpointAfterProviderFailure( + streaming: streaming, + failure: .error, + expectedStatus: "failed") + } + + @Test(arguments: [false, true]) + @MainActor + func `Provider cancellation preserves last complete tool batch`(_ streaming: Bool) async throws { + try await self.assertCheckpointAfterProviderFailure( + streaming: streaming, + failure: .cancellation, + expectedStatus: "cancelled") + } + + @Test(arguments: [false, true]) + @MainActor + func `Parent cancellation persists a balanced in-flight tool batch`(_ streaming: Bool) async throws { + let provider = MultiSleepProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let model = LanguageModel.openai(.gpt55) + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: model) + let context = try await agentService.prepareSession( + task: "Run three sleeps.", + model: model, + label: "parent-cancellation-test", + logBehavior: .verboseOnly) + let firstCompletion = FirstToolCompletionGate() + let eventHandler = EventHandler { event in + if case .toolCallCompleted = event { + await firstCompletion.signal() + } + } + + let task = Task { @MainActor in + if streaming { + try await agentService.executeWithStreaming( + context: context, + model: model, + maxSteps: 1, + streamingDelegate: StreamingEventDelegate { _ in }, + eventHandler: eventHandler, + enhancementOptions: nil) + } else { + try await agentService.executeWithoutStreaming( + context: context, + model: model, + maxSteps: 1, + eventHandler: eventHandler, + enhancementOptions: nil) + } + } + + await firstCompletion.wait() + task.cancel() + await #expect(throws: CancellationError.self) { + _ = try await task.value + } + + let session = try #require(try await agentService.getSessionInfo(sessionId: context.id)) + #expect(session.metadata.customData["status"] == "cancelled") + #expect(session.metadata.toolCallCount == 3) + let toolResults = session.messages.flatMap { message in + message.content.compactMap { part -> AgentToolResult? in + if case let .toolResult(result) = part { + result + } else { + nil + } + } + } + #expect(toolResults.map(\.toolCallId) == ["first-sleep", "second-sleep", "third-sleep"]) + #expect(toolResults.map(\.isError) == [false, true, true]) + #expect(session.messages.containsToolCall(id: "first-sleep")) + #expect(session.messages.containsToolCall(id: "second-sleep")) + #expect(session.messages.containsToolCall(id: "third-sleep")) + try await agentService.deleteSession(id: context.id) + } + + @Test(arguments: [false, true]) + @MainActor + func `Noncooperative provider cannot complete after parent cancellation`(_ streaming: Bool) async throws { + let gate = NoncooperativeProviderGate() + let provider = NoncooperativeCancellationProvider(gate: gate) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let model = LanguageModel.anthropic(.sonnet45) + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: model) + let context = try await agentService.prepareSession( + task: "Wait for a provider response.", + model: model, + label: "provider-parent-cancellation-test", + logBehavior: .verboseOnly) + let eventHandler = EventHandler { event in + if streaming, case .assistantMessage = event { + await gate.markStarted() + } + } + + let task = Task { @MainActor in + if streaming { + try await agentService.executeWithStreaming( + context: context, + model: model, + maxSteps: 1, + streamingDelegate: StreamingEventDelegate { _ in }, + eventHandler: eventHandler, + enhancementOptions: nil) + } else { + try await agentService.executeWithoutStreaming( + context: context, + model: model, + maxSteps: 1, + eventHandler: eventHandler, + enhancementOptions: nil) + } + } + + await gate.waitForStart() + task.cancel() + await gate.release() + await #expect(throws: CancellationError.self) { + _ = try await task.value + } + + let session = try #require(try await agentService.getSessionInfo(sessionId: context.id)) + #expect(session.metadata.customData["status"] == "cancelled") + #expect(session.metadata.totalTokens == (streaming ? 0 : 5)) + #expect(session.metadata.totalCost == (streaming ? nil : 0)) + #expect(!session.messages.containsText(NoncooperativeCancellationProvider.unsafeText)) + try await agentService.deleteSession(id: context.id) + } + + @Test + @MainActor + func `Streaming usage accumulates across tool turns`() async throws { + let provider = TwoTurnUsageProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let result = try await agentService.executeTask( + "Use a tool, then finish.", + maxSteps: 2, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + + #expect(result.usage?.inputTokens == 8) + #expect(result.usage?.outputTokens == 10) + #expect(abs((result.usage?.cost?.input ?? 0) - 0.4) < 0.000_001) + #expect(abs((result.usage?.cost?.output ?? 0) - 0.6) < 0.000_001) + + let sessionId = try #require(result.sessionId) + let session = try #require(try await agentService.getSessionInfo(sessionId: sessionId)) + #expect(session.metadata.totalTokens == 18) + #expect(abs((session.metadata.totalCost ?? 0) - 1.0) < 0.000_001) + try await agentService.deleteSession(id: sessionId) + } + + @Test(arguments: [false, true]) + @MainActor + func `Mixed known and unknown turn costs omit the aggregate cost`(_ streaming: Bool) async throws { + let provider = TwoTurnCostProvider( + firstCost: nil, + secondCost: .init(input: 0.3, output: 0.4)) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let result = try await agentService.executeTask( + "Use a tool, then finish.", + maxSteps: 2, + model: .openai(.gpt55), + eventDelegate: streaming ? delegate : nil, + enhancementOptions: nil) + + #expect(result.usage?.inputTokens == 8) + #expect(result.usage?.outputTokens == 10) + #expect(result.usage?.cost == nil) + let sessionId = try #require(result.sessionId) + let session = try #require(try await agentService.getSessionInfo(sessionId: sessionId)) + #expect(session.metadata.totalTokens == 18) + #expect(session.metadata.totalCost == nil) + try await agentService.deleteSession(id: sessionId) + } + + @Test(arguments: [false, true]) + @MainActor + func `Explicit zero turn costs remain known`(_ streaming: Bool) async throws { + let provider = TwoTurnCostProvider( + firstCost: .init(input: 0, output: 0), + secondCost: .init(input: 0, output: 0)) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let result = try await agentService.executeTask( + "Use a tool, then finish.", + maxSteps: 2, + model: .openai(.gpt55), + eventDelegate: streaming ? delegate : nil, + enhancementOptions: nil) + + let cost = try #require(result.usage?.cost) + #expect(cost.input == 0) + #expect(cost.output == 0) + #expect(cost.total == 0) + let sessionId = try #require(result.sessionId) + let session = try #require(try await agentService.getSessionInfo(sessionId: sessionId)) + #expect(session.metadata.totalCost == 0) + try await agentService.deleteSession(id: sessionId) + } + + @Test + @MainActor + func `Unknown zero-token cost remains unknown across session continuation`() async throws { + let model = LanguageModel.openai(.gpt55) + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: model) + let context = try await agentService.prepareSession( + task: "Start cost coverage test.", + model: model, + label: "cost-coverage-test", + logBehavior: .verboseOnly) + + do { + try agentService.saveExecutionSession( + context: context, + model: model, + finalMessages: context.messages, + endTime: Date(), + toolCallCount: 0, + usage: Usage(inputTokens: 0, outputTokens: 0, cost: nil), + status: "failed") + + let firstSession = try #require(try await agentService.getSessionInfo(sessionId: context.id)) + let continuation = agentService.makeContinuationContext( + from: firstSession, + userMessage: "Continue.", + model: model) + try agentService.saveExecutionSession( + context: continuation, + model: model, + finalMessages: continuation.messages, + endTime: Date(), + toolCallCount: 0, + usage: Usage( + inputTokens: 0, + outputTokens: 0, + cost: .init(input: 0, output: 0)), + status: "completed") + + let finalSession = try #require(try await agentService.getSessionInfo(sessionId: context.id)) + #expect(finalSession.metadata.totalCost == nil) + #expect(finalSession.metadata.customData["agent_usage_observed"] == "true") + #expect(finalSession.metadata.customData["agent_usage_cost_complete"] == "false") + try await agentService.deleteSession(id: context.id) + } catch { + try? await agentService.deleteSession(id: context.id) + throw error + } + } + + @Test(arguments: [false, true]) + @MainActor + func `Abnormal billed response checkpoints usage without unsafe assistant history`(_ streaming: Bool) async throws { + let provider = AbnormalBilledResponseProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let model = LanguageModel.openai(.gpt55) + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: model) + let context = try await agentService.prepareSession( + task: "Return a billed but incomplete response.", + model: model, + label: "abnormal-usage-test", + logBehavior: .verboseOnly) + + do { + await #expect(throws: TachikomaError.self) { + if streaming { + _ = try await agentService.executeWithStreaming( + context: context, + model: model, + maxSteps: 1, + streamingDelegate: StreamingEventDelegate { _ in }, + enhancementOptions: nil) + } else { + _ = try await agentService.executeWithoutStreaming( + context: context, + model: model, + maxSteps: 1, + enhancementOptions: nil) + } + } + + let session = try #require(try await agentService.getSessionInfo(sessionId: context.id)) + #expect(session.metadata.customData["status"] == "failed") + #expect(session.metadata.totalTokens == 13) + #expect(abs((session.metadata.totalCost ?? 0) - 0.3) < 0.000_001) + #expect(!session.messages.containsText(AbnormalBilledResponseProvider.unsafeText)) + try await agentService.deleteSession(id: context.id) + } catch { + try? await agentService.deleteSession(id: context.id) + throw error + } + } + + @Test(arguments: [FinishReason.length, .error]) + @MainActor + func `Nonstreaming abnormal tool response fails before tool execution`(_ finishReason: FinishReason) async throws { + let provider = MalformedToolResponseProvider( + finishReason: finishReason, + toolCalls: [TerminalTool.done.call]) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .anthropic(.fable5)) + + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.executeTask( + "Return an incomplete tool response.", + maxSteps: 1, + model: .anthropic(.fable5), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains(finishReason.rawValue)) + #expect(provider.generateRequestCount == 1) + #expect(provider.streamRequestCount == 0) + #expect(!delegate.events.containsToolEvent) + #expect(!delegate.events.containsCompletedEvent) + } + + @Test(arguments: [FinishReason.length, .error]) + @MainActor + func `Streaming abnormal tool response fails before tool execution`(_ finishReason: FinishReason) async throws { + let provider = MalformedToolResponseProvider( + finishReason: finishReason, + toolCalls: [TerminalTool.done.call]) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.executeTask( + "Return an incomplete tool response.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(error.localizedDescription.contains(finishReason.rawValue)) + #expect(provider.generateRequestCount == 0) + #expect(provider.streamRequestCount == 1) + #expect(!delegate.events.containsToolEvent) + #expect(!delegate.events.containsCompletedEvent) + } + + @Test(arguments: [false, true]) + @MainActor + func `Successful done after a failed call requires model continuation`(_ streaming: Bool) async throws { + let provider = ErrorThenDoneProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let delegate = StepLimitEventDelegate() + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + + let thrownError = await #expect(throws: PeekabooAgentService.AgentStepLimitExceededError.self) { + _ = try await agentService.executeTask( + "Do not hide a failed tool call.", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: streaming ? delegate : nil, + enhancementOptions: nil) + } + let error = try #require(thrownError) + + #expect(provider.generateRequestCount == (streaming ? 0 : 1)) + #expect(provider.streamRequestCount == (streaming ? 1 : 0)) + #expect(!delegate.events.containsCompletedEvent) + + let loadedSession = try await agentService.getSessionInfo(sessionId: error.sessionId) + let session = try #require(loadedSession) + #expect(session.metadata.customData["status"] == "max_steps_exhausted") + let toolResults = session.messages.flatMap { message in + message.content.compactMap { part -> AgentToolResult? in + if case let .toolResult(toolResult) = part { + toolResult + } else { + nil + } + } + } + #expect(toolResults.map(\.toolCallId) == ["unknown-call", "done-call"]) + #expect(toolResults.map(\.isError) == [true, false]) + #expect(agentService.turnBoundaryStopReason(from: toolResults) == nil) + + try await agentService.deleteSession(id: error.sessionId) + } + + @MainActor + private func assertCheckpointAfterProviderFailure( + streaming: Bool, + failure: ProviderFailure, + expectedStatus: String) async throws + { + let provider = ToolThenFailureProvider(failure: failure) + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { TachikomaConfiguration.default = previousConfiguration } + + let model = LanguageModel.openai(.gpt55) + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: model) + let context = try await agentService.prepareSession( + task: "Use a tool, then encounter a provider failure.", + model: model, + label: "checkpoint-test", + logBehavior: .verboseOnly) + + do { + if failure == .cancellation { + await #expect(throws: CancellationError.self) { + if streaming { + _ = try await agentService.executeWithStreaming( + context: context, + model: model, + maxSteps: 2, + streamingDelegate: StreamingEventDelegate { _ in }, + enhancementOptions: nil) + } else { + _ = try await agentService.executeWithoutStreaming( + context: context, + model: model, + maxSteps: 2, + enhancementOptions: nil) + } + } + } else { + await #expect(throws: TachikomaError.self) { + if streaming { + _ = try await agentService.executeWithStreaming( + context: context, + model: model, + maxSteps: 2, + streamingDelegate: StreamingEventDelegate { _ in }, + enhancementOptions: nil) + } else { + _ = try await agentService.executeWithoutStreaming( + context: context, + model: model, + maxSteps: 2, + enhancementOptions: nil) + } + } + } + + let session = try #require(try await agentService.getSessionInfo(sessionId: context.id)) + #expect(session.metadata.customData["status"] == expectedStatus) + #expect(session.metadata.toolCallCount == 1) + #expect(session.messages.containsToolCall(id: "checkpoint-tool")) + #expect(session.messages.containsToolResult(id: "checkpoint-tool")) + try await agentService.deleteSession(id: context.id) + } catch { + try? await agentService.deleteSession(id: context.id) + throw error + } + } +} + +enum TerminalTool: CaseIterable, Sendable { + case done + case needInfo + + var call: AgentToolCall { + switch self { + case .done: + AgentToolCall( + id: "done-call", + name: "done", + arguments: ["message": AnyAgentToolValue(string: "Finished export")]) + case .needInfo: + AgentToolCall( + id: "need-info-call", + name: "need_info", + arguments: ["question": AnyAgentToolValue(string: "Which account?")]) + } + } + + var expectedReason: String { + switch self { + case .done: "Finished export" + case .needInfo: "Need more information: Which account?" + } + } +} + +private final class TerminalToolProvider: ModelProvider, @unchecked Sendable { + let modelId = "terminal-tool-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + private let terminalTool: TerminalTool + + init(terminalTool: TerminalTool) { + self.terminalTool = terminalTool + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + ProviderResponse( + text: "", + finishReason: .toolCalls, + toolCalls: [self.terminalTool.call]) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.yield(.tool(self.terminalTool.call)) + continuation.yield(.done(finishReason: .toolCalls)) + continuation.finish() + } + } +} + +private final class PerpetualToolProvider: ModelProvider, @unchecked Sendable { + let modelId = "perpetual-tool-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let lock = NSLock() + private var requests = 0 + + var requestCount: Int { + self.lock.withLock { self.requests } + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + let requestNumber = self.nextRequestNumber() + return ProviderResponse( + text: "tool step \(requestNumber)", + finishReason: .toolCalls, + toolCalls: [self.toolCall(for: requestNumber)]) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + let requestNumber = self.nextRequestNumber() + let toolCall = self.toolCall(for: requestNumber) + return AsyncThrowingStream { continuation in + continuation.yield(.text("tool step \(requestNumber)")) + continuation.yield(.tool(toolCall)) + continuation.yield(.done(finishReason: .toolCalls)) + continuation.finish() + } + } + + private func nextRequestNumber() -> Int { + self.lock.withLock { + self.requests += 1 + return self.requests + } + } + + private func toolCall(for requestNumber: Int) -> AgentToolCall { + AgentToolCall( + id: "tool-call-\(requestNumber)", + name: "missing_test_tool", + arguments: [:]) + } +} + +private final class MalformedToolResponseProvider: ModelProvider, @unchecked Sendable { + let modelId = "malformed-tool-response-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let finishReason: FinishReason + private let toolCalls: [AgentToolCall] + private let text: String + private let lock = NSLock() + private var generateRequests = 0 + private var streamRequests = 0 + + init(finishReason: FinishReason, toolCalls: [AgentToolCall], text: String = "") { + self.finishReason = finishReason + self.toolCalls = toolCalls + self.text = text + } + + var generateRequestCount: Int { + self.lock.withLock { self.generateRequests } + } + + var streamRequestCount: Int { + self.lock.withLock { self.streamRequests } + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + self.lock.withLock { + self.generateRequests += 1 + } + return ProviderResponse( + text: self.text, + finishReason: self.finishReason, + toolCalls: self.toolCalls) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + self.lock.withLock { + self.streamRequests += 1 + } + return AsyncThrowingStream { continuation in + if !self.text.isEmpty { + continuation.yield(.text(self.text)) + } + for toolCall in self.toolCalls { + continuation.yield(.tool(toolCall)) + } + continuation.yield(.done(finishReason: self.finishReason)) + continuation.finish() + } + } +} + +private final class ErrorThenDoneProvider: ModelProvider, @unchecked Sendable { + let modelId = "error-then-done-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let lock = NSLock() + private var generateRequests = 0 + private var streamRequests = 0 + + var generateRequestCount: Int { + self.lock.withLock { self.generateRequests } + } + + var streamRequestCount: Int { + self.lock.withLock { self.streamRequests } + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + self.lock.withLock { + self.generateRequests += 1 + } + return ProviderResponse( + text: "", + finishReason: .toolCalls, + toolCalls: self.toolCalls) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + self.lock.withLock { + self.streamRequests += 1 + } + return AsyncThrowingStream { continuation in + for toolCall in self.toolCalls { + continuation.yield(.tool(toolCall)) + } + continuation.yield(.done(finishReason: .toolCalls)) + continuation.finish() + } + } + + private var toolCalls: [AgentToolCall] { + [ + AgentToolCall( + id: "unknown-call", + name: "missing_test_tool", + arguments: [:]), + TerminalTool.done.call, + ] + } +} + +private final class EmptyTerminalAfterToolProvider: ModelProvider, @unchecked Sendable { + let modelId = "empty-terminal-after-tool-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let lock = NSLock() + private var generateRequests = 0 + private var streamRequests = 0 + + var generateRequestCount: Int { + self.lock.withLock { self.generateRequests } + } + + var streamRequestCount: Int { + self.lock.withLock { self.streamRequests } + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + let requestNumber = self.lock.withLock { + self.generateRequests += 1 + return self.generateRequests + } + return self.response(for: requestNumber) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + let requestNumber = self.lock.withLock { + self.streamRequests += 1 + return self.streamRequests + } + let response = self.response(for: requestNumber) + return AsyncThrowingStream { continuation in + for toolCall in response.toolCalls ?? [] { + continuation.yield(.tool(toolCall)) + } + continuation.yield(.done(finishReason: response.finishReason)) + continuation.finish() + } + } + + private func response(for requestNumber: Int) -> ProviderResponse { + if requestNumber == 1 { + return ProviderResponse( + text: "", + finishReason: .toolCalls, + toolCalls: [AgentToolCall( + id: "missing-tool", + name: "missing_test_tool", + arguments: [:])]) + } + + return ProviderResponse(text: "", finishReason: .stop) + } +} + +private enum ProviderFailure: Equatable, Sendable { + case error + case cancellation +} + +private final class ToolThenFailureProvider: ModelProvider, @unchecked Sendable { + let modelId = "tool-then-failure-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let failure: ProviderFailure + private let lock = NSLock() + private var generateRequests = 0 + private var streamRequests = 0 + + init(failure: ProviderFailure) { + self.failure = failure + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + let requestNumber = self.lock.withLock { + self.generateRequests += 1 + return self.generateRequests + } + if requestNumber == 1 { + return self.toolResponse + } + return try self.throwFailure() + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + let requestNumber = self.lock.withLock { + self.streamRequests += 1 + return self.streamRequests + } + return AsyncThrowingStream { continuation in + if requestNumber == 1 { + continuation.yield(.tool(self.toolCall)) + continuation.yield(.done(finishReason: .toolCalls)) + continuation.finish() + } else { + switch self.failure { + case .error: + continuation.finish(throwing: TachikomaError.apiError("Synthetic provider failure")) + case .cancellation: + continuation.finish(throwing: CancellationError()) + } + } + } + } + + private var toolCall: AgentToolCall { + AgentToolCall(id: "checkpoint-tool", name: "missing_checkpoint_tool", arguments: [:]) + } + + private var toolResponse: ProviderResponse { + ProviderResponse(text: "", finishReason: .toolCalls, toolCalls: [self.toolCall]) + } + + private func throwFailure() throws -> ProviderResponse { + switch self.failure { + case .error: + throw TachikomaError.apiError("Synthetic provider failure") + case .cancellation: + throw CancellationError() + } + } +} + +private final class TwoTurnUsageProvider: ModelProvider, @unchecked Sendable { + let modelId = "two-turn-usage-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let lock = NSLock() + private var requests = 0 + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + self.response(for: self.nextRequestNumber()) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + let response = self.response(for: self.nextRequestNumber()) + return AsyncThrowingStream { continuation in + if !response.text.isEmpty { + continuation.yield(.text(response.text)) + } + for toolCall in response.toolCalls ?? [] { + continuation.yield(.tool(toolCall)) + } + continuation.yield(.done(usage: response.usage, finishReason: response.finishReason)) + continuation.finish() + } + } + + private func nextRequestNumber() -> Int { + self.lock.withLock { + self.requests += 1 + return self.requests + } + } + + private func response(for requestNumber: Int) -> ProviderResponse { + if requestNumber == 1 { + return ProviderResponse( + text: "", + usage: Usage( + inputTokens: 3, + outputTokens: 4, + cost: .init(input: 0.1, output: 0.2)), + finishReason: .toolCalls, + toolCalls: [AgentToolCall( + id: "usage-tool", + name: "missing_usage_tool", + arguments: [:])]) + } + + return ProviderResponse( + text: "Finished", + usage: Usage( + inputTokens: 5, + outputTokens: 6, + cost: .init(input: 0.3, output: 0.4)), + finishReason: .stop) + } +} + +private actor FirstToolCompletionGate { + private var completed = false + private var waiters: [CheckedContinuation] = [] + + func signal() { + guard !self.completed else { return } + self.completed = true + let waiters = self.waiters + self.waiters.removeAll() + for waiter in waiters { + waiter.resume() + } + } + + func wait() async { + if self.completed { + return + } + await withCheckedContinuation { continuation in + self.waiters.append(continuation) + } + } +} + +private final class MultiSleepProvider: ModelProvider, @unchecked Sendable { + let modelId = "multi-sleep-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + ProviderResponse(text: "", finishReason: .toolCalls, toolCalls: self.toolCalls) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + for toolCall in self.toolCalls { + continuation.yield(.tool(toolCall)) + } + continuation.yield(.done(finishReason: .toolCalls)) + continuation.finish() + } + } + + private var toolCalls: [AgentToolCall] { + [ + AgentToolCall( + id: "first-sleep", + name: "sleep", + arguments: ["duration": AnyAgentToolValue(double: 1)]), + AgentToolCall( + id: "second-sleep", + name: "sleep", + arguments: ["duration": AnyAgentToolValue(double: 30000)]), + AgentToolCall( + id: "third-sleep", + name: "sleep", + arguments: ["duration": AnyAgentToolValue(double: 1)]), + ] + } +} + +private actor NoncooperativeProviderGate { + private var started = false + private var released = false + private var startWaiters: [CheckedContinuation] = [] + private var releaseWaiters: [CheckedContinuation] = [] + + func markStarted() { + self.started = true + let waiters = self.startWaiters + self.startWaiters.removeAll() + for waiter in waiters { + waiter.resume() + } + } + + func waitForStart() async { + if self.started { + return + } + await withCheckedContinuation { continuation in + self.startWaiters.append(continuation) + } + } + + func waitForRelease() async { + if self.released { + return + } + await withCheckedContinuation { continuation in + self.releaseWaiters.append(continuation) + } + } + + func release() { + self.released = true + let waiters = self.releaseWaiters + self.releaseWaiters.removeAll() + for waiter in waiters { + waiter.resume() + } + } +} + +private final class NoncooperativeCancellationProvider: ModelProvider, @unchecked Sendable { + static let unsafeText = "response returned after cancellation" + + let modelId = "noncooperative-cancellation-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let gate: NoncooperativeProviderGate + + init(gate: NoncooperativeProviderGate) { + self.gate = gate + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + await self.gate.markStarted() + await self.gate.waitForRelease() + return ProviderResponse( + text: Self.unsafeText, + usage: self.usage, + finishReason: .stop) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { + continuation.yield(.text(Self.unsafeText)) + await self.gate.waitForRelease() + continuation.yield(.done(usage: self.usage, finishReason: .stop)) + continuation.finish() + } + } + } + + private var usage: Usage { + Usage( + inputTokens: 2, + outputTokens: 3, + cost: .init(input: 0, output: 0)) + } +} + +private final class TwoTurnCostProvider: ModelProvider, @unchecked Sendable { + let modelId = "two-turn-cost-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let firstCost: Usage.Cost? + private let secondCost: Usage.Cost? + private let lock = NSLock() + private var requests = 0 + + init(firstCost: Usage.Cost?, secondCost: Usage.Cost?) { + self.firstCost = firstCost + self.secondCost = secondCost + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + self.response(for: self.nextRequestNumber()) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + let response = self.response(for: self.nextRequestNumber()) + return AsyncThrowingStream { continuation in + if !response.text.isEmpty { + continuation.yield(.text(response.text)) + } + for toolCall in response.toolCalls ?? [] { + continuation.yield(.tool(toolCall)) + } + continuation.yield(.done(usage: response.usage, finishReason: response.finishReason)) + continuation.finish() + } + } + + private func nextRequestNumber() -> Int { + self.lock.withLock { + self.requests += 1 + return self.requests + } + } + + private func response(for requestNumber: Int) -> ProviderResponse { + if requestNumber == 1 { + return ProviderResponse( + text: "", + usage: Usage(inputTokens: 3, outputTokens: 4, cost: self.firstCost), + finishReason: .toolCalls, + toolCalls: [AgentToolCall( + id: "cost-tool", + name: "missing_cost_tool", + arguments: [:])]) + } + + return ProviderResponse( + text: "Finished", + usage: Usage(inputTokens: 5, outputTokens: 6, cost: self.secondCost), + finishReason: .stop) + } +} + +private final class AbnormalBilledResponseProvider: ModelProvider, @unchecked Sendable { + static let unsafeText = "unsafe partial assistant response" + + let modelId = "abnormal-billed-response-provider" + let baseURL: String? = nil + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + self.response + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.yield(.text(Self.unsafeText)) + continuation.yield(.done(usage: self.response.usage, finishReason: .length)) + continuation.finish() + } + } + + private var response: ProviderResponse { + ProviderResponse( + text: Self.unsafeText, + usage: Usage( + inputTokens: 8, + outputTokens: 5, + cost: .init(input: 0.2, output: 0.1)), + finishReason: .length) + } +} + +@MainActor +private final class StepLimitEventDelegate: AgentEventDelegate { + private(set) var events: [AgentEvent] = [] + + func agentDidEmitEvent(_ event: AgentEvent) { + self.events.append(event) + } +} + +extension [AgentEvent] { + fileprivate var containsCompletedEvent: Bool { + self.contains { event in + if case .completed = event { + true + } else { + false + } + } + } + + fileprivate var containsToolEvent: Bool { + self.contains { event in + switch event { + case .toolCallStarted, .toolCallUpdated, .toolCallCompleted: + true + default: + false + } + } + } +} + +extension [ModelMessage] { + fileprivate func containsText(_ expected: String) -> Bool { + self.contains { message in + message.content.contains { part in + if case let .text(text) = part { + text == expected + } else { + false + } + } + } + } + + fileprivate func containsToolCall(id: String) -> Bool { + self.contains { message in + message.content.contains { part in + if case let .toolCall(toolCall) = part { + toolCall.id == id + } else { + false + } + } + } + } + + fileprivate func containsToolResult(id: String) -> Bool { + self.contains { message in + message.content.contains { part in + if case let .toolResult(toolResult) = part { + toolResult.toolCallId == id + } else { + false + } + } + } + } +} From 3a3e5fd3bceace92674faeba21a163d9c45c5065 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:07:55 +0100 Subject: [PATCH 03/11] test(agent): cover model and session continuity --- .../AgentSessionManagerStorageTests.swift | 130 +++++++ .../AgentTurnBoundaryTests.swift | 18 +- .../PeekabooAIServiceProviderTests.swift | 6 + ...ooAgentOllamaStreamingReasoningTests.swift | 87 +++++ ...kabooAgentResumeModelContinuityTests.swift | 350 ++++++++++++++++++ .../PeekabooAgentServiceModelTests.swift | 163 ++++++-- ...abooAgentStreamingContentFilterTests.swift | 19 +- ...AgentStreamingReasoningBoundaryTests.swift | 25 +- 8 files changed, 757 insertions(+), 41 deletions(-) create mode 100644 Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentSessionManagerStorageTests.swift create mode 100644 Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift create mode 100644 Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift diff --git a/Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentSessionManagerStorageTests.swift b/Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentSessionManagerStorageTests.swift new file mode 100644 index 000000000..f256290af --- /dev/null +++ b/Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentSessionManagerStorageTests.swift @@ -0,0 +1,130 @@ +import Foundation +import Tachikoma +import Testing +@testable import PeekabooAgentRuntime + +@Suite("Agent session persistence", .serialized) +struct AgentSessionManagerStorageTests { + @Test + @MainActor + func `Session operations reject traversal IDs without touching outside files`() async throws { + let root = try Self.makeTemporaryDirectory() + defer { try? FileManager.default.removeItem(at: root) } + + let sessionDirectory = root.appendingPathComponent("sessions", isDirectory: true) + let manager = try AgentSessionManager(sessionDirectory: sessionDirectory) + let outsideFile = root.appendingPathComponent("outside.json") + let sentinel = Data("outside-session-data".utf8) + try sentinel.write(to: outsideFile) + + #expect(throws: AgentSessionManagerError.self) { + try manager.saveSession(Self.session(id: "../outside")) + } + #expect(try Data(contentsOf: outsideFile) == sentinel) + + await #expect(throws: AgentSessionManagerError.self) { + _ = try await manager.loadSession(id: "../outside") + } + #expect(try Data(contentsOf: outsideFile) == sentinel) + + await #expect(throws: AgentSessionManagerError.self) { + try await manager.deleteSession(id: "../outside") + } + #expect(try Data(contentsOf: outsideFile) == sentinel) + } + + @Test + @MainActor + func `Session IDs cannot contain path separators or ambiguous path components`() throws { + let root = try Self.makeTemporaryDirectory() + defer { try? FileManager.default.removeItem(at: root) } + + let manager = try AgentSessionManager(sessionDirectory: root) + let unsafeIDs = ["", ".", "..", "nested/session", "nested\\session", "/tmp/session"] + + for id in unsafeIDs { + #expect(throws: AgentSessionManagerError.self) { + try manager.saveSession(Self.session(id: id)) + } + } + #expect(manager.listSessions().isEmpty) + } + + @Test + @MainActor + func `Stored session identity must match its contained file`() async throws { + let root = try Self.makeTemporaryDirectory() + defer { try? FileManager.default.removeItem(at: root) } + + let encoded = try JSONEncoder().encode(Self.session(id: "../outside")) + try encoded.write(to: root.appendingPathComponent("safe.json")) + let manager = try AgentSessionManager(sessionDirectory: root) + + await #expect(throws: AgentSessionManagerError.self) { + _ = try await manager.loadSession(id: "safe") + } + #expect(manager.listSessions().isEmpty) + } + + @Test + @MainActor + func `Saving a session atomically replaces its previous snapshot`() async throws { + let root = try Self.makeTemporaryDirectory() + defer { try? FileManager.default.removeItem(at: root) } + + let manager = try AgentSessionManager(sessionDirectory: root) + try manager.saveSession(Self.session(id: "replace-me", totalTokens: 10)) + try manager.saveSession(Self.session(id: "replace-me", totalTokens: 25)) + + let freshManager = try AgentSessionManager(sessionDirectory: root) + let loaded = try #require(try await freshManager.loadSession(id: "replace-me")) + #expect(loaded.metadata.totalTokens == 25) + + let files = try FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) + #expect(files.map(\.lastPathComponent) == ["replace-me.json"]) + } + + @Test + @MainActor + func `Session summaries reflect persisted lifecycle status`() throws { + let root = try Self.makeTemporaryDirectory() + defer { try? FileManager.default.removeItem(at: root) } + + let manager = try AgentSessionManager(sessionDirectory: root) + try manager.saveSession(Self.session(id: "active", status: "active")) + try manager.saveSession(Self.session(id: "completed", status: "completed")) + try manager.saveSession(Self.session(id: "failed", status: "failed")) + try manager.saveSession(Self.session(id: "cancelled", status: "cancelled")) + try manager.saveSession(Self.session(id: "resumable", status: "max_steps_exhausted")) + + let statuses = Dictionary(uniqueKeysWithValues: manager.listSessions().map { ($0.id, $0.status) }) + #expect(statuses["active"] == .active) + #expect(statuses["completed"] == .completed) + #expect(statuses["failed"] == .failed) + #expect(statuses["cancelled"] == .failed) + #expect(statuses["resumable"] == .active) + } + + private static func makeTemporaryDirectory() throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("AgentSessionManagerStorageTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + + private static func session( + id: String, + status: String? = nil, + totalTokens: Int = 0) -> AgentSession + { + let now = Date() + let customData = status.map { ["status": $0] } ?? [:] + return AgentSession( + id: id, + modelName: "test-model", + messages: [.user("Test session")], + metadata: SessionMetadata(totalTokens: totalTokens, customData: customData), + createdAt: now, + updatedAt: now) + } +} diff --git a/Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentTurnBoundaryTests.swift b/Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentTurnBoundaryTests.swift index d430efb90..835f417b8 100644 --- a/Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentTurnBoundaryTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooAgentRuntimeTests/AgentTurnBoundaryTests.swift @@ -44,12 +44,26 @@ struct AgentTurnBoundaryTests { } @Test - func `non UI tools do not stop after perceive`() { + func `completion tools stop after successful execution`() { + let boundary = AgentTurnBoundary() + + #expect(boundary.record( + toolName: "done", + arguments: ["message": AnyAgentToolValue(string: "Finished export")]) == + .stopAfterSuccessfulTool(reason: "Finished export")) + #expect(boundary.record( + toolName: "need-info", + arguments: ["question": AnyAgentToolValue(string: "Which account?")]) == + .stopAfterSuccessfulTool(reason: "Need more information: Which account?")) + } + + @Test + func `non UI and invalid completion tools do not stop after perceive`() { let boundary = AgentTurnBoundary() #expect(boundary.record(toolName: "watch") == .continueTurn) #expect(boundary.record(toolName: "sleep") == .continueTurn) - #expect(boundary.record(toolName: "done") == .continueTurn) + #expect(boundary.record(toolName: "need_info", arguments: [:]) == .continueTurn) } @Test diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAIServiceProviderTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAIServiceProviderTests.swift index e4fe02235..354c2f52b 100644 --- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAIServiceProviderTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAIServiceProviderTests.swift @@ -4,6 +4,7 @@ import Testing @testable import PeekabooAutomation @Suite(.serialized) +// swiftlint:disable:next type_body_length struct PeekabooAIServiceProviderTests { @Test @MainActor @@ -876,6 +877,8 @@ struct PeekabooAIServiceProviderTests { "GOOGLE_API_KEY", "MINIMAX_API_KEY", "MINIMAX_CN_API_KEY", + "MOONSHOT_API_KEY", + "KIMI_API_KEY", "OPENROUTER_API_KEY", "X_AI_API_KEY", "XAI_API_KEY", @@ -943,6 +946,8 @@ struct PeekabooAIServiceProviderTests { "GOOGLE_API_KEY", "MINIMAX_API_KEY", "MINIMAX_CN_API_KEY", + "MOONSHOT_API_KEY", + "KIMI_API_KEY", "OPENROUTER_API_KEY", "X_AI_API_KEY", "XAI_API_KEY", @@ -991,6 +996,7 @@ struct PeekabooAIServiceProviderTests { TachikomaConfiguration.current.removeAPIKey(for: .google) TachikomaConfiguration.current.removeAPIKey(for: .minimax) TachikomaConfiguration.current.removeAPIKey(for: .minimaxCN) + TachikomaConfiguration.current.removeAPIKey(for: .kimi) TachikomaConfiguration.current.removeAPIKey(for: .grok) TachikomaConfiguration.current.removeAPIKey(for: .custom("openrouter")) TachikomaConfiguration.current.removeBaseURL(for: .ollama) diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift new file mode 100644 index 000000000..1b1e8e39a --- /dev/null +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift @@ -0,0 +1,87 @@ +import Foundation +import Tachikoma +import Testing +@testable import PeekabooAgentRuntime +@testable import PeekabooCore + +@Suite(.serialized) +struct PeekabooAgentOllamaStreamingReasoningTests { + @Test + @MainActor + func `Streaming Ollama thinking is not replayed to an injected provider`() async throws { + let provider = StreamingOllamaReasoningReplayProvider() + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setBaseURL("http://localhost:11434", for: .ollama) + configuration.setProviderFactoryOverride { _, _ in provider } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configuration + defer { + TachikomaConfiguration.default = previousConfiguration + } + + let model = LanguageModel.ollama(.custom("qwen3:8b")) + let agentService = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: model) + + _ = try await agentService.executeTask( + "Use a tool, then continue.", + maxSteps: 2, + model: model, + eventDelegate: OllamaNoopAgentEventDelegate(), + enhancementOptions: nil) + + let secondRequestMessages = try #require(provider.secondRequestMessages) + #expect(!secondRequestMessages.contains { $0.channel == .thinking }) + } +} + +private final class StreamingOllamaReasoningReplayProvider: ModelProvider, @unchecked Sendable { + let modelId = "qwen3:8b" + let baseURL: String? = "http://localhost:11434" + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let lock = NSLock() + private var requestCount = 0 + private var capturedSecondRequestMessages: [ModelMessage]? + + var secondRequestMessages: [ModelMessage]? { + self.lock.withLock { self.capturedSecondRequestMessages } + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + ProviderResponse(text: "done", finishReason: .stop) + } + + func streamText(request: ProviderRequest) async throws -> AsyncThrowingStream { + let requestNumber = self.lock.withLock { + self.requestCount += 1 + if self.requestCount == 2 { + self.capturedSecondRequestMessages = request.messages + } + return self.requestCount + } + + return AsyncThrowingStream { continuation in + if requestNumber == 1 { + continuation.yield(.reasoning("streamed Ollama thinking", type: "ollama_thinking")) + continuation.yield(.tool(AgentToolCall( + id: "missing-tool", + name: "missing_test_tool", + arguments: [:]))) + continuation.yield(.done(finishReason: .toolCalls)) + } else { + continuation.yield(.text("done")) + continuation.yield(.done(finishReason: .stop)) + } + continuation.finish() + } + } +} + +@MainActor +private final class OllamaNoopAgentEventDelegate: AgentEventDelegate { + func agentDidEmitEvent(_: AgentEvent) {} +} diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift new file mode 100644 index 000000000..4d80d3c8d --- /dev/null +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift @@ -0,0 +1,350 @@ +import Foundation +import PeekabooFoundation +import Tachikoma +import Testing +@testable import PeekabooAgentRuntime +@testable import PeekabooCore + +@Suite(.serialized, .tags(.safe)) +struct PeekabooAgentResumeModelContinuityTests { + @Test + @MainActor + func `Resume without override preserves stored Ollama model`() throws { + let service = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let model = LanguageModel.ollama(.custom("qwen3.5:9b")) + let identity = service.persistedModelIdentity(for: model) + let session = Self.session( + modelName: identity.displayName, + modelSelection: identity.selection, + modelEndpointIdentity: identity.endpointIdentity, + modelProviderIdentity: identity.providerIdentity) + + let resolved = try service.resolveContinuationModel(explicitModel: nil, session: session) + + #expect(resolved == .ollama(.custom("qwen3.5:9b"))) + } + + @Test + @MainActor + func `Explicit resume model overrides stored selection`() throws { + let service = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let session = Self.session( + modelName: "Custom/removed-provider/private-model", + modelSelection: "removed-provider/private-model") + + let resolved = try service.resolveContinuationModel( + explicitModel: .anthropic(.opus48), + session: session) + + #expect(resolved == .anthropic(.opus48)) + } + + @Test + @MainActor + func `Legacy Ollama session without endpoint proof fails closed`() throws { + let service = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let session = Self.session(modelName: "Ollama/qwen3.5:9b") + + #expect(throws: PeekabooError.self) { + _ = try service.resolveContinuationModel(explicitModel: nil, session: session) + } + } + + @Test + @MainActor + func `Unresolved legacy custom model fails closed`() throws { + let service = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let session = Self.session(modelName: "Custom/removed-provider/private-model") + + let error = #expect(throws: PeekabooError.self) { + _ = try service.resolveContinuationModel(explicitModel: nil, session: session) + } + + #expect(error?.localizedDescription.contains("can no longer be verified safely") == true) + #expect(error?.localizedDescription.contains("explicit model") == true) + } + + @Test(arguments: [ + LanguageModel.azureOpenAI( + deployment: "private", + endpoint: "https://azure.tenant.example"), + .openaiCompatible( + modelId: "private", + baseURL: "https://openai.tenant.example/v1"), + .anthropicCompatible( + modelId: "private", + baseURL: "https://anthropic.tenant.example/v1"), + ]) + @MainActor + func `Legacy endpoint-bearing model cannot fall through to current default`(_ legacyModel: LanguageModel) throws { + let service = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .openai(.gpt55)) + let session = Self.session(modelName: legacyModel.description) + + #expect(throws: PeekabooError.self) { + _ = try service.resolveContinuationModel(explicitModel: nil, session: session) + } + + let explicit = try service.resolveContinuationModel( + explicitModel: .ollama(.custom("qwen3.5:9b")), + session: session) + #expect(explicit == .ollama(.custom("qwen3.5:9b"))) + } + + @Test + @MainActor + func `Unregistered custom model has no implicit persisted selection`() throws { + let service = try PeekabooAgentService(services: PeekabooServices()) + let model = LanguageModel.custom(provider: CredentialBearingTestProvider()) + + #expect(service.persistedModelSelection(for: model) == nil) + } + + @Test + @MainActor + func `Bare custom model identifier cannot resume as a built in provider`() throws { + let service = try PeekabooAgentService( + services: PeekabooServices(), + defaultModel: .anthropic(.opus48)) + let customModel = LanguageModel.custom(provider: BareCustomTestProvider()) + + #expect(service.persistedModelSelection(for: customModel) == nil) + + let legacySession = Self.session(modelName: "Custom/gpt-5.5") + #expect(throws: PeekabooError.self) { + _ = try service.resolveContinuationModel(explicitModel: nil, session: legacySession) + } + + let invalidPersistedSession = Self.session( + modelName: "Custom/gpt-5.5", + modelSelection: "gpt-5.5", + modelEndpointIdentity: service.currentEndpointIdentity(for: .openai(.gpt55)), + modelProviderIdentity: "openai") + #expect(throws: PeekabooError.self) { + _ = try service.resolveContinuationModel(explicitModel: nil, session: invalidPersistedSession) + } + } + + @Test + @MainActor + func `Implicit resume rejects Ollama endpoint drift`() throws { + let originalConfiguration = TachikomaConfiguration.default + let originalEndpoint = getenv("PEEKABOO_OLLAMA_BASE_URL").map { String(cString: $0) } + defer { + TachikomaConfiguration.default = originalConfiguration + if let originalEndpoint { + setenv("PEEKABOO_OLLAMA_BASE_URL", originalEndpoint, 1) + } else { + unsetenv("PEEKABOO_OLLAMA_BASE_URL") + } + } + + let model = LanguageModel.ollama(.custom("qwen3.5:9b")) + setenv("PEEKABOO_OLLAMA_BASE_URL", "http://127.0.0.1:11434", 1) + let configurationA = TachikomaConfiguration(loadFromEnvironment: false) + TachikomaConfiguration.default = configurationA + let services = PeekabooServices() + services.configuration.applyAIProviderKeys(to: configurationA) + let service = try PeekabooAgentService(services: services, defaultModel: model) + let identity = service.persistedModelIdentity(for: model) + let session = Self.session( + modelName: identity.displayName, + modelSelection: identity.selection, + modelEndpointIdentity: identity.endpointIdentity, + modelProviderIdentity: identity.providerIdentity) + + setenv("PEEKABOO_OLLAMA_BASE_URL", "http://127.0.0.1:22434", 1) + let configurationB = TachikomaConfiguration(loadFromEnvironment: false) + TachikomaConfiguration.default = configurationB + services.configuration.applyAIProviderKeys(to: configurationB) + #expect(identity.endpointIdentity != service.currentEndpointIdentity(for: model)) + + #expect(throws: PeekabooError.self) { + _ = try service.resolveContinuationModel(explicitModel: nil, session: session) + } + #expect(try service.resolveContinuationModel(explicitModel: model, session: session) == model) + } + + @Test + @MainActor + func `Prepared session keeps its provider when global configuration changes`() async throws { + let providerA = CountingContinuityProvider(modelId: "provider-a") + let providerB = CountingContinuityProvider(modelId: "provider-b") + let configurationA = TachikomaConfiguration(loadFromEnvironment: false) + configurationA.setProviderFactoryOverride { _, _ in providerA } + let configurationB = TachikomaConfiguration(loadFromEnvironment: false) + configurationB.setProviderFactoryOverride { _, _ in providerB } + + let previousConfiguration = TachikomaConfiguration.default + TachikomaConfiguration.default = configurationA + defer { TachikomaConfiguration.default = previousConfiguration } + + let model = LanguageModel.openai(.gpt55) + let service = try PeekabooAgentService(services: PeekabooServices(), defaultModel: model) + let context = try await service.prepareSession( + task: "Answer once.", + model: model, + label: "provider-continuity-test", + logBehavior: .verboseOnly) + + TachikomaConfiguration.default = configurationB + let result = try await service.executeWithoutStreaming( + context: context, + model: model, + maxSteps: 1, + enhancementOptions: nil) + try await service.deleteSession(id: context.id) + + #expect(result.content == "provider-a response") + #expect(providerA.requestCount == 1) + #expect(providerB.requestCount == 0) + } + + @Test + @MainActor + func `Endpoint credentials never appear in persisted model display`() throws { + let service = try PeekabooAgentService(services: PeekabooServices()) + let model = LanguageModel.openaiCompatible( + modelId: "private-model", + baseURL: "https://alice:secret@example.test/v1?token=hidden") + let identity = service.persistedModelIdentity(for: model) + let session = Self.session( + modelName: identity.displayName, + modelSelection: identity.selection, + modelEndpointIdentity: identity.endpointIdentity, + modelProviderIdentity: identity.providerIdentity) + + let encoded = try JSONEncoder().encode(session) + let json = try #require(String(data: encoded, encoding: .utf8)) + + #expect(identity.displayName == "OpenAI-Compatible/private-model") + #expect(!json.contains("alice")) + #expect(!json.contains("secret")) + #expect(!json.contains("hidden")) + #expect(!json.contains("example.test")) + } + + @Test + @MainActor + func `Nonroundtrippable models have no implicit persisted selection`() throws { + let service = try PeekabooAgentService(services: PeekabooServices()) + + #expect(service.persistedModelSelection(for: .azureOpenAI(deployment: "private")) == nil) + #expect(service.persistedModelSelection(for: .openaiCompatible( + modelId: "private", + baseURL: "https://tenant.example/v1")) == nil) + #expect(service.persistedModelSelection(for: .anthropicCompatible( + modelId: "private", + baseURL: "https://tenant.example/v1")) == nil) + #expect(service.persistedModelSelection(for: .together(modelId: "private")) == nil) + #expect(service.persistedModelSelection(for: .replicate(modelId: "private")) == nil) + } + + @Test + func `Legacy session JSON decodes without model selection`() throws { + let session = Self.session(modelName: "Ollama/qwen3.5:9b") + let encoded = try JSONEncoder().encode(session) + let object = try #require(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + + #expect(object["modelSelection"] == nil) + #expect(object["modelEndpointIdentity"] == nil) + #expect(object["modelProviderIdentity"] == nil) + + let decoded = try JSONDecoder().decode(AgentSession.self, from: encoded) + #expect(decoded.modelSelection == nil) + #expect(decoded.modelEndpointIdentity == nil) + #expect(decoded.modelProviderIdentity == nil) + #expect(decoded.modelName == session.modelName) + } + + private static func session( + modelName: String, + modelSelection: String? = nil, + modelEndpointIdentity: String? = nil, + modelProviderIdentity: String? = nil) -> AgentSession + { + let now = Date() + return AgentSession( + id: "resume-model-\(UUID().uuidString)", + modelName: modelName, + modelSelection: modelSelection, + modelEndpointIdentity: modelEndpointIdentity, + modelProviderIdentity: modelProviderIdentity, + messages: [.system("Test system prompt"), .user("Test task")], + metadata: SessionMetadata(), + createdAt: now, + updatedAt: now) + } +} + +private struct CredentialBearingTestProvider: ModelProvider { + let modelId = "local-proxy/private-model" + let baseURL: String? = "http://localhost:1234/v1" + let apiKey: String? = "test-secret" + let capabilities = ModelCapabilities() + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + ProviderResponse(text: "unused", finishReason: .stop) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.finish() + } + } +} + +private struct BareCustomTestProvider: ModelProvider { + let modelId = "gpt-5.5" + let baseURL: String? = "http://localhost:1234/v1" + let apiKey: String? = "test-secret" + let capabilities = ModelCapabilities() + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + ProviderResponse(text: "unused", finishReason: .stop) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.finish() + } + } +} + +private final class CountingContinuityProvider: ModelProvider, @unchecked Sendable { + let modelId: String + let baseURL: String? = "http://localhost:1234/v1" + let apiKey: String? = nil + let capabilities = ModelCapabilities() + + private let lock = NSLock() + private var requests = 0 + + init(modelId: String) { + self.modelId = modelId + } + + var requestCount: Int { + self.lock.withLock { self.requests } + } + + func generateText(request _: ProviderRequest) async throws -> ProviderResponse { + self.lock.withLock { self.requests += 1 } + return ProviderResponse(text: "\(self.modelId) response", finishReason: .stop) + } + + func streamText(request _: ProviderRequest) async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.finish() + } + } +} diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentServiceModelTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentServiceModelTests.swift index f3bce1f67..87e2570d5 100644 --- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentServiceModelTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentServiceModelTests.swift @@ -1,4 +1,5 @@ import Foundation +import PeekabooFoundation import Tachikoma import Testing @testable import PeekabooAgentRuntime @@ -212,12 +213,14 @@ extension PeekabooAgentServiceTests { services: PeekabooServices(), defaultModel: .anthropic(.fable5)) - _ = try await agentService.executeTask( - "Use a tool once.", - maxSteps: 1, - model: .anthropic(.fable5), - eventDelegate: delegate, - enhancementOptions: nil) + await #expect(throws: PeekabooAgentService.AgentStepLimitExceededError.self) { + _ = try await agentService.executeTask( + "Use a tool once.", + maxSteps: 1, + model: .anthropic(.fable5), + eventDelegate: delegate, + enhancementOptions: nil) + } let startIndex = try #require(delegate.events.firstIndexOfToolStart("missing_test_tool")) let completionIndex = try #require(delegate.events.firstIndexOfToolCompletion("missing_test_tool")) @@ -539,7 +542,7 @@ extension PeekabooAgentServiceTests { @Test @MainActor - func `Nonstreaming native reasoning only response records assistant boundary`() async throws { + func `Nonstreaming reasoning only terminal response is rejected`() async throws { let provider = NativeReasoningOnlyProvider() let configuration = TachikomaConfiguration(loadFromEnvironment: false) configuration.setProviderFactoryOverride { _, _ in provider } @@ -554,18 +557,21 @@ extension PeekabooAgentServiceTests { services: PeekabooServices(), defaultModel: .anthropic(.fable5)) - let result = try await agentService.executeTask( - "think only", - maxSteps: 1, + let loopConfiguration = PeekabooAgentService.StreamingLoopConfiguration( model: .anthropic(.fable5), + provider: provider, + tools: [], + sessionId: "reasoning-only-test", + eventHandler: nil, enhancementOptions: nil) + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.runGenerationLoop( + configuration: loopConfiguration, + maxSteps: 1, + initialMessages: [.user("think only")]) + } - let thinkingIndex = try #require(result.messages.firstIndex { $0.channel == .thinking }) - let boundaryMessage = try #require(result.messages.dropFirst(thinkingIndex + 1).first) - - #expect(boundaryMessage.role == .assistant) - #expect(boundaryMessage.content == [.text("")]) - #expect(boundaryMessage.metadata?.customData?["tachikoma.internal.boundary"] == "reasoning_only") + #expect(thrownError?.localizedDescription.contains("empty terminal response") == true) } @Test @@ -1873,6 +1879,103 @@ extension PeekabooAgentServiceTests { #expect(agentService.defaultModel == customModel.description) } + @Test + @MainActor + func `Resume rejects stored custom model without tool support`() throws { + try self.withIsolatedAgentEnvironment( + ["PEEKABOO_CUSTOM_PROVIDER_KEY": "resolved-secret"], + configurationJSON: """ + { + "customProviders": { + "text-proxy": { + "name": "Text Proxy", + "type": "openai", + "enabled": true, + "options": { + "baseURL": "http://localhost:8317/v1", + "apiKey": "test-key" + }, + "models": { + "text-only": { + "name": "Text only", + "supportsTools": false + } + } + } + } + } + """) { + let service = try PeekabooAgentService( + services: self.makeServices(), + defaultModel: .openai(.gpt55)) + let now = Date() + let session = AgentSession( + id: "resume-text-only-\(UUID().uuidString)", + modelName: "Custom/text-proxy/text-only", + modelSelection: "text-proxy/text-only", + messages: [.system("Test system prompt"), .user("Test task")], + metadata: SessionMetadata(), + createdAt: now, + updatedAt: now) + + #expect(throws: PeekabooError.self) { + _ = try service.resolveContinuationModel(explicitModel: nil, session: session) + } + } + } + + @Test + @MainActor + func `Stored custom provider model round trips without credentials`() throws { + try self.withIsolatedAgentEnvironment( + ["PEEKABOO_CUSTOM_PROVIDER_KEY": "resolved-secret"], + configurationJSON: """ + { + "customProviders": { + "local-proxy": { + "name": "Local Proxy", + "type": "openai", + "enabled": true, + "options": { + "baseURL": "http://localhost:8317/v1", + "apiKey": "test-key" + }, + "models": { + "mini": { + "name": "Mini", + "supportsTools": true + } + } + } + } + } + """) { + let service = try PeekabooAgentService( + services: self.makeServices(), + defaultModel: .openai(.gpt55)) + let configured = try #require(service.resolveConfiguredModel("local-proxy/mini")) + let identity = service.persistedModelIdentity(for: configured) + let selection = try #require(identity.selection) + let now = Date() + let session = AgentSession( + id: "resume-custom-\(UUID().uuidString)", + modelName: identity.displayName, + modelSelection: selection, + modelEndpointIdentity: identity.endpointIdentity, + modelProviderIdentity: identity.providerIdentity, + messages: [.system("Test system prompt"), .user("Test task")], + metadata: SessionMetadata(), + createdAt: now, + updatedAt: now) + + let resolved = try service.resolveContinuationModel(explicitModel: nil, session: session) + + #expect(selection == "local-proxy/mini") + #expect(!selection.contains("resolved-secret")) + #expect(resolved.modelId == configured.modelId) + } + } + private func withIsolatedAgentEnvironment( _ overrides: [String: String], configurationJSON: String? = nil, @@ -1893,6 +1996,8 @@ extension PeekabooAgentServiceTests { "PEEKABOO_MISSING_PROVIDER_KEY", "MINIMAX_API_KEY", "MINIMAX_CN_API_KEY", + "MOONSHOT_API_KEY", + "KIMI_API_KEY", "PEEKABOO_OLLAMA_BASE_URL", "OLLAMA_BASE_URL", ] @@ -1936,6 +2041,8 @@ extension PeekabooAgentServiceTests { unsetenv("PEEKABOO_MISSING_PROVIDER_KEY") unsetenv("MINIMAX_API_KEY") unsetenv("MINIMAX_CN_API_KEY") + unsetenv("MOONSHOT_API_KEY") + unsetenv("KIMI_API_KEY") unsetenv("PEEKABOO_OLLAMA_BASE_URL") unsetenv("OLLAMA_BASE_URL") TachikomaConfiguration.current.removeAPIKey(for: .grok) @@ -1967,6 +2074,8 @@ extension PeekabooAgentServiceTests { "PEEKABOO_MISSING_PROVIDER_KEY", "MINIMAX_API_KEY", "MINIMAX_CN_API_KEY", + "MOONSHOT_API_KEY", + "KIMI_API_KEY", "PEEKABOO_OLLAMA_BASE_URL", "OLLAMA_BASE_URL", ] @@ -2010,6 +2119,8 @@ extension PeekabooAgentServiceTests { unsetenv("PEEKABOO_MISSING_PROVIDER_KEY") unsetenv("MINIMAX_API_KEY") unsetenv("MINIMAX_CN_API_KEY") + unsetenv("MOONSHOT_API_KEY") + unsetenv("KIMI_API_KEY") unsetenv("PEEKABOO_OLLAMA_BASE_URL") unsetenv("OLLAMA_BASE_URL") for (key, value) in overrides { @@ -2263,19 +2374,29 @@ struct PeekabooAgentResumeTests { createdAt: now, updatedAt: now)) - do { + let thrownError = await #expect(throws: PeekabooAgentService.AgentStepLimitExceededError.self) { _ = try await agentService.resumeSession( sessionId: sessionId, model: .openai(.gpt55), maxSteps: 1, enhancementOptions: nil) - try await agentService.deleteSession(id: sessionId) - } catch { - try? await agentService.deleteSession(id: sessionId) - throw error } + let error = try #require(thrownError) #expect(provider.requestCount == 1) + #expect(error.sessionId == sessionId) + let loadedSession = try await agentService.getSessionInfo(sessionId: sessionId) + #expect(loadedSession?.metadata.customData["status"] == "max_steps_exhausted") + #expect(loadedSession?.messages.contains { message in + message.content.contains { part in + if case .toolResult = part { + true + } else { + false + } + } + } == true) + try await agentService.deleteSession(id: sessionId) } } diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingContentFilterTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingContentFilterTests.swift index f95a31680..8cfb30c86 100644 --- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingContentFilterTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingContentFilterTests.swift @@ -130,16 +130,19 @@ struct PeekabooAgentStreamingContentFilterTests { services: PeekabooServices(), defaultModel: .openai(.gpt55)) - let response = try await agentService.executeTask( - "trigger nil finish stream", - maxSteps: 1, - model: .openai(.gpt55), - eventDelegate: delegate, - enhancementOptions: nil) - - #expect(response.content == "assistant text") + let thrownError = await #expect(throws: PeekabooAgentService.AgentStepLimitExceededError.self) { + _ = try await agentService.executeTask( + "trigger nil finish stream", + maxSteps: 1, + model: .openai(.gpt55), + eventDelegate: delegate, + enhancementOptions: nil) + } + let error = try #require(thrownError) + #expect(delegate.events.containsAssistantMessage("assistant text")) #expect(delegate.events.firstToolStart(named: "terminal_test_tool") != nil) + try await agentService.deleteSession(id: error.sessionId) } } diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingReasoningBoundaryTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingReasoningBoundaryTests.swift index 738ec4353..1eba5d40d 100644 --- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingReasoningBoundaryTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentStreamingReasoningBoundaryTests.swift @@ -8,7 +8,7 @@ import Testing struct PeekabooAgentStreamingReasoningBoundaryTests { @Test @MainActor - func `Streaming native reasoning only response records assistant boundary`() async throws { + func `Streaming reasoning only terminal response is rejected`() async throws { let provider = StreamingReasoningOnlyProvider() let configuration = TachikomaConfiguration(loadFromEnvironment: false) configuration.setProviderFactoryOverride { _, _ in provider } @@ -23,16 +23,21 @@ struct PeekabooAgentStreamingReasoningBoundaryTests { services: PeekabooServices(), defaultModel: .anthropic(.opus47)) - let result = try await agentService.executeTaskStreaming( - "think only", - model: .anthropic(.opus47)) { _ in } - - let thinkingIndex = try #require(result.messages.firstIndex { $0.channel == .thinking }) - let boundaryMessage = try #require(result.messages.dropFirst(thinkingIndex + 1).first) + let loopConfiguration = PeekabooAgentService.StreamingLoopConfiguration( + model: .anthropic(.opus47), + provider: provider, + tools: [], + sessionId: "reasoning-only-stream-test", + eventHandler: nil, + enhancementOptions: nil) + let thrownError = await #expect(throws: TachikomaError.self) { + _ = try await agentService.runStreamingLoop( + configuration: loopConfiguration, + maxSteps: 1, + initialMessages: [.user("think only")]) + } - #expect(boundaryMessage.role == .assistant) - #expect(boundaryMessage.content == [.text("")]) - #expect(boundaryMessage.metadata?.customData?["tachikoma.internal.boundary"] == "reasoning_only") + #expect(thrownError?.localizedDescription.contains("empty terminal response") == true) } @Test From 0677c244c3baf9747428985a1a4d15523ed32a4f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:07:59 +0100 Subject: [PATCH 04/11] fix(cli): validate agent models and preserve provider config --- .../PeekabooCLI/CLI/PeekabooEntryPoint.swift | 3 +- .../Commands/AI/AgentChatEventDelegate.swift | 2 + .../Commands/AI/AgentChatLaunchPolicy.swift | 19 +- .../PeekabooCLI/Commands/AI/AgentChatUI.swift | 6 + .../Commands/AI/AgentCommand+Chat.swift | 22 +- .../Commands/AI/AgentCommand+Execution.swift | 16 +- .../AI/AgentCommand+ModelParsing.swift | 53 +++ .../Commands/AI/AgentCommand+Sessions.swift | 7 +- .../Commands/AI/AgentCommand.swift | 50 ++- .../Commands/AI/AgentOutputDelegate.swift | 2 + .../ConfigCommand+ProviderManagement.swift | 56 +-- .../AgentCommandBasicTests.swift | 1 + ...entCommandValidationIntegrationTests.swift | 45 +++ .../CLIRuntimeSmokeTests.swift | 4 +- .../AgentChatLaunchPolicyTests.swift | 16 + .../AgentCommandModelParsingTests.swift | 171 +++++++++ .../AgentCommandStepLimitTests.swift | 50 +++ .../ConfigModelsProviderTests.swift | 355 ++++++++++++++++++ 18 files changed, 832 insertions(+), 46 deletions(-) create mode 100644 Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift create mode 100644 Apps/CLI/Tests/CoreCLITests/AgentCommandStepLimitTests.swift create mode 100644 Apps/CLI/Tests/CoreCLITests/ConfigModelsProviderTests.swift diff --git a/Apps/CLI/Sources/PeekabooCLI/CLI/PeekabooEntryPoint.swift b/Apps/CLI/Sources/PeekabooCLI/CLI/PeekabooEntryPoint.swift index c7b5aae1b..db2fed594 100644 --- a/Apps/CLI/Sources/PeekabooCLI/CLI/PeekabooEntryPoint.swift +++ b/Apps/CLI/Sources/PeekabooCLI/CLI/PeekabooEntryPoint.swift @@ -42,7 +42,8 @@ func executePeekabooCLI(arguments: [String]) async -> Int32 { } private func containsJSONOutputFlag(_ arguments: [String]) -> Bool { - arguments.contains("--json") || arguments.contains("-j") || arguments.contains("--json-output") + arguments.contains("--json") || arguments.contains("-j") || arguments.contains("--json-output") || + arguments.contains("--jsonOutput") } private func commanderErrorMessage(_ error: CommanderProgramError) -> String { diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatEventDelegate.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatEventDelegate.swift index 4f27e664e..6f06cfede 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatEventDelegate.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatEventDelegate.swift @@ -5,6 +5,7 @@ import PeekabooAgentRuntime final class AgentChatEventDelegate: AgentEventDelegate { private weak var ui: AgentChatUI? private var lastToolArguments: [String: [String: Any]] = [:] + private(set) var hasReceivedError = false init(ui: AgentChatUI) { self.ui = ui @@ -28,6 +29,7 @@ final class AgentChatEventDelegate: AgentEventDelegate { case .verificationCompleted, .desktopContextRefreshed: break case let .error(message): + self.hasReceivedError = true ui.showError(message) case .completed: ui.finishStreaming() diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift index b6b555de2..29a0bd040 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift @@ -17,6 +17,23 @@ struct AgentChatLaunchContext { let listSessions: Bool let normalizedTaskInput: String? let capabilities: TerminalCapabilities + let hasSessionResumption: Bool + + init( + chatFlag: Bool, + hasTaskInput: Bool, + listSessions: Bool, + normalizedTaskInput: String?, + capabilities: TerminalCapabilities, + hasSessionResumption: Bool = false + ) { + self.chatFlag = chatFlag + self.hasTaskInput = hasTaskInput + self.listSessions = listSessions + self.normalizedTaskInput = normalizedTaskInput + self.capabilities = capabilities + self.hasSessionResumption = hasSessionResumption + } } /// Determines how the agent should launch chat mode based on flags and terminal context. @@ -27,7 +44,7 @@ struct AgentChatLaunchPolicy { return .interactive(initialPrompt: context.normalizedTaskInput) } - if context.hasTaskInput || context.listSessions { + if context.hasTaskInput || context.listSessions || context.hasSessionResumption { return .none } diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatUI.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatUI.swift index 8d369f23a..45ed53964 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatUI.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatUI.swift @@ -154,6 +154,12 @@ final class AgentChatUI { self.requestRender() } + func updateSessionId(_ sessionId: String) { + self.sessionId = sessionId + self.sessionLine.text = AgentChatUI.sessionDescription(for: sessionId, queueMode: self.queueMode) + self.requestRender() + } + func showToolStart(name: String, summary: String?, icon: String?, displayName: String?) { let label = displayName ?? name let detail = summary.flatMap { $0.isEmpty ? nil : $0 } diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift index 90fd605ad..e4f1139c8 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift @@ -236,7 +236,13 @@ extension AgentCommand { } catch is CancellationError { chatUI.showCancelled() } catch { - chatUI.showError(error.localizedDescription) + if let sessionId = self.stepLimitSessionId(from: error) { + activeSessionId = sessionId + chatUI.updateSessionId(sessionId) + } + if !tuiDelegate.hasReceivedError { + chatUI.showError(error.localizedDescription) + } } currentRun = nil @@ -359,7 +365,8 @@ extension AgentCommand { task: batchedInput, requestedModel: requestedModel, maxSteps: self.resolvedMaxSteps, - queueMode: queueMode + queueMode: queueMode, + preserveStepLimitError: true ) } } @@ -381,6 +388,13 @@ extension AgentCommand { } catch is CancellationError { cancelMonitor.stop() return + } catch { + cancelMonitor.stop() + if let sessionId = self.stepLimitSessionId(from: error) { + context.sessionId = sessionId + return + } + throw error } if let updatedSessionId = result.sessionId { @@ -390,6 +404,10 @@ extension AgentCommand { self.printChatTurnSummary(result) } + func stepLimitSessionId(from error: any Error) -> String? { + (error as? PeekabooAgentService.AgentStepLimitExceededError)?.sessionId + } + private func printChatTurnSummary(_ result: AgentExecutionResult) { guard !self.quiet else { return } let duration = String(format: "%.1fs", result.metadata.executionTime) diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Execution.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Execution.swift index 427212067..c2b7bfb80 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Execution.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Execution.swift @@ -112,7 +112,8 @@ extension AgentCommand { task: String, requestedModel: LanguageModel?, maxSteps: Int, - queueMode: QueueMode + queueMode: QueueMode, + preserveStepLimitError: Bool = false ) async throws -> AgentExecutionResult { let outputDelegate = self.makeDisplayDelegate(for: task) let streamingDelegate = self.makeStreamingDelegate(using: outputDelegate) @@ -139,8 +140,15 @@ extension AgentCommand { + "session=\(sessionId) tokens=\(finalTokens)" ) return result + } catch let error as PeekabooAgentService.AgentStepLimitExceededError where preserveStepLimitError { + if outputDelegate?.hasReceivedError != true { + self.printAgentExecutionError("Agent execution failed: \(error.localizedDescription)") + } + throw error } catch { - self.printAgentExecutionError("Agent execution failed: \(error.localizedDescription)") + if outputDelegate?.hasReceivedError != true { + self.printAgentExecutionError("Agent execution failed: \(error.localizedDescription)") + } throw ExitCode.failure } } @@ -159,6 +167,10 @@ extension AgentCommand { self.maxSteps ?? 100 } + func validatedMaxStepCount() throws -> Int { + try AgentStepBudget.validate(self.resolvedMaxSteps) + } + func resolvedQueueMode() throws -> QueueMode { guard let raw = self.queueMode?.trimmingCharacters(in: .whitespacesAndNewlines), !raw.isEmpty else { return .oneAtATime diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+ModelParsing.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+ModelParsing.swift index f8201348a..d547cae5b 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+ModelParsing.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+ModelParsing.swift @@ -5,6 +5,10 @@ import Tachikoma @available(macOS 14.0, *) extension AgentCommand { + func shouldUsePersistedSessionModel(requestedModel: LanguageModel?) -> Bool { + (self.resume || self.resumeSession != nil) && requestedModel == nil + } + @MainActor func parseModelString( _ modelString: String, @@ -124,6 +128,15 @@ extension AgentCommand { @MainActor func validatedModelSelection(configuration: PeekabooCore.ConfigurationManager? = nil) throws -> LanguageModel? { guard let modelString = self.model else { return nil } + + if let configuration, + let configuredModel = PeekabooAIService(configuration: configuration) + .resolveConfiguredModel(modelString), + case .custom = configuredModel, + !configuredModel.supportsTools { + throw Self.configuredCustomModelToolCapabilityError(modelString) + } + guard let parsed = self.parseModelString(modelString, configuration: configuration) else { // A model that parses but lacks tool support is a real, installed model — // saying "unsupported" and listing an allowlist implies the name is wrong. @@ -141,6 +154,46 @@ extension AgentCommand { return parsed } + @MainActor + func unavailableImplicitCustomModelToolCapabilityError( + from service: PeekabooAIService, + configuration: PeekabooCore.ConfigurationManager + ) -> PeekabooError? { + let selections: [String] + if configuration.hasExplicitAIProviderList() { + selections = configuration.getAIProviders() + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + } else if let defaultModel = configuration.getAgentModel()? + .trimmingCharacters(in: .whitespacesAndNewlines), + !defaultModel.isEmpty { + selections = [defaultModel] + } else { + return nil + } + + for selection in selections { + guard let model = service.resolveConfiguredModel(selection), + case .custom = model, + !model.supportsTools + else { + continue + } + return Self.configuredCustomModelToolCapabilityError(selection) + } + + return nil + } + + private static func configuredCustomModelToolCapabilityError(_ modelString: String) -> PeekabooError { + PeekabooError.invalidInput( + "Model '\(modelString)' is configured with supportsTools: false, but `peekaboo agent` " + + "requires tool calling. Choose a tool-capable model, or set this model's supportsTools " + + "setting to true only if the endpoint actually supports tool calls." + ) + } + private static let supportedOpenAIInputs: Set = [ .gpt56Sol, .gpt56Terra, diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Sessions.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Sessions.swift index 7056e0457..809b75e77 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Sessions.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Sessions.swift @@ -1,3 +1,4 @@ +import Commander import Foundation import PeekabooAgentRuntime import PeekabooCore @@ -233,8 +234,10 @@ extension AgentCommand { ) self.displayResult(result, delegate: outputDelegate) } catch { - self.printAgentExecutionError("Failed to resume session: \(error.localizedDescription)") - throw error + if outputDelegate?.hasReceivedError != true { + self.printAgentExecutionError("Failed to resume session: \(error.localizedDescription)") + } + throw ExitCode.failure } } } diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand.swift index 50c01d750..8d954f041 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand.swift @@ -80,7 +80,7 @@ struct AgentCommand: RuntimeOptionsConfigurable { @Flag(name: .long, help: "Dry run - show planned steps without executing") var dryRun = false - @Option(name: .long, help: "Maximum number of steps the agent can take") + @Option(name: .long, help: "Maximum model turns before failing (1-100, default 100)") var maxSteps: Int? @Option(name: .long, help: "Queue mode for queued prompts: one-at-a-time (default) or all") @@ -214,7 +214,15 @@ extension AgentCommand { mutating func runInternal(runtime: CommandRuntime) async throws { if self.isAgentDisabled() { self.emitAgentUnavailableMessage() - return + throw ExitCode.failure + } + + let maxSteps: Int + do { + maxSteps = try self.validatedMaxStepCount() + } catch { + self.printAgentExecutionError(error.localizedDescription) + throw ExitCode.failure } let services = runtime.services @@ -241,6 +249,7 @@ extension AgentCommand { configuration: services.configuration, existingAgentModel: existingAgentModel ) + let usesPersistedSessionModel = self.shouldUsePersistedSessionModel(requestedModel: requestedModel) if self.listSessions { let listingModel = selectedModel ?? existingAgentModel ?? .anthropic(.opus48) let agentService: any AgentServiceProtocol = if let existing = existingAgent { @@ -256,22 +265,32 @@ extension AgentCommand { return } - guard let selectedModel else { + if selectedModel == nil, !usesPersistedSessionModel { + if let capabilityError = self.unavailableImplicitCustomModelToolCapabilityError( + from: configuredAIService, + configuration: services.configuration + ) { + self.printAgentExecutionError(capabilityError.localizedDescription) + throw ExitCode.failure + } self.emitAgentUnavailableMessage() - return + throw ExitCode.failure } - guard self.hasCredentials(for: selectedModel) || self.isLocalModel(selectedModel) else { + let serviceDefaultModel = selectedModel ?? existingAgentModel ?? .anthropic(.opus48) + if !usesPersistedSessionModel, + !self.hasCredentials(for: serviceDefaultModel), + !self.isLocalModel(serviceDefaultModel) { if requestedModel != nil { - let providerName = self.providerDisplayName(for: selectedModel) - let envVar = self.providerEnvironmentVariable(for: selectedModel) + let providerName = self.providerDisplayName(for: serviceDefaultModel) + let envVar = self.providerEnvironmentVariable(for: serviceDefaultModel) self.printAgentExecutionError( "Missing API key for \(providerName). Set \(envVar) and retry." ) } else { self.emitAgentUnavailableMessage() } - return + throw ExitCode.failure } let agentService: any AgentServiceProtocol = if let existing = existingAgent { @@ -279,7 +298,7 @@ extension AgentCommand { } else { try PeekabooAgentService( services: services, - defaultModel: selectedModel, + defaultModel: serviceDefaultModel, snapshotMutationCoordinator: mutationCoordinator ) } @@ -296,8 +315,10 @@ extension AgentCommand { throw PeekabooError.commandFailed("Agent service not properly initialized") } - guard self.ensureAgentHasCredentials(selectedModel: selectedModel) else { - return + if !usesPersistedSessionModel { + guard self.ensureAgentHasCredentials(selectedModel: serviceDefaultModel) else { + throw ExitCode.failure + } } let chatPolicy = AgentChatLaunchPolicy() @@ -306,7 +327,8 @@ extension AgentCommand { hasTaskInput: self.hasTaskInput, listSessions: self.listSessions, normalizedTaskInput: self.normalizedTaskInput, - capabilities: terminalCapabilities + capabilities: terminalCapabilities, + hasSessionResumption: self.resume || self.resumeSession != nil ) let queueMode: QueueMode @@ -337,7 +359,7 @@ extension AgentCommand { if try await self.handleSessionResumption( peekabooAgent, requestedModel: requestedModel, - maxSteps: self.maxSteps ?? 100, + maxSteps: maxSteps, queueMode: queueMode ) { return @@ -351,7 +373,7 @@ extension AgentCommand { peekabooAgent, task: executionTask, requestedModel: requestedModel, - maxSteps: self.maxSteps ?? 100, + maxSteps: maxSteps, queueMode: queueMode ) } diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentOutputDelegate.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentOutputDelegate.swift index 1cd3802fa..0c8ca05ed 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentOutputDelegate.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentOutputDelegate.swift @@ -29,6 +29,7 @@ final class AgentOutputDelegate: PeekabooCore.AgentEventDelegate { private var hasReceivedContent = false private var isThinking = false private var hasShownFinalSummary = false + private(set) var hasReceivedError = false private let startTime = Date() // MARK: - Initialization @@ -70,6 +71,7 @@ extension AgentOutputDelegate { break case let .error(message): + self.hasReceivedError = true self.handleError(message) case let .completed(summary, usage): diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/Core/ConfigCommand+ProviderManagement.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/Core/ConfigCommand+ProviderManagement.swift index 6626bd85f..b3819dec2 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/Core/ConfigCommand+ProviderManagement.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/Core/ConfigCommand+ProviderManagement.swift @@ -295,29 +295,33 @@ extension ConfigCommand { throw ExitCode.failure } - let manager = self.configManager - let providerId = self.providerId - let modelResult: Result<(models: [String], error: String?), TimeoutError> = await withTimeout( - ConfigCommandTimeouts.network - ) { - await manager.discoverModelsForCustomProvider(id: providerId) - } - let models: [String] let apiError: String? - switch modelResult { - case .failure: - models = [] - apiError = "Model discovery timed out" - case let .success(tuple): - if self.discover && provider.type == .openai { + if self.discover && provider.type == .openai { + let manager = self.configManager + let providerId = self.providerId + let modelResult: Result<(models: [String], error: String?), TimeoutError> = await withTimeout( + ConfigCommandTimeouts.network + ) { + await manager.discoverModelsForCustomProvider(id: providerId) + } + switch modelResult { + case .failure: + models = [] + apiError = "Model discovery timed out" + case let .success(tuple): models = tuple.models apiError = tuple.error - } else { - models = provider.models?.keys.map { String($0) } ?? [] - apiError = tuple.error } + } else { + models = provider.models?.keys.map { String($0) } ?? [] + apiError = nil + } + + let saved = self.save && apiError == nil + if saved { + try self.saveModels(models, for: self.providerId, existing: provider) } if self.jsonOutput { @@ -325,10 +329,14 @@ extension ConfigCommand { "providerId": providerId, "models": models, "source": discover && provider.type == .openai ? "api" : "configuration", - "error": apiError as Any + "error": apiError as Any, + "saved": saved ] let output = SuccessOutput(success: apiError == nil, data: data) outputJSON(output, logger: self.logger) + if apiError != nil { + throw ExitCode.failure + } return } @@ -359,8 +367,11 @@ extension ConfigCommand { } } - if self.save, apiError == nil { - try self.saveModels(models, for: providerId, existing: provider) + if saved { + print("[ok] Saved \(models.count) model(s) to configuration") + } + if apiError != nil { + throw ExitCode.failure } } @@ -384,7 +395,9 @@ extension ConfigCommand { existing provider: Configuration.CustomProvider ) throws { let modelDefinitions = Dictionary( - uniqueKeysWithValues: models.map { ($0, Configuration.ModelDefinition(name: $0)) } + uniqueKeysWithValues: models.map { modelID in + (modelID, provider.models?[modelID] ?? Configuration.ModelDefinition(name: modelID)) + } ) let updated = Configuration.CustomProvider( name: provider.name, @@ -395,7 +408,6 @@ extension ConfigCommand { enabled: provider.enabled ) try self.configManager.addCustomProvider(updated, id: providerId) - print("[ok] Saved \(models.count) model(s) to configuration") } } } diff --git a/Apps/CLI/Tests/CLIAutomationTests/AgentCommandBasicTests.swift b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandBasicTests.swift index 6be2add57..5a8bd8653 100644 --- a/Apps/CLI/Tests/CLIAutomationTests/AgentCommandBasicTests.swift +++ b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandBasicTests.swift @@ -19,5 +19,6 @@ struct AgentCommandBasicTests { #expect(result.exitStatus == 0) #expect(result.combinedOutput.contains("gpt-5.6")) #expect(result.combinedOutput.contains("claude-sonnet-5")) + #expect(result.combinedOutput.contains("Maximum model turns before failing (1-100, default 100)")) } } diff --git a/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift new file mode 100644 index 000000000..7045f8883 --- /dev/null +++ b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift @@ -0,0 +1,45 @@ +import Foundation +import Testing +@testable import PeekabooCLI + +@Suite(.serialized, .tags(.safe)) +struct AgentCommandValidationIntegrationTests { + @Test(arguments: [0, 101]) + func `Invalid max steps returns one JSON error`(_ maxSteps: Int) async throws { + let result = try await InProcessCommandRunner.runShared( + ["agent", "test task", "--max-steps", String(maxSteps), "--json"], + allowedExitCodes: [1] + ) + + #expect(result.exitStatus == 1) + + let data = try #require(result.stdout.trimmingCharacters(in: .whitespacesAndNewlines).data(using: .utf8)) + let payload = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + #expect(payload.count == 2) + #expect(payload["success"] as? Bool == false) + let message = try #require(payload["error"] as? String) + #expect(message.contains("between 1 and 100")) + #expect(message.contains("received \(maxSteps)")) + } + + @Test + func `Missing resume session returns one JSON error`() async throws { + let result = try await InProcessCommandRunner.runShared( + [ + "agent", + "--resume-session", + "missing-session-\(UUID().uuidString)", + "--jsonOutput", + ], + allowedExitCodes: [1] + ) + + #expect(result.exitStatus == 1) + + let data = try #require(result.stdout.trimmingCharacters(in: .whitespacesAndNewlines).data(using: .utf8)) + let payload = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + #expect(payload.count == 2) + #expect(payload["success"] as? Bool == false) + #expect((payload["error"] as? String)?.contains("Failed to resume session") == true) + } +} diff --git a/Apps/CLI/Tests/CLIRuntimeTests/CLIRuntimeSmokeTests.swift b/Apps/CLI/Tests/CLIRuntimeTests/CLIRuntimeSmokeTests.swift index 581d6a8ff..0e521b832 100644 --- a/Apps/CLI/Tests/CLIRuntimeTests/CLIRuntimeSmokeTests.swift +++ b/Apps/CLI/Tests/CLIRuntimeTests/CLIRuntimeSmokeTests.swift @@ -388,14 +388,14 @@ struct CLIRuntimeSmokeTests { } @Test - func `peekaboo agent warns when no provider credentials exist`() async throws { + func `peekaboo agent fails when no provider credentials exist`() async throws { guard Self.ensureLocalRuntimeAvailable() else { return } let result = try await TestChildProcess.runPeekaboo([ "agent", "list files", "--dry-run" ], environment: ["PEEKABOO_DISABLE_AGENT": "1", "PEEKABOO_NO_REMOTE": "1"]) - #expect(result.status == .exited(0)) + #expect(result.status == .exited(1)) #expect(result.standardOutput.contains("Agent service not available")) } diff --git a/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift b/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift index f96681256..c1fd8e8e2 100644 --- a/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift +++ b/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift @@ -97,4 +97,20 @@ struct AgentChatLaunchPolicyTests { #expect(ci == .helpOnly) } + + @Test + func `Taskless session resume bypasses noninteractive help`() { + let strategy = self.policy.strategy( + for: AgentChatLaunchContext( + chatFlag: false, + hasTaskInput: false, + listSessions: false, + normalizedTaskInput: nil, + capabilities: self.makeCaps(interactive: false, piped: true), + hasSessionResumption: true + ) + ) + + #expect(strategy == .none) + } } diff --git a/Apps/CLI/Tests/CoreCLITests/AgentCommandModelParsingTests.swift b/Apps/CLI/Tests/CoreCLITests/AgentCommandModelParsingTests.swift index 066c4556c..6359c6fce 100644 --- a/Apps/CLI/Tests/CoreCLITests/AgentCommandModelParsingTests.swift +++ b/Apps/CLI/Tests/CoreCLITests/AgentCommandModelParsingTests.swift @@ -8,6 +8,32 @@ import Testing @Suite(.tags(.safe)) @MainActor struct AgentCommandTests { + @Test + func `Resume without model uses persisted session selection`() throws { + let latest = try AgentCommand.parse(["continue", "--resume"]) + let specific = try AgentCommand.parse([ + "continue", + "--resume-session", + "session-id", + ]) + + #expect(latest.shouldUsePersistedSessionModel(requestedModel: nil)) + #expect(specific.shouldUsePersistedSessionModel(requestedModel: nil)) + } + + @Test + func `Explicit resume model still requires override preflight`() throws { + let command = try AgentCommand.parse([ + "continue", + "--resume-session", + "session-id", + "--model", + "ollama/qwen3.5:9b", + ]) + + #expect(!command.shouldUsePersistedSessionModel(requestedModel: .ollama(.custom("qwen3.5:9b")))) + } + @Test func `Supported OpenAI aliases map to GPT-5.5`() throws { let command = try AgentCommand.parse([]) @@ -110,6 +136,52 @@ struct AgentCommandTests { } } + @Test + @MainActor + func `Configured custom model with tools disabled gives safe agent guidance`() throws { + try self.withIsolatedConfiguration( + """ + { + "customProviders": { + "local-proxy": { + "name": "Local Proxy", + "type": "openai", + "enabled": true, + "options": { + "baseURL": "http://localhost:8317/v1", + "apiKey": "test-key" + }, + "models": { + "text-only": { + "name": "Text Only", + "supportsVision": false, + "supportsTools": false + } + } + } + } + } + """, + environment: ["PEEKABOO_CUSTOM_PROVIDER_KEY": "resolved-secret"] + ) { + var command = try AgentCommand.parse([]) + command.model = "local-proxy/text-only" + + do { + _ = try command.validatedModelSelection( + configuration: PeekabooCore.ConfigurationManager.shared + ) + Issue.record("expected a validation error for a configured tool-incapable model") + } catch let error as PeekabooError { + let message = error.localizedDescription + #expect(message.contains("configured with supportsTools: false")) + #expect(message.contains("requires tool calling")) + #expect(!message.contains("Allowed values")) + #expect(!message.contains("--analyze")) + } + } + } + @Test func `Current Gemini models are accepted`() throws { let command = try AgentCommand.parse([]) @@ -306,6 +378,105 @@ struct AgentCommandTests { } } + @Test + func `Implicit custom default with tools disabled gives safe agent guidance`() throws { + try self.withIsolatedConfiguration( + """ + { + "agent": { + "defaultModel": "local-proxy/text-only" + }, + "customProviders": { + "local-proxy": { + "name": "Local Proxy", + "type": "openai", + "enabled": true, + "options": { + "baseURL": "http://localhost:8317/v1", + "apiKey": "test-key" + }, + "models": { + "text-only": { + "name": "Text Only", + "supportsTools": false, + "supportsVision": false + } + } + } + } + } + """, + environment: ["PEEKABOO_CUSTOM_PROVIDER_KEY": "resolved-secret"] + ) { + let command = try AgentCommand.parse([]) + let configuration = PeekabooCore.ConfigurationManager.shared + let service = PeekabooAIService(configuration: configuration) + + #expect(command + .implicitToolModel(from: service, configuration: configuration, existingAgentModel: nil) == nil) + let error = try #require(command.unavailableImplicitCustomModelToolCapabilityError( + from: service, + configuration: configuration + )) + let message = error.localizedDescription + #expect(message.contains("local-proxy/text-only")) + #expect(message.contains("configured with supportsTools: false")) + #expect(message.contains("requires tool calling")) + #expect(!message.contains("Allowed values")) + #expect(!message.contains("--analyze")) + } + } + + @Test + func `Explicit custom provider list with tools disabled gives safe agent guidance`() throws { + try self.withIsolatedConfiguration( + """ + { + "aiProviders": { + "providers": "local-proxy/text-only" + }, + "customProviders": { + "local-proxy": { + "name": "Local Proxy", + "type": "openai", + "enabled": true, + "options": { + "baseURL": "http://localhost:8317/v1", + "apiKey": "test-key" + }, + "models": { + "text-only": { + "name": "Text Only", + "supportsTools": false, + "supportsVision": false + } + } + } + } + } + """, + environment: ["PEEKABOO_CUSTOM_PROVIDER_KEY": "resolved-secret"] + ) { + let command = try AgentCommand.parse([]) + let configuration = PeekabooCore.ConfigurationManager.shared + let service = PeekabooAIService(configuration: configuration) + + #expect(configuration.hasExplicitAIProviderList()) + #expect(command + .implicitToolModel(from: service, configuration: configuration, existingAgentModel: nil) == nil) + let error = try #require(command.unavailableImplicitCustomModelToolCapabilityError( + from: service, + configuration: configuration + )) + let message = error.localizedDescription + #expect(message.contains("local-proxy/text-only")) + #expect(message.contains("configured with supportsTools: false")) + #expect(message.contains("requires tool calling")) + #expect(!message.contains("Allowed values")) + #expect(!message.contains("--analyze")) + } + } + @Test func `Implicit selection does not escape explicit provider lists`() throws { try self.withIsolatedConfiguration( diff --git a/Apps/CLI/Tests/CoreCLITests/AgentCommandStepLimitTests.swift b/Apps/CLI/Tests/CoreCLITests/AgentCommandStepLimitTests.swift new file mode 100644 index 000000000..3656656ff --- /dev/null +++ b/Apps/CLI/Tests/CoreCLITests/AgentCommandStepLimitTests.swift @@ -0,0 +1,50 @@ +import Foundation +import PeekabooAgentRuntime +import PeekabooFoundation +import Testing +@testable import PeekabooCLI + +@Suite(.tags(.safe)) +struct AgentCommandStepLimitTests { + @Test(arguments: [1, 100]) + func `Agent accepts supported step limits`(_ maxSteps: Int) throws { + var command = try AgentCommand.parse([]) + command.maxSteps = maxSteps + + #expect(try command.validatedMaxStepCount() == maxSteps) + } + + @Test(arguments: [-1, 0, 101]) + func `Agent rejects unsupported step limits`(_ maxSteps: Int) throws { + var command = try AgentCommand.parse([]) + command.maxSteps = maxSteps + + let error = #expect(throws: PeekabooError.self) { + try command.validatedMaxStepCount() + } + + if case let .invalidInput(message) = error { + #expect(message.contains("between 1 and 100")) + #expect(message.contains("received \(maxSteps)")) + } else { + Issue.record("Expected invalidInput error") + } + } + + @Test + func `Agent defaults to maximum supported step limit`() throws { + let command = try AgentCommand.parse([]) + + #expect(try command.validatedMaxStepCount() == 100) + } + + @Test + func `Chat recovers the saved session from step exhaustion`() throws { + let command = try AgentCommand.parse([]) + let sessionId = UUID().uuidString + let error = PeekabooAgentService.AgentStepLimitExceededError(maxSteps: 1, sessionId: sessionId) + + #expect(command.stepLimitSessionId(from: error) == sessionId) + #expect(command.stepLimitSessionId(from: PeekabooError.commandFailed("other")) == nil) + } +} diff --git a/Apps/CLI/Tests/CoreCLITests/ConfigModelsProviderTests.swift b/Apps/CLI/Tests/CoreCLITests/ConfigModelsProviderTests.swift new file mode 100644 index 000000000..78645edd1 --- /dev/null +++ b/Apps/CLI/Tests/CoreCLITests/ConfigModelsProviderTests.swift @@ -0,0 +1,355 @@ +import Commander +import Foundation +import Network +import PeekabooCore +import Testing +@testable import PeekabooCLI + +@Suite(.tags(.safe), .serialized) +struct ConfigModelsProviderTests { + @Test + func `Saving provider models preserves configured model capabilities`() async throws { + try await self.withTempConfigDir { + let model = Configuration.ModelDefinition( + name: "Configured Model", + maxTokens: 32768, + supportsTools: false, + supportsVision: false, + parameters: ["reasoning": "high"] + ) + let provider = Configuration.CustomProvider( + name: "Anthropic Proxy", + type: .anthropic, + options: .init( + baseURL: "https://api.example.com/v1", + apiKey: "test-key" + ), + models: ["configured-model": model] + ) + try PeekabooCore.ConfigurationManager.shared.addCustomProvider(provider, id: "anthropic-proxy") + + var command = ConfigCommand.ModelsProviderCommand() + command.providerId = "anthropic-proxy" + command.save = true + try await command.run(using: self.makeRuntime()) + + let saved = try #require( + PeekabooCore.ConfigurationManager.shared + .getCustomProvider(id: "anthropic-proxy")? + .models?["configured-model"] + ) + #expect(saved.name == "Configured Model") + #expect(saved.maxTokens == 32768) + #expect(saved.supportsTools == false) + #expect(saved.supportsVision == false) + #expect(saved.parameters == ["reasoning": "high"]) + } + } + + @Test + func `Save without discover does not call the provider API`() async throws { + try await self.withTempConfigDir { + let server = try await ModelsProviderHTTPServer.start( + statusCode: 500, + body: #"{"error":"must not be called"}"# + ) + defer { server.stop() } + + let provider = Configuration.CustomProvider( + name: "OpenAI Proxy", + type: .openai, + options: .init(baseURL: server.baseURL, apiKey: "test-key"), + models: ["configured-model": .init(name: "Configured Model", supportsTools: false)] + ) + try PeekabooCore.ConfigurationManager.shared.addCustomProvider(provider, id: "openai-proxy") + + var command = ConfigCommand.ModelsProviderCommand() + command.providerId = "openai-proxy" + command.save = true + let output = try await captureStandardOutput { + try await command.run(using: self.makeRuntime(jsonOutput: true)) + } + + let response = try JSONDecoder().decode( + ModelsProviderJSONResponse.self, + from: Data(output.utf8) + ) + #expect(response.success) + #expect(response.data.saved) + #expect(response.data.models == ["configured-model"]) + + let saved = try #require( + PeekabooCore.ConfigurationManager.shared + .getCustomProvider(id: "openai-proxy")? + .models?["configured-model"] + ) + #expect(saved.name == "Configured Model") + #expect(saved.supportsTools == false) + } + } + + @Test + func `Discover save JSON preserves metadata and emits one JSON document`() async throws { + try await self.withTempConfigDir { + let server = try await ModelsProviderHTTPServer.start( + statusCode: 200, + body: #"{"data":[{"id":"configured-model"},{"id":"new-model"}]}"# + ) + defer { server.stop() } + + let model = Configuration.ModelDefinition( + name: "Configured Model", + maxTokens: 32768, + supportsTools: false, + supportsVision: false, + parameters: ["reasoning": "high"] + ) + let provider = Configuration.CustomProvider( + name: "OpenAI Proxy", + type: .openai, + options: .init(baseURL: server.baseURL, apiKey: "test-key"), + models: ["configured-model": model] + ) + try PeekabooCore.ConfigurationManager.shared.addCustomProvider(provider, id: "openai-proxy") + + var command = ConfigCommand.ModelsProviderCommand() + command.providerId = "openai-proxy" + command.discover = true + command.save = true + let output = try await captureStandardOutput { + try await command.run(using: self.makeRuntime(jsonOutput: true)) + } + + let response = try JSONDecoder().decode( + ModelsProviderJSONResponse.self, + from: Data(output.utf8) + ) + #expect(response.success, "Command output: \(output)") + #expect(response.data.saved) + #expect(Set(response.data.models) == ["configured-model", "new-model"]) + #expect(!output.contains("[ok]")) + + let savedProvider = try #require( + PeekabooCore.ConfigurationManager.shared.getCustomProvider(id: "openai-proxy") + ) + let savedExisting = try #require(savedProvider.models?["configured-model"]) + #expect(savedExisting.name == "Configured Model") + #expect(savedExisting.maxTokens == 32768) + #expect(savedExisting.supportsTools == false) + #expect(savedExisting.supportsVision == false) + #expect(savedExisting.parameters == ["reasoning": "high"]) + + let savedNew = try #require(savedProvider.models?["new-model"]) + #expect(savedNew.name == "new-model") + #expect(savedNew.maxTokens == nil) + #expect(savedNew.supportsTools == nil) + #expect(savedNew.supportsVision == nil) + #expect(savedNew.parameters == nil) + } + } + + @Test + func `Discover API error never saves models`() async throws { + try await self.withTempConfigDir { + let server = try await ModelsProviderHTTPServer.start( + statusCode: 500, + body: #"{"error":"mock failure"}"# + ) + defer { server.stop() } + + let model = Configuration.ModelDefinition( + name: "Configured Model", + maxTokens: 32768, + supportsTools: false, + parameters: ["reasoning": "high"] + ) + let provider = Configuration.CustomProvider( + name: "OpenAI Proxy", + type: .openai, + options: .init(baseURL: server.baseURL, apiKey: "test-key"), + models: ["configured-model": model] + ) + try PeekabooCore.ConfigurationManager.shared.addCustomProvider(provider, id: "openai-proxy") + + var command = ConfigCommand.ModelsProviderCommand() + command.providerId = "openai-proxy" + command.discover = true + command.save = true + let output = try await captureStandardOutput { + let exitCode = await #expect(throws: ExitCode.self) { + try await command.run(using: self.makeRuntime(jsonOutput: true)) + } + #expect(exitCode == .failure) + } + + let response = try JSONDecoder().decode( + ModelsProviderJSONResponse.self, + from: Data(output.utf8) + ) + #expect(!response.success) + #expect(!response.data.saved) + + let savedProvider = try #require( + PeekabooCore.ConfigurationManager.shared.getCustomProvider(id: "openai-proxy") + ) + #expect(savedProvider.models?.count == 1) + let unchanged = try #require(savedProvider.models?["configured-model"]) + #expect(unchanged.name == "Configured Model") + #expect(unchanged.maxTokens == 32768) + #expect(unchanged.supportsTools == false) + #expect(unchanged.parameters == ["reasoning": "high"]) + } + } + + private func makeRuntime(jsonOutput: Bool = false) -> CommandRuntime { + CommandRuntime( + configuration: .init(verbose: false, jsonOutput: jsonOutput, logLevel: nil), + services: PeekabooServices() + ) + } + + private func captureStandardOutput(_ body: () async throws -> Void) async throws -> String { + let pipe = Pipe() + let originalStandardOutput = dup(STDOUT_FILENO) + guard originalStandardOutput >= 0 else { + throw CocoaError(.fileWriteUnknown) + } + guard dup2(pipe.fileHandleForWriting.fileDescriptor, STDOUT_FILENO) >= 0 else { + close(originalStandardOutput) + throw CocoaError(.fileWriteUnknown) + } + pipe.fileHandleForWriting.closeFile() + + do { + try await body() + fflush(nil) + _ = dup2(originalStandardOutput, STDOUT_FILENO) + close(originalStandardOutput) + let data = pipe.fileHandleForReading.readDataToEndOfFile() + pipe.fileHandleForReading.closeFile() + return String(data: data, encoding: .utf8) ?? "" + } catch { + fflush(nil) + _ = dup2(originalStandardOutput, STDOUT_FILENO) + close(originalStandardOutput) + pipe.fileHandleForReading.closeFile() + throw error + } + } + + private func withTempConfigDir(_ body: () async throws -> Void) async throws { + let tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent("peekaboo-cli-config-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + + let environmentKeys = [ + "PEEKABOO_CONFIG_DIR", + "PEEKABOO_CONFIG_NONINTERACTIVE", + "PEEKABOO_CONFIG_DISABLE_MIGRATION", + ] + let previous = Dictionary(uniqueKeysWithValues: environmentKeys.map { key in + (key, getenv(key).map { String(cString: $0) }) + }) + + setenv("PEEKABOO_CONFIG_DIR", tempDir.path, 1) + setenv("PEEKABOO_CONFIG_NONINTERACTIVE", "1", 1) + setenv("PEEKABOO_CONFIG_DISABLE_MIGRATION", "1", 1) + PeekabooCore.ConfigurationManager.shared.resetForTesting() + + defer { + for key in environmentKeys { + if case let value?? = previous[key] { + setenv(key, value, 1) + } else { + unsetenv(key) + } + } + PeekabooCore.ConfigurationManager.shared.resetForTesting() + try? FileManager.default.removeItem(at: tempDir) + } + + try await body() + } +} + +private struct ModelsProviderJSONResponse: Decodable { + let success: Bool + let data: ModelsProviderJSONData +} + +private struct ModelsProviderJSONData: Decodable { + let models: [String] + let saved: Bool +} + +private final class ModelsProviderHTTPServer { + private let listener: NWListener + + private init(listener: NWListener) { + self.listener = listener + } + + var baseURL: String { + guard let port = listener.port else { + preconditionFailure("HTTP test server must be ready before use") + } + return "http://127.0.0.1:\(port.rawValue)" + } + + static func start(statusCode: Int, body: String) async throws -> ModelsProviderHTTPServer { + let listener = try NWListener(using: .tcp, on: .any) + let queue = DispatchQueue(label: "peekaboo.tests.models-provider-http") + let response = response(statusCode: statusCode, body: body) + + listener.newConnectionHandler = { connection in + connection.start(queue: queue) + connection.receive(minimumIncompleteLength: 1, maximumLength: 64 * 1024) { _, _, _, _ in + connection.send( + content: response, + contentContext: .finalMessage, + isComplete: true, + completion: .contentProcessed { _ in + connection.cancel() + } + ) + } + } + + try await withCheckedThrowingContinuation { continuation in + listener.stateUpdateHandler = { state in + switch state { + case .ready: + listener.stateUpdateHandler = nil + continuation.resume() + case let .failed(error): + listener.stateUpdateHandler = nil + continuation.resume(throwing: error) + case .cancelled: + listener.stateUpdateHandler = nil + continuation.resume(throwing: CancellationError()) + default: + break + } + } + listener.start(queue: queue) + } + + return ModelsProviderHTTPServer(listener: listener) + } + + func stop() { + self.listener.cancel() + } + + private static func response(statusCode: Int, body: String) -> Data { + let reason = statusCode == 200 ? "OK" : "Internal Server Error" + let bodyData = Data(body.utf8) + let headers = "HTTP/1.1 \(statusCode) \(reason)\r\n" + + "Content-Type: application/json\r\n" + + "Content-Length: \(bodyData.count)\r\n" + + "Connection: close\r\n\r\n" + var data = Data(headers.utf8) + data.append(bodyData) + return data + } +} From a8cbe830b5311b9096de4ede39f813fdc939013b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:07:59 +0100 Subject: [PATCH 05/11] docs(agent): document Ollama execution guarantees --- Apps/CLI/CHANGELOG.md | 3 + CHANGELOG.md | 3 + README.md | 2 +- docs/MCP.md | 3 +- docs/commands/agent.md | 8 +- docs/configuration.md | 6 +- docs/providers.md | 27 +- docs/providers/README.md | 5 +- docs/providers/ollama-models.md | 157 ++----- docs/providers/ollama.md | 780 ++++---------------------------- 10 files changed, 173 insertions(+), 821 deletions(-) diff --git a/Apps/CLI/CHANGELOG.md b/Apps/CLI/CHANGELOG.md index 8cdf1400b..26f1d7b13 100644 --- a/Apps/CLI/CHANGELOG.md +++ b/Apps/CLI/CHANGELOG.md @@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `peekaboo agent --model` now accepts GPT-5.6 Sol, Terra, and Luna (`gpt-5.6` selects Sol) plus Claude Sonnet 5. ### Fixed +- Resuming an agent session without `--model` now preserves its credential-free provider-qualified model selection instead of silently switching to the current default; ambiguous legacy sessions fail closed and require an explicit override. +- Multi-step Ollama agent runs now preserve native tool-call history and recursive schemas, surface streamed server errors, and fail with a resumable saved session when pending tool work exhausts the validated `1...100` step budget. +- Custom-provider models marked `supportsTools: false` now get actionable agent guidance, and `config models-provider --save` preserves existing model capabilities, limits, and parameters in both human and JSON modes. - OpenRouter, Together, and OpenAI-compatible GPT-5.6 routes now preserve the 372K context/128K output capability profile, omit unsupported temperature, and recognize routing suffixes such as `:online`. - Adding a macOS application bundle to the Dock now places it with applications instead of mistaking its on-disk directory for a folder. - Bare `peekaboo paste` now pastes the current clipboard, while payload-only flags without a payload fail validation even when `--restore-delay-ms` explicitly uses its 150ms default; `list apps` also accepts the `app list` visibility flags and emits preferred snake_case keys alongside legacy keys. diff --git a/CHANGELOG.md b/CHANGELOG.md index 735389bbd..b7e5651cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ ### Fixed - The Mac app's status bar menu now follows the system light/dark mode. It previously inherited the menu bar's wallpaper-derived vibrant appearance, which could render a dark menu while the system was in light mode (and vice versa). +- Resuming an agent session without an explicit model now preserves its credential-free provider-qualified model selection instead of silently switching to the current default and potentially sending saved context to a different provider; ambiguous legacy sessions fail closed and require an explicit override. +- Multi-step Ollama agent runs now replay assistant tool calls and named results, preserve recursive arguments and array schemas, surface HTTP-200 stream errors, and fail with a resumable saved session instead of claiming success when pending tool work exhausts the step budget. +- Custom-provider models marked `supportsTools: false` now get actionable agent guidance, and `config models-provider --save` preserves existing capabilities, limits, and parameters instead of resetting model definitions, including in JSON mode. - OpenRouter, Together, and OpenAI-compatible GPT-5.6 routes now preserve the 372K context/128K output capability profile, omit unsupported temperature, and recognize routing suffixes such as `:online`. - Adding a macOS application bundle to the Dock now places it with applications instead of mistaking its on-disk directory for a folder. - Synchronous default MCP tool-context access now fails fast off the main thread, with an async main-actor accessor for background callers. Thanks @SebTardif for #253. diff --git a/README.md b/README.md index d6e5e2bc9..40af98e0e 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ macOS Settings UI reads and writes the same values. - Testing guide: [docs/testing/tools.md](docs/testing/tools.md) - MCP setup: [docs/commands/mcp.md](docs/commands/mcp.md) - Permissions: [docs/permissions.md](docs/permissions.md) -- Ollama/local models: [docs/ollama.md](docs/ollama.md) +- Ollama/local models: [docs/providers/ollama.md](docs/providers/ollama.md) - Agent chat loop: [docs/agent-chat.md](docs/agent-chat.md) - Service API reference: [docs/service-api-reference.md](docs/service-api-reference.md) diff --git a/docs/MCP.md b/docs/MCP.md index 654966225..1ce94323a 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -88,7 +88,8 @@ Common environment variables: - `OPENAI_API_KEY`: OpenAI API key for GPT models. - `ANTHROPIC_API_KEY`: Anthropic API key for Claude models. - `X_AI_API_KEY` or `XAI_API_KEY`: xAI API key for Grok models. -- `PEEKABOO_OLLAMA_BASE_URL`: Ollama server URL, defaults to `http://localhost:11434`. +- `PEEKABOO_OLLAMA_BASE_URL` / `OLLAMA_BASE_URL`: native Ollama server base. The Peekaboo-specific variable wins, + then the Ollama variable, config, and finally `http://localhost:11434`; do not append `/v1`. ## Verify client setup diff --git a/docs/commands/agent.md b/docs/commands/agent.md index d9e8c288e..3ee83c8dc 100644 --- a/docs/commands/agent.md +++ b/docs/commands/agent.md @@ -15,7 +15,7 @@ read_when: | `[task]` | Optional free-form task description. Required unless you pass `--resume`/`--resume-session`. | | `--chat` | Force the interactive chat loop even when stdin/stdout are not TTYs. | | `--dry-run` | Emit the planned steps without actually invoking tools. | -| `--max-steps ` | Cap how many tool invocations the agent may issue before aborting (default: 100). | +| `--max-steps ` | Cap model turns to `1...100` (default: 100). One turn may contain multiple tool calls. | | `--model gpt-5.6|gpt-5.5|claude-fable-5|claude-sonnet-5|gemini-3-flash|minimax|minimax-cn/|openrouter//|ollama/|lmstudio/` | Override the default model (`gpt-5.5`). Input is validated against supported hosted providers and local model providers. | | `--resume` / `--resume-session ` | Continue the most recent session or a specific session ID. | | `--list-sessions` | Print cached sessions (id, task, timestamps, message count) instead of running anything. | @@ -31,6 +31,12 @@ read_when: - Audio flags wire into Tachikoma’s audio stack: `--audio` opens the microphone, `--audio-file` loads a WAV/CAF file, and `--realtime` enables low-latency streaming (OpenAI-only). - Generation uses `agent.temperature` and `agent.maxTokens` from the shared config written by the macOS Settings UI. Token requests are capped to model capability; unsupported temperature controls are omitted automatically. +- A run saves its session and fails when its final permitted turn still requests tools and therefore needs another + model turn to interpret their results. This avoids reporting an empty success when the step budget expires with + pending work; resume the reported session to continue. +- Native `ollama/` runs replay each assistant tool call and named tool result on the next model turn. Ollama + support is model-dependent, and current native stream output is response-buffered rather than token-live. See the + [Ollama guide](../providers/ollama.md). ## Chat mode diff --git a/docs/configuration.md b/docs/configuration.md index 3bf08a07d..fd8564ec6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -33,7 +33,7 @@ Peekaboo resolves settings in this order (highest → lowest): | MiniMax API Key | credentials file | `MINIMAX_API_KEY` | Required for MiniMax international; also works as fallback for MiniMax China. | | MiniMax China API Key | credentials file | `MINIMAX_CN_API_KEY` | Optional China-specific key for `minimax-cn/...` models. | | Kimi API Key | credentials file | `MOONSHOT_API_KEY` / `KIMI_API_KEY` | Required for Kimi models; `MOONSHOT_API_KEY` takes precedence. | -| Ollama URL | `aiProviders.ollamaBaseUrl` | `PEEKABOO_OLLAMA_BASE_URL` | Base URL for local/remote Ollama (default `http://localhost:11434`). | +| Ollama URL | `aiProviders.ollamaBaseUrl` | `PEEKABOO_OLLAMA_BASE_URL`, `OLLAMA_BASE_URL` | Native Ollama server base; precedence is Peekaboo env, Ollama env, config, then `http://localhost:11434`. Do not append `/v1`. | | Default Save Path | `defaults.savePath` | `PEEKABOO_DEFAULT_SAVE_PATH` | Directory for screenshots (supports `~`). | | Log Level | `logging.level` | `PEEKABOO_LOG_LEVEL` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` (default `info`). | | Log Path | `logging.path` | `PEEKABOO_LOG_FILE` | Custom log destination (default `/tmp/peekaboo-mcp.log` for MCP; CLI uses stderr). | @@ -55,7 +55,9 @@ Peekaboo resolves settings in this order (highest → lowest): - `PEEKABOO_AI_PROVIDERS`: `provider/model` CSV. Example: `openai/gpt-5.5,anthropic/claude-opus-4-8,grok/grok-4.3,ollama/llava:latest`. - `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GROK_API_KEY` | `X_AI_API_KEY` | `XAI_API_KEY`, `GEMINI_API_KEY`, `MINIMAX_API_KEY`, `MINIMAX_CN_API_KEY`, `MOONSHOT_API_KEY` | `KIMI_API_KEY`: required for their respective providers when using API keys. -- `PEEKABOO_OLLAMA_BASE_URL`: change when your Ollama daemon isn’t on `localhost:11434`. +- Ollama native endpoint precedence: `PEEKABOO_OLLAMA_BASE_URL` > `OLLAMA_BASE_URL` > + `aiProviders.ollamaBaseUrl` > `http://localhost:11434`. Set the server base without `/api/chat` or `/v1`; see the + [Ollama provider guide](providers/ollama.md#endpoint-selection). ## Defaults & Paths diff --git a/docs/providers.md b/docs/providers.md index df0c86b6c..46f8e6091 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -26,10 +26,10 @@ pages instead of duplicating provider lists in multiple places. | **MiniMax China** | MiniMax-M3, MiniMax-M2.7, MiniMax-M2.7-highspeed | `MINIMAX_CN_API_KEY` or `MINIMAX_API_KEY` | | **Kimi** | kimi-k2.6, kimi-k2.7-code, kimi-k2.7-code-highspeed | `MOONSHOT_API_KEY` or `KIMI_API_KEY` | | **OpenRouter** | any tool-calling OpenRouter model ID | `OPENROUTER_API_KEY` | -| **Ollama** | any local model with tool-calling | runs at `http://localhost:11434` | +| **Ollama** | `ollama/` | No key; native server defaults to `http://localhost:11434` | | **LM Studio** | any local OpenAI-compatible model with tool-calling | runs at `http://localhost:1234/v1` | -Other Tachikoma-supported providers also work — see the [Tachikoma docs](https://github.com/steipete/Tachikoma) for the full list. +Other Tachikoma-supported providers also work — see the [Tachikoma docs](https://github.com/openclaw/Tachikoma) for the full list. ## Credentials @@ -67,7 +67,7 @@ peekaboo agent --model minimax-cn/MiniMax-M3 "summarize this window" peekaboo agent --model kimi/kimi-k2.7-code "summarize this window" peekaboo agent --model openrouter/xiaomi/mimo-v2.5-pro "summarize this window" peekaboo agent --model gpt-5-mini "click Continue and wait for the dialog" -peekaboo agent --model ollama/llama3.1:8b "describe this screenshot" +peekaboo agent --model ollama/llama3.1:8b "open System Settings" peekaboo agent --model lmstudio/openai/gpt-oss-120b "summarize this window" ``` @@ -79,21 +79,30 @@ capabilities; Peekaboo currently catalogs Fable 5 and Sonnet 5 with 1M context w ## Tool calling -The agent requires a tool-calling capable model. If the selected model does not support tools, Peekaboo rejects it and points to `peekaboo image --analyze` / `see --analyze` for vision-only use. Choose a tool-capable model for agent runs. +The agent requires a tool-calling capable model. Peekaboo rejects a configured model marked `supportsTools: false`; +only opt that model into tools when its endpoint actually implements tool calling. Vision is a separate capability, so +use `peekaboo image --analyze` / `see --analyze` only with a model that also supports vision. Ollama capabilities vary +by model and tag; see the [Ollama provider guide](providers/ollama.md) before assuming a locally installed model +supports tools. -## Local-only mode +## On-device Ollama mode -Want everything on-device? Run an Ollama model with tool calling and point the CLI at it: +To keep model inference on-device, run an Ollama model with tool calling and select it explicitly: ```bash -ollama run llama3.1:8b +ollama pull llama3.1:8b peekaboo agent --model ollama/llama3.1:8b "open System Settings" ``` -No network requests leave the machine. Captures, AX queries, and reasoning all stay local. +The loopback endpoint only guarantees that Peekaboo talks to the local Ollama daemon. Ollama cloud-model tags can be +automatically offloaded by that daemon. For strict on-device inference, select a locally installed model and disable +Ollama cloud features with `OLLAMA_NO_CLOUD=1` or `disable_ollama_cloud` in `~/.ollama/server.json`. Model downloads, +network-capable tools, and a remote Ollama base URL remain separate network paths. See the +[Ollama privacy boundary](providers/ollama.md#privacy-boundary) and Ollama's +[cloud documentation](https://docs.ollama.com/cloud). ## Troubleshooting - **"401 Unauthorized"** — credential isn't set, or env var overrides the saved one. Run `peekaboo config get-credential `. -- **"context length exceeded"** — long sessions accumulate screenshots. Start a fresh session with `peekaboo agent --new`. +- **"context length exceeded"** — long sessions accumulate history. Start a fresh run without `--resume`. - **"no tool-call support"** — pick a different model. The error log lists the providers and models with confirmed tool-calling. diff --git a/docs/providers/README.md b/docs/providers/README.md index 33067a689..f04a04533 100644 --- a/docs/providers/README.md +++ b/docs/providers/README.md @@ -14,7 +14,8 @@ read_when: - **MiniMax China** — use `minimax-cn/...` with `MINIMAX_CN_API_KEY` or the shared `MINIMAX_API_KEY`; routes to `api.minimaxi.com`. - **Kimi** — use `kimi/...` with `MOONSHOT_API_KEY` or `KIMI_API_KEY`; supports Kimi K2.6 and K2.7 Code through Moonshot's coding API. - **Grok** — `grok.md`: Grok 4 implementation guide and checkpoints. -- **Ollama** — `ollama.md`: local model configuration; `ollama-models.md` for model catalog notes. +- **Ollama** — `ollama.md`: native API, endpoints, tool history, and streaming behavior; `ollama-models.md` for + capability-first model selection. Use [`docs/providers.md`](../providers.md) as the central reference for the user-facing provider list, configuration syntax, and environment variable reference. @@ -30,7 +31,7 @@ configuration syntax, and environment variable reference. | MiniMax China | Yes | Model-dependent | Yes | No | API key | | Kimi | Yes | Yes | Yes | No | API key | | Grok | Yes | Limited | Yes | No | API key | -| Ollama | Yes (via local server) | Model-dependent | Yes | **Yes** (local) | None (local daemon) | +| Ollama | Model-dependent | Model-dependent | Response-buffered NDJSON | Model-dependent; disable cloud for strict local use | None for local models | | LM Studio | Yes (OpenAI-compatible local server) | Model-dependent | Yes | **Yes** (local) | None by default | See individual pages for model lists, quirks, and test coverage expectations. diff --git a/docs/providers/ollama-models.md b/docs/providers/ollama-models.md index b9c69ee53..6d113677e 100644 --- a/docs/providers/ollama-models.md +++ b/docs/providers/ollama-models.md @@ -1,137 +1,58 @@ --- -summary: 'Review Ollama Models Guide guidance' +summary: 'Choose an Ollama model for Peekaboo by verified tools, vision, context, and hardware capabilities.' read_when: - - 'planning work related to ollama models guide' - - 'debugging or extending features described here' + - 'choosing an Ollama model for agent automation or image analysis' + - 'debugging an Ollama model capability mismatch' --- -# Ollama Models Guide +# Choosing an Ollama model -This guide provides an overview of Ollama models that excel at specific tasks, particularly tool/function calling and vision capabilities. +Ollama's catalog and model capabilities change independently of Peekaboo. Treat model names, parameter counts, and +memory estimates as discovery hints—not proof that a model can drive the agent. Verify the exact tag you install. -## Models for Tool/Function Calling +## Match the capability to the command -### By VRAM Requirements +| Peekaboo use | Required Ollama capability | Notes | +| --- | --- | --- | +| `peekaboo agent` | Tools/function calling | Required even when the task sounds text-only, because the agent acts through tools. | +| Image analysis | Vision | A vision-only model can analyze an image but cannot necessarily run the agent. | +| Visual agent task | Tools; vision if the selected workflow sends images to the model | Tool support and vision support are independent. | -#### 64 GB+ VRAM -- **Llama 3 Groq Tool-Use 70B** - - Most accurate JSON output - - Handles multi-tool and nested calls - - Huge context window - - Best choice for complex automation tasks - -#### 32 GB VRAM -- **Mixtral 8×7B Instruct** - - Native tool-calling flag support - - MoE (Mixture of Experts) architecture for speed - - 46B active parameters providing near-GPT-3.5 quality - - Good balance of performance and capability - -#### 24 GB VRAM -- **Mistral Small 3.1 24B** - - Explicit "low-latency function calling" in documentation - - Fits on single RTX 4090 or Apple Silicon 32GB - - Excellent for production deployments - -#### <16 GB VRAM -- **Functionary-Small v3.1 (8B)** - - Fine-tuned solely for JSON schema compliance - - Great for rapid prototyping - - Reliable structured output - -#### Laptop-class (8-12 GB) -- **Phi-3 Mini / Gemma 3.1-3B** - - Tiny models that respond in JSON with careful prompting - - Good for IoT agents and edge devices - - Requires more prompt engineering - -## Vision Models (Image Chat/OCR/Diagram Q&A) - -### By VRAM Requirements - -#### 7-34B Options -- **LLaVA 1.6** - - Big improvement in resolution (up to 672×672) - - Much better OCR than v1.5 - - Simple CLI: `ollama run llava` - - Recommended for general vision tasks - -#### 24B -- **Mistral Small 3.1 Vision** - - Same text skills as tool-calling version plus vision - - Supports 128k tokens - - Can process long PDF pages as images or text chunks - - Best for document + vision hybrid tasks - -#### 2B -- **Granite 3.2-Vision** - - Specialized for documents: tables, charts, invoices - - Works on machines with <8GB VRAM - - Excellent for business document processing - -#### 1.8B -- **Moondream 2** - - Ridiculously small model - - Runs on Raspberry Pi-class devices - - Still captions everyday photos decently - - Perfect for edge computing - -#### 7B -- **BakLLaVA** - - Mistral-based fork of LLaVA - - Better reasoning than LLaVA-7B - - Heavier than Moondream but more capable - -## Usage in Peekaboo - -### Recommended Models for Agent Tasks - -1. **Best Overall**: `llama3.3` (or aliases: `llama`, `llama3`) - - Excellent tool calling support - - Good balance of speed and accuracy - - Works well with Peekaboo's automation tools - -2. **For Vision Tasks**: `llava` or `mistral-small:3.1-vision` - - Note: Vision models typically don't support tool calling - - Use for image analysis tasks only - -3. **For Limited Resources**: `mistral-nemo` or `firefunction-v2` - - Smaller models with tool support - - Good for testing and development - -### Example Usage +Use Ollama's current [tools filter](https://ollama.com/search?c=tools) and +[vision filter](https://ollama.com/search?c=vision), then inspect the exact local tag: ```bash -# Tool calling with llama3.3 -PEEKABOO_AI_PROVIDERS="ollama/llama3.3" ./scripts/peekaboo-wait.sh agent "Click on the Apple menu" - -# Vision analysis with llava -PEEKABOO_AI_PROVIDERS="ollama/llava" ./scripts/peekaboo-wait.sh analyze screenshot.png "What's in this image?" - -# Using model shortcuts -PEEKABOO_AI_PROVIDERS="ollama/llama" ./scripts/peekaboo-wait.sh agent "Type hello world" +ollama list +ollama show llama3.1:8b ``` -## Important Notes - -1. **Tool Calling Support**: Not all models support tool/function calling. Check the model's capabilities before using with Peekaboo's agent command. - -2. **First Run**: Models need to be downloaded on first use. This can take several minutes depending on model size and internet speed. +The model page is the capability authority. A model that merely produces JSON is not equivalent to one implementing +Ollama's tool-calling protocol. -3. **Performance**: Local inference speed depends heavily on your hardware. GPU acceleration (NVIDIA CUDA or Apple Metal) significantly improves performance. +## Smoke-test the selected tag -4. **Memory Usage**: Ensure you have sufficient VRAM/RAM for your chosen model. The VRAM requirements listed are minimums for reasonable performance. +```bash +MODEL=llama3.1:8b +ollama pull "$MODEL" -5. **Context Length**: Larger models generally support longer context windows, important for complex automation tasks. +# A small budget proves the native route and one tool round trip. +peekaboo agent --model "ollama/$MODEL" --max-steps 2 \ + "Read the title of the frontmost window and report it" +``` -## Model Selection Tips +The model must return a structured tool call, accept the named tool result on the next turn, and then produce a final +answer. If it emits prose that only describes a tool call, rejects the schema, or repeatedly calls the same tool, +choose a stronger tool-capable tag. -- **For automation/agent tasks**: Choose models with explicit tool calling support -- **For simple tasks**: Smaller models (8B-24B) are often sufficient -- **For complex reasoning**: Larger models (70B+) provide better accuracy -- **For vision tasks**: LLaVA 1.6 is a solid default choice -- **For edge devices**: Consider Moondream 2 or Phi-3 Mini +## Resource considerations -## Troubleshooting +- Pick a quantization and parameter size that fits available unified memory or VRAM; otherwise Ollama may evict the + model or respond too slowly for an interactive automation loop. +- Longer agent sessions replay conversation and tool history. Prefer a context window large enough for the task and + start a fresh session when old context is no longer useful. +- Small models can be fast but less reliable with nested schemas and multi-step planning. Test the actual Peekaboo + tool set rather than inferring reliability from a generic benchmark. +- Pulling a model requires network access and disk space even when subsequent inference runs locally. -If a model returns HTTP 400 errors when used with Peekaboo's agent command, it likely doesn't support tool calling. Switch to a model from the tool calling list above. \ No newline at end of file +See [Ollama](ollama.md) for endpoint precedence, the native `/api/chat` route, step-budget behavior, and the qualified +privacy boundary. diff --git a/docs/providers/ollama.md b/docs/providers/ollama.md index ff32c2199..3a0a17464 100644 --- a/docs/providers/ollama.md +++ b/docs/providers/ollama.md @@ -1,735 +1,141 @@ --- -summary: 'Configure Peekaboo to use local Ollama models (llama3, llava, Ultrathink) and track the remaining implementation work.' +summary: 'Configure and run Peekaboo with Ollama through its native chat API.' read_when: - - 'running Peekaboo with local models' - - 'debugging or extending the Ollama provider' + - 'running Peekaboo with a local or remote Ollama server' + - 'debugging Ollama tool calls, streaming, or endpoint selection' --- -# Ollama Ultrathink Integration Plan for Peekaboo +# Ollama -## Overview +Peekaboo supports Ollama directly. A model ID in the form `ollama/` selects the native Ollama provider and +sends chat requests to `/api/chat`; no OpenAI-compatible adapter or API key is required. -This document outlines the plan for completing Ollama support in Peekaboo and adding the Ultrathink model. Currently, Ollama has basic provider infrastructure but lacks full implementation, particularly for the agent command and streaming responses. - -## Quick Start (Local Only) - -For privacy-focused automation runs you can aim Peekaboo at a local Ollama daemon: +## Quick start ```bash -# Install and start Ollama brew install ollama ollama serve - -# Grab recommended models -ollama pull llama3.3 # ✅ Supports tool calling -ollama pull llava:latest # Vision-only (no tools) - -# Point Peekaboo at the server -PEEKABOO_AI_PROVIDERS="ollama/llama3.3" peekaboo agent "Click the Submit button" -PEEKABOO_AI_PROVIDERS="ollama/llava:latest" peekaboo image --analyze "Describe this UI" - -# Persist in config (optional) -peekaboo config set aiProviders.providers "ollama/llama3.3" -peekaboo config set aiProviders.ollamaBaseUrl "http://localhost:11434" -``` - -### Recommended Models - -- **Automation (tool calling):** `llama3.3` (best) or `llama3.2`. These understand tool metadata and can drive GUI automation. -- **Vision-only:** `llava:latest`, `bakllava` – use for `image --analyze`, but they cannot execute tools. -- **Ultrathink / other heavy models:** follow the implementation plan below to ensure streaming + tool calling support before enabling by default. - -**Environment variables** - -- `PEEKABOO_AI_PROVIDERS="ollama/`" – enables Ollama providers globally. -- `PEEKABOO_OLLAMA_BASE_URL` – override the default `http://localhost:11434` when your daemon runs on another host. - -> Note: The CLI only accepts models that advertise tool support when you run `peekaboo agent`. If a model is vision-only you can still use `peekaboo image --analyze` via the same provider string. - -## Current State - -### Existing Implementation -- ✅ `OllamaProvider.swift` with basic structure -- ✅ Server availability checks -- ✅ Model listing capability -- ✅ Image analysis via `/api/chat` with `messages[].images` (used by `peekaboo image --analyze`) -- ❌ No `peekaboo agent` support yet (agent runtime still assumes cloud providers) -- ⚠️ Streaming is basic and may not truly stream token-by-token -- ⚠️ Tool calling support is partial/experimental and model-dependent - -### Model Support -Supports vision models like `llava:latest` and `qwen2.5vl:latest` for `peekaboo image --analyze`, plus text models like `llama3.3` for local text generation. - -## Ollama API Overview - -Ollama provides a REST API with two main approaches: -- **Native API**: Base URL `http://localhost:11434` - - Chat endpoint: `/api/chat` (primary for conversations) - - Generate endpoint: `/api/generate` (for simple completions) - - Streaming: JSON objects (not SSE), streaming enabled by default - - Tool calling: Supported via `tools` parameter (model-dependent) -- **OpenAI Compatibility**: `/v1/chat/completions` - - Full OpenAI Chat Completions API compatibility - - Easier integration with existing OpenAI tooling - -## Implementation Plan - -### Phase 1: Complete Core Provider (1-2 days) - -1. **Move OllamaProvider to Core** - - Move from `Apps/CLI/Sources/peekaboo/AIProviders/` to `Core/PeekabooCore/Sources/PeekabooCore/AI/Ollama/` - - Align with OpenAI/Anthropic structure - -2. **Create Ollama Types** - - Location: `Core/PeekabooCore/Sources/PeekabooCore/AI/Ollama/OllamaTypes.swift` - ```swift - struct OllamaChatRequest - struct OllamaChatResponse - struct OllamaMessage - struct OllamaToolCall - struct OllamaStreamChunk - ``` - -3. **Implement Basic Chat** - - Update `OllamaProvider` to implement full `AIProvider` protocol - - Add chat completion support - - Handle authentication (none required for local) - -### Phase 2: Create OllamaModel (2-3 days) - -1. **Create OllamaModel.swift** - - Location: `Core/PeekabooCore/Sources/PeekabooCore/AI/Models/OllamaModel.swift` - - Implement `ModelInterface` protocol - - Message conversion logic - - System prompt handling - -2. **Message Format Conversion** - ```swift - // Peekaboo → Ollama - SystemMessageItem → messages[].content with role "system" - UserMessageItem → messages[].content with role "user" - AssistantMessageItem → messages[].content with role "assistant" - ToolMessageItem → Not directly supported, convert to user message - ``` - -3. **Image Support** - - Convert base64 images to Ollama format - - Support multimodal models (llava, bakllava) - - Handle text-only models gracefully - -### Phase 3: Streaming Implementation (1-2 days) - -**Critical**: Ollama uses newline-delimited JSON streaming, NOT Server-Sent Events (SSE)! - -1. **JSON Streaming Parser** - - Parse newline-delimited JSON objects - - Handle partial chunks and buffering - - Robust error recovery for malformed JSON - - Convert to Peekaboo's `StreamEvent` types - -2. **Stream Integration** - ```swift - func getStreamedResponse(messages: [MessageItem], tools: [ToolDefinition]?) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { - do { - let url = baseURL.appendingPathComponent("api/chat") - var request = URLRequest(url: url) - request.httpMethod = "POST" - request.setValue("application/json", forHTTPHeaderField: "Content-Type") - - let body = OllamaChatRequest( - model: model, - messages: convertMessages(messages), - tools: convertTools(tools), - stream: true - ) - request.httpBody = try JSONEncoder().encode(body) - - let (bytes, _) = try await URLSession.shared.bytes(for: request) - let parser = OllamaStreamParser() - - for try await line in bytes.lines { - let events = parser.parse(data: line.data(using: .utf8)!) - for event in events { - continuation.yield(mapToStreamEvent(event)) - } - } - - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - } - } - ``` - -3. **Event Mapping** - ```swift - // Ollama streaming events → Peekaboo StreamEvents - - message.content deltas → .contentDelta(String) - - tool_calls → .toolCall(ToolCall) - - done: true → .finished - - error responses → .error(Error) - ``` - -4. **Streaming States** - - Content streaming (text generation) - - Tool call streaming (function invocations) - - Mixed content/tool streaming - - Completion with statistics - -### Phase 4: Tool Calling Support (2 days) - -**Update (2025)**: Ollama now has official tool calling support with streaming capabilities! - -1. **Tool Definition Conversion** - - Convert `ToolDefinition` to Ollama function format - - Handle parameter schemas - - Support required/optional parameters - - Use improved parser that understands tool call structure - -2. **Tool Execution Flow** - - Parse tool calls from responses - - Format tool results - - Handle multi-turn conversations - - Support streaming with tool calls - -3. **Supported Models** - Models with verified tool calling support (as of 2025): - - **Llama 3.1** (8b, 70b, 405b) - Primary recommendation - - **Mistral Nemo** - Reliable for tools - - **Firefunction v2** - Optimized for function calling - - **Command-R+** - Good tool support - - **Qwen models** - Varying support by version - - **DeepSeek-R1** - New reasoning model with tool support - - **Important**: Tool calling support is model-dependent. Not all Ollama models support tools. Always check model capabilities before assuming tool support. - -4. **Implementation Tips** - - Use context window of 32k+ for better tool calling performance - - New streaming parser handles tool calls without blocking - - Python library v0.4+ supports direct function passing - -### Phase 5: Model Registration (1 day) - -1. **Register Ollama Models** - ```swift - // In ModelProvider.swift - registerOllamaModels() - ``` - -2. **Model Definitions** - ```swift - // Text generation models with tool calling (verified 2025) - - ollama/llama3.1:8b ✅ Tool calling - - ollama/llama3.1:70b ✅ Tool calling - - ollama/llama3.1:405b ✅ Tool calling - - ollama/mistral-nemo ✅ Tool calling - - ollama/firefunction-v2 ✅ Tool calling (optimized) - - ollama/command-r-plus ✅ Tool calling - - ollama/deepseek-r1 ✅ Tool calling (reasoning model) - - // Text generation models (limited/no tool support) - - ollama/ultrathink ❓ TBD when released - - ollama/llama3.2 ❌ No official tool support - - ollama/qwen2.5 ⚠️ Variable by version - - ollama/phi3 ❌ No tool calling - - ollama/mistral:7b ❌ Use mistral-nemo for tools - - // Multimodal models (no tool support) - - ollama/llava:latest ❌ Vision only - - ollama/bakllava ❌ Vision only - - ollama/llava-llama3 ❌ Vision only - ``` - -3. **Dynamic Model Discovery** - - Query `/api/tags` for available models - - Cache model list - - Refresh periodically - -### Phase 6: Ultrathink-Specific Features (1-2 days) - -1. **Model Characteristics** - - Extended context window support - - Reasoning traces (if supported) - - Performance optimizations - -2. **Special Parameters** - ```swift - struct UltrathinkOptions { - var temperature: Double = 0.7 - var num_predict: Int = 4096 - var num_ctx: Int = 32768 // Extended context - var reasoning_mode: String? = "detailed" - } - ``` - -3. **Reasoning Support** - - Check if Ultrathink supports reasoning traces - - Implement similar to GPT-5 reasoning summaries - - Display thinking indicators - -### Phase 7: Testing & Integration (2 days) - -1. **Unit Tests** - - Test message conversion - - Mock Ollama responses - - Error scenarios - -2. **Integration Tests** - - Test with local Ollama instance - - Verify streaming - - Tool calling scenarios - -3. **Performance Testing** - - Benchmark vs OpenAI/Anthropic - - Memory usage with large contexts - - Streaming latency - -## Technical Implementation Details - -### Streaming Parser Implementation - -```swift -class OllamaStreamParser { - private var buffer = "" - - func parse(data: Data) -> [OllamaStreamEvent] { - guard let text = String(data: data, encoding: .utf8) else { return [] } - buffer += text - - var events: [OllamaStreamEvent] = [] - let lines = buffer.split(separator: "\n", omittingEmptySubsequences: false) - - // Keep incomplete line in buffer - if !buffer.hasSuffix("\n") && !lines.isEmpty { - buffer = String(lines.last!) - for line in lines.dropLast() { - if let event = parseJSONLine(String(line)) { - events.append(event) - } - } - } else { - buffer = "" - for line in lines where !line.isEmpty { - if let event = parseJSONLine(String(line)) { - events.append(event) - } - } - } - - return events - } - - private func parseJSONLine(_ line: String) -> OllamaStreamEvent? { - guard let data = line.data(using: .utf8), - let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { - return nil - } - - // Parse based on content - if json["done"] as? Bool == true { - return .completed(stats: parseStats(json)) - } else if let message = json["message"] as? [String: Any] { - if let content = message["content"] as? String, !content.isEmpty { - return .contentDelta(content) - } - if let toolCalls = message["tool_calls"] as? [[String: Any]] { - return .toolCall(parseToolCalls(toolCalls)) - } - } - - return nil - } -} ``` -### API Endpoints +Leave that terminal running. In another terminal: -```swift -// Chat completion -POST /api/chat -{ - "model": "llama3.1", - "messages": [ - {"role": "system", "content": "You are a helpful assistant"}, - {"role": "user", "content": "Hello"} - ], - "stream": true, - "tools": [...], - "options": { - "temperature": 0.7, - "num_predict": 4096, - "num_ctx": 32768 - }, - "keep_alive": "5m" -} +```bash +MODEL=llama3.1:8b +ollama pull "$MODEL" +peekaboo agent --model "ollama/$MODEL" "Open System Settings" +``` -// Model listing -GET /api/tags +`peekaboo agent` requires a model that supports tool calling. Ollama capabilities are model-dependent: a model that +can generate text or inspect images does not necessarily accept tools. Check the model's current Ollama page or +`ollama show "$MODEL"` before using it for automation. See [Choosing a model](ollama-models.md) for a capability-first +selection guide. -// Model info -GET /api/show/{modelname} -``` +## Endpoint selection -### Streaming Format +The built-in provider resolves its base URL in this order, highest priority first: -```swift -// Standard content streaming (newline-delimited JSON) -{"model":"llama3.1","created_at":"2025-01-26T12:00:00Z","message":{"role":"assistant","content":"Hello"},"done":false} -{"model":"llama3.1","created_at":"2025-01-26T12:00:01Z","message":{"role":"assistant","content":" there"},"done":false} -{"model":"llama3.1","created_at":"2025-01-26T12:00:02Z","message":{"role":"assistant","content":"!"},"done":false} -{"model":"llama3.1","created_at":"2025-01-26T12:00:03Z","done":true,"done_reason":"stop","total_duration":1234567890,"load_duration":123456,"prompt_eval_count":10,"prompt_eval_duration":123456,"eval_count":3,"eval_duration":234567} +1. `PEEKABOO_OLLAMA_BASE_URL` +2. `OLLAMA_BASE_URL` +3. `aiProviders.ollamaBaseUrl` in `~/.peekaboo/config.json` +4. `http://localhost:11434` -// Streaming with tool calls -{"model":"llama3.1","created_at":"2025-01-26T12:00:00Z","message":{"role":"assistant","content":""},"done":false} -{"model":"llama3.1","created_at":"2025-01-26T12:00:01Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"get_weather","arguments":{"city":"Toronto"}}}]},"done":false} -{"model":"llama3.1","created_at":"2025-01-26T12:00:02Z","done":true,"done_reason":"stop","total_duration":987654321} +Set the server base only; do not append `/api/chat` or `/v1`: -// Mixed content and tool streaming -{"model":"llama3.1","created_at":"2025-01-26T12:00:00Z","message":{"role":"assistant","content":"Let me check the weather"},"done":false} -{"model":"llama3.1","created_at":"2025-01-26T12:00:01Z","message":{"role":"assistant","content":" for you.","tool_calls":[{"function":{"name":"get_weather","arguments":{"city":"Toronto"}}}]},"done":false} -{"model":"llama3.1","created_at":"2025-01-26T12:00:02Z","done":true} +```bash +PEEKABOO_OLLAMA_BASE_URL=http://192.168.1.20:11434 \ + peekaboo agent --model ollama/llama3.1:8b "Summarize the frontmost window" ``` -### Tool Calling Format - -```swift -// Request with tools -{ - "model": "llama3.1", - "messages": [{"role": "user", "content": "What's the weather in Toronto?"}], - "tools": [{ - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather for a city", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The name of the city" - } - }, - "required": ["city"] - } - } - }], - "stream": true -} +To persist the built-in provider endpoint: -// Response with tool call +```json { - "model": "llama3.1", - "message": { - "role": "assistant", - "content": "", - "tool_calls": [{ - "function": { - "name": "get_current_weather", - "arguments": { - "city": "Toronto" - } - } - }] + "aiProviders": { + "ollamaBaseUrl": "http://192.168.1.20:11434" } } ``` -### Error Handling - -```swift -enum OllamaError: Error { - case serverNotRunning - case modelNotFound(String) - case modelNotLoaded(String) - case contextLengthExceeded - case streamingError(String) - case malformedJSON(String) - case connectionLost - case toolCallFailed(String) -} +### Native API versus OpenAI compatibility -// Streaming error scenarios -extension OllamaStreamParser { - func handleStreamingErrors(_ error: Error) -> StreamEvent { - switch error { - case URLError.networkConnectionLost: - return .error(OllamaError.connectionLost) - case let DecodingError.dataCorrupted(context): - return .error(OllamaError.malformedJSON(context.debugDescription)) - default: - return .error(OllamaError.streamingError(error.localizedDescription)) - } - } -} +Use `ollama/` for Peekaboo's native integration. It targets `/api/chat`, understands Ollama's newline-delimited +stream format, and preserves Ollama's native tool-call history. -// Error recovery strategies -class OllamaStreamHandler { - func recoverFromError(_ error: OllamaError) async throws { - switch error { - case .serverNotRunning: - throw error // Can't recover, user must start Ollama - case .modelNotFound(let model): - // Suggest pulling the model - print("Model '\(model)' not found. Run: ollama pull \(model)") - throw error - case .connectionLost: - // Retry with exponential backoff - try await Task.sleep(nanoseconds: 1_000_000_000) - // Retry logic here - case .malformedJSON: - // Continue parsing, skip malformed line - break - default: - throw error - } - } -} -``` +Ollama also exposes an OpenAI-compatible endpoint at `/v1`. Use that route only when testing the compatibility layer +or when a custom-provider workflow requires it: -### Conversation Context and Session Management - -**Important**: Ollama is stateless - it does NOT maintain conversation history between API calls. You must manage context yourself. - -```swift -// Managing conversation history -class OllamaConversationManager { - private var messages: [OllamaMessage] = [] - - func addUserMessage(_ content: String) { - messages.append(OllamaMessage(role: "user", content: content)) - } - - func addAssistantMessage(_ content: String) { - messages.append(OllamaMessage(role: "assistant", content: content)) - } - - func addSystemMessage(_ content: String) { - // System messages should typically be first - messages.insert(OllamaMessage(role: "system", content: content), at: 0) - } - - func getChatRequest(newMessage: String) -> OllamaChatRequest { - addUserMessage(newMessage) - - return OllamaChatRequest( - model: model, - messages: messages, // Send full history - stream: true, - options: ["num_ctx": 32768] // Ensure large context window - ) - } - - func trimHistory(maxMessages: Int = 50) { - // Keep system message + recent messages - if messages.count > maxMessages { - let systemMessages = messages.filter { $0.role == "system" } - let recentMessages = Array(messages.suffix(maxMessages - systemMessages.count)) - messages = systemMessages + recentMessages - } - } -} +```bash +peekaboo config add-provider ollama-openai \ + --type openai \ + --name "Ollama via OpenAI compatibility" \ + --base-url "http://localhost:11434/v1" \ + --api-key "dummy-key" + +peekaboo config models-provider ollama-openai --discover --save +peekaboo agent --model ollama-openai/llama3.1:8b "Open System Settings" ``` -#### Key Differences from Cloud Providers +The two routes have different wire formats and configuration. Do not give `/v1` to +`PEEKABOO_OLLAMA_BASE_URL`; that variable configures the native route. -1. **No Session IDs**: Unlike OpenAI/Anthropic, Ollama has no session management -2. **Manual History**: You must send the complete conversation history with each request -3. **Context Limits**: Be mindful of model context windows (varies by model) -4. **Memory Usage**: Larger contexts use more VRAM/RAM +## Tool calling and agent turns -#### Context Parameter (Deprecated) +Peekaboo follows Ollama's [multi-turn tool-calling contract](https://docs.ollama.com/capabilities/tool-calling): -The old `/api/generate` endpoint used a `context` parameter (array of tokens) to maintain state: -```swift -// OLD WAY - DEPRECATED -{ - "model": "llama2", - "prompt": "continue our conversation", - "context": [1, 2, 3, ...] // Token array from previous response -} -``` +1. Peekaboo sends the conversation and available tool schemas. +2. Ollama returns an assistant message containing one or more function calls. +3. Peekaboo executes those calls. +4. The next request replays the assistant's function calls followed by named `tool` result messages. +5. The loop repeats until the model returns a final answer without more tool calls. -**Use `/api/chat` with full message history instead** for better compatibility and clearer conversation management. - -#### Best Practices - -1. **Persistent Storage**: Store conversations in a database for multi-session support -2. **Context Pruning**: Implement sliding window or importance-based pruning for long conversations -3. **System Prompts**: Include system messages at the start of each conversation -4. **Error Recovery**: Save conversation state periodically to recover from crashes - -```swift -// Example: Peekaboo integration -extension OllamaModel { - func continueConversation(sessionId: String, newMessage: String) async throws -> String { - // Load conversation from storage - let history = try await loadConversationHistory(sessionId) - - // Add new message - history.append(MessageItem.user(newMessage)) - - // Send full history to Ollama - let response = try await getResponse(messages: history, tools: nil) - - // Save updated conversation - history.append(MessageItem.assistant(response)) - try await saveConversationHistory(sessionId, history) - - return response - } -} -``` +Ollama does not retain this state between requests. Peekaboo therefore sends the canonical assistant call and tool +result history on every follow-up, including nested arguments, array schemas, and parallel calls. -## Configuration +`--max-steps` limits model turns, not individual tool invocations. The CLI accepts `1...100` and defaults to `100`; +one turn may contain several parallel tool calls. If the last permitted turn still requests tools and needs another +model turn to interpret their results, Peekaboo saves the session and reports step-budget exhaustion instead of +returning an empty success. Resume that session to continue from the preserved tool results. -### User Setup -```bash -# Install Ollama -curl -fsSL https://ollama.com/install.sh | sh +## Streaming behavior -# Pull Ultrathink model (when available) -ollama pull ultrathink +Ollama's native chat API returns newline-delimited JSON when streaming. Peekaboo asks for that format and validates +each chunk, including [errors delivered after an HTTP 200 response](https://docs.ollama.com/api/errors). -# Pull recommended models with tool support -ollama pull llama3.1 # Primary recommendation for tools -ollama pull mistral-nemo # Good tool support -ollama pull firefunction-v2 # Optimized for functions -ollama pull command-r-plus # Alternative with tools +The current Tachikoma transport buffers the HTTP response before emitting its decoded deltas. Tool calls and errors +follow streaming semantics, but CLI text appears response-by-response rather than token-by-token. Do not rely on +token-live output for progress or timeout decisions. -# Pull other models -ollama pull deepseek-r1 # Reasoning model -ollama pull llava:latest # Multimodal (no tools) +## Privacy boundary -# Verify models -ollama list -``` +When the resolved base URL points to a daemon on the same Mac and the selected model is local, prompts, screenshots +included in model requests, tool history, and model responses are processed on that Mac. A loopback URL alone is not +a local-only guarantee: Ollama cloud-model tags are automatically offloaded through the daemon. -### Peekaboo Configuration -```bash -# Use Ollama models -./peekaboo agent "analyze this code" --model ollama/llama3.1 -./peekaboo agent "analyze this code" --model ollama/ultrathink # When available +For strict local-only operation, use a locally installed model and disable Ollama cloud features with +`OLLAMA_NO_CLOUD=1` or `"disable_ollama_cloud": true` in `~/.ollama/server.json`, then restart Ollama. See Ollama's +[cloud guide](https://docs.ollama.com/cloud) and [FAQ](https://docs.ollama.com/faq). Other network boundaries remain: -# Set as default -PEEKABOO_AI_PROVIDERS="ollama/llama3.1" ./peekaboo agent "help me" +- `ollama pull` contacts Ollama's model registry. +- A remote `PEEKABOO_OLLAMA_BASE_URL` or `OLLAMA_BASE_URL` sends model data to that host. +- A cloud model sends prompts and responses to Ollama Cloud even when Peekaboo connects to `localhost`. +- Tools chosen by the model can launch apps or perform actions that use the network. +- Other configured cloud providers remain separate network paths; an explicit `--model ollama/...` keeps model + selection on Ollama for that run. -# Multiple providers -PEEKABOO_AI_PROVIDERS="ollama/llama3.1,openai/gpt-4.1" ./peekaboo agent "task" +Review the endpoint and enabled tools before using sensitive data. -# OpenAI compatibility mode (alternative approach) -OLLAMA_OPENAI_COMPAT=true ./peekaboo agent "task" --model ollama/llama3.1 -``` +## Troubleshooting -### OpenAI Compatibility Mode +- **Connection refused:** start `ollama serve`, then verify the resolved URL with `peekaboo config show --effective`. +- **Model not found:** run `ollama list`, then `ollama pull ` if necessary. +- **Tool-call or schema error:** the selected model may not support tools reliably. Choose a model marked for tools in + the current [Ollama library](https://ollama.com/search?c=tools). +- **Vision works but `agent` fails:** vision and tools are independent capabilities; choose a tool-capable model for + `peekaboo agent`. +- **Run stops at the limit:** increase `--max-steps` within `1...100`, or reduce the task scope. A limit failure means + the model still had pending work, not that the completed tool results were lost; the error identifies the saved + session that can be resumed. -Ollama also provides OpenAI API compatibility at `http://localhost:11434/v1/chat/completions`. This allows using Ollama with tools expecting OpenAI's API format. Benefits: -- Use existing OpenAI client libraries -- Simplified integration -- Consistent API format across providers - -## Key Differences from Cloud Providers - -1. **Local Execution**: No API keys required -2. **Model Management**: Must pull models before use -3. **Performance**: Depends on local hardware -4. **Privacy**: All data stays local -5. **Availability**: No rate limits or quotas -6. **Cost**: Free after initial hardware investment - -## Success Criteria - -- [ ] Basic chat completions working -- [ ] Streaming responses functional -- [ ] Tool calling implemented for: - - [ ] Llama 3.1 (primary tool-calling model) - - [ ] Qwen 2.5, Mistral, DeepSeek-R1 - - [ ] Capability detection for unsupported models -- [ ] Image analysis working (llava, bakllava) -- [ ] All common Ollama models registered with capability flags -- [ ] Ultrathink model fully supported (when available) -- [ ] Performance acceptable for local execution -- [ ] Graceful handling of server unavailability -- [ ] Model-specific optimizations (32k+ context for tool calling) - -## Timeline - -- Phase 1-2: 3-5 days (Core implementation) -- Phase 3-4: 3-4 days (Streaming & tools) -- Phase 5-6: 2-3 days (Models & Ultrathink) -- Phase 7: 2 days (Testing) - -**Total: 10-14 days** - -## Risks & Mitigations - -1. **Risk**: Ultrathink model not yet available - - **Mitigation**: Implement generic Ollama support first, add Ultrathink when released - -2. **Risk**: Tool calling compatibility varies by model - - **Mitigation**: Implement capability detection and graceful degradation - -3. **Risk**: Performance issues with large models - - **Mitigation**: Add configuration for GPU acceleration, implement timeouts - -4. **Risk**: Ollama API changes - - **Mitigation**: Version detection, compatibility layer - -## Verdict: Full Implementation is Ready to Proceed ✅ - -After thorough analysis, **YES** - we can fully implement Ollama with all features working: - -### What Will Work: -1. **Basic Chat Completions** ✅ - Full conversation support via `/api/chat` -2. **Streaming** ✅ - Newline-delimited JSON streaming with proper parsing -3. **Tool Calling** ✅ - Supported by Llama 3.1, Mistral Nemo, and other models -4. **Session Management** ✅ - Already generic via AgentSessionManager -5. **Agent Integration** ✅ - AgentRunner works with any ModelInterface -6. **Image Analysis** ✅ - For multimodal models (llava, bakllava) -7. **Error Handling** ✅ - Comprehensive error recovery strategies - -### Key Implementation Notes: -- **Session persistence** is already provider-agnostic through AgentSessionManager -- **Conversation context** managed by sending full message history (Ollama is stateless) -- **Tool calling** requires model support (Llama 3.1 recommended) -- **Streaming** uses URLSession.bytes with line-by-line JSON parsing -- **No changes needed** to AgentRunner or session infrastructure - -### Implementation Priority: -1. **Phase 1-2**: Core OllamaModel implementation (3-5 days) -2. **Phase 3**: Streaming support (1-2 days) -3. **Phase 4**: Tool calling (2 days) -4. **Phase 5-7**: Model registration & testing (4-5 days) - -**Total: 10-14 days for complete implementation** - -## Next Steps - -1. Begin Phase 1: Move OllamaProvider to Core and create OllamaModel -2. Implement ModelInterface protocol conformance -3. Add streaming support with proper JSON parsing -4. Test with Llama 3.1 for tool calling verification -5. Add Ultrathink support when model becomes available - -## References - -### Official Documentation -- [Ollama Tool Support Blog](https://ollama.com/blog/tool-support) -- [Streaming with Tool Calling](https://ollama.com/blog/streaming-tool) -- [Python Library v0.4 with Functions](https://ollama.com/blog/functions-as-tools) -- [Models with Tool Support](https://ollama.com/search?c=tools) - -### Implementation Examples -- IBM's [Ollama Tool Calling Tutorial](https://www.ibm.com/think/tutorials/local-tool-calling-ollama-granite) -- [Function Calling with Gemma3](https://medium.com/google-cloud/function-calling-with-gemma3-using-ollama-120194577fa6) - -### Key Insights -1. Tool calling officially supported as of 2025 -2. Streaming now works with tool calls (improved parser) -3. 32k+ context window recommended for better tool performance -4. Models page has dedicated "Tools" category -5. Python SDK v0.4+ allows direct function passing +Official references: [chat API](https://docs.ollama.com/api/chat), +[tool calling](https://docs.ollama.com/capabilities/tool-calling), and +[OpenAI compatibility](https://docs.ollama.com/api/openai-compatibility). From c6cf1b49de18ca32354d179821fa6ccc2a6de5bc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:28:03 +0100 Subject: [PATCH 06/11] fix(agent): bind Ollama reasoning replay to authentication --- .../PeekabooAgentService+Execution.swift | 4 +- .../PeekabooAgentService+Streaming.swift | 44 ++++++++++++++----- ...ooAgentOllamaStreamingReasoningTests.swift | 37 ++++++++++++++++ Tachikoma | 2 +- 4 files changed, 74 insertions(+), 13 deletions(-) diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift index 830c91b7d..1a0d86d19 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Execution.swift @@ -476,7 +476,8 @@ extension PeekabooAgentService { messages: state.messages.sanitizedForProviderContext( model: configuration.model, configuration: resolvedConfiguration, - peekabooConfiguration: self.services.configuration), + peekabooConfiguration: self.services.configuration, + provider: provider), tools: configuration.tools.isEmpty ? nil : configuration.tools, settings: self.generationSettings(for: configuration.model)) let response = try await provider.generateText(request: request) @@ -506,6 +507,7 @@ extension PeekabooAgentService { model: configuration.model, configuration: resolvedConfiguration, peekabooConfiguration: self.services.configuration, + provider: provider, to: &state.messages) if toolCalls.isEmpty { diff --git a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift index c1a4ad98c..54b818e85 100644 --- a/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift +++ b/Core/PeekabooCore/Sources/PeekabooAgentRuntime/Agent/PeekabooAgentService+Streaming.swift @@ -251,7 +251,8 @@ extension PeekabooAgentService { let shouldReplayReasoning = ReasoningReplayTarget( model: configuration.model, configuration: resolvedConfiguration, - peekabooConfiguration: self.services.configuration) != nil + peekabooConfiguration: self.services.configuration, + provider: configuration.provider) != nil if shouldReplayReasoning { for block in output.reasoningBlocks { self.appendReasoningBlock( @@ -262,6 +263,7 @@ extension PeekabooAgentService { model: configuration.model, configuration: resolvedConfiguration, peekabooConfiguration: self.services.configuration, + provider: configuration.provider, to: &state.messages) } } @@ -442,6 +444,7 @@ extension PeekabooAgentService { model: LanguageModel, configuration: TachikomaConfiguration, peekabooConfiguration: ConfigurationManager? = nil, + provider: (any ModelProvider)? = nil, to messages: inout [ModelMessage]) { messages.append(ModelMessage( @@ -452,7 +455,8 @@ extension PeekabooAgentService { for: block, model: model, configuration: configuration, - peekabooConfiguration: peekabooConfiguration)))) + peekabooConfiguration: peekabooConfiguration, + provider: provider)))) } func appendResponseHistory( @@ -460,6 +464,7 @@ extension PeekabooAgentService { model: LanguageModel, configuration: TachikomaConfiguration, peekabooConfiguration: ConfigurationManager? = nil, + provider: (any ModelProvider)? = nil, to messages: inout [ModelMessage]) -> Bool { @@ -476,7 +481,8 @@ extension PeekabooAgentService { for: block, model: model, configuration: configuration, - peekabooConfiguration: peekabooConfiguration)))) + peekabooConfiguration: peekabooConfiguration, + provider: provider)))) } let toolCalls = response.toolCalls ?? [] @@ -507,7 +513,8 @@ extension PeekabooAgentService { for block: ProviderReasoningBlock, model: LanguageModel, configuration: TachikomaConfiguration, - peekabooConfiguration: ConfigurationManager? = nil) + peekabooConfiguration: ConfigurationManager? = nil, + provider: (any ModelProvider)? = nil) -> [String: String] { if let rawJSON = block.rawJSON, @@ -536,7 +543,8 @@ extension PeekabooAgentService { let target = ReasoningReplayTarget( model: model, configuration: configuration, - peekabooConfiguration: peekabooConfiguration), + peekabooConfiguration: peekabooConfiguration, + provider: provider), target.provider == "kimi" { var metadata = [ @@ -555,7 +563,8 @@ extension PeekabooAgentService { let target = ReasoningReplayTarget( model: model, configuration: configuration, - peekabooConfiguration: peekabooConfiguration), + peekabooConfiguration: peekabooConfiguration, + provider: provider), target.provider == "ollama" { var metadata = [ @@ -1076,13 +1085,15 @@ extension [ModelMessage] { func sanitizedForProviderContext( model: LanguageModel, configuration: TachikomaConfiguration, - peekabooConfiguration: ConfigurationManager? = nil) + peekabooConfiguration: ConfigurationManager? = nil, + provider: (any ModelProvider)? = nil) -> [ModelMessage] { let target = ReasoningReplayTarget( model: model, configuration: configuration, - peekabooConfiguration: peekabooConfiguration) + peekabooConfiguration: peekabooConfiguration, + provider: provider) var previousSourceWasRetainedThinking = false var sanitizedMessages: [ModelMessage] = [] sanitizedMessages.reserveCapacity(self.count) @@ -1176,12 +1187,15 @@ private struct ReasoningReplayTarget { let baseURL: String? let allowsReasoningBoundaries: Bool let allowsLegacyUnknown: Bool + private var verifiedEndpointIdentity: String? init?( model: LanguageModel, configuration: TachikomaConfiguration, - peekabooConfiguration: ConfigurationManager? = nil) + peekabooConfiguration: ConfigurationManager? = nil, + provider: (any ModelProvider)? = nil) { + self.verifiedEndpointIdentity = nil switch model { case let .anthropic(anthropicModel): self.provider = "anthropic" @@ -1220,11 +1234,19 @@ private struct ReasoningReplayTarget { self.allowsReasoningBoundaries = true self.allowsLegacyUnknown = false case let .ollama(model): + guard + let ollamaProvider = provider as? OllamaProvider, + ollamaProvider.modelId == model.modelId, + let replayIdentity = ollamaProvider.reasoningReplayIdentity + else { + return nil + } self.provider = "ollama" self.modelId = model.modelId - self.baseURL = (try? configuration.makeProvider(for: .ollama(model)))?.baseURL + self.baseURL = ollamaProvider.baseURL self.allowsReasoningBoundaries = true self.allowsLegacyUnknown = false + self.verifiedEndpointIdentity = replayIdentity case let .custom(provider): if let anthropicProvider = provider as? AnthropicProvider { self.provider = "anthropic" @@ -1289,6 +1311,6 @@ private struct ReasoningReplayTarget { } var endpointIdentity: String? { - PeekabooAgentService.canonicalEndpointIdentity(self.baseURL) + self.verifiedEndpointIdentity ?? PeekabooAgentService.canonicalEndpointIdentity(self.baseURL) } } diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift index 1b1e8e39a..4f1292f12 100644 --- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentOllamaStreamingReasoningTests.swift @@ -6,6 +6,43 @@ import Testing @Suite(.serialized) struct PeekabooAgentOllamaStreamingReasoningTests { + @Test(arguments: [false, true]) + @MainActor + func `Ollama thinking replay follows the provider authentication boundary`(_ authenticated: Bool) throws { + let configuration = TachikomaConfiguration(loadFromEnvironment: false) + configuration.setBaseURL("http://127.0.0.1:11434", for: .ollama) + if authenticated { + configuration.setAPIKey("test-key", for: .ollama) + } + + let model = LanguageModel.ollama(.custom("qwen3:8b")) + let provider = try OllamaProvider(model: .custom("qwen3:8b"), configuration: configuration) + let agentService = try PeekabooAgentService(services: PeekabooServices(), defaultModel: model) + var messages: [ModelMessage] = [] + + agentService.appendReasoningBlock( + ProviderReasoningBlock(text: "private thinking", type: "ollama_thinking"), + model: model, + configuration: configuration, + provider: provider, + to: &messages) + + let customData = try #require(messages.first?.metadata?.customData) + let sanitized = messages.sanitizedForProviderContext( + model: model, + configuration: configuration, + provider: provider) + if authenticated { + #expect(provider.reasoningReplayIdentity == nil) + #expect(customData["ollama.thinking"] == nil) + #expect(sanitized.isEmpty) + } else { + #expect(customData["ollama.thinking"] == "private thinking") + #expect(customData["tachikoma.reasoning.base_url"] == provider.reasoningReplayIdentity) + #expect(sanitized.count == 1) + } + } + @Test @MainActor func `Streaming Ollama thinking is not replayed to an injected provider`() async throws { diff --git a/Tachikoma b/Tachikoma index 06385196f..aa34da4b9 160000 --- a/Tachikoma +++ b/Tachikoma @@ -1 +1 @@ -Subproject commit 06385196fc5c5f3f9834d3324c32ec40ed69e486 +Subproject commit aa34da4b9eb40d5a3aecf40fd5c6bcc607322310 From c7762c50bfd93296aff213602da4586812658efa Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:43:50 +0100 Subject: [PATCH 07/11] test(agent): keep resume continuity credential-free --- .../PeekabooAgentResumeModelContinuityTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift index 4d80d3c8d..a51b1a3a2 100644 --- a/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift +++ b/Core/PeekabooCore/Tests/PeekabooTests/PeekabooAgentResumeModelContinuityTests.swift @@ -37,10 +37,10 @@ struct PeekabooAgentResumeModelContinuityTests { modelSelection: "removed-provider/private-model") let resolved = try service.resolveContinuationModel( - explicitModel: .anthropic(.opus48), + explicitModel: .lmstudio(.gptOSS120B), session: session) - #expect(resolved == .anthropic(.opus48)) + #expect(resolved == .lmstudio(.gptOSS120B)) } @Test From d4d46b86b35b229298d684df24b90196583a1cb9 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:53:10 +0100 Subject: [PATCH 08/11] fix(cli): route taskless resume into chat --- .../PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift | 6 +++++- .../Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift | 5 ++++- .../AgentCommandValidationIntegrationTests.swift | 1 + .../CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift index 29a0bd040..fb642473a 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentChatLaunchPolicy.swift @@ -44,10 +44,14 @@ struct AgentChatLaunchPolicy { return .interactive(initialPrompt: context.normalizedTaskInput) } - if context.hasTaskInput || context.listSessions || context.hasSessionResumption { + if context.hasTaskInput || context.listSessions { return .none } + if context.hasSessionResumption { + return .interactive(initialPrompt: nil) + } + if context.capabilities.isInteractive && !context.capabilities.isPiped && !context.capabilities.isCI { return .interactive(initialPrompt: nil) } diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift index e4f1139c8..ca938ee55 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift @@ -3,6 +3,7 @@ // PeekabooCLI // +import Commander import Foundation import PeekabooAgentRuntime import PeekabooCore @@ -55,7 +56,9 @@ extension AgentCommand { capabilities: TerminalCapabilities, queueMode: QueueMode ) async throws { - guard self.ensureChatModePreconditions() else { return } + guard self.ensureChatModePreconditions() else { + throw ExitCode.failure + } if capabilities.isInteractive && !capabilities.isPiped { do { diff --git a/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift index 7045f8883..ecc8d9cb6 100644 --- a/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift +++ b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift @@ -29,6 +29,7 @@ struct AgentCommandValidationIntegrationTests { "agent", "--resume-session", "missing-session-\(UUID().uuidString)", + "continue the task", "--jsonOutput", ], allowedExitCodes: [1] diff --git a/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift b/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift index c1fd8e8e2..d56e1045e 100644 --- a/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift +++ b/Apps/CLI/Tests/CoreCLITests/AgentChatLaunchPolicyTests.swift @@ -99,7 +99,7 @@ struct AgentChatLaunchPolicyTests { } @Test - func `Taskless session resume bypasses noninteractive help`() { + func `Taskless session resume enters chat even without an interactive terminal`() { let strategy = self.policy.strategy( for: AgentChatLaunchContext( chatFlag: false, @@ -111,6 +111,6 @@ struct AgentChatLaunchPolicyTests { ) ) - #expect(strategy == .none) + #expect(strategy == .interactive(initialPrompt: nil)) } } From 473f55300f0081edf2c9a5c9189f6913b92bfc91 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 03:57:01 +0100 Subject: [PATCH 09/11] fix(cli): fail invalid chat session resume --- .../PeekabooCLI/Commands/AI/AgentCommand+Chat.swift | 6 ++++-- .../AgentCommandValidationIntegrationTests.swift | 11 ++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift index ca938ee55..bccf83394 100644 --- a/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift +++ b/Apps/CLI/Sources/PeekabooCLI/Commands/AI/AgentCommand+Chat.swift @@ -70,6 +70,8 @@ extension AgentCommand { queueMode: queueMode ) return + } catch is ExitCode { + throw ExitCode.failure } catch { self.printAgentExecutionError( "Failed to launch TauTUI chat: \(error.localizedDescription). Falling back to basic chat." @@ -104,7 +106,7 @@ extension AgentCommand { turnContext.sessionId = try await self.initialChatSessionId(agentService) } catch { self.printAgentExecutionError(error.localizedDescription) - return + throw ExitCode.failure } self.printChatWelcome( @@ -160,7 +162,7 @@ extension AgentCommand { activeSessionId = try await self.initialChatSessionId(agentService) } catch { self.printAgentExecutionError(error.localizedDescription) - return + throw ExitCode.failure } let chatUI = AgentChatUI( diff --git a/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift index ecc8d9cb6..3ffae730e 100644 --- a/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift +++ b/Apps/CLI/Tests/CLIAutomationTests/AgentCommandValidationIntegrationTests.swift @@ -23,24 +23,17 @@ struct AgentCommandValidationIntegrationTests { } @Test - func `Missing resume session returns one JSON error`() async throws { + func `Taskless missing resume session returns a failing error`() async throws { let result = try await InProcessCommandRunner.runShared( [ "agent", "--resume-session", "missing-session-\(UUID().uuidString)", - "continue the task", - "--jsonOutput", ], allowedExitCodes: [1] ) #expect(result.exitStatus == 1) - - let data = try #require(result.stdout.trimmingCharacters(in: .whitespacesAndNewlines).data(using: .utf8)) - let payload = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) - #expect(payload.count == 2) - #expect(payload["success"] as? Bool == false) - #expect((payload["error"] as? String)?.contains("Failed to resume session") == true) + #expect(result.stdout.contains("Session not found or expired")) } } From ad933ac66f8d1e446d20b9cb8f243dacde3946ec Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 04:01:40 +0100 Subject: [PATCH 10/11] docs(agent): correct Ollama streaming behavior --- docs/providers/README.md | 2 +- docs/providers/ollama.md | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/providers/README.md b/docs/providers/README.md index f04a04533..780d3ec44 100644 --- a/docs/providers/README.md +++ b/docs/providers/README.md @@ -31,7 +31,7 @@ configuration syntax, and environment variable reference. | MiniMax China | Yes | Model-dependent | Yes | No | API key | | Kimi | Yes | Yes | Yes | No | API key | | Grok | Yes | Limited | Yes | No | API key | -| Ollama | Model-dependent | Model-dependent | Response-buffered NDJSON | Model-dependent; disable cloud for strict local use | None for local models | +| Ollama | Model-dependent | Model-dependent | Incremental NDJSON | Model-dependent; disable cloud for strict local use | None for local models | | LM Studio | Yes (OpenAI-compatible local server) | Model-dependent | Yes | **Yes** (local) | None by default | See individual pages for model lists, quirks, and test coverage expectations. diff --git a/docs/providers/ollama.md b/docs/providers/ollama.md index 3a0a17464..3dcdfc915 100644 --- a/docs/providers/ollama.md +++ b/docs/providers/ollama.md @@ -101,9 +101,8 @@ returning an empty success. Resume that session to continue from the preserved t Ollama's native chat API returns newline-delimited JSON when streaming. Peekaboo asks for that format and validates each chunk, including [errors delivered after an HTTP 200 response](https://docs.ollama.com/api/errors). -The current Tachikoma transport buffers the HTTP response before emitting its decoded deltas. Tool calls and errors -follow streaming semantics, but CLI text appears response-by-response rather than token-by-token. Do not rely on -token-live output for progress or timeout decisions. +Tachikoma parses each NDJSON chunk as it arrives and emits text deltas incrementally. The exact chunk cadence depends +on the model and Ollama server; tool calls become actionable only after Ollama has supplied their complete arguments. ## Privacy boundary From fb0b0fdb1eff0e3226cb3509c5b704105a3c797d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 11 Jul 2026 04:03:18 +0100 Subject: [PATCH 11/11] docs(agent): align Ollama command streaming notes --- docs/commands/agent.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/agent.md b/docs/commands/agent.md index 3ee83c8dc..00adb5bb2 100644 --- a/docs/commands/agent.md +++ b/docs/commands/agent.md @@ -35,7 +35,7 @@ read_when: model turn to interpret their results. This avoids reporting an empty success when the step budget expires with pending work; resume the reported session to continue. - Native `ollama/` runs replay each assistant tool call and named tool result on the next model turn. Ollama - support is model-dependent, and current native stream output is response-buffered rather than token-live. See the + support is model-dependent, and native text arrives incrementally with a model-dependent chunk cadence. See the [Ollama guide](../providers/ollama.md). ## Chat mode