diff --git a/multimodal/tarko/agent-interface/src/agent-event-stream.ts b/multimodal/tarko/agent-interface/src/agent-event-stream.ts index 74920e4259..d3dcca05e4 100644 --- a/multimodal/tarko/agent-interface/src/agent-event-stream.ts +++ b/multimodal/tarko/agent-interface/src/agent-event-stream.ts @@ -118,8 +118,23 @@ export namespace AgentEventStream { /** How the response was finished */ finishReason?: string; - /** Time taken to generate this response */ - elapsedMs?: number; + /** + * Time to First Token (TTFT) in milliseconds - time from request start to first content chunk. + * The time it takes for the model to return the first token of the response after it receives the prompt. + * + * @see https://modal.com/llm-almanac/how-to-benchmark + * @see https://cloud.google.com/vertex-ai/generative-ai/docs/learn/prompt-best-practices + */ + ttftMs?: number; + + /** + * Time to Last Token (TTLT) in milliseconds - time from request start to response completion. + * The overall time taken by the model to process the prompt and generate the complete response. + * + * @see https://modal.com/llm-almanac/how-to-benchmark + * @see https://cloud.google.com/vertex-ai/generative-ai/docs/learn/prompt-best-practices + */ + ttltMs?: number; /** * Unique message identifier that links streaming messages to their final message diff --git a/multimodal/tarko/agent-interface/src/agent-options.ts b/multimodal/tarko/agent-interface/src/agent-options.ts index dd0c15ff39..01669f74a6 100644 --- a/multimodal/tarko/agent-interface/src/agent-options.ts +++ b/multimodal/tarko/agent-interface/src/agent-options.ts @@ -184,6 +184,19 @@ export interface AgentMemoryOptions { enableStreamingToolCallEvents?: boolean; } +/** + * Metric configuration options for performance monitoring + */ +export interface AgentMetricOptions { + /** + * Whether to enable metric collection (TTFT, TTLT, etc.) + * When disabled, timing metrics will not be collected or included in event streams. + * + * @defaultValue `false` + */ + enable?: boolean; +} + /** * Miscellaneous configuration options for logging and debugging */ @@ -194,6 +207,11 @@ export interface AgentMiscOptions { * @defaultValue `LogLevel.INFO` in development, `LogLevel.WARN` in production */ logLevel?: LogLevel; + + /** + * Metric collection settings + */ + metric?: AgentMetricOptions; } /** diff --git a/multimodal/tarko/agent-snapshot/src/utils/snapshot-normalizer.ts b/multimodal/tarko/agent-snapshot/src/utils/snapshot-normalizer.ts index ea50c0557c..f6a72f4a9e 100644 --- a/multimodal/tarko/agent-snapshot/src/utils/snapshot-normalizer.ts +++ b/multimodal/tarko/agent-snapshot/src/utils/snapshot-normalizer.ts @@ -42,6 +42,8 @@ const DEFAULT_CONFIG: AgentNormalizerConfig = { { pattern: 'toolCallId', replacement: '<>' }, { pattern: 'sessionId', replacement: '<>' }, { pattern: 'messageId', replacement: '<>' }, + { pattern: 'ttftMs', replacement: '<>' }, + { pattern: 'ttltMs', replacement: '<>' }, { pattern: /Time$/, replacement: '<>' }, ], fieldsToIgnore: [], diff --git a/multimodal/tarko/agent-web-ui/src/common/state/actions/eventProcessors/handlers/MessageHandler.ts b/multimodal/tarko/agent-web-ui/src/common/state/actions/eventProcessors/handlers/MessageHandler.ts index 5d2691b09c..a856a36253 100644 --- a/multimodal/tarko/agent-web-ui/src/common/state/actions/eventProcessors/handlers/MessageHandler.ts +++ b/multimodal/tarko/agent-web-ui/src/common/state/actions/eventProcessors/handlers/MessageHandler.ts @@ -93,6 +93,8 @@ export class AssistantMessageHandler toolCalls: event.toolCalls, finishReason: event.finishReason, isStreaming: false, + ttftMs: event.ttftMs, + ttltMs: event.ttltMs, }; return { @@ -114,6 +116,8 @@ export class AssistantMessageHandler toolCalls: event.toolCalls, finishReason: event.finishReason, messageId: messageId, + ttftMs: event.ttftMs, + ttltMs: event.ttltMs, }, ], }; diff --git a/multimodal/tarko/agent-web-ui/src/common/types/index.ts b/multimodal/tarko/agent-web-ui/src/common/types/index.ts index 165240d76b..37da49b0fc 100644 --- a/multimodal/tarko/agent-web-ui/src/common/types/index.ts +++ b/multimodal/tarko/agent-web-ui/src/common/types/index.ts @@ -11,8 +11,6 @@ export type { SanitizedAgentOptions, WorkspaceInfo, SessionItemInfo }; export type { ChatCompletionContentPart, ChatCompletionMessageToolCall }; - - /** * Tool result type with categorization and timing information */ @@ -46,6 +44,8 @@ export interface Message { description?: string; // Added for environment inputs isDeepResearch?: boolean; // Added for final answer events title?: string; // Added for research report title + ttftMs?: number; // Time to First Token (TTFT) in milliseconds + ttltMs?: number; // Total response time in milliseconds // System message specific properties level?: 'info' | 'warning' | 'error'; @@ -102,5 +102,3 @@ export interface ReplayEventMarker { position: number; // 0-1 normalized position on timeline content?: string | any; } - - diff --git a/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageFooter.tsx b/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageFooter.tsx new file mode 100644 index 0000000000..ae9c48d675 --- /dev/null +++ b/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageFooter.tsx @@ -0,0 +1,115 @@ +import React from 'react'; +import { FiClock, FiCheck, FiCopy, FiZap, FiActivity } from 'react-icons/fi'; +import { Tooltip, TooltipProps } from '@mui/material'; +import { formatTimestamp } from '@/common/utils/formatters'; +import { Message as MessageType, ChatCompletionContentPart } from '@/common/types'; +import { useCopyToClipboard } from '../hooks/useCopyToClipboard'; + +interface MessageFooterProps { + message: MessageType; + className?: string; +} + +/** + * MessageFooter Component + * Displays timestamp, copy functionality, and TTFT information for messages + */ +export const MessageFooter: React.FC = ({ message, className = '' }) => { + const { isCopied, copyToClipboard } = useCopyToClipboard(); + const showTTFT = message.role === 'assistant' && message.ttftMs !== undefined; + + const handleCopy = () => { + const textToCopy = + typeof message.content === 'string' + ? message.content + : (message.content as ChatCompletionContentPart[]) + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join('\n'); + + copyToClipboard(textToCopy); + }; + + // Helper function to format elapsed time for display (always in ms for precision) + const formatElapsedTime = (ms: number): string => { + return `${ms}ms`; + }; + + // Tooltip styling for consistent appearance + const tooltipProps: Partial = { + arrow: true, + componentsProps: { + tooltip: { + sx: { + backgroundColor: '#000000', + color: '#ffffff', + fontSize: '13px', + fontWeight: 500, + padding: '8px 12px', + borderRadius: '6px', + boxShadow: '0 4px 12px rgba(0, 0, 0, 0.3)', + '.MuiTooltip-arrow': { + color: '#000000', + }, + }, + }, + }, + }; + + return ( +
+
+
+ {/* Timestamp */} +
+ + {formatTimestamp(message.timestamp)} +
+ + {/* TTFT & TTLT Display with icons and consistent styling */} + {showTTFT && ( +
+ {/* TTFT */} + +
+ + + {formatElapsedTime(message.ttftMs!)} + +
+ + + {/* TTLT (if different from TTFT) */} + {message.ttltMs && message.ttltMs !== message.ttftMs && ( + +
+ + + {formatElapsedTime(message.ttltMs)} + +
+ + )} +
+ )} +
+ + {/* Copy functionality */} + +
+
+ ); +}; diff --git a/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageGroup.tsx b/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageGroup.tsx index 3abf18549a..7baf2c9c70 100644 --- a/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageGroup.tsx +++ b/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageGroup.tsx @@ -1,15 +1,12 @@ import React from 'react'; import { Message as MessageType } from '@/common/types'; import { Message } from '../index'; -import { FiClock } from 'react-icons/fi'; -import { formatTimestamp } from '@/common/utils/formatters'; import { isMultimodalContent } from '@/common/utils/typeGuards'; -import { MessageTimestamp } from './MessageTimestamp'; +import { MessageFooter } from './MessageFooter'; import { ThinkingAnimation } from './ThinkingAnimation'; import { SkeletonLoader } from './SkeletonLoader'; import { useAtomValue } from 'jotai'; import { agentStatusAtom } from '@/common/state/atoms/ui'; -import { AgentProcessingPhase } from '@tarko/interface'; import { getAgentTitle } from '@/common/constants'; interface MessageGroupProps { @@ -117,25 +114,8 @@ export const MessageGroup: React.FC = ({ messages, isThinking )} - {/* Timestamp and copy functionality */} - {!isThinking && lastResponseMessage && ( -
-
-
- - {formatTimestamp(lastResponseMessage.timestamp)} -
- - {/* Integrated copy function button - now uses the last message */} - -
-
- )} + {/* Message footer with timestamp, TTFT, and copy functionality */} + {!isThinking && lastResponseMessage && } ); }; diff --git a/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageTimestamp.tsx b/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageTimestamp.tsx deleted file mode 100644 index d09d87b93c..0000000000 --- a/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/components/MessageTimestamp.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import { ChatCompletionContentPart } from '@tarko/agent-interface'; -import React from 'react'; -import { FiCheck, FiCopy } from 'react-icons/fi'; -import { formatTimestamp } from '@/common/utils/formatters'; -import { useCopyToClipboard } from '../hooks/useCopyToClipboard'; - -interface MessageTimestampProps { - timestamp: number; - content: string | ChatCompletionContentPart[]; - role: string; - inlineStyle?: boolean; // New property for inline display mode -} - -/** - * Component for displaying message timestamp and copy functionality - * - * Design principles: - * - Unobtrusive placement to reduce visual noise - * - Accessible on hover for contextual actions - * - Clear visual feedback for copy operations - */ -export const MessageTimestamp: React.FC = ({ - timestamp, - content, - role, - inlineStyle = false, -}) => { - const { isCopied, copyToClipboard } = useCopyToClipboard(); - - const handleCopy = () => { - const textToCopy = - typeof content === 'string' - ? content - : content - .filter((part) => part.type === 'text') - .map((part) => part.text) - .join('\n'); - - copyToClipboard(textToCopy); - }; - - if (inlineStyle) { - // Inline style mode, only show copy button - return ( - - ); - } - - // Original floating style - return ( -
- {formatTimestamp(timestamp)} - -
- ); -}; diff --git a/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/index.tsx b/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/index.tsx index a65753410e..5630d023c2 100644 --- a/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/index.tsx +++ b/multimodal/tarko/agent-web-ui/src/standalone/chat/Message/index.tsx @@ -12,13 +12,12 @@ import { SystemMessage } from './components/SystemMessage'; import { MultimodalContent } from './components/MultimodalContent'; import { ToolCalls } from './components/ToolCalls'; import { ThinkingToggle } from './components/ThinkingToggle'; -import { MessageTimestamp } from './components/MessageTimestamp'; + import { useAtomValue } from 'jotai'; import { replayStateAtom } from '@/common/state/atoms/replay'; import { ReportFileEntry } from './components/ReportFileEntry'; import { messagesAtom } from '@/common/state/atoms/message'; - interface MessageProps { message: MessageType; shouldDisplayTimestamp?: boolean; @@ -74,8 +73,6 @@ export const Message: React.FC = ({ } }; - - // Render content based on type const renderContent = () => { if (isMultimodal) { @@ -131,8 +128,6 @@ export const Message: React.FC = ({ baseClasses = 'message-assistant'; } - - return baseClasses; }; @@ -146,8 +141,6 @@ export const Message: React.FC = ({ return imageContents.length > 0 && textContents.length === 0; }, [message.content]); - - // Determine which prose class should be used, based on message type and dark mode const getProseClasses = () => { if (message.role === 'user') { @@ -185,8 +178,6 @@ export const Message: React.FC = ({
{renderContent()}
- - {isFinalAnswer && message.title && typeof message.content === 'string' && ( = ({ )} - {/* Timestamp and copy button - only for main messages */} - {message.role !== 'system' && - !isInGroup && - shouldDisplayTimestamp && - !replayState.isActive && ( - - )} + ); }; diff --git a/multimodal/tarko/agent/snapshot/__snapshots__/index.test.ts.snap b/multimodal/tarko/agent/snapshot/__snapshots__/index.test.ts.snap index a8e1147aba..72bdbe7c7c 100644 --- a/multimodal/tarko/agent/snapshot/__snapshots__/index.test.ts.snap +++ b/multimodal/tarko/agent/snapshot/__snapshots__/index.test.ts.snap @@ -66,7 +66,9 @@ exports[`AgentSnapshot tests > should match snapshot for gui-agent/basic 2`] = ` } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -152,7 +154,9 @@ exports[`AgentSnapshot tests > should match snapshot for gui-agent/basic 2`] = ` } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -238,7 +242,9 @@ exports[`AgentSnapshot tests > should match snapshot for gui-agent/basic 2`] = ` } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -314,7 +320,9 @@ exports[`AgentSnapshot tests > should match snapshot for gui-agent/basic 2`] = ` "content": "Agent TARS is Bytedance's new open-source approach for automating complex tasks by visually interpreting web content and interacting with the command line and file system.", "rawContent": "Agent TARS is Bytedance's new open-source approach for automating complex tasks by visually interpreting web content and interacting with the command line and file system.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -336,7 +344,9 @@ exports[`AgentSnapshot tests > should match snapshot for gui-agent/basic 3`] = ` "content": "Agent TARS is Bytedance's new open-source approach for automating complex tasks by visually interpreting web content and interacting with the command line and file system.", "rawContent": "Agent TARS is Bytedance's new open-source approach for automating complex tasks by visually interpreting web content and interacting with the command line and file system.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" } `; @@ -618,7 +628,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls 2` } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -919,7 +931,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls 2` } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -1851,7 +1865,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls 2` "content": "The weather information for Boston is now available, so we can directly respond with the details.\\nIn Boston, the weather today is Sunny. The temperature is 70°F (21°C), with a 10% chance of precipitation, 45% humidity, and wind speed of 5 mph.In Boston, the weather today is Sunny. The temperature is 70°F (21°C), with a 10% chance of precipitation, 45% humidity, and wind speed of 5 mph.", "rawContent": "", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -1943,7 +1959,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls-pr } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2130,7 +2148,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls-pr } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2542,7 +2562,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls-pr "content": "Today in Boston, the weather is sunny with a temperature of 70°F (21°C). The precipitation chance is 10%, humidity is 45%, and the wind is blowing at 5 mph.", "rawContent": "Today in Boston, the weather is sunny with a temperature of 70°F (21°C). The precipitation chance is 10%, humidity is 45%, and the wind is blowing at 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2628,7 +2650,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls-st } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2715,7 +2739,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls-st } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2823,7 +2849,9 @@ exports[`AgentSnapshot tests > should match snapshot for streaming/tool-calls-st "content": "The weather in Boston today is sunny with a temperature of 70°F (21°C). There's low precipitation chance at 10%, humidity is at 45%, and a light wind of 5 mph.", "rawContent": "The weather in Boston today is sunny with a temperature of 70°F (21°C). There's low precipitation chance at 10%, humidity is at 45%, and a light wind of 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2885,7 +2913,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/basic 2`] = } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2932,7 +2962,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/basic 2`] = } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -2984,7 +3016,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/basic 2`] = "content": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "rawContent": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3006,7 +3040,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/basic 3`] = "content": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "rawContent": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" } `; @@ -3055,7 +3091,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/prompt-engin } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3102,7 +3140,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/prompt-engin } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3154,7 +3194,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/prompt-engin "content": "In Boston today, the weather is sunny with a temperature of 70°F (21°C). The precipitation chance is 10%, humidity is 45%, and the wind speed is 5 mph.", "rawContent": "In Boston today, the weather is sunny with a temperature of 70°F (21°C). The precipitation chance is 10%, humidity is 45%, and the wind speed is 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3176,7 +3218,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/prompt-engin "content": "In Boston today, the weather is sunny with a temperature of 70°F (21°C). The precipitation chance is 10%, humidity is 45%, and the wind speed is 5 mph.", "rawContent": "In Boston today, the weather is sunny with a temperature of 70°F (21°C). The precipitation chance is 10%, humidity is 45%, and the wind speed is 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" } `; @@ -3225,7 +3269,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3272,7 +3318,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3324,7 +3372,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o "content": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "rawContent": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3346,7 +3396,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o "content": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "rawContent": "Today in Boston, the weather is Sunny with a temperature of 70°F (21°C). Precipitation is 10%, humidity is 45%, and wind speed is 5 mph.", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" } `; @@ -3395,7 +3447,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3442,7 +3496,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o } ], "finishReason": "tool_calls", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3494,7 +3550,9 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o "content": "Today in Boston, it's 70°F (21°C) and sunny with only a 10% chance of precipitation. The humidity is at 45% with light winds at 5 mph. It's a beautiful day!", "rawContent": "Today in Boston, it's 70°F (21°C) and sunny with only a 10% chance of precipitation. The humidity is at 45% with light winds at 5 mph. It's a beautiful day!", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" }, { "id": "<>", @@ -3516,6 +3574,8 @@ exports[`AgentSnapshot tests > should match snapshot for tool-calls/structured-o "content": "Today in Boston, it's 70°F (21°C) and sunny with only a 10% chance of precipitation. The humidity is at 45% with light winds at 5 mph. It's a beautiful day!", "rawContent": "Today in Boston, it's 70°F (21°C) and sunny with only a 10% chance of precipitation. The humidity is at 45% with light winds at 5 mph. It's a beautiful day!", "finishReason": "stop", - "messageId": "<>" + "messageId": "<>", + "ttftMs": "<>", + "ttltMs": "<>" } `; diff --git a/multimodal/tarko/agent/src/agent/agent-runner.ts b/multimodal/tarko/agent/src/agent/agent-runner.ts index c430cd8575..cec0e00614 100644 --- a/multimodal/tarko/agent/src/agent/agent-runner.ts +++ b/multimodal/tarko/agent/src/agent/agent-runner.ts @@ -44,6 +44,7 @@ interface AgentRunnerOptions { agent: Agent; contextAwarenessOptions?: AgentContextAwarenessOptions; enableStreamingToolCallEvents: boolean; + enableMetrics: boolean; } /** @@ -65,6 +66,7 @@ export class AgentRunner { private agent: Agent; private contextAwarenessOptions?: AgentContextAwarenessOptions; private enableStreamingToolCallEvents: boolean; + private enableMetrics: boolean; private logger = getLogger('AgentRunner'); // Specialized components @@ -85,6 +87,7 @@ export class AgentRunner { this.agent = options.agent; this.contextAwarenessOptions = options.contextAwarenessOptions; this.enableStreamingToolCallEvents = options.enableStreamingToolCallEvents; + this.enableMetrics = options.enableMetrics; // Initialize the specialized components this.toolProcessor = new ToolProcessor(this.agent, this.toolManager, this.eventStream); @@ -99,6 +102,7 @@ export class AgentRunner { this.top_p, this.contextAwarenessOptions, this.enableStreamingToolCallEvents, + this.enableMetrics, ); this.loopExecutor = new LoopExecutor( @@ -196,6 +200,7 @@ export class AgentRunner { return this.eventStream.createEvent('assistant_message', { content: 'Request was aborted', finishReason: 'abort', + ttltMs: 0, // Immediate abort, no processing time }); } else { // Handle other types of errors diff --git a/multimodal/tarko/agent/src/agent/agent.ts b/multimodal/tarko/agent/src/agent/agent.ts index 93a6e1b789..ddc7d53667 100644 --- a/multimodal/tarko/agent/src/agent/agent.ts +++ b/multimodal/tarko/agent/src/agent/agent.ts @@ -159,6 +159,7 @@ export class Agent agent: this, contextAwarenessOptions: contextAwarenessOptions, enableStreamingToolCallEvents: options.enableStreamingToolCallEvents ?? false, + enableMetrics: options.metric?.enable ?? false, }); // Initialize execution controller diff --git a/multimodal/tarko/agent/src/agent/runner/llm-processor.ts b/multimodal/tarko/agent/src/agent/runner/llm-processor.ts index 009cb626cc..2c2877daea 100644 --- a/multimodal/tarko/agent/src/agent/runner/llm-processor.ts +++ b/multimodal/tarko/agent/src/agent/runner/llm-processor.ts @@ -39,6 +39,7 @@ export class LLMProcessor { private messageHistory: MessageHistory; private llmClient?: OpenAI; private enableStreamingToolCallEvents: boolean; + private enableMetrics: boolean; constructor( private agent: Agent, @@ -50,12 +51,14 @@ export class LLMProcessor { private top_p?: number, private contextAwarenessOptions?: AgentContextAwarenessOptions, enableStreamingToolCallEvents = false, + enableMetrics = false, ) { this.messageHistory = new MessageHistory( this.eventStream, this.contextAwarenessOptions?.maxImagesCount, ); this.enableStreamingToolCallEvents = enableStreamingToolCallEvents; + this.enableMetrics = enableMetrics; } /** @@ -203,7 +206,7 @@ export class LLMProcessor { }; // Process the request - const startTime = Date.now(); + const startTime = this.enableMetrics ? Date.now() : 0; await this.sendRequest( resolvedModel, @@ -211,11 +214,14 @@ export class LLMProcessor { sessionId, toolCallEngine, streamingMode, + startTime, abortSignal, ); - const duration = Date.now() - startTime; - this.logger.info(`[LLM] Response received | Duration: ${duration}ms`); + if (this.enableMetrics) { + const duration = Date.now() - startTime; + this.logger.info(`[LLM] Response received | Duration: ${duration}ms`); + } } /** @@ -227,6 +233,7 @@ export class LLMProcessor { sessionId: string, toolCallEngine: ToolCallEngine, streamingMode: boolean, + requestStartTime: number, abortSignal?: AbortSignal, ): Promise { // Check if operation was aborted @@ -260,6 +267,7 @@ export class LLMProcessor { sessionId, toolCallEngine, streamingMode, + requestStartTime, abortSignal, ); } @@ -274,6 +282,7 @@ export class LLMProcessor { sessionId: string, toolCallEngine: ToolCallEngine, streamingMode: boolean, + requestStartTime: number, abortSignal?: AbortSignal, ): Promise { // Collect all chunks for final onLLMResponse call @@ -285,6 +294,10 @@ export class LLMProcessor { // Generate a unique message ID to correlate streaming messages with final message const messageId = `msg_${Date.now()}_${Math.random().toString(36).substring(2, 10)}`; + // Track TTFT (Time to First Token) only if metrics are enabled + let firstTokenTime: number | null = null; + let hasReceivedFirstContent = false; + this.logger.info(`llm stream start`); // Process each incoming chunk @@ -300,6 +313,20 @@ export class LLMProcessor { // Process the chunk using the tool call engine const chunkResult = toolCallEngine.processStreamingChunk(chunk, processingState); + // Track first token time only if metrics are enabled + if ( + this.enableMetrics && + !hasReceivedFirstContent /* && (chunkResult.content || chunkResult.reasoningContent) */ + ) { + firstTokenTime = Date.now(); + hasReceivedFirstContent = true; + if (requestStartTime > 0) { + // Only calculate if we have a valid start time + const ttft = firstTokenTime - requestStartTime; + this.logger.info(`[LLM] First token received | TTFT: ${ttft}ms`); + } + } + // Only send streaming events in streaming mode if (streamingMode) { // Send reasoning content if any @@ -356,6 +383,16 @@ export class LLMProcessor { this.logger.infoWithData('Finalized Response', parsedResponse, JSON.stringify); + // Calculate timing metrics only if enabled + let ttftMs: number | undefined; + let ttltMs: number | undefined; + + if (this.enableMetrics && requestStartTime > 0) { + ttltMs = Date.now() - requestStartTime; + ttftMs = firstTokenTime ? firstTokenTime - requestStartTime : ttltMs; + this.logger.info(`[LLM] Response timing | TTFT: ${ttftMs}ms | Total: ${ttltMs}ms`); + } + // Create the final events based on processed content this.createFinalEvents( parsedResponse.content || '', @@ -364,6 +401,8 @@ export class LLMProcessor { parsedResponse.reasoningContent || '', parsedResponse.finishReason || 'stop', messageId, // Pass the message ID to final events + ttftMs, // Pass the TTFT only if metrics were calculated + ttltMs, // Pass the TTLT only if metrics were calculated ); // Call response hooks with session ID @@ -420,6 +459,8 @@ export class LLMProcessor { reasoningBuffer: string, finishReason: string, messageId?: string, + ttftMs?: number, + ttltMs?: number, ): void { // If we have complete content, create a consolidated assistant message event if (content || currentToolCalls.length > 0) { @@ -429,6 +470,8 @@ export class LLMProcessor { toolCalls: currentToolCalls.length > 0 ? currentToolCalls : undefined, finishReason: finishReason, messageId: messageId, // Include the message ID in the final message + ttftMs: ttftMs, // Include the TTFT (Time to First Token) for display + ttltMs: ttltMs, // Include the total response time for analytics }); this.eventStream.sendEvent(assistantEvent);