deepeval/typescript/src/tracing/api.ts at 3898ea53f1fe1ea609ea4a3588b025cbd5ba0836 · A-Vamshi/deepeval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import { Prompt } from "../prompt";

export enum SpanApiType {
  BASE = "base",
  AGENT = "agent",
  LLM = "llm",
  RETRIEVER = "retriever",
  TOOL = "tool",
}

export enum TraceSpanApiStatus {
  SUCCESS = "SUCCESS",
  ERROR = "ERRORED",
}

export interface ToolCall {
  name: string;
  description?: string;
  inputParameters?: Record<string, any>;
  output?: any;
  reasoning?: string;
}

export interface LLMTestCase {
  input: string;
  actualOutput: string;
  expectedOutput?: string;
  context?: string[];
  retrievalContext?: string[];
  toolsCalled?: ToolCall[];
  expectedTools?: ToolCall[];
}

export interface BaseApiSpan {
  uuid: string;
  name?: string;
  status: TraceSpanApiStatus;
  type: SpanApiType | string;
  traceUuid: string;
  parentUuid?: string;
  startTime: string;
  endTime: string;
  input?: any;
  output?: any;
  error?: string;

  // agents
  availableTools?: string[];
  agentHandoffs?: string[];

  // tools
  description?: string;

  // retriever
  embedder?: string;
  topK?: number;
  chunkSize?: number;

  // llm
  model?: string;
  inputTokenCount?: number;
  outputTokenCount?: number;
  costPerInputToken?: number;
  costPerOutputToken?: number;
  prompt?: Prompt;
  promptCommitHash?: string;
  promptAlias?: string;
  promptLabel?: string;
  promptVersion?: string;

  // evals
  llmTestCase?: LLMTestCase;
  metrics?: string[];
  metricsData?: unknown[];

  // metadata
  metadata?: Record<string, any>;
  metricCollection?: string;

  // additional test case params
  expectedOutput?: string;
  retrievalContext?: string[];
  context?: string[];
  toolsCalled?: ToolCall[];
  expectedTools?: ToolCall[];
}

export interface TraceApi {
  uuid: string;
  status?: TraceSpanApiStatus;
  baseSpans: BaseApiSpan[];
  agentSpans: BaseApiSpan[];
  llmSpans: BaseApiSpan[];
  retrieverSpans: BaseApiSpan[];
  toolSpans: BaseApiSpan[];
  startTime: string;
  endTime: string;
  environment: string;
  metadata?: Record<string, any>;
  tags?: string[];
  threadId?: string;
  userId?: string;
  testCaseId?: string;
  testRunId?: string;
  turnId?: string;
  input?: any;
  output?: any;
  name?: string;

  // additional test case params
  expectedOutput?: string;
  retrievalContext?: string[];
  context?: string[];
  toolsCalled?: ToolCall[];
  expectedTools?: ToolCall[];
  metricCollection?: string;
  metricsData?: unknown[];

  // Don't serialize this
  confidentApiKey?: string;
}