Skip to content

Commit 2a6cd63

Browse files
committed
test(llm-tracing): add integration test for llm-dominates-request rule
Replace weak no-op anomaly test with a real end-to-end assertion: emits 'request' on agent with 1ms HTTP duration, runs OpenAI mock (10ms delay) inside runWithContext with matching traceId, then asserts 'anomaly' event with type 'llm-dominates-request' fires on the agent.
1 parent 09ca4a7 commit 2a6cd63

1 file changed

Lines changed: 49 additions & 15 deletions

File tree

packages/agent/tests/integration/llm-tracing.test.ts

Lines changed: 49 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { describe, it, beforeEach, afterEach } from "node:test";
22
import assert from "node:assert/strict";
33
import { ArgusAgent } from "../../src/argus-agent.ts";
44
import { requireRef } from "../../src/instrumentation/drivers/_require.ts";
5+
import { runWithContext } from "../../src/instrumentation/correlation.ts";
56
import type { LLMEvent } from "../../src/instrumentation/llm/types.ts";
67

78
function makeMockOpenAI() {
@@ -127,23 +128,56 @@ describe("ArgusAgent LLM tracing integration", () => {
127128
assert.strictEqual(events.length, 1, "no new events after stop");
128129
});
129130

130-
it("n-llm-calls anomaly fires as 'anomaly' event on agent", async () => {
131+
it("llm-dominates-request fires as 'anomaly' when LLM exceeds 80% of HTTP duration", async () => {
132+
// Slow mock: 10ms delay → LLM durationMs ≈ 10ms, easily > 80% of the 1ms HTTP duration
133+
const slowProto = {
134+
create: async (_params: Record<string, unknown>) => {
135+
await new Promise<void>((r) => setTimeout(r, 10));
136+
return {
137+
model: "gpt-4o",
138+
choices: [{ message: { content: "ok" } }],
139+
usage: { prompt_tokens: 5, completion_tokens: 3 },
140+
};
141+
},
142+
};
143+
const SlowMock = { prototype: { chat: { completions: slowProto } } };
144+
requireRef.current = Object.assign(
145+
(id: string) => (id === "openai" ? { default: SlowMock } : originalRequire(id)),
146+
originalRequire,
147+
) as typeof originalRequire;
148+
131149
agent = ArgusAgent.create().withLLMTracing({ providers: ["openai"] });
132150
await agent.start();
133151

134-
const anomalies: unknown[] = [];
135-
agent.on("anomaly", (a) => anomalies.push(a));
136-
137-
const mock = requireRef.current("openai") as { default: ReturnType<typeof makeMockOpenAI> };
138-
// Three calls with same traceId (context not set, so traceId is undefined — skip traceId rule)
139-
// Use costUsd spike instead: need 5 baseline calls first
140-
for (let i = 0; i < 5; i++) {
141-
await mock.default.prototype.chat.completions.create({
142-
model: "gpt-4o",
143-
messages: [{ role: "user", content: "hello" }],
144-
});
145-
}
146-
// No anomaly yet from n-llm-calls since no traceId — but no crash either
147-
assert.ok(agent.isRunning, "agent still running after multiple LLM calls");
152+
const anomalies: Record<string, unknown>[] = [];
153+
agent.on("anomaly", (a) => anomalies.push(a as Record<string, unknown>));
154+
155+
// W3C 128-bit traceId (32 lowercase hex chars)
156+
const traceId = "0af7651916cd43dd8448eb211c80319c";
157+
158+
// Record a 1ms HTTP request — LLM will take ~10ms, far exceeding the 80% threshold
159+
agent.emit("request", { traceId, durationMs: 1 });
160+
161+
// Run the LLM call inside an async context with the matching traceId
162+
await runWithContext(
163+
{
164+
requestId: "req-dom-1",
165+
traceId,
166+
spanId: "ab12cd34ef56ab12",
167+
method: "POST",
168+
url: "/api/chat",
169+
startedAt: Date.now(),
170+
},
171+
() =>
172+
SlowMock.prototype.chat.completions.create({
173+
model: "gpt-4o",
174+
messages: [{ role: "user", content: "summarize" }],
175+
}),
176+
);
177+
178+
assert.ok(
179+
anomalies.some((a) => a.type === "llm-dominates-request"),
180+
"should emit llm-dominates-request anomaly when LLM dominates HTTP request time",
181+
);
148182
});
149183
});

0 commit comments

Comments
 (0)