DurableAgent: various compatibility fixes

.changeset/durable-agent-improvements.md

@@ -0,0 +1,5 @@
+---
+'@workflow/ai': patch
+---
+
+Add type helpers (`InferDurableAgentTools`, `InferDurableAgentUIMessage`), support `prepareStep` on `DurableAgent` constructor, fix `supportedUrls` causing `AI_DownloadError` for image URLs, and add telemetry span support for `experimental_telemetry`. Fix `LanguageModelV3ToolResultOutput` breaking response when not json compatible.

packages/ai/package.json

@@ -72,9 +72,15 @@
     "workflow": "workspace:*"
   },
   "peerDependencies": {
+    "@opentelemetry/api": "^1.0.0",
     "ai": "^6",
     "workflow": "workspace:^"
   },
+  "peerDependenciesMeta": {
+    "@opentelemetry/api": {
+      "optional": true
+    }
+  },
   "dependencies": {
     "@ai-sdk/provider": "^3.0.0",
     "@workflow/serde": "workspace:^",

packages/ai/src/agent/do-stream-step.ts

@@ -21,6 +21,7 @@ import type {
   StreamTextTransform,
   TelemetrySettings,
 } from './durable-agent.js';
+import { recordSpan } from './telemetry.js';
 import type { CompatibleLanguageModel } from './types.js';
 
 export type FinishPart = Extract<LanguageModelV3StreamPart, { type: 'finish' }>;
@@ -163,7 +164,39 @@ export async function doStreamStep(
     }),
   };
 
-  const result = await model.doStream(callOptions);
+  const result = await recordSpan({
+    name: 'ai.streamText.doStream',
+    telemetry: options?.experimental_telemetry,
+    attributes: {
+      'ai.model.provider': model.provider,
+      'ai.model.id': model.modelId,
+      // gen_ai semantic convention attributes
+      'gen_ai.system': model.provider,
+      'gen_ai.request.model': model.modelId,
+      ...(options?.maxOutputTokens !== undefined && {
+        'gen_ai.request.max_tokens': options.maxOutputTokens,
+      }),
+      ...(options?.temperature !== undefined && {
+        'gen_ai.request.temperature': options.temperature,
+      }),
+      ...(options?.topP !== undefined && {
+        'gen_ai.request.top_p': options.topP,
+      }),
+      ...(options?.topK !== undefined && {
+        'gen_ai.request.top_k': options.topK,
+      }),
+      ...(options?.frequencyPenalty !== undefined && {
+        'gen_ai.request.frequency_penalty': options.frequencyPenalty,
+      }),
+      ...(options?.presencePenalty !== undefined && {
+        'gen_ai.request.presence_penalty': options.presencePenalty,
+      }),
+      ...(options?.stopSequences !== undefined && {
+        'gen_ai.request.stop_sequences': options.stopSequences,
+      }),
+    },
+    fn: () => model!.doStream(callOptions),
+  });
 
   let finish: FinishPart | undefined;
   const toolCalls: LanguageModelV3ToolCall[] = [];

packages/ai/src/agent/durable-agent-compat.test.ts

@@ -294,6 +294,82 @@ describe('DurableAgent (ToolLoopAgent compat)', () => {
       `);
     });
 
+    it('should use prepareStep from constructor', async () => {
+      const stepNumbers: number[] = [];
+
+      const agent = new DurableAgent({
+        model: asModelFactory(mockModel),
+        prepareStep: ({ stepNumber }) => {
+          stepNumbers.push(stepNumber);
+          return {};
+        },
+      });
+
+      const { writable } = createMockWritable();
+
+      await agent.stream({
+        messages: [{ role: 'user' as const, content: 'Hello' }],
+        writable,
+      });
+
+      expect(stepNumbers).toEqual([0]);
+    });
+
+    it('should prefer stream prepareStep over constructor prepareStep', async () => {
+      const source: string[] = [];
+
+      const agent = new DurableAgent({
+        model: asModelFactory(mockModel),
+        prepareStep: () => {
+          source.push('constructor');
+          return {};
+        },
+      });
+
+      const { writable } = createMockWritable();
+
+      await agent.stream({
+        messages: [{ role: 'user' as const, content: 'Hello' }],
+        writable,
+        prepareStep: () => {
+          source.push('stream');
+          return {};
+        },
+      });
+
+      // Stream-level prepareStep should override constructor-level
+      expect(source).toEqual(['stream']);
+    });
+
+    it('should call constructor prepareStep on each step in multi-step', async () => {
+      const toolCallModel = createToolCallStreamMockModel();
+      const stepNumbers: number[] = [];
+
+      const agent = new DurableAgent({
+        model: asModelFactory(toolCallModel),
+        tools: {
+          testTool: tool({
+            inputSchema: z.object({ value: z.string() }),
+            execute: async ({ value }: { value: string }) => `result: ${value}`,
+          }),
+        },
+        prepareStep: ({ stepNumber }) => {
+          stepNumbers.push(stepNumber);
+          return {};
+        },
+      });
+
+      const { writable } = createMockWritable();
+
+      await agent.stream({
+        messages: [{ role: 'user' as const, content: 'Hello' }],
+        writable,
+      });
+
+      // prepareStep called for both the tool-call step and the final text step
+      expect(stepNumbers).toEqual([0, 1]);
+    });
+
     it('should pass abortSignal to streamText', async () => {
       const abortController = new AbortController();
 

packages/ai/src/agent/durable-agent-types.test.ts

@@ -0,0 +1,40 @@
+import { type InferUITools, tool, type UIMessage } from 'ai';
+import { describe, expectTypeOf, it } from 'vitest';
+import { z } from 'zod';
+import type {
+  DurableAgent,
+  InferDurableAgentTools,
+  InferDurableAgentUIMessage,
+} from './durable-agent.js';
+
+const getWeather = tool({
+  description: 'Get weather for a location',
+  inputSchema: z.object({ location: z.string() }),
+  execute: async ({ location }) => `Weather in ${location}`,
+});
+
+type WeatherAgent = DurableAgent<{
+  getWeather: typeof getWeather;
+}>;
+
+describe('InferDurableAgentTools', () => {
+  it('infers the tools from a durable agent', () => {
+    expectTypeOf<InferDurableAgentTools<WeatherAgent>>().toEqualTypeOf<{
+      getWeather: typeof getWeather;
+    }>();
+  });
+});
+
+describe('InferDurableAgentUIMessage', () => {
+  it('infers the UI message type from a durable agent', () => {
+    expectTypeOf<
+      InferDurableAgentUIMessage<WeatherAgent, { threadId: string }>
+    >().toEqualTypeOf<
+      UIMessage<
+        { threadId: string },
+        never,
+        InferUITools<InferDurableAgentTools<WeatherAgent>>
+      >
+    >();
+  });
+});