Skip to content

Commit 1851805

Browse files
Godzilla675Godzilla675
authored
Merge pull request #98 from Godzilla675/copilot/fix-broken-ai-functions
Fix broken multi-turn tool recursion and Anthropic tool result grouping
2 parents 476b670 + 603d6cf commit 1851805

4 files changed

Lines changed: 340 additions & 117 deletions

File tree

‎apps/backend/src/routes/copilot.ts‎

Lines changed: 16 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,10 @@ export function createCopilotRouter(
6565
toolCalls: currentResult.toolCalls
6666
});
6767

68-
// Execute tools
68+
// Execute all tools and collect results into a single message
69+
// (required for Anthropic parallel tool use)
70+
const toolResults: { toolCallId: string; toolName: string; result: any; isError?: boolean }[] = [];
71+
6972
for (const call of currentResult.toolCalls) {
7073
const toolName = call.toolName;
7174
const toolCallId = call.toolCallId;
@@ -75,45 +78,26 @@ export function createCopilotRouter(
7578
if (serverName) {
7679
console.log(`Executing tool ${toolName} on server ${serverName} with args:`, call.args);
7780
const toolResult = await mcpClientManager.callTool(serverName, toolName, call.args);
78-
79-
messages.push({
80-
role: 'user',
81-
content: `Tool '${toolName}' result: ${JSON.stringify(toolResult)}`,
82-
toolResults: [{
83-
toolCallId,
84-
toolName,
85-
result: toolResult
86-
}]
87-
});
81+
toolResults.push({ toolCallId, toolName, result: toolResult });
8882
} else {
89-
messages.push({
90-
role: 'user',
91-
content: `Tool '${toolName}' not found.`,
92-
toolResults: [{
93-
toolCallId,
94-
toolName,
95-
result: { error: `Tool '${toolName}' not found.` },
96-
isError: true
97-
}]
98-
});
83+
toolResults.push({ toolCallId, toolName, result: { error: `Tool '${toolName}' not found.` }, isError: true });
9984
}
10085
} catch (error: any) {
10186
console.error(`Tool execution failed: ${toolName}`, error);
102-
messages.push({
103-
role: 'user',
104-
content: `Tool '${toolName}' failed: ${error.message}`,
105-
toolResults: [{
106-
toolCallId,
107-
toolName,
108-
result: { error: error.message },
109-
isError: true
110-
}]
111-
});
87+
toolResults.push({ toolCallId, toolName, result: { error: error.message }, isError: true });
11288
}
11389
}
11490

91+
// Add all tool results as a single user message (content is empty
92+
// because providers use the structured toolResults, not the text field)
93+
messages.push({
94+
role: 'user',
95+
content: '',
96+
toolResults
97+
});
98+
11599
// Call LLM again
116-
currentResult = await orchestrator.chat(messages, tools as any, undefined, { model });
100+
currentResult = await orchestrator.chat(messages, tools as any, executeTool, { model });
117101
}
118102

119103
res.json({ content: currentResult.content });

‎apps/backend/src/test/rest-tool-grouping-test.ts‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
import express from 'express';
2+
import http from 'http';
3+
import { createCopilotRouter } from '../routes/copilot.js';
4+
import { LLMOrchestrator } from '../llm/orchestrator.js';
5+
import { ProjectManager } from '../project/state.js';
6+
import { LLMConfig, Message } from '@ai-video-editor/shared-types';
7+
import { LLMProviderInterface, MCPTool, StreamChunk, ToolCall, ToolExecutor, LLMProviderOptions } from '../llm/types.js';
8+
import fs from 'fs';
9+
10+
// Track messages passed to chat() to verify tool result grouping
11+
let chatCallCount = 0;
12+
const chatMessageHistory: Message[][] = [];
13+
14+
class MockToolProvider implements LLMProviderInterface {
15+
async getModels(): Promise<string[]> {
16+
return ['mock-model'];
17+
}
18+
19+
async chat(messages: Message[]): Promise<{ content: string; toolCalls?: ToolCall[] }> {
20+
chatCallCount++;
21+
chatMessageHistory.push([...messages]);
22+
23+
if (chatCallCount === 1) {
24+
// First call: return two parallel tool calls
25+
return {
26+
content: 'I will search for both.',
27+
toolCalls: [
28+
{ toolName: 'search_images', toolCallId: 'call-1', args: { query: 'sunset' } },
29+
{ toolName: 'search_images', toolCallId: 'call-2', args: { query: 'beach' } }
30+
]
31+
};
32+
} else {
33+
// Second call: final response
34+
return { content: 'Found both images!' };
35+
}
36+
}
37+
38+
async *streamChat(messages: Message[]): AsyncIterable<StreamChunk> {
39+
yield { done: false, content: 'streamed' };
40+
yield { done: true };
41+
}
42+
}
43+
44+
async function runTest() {
45+
console.log('=== REST API Tool Result Grouping Test ===');
46+
47+
const projectDir = './test-rest-tool-grouping';
48+
if (fs.existsSync(projectDir)) {
49+
fs.rmSync(projectDir, { recursive: true, force: true });
50+
}
51+
52+
const mockConfig: LLMConfig = {
53+
provider: 'copilot',
54+
apiKey: '',
55+
model: 'mock-model'
56+
};
57+
58+
const orchestrator = new LLMOrchestrator(mockConfig);
59+
(orchestrator as any).provider = new MockToolProvider();
60+
61+
const projectManager = new ProjectManager(projectDir);
62+
63+
const app = express();
64+
app.use(express.json());
65+
app.use('/api/copilot', createCopilotRouter(orchestrator, projectManager));
66+
67+
const server = http.createServer(app);
68+
const port = 3096;
69+
await new Promise<void>(resolve => server.listen(port, resolve));
70+
console.log(`Test server on port ${port}`);
71+
72+
try {
73+
chatCallCount = 0;
74+
chatMessageHistory.length = 0;
75+
76+
// Test: POST /chat triggers parallel tool calls and groups results
77+
console.log('--- Test: POST /api/copilot/chat with parallel tool calls ---');
78+
const chatRes = await fetch(`http://localhost:${port}/api/copilot/chat`, {
79+
method: 'POST',
80+
headers: { 'Content-Type': 'application/json' },
81+
body: JSON.stringify({ content: 'Find sunset and beach images' })
82+
});
83+
84+
if (!chatRes.ok) throw new Error(`Chat endpoint failed: ${chatRes.statusText}`);
85+
const chatData = await chatRes.json();
86+
87+
// Verify the final response
88+
if (chatData.content !== 'Found both images!') {
89+
throw new Error(`Unexpected response: ${chatData.content}`);
90+
}
91+
console.log('[PASS] Final response received:', chatData.content);
92+
93+
// Verify chat was called twice (initial + after tool results)
94+
if (chatCallCount !== 2) {
95+
throw new Error(`Expected 2 chat calls, got ${chatCallCount}`);
96+
}
97+
console.log('[PASS] Chat called correct number of times:', chatCallCount);
98+
99+
// Verify the second call has tool results grouped in a SINGLE user message
100+
const secondCallMessages = chatMessageHistory[1];
101+
const toolResultMessages = secondCallMessages.filter(
102+
m => m.role === 'user' && m.toolResults && m.toolResults.length > 0
103+
);
104+
105+
if (toolResultMessages.length !== 1) {
106+
throw new Error(`Expected 1 grouped tool result message, got ${toolResultMessages.length} separate messages`);
107+
}
108+
console.log('[PASS] Tool results grouped into single message');
109+
110+
// Verify the grouped message has both tool results
111+
const groupedResults = toolResultMessages[0].toolResults!;
112+
if (groupedResults.length !== 2) {
113+
throw new Error(`Expected 2 tool results in grouped message, got ${groupedResults.length}`);
114+
}
115+
116+
if (groupedResults[0].toolCallId !== 'call-1' || groupedResults[1].toolCallId !== 'call-2') {
117+
throw new Error(`Tool result IDs don't match: ${JSON.stringify(groupedResults)}`);
118+
}
119+
console.log('[PASS] Grouped message contains both tool results with correct IDs');
120+
121+
console.log('=== All REST API tool grouping tests passed ===');
122+
} catch (err) {
123+
console.error('Test failed:', err);
124+
process.exit(1);
125+
} finally {
126+
server.close();
127+
if (fs.existsSync(projectDir)) {
128+
fs.rmSync(projectDir, { recursive: true, force: true });
129+
}
130+
}
131+
}
132+
133+
runTest().catch(err => {
134+
console.error('Test error:', err);
135+
process.exit(1);
136+
});

‎apps/backend/src/test/websocket-tool-recursion-test.ts‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
import http from 'http';
2+
import express from 'express';
3+
import { WebSocketServer, WebSocket } from 'ws';
4+
import { WebSocketHandler } from '../websocket/handler.js';
5+
import { ProjectManager } from '../project/state.js';
6+
import { LLMOrchestrator } from '../llm/orchestrator.js';
7+
import { LLMConfig, Message } from '@ai-video-editor/shared-types';
8+
import { LLMProviderInterface, MCPTool, StreamChunk, ToolCall, ToolExecutor, LLMProviderOptions } from '../llm/types.js';
9+
10+
// Mock provider that returns tool calls on the first call, then text on the second
11+
let streamChatCallCount = 0;
12+
13+
class MockToolProvider implements LLMProviderInterface {
14+
async getModels(): Promise<string[]> {
15+
return ['mock-model'];
16+
}
17+
18+
async chat(messages: Message[]): Promise<{ content: string; toolCalls?: ToolCall[] }> {
19+
return { content: 'Mock chat response' };
20+
}
21+
22+
async *streamChat(messages: Message[]): AsyncIterable<StreamChunk> {
23+
streamChatCallCount++;
24+
25+
if (streamChatCallCount === 1) {
26+
// First call: return text + a tool call
27+
yield { done: false, content: 'Let me search for that. ' };
28+
yield {
29+
done: false,
30+
toolCall: {
31+
toolName: 'search_images',
32+
toolCallId: 'call-1',
33+
args: { query: 'sunset' }
34+
}
35+
};
36+
yield { done: true };
37+
} else if (streamChatCallCount === 2) {
38+
// Second call (after tool result): return another tool call for multi-turn
39+
yield { done: false, content: 'Found images. Now downloading. ' };
40+
yield {
41+
done: false,
42+
toolCall: {
43+
toolName: 'download_asset',
44+
toolCallId: 'call-2',
45+
args: { url: 'https://example.com/sunset.jpg' }
46+
}
47+
};
48+
yield { done: true };
49+
} else {
50+
// Third call: final text response
51+
yield { done: false, content: 'Done! The sunset image has been added.' };
52+
yield { done: true };
53+
}
54+
}
55+
}
56+
57+
async function runTest() {
58+
console.log('=== WebSocket Multi-Turn Tool Recursion Test ===');
59+
60+
const projectDir = './test-ws-tool-recursion';
61+
const fs = await import('fs');
62+
if (fs.existsSync(projectDir)) {
63+
fs.rmSync(projectDir, { recursive: true, force: true });
64+
}
65+
66+
const app = express();
67+
const server = http.createServer(app);
68+
const wss = new WebSocketServer({ server });
69+
const projectManager = new ProjectManager(projectDir);
70+
71+
const mockConfig: LLMConfig = {
72+
provider: 'copilot',
73+
apiKey: '',
74+
model: 'mock-model'
75+
};
76+
77+
const orchestrator = new LLMOrchestrator(mockConfig);
78+
(orchestrator as any).provider = new MockToolProvider();
79+
80+
const wsHandler = new WebSocketHandler(wss, projectManager, orchestrator);
81+
82+
const port = 3097;
83+
await new Promise<void>(resolve => server.listen(port, resolve));
84+
console.log(`Test server on port ${port}`);
85+
86+
try {
87+
const ws = new WebSocket(`ws://localhost:${port}`);
88+
await new Promise<void>((resolve, reject) => {
89+
ws.on('open', resolve);
90+
ws.on('error', reject);
91+
});
92+
93+
streamChatCallCount = 0;
94+
95+
const responses: any[] = [];
96+
const toolCalls: any[] = [];
97+
const toolResults: any[] = [];
98+
let doneCount = 0;
99+
100+
const responsePromise = new Promise<void>((resolve) => {
101+
ws.on('message', (data) => {
102+
const msg = JSON.parse(data.toString());
103+
if (msg.type === 'copilot.response') {
104+
responses.push(msg.payload);
105+
if (msg.payload.done) {
106+
doneCount++;
107+
setTimeout(() => resolve(), 500);
108+
}
109+
} else if (msg.type === 'copilot.tool_call') {
110+
toolCalls.push(msg.payload);
111+
} else if (msg.type === 'copilot.tool_result') {
112+
toolResults.push(msg.payload);
113+
}
114+
});
115+
});
116+
117+
ws.send(JSON.stringify({
118+
type: 'copilot.message',
119+
payload: { content: 'Add a sunset image', model: 'mock-model' }
120+
}));
121+
122+
await responsePromise;
123+
124+
// Verify: streamChat should be called 3 times (initial + 2 tool recursions)
125+
console.log(`streamChat called ${streamChatCallCount} time(s)`);
126+
if (streamChatCallCount !== 3) {
127+
throw new Error(`Expected streamChat to be called 3 times for multi-turn tool use, but was called ${streamChatCallCount} times`);
128+
}
129+
130+
// Verify: exactly ONE done:true signal
131+
if (doneCount !== 1) {
132+
throw new Error(`Expected 1 done signal, got ${doneCount}`);
133+
}
134+
135+
// Verify: 2 tool calls were made
136+
if (toolCalls.length !== 2) {
137+
throw new Error(`Expected 2 tool calls, got ${toolCalls.length}: ${JSON.stringify(toolCalls)}`);
138+
}
139+
if (toolCalls[0].tool !== 'search_images') {
140+
throw new Error(`Expected first tool call to be 'search_images', got '${toolCalls[0].tool}'`);
141+
}
142+
if (toolCalls[1].tool !== 'download_asset') {
143+
throw new Error(`Expected second tool call to be 'download_asset', got '${toolCalls[1].tool}'`);
144+
}
145+
146+
// Verify: 2 tool results were returned
147+
if (toolResults.length !== 2) {
148+
throw new Error(`Expected 2 tool results, got ${toolResults.length}`);
149+
}
150+
151+
// Verify: all content was received
152+
const contentResponses = responses.filter(r => r.content && r.content.length > 0 && !r.done);
153+
const fullContent = contentResponses.map(r => r.content).join('');
154+
if (!fullContent.includes('Let me search for that.')) {
155+
throw new Error(`Missing first turn content in: "${fullContent}"`);
156+
}
157+
if (!fullContent.includes('Done! The sunset image has been added.')) {
158+
throw new Error(`Missing final turn content in: "${fullContent}"`);
159+
}
160+
161+
console.log('[PASS] Multi-turn tool recursion works correctly (3 LLM calls)');
162+
console.log('[PASS] Single done signal sent');
163+
console.log(`[PASS] 2 tool calls executed: ${toolCalls.map(t => t.tool).join(', ')}`);
164+
console.log(`[PASS] Full content received: "${fullContent}"`);
165+
166+
ws.close();
167+
} catch (err) {
168+
console.error('Test failed:', err);
169+
process.exit(1);
170+
} finally {
171+
server.close();
172+
const fs = await import('fs');
173+
if (fs.existsSync(projectDir)) {
174+
fs.rmSync(projectDir, { recursive: true, force: true });
175+
}
176+
}
177+
178+
console.log('=== All multi-turn tool recursion tests passed ===');
179+
}
180+
181+
runTest().catch(err => {
182+
console.error('Test error:', err);
183+
process.exit(1);
184+
});

0 commit comments

Comments
 (0)