From 18db5043e2fafc2a9a0515a5becee4ad6b614a64 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 18 Feb 2026 17:28:57 +0000 Subject: [PATCH 01/11] chore: bump bedrock model for data copilot Signed-off-by: Joana Maia --- frontend/lib/chat/Readme.md | 2 +- frontend/lib/chat/chart/generator.ts | 2 +- frontend/lib/chat/data-copilot.ts | 2 +- frontend/lib/chat/tests/auditor.test.ts | 2 +- frontend/lib/chat/tests/router.test.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/lib/chat/Readme.md b/frontend/lib/chat/Readme.md index 07448e942..f136c8365 100644 --- a/frontend/lib/chat/Readme.md +++ b/frontend/lib/chat/Readme.md @@ -268,7 +268,7 @@ const mcpClient = await createMCPClient({ # Model Configuration -Currently using: `us.anthropic.claude-sonnet-4-20250514-v1:0` via AWS Bedrock +Currently using: `anthropic.claude-opus-4-6-v1` via AWS Bedrock ## Data Flow Example diff --git a/frontend/lib/chat/chart/generator.ts b/frontend/lib/chat/chart/generator.ts index d7f35d7b4..0a7f355b0 100644 --- a/frontend/lib/chat/chart/generator.ts +++ b/frontend/lib/chat/chart/generator.ts @@ -43,7 +43,7 @@ const chartColors = { lines: [lfxColors.positive[500], lfxColors.negative[500], lfxColors.brand[300]], }, }; -const model = bedrock('us.anthropic.claude-sonnet-4-20250514-v1:0'); +const model = bedrock('anthropic.claude-opus-4-6-v1'); export async function generateChartConfig( results: Result[], diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 388e52f26..671432b9f 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -63,7 +63,7 @@ export class DataCopilot { private model: LanguageModelV1; /** Bedrock model identifier */ - private readonly BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; + private readonly BEDROCK_MODEL_ID = 'anthropic.claude-opus-4-6-v1'; /** Maximum number of auditor retry attempts */ private readonly MAX_AUDITOR_RETRIES = 1; diff --git a/frontend/lib/chat/tests/auditor.test.ts b/frontend/lib/chat/tests/auditor.test.ts index 9b351b381..6d864b43a 100644 --- a/frontend/lib/chat/tests/auditor.test.ts +++ b/frontend/lib/chat/tests/auditor.test.ts @@ -36,7 +36,7 @@ describe('Auditor Agent', () => { region: process.env.NUXT_AWS_BEDROCK_REGION, }); - const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; + const BEDROCK_MODEL_ID = 'anthropic.claude-opus-4-6-v1'; model = bedrock(BEDROCK_MODEL_ID); }, 30000); diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index b9f39b1cd..83479e54a 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -46,7 +46,7 @@ describe('Router Agent', () => { }); // Initialize model once, like DataCopilot does in constructor - const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; + const BEDROCK_MODEL_ID = 'anthropic.claude-opus-4-6-v1'; model = bedrock(BEDROCK_MODEL_ID); // Initialize MCP client to get real tools - same as DataCopilot From db49be85c0f71115a88b7a27c747ba5cec75c154 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Thu, 19 Feb 2026 10:58:04 +0000 Subject: [PATCH 02/11] chore: add region prefix to model id Signed-off-by: Joana Maia --- frontend/lib/chat/Readme.md | 2 +- frontend/lib/chat/chart/generator.ts | 2 +- frontend/lib/chat/data-copilot.ts | 2 +- frontend/lib/chat/tests/auditor.test.ts | 2 +- frontend/lib/chat/tests/router.test.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/lib/chat/Readme.md b/frontend/lib/chat/Readme.md index f136c8365..1fdcfe124 100644 --- a/frontend/lib/chat/Readme.md +++ b/frontend/lib/chat/Readme.md @@ -268,7 +268,7 @@ const mcpClient = await createMCPClient({ # Model Configuration -Currently using: `anthropic.claude-opus-4-6-v1` via AWS Bedrock +Currently using: `us.anthropic.claude-opus-4-6-v1` via AWS Bedrock ## Data Flow Example diff --git a/frontend/lib/chat/chart/generator.ts b/frontend/lib/chat/chart/generator.ts index 0a7f355b0..734a65abb 100644 --- a/frontend/lib/chat/chart/generator.ts +++ b/frontend/lib/chat/chart/generator.ts @@ -43,7 +43,7 @@ const chartColors = { lines: [lfxColors.positive[500], lfxColors.negative[500], lfxColors.brand[300]], }, }; -const model = bedrock('anthropic.claude-opus-4-6-v1'); +const model = bedrock('us.anthropic.claude-opus-4-6-v1'); export async function generateChartConfig( results: Result[], diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 671432b9f..a9ba16632 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -63,7 +63,7 @@ export class DataCopilot { private model: LanguageModelV1; /** Bedrock model identifier */ - private readonly BEDROCK_MODEL_ID = 'anthropic.claude-opus-4-6-v1'; + private readonly BEDROCK_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; /** Maximum number of auditor retry attempts */ private readonly MAX_AUDITOR_RETRIES = 1; diff --git a/frontend/lib/chat/tests/auditor.test.ts b/frontend/lib/chat/tests/auditor.test.ts index 6d864b43a..932fdaa91 100644 --- a/frontend/lib/chat/tests/auditor.test.ts +++ b/frontend/lib/chat/tests/auditor.test.ts @@ -36,7 +36,7 @@ describe('Auditor Agent', () => { region: process.env.NUXT_AWS_BEDROCK_REGION, }); - const BEDROCK_MODEL_ID = 'anthropic.claude-opus-4-6-v1'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; model = bedrock(BEDROCK_MODEL_ID); }, 30000); diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index 83479e54a..ede889e2b 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -46,7 +46,7 @@ describe('Router Agent', () => { }); // Initialize model once, like DataCopilot does in constructor - const BEDROCK_MODEL_ID = 'anthropic.claude-opus-4-6-v1'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; model = bedrock(BEDROCK_MODEL_ID); // Initialize MCP client to get real tools - same as DataCopilot From d5dac303ecc106f102dc172ca8fab75b5449ad3e Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 25 Feb 2026 12:27:28 +0000 Subject: [PATCH 03/11] chore: test different models Signed-off-by: Joana Maia --- frontend/lib/chat/data-copilot.ts | 35 ++++++++++++++++++------- frontend/lib/chat/instructions.ts | 11 ++++++-- frontend/lib/chat/tests/auditor.test.ts | 2 +- frontend/lib/chat/tests/router.test.ts | 10 +++---- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index a9ba16632..6d3bc8fe8 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -59,11 +59,17 @@ export class DataCopilot { /** Tinybird MCP server URL */ private tbMcpUrl: string = ''; - /** Amazon Bedrock language model instance */ + /** Amazon Bedrock language model instance for routing, auditing, and pipe agents */ private model: LanguageModelV1; - /** Bedrock model identifier */ - private readonly BEDROCK_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; + /** Amazon Bedrock language model instance for text-to-SQL (higher reasoning capacity) */ + private sqlModel: LanguageModelV1; + + /** Bedrock model identifier for general agents */ + private readonly BEDROCK_SONNET_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; + + /** Bedrock model identifier for text-to-SQL and pipe agent */ + private readonly BEDROCK_OPUS_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; /** Maximum number of auditor retry attempts */ private readonly MAX_AUDITOR_RETRIES = 1; @@ -72,7 +78,8 @@ export class DataCopilot { private readonly MAX_SQL_RETRIES = 2; constructor() { - this.model = bedrock(this.BEDROCK_MODEL_ID); + this.model = bedrock(this.BEDROCK_SONNET_MODEL_ID); + this.sqlModel = bedrock(this.BEDROCK_OPUS_MODEL_ID); this.tbMcpUrl = `https://mcp.tinybird.co?token=${process.env.NUXT_INSIGHTS_DATA_COPILOT_TINYBIRD_TOKEN}&host=${process.env.NUXT_TINYBIRD_BASE_URL}`; } @@ -170,10 +177,18 @@ export class DataCopilot { instructions?: string, ): Promise { const chatRepo = new ChatRepository(insightsDbPool); + + let model: string | undefined = this.BEDROCK_SONNET_MODEL_ID; + + if (agent === 'EXECUTE_INSTRUCTIONS') { + model = undefined; + } else if (agent === 'TEXT_TO_SQL' || agent === 'PIPE' || agent === 'CHART') { + model = this.BEDROCK_OPUS_MODEL_ID; + } await chatRepo.saveAgentStep({ chatResponseId, agent, - model: agent === 'EXECUTE_INSTRUCTIONS' ? undefined : this.BEDROCK_MODEL_ID, + model, response, inputTokens: response?.usage?.promptTokens || 0, outputTokens: response?.usage?.completionTokens || 0, @@ -248,7 +263,7 @@ export class DataCopilot { const agent = new TextToSqlAgent(); return agent.execute({ - model: this.model, + model: this.sqlModel, messages, tools: followUpTools, date, @@ -768,7 +783,7 @@ export class DataCopilot { userPrompt: currentQuestion, inputTokens: 0, outputTokens: 0, - model: this.BEDROCK_MODEL_ID, + model: this.BEDROCK_SONNET_MODEL_ID, conversationId: conversationId || '', routerResponse: RouterDecisionAction.STOP, routerReason: '', @@ -866,7 +881,7 @@ export class DataCopilot { routerReason: routerOutput.reasoning, pipeInstructions: undefined, sqlQuery: undefined, - model: this.BEDROCK_MODEL_ID, + model: this.BEDROCK_SONNET_MODEL_ID, conversationId: conversationId, }, insightsDbPool, @@ -901,7 +916,7 @@ export class DataCopilot { clarificationQuestion: routerOutput.clarification_question || undefined, pipeInstructions: undefined, sqlQuery: undefined, - model: this.BEDROCK_MODEL_ID, + model: this.BEDROCK_SONNET_MODEL_ID, conversationId: conversationId, }, insightsDbPool, @@ -1246,7 +1261,7 @@ export class DataCopilot { routerReason: routerOutput.reasoning, pipeInstructions, sqlQuery, - model: this.BEDROCK_MODEL_ID, + model: this.BEDROCK_SONNET_MODEL_ID, conversationId: conversationId, }, insightsDbPool, diff --git a/frontend/lib/chat/instructions.ts b/frontend/lib/chat/instructions.ts index 6a292ae83..83a6732da 100644 --- a/frontend/lib/chat/instructions.ts +++ b/frontend/lib/chat/instructions.ts @@ -28,6 +28,9 @@ async function executeTinybirdPipe( ? `${tinybirdBaseUrl}/v0/pipes/${pipeName}.json?${params}` : `${tinybirdBaseUrl}/v0/pipes/${pipeName}.json`; + console.warn(`🔍 [Tinybird] Calling pipe: ${pipeName}`); + console.warn(`🔍 [Tinybird] URL: ${url}`); + try { const response = await ofetch(url, { headers: { @@ -35,8 +38,12 @@ async function executeTinybirdPipe( }, }); - // TinyBird response format has data array - return response.data || []; + const data = response.data || []; + console.warn( + `✅ [Tinybird] ${pipeName} returned ${data.length} rows:`, + JSON.stringify(data.slice(0, 3)), + ); + return data; } catch (error) { console.error(`Error executing TinyBird pipe ${pipeName}:`, error); return []; diff --git a/frontend/lib/chat/tests/auditor.test.ts b/frontend/lib/chat/tests/auditor.test.ts index 932fdaa91..9b351b381 100644 --- a/frontend/lib/chat/tests/auditor.test.ts +++ b/frontend/lib/chat/tests/auditor.test.ts @@ -36,7 +36,7 @@ describe('Auditor Agent', () => { region: process.env.NUXT_AWS_BEDROCK_REGION, }); - const BEDROCK_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; model = bedrock(BEDROCK_MODEL_ID); }, 30000); diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index ede889e2b..1dfa0cda7 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -46,7 +46,7 @@ describe('Router Agent', () => { }); // Initialize model once, like DataCopilot does in constructor - const BEDROCK_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; model = bedrock(BEDROCK_MODEL_ID); // Initialize MCP client to get real tools - same as DataCopilot @@ -201,7 +201,7 @@ describe('Router Agent', () => { console.warn(`✅ "${query}" → ${result.next_action}`); console.warn(`🔍 Reasoning: ${result.reasoning}`); }, - 15000, + 360000, ); }); @@ -226,7 +226,7 @@ describe('Router Agent', () => { console.warn(`✅ "${query}" → ${result.next_action}`); console.warn(`🔍 Reasoning: ${result.reasoning}`); }, - 15000, + 360000, ); }); @@ -251,7 +251,7 @@ describe('Router Agent', () => { console.warn(`✅ "${query}" → ${result.next_action}`); console.warn(`🔍 Reasoning: ${result.reasoning}`); }, - 15000, + 360000, ); }); @@ -274,7 +274,7 @@ describe('Router Agent', () => { console.warn(`🔍 Reasoning: ${result.reasoning}`); console.warn(`❓ Clarification: ${result.clarification_question}`); }, - 15000, + 360000, ); }); }); From 25595b24e152b6ae46c8ca9d4f3e148262398ce8 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 25 Feb 2026 12:28:40 +0000 Subject: [PATCH 04/11] chore: revert timeout Signed-off-by: Joana Maia --- frontend/lib/chat/tests/router.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index 1dfa0cda7..b9f39b1cd 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -201,7 +201,7 @@ describe('Router Agent', () => { console.warn(`✅ "${query}" → ${result.next_action}`); console.warn(`🔍 Reasoning: ${result.reasoning}`); }, - 360000, + 15000, ); }); @@ -226,7 +226,7 @@ describe('Router Agent', () => { console.warn(`✅ "${query}" → ${result.next_action}`); console.warn(`🔍 Reasoning: ${result.reasoning}`); }, - 360000, + 15000, ); }); @@ -251,7 +251,7 @@ describe('Router Agent', () => { console.warn(`✅ "${query}" → ${result.next_action}`); console.warn(`🔍 Reasoning: ${result.reasoning}`); }, - 360000, + 15000, ); }); @@ -274,7 +274,7 @@ describe('Router Agent', () => { console.warn(`🔍 Reasoning: ${result.reasoning}`); console.warn(`❓ Clarification: ${result.clarification_question}`); }, - 360000, + 15000, ); }); }); From 01c24832a7e79aeae413aa7ec5d14f6cc92f8f99 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 25 Feb 2026 12:36:04 +0000 Subject: [PATCH 05/11] fix: models Signed-off-by: Joana Maia --- frontend/lib/chat/data-copilot.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 6d3bc8fe8..109a8b20a 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -59,11 +59,11 @@ export class DataCopilot { /** Tinybird MCP server URL */ private tbMcpUrl: string = ''; - /** Amazon Bedrock language model instance for routing, auditing, and pipe agents */ - private model: LanguageModelV1; + /** Amazon Bedrock language model instance for routing and auditing (Sonnet) */ + private sonnetModel: LanguageModelV1; - /** Amazon Bedrock language model instance for text-to-SQL (higher reasoning capacity) */ - private sqlModel: LanguageModelV1; + /** Amazon Bedrock language model instance for text-to-SQL, pipe, and chart agents (Opus) */ + private opusModel: LanguageModelV1; /** Bedrock model identifier for general agents */ private readonly BEDROCK_SONNET_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; @@ -78,8 +78,8 @@ export class DataCopilot { private readonly MAX_SQL_RETRIES = 2; constructor() { - this.model = bedrock(this.BEDROCK_SONNET_MODEL_ID); - this.sqlModel = bedrock(this.BEDROCK_OPUS_MODEL_ID); + this.sonnetModel = bedrock(this.BEDROCK_SONNET_MODEL_ID); + this.opusModel = bedrock(this.BEDROCK_OPUS_MODEL_ID); this.tbMcpUrl = `https://mcp.tinybird.co?token=${process.env.NUXT_INSIGHTS_DATA_COPILOT_TINYBIRD_TOKEN}&host=${process.env.NUXT_TINYBIRD_BASE_URL}`; } @@ -222,7 +222,7 @@ export class DataCopilot { }: Omit) { const agent = new RouterAgent(); return agent.execute({ - model: this.model, + model: this.sonnetModel, messages, tools: this.tbTools, toolsOverview: this.toolsOverview, @@ -263,7 +263,7 @@ export class DataCopilot { const agent = new TextToSqlAgent(); return agent.execute({ - model: this.sqlModel, + model: this.opusModel, messages, tools: followUpTools, date, @@ -308,7 +308,7 @@ export class DataCopilot { } const agent = new PipeAgent(); return agent.execute({ - model: this.model, + model: this.opusModel, messages, tools: followUpTools, date, @@ -344,7 +344,7 @@ export class DataCopilot { const dataSummary = generateDataSummary(data); const agent = new AuditorAgent(); return agent.execute({ - model: this.model, + model: this.sonnetModel, messages, originalQuestion, reformulatedQuestion, From b83314d7df5121ef54dc3a9170331d1ccf073cae Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 25 Feb 2026 19:03:46 +0000 Subject: [PATCH 06/11] chore: further improvements Signed-off-by: Joana Maia --- frontend/lib/chat/data-copilot.ts | 61 ++++++++++++++++++++++--- frontend/lib/chat/instructions.ts | 16 +++---- frontend/lib/chat/prompts/auditor.ts | 15 +++--- frontend/lib/chat/types.ts | 3 +- frontend/lib/chat/utils/data-summary.ts | 10 +++- frontend/server/api/chat/chart.ts | 6 ++- 6 files changed, 85 insertions(+), 26 deletions(-) diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 109a8b20a..0e33d6df3 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -8,6 +8,8 @@ import type { Pool } from 'pg'; import type { ChatResponse, IChatResponseDb } from '../../server/repo/chat.repo'; import { ChatRepository } from '../../server/repo/chat.repo'; +import { getBucketIdForProject } from '../../server/data/tinybird/bucket-cache'; +import { fetchFromTinybird } from '../../server/data/tinybird/tinybird'; import { TextToSqlAgent, PipeAgent, RouterAgent, AuditorAgent } from './agents'; import { executePipeInstructions, executeTextToSqlInstructions } from './instructions'; import type { @@ -56,6 +58,9 @@ export class DataCopilot { /** Human-readable overview of tools for router agent decision making */ private toolsOverview: string = ''; + /** Tinybird bucketId for the current project — fetched once and reused across all pipe calls */ + private bucketId: number | null = null; + /** Tinybird MCP server URL */ private tbMcpUrl: string = ''; @@ -95,10 +100,35 @@ export class DataCopilot { }), }); - this.tbTools = await this.mcpClient.tools({}); + const allTools = await this.mcpClient.tools({}); + + // Filter out tools with empty descriptions — Bedrock rejects them with a validation error + this.tbTools = Object.fromEntries( + Object.entries(allTools).filter(([_, tool]: [string, any]) => { + const description = + (tool?.description as string) || (tool?.meta?.description as string) || ''; + return description.trim().length > 0; + }), + ); + this.buildToolsOverview(); } + /** + * Fetch and cache the Tinybird bucketId for a project. + * Delegates to getBucketIdForProject which handles Redis caching and stampede prevention. + */ + private async fetchBucketId(project: string): Promise { + if (!project) return; + try { + this.bucketId = await getBucketIdForProject(project, fetchFromTinybird); + console.warn(`🪣 [DataCopilot] bucketId for "${project}": ${this.bucketId}`); + } catch (error) { + console.error(`[DataCopilot] Failed to fetch bucketId for "${project}":`, error); + this.bucketId = null; + } + } + /** * Build human-readable overview of available tools for the router agent */ @@ -182,7 +212,9 @@ export class DataCopilot { if (agent === 'EXECUTE_INSTRUCTIONS') { model = undefined; - } else if (agent === 'TEXT_TO_SQL' || agent === 'PIPE' || agent === 'CHART') { + // For now, we use the Opus model for text-to-SQL and auditor. + // The model is currently too slow for both the pipe and router agents. + } else if (agent === 'TEXT_TO_SQL' || agent === 'AUDITOR') { model = this.BEDROCK_OPUS_MODEL_ID; } await chatRepo.saveAgentStep({ @@ -303,12 +335,21 @@ export class DataCopilot { const followUpTools: Record = {}; for (const toolName of toolNames) { if (this.tbTools[toolName]) { - followUpTools[toolName] = this.tbTools[toolName]; + const tool = this.tbTools[toolName] as any; + // Wrap execute to inject bucketId into every MCP tool call during planning + followUpTools[toolName] = + this.bucketId !== null && this.tbTools[toolName]?.execute + ? { + ...this.tbTools[toolName], + execute: async (params: any) => + tool.execute({ bucketId: this.bucketId, ...params }), + } + : tool; } } const agent = new PipeAgent(); return agent.execute({ - model: this.opusModel, + model: this.sonnetModel, messages, tools: followUpTools, date, @@ -344,7 +385,7 @@ export class DataCopilot { const dataSummary = generateDataSummary(data); const agent = new AuditorAgent(); return agent.execute({ - model: this.sonnetModel, + model: this.opusModel, messages, originalQuestion, reformulatedQuestion, @@ -621,7 +662,7 @@ export class DataCopilot { } // Prepare for retry - add feedback to messages and loop - previousFeedback = auditorResult.feedback_to_router; + previousFeedback = auditorResult.feedback_to_router || undefined; attemptNumber++; dataStream.writeData({ @@ -757,6 +798,12 @@ export class DataCopilot { const parametersString = JSON.stringify(parameters || {}); const date = new Date().toISOString().slice(0, 10); + // Fetch bucketId once upfront — required by all Tinybird pipes for data partitioning + const project = (parameters as any)?.project; + if (project) { + await this.fetchBucketId(project); + } + // Build messages from conversation history const { messages, previousWasClarification } = await this.buildMessagesFromConversation( currentQuestion, @@ -1201,7 +1248,7 @@ export class DataCopilot { // Execute the pipes according to the instructions and combine results (don't stream data yet, auditor will do it) const pipeExecutionStart = Date.now(); try { - const combinedData = await executePipeInstructions(pipeOutput.instructions); + const combinedData = await executePipeInstructions(pipeOutput.instructions, this.bucketId); const pipeExecutionTime = (Date.now() - pipeExecutionStart) / 1000; // Track successful pipe execution step diff --git a/frontend/lib/chat/instructions.ts b/frontend/lib/chat/instructions.ts index 83a6732da..f0b62a85e 100644 --- a/frontend/lib/chat/instructions.ts +++ b/frontend/lib/chat/instructions.ts @@ -28,9 +28,6 @@ async function executeTinybirdPipe( ? `${tinybirdBaseUrl}/v0/pipes/${pipeName}.json?${params}` : `${tinybirdBaseUrl}/v0/pipes/${pipeName}.json`; - console.warn(`🔍 [Tinybird] Calling pipe: ${pipeName}`); - console.warn(`🔍 [Tinybird] URL: ${url}`); - try { const response = await ofetch(url, { headers: { @@ -39,10 +36,6 @@ async function executeTinybirdPipe( }); const data = response.data || []; - console.warn( - `✅ [Tinybird] ${pipeName} returned ${data.length} rows:`, - JSON.stringify(data.slice(0, 3)), - ); return data; } catch (error) { console.error(`Error executing TinyBird pipe ${pipeName}:`, error); @@ -51,14 +44,19 @@ async function executeTinybirdPipe( } // Function to execute pipe instructions and combine results -export async function executePipeInstructions(instructions: PipeInstructions): Promise { +export async function executePipeInstructions( + instructions: PipeInstructions, + bucketId?: number | null, +): Promise { // Execute the pipes according to the instructions const pipeResults: Record = {}; // Execute each pipe with its inputs using TinyBird API for (const pipeInstruction of instructions.pipes) { try { - const result = await executeTinybirdPipe(pipeInstruction.name, pipeInstruction.inputs); + const inputs = + bucketId !== null ? { bucketId, ...pipeInstruction.inputs } : pipeInstruction.inputs; + const result = await executeTinybirdPipe(pipeInstruction.name, inputs); pipeResults[pipeInstruction.id] = result; } catch (error) { console.error(`Error executing pipe ${pipeInstruction.name}:`, error); diff --git a/frontend/lib/chat/prompts/auditor.ts b/frontend/lib/chat/prompts/auditor.ts index de1b0cd29..a49979735 100644 --- a/frontend/lib/chat/prompts/auditor.ts +++ b/frontend/lib/chat/prompts/auditor.ts @@ -36,6 +36,8 @@ export const auditorPrompt = ( }) .join('\n'); + const dataSection = `\n## TOP ROWS (first ${dataSummary.topRows.length} of ${dataSummary.rowCount} — use these actual values in your summary)\n\`\`\`json\n${JSON.stringify(dataSummary.topRows, null, 2)}\n\`\`\``; + return `You are an Auditor agent that validates whether retrieved data can answer the user's question. ## USER'S QUESTION @@ -47,6 +49,7 @@ ${reformulatedQuestion} ## DATA SUMMARY **Total Rows:** ${dataSummary.rowCount} **Columns:** ${dataSummary.columns.join(', ')} +**Top Rows:** ${dataSection} **Column Statistics:** ${statsFormatted} @@ -121,12 +124,12 @@ Make a **BINARY decision**: Can this data answer the user's question? **IF is_valid = true:** - Set \`is_valid: true\` -- Write a brief \`summary\` (2-3 sentences) for the user: - - What the data shows - - Key findings based on statistics - - Direct answer to their question - - Example: "Commit activity in 2024 ranged from 0 to 453 per day across 12 companies, - with an average of 87 commits daily." +- Write a conversational \`summary\` (1-3 sentences) for the user: + - If possible, write a summary that directly answers the user's question. + - You SHOULD reference actual values from the TOP ROWS data above — name the specific country, person, organization, etc. NEVER guess, infer, or use external knowledge. + - Unknown / null / placeholder entries: If a top row has a null, empty, "Unknown", or placeholder value (e.g. country code "XX", name "null") in a label column, do NOT treat it as a real result. Explain it represents unattributed or anonymous data (e.g. "contributions where the country of origin is unknown"), then identify and state the top row with a real value as the actual answer. + - ✅ Example: "The top contributor country is the United States with 670 contributors. Note: 5,882 contributions have no country attribution and are listed separately as 'Unknown'." + - Write summary in plain text, not markdown. **IF is_valid = false:** - Set \`is_valid: false\` diff --git a/frontend/lib/chat/types.ts b/frontend/lib/chat/types.ts index 80929c1d5..4c62cf799 100644 --- a/frontend/lib/chat/types.ts +++ b/frontend/lib/chat/types.ts @@ -120,9 +120,10 @@ export const auditorOutputSchema = z.object({ reasoning: z.string().describe('2-3 sentences explaining the validation decision'), feedback_to_router: z .string() + .nullable() .optional() .describe('If invalid, specific guidance for router to fix the issue'), - summary: z.string().optional().describe('If valid, user-friendly summary of findings'), + summary: z.string().nullable().optional().describe('If valid, user-friendly summary of findings'), }); // TypeScript types for agent outputs diff --git a/frontend/lib/chat/utils/data-summary.ts b/frontend/lib/chat/utils/data-summary.ts index c22f4f810..befe86e3c 100644 --- a/frontend/lib/chat/utils/data-summary.ts +++ b/frontend/lib/chat/utils/data-summary.ts @@ -5,6 +5,7 @@ export interface DataSummary { rowCount: number; columns: string[]; columnStats: Record; + topRows: Record[]; } export interface ColumnStats { @@ -32,8 +33,8 @@ export interface ColumnStats { /** * Generate statistical summary of dataset - * Token-efficient: ~400-500 tokens for typical dataset - * No raw data samples sent to LLM - only statistics + * Token-efficient: ~1500-200 tokens for typical dataset + * Top rows of raw data sent to LLM + statistics * * @param data - Array of data rows * @returns Statistical summary optimized for auditor validation @@ -44,6 +45,7 @@ export function generateDataSummary>(data: T[] rowCount: 0, columns: [], columnStats: {}, + topRows: [], }; } @@ -127,9 +129,13 @@ export function generateDataSummary>(data: T[] columnStats[col] = stats; } + const rows = data as Record[]; return { rowCount: data.length, columns, columnStats, + // Ideally the entire response would be available to the auditor. But that would be too costly. + // TODO: Explore a better way to have a proper summary of the data that answers the user's question directly. + topRows: rows.slice(0, 3), }; } diff --git a/frontend/server/api/chat/chart.ts b/frontend/server/api/chat/chart.ts index 198aac118..64c9759fc 100644 --- a/frontend/server/api/chat/chart.ts +++ b/frontend/server/api/chat/chart.ts @@ -4,6 +4,8 @@ import { Pool } from 'pg'; import { generateChartConfig, modifyChartConfig } from '../../../lib/chat/chart/generator'; import { ChatRepository } from '../../repo/chat.repo'; import { Result, Config, DataMapping } from '../../../lib/chat/chart/types'; +import { getBucketIdForProject } from '../../data/tinybird/bucket-cache'; +import { fetchFromTinybird } from '../../data/tinybird/tinybird'; import { PipeInstructions } from '~~/lib/chat/types'; export const maxDuration = 30; @@ -59,7 +61,9 @@ export default defineEventHandler(async (event): Promise Date: Wed, 25 Feb 2026 19:12:37 +0000 Subject: [PATCH 07/11] fix: readme Signed-off-by: Joana Maia --- frontend/lib/chat/Readme.md | 3 ++- frontend/lib/chat/instructions.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/lib/chat/Readme.md b/frontend/lib/chat/Readme.md index 1fdcfe124..620246de1 100644 --- a/frontend/lib/chat/Readme.md +++ b/frontend/lib/chat/Readme.md @@ -268,7 +268,8 @@ const mcpClient = await createMCPClient({ # Model Configuration -Currently using: `us.anthropic.claude-opus-4-6-v1` via AWS Bedrock +Currently using: `us.anthropic.claude-sonnet-4-20250514-v1:0` via AWS Bedrock for Router and Pipe Agent. +Currently using: `us.anthropic.claude-opus-4-6-v1` via AWS Bedrock for Auditor and TextToSql Agent. ## Data Flow Example diff --git a/frontend/lib/chat/instructions.ts b/frontend/lib/chat/instructions.ts index f0b62a85e..87456e957 100644 --- a/frontend/lib/chat/instructions.ts +++ b/frontend/lib/chat/instructions.ts @@ -35,8 +35,8 @@ async function executeTinybirdPipe( }, }); - const data = response.data || []; - return data; + // TinyBird response format has data array + return response.data || []; } catch (error) { console.error(`Error executing TinyBird pipe ${pipeName}:`, error); return []; From 22e0e71866573592177f8a75c546784a00403719 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Wed, 25 Feb 2026 19:18:13 +0000 Subject: [PATCH 08/11] fix: imports Signed-off-by: Joana Maia --- frontend/lib/chat/data-copilot.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 0e33d6df3..f2ad63281 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -5,11 +5,10 @@ import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; import { experimental_createMCPClient as createMCPClient, type LanguageModelV1 } from 'ai'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import type { Pool } from 'pg'; +import { ofetch } from 'ofetch'; import type { ChatResponse, IChatResponseDb } from '../../server/repo/chat.repo'; import { ChatRepository } from '../../server/repo/chat.repo'; -import { getBucketIdForProject } from '../../server/data/tinybird/bucket-cache'; -import { fetchFromTinybird } from '../../server/data/tinybird/tinybird'; import { TextToSqlAgent, PipeAgent, RouterAgent, AuditorAgent } from './agents'; import { executePipeInstructions, executeTextToSqlInstructions } from './instructions'; import type { @@ -115,13 +114,21 @@ export class DataCopilot { } /** - * Fetch and cache the Tinybird bucketId for a project. - * Delegates to getBucketIdForProject which handles Redis caching and stampede prevention. + * Fetch and store the Tinybird bucketId for a project. + * Uses ofetch directly to stay outside the Nuxt server context (no useStorage/createError). */ private async fetchBucketId(project: string): Promise { if (!project) return; + const tinybirdBaseUrl = + process.env.NUXT_TINYBIRD_BASE_URL || 'https://api.us-west-2.aws.tinybird.co'; + const tinybirdToken = process.env.NUXT_INSIGHTS_DATA_COPILOT_TINYBIRD_TOKEN; + if (!tinybirdToken) return; try { - this.bucketId = await getBucketIdForProject(project, fetchFromTinybird); + const response = await ofetch( + `${tinybirdBaseUrl}/v0/pipes/project_buckets.json?project=${encodeURIComponent(project)}`, + { headers: { Authorization: `Bearer ${tinybirdToken}` }, timeout: 10_000 }, + ); + this.bucketId = response.data?.[0]?.bucketId ?? null; console.warn(`🪣 [DataCopilot] bucketId for "${project}": ${this.bucketId}`); } catch (error) { console.error(`[DataCopilot] Failed to fetch bucketId for "${project}":`, error); From 88593d0daef6b910deef0b1e79a5374f6912c8d9 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Mon, 9 Mar 2026 11:58:23 +0000 Subject: [PATCH 09/11] fix: address PR comments Signed-off-by: Joana Maia --- frontend/lib/chat/data-copilot.ts | 15 +++++++-------- frontend/lib/chat/instructions.ts | 3 +-- frontend/lib/chat/utils/data-summary.ts | 4 +--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index f2ad63281..84af98284 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -63,14 +63,14 @@ export class DataCopilot { /** Tinybird MCP server URL */ private tbMcpUrl: string = ''; - /** Amazon Bedrock language model instance for routing and auditing (Sonnet) */ + /** Amazon Bedrock language model instance for routing and piping agents (Sonnet) */ private sonnetModel: LanguageModelV1; - /** Amazon Bedrock language model instance for text-to-SQL, pipe, and chart agents (Opus) */ + /** Amazon Bedrock language model instance for text-to-SQL, auditor, and chart agents (Opus) */ private opusModel: LanguageModelV1; /** Bedrock model identifier for general agents */ - private readonly BEDROCK_SONNET_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; + private readonly BEDROCK_SONNET_MODEL_ID = 'us.anthropic.claude-sonnet-4-6'; /** Bedrock model identifier for text-to-SQL and pipe agent */ private readonly BEDROCK_OPUS_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; @@ -126,10 +126,9 @@ export class DataCopilot { try { const response = await ofetch( `${tinybirdBaseUrl}/v0/pipes/project_buckets.json?project=${encodeURIComponent(project)}`, - { headers: { Authorization: `Bearer ${tinybirdToken}` }, timeout: 10_000 }, + { headers: { Authorization: `Bearer ${tinybirdToken}` }, timeout: 10000 }, ); this.bucketId = response.data?.[0]?.bucketId ?? null; - console.warn(`🪣 [DataCopilot] bucketId for "${project}": ${this.bucketId}`); } catch (error) { console.error(`[DataCopilot] Failed to fetch bucketId for "${project}":`, error); this.bucketId = null; @@ -215,12 +214,12 @@ export class DataCopilot { ): Promise { const chatRepo = new ChatRepository(insightsDbPool); + // For now, we use the Opus model for TextToSql and Auditor Agents. + // The model is currently too slow for both the Pipe and Router agents. let model: string | undefined = this.BEDROCK_SONNET_MODEL_ID; if (agent === 'EXECUTE_INSTRUCTIONS') { model = undefined; - // For now, we use the Opus model for text-to-SQL and auditor. - // The model is currently too slow for both the pipe and router agents. } else if (agent === 'TEXT_TO_SQL' || agent === 'AUDITOR') { model = this.BEDROCK_OPUS_MODEL_ID; } @@ -345,7 +344,7 @@ export class DataCopilot { const tool = this.tbTools[toolName] as any; // Wrap execute to inject bucketId into every MCP tool call during planning followUpTools[toolName] = - this.bucketId !== null && this.tbTools[toolName]?.execute + !!this.bucketId && this.tbTools[toolName]?.execute ? { ...this.tbTools[toolName], execute: async (params: any) => diff --git a/frontend/lib/chat/instructions.ts b/frontend/lib/chat/instructions.ts index 87456e957..b65425056 100644 --- a/frontend/lib/chat/instructions.ts +++ b/frontend/lib/chat/instructions.ts @@ -54,8 +54,7 @@ export async function executePipeInstructions( // Execute each pipe with its inputs using TinyBird API for (const pipeInstruction of instructions.pipes) { try { - const inputs = - bucketId !== null ? { bucketId, ...pipeInstruction.inputs } : pipeInstruction.inputs; + const inputs = !!bucketId ? { bucketId, ...pipeInstruction.inputs } : pipeInstruction.inputs; const result = await executeTinybirdPipe(pipeInstruction.name, inputs); pipeResults[pipeInstruction.id] = result; } catch (error) { diff --git a/frontend/lib/chat/utils/data-summary.ts b/frontend/lib/chat/utils/data-summary.ts index befe86e3c..cf1d548c8 100644 --- a/frontend/lib/chat/utils/data-summary.ts +++ b/frontend/lib/chat/utils/data-summary.ts @@ -33,7 +33,7 @@ export interface ColumnStats { /** * Generate statistical summary of dataset - * Token-efficient: ~1500-200 tokens for typical dataset + * Token-efficient: ~1500-2000 tokens for typical dataset * Top rows of raw data sent to LLM + statistics * * @param data - Array of data rows @@ -134,8 +134,6 @@ export function generateDataSummary>(data: T[] rowCount: data.length, columns, columnStats, - // Ideally the entire response would be available to the auditor. But that would be too costly. - // TODO: Explore a better way to have a proper summary of the data that answers the user's question directly. topRows: rows.slice(0, 3), }; } From cb5ca712877766c43c1f564c1bfb96bc13579167 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Mon, 9 Mar 2026 12:01:08 +0000 Subject: [PATCH 10/11] fix: tests model and documentation Signed-off-by: Joana Maia --- frontend/lib/chat/Readme.md | 2 +- frontend/lib/chat/tests/auditor.test.ts | 2 +- frontend/lib/chat/tests/router.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/lib/chat/Readme.md b/frontend/lib/chat/Readme.md index 620246de1..1e8634ffc 100644 --- a/frontend/lib/chat/Readme.md +++ b/frontend/lib/chat/Readme.md @@ -268,7 +268,7 @@ const mcpClient = await createMCPClient({ # Model Configuration -Currently using: `us.anthropic.claude-sonnet-4-20250514-v1:0` via AWS Bedrock for Router and Pipe Agent. +Currently using: `us.anthropic.claude-sonnet-4-6` via AWS Bedrock for Router and Pipe Agent. Currently using: `us.anthropic.claude-opus-4-6-v1` via AWS Bedrock for Auditor and TextToSql Agent. ## Data Flow Example diff --git a/frontend/lib/chat/tests/auditor.test.ts b/frontend/lib/chat/tests/auditor.test.ts index 9b351b381..352e7514e 100644 --- a/frontend/lib/chat/tests/auditor.test.ts +++ b/frontend/lib/chat/tests/auditor.test.ts @@ -36,7 +36,7 @@ describe('Auditor Agent', () => { region: process.env.NUXT_AWS_BEDROCK_REGION, }); - const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-6'; model = bedrock(BEDROCK_MODEL_ID); }, 30000); diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index b9f39b1cd..19a325ec1 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -46,7 +46,7 @@ describe('Router Agent', () => { }); // Initialize model once, like DataCopilot does in constructor - const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-6'; model = bedrock(BEDROCK_MODEL_ID); // Initialize MCP client to get real tools - same as DataCopilot From cef9536782b1f5512341e7fae433125505f2abfb Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Mon, 9 Mar 2026 12:07:36 +0000 Subject: [PATCH 11/11] chore: revert model Signed-off-by: Joana Maia --- frontend/lib/chat/Readme.md | 2 +- frontend/lib/chat/data-copilot.ts | 2 +- frontend/lib/chat/tests/auditor.test.ts | 2 +- frontend/lib/chat/tests/router.test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/lib/chat/Readme.md b/frontend/lib/chat/Readme.md index 1e8634ffc..620246de1 100644 --- a/frontend/lib/chat/Readme.md +++ b/frontend/lib/chat/Readme.md @@ -268,7 +268,7 @@ const mcpClient = await createMCPClient({ # Model Configuration -Currently using: `us.anthropic.claude-sonnet-4-6` via AWS Bedrock for Router and Pipe Agent. +Currently using: `us.anthropic.claude-sonnet-4-20250514-v1:0` via AWS Bedrock for Router and Pipe Agent. Currently using: `us.anthropic.claude-opus-4-6-v1` via AWS Bedrock for Auditor and TextToSql Agent. ## Data Flow Example diff --git a/frontend/lib/chat/data-copilot.ts b/frontend/lib/chat/data-copilot.ts index 84af98284..423b8b9a4 100644 --- a/frontend/lib/chat/data-copilot.ts +++ b/frontend/lib/chat/data-copilot.ts @@ -70,7 +70,7 @@ export class DataCopilot { private opusModel: LanguageModelV1; /** Bedrock model identifier for general agents */ - private readonly BEDROCK_SONNET_MODEL_ID = 'us.anthropic.claude-sonnet-4-6'; + private readonly BEDROCK_SONNET_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; /** Bedrock model identifier for text-to-SQL and pipe agent */ private readonly BEDROCK_OPUS_MODEL_ID = 'us.anthropic.claude-opus-4-6-v1'; diff --git a/frontend/lib/chat/tests/auditor.test.ts b/frontend/lib/chat/tests/auditor.test.ts index 352e7514e..9b351b381 100644 --- a/frontend/lib/chat/tests/auditor.test.ts +++ b/frontend/lib/chat/tests/auditor.test.ts @@ -36,7 +36,7 @@ describe('Auditor Agent', () => { region: process.env.NUXT_AWS_BEDROCK_REGION, }); - const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-6'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; model = bedrock(BEDROCK_MODEL_ID); }, 30000); diff --git a/frontend/lib/chat/tests/router.test.ts b/frontend/lib/chat/tests/router.test.ts index 19a325ec1..b9f39b1cd 100644 --- a/frontend/lib/chat/tests/router.test.ts +++ b/frontend/lib/chat/tests/router.test.ts @@ -46,7 +46,7 @@ describe('Router Agent', () => { }); // Initialize model once, like DataCopilot does in constructor - const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-6'; + const BEDROCK_MODEL_ID = 'us.anthropic.claude-sonnet-4-20250514-v1:0'; model = bedrock(BEDROCK_MODEL_ID); // Initialize MCP client to get real tools - same as DataCopilot