Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions lumen/ai/agents/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,18 @@ async def _validate_sql(
feedback = f"{type(e).__name__}: {e!s}"
if "KeyError" in feedback:
feedback += " The data does not exist; select from available data sources."
elif "not found" in str(e).lower():
# A referenced table does not exist (e.g. the model used the
# source name instead of one of its tables, common for
# multi-table sources like XArraySQLSource). Surface the
# actual table names so the retry stops guessing.
available = source.get_tables()

@ahuang11 ahuang11 Jun 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens if there are over 100 of tables in a database? Won't that consume too much context? I think we may need to do similarity search, perhaps through Metaset?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch on the ordering. It's already capped at [:50] so context stays bounded, but get_tables() comes back in arbitrary order, so with a lot of tables the slice can drop the one the model actually needs.

The metaset's a better fit, and it's already in context here. get_top_tables() ranks by similarity to the query, so I'll pull the relevant names for that source from it and only fall back to get_tables() when it's empty. Pushing that shortly.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we need to mention it's truncated if it's over 50

if available:
names = ", ".join(repr(t) for t in available[:50])
feedback += (
f" Use one of these exact table names, not the "
f"source name: {names}."
)

retry_result = await self.revise(
feedback, messages, context, spec=sql_query, language=f"sql.{source.dialect}",
Expand Down
87 changes: 87 additions & 0 deletions lumen/tests/ai/test_sql_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
from unittest.mock import AsyncMock, MagicMock

import pytest

try:
import lumen.ai # noqa
except ModuleNotFoundError:
pytest.skip(
"lumen.ai could not be imported, skipping tests.",
allow_module_level=True,
)

from lumen.ai.agents.sql import SQLAgent


@pytest.mark.asyncio
async def test_validate_sql_surfaces_table_names_on_not_found():
"""On a 'table not found' error the retry feedback should list the
source's real table names so the model stops using the source name
(the failure mode for multi-table sources like XArraySQLSource)."""
agent = SQLAgent()

source = MagicMock()
source.dialect = "duckdb"
source.get_tables.return_value = ["CMI_C01", "DQF_C01"]

attempts = {"n": 0}

def execute(sql, *a, **k):
attempts["n"] += 1
if attempts["n"] == 1:
# First attempt: model referenced the source name as a table.
raise ValueError("Error during planning: table 'MySource' not found")
return None # Corrected query validates.

source.execute.side_effect = execute

captured = {}

async def fake_revise(feedback, *a, **k):
captured["feedback"] = feedback
return "SELECT * FROM CMI_C01"

agent.revise = fake_revise

result = await agent._validate_sql(
context={},
sql_query='SELECT * FROM "MySource"',
expr_slug="x",
source=source,
messages=[],
step=MagicMock(),
max_retries=2,
)

# Recovered using the corrected SQL.
assert "CMI_C01" in result
# The retry feedback surfaced the real table names + the hint.
assert "CMI_C01" in captured["feedback"]
assert "DQF_C01" in captured["feedback"]
assert "not the source name" in captured["feedback"]


@pytest.mark.asyncio
async def test_validate_sql_valid_query_no_retry():
"""A valid query validates on the first try with no retry/feedback."""
agent = SQLAgent()

source = MagicMock()
source.dialect = "duckdb"
source.execute.return_value = None # valid

agent.revise = AsyncMock(
side_effect=AssertionError("revise should not be called")
)

result = await agent._validate_sql(
context={},
sql_query="SELECT * FROM CMI_C01",
expr_slug="x",
source=source,
messages=[],
step=MagicMock(),
max_retries=2,
)
assert "CMI_C01" in result
source.get_tables.assert_not_called()
Loading