feat:nao test: assert the agent performed a specific action

apps/backend/src/agents/tools/index.ts

@@ -55,7 +55,6 @@ export const getTools = (
 	agentSettings: AgentSettings | null,
 	extraTools?: Record<string, unknown>,
 	options: {
-		testMode?: boolean;
 		mcpEnabled?: boolean;
 		mcpServers?: string[] | null;
 		excludeFollowUps?: boolean;
@@ -89,14 +88,7 @@ export const getTools = (
 			}
 		: {};
 
-	const {
-		execute_python,
-		execute_sandboxed_code,
-		clarification: clarificationTool,
-		suggest_follow_ups,
-		write: writeTool,
-		...rest
-	} = tools;
+	const { execute_python, execute_sandboxed_code, suggest_follow_ups, write: writeTool, ...rest } = tools;
 	const baseTools = {
 		...rest,
 		...(isStorageEnabled() && { write: writeTool }),
@@ -105,7 +97,6 @@ export const getTools = (
 
 	const allTools = {
 		...baseTools,
-		...(!options.testMode && { clarification: clarificationTool }),
 		...mcpTools,
 		...(agentSettings?.experimental?.pythonSandboxing && execute_python && { execute_python }),
 		...(agentSettings?.experimental?.sandboxes && execute_sandboxed_code && { execute_sandboxed_code }),

apps/backend/src/components/ai/system-prompt.tsx

@@ -29,7 +29,6 @@ type SystemPromptProps = {
 	/** Names of MCP servers the agent is allowed to call (tools discovered as on-disk specs). */
 	mcpServers?: string[];
 	timezone?: string;
-	testMode?: boolean;
 	/** Names of the tools in the run's tool set — rules for surface-dependent tools (e.g. display_map) are only emitted when the tool is present. Omit to include every rule. */
 	toolNames?: string[];
 	options?: SystemPromptOptions;
@@ -52,7 +51,6 @@ export function SystemPrompt({
 	customCharts = [],
 	mcpServers = [],
 	timezone,
-	testMode,
 	toolNames,
 	options = {},
 }: SystemPromptProps) {
@@ -106,15 +104,17 @@ export function SystemPrompt({
 						researching.
 					</ListItem>,
 					<ListItem>If you can execute a SQL query, use the execute_sql tool for it.</ListItem>,
-					!testMode && (
-						<ListItem>
-							Use the <Bold>clarification</Bold> tool when the user's request is genuinely ambiguous and
-							proceeding would likely produce the wrong result (e.g. multiple plausible tables, unclear
-							time range, undefined metric). If you need to ask another clarifying question after the user
-							answers, call the <Bold>clarification</Bold> tool again instead of asking in plain text,
-							bullet lists, or examples.
-						</ListItem>
-					),
+					...(hasTool('clarification')
+						? [
+								<ListItem>
+									Use the <Bold>clarification</Bold> tool when the user's request is genuinely
+									ambiguous and proceeding would likely produce the wrong result (e.g. multiple
+									plausible tables, unclear time range, undefined metric). If you need to ask another
+									clarifying question after the user answers, call the <Bold>clarification</Bold> tool
+									again instead of asking in plain text, bullet lists, or examples.
+								</ListItem>,
+							]
+						: []),
 					...dialectToolCallRules,
 				]}
 			</List>

apps/backend/src/handlers/automation.handler.ts

@@ -124,7 +124,7 @@ async function finishAutomationRun(automation: AutomationWithSchedule, run: DBAu
 			{
 				excludeFollowUps: true,
 				supportsCustomCharts: false,
-				tools: ({ chat: agentChat, agentSettings, webTools }) =>
+				tools: ({ agentSettings, webTools }) =>
 					getTools(
 						agentSettings,
 						{
@@ -139,7 +139,6 @@ async function finishAutomationRun(automation: AutomationWithSchedule, run: DBAu
 							}),
 						},
 						{
-							testMode: agentChat.testMode,
 							mcpEnabled: automation.mcpEnabled,
 							mcpServers: automation.mcpServers,
 							excludeFollowUps: true,

apps/backend/src/routes/test.ts

@@ -39,7 +39,8 @@ export const testRoutes = async (app: App) => {
 				body: z.object({
 					prompt: z.string(),
 					model: llmSelectedModelSchema,
-					sql: z.string(),
+					// Optional: assertion-only tests omit reference SQL
+					sql: z.string().optional().default(''),
 					meta: z
 						.object({
 							costs: customModelCostSchema,

apps/backend/src/services/agent.ts

@@ -101,7 +101,6 @@ export interface AgentRunResult {
 
 export type AgentChat = Pick<DBChat, 'id' | 'projectId' | 'userId'> & {
 	forkMetadata?: ForkMetadata | null;
-	testMode?: boolean;
 };
 
 /** Dependencies a tool resolver receives once a run's context has been resolved. */
@@ -119,26 +118,25 @@ export interface AgentToolsContext {
 export type AgentToolsResolver = (context: AgentToolsContext) => AgentTools | Promise<AgentTools>;
 
 /** Default tool set for interactive runs: all built-ins, MCP tools and web search. */
-export const defaultAgentTools: AgentToolsResolver = ({ chat, agentSettings, webTools, customBoundaries }) =>
-	getTools(agentSettings, webTools ?? {}, { testMode: chat.testMode, customBoundaries });
+export const defaultAgentTools: AgentToolsResolver = ({ agentSettings, webTools, customBoundaries }) =>
+	getTools(agentSettings, webTools ?? {}, { customBoundaries });
 
 /** Default tool set minus the given built-ins — for runs whose surface cannot render them. */
 export const defaultAgentToolsExcluding =
 	(excludeBuiltinTools: string[]): AgentToolsResolver =>
-	({ chat, agentSettings, webTools, customBoundaries }) =>
-		getTools(agentSettings, webTools ?? {}, { testMode: chat.testMode, excludeBuiltinTools, customBoundaries });
+	({ agentSettings, webTools, customBoundaries }) =>
+		getTools(agentSettings, webTools ?? {}, { excludeBuiltinTools, customBoundaries });
 
 /**
  * Admin-mode tool set: the same `execute_sql` tool the chat already uses (it
  * runs against nao's own app database when `ToolContext.adminMode` is set),
  * plus charting and follow-ups. Excludes the filesystem context tools.
  */
-export const adminAgentTools: AgentToolsResolver = ({ chat, agentSettings }) =>
+export const adminAgentTools: AgentToolsResolver = ({ agentSettings }) =>
 	getTools(
 		agentSettings,
 		{},
 		{
-			testMode: chat.testMode,
 			builtinToolAllowlist: [
 				'execute_sql',
 				'read_query_result',
@@ -273,9 +271,7 @@ export class AgentService {
 		const agentTools = await resolveTools({ chat, agentSettings, toolContext, webTools, customBoundaries });
 		const stopWhen: StopCondition<AgentTools>[] = options.excludeFollowUps
 			? [stepCountIs(options.maxSteps ?? 20)]
-			: chat.testMode
-				? [hasToolCall('suggest_follow_ups')]
-				: [hasToolCall('suggest_follow_ups'), hasToolCall('clarification')];
+			: [hasToolCall('suggest_follow_ups'), hasToolCall('clarification')];
 		const agent = new AgentManager(
 			chat,
 			modelConfig,
@@ -613,7 +609,6 @@ class AgentManager {
 				customCharts,
 				mcpServers,
 				timezone,
-				testMode: this.chat.testMode,
 				toolNames: Object.keys(this._agentTools),
 				options: { canGrepSavedFiles: canGrepUserFiles() },
 			}),

apps/backend/src/services/test-agent.service.ts

@@ -54,7 +54,6 @@ export class TestAgentService extends AgentService {
 			messages: [userMessage],
 			userId: 'test',
 			projectId,
-			testMode: true,
 		};
 
 		const agent = await this.create(tempChat, modelSelection);

cli/README.md

@@ -212,7 +212,19 @@ databases:
 nao test
 ```
 
-Runs test cases defined as YAML files in `tests/`. Each test has a `name`, `prompt`, and expected `sql`. Results are saved to `tests/outputs/`.
+Runs test cases defined as YAML files in `tests/`. Each test has a `name`, `prompt`, and optional expected `sql` and/or `assertions`. Results are saved to `tests/outputs/`.
+
+Final-output checks use reference `sql` (dataframe equality). Intermediate agent actions use `assertions` against the run's tool-call trace — for example, requiring a clarifying follow-up:
+
+```yaml
+name: ambiguous_revenue_period
+prompt: What was the revenue?
+assertions:
+  - type: tool_call
+    tool: clarification
+```
+
+`tool_call` assertions can also require a specific tool (e.g. `execute_sql`), optional arg subset match via `args`, and `min_count`. SQL verification and assertions can be combined; the run passes only if every check passes.
 
 Options:
 

cli/nao_core/commands/test/assertions.py

@@ -0,0 +1,149 @@
+"""Extensible action/step assertions for `nao test`.
+
+Assertions check intermediate agent behavior (tool calls, steps) independently
+of final-output dataframe verification.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, cast
+
+
+class AssertionConfigError(ValueError):
+    """Raised when an assertion definition in a test YAML is invalid."""
+
+
+@dataclass(frozen=True)
+class ToolCallAssertion:
+    """Require that a named tool was invoked during the agentic loop.
+
+    Optional ``args`` values must appear as a subset of the tool call's args
+    (nested dicts are matched recursively; lists require equality).
+    """
+
+    tool: str
+    args: dict[str, Any] | None = None
+    min_count: int = 1
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ToolCallAssertion:
+        tool = data.get("tool")
+        if not isinstance(tool, str) or not tool.strip():
+            raise AssertionConfigError("tool_call assertion requires a non-empty 'tool' string")
+
+        args = data.get("args")
+        if args is not None and not isinstance(args, dict):
+            raise AssertionConfigError("tool_call assertion 'args' must be a mapping when provided")
+
+        min_count = data.get("min_count", 1)
+        if not isinstance(min_count, int) or isinstance(min_count, bool) or min_count < 1:
+            raise AssertionConfigError("tool_call assertion 'min_count' must be an integer >= 1")
+
+        unknown = set(data) - {"type", "tool", "args", "min_count"}
+        if unknown:
+            raise AssertionConfigError(f"unknown tool_call assertion fields: {sorted(unknown)}")
+
+        return cls(tool=tool.strip(), args=args, min_count=min_count)
+
+
+Assertion = ToolCallAssertion
+
+
+def parse_assertions(raw: Any) -> list[Assertion]:
+    """Parse the optional ``assertions`` list from a test YAML document."""
+    if raw is None:
+        return []
+    if not isinstance(raw, list):
+        raise AssertionConfigError("'assertions' must be a list")
+
+    assertions: list[Assertion] = []
+    for index, item in enumerate(raw):
+        if not isinstance(item, dict):
+            raise AssertionConfigError(f"assertions[{index}] must be a mapping")
+        entry = cast(dict[str, Any], item)
+        assertion_type = entry.get("type")
+        if assertion_type == "tool_call":
+            assertions.append(ToolCallAssertion.from_dict(entry))
+        elif assertion_type is None:
+            raise AssertionConfigError(f"assertions[{index}] is missing 'type'")
+        else:
+            raise AssertionConfigError(f"assertions[{index}] has unknown type {assertion_type!r}; supported: tool_call")
+    return assertions
+
+
+def _args_match(expected: Any, actual: Any) -> bool:
+    """Return True when ``expected`` is a subset of ``actual`` (dicts recursive)."""
+    if isinstance(expected, dict):
+        if not isinstance(actual, dict):
+            return False
+        return all(key in actual and _args_match(value, actual[key]) for key, value in expected.items())
+    return expected == actual
+
+
+def evaluate_tool_call_assertion(
+    assertion: ToolCallAssertion,
+    tool_calls: list[dict[str, Any]] | None,
+) -> tuple[bool, str]:
+    """Evaluate a single tool_call assertion against recorded tool calls."""
+    calls = tool_calls or []
+    matches = [
+        call
+        for call in calls
+        if call.get("toolName") == assertion.tool
+        and (assertion.args is None or _args_match(assertion.args, call.get("args") or {}))
+    ]
+    count = len(matches)
+    if count >= assertion.min_count:
+        if assertion.min_count == 1 and assertion.args is None:
+            return True, f"tool_call: {assertion.tool}"
+        detail = f"tool_call: {assertion.tool} (x{count}"
+        if assertion.min_count > 1:
+            detail += f", min {assertion.min_count}"
+        if assertion.args is not None:
+            detail += ", args matched"
+        detail += ")"
+        return True, detail
+
+    if assertion.args is not None:
+        same_tool = sum(1 for call in calls if call.get("toolName") == assertion.tool)
+        if same_tool:
+            return (
+                False,
+                f"missing tool_call: {assertion.tool} with args {assertion.args} "
+                f"(found {count} matching, need >= {assertion.min_count}; {same_tool} total)",
+            )
+        return False, f"missing tool_call: {assertion.tool} with args {assertion.args}"
+
+    if assertion.min_count > 1:
+        return (
+            False,
+            f"missing tool_call: {assertion.tool} (found {count}, need >= {assertion.min_count})",
+        )
+    return False, f"missing tool_call: {assertion.tool}"
+
+
+def evaluate_assertions(
+    assertions: list[Assertion],
+    tool_calls: list[dict[str, Any]] | None,
+) -> tuple[bool, str]:
+    """Evaluate all assertions. Returns (passed, combined message)."""
+    if not assertions:
+        return True, ""
+
+    messages: list[str] = []
+    all_passed = True
+    for assertion in assertions:
+        if isinstance(assertion, ToolCallAssertion):
+            passed, message = evaluate_tool_call_assertion(assertion, tool_calls)
+        else:  # pragma: no cover - exhaustive for current Assertion union
+            passed, message = False, f"unsupported assertion: {assertion!r}"
+        messages.append(message)
+        all_passed = all_passed and passed
+
+    return all_passed, "; ".join(messages)
+
+
+def combine_check_messages(*parts: str) -> str:
+    """Join non-empty check messages with '; '."""
+    return "; ".join(part for part in parts if part)

cli/nao_core/commands/test/case.py

@@ -1,10 +1,12 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 
 import yaml
 
 from nao_core.ui import UI
 
+from .assertions import Assertion, parse_assertions
+
 TESTS_FOLDER = "tests/"
 
 
@@ -15,7 +17,8 @@ class TestCase:
     name: str
     prompt: str
     file_path: Path
-    sql: str
+    sql: str | None = None
+    assertions: list[Assertion] = field(default_factory=list)
 
     @classmethod
     def from_yaml(cls, file_path: Path) -> "TestCase":
@@ -28,6 +31,7 @@ def from_yaml(cls, file_path: Path) -> "TestCase":
             prompt=data["prompt"],
             sql=data.get("sql"),
             file_path=file_path,
+            assertions=parse_assertions(data.get("assertions")),
         )
 
 

cli/nao_core/commands/test/client.py

@@ -128,7 +128,8 @@ def run_test(
                 "modelId": model_id,
             },
             "prompt": test_case.prompt,
-            "sql": test_case.sql,
+            # sql is optional: assertion-only tests may omit reference SQL
+            "sql": test_case.sql or "",
         }
 
         cost_payload = serialize_model_costs(costs)