fix(agent): re-arm the domain of recently-used tools for anaphoric follow-ups

A vague reply right after a real tool turn ('do another', 'add a couple
more to it', 'try again') references the prior action only anaphorically:
no domain keyword, and not a terse continuation. _classify_agent_request
returns domains=[], the just-used tool is stripped from selection, and a
small model confabulates the action instead — observed live as a
fabricated image link, and as a wrong-tool cascade that built a phantom
document while claiming to update a note.

When a turn matches NO domain (and isn't a continuation), re-arm the
domain of any tool a recent assistant turn actually used, reusing
_DOMAIN_TOOL_MAP as the tool->domain reverse map — no per-domain special
casing. Deliberately narrow: keyword-matched turns are untouched,
casual/greeting turns exit before the re-arm, explicit continuations
already inherit context, and unmapped (user-registered MCP) tool names
are inert. Tool use is read from persisted tool_events metadata (survives
session reload) with a generated-image URL fallback. Disabled tools stay
disabled: domain seeding feeds the same schema filter as every other
selection path.

Co-Authored-By: Claude <noreply@anthropic.com>

Author: nsgds

src/agent_loop.py

@@ -952,6 +952,39 @@ def _is_contextual_retry_continuation(messages: List[Dict], text: str) -> bool:
     return bool(_COOKBOOK_CONTEXT_RE.search(recent))
 
 
+def _assistant_recently_used_tools(messages: List[Dict], lookback: int = 4) -> set:
+    """Tool names used by the last ``lookback`` assistant turns.
+
+    An anaphoric follow-up ("do another", "reply to the next one too", "add a
+    couple more") refers to a just-completed tool action but contains no domain
+    keyword, so it matches nothing and hits the low_signal short-circuit that
+    strips every tool — leaving a small model to confabulate a fake action
+    instead of repeating the real one. Re-arming the domain of a recently-used
+    tool keeps that tool offered. Read from recorded ``tool_events`` metadata,
+    with a generated-image URL fallback (image tools don't always thread the
+    metadata into the message list).
+    """
+    used: set = set()
+    checked = 0
+    for msg in reversed(messages):
+        if msg.get("role") != "assistant":
+            continue
+        checked += 1
+        if checked > lookback:
+            break
+        events = (msg.get("metadata") or {}).get("tool_events") or []
+        if isinstance(events, list):
+            for e in events:
+                if isinstance(e, dict) and e.get("tool"):
+                    used.add(e["tool"])
+        content = msg.get("content", "")
+        if isinstance(content, list):
+            content = " ".join(b.get("text", "") for b in content if isinstance(b, dict))
+        if "/api/generated-image/" in str(content or ""):
+            used.update(("generate_image", "edit_image"))
+    return used
+
+
 def _assistant_requested_followup(messages: List[Dict]) -> bool:
     """True when the previous assistant turn asked for missing task details.
 
@@ -1077,6 +1110,25 @@ def has(*patterns: str) -> bool:
            r"\b(?:home ?assistant|miniflux|gitea|linkding|jellyfin)\b"):
         domains.add("integrations")
 
+    # Anaphoric-follow-up re-arm (domain-agnostic). A vague reply right after a
+    # real tool turn ("do another", "reply to the next one too", "add a couple
+    # more") references the prior action only anaphorically — no keyword — so it
+    # matches no domain and would be stripped of every tool, leaving a small
+    # model to confabulate a fake action. If a recent assistant turn actually
+    # used a tool, re-arm that tool's domain so it stays offered, reusing the
+    # existing _DOMAIN_TOOL_MAP as the tool->domain map (no per-domain special
+    # casing). Deliberately narrow: only when THIS turn matched nothing (the
+    # keyword classifier already handled turns with their own intent, and
+    # widening every keyword-matched turn with historical domains would bloat
+    # tool selection), and not for explicit continuations, which already
+    # inherit recent context for both retrieval and domain matching.
+    if not continuation and not domains:
+        _recent_tools = _assistant_recently_used_tools(messages)
+        if _recent_tools:
+            for _dom, _dom_tools in _DOMAIN_TOOL_MAP.items():
+                if _recent_tools & _dom_tools:
+                    domains.add(_dom)
+
     low_signal = not continuation and not domains
     return {
         "low_signal": low_signal,

tests/test_anaphoric_followup_rearm.py

@@ -0,0 +1,116 @@
+"""A vague follow-up right after a real tool turn ("do another", "add a couple
+more to it", "try again") references the prior action only anaphorically — no
+domain keyword and not a terse continuation — so `_classify_agent_request`
+returns domains=[] and the just-used tool is stripped from selection. A small
+model, asked to repeat an action it no longer has, confabulates: observed live
+as a fabricated image link (images) and a wrong-tool cascade that built a
+phantom document while claiming to update a note (notes).
+
+Fix: when a turn matches NO domain (and isn't a continuation), re-arm the
+domain of any tool a recent assistant turn actually used, reusing
+_DOMAIN_TOOL_MAP as the tool->domain reverse map. Deliberately narrow — turns
+that matched a domain of their own are untouched, casual/greeting turns exit
+before the re-arm, and explicit continuations already inherit context.
+
+The classifier is deterministic (no embeddings/DB), so it is exercised directly.
+"""
+
+from src.agent_loop import (
+    _assistant_recently_used_tools,
+    _classify_agent_request,
+    _DOMAIN_TOOL_MAP,
+)
+
+
+def _tool_turn(tool, text="Done."):
+    return {"role": "assistant", "content": text,
+            "metadata": {"tool_events": [{"round": 1, "tool": tool, "exit_code": 0}]}}
+
+
+def _plain(role, text):
+    return {"role": role, "content": text}
+
+
+def _classify(msgs):
+    return _classify_agent_request(msgs, msgs[-1]["content"])
+
+
+# --- _assistant_recently_used_tools -----------------------------------------
+
+def test_collects_tools_from_tool_events():
+    msgs = [_plain("user", "make a note to buy milk"), _tool_turn("manage_notes"),
+            _plain("user", "add a couple more to it")]
+    assert "manage_notes" in _assistant_recently_used_tools(msgs)
+
+
+def test_generated_image_url_fallback_maps_to_image_tools():
+    msgs = [_plain("user", "make an image"),
+            {"role": "assistant", "content": "Direct link: https://x/api/generated-image/abc.png"},
+            _plain("user", "do another one")]
+    used = _assistant_recently_used_tools(msgs)
+    assert {"generate_image", "edit_image"} <= used
+
+
+def test_lookback_bound():
+    """Tool turns older than the lookback window are not collected."""
+    msgs = [_plain("user", "note please"), _tool_turn("manage_notes")]
+    for i in range(5):  # 5 newer assistant turns push the tool turn out (lookback=4)
+        msgs += [_plain("user", f"q{i}"), _plain("assistant", f"a{i}")]
+    msgs += [_plain("user", "add more to it")]
+    assert "manage_notes" not in _assistant_recently_used_tools(msgs)
+
+
+# --- the classifier re-arm ---------------------------------------------------
+
+def test_anaphoric_followup_rearms_used_tools_domain():
+    """The live repro: notes turn, then a keyword-free follow-up."""
+    msgs = [_plain("user", 'Make a note titled "Grocery run" to buy milk and eggs.'),
+            _tool_turn("manage_notes"),
+            _plain("user", "actually add a couple more to it — bread and coffee")]
+    intent = _classify(msgs)
+    assert "notes_calendar_tasks" in intent["domains"]
+    assert intent["low_signal"] is False
+
+
+def test_rearm_covers_every_mapped_domain():
+    """The reverse map must work for each domain's tools, not just one."""
+    for domain, tools in _DOMAIN_TOOL_MAP.items():
+        tool = sorted(tools)[0]
+        msgs = [_plain("user", "please handle this"), _tool_turn(tool),
+                _plain("user", "hmm, now do the other one as well")]
+        intent = _classify(msgs)
+        assert domain in intent["domains"], f"{tool} should re-arm {domain}"
+
+
+def test_turn_with_its_own_domain_is_untouched():
+    """A follow-up that matched a domain keeps ONLY its own match — recent tool
+    use must not piggyback extra domains onto keyword-matched turns."""
+    msgs = [_plain("user", "make a note to buy milk"), _tool_turn("manage_notes"),
+            _plain("user", "now reply to the latest email in my inbox")]
+    intent = _classify(msgs)
+    assert "email" in intent["domains"]
+    assert "notes_calendar_tasks" not in intent["domains"]
+
+
+def test_casual_turn_does_not_rearm():
+    """Greetings/thanks exit on the casual path before the re-arm."""
+    msgs = [_plain("user", "make a note to buy milk"), _tool_turn("manage_notes"),
+            _plain("user", "thanks!")]
+    intent = _classify(msgs)
+    assert intent["domains"] == set()
+    assert intent["low_signal"] is True
+
+
+def test_unmapped_mcp_tool_is_inert():
+    """A user-registered qualified MCP tool maps to no domain — no re-arm."""
+    msgs = [_plain("user", "check the weather"), _tool_turn("mcp__weather__get_forecast"),
+            _plain("user", "and tomorrow as well?")]
+    intent = _classify(msgs)
+    assert intent["domains"] == set()
+
+
+def test_fresh_chat_unaffected():
+    """No assistant history -> nothing to re-arm; fresh-turn behavior unchanged."""
+    msgs = [_plain("user", "hello there, what can you do?")]
+    intent = _classify(msgs)
+    assert intent["domains"] == set()