fix: improve chat accuracy and keyword fallback

- Add date/portal extraction before keyword classification in fallback
- Partial portal name matching (e.g. 'Löwenbräu' → 'Löwenbräu-Festzelt')
- Include actual dates in _compress_availability and MCP tool results
- Add check_date param for on-demand deep-scanning in monitor_availability
- Rewrite SCANNER_INSTRUCTIONS with accuracy and action rules
- Lower LLM temperature to 0.2 to reduce hallucination
- Show accumulated thinking steps in chat UI
- Add eval_chat_accuracy.py test suite (6/6 passing)

Author: annawiewer

src/wiesn_agent/api.py

@@ -51,6 +51,9 @@
 # Stores both user and agent messages for the chat panel.
 _chat_log: deque[dict] = deque(maxlen=200)
 
+# ── Thinking status (broadcast to SSE clients) ───
+_thinking_status: str = ""
+
 
 # ── Persistence helpers ───────────────────────────
 
@@ -464,6 +467,70 @@ class ChatMessage(BaseModel):
 }
 
 
+_MONTH_NAMES = {
+    "januar": "01", "februar": "02", "märz": "03", "april": "04",
+    "mai": "05", "juni": "06", "juli": "07", "august": "08",
+    "september": "09", "oktober": "10", "november": "11", "dezember": "12",
+}
+
+
+def _extract_date(text: str) -> str:
+    """Extract a date from user text. Returns YYYY-MM-DD or DD.MM.YYYY or empty string."""
+    # Match DD.MM or DD.MM.YYYY
+    m = re.search(r'(\d{1,2})\.(\d{1,2})(?:\.(\d{4}))?', text)
+    if m:
+        day = int(m.group(1))
+        month = int(m.group(2))
+        year = int(m.group(3)) if m.group(3) else 2026
+        return f"{year}-{month:02d}-{day:02d}"
+
+    # Match "DD. Monat" or "DD Monat"
+    m = re.search(r'(\d{1,2})\.?\s+(januar|februar|märz|april|mai|juni|juli|august|september|oktober|november|dezember)', text.lower())
+    if m:
+        day = int(m.group(1))
+        month = int(_MONTH_NAMES[m.group(2)])
+        return f"2026-{month:02d}-{day:02d}"
+
+    return ""
+
+
+def _date_matches(iso_date: str, text: str) -> bool:
+    """Check if an ISO date (YYYY-MM-DD) matches a date in text (various formats)."""
+    try:
+        from datetime import datetime as dt
+        parsed = dt.strptime(iso_date, "%Y-%m-%d")
+        # DD.MM.YYYY format
+        if parsed.strftime("%d.%m.%Y") in text:
+            return True
+        # German month name
+        months_de = ["Januar", "Februar", "März", "April", "Mai", "Juni",
+                     "Juli", "August", "September", "Oktober", "November", "Dezember"]
+        german = f"{parsed.day}. {months_de[parsed.month - 1]}"
+        if german in text:
+            return True
+    except (ValueError, IndexError):
+        pass
+    return False
+
+
+def _find_portal(text: str, config) -> str | None:
+    """Find a portal name mentioned in text, with partial matching."""
+    lower = text.lower()
+    # Exact match first
+    for portal in config.portale:
+        if portal.name.lower() in lower:
+            return portal.name
+    # Partial match: split portal name on spaces/hyphens and check core parts
+    for portal in config.portale:
+        parts = re.split(r'[\s\-]+', portal.name.lower())
+        # Match if any distinctive part (>3 chars, not generic) appears
+        generic = {"fest", "zelt", "festzelt", "wiesn"}
+        for part in parts:
+            if len(part) > 3 and part not in generic and part in lower:
+                return portal.name
+    return None
+
+
 def _classify_intent(text: str) -> str:
     """Classify user message into an intent. Returns intent name or 'unknown'."""
     lower = text.lower().strip()
@@ -534,14 +601,11 @@ async def post_chat(body: ChatMessage):
         }
 
         def _on_tool_progress(tool_name: str, tool_args: dict) -> None:
+            global _thinking_status
             portal = tool_args.get("portal_name") or tool_args.get("name") or ""
             label = _TOOL_LABELS.get(tool_name, tool_name)
             detail = f" — {portal}" if portal else ""
-            _chat_log.append({
-                "timestamp": datetime.now().isoformat(),
-                "role": "thinking",
-                "message": f"{label}{detail}",
-            })
+            _thinking_status = f"{label}{detail}"
 
         history = list(_chat_log)[:-1]  # exclude current message (already in prompt)
         reply_text = await llm_chat(
@@ -550,9 +614,8 @@ def _on_tool_progress(tool_name: str, tool_args: dict) -> None:
             on_progress=_on_tool_progress,
         )
 
-        # Remove thinking entries before adding final reply
-        while _chat_log and _chat_log[-1].get("role") == "thinking":
-            _chat_log.pop()
+        # Clear thinking status
+        _thinking_status = ""
 
         reply = _chat_reply(reply_text)
         return {"user": user_entry, "reply": reply}
@@ -563,18 +626,78 @@ def _on_tool_progress(tool_name: str, tool_args: dict) -> None:
         logger.warning("LLM chat error, falling back to keywords: %s", e, exc_info=True)
 
     # ── Keyword fallback ──────────────────────────
+    # First, check for date/portal mentions (more specific than keyword intents)
+    snapshots = load_snapshots()
+    config = _load_config()
+    mentioned_date = _extract_date(text)
+    mentioned_portal = _find_portal(text, config)
+
+    if mentioned_portal and mentioned_date:
+        snap = snapshots.get(mentioned_portal)
+        has_date = False
+        if snap:
+            for d in snap.datum_options:
+                val = d.get("value", d.get("text", ""))
+                txt = d.get("text", d.get("value", ""))
+                if mentioned_date in val or mentioned_date in txt or _date_matches(mentioned_date, txt):
+                    has_date = True
+                    break
+        if has_date:
+            reply = _chat_reply(
+                f"**{mentioned_portal}** hat den **{mentioned_date}** als auswählbares Datum. "
+                f"Abend-Slots sind nicht bestätigt (dafür ist ein Deep-Scan nötig)."
+            )
+        else:
+            reply = _chat_reply(f"**{mentioned_portal}** hat den **{mentioned_date}** leider **nicht** verfügbar.")
+        return {"user": user_entry, "reply": reply}
+
+    if mentioned_date:
+        with_date = []
+        without_date = []
+        for name, snap in snapshots.items():
+            found = False
+            for d in snap.datum_options:
+                val = d.get("value", d.get("text", ""))
+                txt = d.get("text", d.get("value", ""))
+                if mentioned_date in val or mentioned_date in txt or _date_matches(mentioned_date, txt):
+                    found = True
+                    break
+            if found:
+                with_date.append(name)
+            else:
+                without_date.append(name)
+
+        if with_date:
+            reply = _chat_reply(
+                f"**{len(with_date)}** Zelte haben den **{mentioned_date}** als auswählbares Datum: "
+                f"{', '.join(with_date)}.\n\n"
+                f"**{len(without_date)}** Zelte haben diesen Tag nicht."
+            )
+        else:
+            reply = _chat_reply(f"Kein Zelt hat den **{mentioned_date}** verfügbar.")
+        return {"user": user_entry, "reply": reply}
+
+    if mentioned_portal:
+        snap = snapshots.get(mentioned_portal)
+        if snap and snap.datum_options:
+            dates = [d.get("text", d.get("value", "")) for d in snap.datum_options]
+            reply = _chat_reply(
+                f"**{mentioned_portal}** hat **{len(dates)}** auswählbare Termine:\n"
+                + ", ".join(dates)
+            )
+        elif snap:
+            reply = _chat_reply(f"**{mentioned_portal}** hat aktuell **keine** verfügbaren Termine.")
+        else:
+            reply = _chat_reply(f"**{mentioned_portal}** wurde noch nicht gescannt.")
+        return {"user": user_entry, "reply": reply}
+
+    # Fall back to keyword classification for generic intents
     intent = _classify_intent(text)
 
     # ── Intent: Scan ──────────────────────────────
     if intent == "scan":
         config = _load_config()
-        # Check if a specific portal is mentioned
-        target_portal = None
-        lower = text.lower()
-        for portal in config.portale:
-            if portal.name.lower() in lower:
-                target_portal = portal.name
-                break
+        target_portal = _find_portal(text, config)
 
         if target_portal:
             reply = _chat_reply(f"Starting scan for **{target_portal}**...")
@@ -652,7 +775,7 @@ def _on_tool_progress(tool_name: str, tool_args: dict) -> None:
         reply = _chat_reply("Portals:\n" + "\n".join(lines))
         return {"user": user_entry, "reply": reply}
 
-    # ── Default: unrecognized → show help ─────────
+    # ── Truly unrecognized → show help ──
     reply = _chat_reply(
         "I didn't quite catch that. Here's what I can help with:\n"
         "- **scan** — Start scanning portals\n"
@@ -692,10 +815,15 @@ async def event_generator():
         # Use a snapshot of the current deque to avoid index race conditions
         # when items are evicted from the maxlen ring buffer.
         last_seen = len(list(_chat_log))
+        prev_thinking = ""
         yield f"data: {json.dumps({'type': 'connected', 'count': last_seen})}\n\n"
         while True:
             if await request.is_disconnected():
                 break
+            # Broadcast thinking status changes immediately
+            if _thinking_status != prev_thinking:
+                prev_thinking = _thinking_status
+                yield f"data: {json.dumps({'role': 'thinking', 'message': _thinking_status})}\n\n"
             snapshot = list(_chat_log)
             if len(snapshot) > last_seen:
                 for item in snapshot[last_seen:]:
@@ -704,7 +832,7 @@ async def event_generator():
             elif len(snapshot) < last_seen:
                 # Buffer wrapped — reset
                 last_seen = len(snapshot)
-            await asyncio.sleep(0.5)
+            await asyncio.sleep(0.3)
 
     return StreamingResponse(event_generator(), media_type="text/event-stream")
 

src/wiesn_agent/chat_agent.py

@@ -77,14 +77,35 @@
 {config}
 
 ## Your Tools
-- `monitor_availability(portal_name)` — scan date dropdowns, compare with saved \
+- `monitor_availability(portal_name, check_date)` — scan date dropdowns, compare with saved \
 snapshots, deep-scan time slots.
   - Use `portal_name="all"` for ALL portals (preferred!).
   - Use a specific name (e.g. `"Hacker-Festzelt"`) for one portal.
   - **NEVER** call separately for each portal — use "all" instead!
+  - Use `check_date="2026-09-25"` to deep-scan time slots for a specific date.\
+    **When the user asks about a specific date, ALWAYS use check_date!**
 - `check_portal(name)` — navigate to one portal and get page info.
 - `check_all_portals()` — quick check of all portals.
 
+## Accuracy Rules (CRITICAL — HIGHEST PRIORITY)
+1. **NEVER guess or assume availability.** Only state what the tool result \
+explicitly contains.
+2. The tool result lists EXACT dates per portal. If the user asks about a \
+specific date (e.g. 25.9), look for exactly that date in the tool output. \
+If it's NOT listed → that portal does NOT have it. Say so.
+3. **Evening/abends slots** are only confirmed when the tool result explicitly \
+lists `abend_slots` for that date. A date in `datum_options` does NOT mean \
+evening slots exist — it only means the date dropdown contains it.
+4. If unsure, say "not confirmed" — never say "available" without proof.
+5. **Accuracy is the core product value. Wrong data = broken trust.**
+
+## Action Rules
+1. When the user asks about availability, **immediately call the tool** — \
+do NOT ask "shall I check?" or "want me to look?".
+2. Always answer with specific data. **NEVER end your reply with a question** \
+like "Soll ich prüfen?" or "Möchtest du...?". Just give the answer.
+3. Only ask before WRITING/SUBMITTING — never before reading/checking.
+
 ## Background Monitoring
 A background scanner runs automatically every few minutes. \
 When the user asks "Status" or "Übersicht", you can answer from your knowledge — \
@@ -244,7 +265,11 @@ def _classify(self, text: str) -> str:
 # ── Availability compression ─────────────────────
 
 def _compress_availability(raw: str) -> str:
-    """Compress monitor_availability JSON into a pre-formatted summary."""
+    """Compress monitor_availability JSON into a pre-formatted summary.
+
+    Includes actual date values so the LLM can accurately answer
+    date-specific queries without guessing.
+    """
     import json
 
     try:
@@ -260,16 +285,21 @@ def _compress_availability(raw: str) -> str:
     without_dates: list[str] = []
     with_new: list[str] = []
     errors: list[str] = []
+    # Collect per-portal date details for accuracy
+    date_details: list[str] = []
 
     for r in results:
         name = r.get("portal", "?")
         count = r.get("datum_count", 0)
         new = r.get("new_dates", [])
+        dates = r.get("dates", [])  # actual date values
 
         if r.get("error"):
             errors.append(name)
         elif count > 0:
             with_dates.append(f"{name} ({count})")
+            if dates:
+                date_details.append(f"{name}: {', '.join(dates)}")
             if new and not r.get("is_first_scan"):
                 with_new.append(f"{name}: +{len(new)} new")
         else:
@@ -279,6 +309,10 @@ def _compress_availability(raw: str) -> str:
 
     lines = [
         "RELAY THIS SUMMARY TO THE USER (translate to their language, keep it compact):",
+        "ACCURACY RULE: A date in 'Available dates per portal' means the date is SELECTABLE in the dropdown.",
+        "It does NOT mean evening/abends slots are available — evening confirmation requires 'deep_scan'.",
+        "If the user asks about evening: only confirm if 'Evening:' lines exist below for that date.",
+        "If no 'Evening:' line exists → say 'date is selectable but evening slots are not yet confirmed'.",
         "",
         f"{len(with_dates)} of {total} tents have open dates: {', '.join(with_dates)}.",
     ]
@@ -291,13 +325,33 @@ def _compress_availability(raw: str) -> str:
     if errors:
         lines.append(f"Errors: {', '.join(errors)}.")
 
+    # Include exact dates per portal so LLM can answer date-specific queries
+    if date_details:
+        lines.append("")
+        lines.append("Available dates per portal (selectable in dropdown, NOT confirmed evening):")
+        lines.extend(date_details)
+
+    has_evening = False
+    deep_scanned_no_evening: list[str] = []
     for r in results:
         ds = r.get("deep_scan")
         if ds:
             for d in ds:
                 slots = d.get("abend_slots", [])
                 if slots:
-                    lines.append(f"Evening: {r['portal']} — {d['datum']}: {', '.join(slots)}")
+                    lines.append(f"CONFIRMED Evening: {r['portal']} — {d['datum']}: {', '.join(slots)}")
+                    has_evening = True
+                else:
+                    deep_scanned_no_evening.append(r.get("portal", "?"))
+
+    if deep_scanned_no_evening:
+        lines.append("")
+        lines.append(f"Deep-scanned but NO evening slots: {', '.join(deep_scanned_no_evening)}.")
+        lines.append("These tents have the date selectable but NO evening time slots are available.")
+
+    if not has_evening and not deep_scanned_no_evening:
+        lines.append("")
+        lines.append("No evening slots confirmed in this scan. Dates above are only selectable in the dropdown — time slot availability is unknown until deep-scanned.")
 
     return "\n".join(lines)
 
@@ -425,7 +479,7 @@ def _build_workflow(self) -> WorkflowAgent:
         )
 
         config_ctx = self._build_config_context()
-        agent_opts = OpenAIChatCompletionOptions(temperature=0.4, max_tokens=2048)
+        agent_opts = OpenAIChatCompletionOptions(temperature=0.2, max_tokens=2048)
 
         # Shared middleware: progress reporting + result compression.
         # Captures self._on_progress which is updated before each chat() call.