add module 9 files

Author: santana-ai

module-09-tracing/demos/.env.example

@@ -0,0 +1,6 @@
+OPENAI_API_KEY=sk-...
+
+# Langfuse tracing — see LANGFUSE.md to start the local stack (http://localhost:3000).
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+LANGFUSE_HOST=http://localhost:3000

module-09-tracing/demos/.gitignore

@@ -0,0 +1,6 @@
+/state.db
+/last_run_id.txt
+/logs/runs.jsonl
+.venv
+__pycache__/
+.env

module-09-tracing/demos/.python-version

@@ -0,0 +1 @@
+3.12

module-09-tracing/demos/LANGFUSE.md

@@ -0,0 +1,48 @@
+# Local Langfuse for the tracing demo
+
+`demo_02.py` ships its span tree to Langfuse. Run Langfuse locally with its official
+Docker stack, pinned to a release:
+
+```bash
+git clone --branch v3.130.0 https://github.com/langfuse/langfuse.git
+cd langfuse
+docker compose up -d          # UI on http://localhost:3000 after ~2-3 min
+```
+
+Then:
+
+1. Open `http://localhost:3000`, create an account (local, not the cloud), create a project.
+2. Copy the project's **public** and **secret** keys into this demo's `.env`:
+
+   ```
+   LANGFUSE_PUBLIC_KEY=pk-lf-...
+   LANGFUSE_SECRET_KEY=sk-lf-...
+   LANGFUSE_HOST=http://localhost:3000
+   ```
+
+3. Run the traced demo and open the run in the UI:
+
+   ```bash
+   uv run python demo_02.py --case traced-run
+   ```
+
+This is a **separate** Docker stack from any RealThor container — run it from the cloned
+`langfuse/` directory with its own `docker compose`. Pin the release tag; the self-hosting
+stack evolves. Current quickstart: https://langfuse.com/self-hosting/deployment/docker-compose
+
+The backend is chosen entirely by env vars, so switching to Langfuse Cloud is just a `.env`
+change: set `LANGFUSE_HOST=https://us.cloud.langfuse.com` (or the EU host) and use the keys
+from your cloud project.
+
+## What to look at once it's open
+
+After `demo_02.py` runs, go to **Tracing → Traces** and filter by the `north-ibuyer` tag.
+Open the trace and read the span tree:
+
+- **Spans** = tool calls (`get_demand_score`, `get_supply_snapshot`) — click one to see its
+  input arguments and the row it returned.
+- **Generations** = LLM calls — these also show the model, token counts, cost, and latency.
+  The final generation is where the model turned the tool results into its recommendation.
+
+Top to bottom, the tree is the reasoning path: what the agent fetched, in what order, and
+how it decided.

module-09-tracing/demos/config.yaml

@@ -0,0 +1,24 @@
+agent:
+  name: RealThor
+  version: "0.9.0"
+
+model:
+  name: gpt-4o-mini
+  # Non-zero: the recommendation can vary run to run — the non-determinism that makes a
+  # single run worth tracing rather than just re-running.
+  temperature: 0.7
+
+debug:
+  state_db: state.db
+
+tracing:
+  enabled: true
+
+logging:
+  log_dir: logs
+
+paths:
+  prompt: prompts/system_prompt_v9.md
+  demand_data: data/demand_signals.csv
+  supply_data: data/supply_snapshot.csv
+  velocity_data: data/listing_velocity.csv

module-09-tracing/demos/data/demand_signals.csv

@@ -0,0 +1,21 @@
+region,property_type,bedrooms,search_volume,lead_count,recent_sales,demand_score
+west,apartment,1,890,64,24,0.77
+west,apartment,2,1240,86,34,0.91
+west,house,3,520,38,14,0.63
+west,studio,1,720,51,19,0.69
+north,apartment,1,620,31,12,0.54
+north,apartment,2,940,71,26,0.80
+north,house,2,310,22,8,0.41
+north,house,3,280,19,6,0.37
+south,apartment,2,780,58,19,0.72
+south,house,3,710,42,18,0.68
+south,house,4,390,27,9,0.49
+south,studio,1,810,58,22,0.78
+east,apartment,1,650,47,17,0.66
+east,apartment,2,980,73,21,0.82
+east,studio,1,570,35,13,0.59
+east,house,3,460,33,11,0.55
+center,studio,1,430,19,8,0.47
+center,apartment,1,680,45,16,0.65
+center,apartment,2,510,37,12,0.56
+center,house,2,490,33,11,0.48

module-09-tracing/demos/data/listing_velocity.csv

@@ -0,0 +1,21 @@
+region,property_type,bedrooms,avg_days_to_offer,avg_days_to_close,velocity_score,data_confidence
+west,apartment,1,8,21,0.82,high
+west,apartment,2,5,18,0.91,high
+west,house,3,19,34,0.63,medium
+west,studio,1,11,24,0.71,medium
+north,apartment,1,22,41,0.52,medium
+north,apartment,2,12,29,0.78,high
+north,house,2,38,58,0.39,low
+north,house,3,44,63,0.34,low
+south,apartment,2,16,32,0.68,medium
+south,house,3,18,35,0.65,medium
+south,house,4,29,47,0.48,medium
+south,studio,1,9,22,0.8,high
+east,apartment,1,21,38,0.62,medium
+east,apartment,2,13,27,0.77,high
+east,studio,1,31,51,0.55,low
+east,house,3,24,42,0.58,medium
+center,studio,1,44,64,0.43,low
+center,apartment,1,26,45,0.61,medium
+center,apartment,2,33,53,0.52,medium
+center,house,2,37,57,0.46,low

module-09-tracing/demos/data/supply_snapshot.csv

@@ -0,0 +1,21 @@
+region,property_type,bedrooms,active_listings,avg_days_on_market,supply_score
+west,apartment,1,42,31,0.44
+west,apartment,2,18,21,0.22
+west,house,3,67,48,0.72
+west,studio,1,48,36,0.52
+north,apartment,1,77,54,0.81
+north,apartment,2,24,28,0.31
+north,house,2,88,62,0.88
+north,house,3,94,68,0.91
+south,apartment,2,39,35,0.48
+south,house,3,36,39,0.55
+south,house,4,52,44,0.62
+south,studio,1,22,19,0.27
+east,apartment,1,53,43,0.51
+east,apartment,2,29,28,0.37
+east,studio,1,83,57,0.85
+east,house,3,44,41,0.58
+center,studio,1,91,63,0.88
+center,apartment,1,58,49,0.68
+center,apartment,2,71,52,0.76
+center,house,2,64,51,0.67

module-09-tracing/demos/demo_01.py

@@ -0,0 +1,88 @@
+"""Demo 1 — Logs and state aren't enough.
+
+RealThor already logs every run and can replay one from persisted state (M07-M08). Run
+the North iBuyer question, then look at what those two artifacts actually show: the log
+line says what happened and how it ended; the replay restores the tool trajectory and the
+values each tool returned. Neither records *why* the model weighted margin over demand —
+the reasoning path between the tool results and the decision. That gap is what tracing
+closes in demo_02.
+
+    uv run python demo_01.py --case all
+"""
+
+import json
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+import click
+from dotenv import load_dotenv
+from langchain_core.messages import HumanMessage, ToolMessage
+
+from realthor.agents import build_agent
+from realthor.config import load_config, resolve_paths
+from realthor.replay import build_checkpointer, format_report, reconstruct_context
+
+load_dotenv()
+
+QUESTION = "Should we acquire through iBuyer in north for 3-bedroom houses?"
+
+
+def _log(cfg, run_id, result):
+    """Write the kind of structured log line the monitoring system records per run."""
+    tools = [m.name for m in result["messages"] if isinstance(m, ToolMessage) and m.name]
+    log_dir = Path(cfg["logging"]["log_dir"])
+    log_dir.mkdir(parents=True, exist_ok=True)
+    row = {
+        "run_id": run_id,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "input_text": QUESTION,
+        "success": True,
+        "tools_called": tools,
+        "prompt_version": Path(cfg["paths"]["prompt"]).name,
+        "dataset_version": cfg["agent"]["version"],
+    }
+    with open(log_dir / "runs.jsonl", "a") as f:
+        f.write(json.dumps(row) + "\n")
+    return row
+
+
+@click.command()
+@click.option("--case", default="all", type=click.Choice(["log", "replay", "all"]))
+def main(case):
+    cfg = resolve_paths(load_config("config.yaml"), "config.yaml")
+    log_path = Path(cfg["logging"]["log_dir"]) / "runs.jsonl"
+    state_path = Path("last_run_id.txt")
+
+    if case in ("log", "all"):
+        run_id = str(uuid.uuid4())
+        agent = build_agent(cfg, checkpointer=build_checkpointer(cfg))
+        result = agent.invoke(
+            {"messages": [HumanMessage(QUESTION)]},
+            config={"configurable": {"thread_id": run_id}},
+        )
+        row = _log(cfg, run_id, result)
+        state_path.write_text(run_id)
+        click.echo("=== The log line (logs/runs.jsonl) ===")
+        click.echo(json.dumps(row, indent=2))
+        click.echo("\nThe log says a run happened, which tools it called, and that it succeeded.")
+
+    if case in ("replay", "all"):
+        run_id = state_path.read_text().strip() if state_path.exists() else None
+        if not run_id:
+            click.echo("No run to replay — run `--case log` first.")
+            return
+        click.echo("\n=== The replay (reconstructed from state.db) ===")
+        ctx = reconstruct_context(run_id, build_checkpointer(cfg), log_path)
+        click.echo(format_report(ctx))
+
+    if case == "all":
+        click.echo("\n--- The gap ---")
+        click.echo("The log shows what happened. The replay restores the state and the values.")
+        click.echo("Neither shows HOW the model weighed those values into the recommendation —")
+        click.echo("the reasoning path between the tool results and the decision. That is what a")
+        click.echo("trace records.")
+
+
+if __name__ == "__main__":
+    main()

module-09-tracing/demos/demo_02.py

@@ -0,0 +1,65 @@
+"""Demo 2 — The trace.
+
+Same North iBuyer question, now with a Langfuse CallbackHandler attached to the invoke.
+The handler records a span tree — one span per model call and tool call — and ships it to
+Langfuse. Open the run in the UI (http://localhost:3000 for the local stack) and the
+reasoning path is finally visible: the sequential tool spans get_demand_score →
+get_supply_snapshot → recommend_channel, each with the values it returned, showing the low
+demand score was fetched but not weighted in the decision.
+
+Requires a Langfuse backend and LANGFUSE_* keys in .env (see LANGFUSE.md).
+
+    uv run python demo_02.py --case traced-run
+"""
+
+import uuid
+
+import click
+from dotenv import load_dotenv
+from langchain_core.messages import HumanMessage
+
+from realthor.agents import build_agent
+from realthor.config import load_config, resolve_paths
+from realthor.replay import build_checkpointer
+from realthor.tracing import build_tracer, flush_tracer
+
+load_dotenv()
+
+QUESTION = "Should we acquire through iBuyer in north for 3-bedroom houses?"
+
+
+@click.command()
+@click.option("--case", default="traced-run", type=click.Choice(["traced-run"]))
+def main(case):
+    cfg = resolve_paths(load_config("config.yaml"), "config.yaml")
+
+    tracer = build_tracer(cfg)
+    if tracer is None:
+        click.echo("Tracing is off or unconfigured.")
+        click.echo("Set tracing.enabled: true and put LANGFUSE_* keys in .env (see LANGFUSE.md).")
+        return
+
+    run_id = str(uuid.uuid4())
+    agent = build_agent(cfg, checkpointer=build_checkpointer(cfg))
+    try:
+        result = agent.invoke(
+            {"messages": [HumanMessage(QUESTION)]},
+            config={
+                "configurable": {"thread_id": run_id},
+                "callbacks": [tracer],
+                # Tags make this run easy to find and filter in the Langfuse UI.
+                "metadata": {"langfuse_tags": ["demo", "north-ibuyer"]},
+            },
+        )
+        click.echo("=== Traced run ===")
+        click.echo(result["messages"][-1].content)
+    finally:
+        # Flush before the process exits, or the batched spans never reach Langfuse.
+        flush_tracer(tracer)
+
+    click.echo(f"\nRun ID: {run_id}")
+    click.echo("Trace sent to Langfuse — open http://localhost:3000 and inspect the span tree.")
+
+
+if __name__ == "__main__":
+    main()