Skip to content

Commit 73e959b

Browse files
committed
feat: agentic trace (#45)
* feat: agentic trace * feat: pixi demo-trace task * refactor: demo trace * chore: increase inference server setup timeout * fix: demo trace log handling * docs: agentic trace
1 parent febc2f4 commit 73e959b

7 files changed

Lines changed: 419 additions & 5 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ imgui.ini
2020
anomaly_images/
2121
package_condition_images/
2222
models/
23+
runs/
2324
*.bak0
2425
sim/user/
2526
backtrace.log

docs/setup_single_machine.md

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,65 @@ The NPU `vlm_safety` model (`gemma3:4b`) has no GGUF; `flm pull` downloads it fo
113113

114114
---
115115

116+
## Verify your installation
117+
118+
Once the build is done and the weights are downloaded, run one command to exercise
119+
the whole stack end to end:
120+
121+
```shell
122+
pixi run -e single-pc-gpu-and-npu demo-trace
123+
```
124+
125+
This brings up the full demo (sim, stack, inference, agents, HMI), populates the
126+
scene, sends one task to the orchestrator (ship one CPU), waits for it to finish,
127+
saves the agent trace, and shuts everything down. A successful run ends with:
128+
129+
```
130+
● Task complete (marker).
131+
132+
=== Trace saved ===
133+
runs/<timestamp>
134+
log.txt human-readable conversation (orchestrator + subagents)
135+
trace.jsonl one JSON record per event
136+
agents_pane.log raw agents tmux output
137+
manifest.txt task + timestamps
138+
```
139+
140+
Open `runs/<timestamp>/log.txt` to read the orchestrator and subagent conversation
141+
for the task.
142+
143+
Environment knobs:
144+
145+
- `TASK`: the task string sent to the orchestrator (default: ship one CPU)
146+
- `MAX_WAIT`: hard cap in seconds on task execution (default: 900)
147+
- `IDLE`: treat this many seconds of trace inactivity as done (default: 180)
148+
- `SKIP_SCENE=1`: skip scene population
149+
- `TRACE_DIR`: output directory (default: `runs/<timestamp>`)
150+
151+
On a GPU-only box, use `-e single-pc-gpu` and set
152+
`[endpoints.vlm_safety] backend = "gpu"` in `config.toml` first (see above).
153+
154+
The conversation trace is written on any agent run (default-on, to `runs/<timestamp>/`);
155+
`demo-trace` just automates a single task plus teardown. Set `AMM_TRACE=0` to
156+
disable it.
157+
158+
### Richer traces with Langfuse (optional)
159+
160+
The orchestrator is already instrumented with a Langfuse callback. Point it at a
161+
Langfuse instance (self-hosted or cloud) to get a full browsable trace, including
162+
nested subagent spans and token usage, alongside the local `runs/` files:
163+
164+
```shell
165+
export LANGFUSE_PUBLIC_KEY=pk-...
166+
export LANGFUSE_SECRET_KEY=sk-...
167+
export LANGFUSE_HOST=http://localhost:3000 # your Langfuse server
168+
```
169+
170+
Without these keys the callback stays inactive and only the local `runs/` trace is
171+
written.
172+
173+
---
174+
116175
## Developer Setup
117176

118177
### Conventional Commits

pixi.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,10 @@ description = "Clone all repositories (gems and ROS 2 workspace)"
299299
cmd = "bash scripts/demo.sh"
300300
description = "Start full demo: sim → stack → inference → agents+orchestrator → hmi (one tmux session each)"
301301

302+
[feature.profile-single-pc.tasks.demo-trace]
303+
cmd = "bash scripts/demo_trace.sh"
304+
description = "Unattended trace run: start demo, send one task, dump the agent conversation to runs/<timestamp>/, then kill everything"
305+
302306
# ─── Single-PC setup variants: GPU (default) vs GPU+NPU ──────────────────────────
303307
# Share clone/demo above; differ only in what `setup` builds and what
304308
# find-runnables waits on. -gpu builds llama.cpp (Vulkan); -npu also builds

rai_app/agents/agent_orchestrator.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
from rai_app.agents.callbacks import (
4545
AgentActionsCallback,
4646
AgentProgressCallback,
47+
ConversationFileCallback,
4748
OrchestratorTasksNotifier,
4849
)
4950
from rai_app.agents.context_providers import WarehouseContext
@@ -596,6 +597,13 @@ def main():
596597
langfuse_handler = CallbackHandler()
597598

598599
ros2_callback = AgentProgressCallback(connector)
600+
callbacks: List[BaseCallbackHandler] = [langfuse_handler, ros2_callback]
601+
# Full conversation trace (orchestrator + subagents) to runs/<timestamp>/.
602+
# Default-on; set AMM_TRACE=0 to disable or AMM_TRACE_DIR to relocate.
603+
trace_callback = ConversationFileCallback.from_env()
604+
if trace_callback is not None:
605+
callbacks.append(trace_callback)
606+
599607
orchestrator = AgentOrchestrator(
600608
connector=connector,
601609
agent=agent,
@@ -604,7 +612,7 @@ def main():
604612
action_topic="/agent/current_action",
605613
initial_state_creator=get_initial_megamind_state,
606614
recursion_limit=100,
607-
langchain_callbacks=[langfuse_handler, ros2_callback],
615+
langchain_callbacks=callbacks,
608616
)
609617
asyncio.run(orchestrator.orchestrator_loop())
610618

rai_app/agents/callbacks.py

Lines changed: 199 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,16 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16+
import functools
1617
import json
1718
import logging
19+
import os
1820
import time
19-
from typing import Any, Dict, List
21+
from datetime import datetime
22+
from pathlib import Path
23+
from typing import Any, Dict, List, Optional
2024

21-
from langchain_core.callbacks import AsyncCallbackHandler
25+
from langchain_core.callbacks import AsyncCallbackHandler, BaseCallbackHandler
2226
from rai.communication.ros2 import ROS2Connector, ROS2HRIMessage, ROS2Message
2327

2428

@@ -220,3 +224,196 @@ def send_heartbeat(self):
220224
msg_type="std_msgs/msg/Header",
221225
target=self.heartbeat_topic,
222226
)
227+
228+
229+
def _stringify_content(content: Any) -> str:
230+
"""Flatten a message's content to text. Multimodal blocks (VLM image parts)
231+
are summarized as ``[image]`` so the log stays readable and small."""
232+
if content is None:
233+
return ""
234+
if isinstance(content, str):
235+
return content
236+
if isinstance(content, list):
237+
parts = []
238+
for block in content:
239+
if isinstance(block, dict):
240+
btype = block.get("type")
241+
if btype == "text":
242+
parts.append(block.get("text", ""))
243+
elif btype in ("image_url", "image"):
244+
parts.append("[image]")
245+
else:
246+
parts.append(f"[{btype or 'block'}]")
247+
else:
248+
parts.append(str(block))
249+
return "\n".join(p for p in parts if p)
250+
return str(content)
251+
252+
253+
def _guard(fn):
254+
"""Never let a tracing error break the agent run."""
255+
256+
@functools.wraps(fn)
257+
def wrapper(self, *args, **kwargs):
258+
try:
259+
return fn(self, *args, **kwargs)
260+
except Exception as exc: # noqa: BLE001 - tracing must not crash the agent
261+
self.logger.warning(f"trace callback {fn.__name__} failed: {exc}")
262+
263+
return wrapper
264+
265+
266+
class ConversationFileCallback(BaseCallbackHandler):
267+
"""Dump the full LLM conversation (orchestrator + subagents) to a run directory.
268+
269+
Register it in the orchestrator's ``langchain_callbacks`` list. That list is
270+
passed to ``astream(config={"callbacks": ...}, subgraphs=True)``, and LangGraph
271+
propagates config callbacks into every subgraph, so both the megamind
272+
orchestrator and its executor subagents are captured in one place.
273+
274+
Writes two files per run:
275+
- ``log.txt`` human-readable transcript (messages, tool calls, results)
276+
- ``trace.jsonl`` one JSON record per event (machine-readable)
277+
278+
Enabled by default to ``runs/<timestamp>/``. Override the directory with
279+
``AMM_TRACE_DIR``; disable entirely with ``AMM_TRACE=0``.
280+
"""
281+
282+
# Run inline on the event loop so concurrent subgraph events don't interleave
283+
# partial lines in the files.
284+
run_inline = True
285+
286+
def __init__(self, out_dir: str | Path):
287+
self.out_dir = Path(out_dir)
288+
self.out_dir.mkdir(parents=True, exist_ok=True)
289+
self.text_path = self.out_dir / "log.txt"
290+
self.jsonl_path = self.out_dir / "trace.jsonl"
291+
self.logger = logging.getLogger(__name__)
292+
self.logger.info(f"Conversation trace -> {self.out_dir}")
293+
294+
@classmethod
295+
def from_env(cls) -> Optional["ConversationFileCallback"]:
296+
"""Build from env, or return None if disabled (``AMM_TRACE=0``)."""
297+
if os.getenv("AMM_TRACE", "1") == "0":
298+
return None
299+
out_dir = os.getenv("AMM_TRACE_DIR") or os.path.join(
300+
"runs", datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
301+
)
302+
return cls(out_dir)
303+
304+
# ── writers ──────────────────────────────────────────────────────────────
305+
@staticmethod
306+
def _ts() -> str:
307+
return datetime.now().strftime("%H:%M:%S")
308+
309+
def _write_text(self, block: str) -> None:
310+
with open(self.text_path, "a", encoding="utf-8") as f:
311+
f.write(block.rstrip() + "\n")
312+
313+
def _write_json(self, record: Dict[str, Any]) -> None:
314+
record = {"ts": datetime.now().isoformat(timespec="seconds"), **record}
315+
with open(self.jsonl_path, "a", encoding="utf-8") as f:
316+
f.write(json.dumps(record, default=str) + "\n")
317+
318+
@staticmethod
319+
def _node(metadata: Any, tags: Any) -> str:
320+
"""Which graph node / subagent produced this event."""
321+
if isinstance(metadata, dict):
322+
node = metadata.get("langgraph_node")
323+
if node:
324+
return str(node)
325+
for tag in tags or []:
326+
if isinstance(tag, str) and ":" in tag:
327+
return tag
328+
return "?"
329+
330+
# ── callbacks ────────────────────────────────────────────────────────────
331+
@_guard
332+
def on_chat_model_start(
333+
self, serialized, messages, *, tags=None, metadata=None, **kwargs
334+
):
335+
node = self._node(metadata, tags)
336+
rendered = []
337+
for msg_list in messages:
338+
for m in msg_list:
339+
rendered.append(
340+
{
341+
"role": getattr(m, "type", m.__class__.__name__),
342+
"content": _stringify_content(getattr(m, "content", "")),
343+
}
344+
)
345+
lines = [f"[{self._ts()}] [{node}] >> LLM input ({len(rendered)} msgs)"]
346+
for r in rendered:
347+
lines.append(f" {r['role']}: {r['content']}")
348+
self._write_text("\n".join(lines))
349+
self._write_json({"event": "llm_start", "node": node, "messages": rendered})
350+
351+
@_guard
352+
def on_llm_end(self, response, *, tags=None, metadata=None, **kwargs):
353+
node = self._node(metadata, tags)
354+
try:
355+
gen = response.generations[0][0]
356+
except (AttributeError, IndexError):
357+
return
358+
message = getattr(gen, "message", None)
359+
content = _stringify_content(
360+
getattr(message, "content", getattr(gen, "text", ""))
361+
)
362+
tool_calls = getattr(message, "tool_calls", None) or []
363+
lines = [f"[{self._ts()}] [{node}] << LLM output"]
364+
if content:
365+
lines.append(f" {content}")
366+
for tc in tool_calls:
367+
args = json.dumps(tc.get("args", {}), default=str)
368+
lines.append(f" tool_call: {tc.get('name')}({args})")
369+
self._write_text("\n".join(lines))
370+
self._write_json(
371+
{
372+
"event": "llm_end",
373+
"node": node,
374+
"content": content,
375+
"tool_calls": tool_calls,
376+
}
377+
)
378+
379+
@_guard
380+
def on_tool_start(
381+
self, serialized, input_str, *, tags=None, metadata=None, **kwargs
382+
):
383+
name = (serialized or {}).get("name", "tool")
384+
node = self._node(metadata, tags)
385+
self._write_text(f"[{self._ts()}] [{node}] -> TOOL {name}({input_str})")
386+
self._write_json(
387+
{"event": "tool_start", "node": node, "tool": name, "input": input_str}
388+
)
389+
390+
@_guard
391+
def on_tool_end(self, output, *, tags=None, metadata=None, **kwargs):
392+
node = self._node(metadata, tags)
393+
text = _stringify_content(getattr(output, "content", output))
394+
self._write_text(f"[{self._ts()}] [{node}] <- TOOL result: {text}")
395+
self._write_json({"event": "tool_end", "node": node, "output": text})
396+
397+
@_guard
398+
def on_chain_start(self, serialized, inputs, *, parent_run_id=None, **kwargs):
399+
# Only the root run (no parent) marks a task boundary; inner nodes/subgraphs
400+
# start constantly and would be noise.
401+
if parent_run_id is None:
402+
self._write_text(f"[{self._ts()}] ===== TASK START =====")
403+
self._write_json({"event": "task_start"})
404+
405+
@_guard
406+
def on_chain_end(self, outputs, *, parent_run_id=None, **kwargs):
407+
if parent_run_id is None:
408+
self._write_text(f"[{self._ts()}] ===== TASK COMPLETE =====")
409+
self._write_json({"event": "task_end"})
410+
411+
@_guard
412+
def on_llm_error(self, error, **kwargs):
413+
self._write_text(f"[{self._ts()}] !! LLM ERROR: {error}")
414+
self._write_json({"event": "llm_error", "error": str(error)})
415+
416+
@_guard
417+
def on_tool_error(self, error, **kwargs):
418+
self._write_text(f"[{self._ts()}] !! TOOL ERROR: {error}")
419+
self._write_json({"event": "tool_error", "error": str(error)})

scripts/demo.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ wait_topic() {
4242
}
4343

4444
check_inference() {
45-
log "Waiting 10s for inference servers to initialize..."
46-
sleep 10
45+
log "Waiting 15s for inference servers to initialize..."
46+
sleep 15
4747
if bash "$DEMO_ROOT/scripts/smoke_test.sh"; then
4848
return 0
4949
else

0 commit comments

Comments
 (0)