Merge pull request #14 from agentevals-dev/feature/bug-report

Implement bug-report feature

Author: krisztianfekete

src/agentevals/api/app.py

@@ -11,6 +11,8 @@
 from fastapi.responses import StreamingResponse
 
 from .routes import router
+from .debug_routes import debug_router, set_trace_manager as set_debug_trace_manager
+from ..utils.log_buffer import log_buffer
 from agentevals import __version__
 
 try:
@@ -37,6 +39,7 @@
 )
 
 app.include_router(router, prefix="/api")
+app.include_router(debug_router, prefix="/api/debug")
 
 _live_mode = os.getenv("AGENTEVALS_LIVE") == "1"
 _trace_manager = None
@@ -49,6 +52,7 @@
     app.include_router(streaming_router, prefix="/api/streaming")
     _trace_manager = StreamingTraceManager()
     set_trace_manager(_trace_manager)
+    set_debug_trace_manager(_trace_manager)
 
     @app.websocket("/ws/traces")
     async def websocket_endpoint(websocket: WebSocket):
@@ -106,7 +110,10 @@ async def on_startup():
         format="%(levelname)s:%(name)s:%(message)s",
         force=True,
     )
-    logging.getLogger("agentevals").setLevel(log_level)
+    ae_logger = logging.getLogger("agentevals")
+    ae_logger.setLevel(log_level)
+    log_buffer.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
+    ae_logger.addHandler(log_buffer)
     if _trace_manager:
         _trace_manager.start_cleanup_task()
 

src/agentevals/api/debug_routes.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+
+import glob
+import importlib.metadata
+import io
+import json
+import logging
+import os
+import platform
+import sys
+import tempfile
+import zipfile
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter, UploadFile, File as FastAPIFile, HTTPException
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+from agentevals import __version__
+from ..utils.log_buffer import log_buffer
+
+if TYPE_CHECKING:
+    from ..streaming.ws_server import StreamingTraceManager
+
+logger = logging.getLogger(__name__)
+
+debug_router = APIRouter()
+
+_trace_manager: StreamingTraceManager | None = None
+
+
+def set_trace_manager(manager: StreamingTraceManager) -> None:
+    global _trace_manager
+    _trace_manager = manager
+
+
+class FrontendDiagnostics(BaseModel):
+    user_description: str = ""
+    browser_info: dict = {}
+    console_logs: list[dict] = []
+    app_state: dict = {}
+    network_errors: list[dict] = []
+
+
+def _get_package_version(name: str) -> str | None:
+    try:
+        return importlib.metadata.version(name)
+    except importlib.metadata.PackageNotFoundError:
+        return None
+
+
+def _collect_environment() -> dict:
+    packages = [
+        "fastapi", "uvicorn", "google-adk", "google-genai",
+        "opentelemetry-sdk", "opentelemetry-api", "pydantic",
+    ]
+    return {
+        "timestamp": datetime.now(tz=timezone.utc).isoformat(),
+        "agentevals_version": __version__,
+        "python_version": sys.version,
+        "os": platform.system(),
+        "os_version": platform.release(),
+        "machine": platform.machine(),
+        "packages": {
+            p: _get_package_version(p) for p in packages
+        },
+        "config": {
+            "log_level": os.getenv("AGENTEVALS_LOG_LEVEL", "INFO"),
+            "live_mode": os.getenv("AGENTEVALS_LIVE") == "1",
+        },
+        "api_keys": {
+            "google": bool(
+                os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")
+            ),
+            "anthropic": bool(os.getenv("ANTHROPIC_API_KEY")),
+            "openai": bool(os.getenv("OPENAI_API_KEY")),
+        },
+    }
+
+
+def _collect_sessions() -> list[dict]:
+    if not _trace_manager:
+        return []
+
+    sessions_data = []
+    for session in _trace_manager.sessions.values():
+        sessions_data.append(
+            {
+                "session_id": session.session_id,
+                "trace_id": session.trace_id,
+                "eval_set_id": session.eval_set_id,
+                "started_at": session.started_at.isoformat(),
+                "is_complete": session.is_complete,
+                "span_count": len(session.spans),
+                "log_count": len(session.logs),
+                "metadata": session.metadata,
+                "spans": session.spans,
+                "logs": session.logs,
+            }
+        )
+    return sessions_data
+
+
+def _collect_temp_files() -> dict[str, str]:
+    tmp_dir = tempfile.gettempdir()
+    files = {}
+    for pattern in ["agentevals_*.jsonl", "eval_set_*.json"]:
+        for path in glob.glob(os.path.join(tmp_dir, pattern)):
+            try:
+                with open(path) as f:
+                    files[os.path.basename(path)] = f.read()
+            except OSError:
+                logger.debug("Could not read temp file %s", path)
+    return files
+
+
+@debug_router.post("/bundle")
+async def create_debug_bundle(diagnostics: FrontendDiagnostics):
+    timestamp = datetime.now(tz=timezone.utc).strftime("%Y%m%d-%H%M%S")
+    prefix = f"bug-report-{timestamp}"
+
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+        env = _collect_environment()
+        metadata = {
+            **env,
+            "user_description": diagnostics.user_description,
+            "browser_info": diagnostics.browser_info,
+        }
+        zf.writestr(f"{prefix}/metadata.json", json.dumps(metadata, indent=2))
+
+        sessions = _collect_sessions()
+        for s in sessions:
+            sid = s["session_id"]
+            zf.writestr(
+                f"{prefix}/sessions/{sid}/spans.json",
+                json.dumps(s["spans"], indent=2),
+            )
+            zf.writestr(
+                f"{prefix}/sessions/{sid}/logs.json",
+                json.dumps(s["logs"], indent=2),
+            )
+            session_meta = {
+                k: v
+                for k, v in s.items()
+                if k not in ("spans", "logs")
+            }
+            zf.writestr(
+                f"{prefix}/sessions/{sid}/session_meta.json",
+                json.dumps(session_meta, indent=2),
+            )
+
+        zf.writestr(f"{prefix}/backend_logs.txt", log_buffer.get_text())
+
+        temp_files = _collect_temp_files()
+        for filename, content in temp_files.items():
+            zf.writestr(f"{prefix}/temp_files/{filename}", content)
+
+        zf.writestr(
+            f"{prefix}/frontend_state.json",
+            json.dumps(diagnostics.app_state, indent=2),
+        )
+        zf.writestr(
+            f"{prefix}/console_logs.json",
+            json.dumps(diagnostics.console_logs, indent=2),
+        )
+        zf.writestr(
+            f"{prefix}/network_errors.json",
+            json.dumps(diagnostics.network_errors, indent=2),
+        )
+
+    buf.seek(0)
+    return StreamingResponse(
+        buf,
+        media_type="application/zip",
+        headers={
+            "Content-Disposition": f'attachment; filename="bug-report-{timestamp}.zip"'
+        },
+    )
+
+
+@debug_router.post("/load")
+async def load_debug_bundle(file: UploadFile = FastAPIFile(...)):
+    if not _trace_manager:
+        raise HTTPException(
+            status_code=400,
+            detail="Live mode is not enabled. Start with: agentevals serve --dev",
+        )
+
+    content = await file.read()
+    try:
+        zf = zipfile.ZipFile(io.BytesIO(content))
+    except zipfile.BadZipFile:
+        raise HTTPException(status_code=400, detail="Invalid ZIP file")
+
+    session_dirs: dict[str, list[str]] = {}
+    for name in zf.namelist():
+        parts = name.split("/")
+        if len(parts) >= 4 and parts[-3] == "sessions":
+            sid = parts[-2]
+            session_dirs.setdefault(sid, []).append(name)
+
+    if not session_dirs:
+        raise HTTPException(status_code=400, detail="No sessions found in ZIP")
+
+    from ..streaming.session import TraceSession
+
+    loaded = []
+    for sid, files in session_dirs.items():
+        meta_file = next((f for f in files if f.endswith("session_meta.json")), None)
+        spans_file = next((f for f in files if f.endswith("spans.json")), None)
+        logs_file = next((f for f in files if f.endswith("logs.json")), None)
+
+        if not spans_file:
+            continue
+
+        meta = json.loads(zf.read(meta_file)) if meta_file else {}
+        spans = json.loads(zf.read(spans_file))
+        logs = json.loads(zf.read(logs_file)) if logs_file else []
+
+        session = TraceSession(
+            session_id=meta.get("session_id", sid),
+            trace_id=meta.get("trace_id", sid),
+            eval_set_id=meta.get("eval_set_id"),
+            spans=spans,
+            logs=logs,
+            is_complete=True,
+            metadata=meta.get("metadata", {}),
+        )
+
+        _trace_manager.sessions[session.session_id] = session
+
+        await _trace_manager.broadcast_to_ui({
+            "type": "session_started",
+            "session": {
+                "sessionId": session.session_id,
+                "traceId": session.trace_id,
+                "evalSetId": session.eval_set_id,
+                "metadata": session.metadata,
+                "startedAt": session.started_at.isoformat(),
+            },
+        })
+
+        invocations_data = await _trace_manager._extract_invocations(session)
+        await _trace_manager._save_spans_to_temp_file(session)
+
+        await _trace_manager.broadcast_to_ui({
+            "type": "session_complete",
+            "sessionId": session.session_id,
+            "invocations": invocations_data,
+        })
+
+        loaded.append(session.session_id)
+        logger.info("Loaded session from bug report: %s", session.session_id)
+
+    return {"loaded_sessions": loaded, "count": len(loaded)}

src/agentevals/utils/log_buffer.py

@@ -0,0 +1,45 @@
+import logging
+from collections import deque
+from dataclasses import dataclass
+
+
+@dataclass
+class BufferedLogRecord:
+    timestamp: str
+    level: str
+    logger_name: str
+    message: str
+    exc_text: str | None = None
+
+
+class RingBufferLogHandler(logging.Handler):
+
+    def __init__(self, capacity: int = 1000):
+        super().__init__()
+        self._buffer: deque[BufferedLogRecord] = deque(maxlen=capacity)
+
+    def emit(self, record: logging.LogRecord) -> None:
+        from datetime import datetime, timezone
+
+        self._buffer.append(
+            BufferedLogRecord(
+                timestamp=datetime.fromtimestamp(
+                    record.created, tz=timezone.utc
+                ).isoformat(),
+                level=record.levelname,
+                logger_name=record.name,
+                message=self.format(record),
+                exc_text=record.exc_text,
+            )
+        )
+
+    def get_text(self) -> str:
+        lines = []
+        for r in self._buffer:
+            lines.append(f"[{r.timestamp}] {r.level} {r.logger_name}: {r.message}")
+            if r.exc_text:
+                lines.append(r.exc_text)
+        return "\n".join(lines)
+
+
+log_buffer = RingBufferLogHandler(capacity=1000)