Standard logging payload (#441)

* Add observability logging for LiteLLM calls

Signed-off-by: Trevor Grant <[email protected]>

* Add environment-controlled LiteLLM logging modes

Signed-off-by: Trevor Grant <[email protected]>

---------

Signed-off-by: Trevor Grant <[email protected]>

Author: rawkintrevo

webapp/packages/api/user-service/agent_factory/__init__.py

@@ -12,9 +12,12 @@
 
 from services.database_service import get_database_service
 from services.llm_service import call_llm
+from services.litellm_logger import ensure_litellm_logging
 from config import settings
 from config.provider_config import PROVIDER_CONFIG
 
+ensure_litellm_logging()
+
 async def generate_agent_code(request: GenerateCodeRequest):
     """
     Generates agent code based on the provided configuration.

webapp/packages/api/user-service/services/chat_service.py

@@ -7,6 +7,10 @@
 from litellm import acompletion, ModelResponse
 import litellm
 
+from services.litellm_logger import ensure_litellm_logging
+
+ensure_litellm_logging()
+
 # Configure litellm
 litellm.drop_params = True
 litellm.set_verbose = False

webapp/packages/api/user-service/services/litellm_logger.py

@@ -0,0 +1,79 @@
+import json
+import os
+from typing import Any
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger
+
+from services.observability_service import get_observability_service
+
+
+class ObservabilityLiteLLMLogger(CustomLogger):
+    """LiteLLM callback that forwards StandardLoggingPayload to observability."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.observability = get_observability_service()
+        self.logging_mode = _get_logging_mode()
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        await self._log_standard_payload(kwargs, level="INFO")
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        await self._log_standard_payload(kwargs, level="ERROR")
+
+    async def _log_standard_payload(self, kwargs: Any, level: str) -> None:
+        if self.logging_mode == "NONE":
+            return
+
+        payload = kwargs.get("standard_logging_object") if isinstance(kwargs, dict) else None
+        if payload is None:
+            return
+
+        try:
+            serialized_payload = json.loads(json.dumps(payload, default=str))
+        except Exception:
+            serialized_payload = {"error": "Failed to serialize StandardLoggingPayload"}
+
+        status = serialized_payload.get("status", "unknown") if serialized_payload else "unknown"
+
+        if self.logging_mode == "COST_ONLY":
+            serialized_payload = {
+                key: serialized_payload.get(key)
+                for key in ("response_cost", "cost_breakdown")
+                if serialized_payload.get(key) is not None
+            }
+        message = f"LiteLLM call completed with status: {status}"
+
+        self.observability.log(
+            level=level,
+            event_type="litellm_call",
+            message=message,
+            metadata={"standardLoggingPayload": serialized_payload},
+        )
+
+
+_configured_logger = False
+
+
+def _get_logging_mode() -> str:
+    mode = os.getenv("LITELLM_LOGGING_MODE", "NONE").upper()
+    return mode if mode in {"NONE", "ALL", "COST_ONLY"} else "NONE"
+
+
+def ensure_litellm_logging() -> None:
+    """Attach the observability logger to LiteLLM callbacks if not already set."""
+    global _configured_logger
+    if _configured_logger or _get_logging_mode() == "NONE":
+        return
+
+    callbacks = getattr(litellm, "callbacks", [])
+    if not any(isinstance(cb, ObservabilityLiteLLMLogger) for cb in callbacks):
+        callbacks.append(ObservabilityLiteLLMLogger())
+        litellm.callbacks = callbacks
+
+    _configured_logger = True
+
+
+# Ensure the logger is registered when the module is imported.
+ensure_litellm_logging()

webapp/packages/api/user-service/services/llm_service.py

@@ -1,8 +1,12 @@
-import litellm
 import asyncio
 import json
 from config.provider_config import PROVIDER_CONFIG
 from typing import Any, Dict, List, Tuple, Optional
+import litellm
+
+from services.litellm_logger import ensure_litellm_logging
+
+ensure_litellm_logging()
 
 async def call_llm(
     provider: str,