Enhance llm-proxy with richer LLM observability

extensions/composer/llm-proxy/config.schema.json

@@ -34,6 +34,18 @@
       "type": "string",
       "description": "Request header to set with the extracted model name."
     },
+    "use_default_attributes": {
+      "type": "boolean",
+      "description": "Emit the full built-in structured log payload."
+    },
+    "use_default_response_attributes": {
+      "type": "boolean",
+      "description": "Emit the lightweight built-in structured log payload."
+    },
+    "session_id_header": {
+      "type": "string",
+      "description": "Optional request header used to extract a session or conversation ID."
+    },
     "clear_route_cache": {
       "type": "boolean",
       "description": "Clear Envoy route cache after setting metadata, enabling route re-selection. Defaults to false."

extensions/composer/llm-proxy/config_schema_test.go

@@ -20,6 +20,9 @@ func TestConfigSchema(t *testing.T) {
 			],
 			"metadata_namespace": "custom.ns",
 			"llm_model_header": "x-llm-model",
+			"use_default_attributes": true,
+			"use_default_response_attributes": false,
+			"session_id_header": "x-session-id",
 			"clear_route_cache": true
 		}`)
 	})
@@ -33,4 +36,9 @@ func TestConfigSchema(t *testing.T) {
 			]
 		}`)
 	})
+	t.Run("invalid wrong type", func(t *testing.T) {
+		internaltesting.AssertSchemaInvalid(t, "config.schema.json", `{
+			"use_default_attributes": "yes"
+		}`)
+	})
 }

extensions/composer/llm-proxy/llm.go

@@ -4,7 +4,8 @@
 // the root of the repo.
 
 // Package llmproxy implements an HTTP filter that identifies LLM API requests
-// and extracts model, stream, and token-usage information into filter metadata.
+// and extracts model, stream, token-usage, and richer observability attributes
+// into filter metadata.
 package llmproxy
 
 const (
@@ -33,20 +34,46 @@ type LLMRequest interface {
 	GetModel() string
 	// IsStream returns whether the request asks for a streaming (SSE) response.
 	IsStream() bool
+	// GetQuestion returns the user question extracted from the request if available.
+	GetQuestion() string
+	// GetSystem returns the system prompt extracted from the request if available.
+	GetSystem() string
 }
 
 // LLMResponse abstracts over different LLM API non-streaming response formats.
 type LLMResponse interface {
 	// GetUsage returns token-usage information extracted from the response body.
 	// The zero value of LLMUsage indicates that no usage data was present.
 	GetUsage() LLMUsage
+	// GetAnswer returns the textual assistant answer extracted from the response if available.
+	GetAnswer() string
+	// GetReasoning returns provider-specific reasoning content when available.
+	GetReasoning() string
+	// GetToolCalls returns any tool call payload emitted by the model.
+	GetToolCalls() any
+	// GetReasoningTokens returns the number of reasoning tokens when provided.
+	GetReasoningTokens() uint32
+	// GetCachedTokens returns cached input token counts when provided.
+	GetCachedTokens() uint32
+	// GetInputTokenDetails returns provider-specific prompt/input token detail fields.
+	GetInputTokenDetails() any
+	// GetOutputTokenDetails returns provider-specific completion/output token detail fields.
+	GetOutputTokenDetails() any
 }
 
 // LLMResponseChunk abstracts over a single event in an LLM streaming SSE response.
 type LLMResponseChunk interface {
 	// GetUsage returns token-usage information carried by this chunk.
 	// The zero value of LLMUsage indicates that the chunk carries no usage data.
 	GetUsage() LLMUsage
+	// GetAnswer returns any assistant text carried by this chunk.
+	GetAnswer() string
+	// GetReasoning returns any reasoning content carried by this chunk.
+	GetReasoning() string
+	// GetToolCalls returns any tool call delta carried by this chunk.
+	GetToolCalls() any
+	// HasTextToken reports whether the chunk contains a real text token.
+	HasTextToken() bool
 }
 
 // SSEParser incrementally consumes body chunks from an LLM streaming SSE response
@@ -59,6 +86,8 @@ type SSEParser interface {
 	// Finish finalises parsing and returns the accumulated LLMResponse and any
 	// terminal error encountered while processing the stream.
 	Finish() (LLMResponse, error)
+	// SeenTextToken reports whether the stream has emitted a real text token yet.
+	SeenTextToken() bool
 }
 
 // LLMFactory creates the per-API-type parsers for a specific LLM provider.

extensions/composer/llm-proxy/manifest.yaml

@@ -9,7 +9,7 @@ categories:
   - AI
 author: Tetrate
 featured: true
-description: Routes LLM API requests by the module name and monitors token usage and latency via Envoy metadata and metrics.
+description: Routes LLM API requests and emits richer observability metadata, metrics, and optional structured logs.
 longDescription: |
   An HTTP filter plugin that inspects incoming requests against a set of configured
   path-matcher rules to identify the LLM provider API in use (OpenAI Chat Completions,
@@ -18,9 +18,11 @@ longDescription: |
   1. Parses the request body to extract the model name and streaming flag, then
      writes them to Envoy's dynamic filter metadata.
   2. Parses the response body (JSON for non-streaming, SSE for streaming) to extract
-     token-usage information and writes it to filter metadata.
+     token-usage information and richer observability fields such as `question`,
+     `system`, `answer`, `reasoning`, and `tool_calls`.
   3. Records Envoy metrics (counters and histograms) for request counts, token usage,
      time-to-first-token (TTFT), and time-per-output-token (TPOT).
+  4. Optionally emits lightweight or full structured logs for downstream observability.
 
   Requests whose path does not match any rule are passed through without modification.
 
@@ -38,9 +40,20 @@ longDescription: |
   | `kind` | string | API kind: `"openai"`, `"anthropic"`, or `"custom"` |
   | `model` | string | Model name extracted from the request body |
   | `is_stream` | bool | Whether the request asks for a streaming (SSE) response |
+  | `response_type` | string | `"stream"` or `"nonstream"` |
+  | `session_id` | string | Session or conversation ID extracted from `session_id_header`, if configured |
+  | `question` | string | User question extracted from the request body, when available |
+  | `system` | string | System prompt extracted from the request body, when available |
+  | `answer` | string | Assistant response content, when available |
+  | `reasoning` | string | Provider-specific reasoning content, when available |
+  | `tool_calls` | array | Tool calls emitted by the model, when available |
   | `input_tokens` | uint32 | Input / prompt token count from the response |
   | `output_tokens` | uint32 | Output / completion token count from the response |
   | `total_tokens` | uint32 | Total token count from the response |
+  | `reasoning_tokens` | uint32 | Reasoning token count when provided by the upstream API |
+  | `cached_tokens` | uint32 | Cached input token count when provided by the upstream API |
+  | `input_token_details` | object | Provider-specific prompt/input token detail fields |
+  | `output_token_details` | object | Provider-specific completion/output token detail fields |
   | `request_ttft` | int64 | Time to first token in milliseconds |
   | `request_tpot` | int64 | Average time per output token in milliseconds |
 
@@ -67,6 +80,9 @@ longDescription: |
   | `llm_configs[].kind` | string | yes | — | `"openai"`, `"anthropic"`, or `"custom"` |
   | `metadata_namespace` | string | no | `io.builtonenvoy.llm-proxy` | Filter metadata namespace |
   | `llm_model_header` | string | no | `""` | If set, the extracted model name is written to this request header |
+  | `use_default_attributes` | bool | no | `false` | Emit the full built-in structured log payload |
+  | `use_default_response_attributes` | bool | no | `false` | Emit the lightweight built-in structured log payload |
+  | `session_id_header` | string | no | `""` | Optional request header used to extract a session or conversation ID |
   | `clear_route_cache` | bool | no | `false` | Clear the route cache after request parsing so Envoy can re-select the route based on updated metadata |
 
 type: go
@@ -130,3 +146,13 @@ examples:
           "llm_model_header": "x-llm-model",
           "clear_route_cache": true
         }'
+  - title: Full observability output
+    description: |
+      Emit richer metadata and a structured log entry including prompts, answers,
+      tool calls, and provider token-detail fields.
+    code: |
+      boe run --extension llm-proxy \
+        --config '{
+          "use_default_attributes": true,
+          "session_id_header": "x-session-id"
+        }'