[Agent Builder] improve agent builder telemetry to report daily usages, correct total… (elastic#255665)

Author: ppisljar

x-pack/platform/plugins/private/telemetry_collection_xpack/schema/xpack_platform.json

@@ -362,19 +362,19 @@
             "total": {
               "type": "long",
               "_meta": {
-                "description": "Total number of conversations"
+                "description": "Total number of conversations (all-time)"
               }
             },
             "total_rounds": {
               "type": "long",
               "_meta": {
-                "description": "Total conversation rounds across all conversations"
+                "description": "Total conversation rounds across all conversations (all-time)"
               }
             },
             "avg_rounds_per_conversation": {
               "type": "float",
               "_meta": {
-                "description": "Average rounds per conversation"
+                "description": "Average rounds per conversation (all-time)"
               }
             },
             "rounds_distribution": {
@@ -399,13 +399,90 @@
             "tokens_used": {
               "type": "long",
               "_meta": {
-                "description": "Total tokens used across all conversations (input + output)"
+                "description": "Total tokens used (input + output) (all-time)"
+              }
+            },
+            "tokens_input": {
+              "type": "long",
+              "_meta": {
+                "description": "Total input tokens across all conversations (all-time)"
+              }
+            },
+            "tokens_output": {
+              "type": "long",
+              "_meta": {
+                "description": "Total output tokens across all conversations (all-time)"
+              }
+            },
+            "average_tokens_per_conversation": {
+              "type": "float",
+              "_meta": {
+                "description": "Average tokens per conversation (all-time)"
+              }
+            }
+          }
+        },
+        "daily": {
+          "properties": {
+            "total": {
+              "type": "long",
+              "_meta": {
+                "description": "Total number of conversations (daily, last 24h)"
+              }
+            },
+            "total_rounds": {
+              "type": "long",
+              "_meta": {
+                "description": "Total conversation rounds (daily, last 24h)"
+              }
+            },
+            "avg_rounds_per_conversation": {
+              "type": "float",
+              "_meta": {
+                "description": "Average rounds per conversation (daily, last 24h)"
+              }
+            },
+            "rounds_distribution": {
+              "type": "array",
+              "items": {
+                "properties": {
+                  "bucket": {
+                    "type": "keyword",
+                    "_meta": {
+                      "description": "Round count bucket (1-5, 6-10, 11-20, 21-50, 51+)"
+                    }
+                  },
+                  "count": {
+                    "type": "long",
+                    "_meta": {
+                      "description": "Number of conversations in this bucket"
+                    }
+                  }
+                }
+              }
+            },
+            "tokens_used": {
+              "type": "long",
+              "_meta": {
+                "description": "Total tokens used (input + output) (daily, last 24h)"
+              }
+            },
+            "tokens_input": {
+              "type": "long",
+              "_meta": {
+                "description": "Total input tokens (daily, last 24h)"
+              }
+            },
+            "tokens_output": {
+              "type": "long",
+              "_meta": {
+                "description": "Total output tokens (daily, last 24h)"
               }
             },
             "average_tokens_per_conversation": {
               "type": "float",
               "_meta": {
-                "description": "Average tokens per conversation"
+                "description": "Average tokens per conversation (daily, last 24h)"
               }
             }
           }

x-pack/platform/plugins/shared/agent_builder/server/telemetry/agent_builder_telemetry.md

@@ -37,28 +37,43 @@ pipeline. They include:
   - How we count: ES `count` on the agents system index.
 
 ### conversations
+
+All-time conversation metrics (no date filter).
+
 - `conversations.total`
   - What it is: Total number of conversations.
-  - How we count: ES `hits.total` on the conversations index.
+  - How we count: ES `hits.total` (with `track_total_hits: true`) on the conversations index.
 - `conversations.rounds_distribution[]`
   - What it is: Distribution of conversations by number of rounds.
   - How we count: ES scripted terms aggregation using `conversation_rounds` (or `rounds` fallback)
     and bucket labels `1-5`, `6-10`, `11-20`, `21-50`, `51+`.
 - `conversations.total_rounds`
-  - What it is: Approximate total number of rounds across all conversations.
-  - How we count: Derived from `rounds_distribution` buckets by multiplying each bucket count by
-    a midpoint (3, 8, 15, 35, 75) and summing.
+  - What it is: Exact total number of rounds across all conversations.
+  - How we count: ES scripted sum aggregation that counts the actual size of each conversation's
+    `conversation_rounds` (or `rounds`) array.
 - `conversations.avg_rounds_per_conversation`
   - What it is: Average rounds per conversation.
   - How we count: `total_rounds / total`, rounded to two decimals.
 - `conversations.tokens_used`
-  - What it is: Total tokens used across all conversation rounds.
-  - How we count: ES scripted sum aggregation over `conversation_rounds[*].model_usage`
-    (`input_tokens + output_tokens`).
+  - What it is: Total tokens used across all conversation rounds (input + output).
+  - How we count: Sum of `tokens_input` and `tokens_output`.
+- `conversations.tokens_input`
+  - What it is: Total input tokens across all conversation rounds.
+  - How we count: ES scripted sum aggregation over `conversation_rounds[*].model_usage.input_tokens`.
+- `conversations.tokens_output`
+  - What it is: Total output tokens across all conversation rounds.
+  - How we count: ES scripted sum aggregation over `conversation_rounds[*].model_usage.output_tokens`.
 - `conversations.average_tokens_per_conversation`
   - What it is: Average tokens per conversation.
   - How we count: `tokens_used / total`, rounded to two decimals.
 
+### daily
+
+Same structure as `conversations` above, but filtered to the last 24 hours using a
+`created_at >= now-24h` range filter. All fields (`total`, `total_rounds`, `tokens_used`,
+`tokens_input`, `tokens_output`, `rounds_distribution`, etc.) are computed identically but
+only over conversations created in the last day.
+
 ### tokens_by_model[]
 - `tokens_by_model[].model`
   - What it is: LLM model identifier.

x-pack/platform/plugins/shared/agent_builder/server/telemetry/query_utils.test.ts

@@ -439,33 +439,39 @@ describe('query_utils', () => {
     });
 
     describe('getConversationMetrics', () => {
-      it('returns conversation metrics with aggregation data', async () => {
-        esClient.search.mockResolvedValue({
-          took: 1,
-          timed_out: false,
-          _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
-          hits: { total: { value: 100, relation: 'eq' }, hits: [] },
-          aggregations: {
-            rounds_distribution: {
-              buckets: [
-                { key: '1-5', doc_count: 60 },
-                { key: '6-10', doc_count: 25 },
-                { key: '11-20', doc_count: 10 },
-                { key: '21-50', doc_count: 4 },
-                { key: '51+', doc_count: 1 },
-              ],
-            },
-            total_tokens: {
-              value: 50000,
-            },
-          },
-        });
+      const mockConversationResponse = ({
+        totalHits = 100,
+        roundsBuckets = [
+          { key: '1-5', doc_count: 60 },
+          { key: '6-10', doc_count: 25 },
+          { key: '11-20', doc_count: 10 },
+          { key: '21-50', doc_count: 4 },
+          { key: '51+', doc_count: 1 },
+        ],
+        totalRounds = 745,
+        inputTokens = 30000,
+        outputTokens = 20000,
+      } = {}) => ({
+        took: 1,
+        timed_out: false,
+        _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
+        hits: { total: { value: totalHits, relation: 'eq' as const }, hits: [] },
+        aggregations: {
+          rounds_distribution: { buckets: roundsBuckets },
+          total_rounds: { value: totalRounds },
+          total_input_tokens: { value: inputTokens },
+          total_output_tokens: { value: outputTokens },
+        },
+      });
+
+      it('returns conversation metrics with real round counts', async () => {
+        esClient.search.mockResolvedValue(mockConversationResponse());
 
         const result = await queryUtils.getConversationMetrics();
 
         expect(result).toEqual({
           total: 100,
-          total_rounds: 60 * 3 + 25 * 8 + 10 * 15 + 4 * 35 + 1 * 75, // = 180 + 200 + 150 + 140 + 75 = 745
+          total_rounds: 745,
           avg_rounds_per_conversation: 7.45,
           rounds_distribution: [
             { bucket: '1-5', count: 60 },
@@ -475,47 +481,33 @@ describe('query_utils', () => {
             { bucket: '51+', count: 1 },
           ],
           tokens_used: 50000,
+          tokens_input: 30000,
+          tokens_output: 20000,
           average_tokens_per_conversation: 500,
         });
       });
 
       it('handles total as number (not object)', async () => {
         esClient.search.mockResolvedValue({
-          took: 1,
-          timed_out: false,
-          _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
+          ...mockConversationResponse({ totalHits: 50 }),
           hits: { total: 50, hits: [] },
-          aggregations: {
-            rounds_distribution: {
-              buckets: [{ key: '1-5', doc_count: 50 }],
-            },
-            total_tokens: {
-              value: 10000,
-            },
-          },
         });
 
         const result = await queryUtils.getConversationMetrics();
 
         expect(result.total).toBe(50);
-        expect(result.average_tokens_per_conversation).toBe(200);
       });
 
       it('returns 0 for average when no conversations', async () => {
-        esClient.search.mockResolvedValue({
-          took: 1,
-          timed_out: false,
-          _shards: { total: 1, successful: 1, skipped: 0, failed: 0 },
-          hits: { total: { value: 0, relation: 'eq' }, hits: [] },
-          aggregations: {
-            rounds_distribution: {
-              buckets: [],
-            },
-            total_tokens: {
-              value: 0,
-            },
-          },
-        });
+        esClient.search.mockResolvedValue(
+          mockConversationResponse({
+            totalHits: 0,
+            roundsBuckets: [],
+            totalRounds: 0,
+            inputTokens: 0,
+            outputTokens: 0,
+          })
+        );
 
         const result = await queryUtils.getConversationMetrics();
 
@@ -525,10 +517,47 @@ describe('query_utils', () => {
           avg_rounds_per_conversation: 0,
           rounds_distribution: [],
           tokens_used: 0,
+          tokens_input: 0,
+          tokens_output: 0,
           average_tokens_per_conversation: 0,
         });
       });
 
+      it('applies date filter when provided', async () => {
+        esClient.search.mockResolvedValue(
+          mockConversationResponse({
+            totalHits: 10,
+            totalRounds: 25,
+            inputTokens: 1000,
+            outputTokens: 500,
+          })
+        );
+
+        await queryUtils.getConversationMetrics({ gte: '2024-01-01T00:00:00.000Z' });
+
+        expect(esClient.search).toHaveBeenCalledWith(
+          expect.objectContaining({
+            query: {
+              bool: {
+                filter: [{ range: { created_at: { gte: '2024-01-01T00:00:00.000Z' } } }],
+              },
+            },
+          })
+        );
+      });
+
+      it('uses match_all when no date filter', async () => {
+        esClient.search.mockResolvedValue(mockConversationResponse());
+
+        await queryUtils.getConversationMetrics();
+
+        expect(esClient.search).toHaveBeenCalledWith(
+          expect.objectContaining({
+            query: { match_all: {} },
+          })
+        );
+      });
+
       it('does not log warning for index_not_found_exception', async () => {
         const error = {
           message: 'index_not_found_exception',
@@ -557,6 +586,8 @@ describe('query_utils', () => {
           avg_rounds_per_conversation: 0,
           rounds_distribution: [],
           tokens_used: 0,
+          tokens_input: 0,
+          tokens_output: 0,
           average_tokens_per_conversation: 0,
         });
         expect(logger.warn).toHaveBeenCalledWith('Failed to fetch conversation metrics: ES error');