Update Logstash single pipeline dashboard with batch byte_size metrics for p50 p90 max (#18311)

Update the cel script to store p50 p90 and max byte size metrics of the batch for the last minute.
Switch to use the average last one minute metric instead of the lifetime for both batch's byte size and value to be coherent with the other graphed percentiles.
Update the Single Pipeline View to include these new values.

Author: andsel

packages/logstash/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.11.0"
+  changes:
+    - description: Update existing batch graphs with 50th and 90th percentiles during last minute. Replace average lifetime metric with last one minute, for both batch's byte size and value, to be coherent with the other graphed percentiles.. Note that these metrics will only show results for Logstash versions 9.4.0 or later.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/17785
 - version: "2.10.1"
   changes:
     - description: Fixes navigation of batch metrics when the integration is used to monitor an older version of Logstash that doesn't expose.

packages/logstash/data_stream/pipeline/agent/stream/cel.yml.hbs

@@ -63,11 +63,16 @@ program: |
         "batch":{
           "event_count":{
             "current": body.pipelines[pipeline_name].?batch.event_count.current.orValue([]),
-            "average.lifetime": body.pipelines[pipeline_name].?batch.event_count.average.lifetime.orValue([])
+            "average.last_1_minute": body.pipelines[pipeline_name].?batch.event_count.average.last_1_minute.orValue([]),
+            "p50.last_1_minute": body.pipelines[pipeline_name].?batch.event_count.p50.last_1_minute.orValue([]),
+            "p90.last_1_minute": body.pipelines[pipeline_name].?batch.event_count.p90.last_1_minute.orValue([]),
           },
           "byte_size":{
             "current": body.pipelines[pipeline_name].?batch.byte_size.current.orValue([]),
-            "average.lifetime": body.pipelines[pipeline_name].?batch.byte_size.average.lifetime.orValue([]),
+            "average.last_1_minute": body.pipelines[pipeline_name].?batch.byte_size.average.last_1_minute.orValue([]),
+            "p50.last_1_minute": body.pipelines[pipeline_name].?batch.byte_size.p50.last_1_minute.orValue([]),
+            "p90.last_1_minute": body.pipelines[pipeline_name].?batch.byte_size.p90.last_1_minute.orValue([]),
+            "max.last_1_minute": body.pipelines[pipeline_name].?batch.byte_size.max.last_1_minute.orValue([]),
           },
         },
         "events":{

packages/logstash/data_stream/pipeline/fields/fields.yml

@@ -49,19 +49,42 @@
           type: long
           description: Number of events contained in batch by the pipeline
           metric_type: gauge
-        - name: batch.event_count.average.lifetime
+        - name: batch.event_count.average.last_1_minute
           type: long
-          description: Number of average events contained in batch by the pipeline (Lifetime)
+          description: Number of average events contained in batch by the pipeline (last 1 minute)
+          metric_type: gauge  
+        - name: batch.event_count.p50.last_1_minute
+          type: long
+          description: 50th percentile of events per batch for the pipeline (last 1 minute)
+          metric_type: gauge
+        - name: batch.event_count.p90.last_1_minute
+          type: long
+          description: 90th percentile of events per batch for the pipeline (last 1 minute)
           metric_type: gauge
         - name: batch.byte_size.current
           type: long
           unit: byte
           description: Size in bytes of batch by the pipeline
           metric_type: gauge
-        - name: batch.byte_size.average.lifetime
+        - name: batch.byte_size.average.last_1_minute
+          type: long
+          unit: byte
+          description: Average size in bytes of batch by the pipeline (last 1 minute)
+          metric_type: gauge
+        - name: batch.byte_size.p50.last_1_minute
+          type: long
+          unit: byte
+          description: 50th percentile of batch size in bytes for the pipeline (last 1 minute)
+          metric_type: gauge
+        - name: batch.byte_size.p90.last_1_minute
+          type: long
+          unit: byte
+          description: 90th percentile of batch size in bytes for the pipeline (last 1 minute)
+          metric_type: gauge
+        - name: batch.byte_size.max.last_1_minute
           type: long
           unit: byte
-          description: Average size in bytes of batch by the pipeline (Lifetime)
+          description: Maximum batch size in bytes for the pipeline (last 1 minute)
           metric_type: gauge
         - name: events.filtered
           type: long

packages/logstash/docs/README.md

@@ -461,10 +461,15 @@ This is the `pipeline` dataset, which drives the Pipeline dashboard pages.
 | logstash.pipeline.info.ephemeral_id | Ephemeral Id for the running pipeline | keyword |  |  |
 | logstash.pipeline.info.workers | Number of workers for the running pipeline | long |  | gauge |
 | logstash.pipeline.name | Logstash Pipeline id/name | keyword |  |  |
-| logstash.pipeline.total.batch.byte_size.average.lifetime | Average size in bytes of batch by the pipeline (Lifetime) | long | byte | gauge |
+| logstash.pipeline.total.batch.byte_size.average.last_1_minute | Average size in bytes of batch by the pipeline (last 1 minute) | long | byte | gauge |
 | logstash.pipeline.total.batch.byte_size.current | Size in bytes of batch by the pipeline | long | byte | gauge |
-| logstash.pipeline.total.batch.event_count.average.lifetime | Number of average events contained in batch by the pipeline (Lifetime) | long |  | gauge |
+| logstash.pipeline.total.batch.byte_size.max.last_1_minute | Maximum batch size in bytes for the pipeline (last 1 minute) | long | byte | gauge |
+| logstash.pipeline.total.batch.byte_size.p50.last_1_minute | 50th percentile of batch size in bytes for the pipeline (last 1 minute) | long | byte | gauge |
+| logstash.pipeline.total.batch.byte_size.p90.last_1_minute | 90th percentile of batch size in bytes for the pipeline (last 1 minute) | long | byte | gauge |
+| logstash.pipeline.total.batch.event_count.average.last_1_minute | Number of average events contained in batch by the pipeline (last 1 minute) | long |  | gauge |
 | logstash.pipeline.total.batch.event_count.current | Number of events contained in batch by the pipeline | long |  | gauge |
+| logstash.pipeline.total.batch.event_count.p50.last_1_minute | 50th percentile of events per batch for the pipeline (last 1 minute) | long |  | gauge |
+| logstash.pipeline.total.batch.event_count.p90.last_1_minute | 90th percentile of events per batch for the pipeline (last 1 minute) | long |  | gauge |
 | logstash.pipeline.total.events.filtered | Number of events filtered by the pipeline | long |  | counter |
 | logstash.pipeline.total.events.in | Number of events received by the pipeline | long |  | counter |
 | logstash.pipeline.total.events.out | Number of events emitted by the pipeline | long |  | counter |