Add Serverless compatibility (elastic#15003)

spuchol81 · web-flow · commit e2d466b00cd4 · 2025-08-27T09:38:54.000+02:00
* Added a deployment_type parameter (values: deployment, elasticsearch, observability, security).
* Added dedicated dashboards for ECH and serverless consumption breakdown
diff --git a/packages/ess_billing/changelog.yml b/packages/ess_billing/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.5.0"
+  changes:
+    - description: Add Serverless compatibility
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15003
 - version: "1.4.4"
   changes:
     - description: Removed empty ess.billing.quantities field from billing data output.
diff --git a/packages/ess_billing/data_stream/billing/_dev/test/pipeline/test-billing.json b/packages/ess_billing/data_stream/billing/_dev/test/pipeline/test-billing.json
@@ -26,6 +26,7 @@
         "billing": {
           "deployment_id": "4679751e171d403d8603cb1639b67e57",
           "deployment_name": "example-deployment",
+          "deployment_type": "deployment",
           "display_quantity": {
             "formatted_value": "24 hours",
             "type": "default",
@@ -61,6 +62,69 @@
         "forwarded",
         "billing"
       ]
+    },
+    {
+      "@timestamp": "2025-08-20T14:08:02.403Z",
+      "agent": {
+        "ephemeral_id": "082a402e-e376-4e6a-8ae4-9fcd12471589",
+        "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+        "name": "elastic-agent-89987",
+        "type": "filebeat",
+        "version": "9.0.4"
+      },
+      "data_stream": {
+        "dataset": "ess_billing.billing",
+        "namespace": "68465",
+        "type": "metrics"
+      },
+      "ecs": {
+        "version": "8.0.0"
+      },
+      "elastic_agent": {
+        "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+        "snapshot": false,
+        "version": "9.0.4"
+      },
+      "ess": {
+        "billing": {
+          "deployment_id": "b7ffa6ecae534f81a2778ca1ce685254",
+          "deployment_name": "example-serverless",
+          "deployment_type": "observability",
+          "display_quantity": {
+            "value": 0.0055175,
+            "formatted_value": "0.0055 GBs",
+            "type": "default"
+          },
+          "from": "2025-08-19T00:00:00Z",
+          "kind": "",
+          "organization_id": "1111",
+          "quantities": [],
+          "quantity": { 
+            "value": 0.0055175,
+            "formatted_value": "0.0055 GBs"
+          },
+          "rate": { 
+            "value": 0.0,
+            "formatted_value": "0.0000 per GB" 
+          },
+          "name": "Serverless Data Out",
+          "total_ecu": 0.0,
+          "type": "data transfer",
+          "sku": "serverless.data-out",
+          "to": "2025-08-20T00:00:00Z",
+          "unit": "GB"
+        }
+      },
+      "event": {
+        "dataset": "ess_billing.billing"
+      },
+      "input": {
+        "type": "cel"
+      },
+      "tags": [
+        "forwarded",
+        "billing"
+      ]
     }
   ]
 }
diff --git a/packages/ess_billing/data_stream/billing/_dev/test/pipeline/test-billing.json-expected.json b/packages/ess_billing/data_stream/billing/_dev/test/pipeline/test-billing.json-expected.json
@@ -47,6 +47,7 @@
                 "billing": {
                     "deployment_id": "4679751e171d403d8603cb1639b67e57",
                     "deployment_name": "example-deployment",
+                    "deployment_type": "deployment",
                     "display_quantity": {
                         "formatted_value": "24 hours",
                         "type": "default",
@@ -83,6 +84,88 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2025-08-19T00:00:00.000Z",
+            "agent": {
+                "ephemeral_id": "082a402e-e376-4e6a-8ae4-9fcd12471589",
+                "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+                "name": "elastic-agent-89987",
+                "type": "filebeat",
+                "version": "9.0.4"
+            },
+            "cloud": {
+                "account": {
+                    "id": "1111"
+                },
+                "geo": {
+                    "location": {
+                        "lat": 90,
+                        "lon": 135
+                    },
+                    "name": "unknown"
+                },
+                "instance": {
+                    "id": "b7ffa6ecae534f81a2778ca1ce685254",
+                    "name": "example-serverless"
+                },
+                "provider": "serverless",
+                "service": {
+                    "type": "data-out"
+                }
+            },
+            "data_stream": {
+                "dataset": "ess_billing.billing",
+                "namespace": "68465",
+                "type": "metrics"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "elastic_agent": {
+                "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+                "snapshot": false,
+                "version": "9.0.4"
+            },
+            "ess": {
+                "billing": {
+                    "deployment_id": "b7ffa6ecae534f81a2778ca1ce685254",
+                    "deployment_name": "example-serverless",
+                    "deployment_type": "observability",
+                    "display_quantity": {
+                        "formatted_value": "0.0055 GBs",
+                        "type": "default",
+                        "value": 0.0055175
+                    },
+                    "from": "2025-08-19T00:00:00Z",
+                    "kind": "",
+                    "name": "Serverless Data Out",
+                    "organization_id": "1111",
+                    "quantity": {
+                        "formatted_value": "0.0055 GBs",
+                        "value": 0.0055175
+                    },
+                    "rate": {
+                        "formatted_value": "0.0000 per GB",
+                        "value": 0.0
+                    },
+                    "sku": "serverless.data-out",
+                    "to": "2025-08-20T00:00:00Z",
+                    "total_ecu": 0.0,
+                    "type": "data transfer",
+                    "unit": "GB"
+                }
+            },
+            "event": {
+                "created": "2021-11-11T01:02:03.123456789Z",
+                "dataset": "ess_billing.billing"
+            },
+            "input": {
+                "type": "cel"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }
diff --git a/packages/ess_billing/data_stream/billing/agent/stream/cel.yml.hbs b/packages/ess_billing/data_stream/billing/agent/stream/cel.yml.hbs
@@ -1,7 +1,7 @@
 config_version: 2
 interval: "24h"
 {{#if enable_request_tracer}}
-resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson"
+resource.tracer.filename: "/run/service_logs/http-request-trace-*.ndjson"
 resource.tracer.maxbackups: 5
 {{/if}}
 {{#if proxy_url}}
@@ -70,11 +70,13 @@ program: |-
                   "ess": {
                     "billing": line_item.with(
                       {
-                        "deployment_name": instance.name, // Include deployment name
-                        "deployment_id": instance.id, // Include deployment ID
-                        "organization_id": state.organization_id, // Include organization ID
+                        "deployment_name": instance.name,  // Include deployment name
+                        "deployment_id": instance.id,      // Include deployment ID
+                        "deployment_type": instance.type,  // Include deployment type
+                        "organization_id": state.organization_id,  // Include organization ID
                         "from": req.from.format(time_layout.RFC3339),
                         "to": req.to.format(time_layout.RFC3339),
+                        "quantities": null   // overwrite useless quantities to avoid flatten failure
                       }
                     ),
                   },
diff --git a/packages/ess_billing/data_stream/billing/elasticsearch/ingest_pipeline/default.yml b/packages/ess_billing/data_stream/billing/elasticsearch/ingest_pipeline/default.yml
@@ -54,26 +54,49 @@ processors:
       copy_from: ess.billing.deployment_id
       ignore_empty_value: true
       tag: cloud_instance_id
+  - set:
+      # NOTE(2025-08-25): This 'set' processor is necessary to cope with serverless
+      # project names being empty in the billing API. to be removed when Names in billing api fixed
+      field: cloud.instance.name
+      copy_from: ess.billing.deployment_id
   - set:
       field: cloud.instance.name
       copy_from: ess.billing.deployment_name
       ignore_empty_value: true
       tag: cloud_instance_name
+  # =====GROK SECTION=====
+  # Retrieving billed cloud service details like flavors and regions
+  #
+  # These 2 processors are specific to ECH sku's
   - grok:
       field: "ess.billing.sku"
       patterns:
         - "(?:Cloud-Enterprise_)?%{CLOUD_PROVIDER:cloud.provider}\\.%{NOTSPACE:cloud.machine.type}_%{NOTSPACE:cloud.region}_%{NUMBER:ess.billing.ram_per_zone:int}_%{NUMBER:ess.billing.zone_count:int}"
       pattern_definitions:
         CLOUD_PROVIDER: "aws|gcp|azure|global"
-      if: "ctx.ess.billing.type == 'capacity'"
+      if: "ctx.ess.billing.type == 'capacity' && ctx.ess.billing.deployment_type == 'deployment'"
       ignore_failure: true
       tag: grok_sku
   - dissect:
       field: "ess.billing.sku"
       pattern: "%{cloud.provider}.%{}"
-      if: "ctx.ess.billing.type != 'capacity'"
+      if: "ctx.ess.billing.type != 'capacity' && ctx.ess.billing.deployment_type == 'deployment'"
       ignore_failure: true
       tag: dissect_sku
+      
+   # These 2 processors are specific to serverless sku's
+  - grok:
+      field: "ess.billing.sku"
+      patterns:
+        - "%{DATA}\\.(?<cloud.service.type>[^_]+)(?:_(?<cloud.region>.+))?"
+      if: "ctx.ess.billing.deployment_type != 'deployment'"
+      ignore_failure: true
+      tag: grok_sku
+  - set:
+      field: "cloud.provider"
+      if: "ctx.ess.billing.deployment_type != 'deployment'"
+      value: "serverless"
+  # =====GROK SECTION END=====
   - convert:
       field: ess.billing.ram_per_zone
       type: "integer"
@@ -114,6 +137,7 @@ processors:
           'gcp-europe-west9': ['location': ['lon': 4.9041, 'lat': 52.3676], 'name': 'gcp-europe-west9'],
           'gcp-me-west1': ['location': ['lon': 34.8516, 'lat': 31.0461], 'name': 'gcp-me-west1'],
           'aws-us-east-2': ['location': ['lon': -82.9988, 'lat': 39.9612], 'name': 'aws-us-east-2'],
+          'aws-us-east-1': ['location': ['lon': -77.0369, 'lat': 39.9612], 'name': 'aws-us-east-2'],
           'us-east-1': ['location': ['lon': -77.0369, 'lat': 38.9072], 'name': 'us-east-1'],
           'eu-west-1': ['location': ['lon': -6.2603, 'lat': 53.3498], 'name': 'eu-west-1'],
           'us-west-1': ['location': ['lon': -122.4194, 'lat': 37.7749], 'name': 'us-west-1'],
diff --git a/packages/ess_billing/data_stream/billing/fields/fields.yml b/packages/ess_billing/data_stream/billing/fields/fields.yml
@@ -9,12 +9,18 @@
     - name: cloud.machine.type
       type: keyword
       description: The machine type of the instance (e.g., n2.68x16x45).
+    - name: cloud.service.type
+      type: keyword
+      description: The service type of the serverless project.
     - name: deployment_id
       type: keyword
       description: ID of the Elasticsearch Service deployment.
     - name: deployment_name
       type: keyword
       description: Name of the Elasticsearch Service deployment.
+    - name: deployment_type
+      type: keyword
+      description: Type of the Elasticsearch Service deployment.
     - name: display_quantity.formatted_value
       type: keyword
       description: Human-readable representation of the quantity used (e.g., "24 hours").
diff --git a/packages/ess_billing/docs/README.md b/packages/ess_billing/docs/README.md
@@ -140,8 +140,10 @@ An example event for `billing` looks as following:
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
 | ess.billing.cloud.machine.type | The machine type of the instance (e.g., n2.68x16x45). | keyword |
+| ess.billing.cloud.service.type | The service type of the serverless project. | keyword |
 | ess.billing.deployment_id | ID of the Elasticsearch Service deployment. | keyword |
 | ess.billing.deployment_name | Name of the Elasticsearch Service deployment. | keyword |
+| ess.billing.deployment_type | Type of the Elasticsearch Service deployment. | keyword |
 | ess.billing.display_quantity.formatted_value | Human-readable representation of the quantity used (e.g., "24 hours"). | keyword |
 | ess.billing.display_quantity.type | Type of quantity displayed (default or custom). | keyword |
 | ess.billing.display_quantity.value | Actual quantity used (e.g., 24). | float |
diff --git a/packages/ess_billing/kibana/dashboard/ess_billing-billingdashboard.json b/packages/ess_billing/kibana/dashboard/ess_billing-billingdashboard.json
diff --git a/packages/ess_billing/kibana/dashboard/ess_billing-creditsdashboard.json b/packages/ess_billing/kibana/dashboard/ess_billing-creditsdashboard.json
diff --git a/packages/ess_billing/manifest.yml b/packages/ess_billing/manifest.yml
