Skip to content

[aws_ecs_otel] Add ML anomaly detection module #19922

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JM-elastic wants to merge 1 commit into
base: main
Choose a base branch
from
+188 −1
Draft

[aws_ecs_otel] Add ML anomaly detection module #19922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/aws_ecs_otel/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "0.7.0"
changes:
- description: Add ML anomaly detection module for ECS service CPU and memory utilization.
type: enhancement
link: https://github.com/elastic/integrations/pull/19922
- version: "0.6.0"
changes:
- description: Improve ESQL queries in dashboards
Expand Down
182 changes: 182 additions & 0 deletions packages/aws_ecs_otel/kibana/ml_module/aws_ecs_otel-metrics-ml.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
{
"id": "aws_ecs_otel-metrics-ml",
"type": "ml-module",
"migrationVersion": {
"search": "7.9.3"
},
"references": [],
"attributes": {
"id": "aws_ecs_otel-metrics-ml",
"title": "AWS ECS service resources (OpenTelemetry)",
"description": "Detect anomalies in ECS service CPU and memory utilization streamed from CloudWatch via the OpenTelemetry firehose receiver. Memory is modelled per service (within its cluster) against its own history to catch slow leaks that affect a service's tasks before any single task crosses a hard threshold - the ECS analogue of per-workload memory modelling on Kubernetes.",
"type": "AWS metrics",
"logo": {
"icon": "logoAWSMono"
},
"defaultIndexPattern": "metrics-aws.ecs.otel-*",
"query": {
"bool": {
"filter": [
{
"term": {
"Namespace": "AWS/ECS"
}
},
{
"exists": {
"field": "ServiceName"
}
}
],
"must_not": {
"terms": {
"_tier": [
"data_frozen",
"data_cold"
]
}
}
}
},
"jobs": [
{
"id": "aws_ecs_service_resource_anomaly",
"config": {
"groups": [
"aws",
"ecs",
"otel"
],
"description": "AWS ECS: detect services whose CPU or memory utilization has grown unusually relative to that service's own history, partitioned by cluster. Catches slow memory-leak trajectories that affect a service across task recycling before any individual task crosses a hard threshold, and sustained CPU drift. Per-service threshold breaches are covered by the alert rules; this job covers the sub-threshold drift, with the service, cluster, region, and account as influencers.",
"analysis_config": {
"bucket_span": "15m",
"summary_count_field_name": "doc_count",
"detectors": [
{
"detector_description": "Anomalous service memory utilization (leak trajectory)",
"function": "high_mean",
"field_name": "metrics.amazonaws.com/AWS/ECS/MemoryUtilization",
"by_field_name": "ServiceName",
"partition_field_name": "ClusterName"
},
{
"detector_description": "Anomalous service CPU utilization",
"function": "high_mean",
"field_name": "metrics.amazonaws.com/AWS/ECS/CPUUtilization",
"by_field_name": "ServiceName",
"partition_field_name": "ClusterName"
}
],
"influencers": [
"ServiceName",
"ClusterName",
"cloud.region",
"cloud.account.id"
]
},
"analysis_limits": {
"model_memory_limit": "64mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"model_plot_config": {
"enabled": false,
"annotations_enabled": true
},
"custom_settings": {
"created_by": "ml-module-aws-ecs-otel-metrics-ml"
}
}
}
],
"datafeeds": [
{
"id": "datafeed-aws_ecs_service_resource_anomaly",
"job_id": "aws_ecs_service_resource_anomaly",
"config": {
"job_id": "aws_ecs_service_resource_anomaly",
"indices": [
"INDEX_PATTERN_NAME"
],
"indices_options": {
"allow_no_indices": true
},
"query": {
"bool": {
"filter": [
{
"term": {
"stat": "Average"
}
},
{
"terms": {
"MetricName": [
"MemoryUtilization",
"CPUUtilization"
]
}
},
{
"exists": {
"field": "ServiceName"
}
}
]
}
},
"aggregations": {
"buckets": {
"composite": {
"size": 1000,
"sources": [
{
"date": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "900s"
}
}
},
{
"ClusterName": {
"terms": {
"field": "ClusterName"
}
}
},
{
"ServiceName": {
"terms": {
"field": "ServiceName"
}
}
}
]
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
},
"metrics.amazonaws.com/AWS/ECS/MemoryUtilization": {
"avg": {
"field": "metrics.amazonaws.com/AWS/ECS/MemoryUtilization"
}
},
"metrics.amazonaws.com/AWS/ECS/CPUUtilization": {
"avg": {
"field": "metrics.amazonaws.com/AWS/ECS/CPUUtilization"
}
}
}
}
}
}
}
]
}
}
2 changes: 1 addition & 1 deletion packages/aws_ecs_otel/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: 3.6.0
name: aws_ecs_otel
title: "AWS ECS Metrics OpenTelemetry Assets"
version: 0.6.0
version: 0.7.0
source:
license: "Elastic-2.0"
description: "AWS ECS Metrics OpenTelemetry Assets"
Expand Down
Loading