Skip to content

[aws_sqs_otel] Add ML anomaly detection module #19924

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
JM-elastic wants to merge 1 commit into
base: main
Choose a base branch
from
+200 −1
Draft

[aws_sqs_otel] Add ML anomaly detection module #19924

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/aws_sqs_otel/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "0.8.0"
changes:
- description: Add ML anomaly detection module for SQS queue backlog, in-flight, and oldest-message age.
type: enhancement
link: https://github.com/elastic/integrations/pull/19924
- version: "0.7.0"
changes:
- description: Improve ESQL queries in dashboards
Expand Down
194 changes: 194 additions & 0 deletions packages/aws_sqs_otel/kibana/ml_module/aws_sqs_otel-metrics-ml.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
{
"id": "aws_sqs_otel-metrics-ml",
"type": "ml-module",
"migrationVersion": {
"search": "7.9.3"
},
"references": [],
"attributes": {
"id": "aws_sqs_otel-metrics-ml",
"title": "AWS SQS queue backlog (OpenTelemetry)",
"description": "Detect anomalies in SQS queue depth, in-flight messages, and oldest-message age streamed from CloudWatch via the OpenTelemetry firehose receiver. Each queue is modelled against its own history, so a backlog building abnormally for that queue (a stalling or under-scaled consumer) is caught before it crosses the static depth or age thresholds.",
"type": "AWS metrics",
"logo": {
"icon": "logoAWSMono"
},
"defaultIndexPattern": "metrics-aws.sqs.otel-*",
"query": {
"bool": {
"filter": [
{
"term": {
"Namespace": "AWS/SQS"
}
},
{
"exists": {
"field": "QueueName"
}
}
],
"must_not": {
"terms": {
"_tier": [
"data_frozen",
"data_cold"
]
}
}
}
},
"jobs": [
{
"id": "aws_sqs_queue_backlog_anomaly",
"config": {
"groups": [
"aws",
"sqs",
"otel"
],
"description": "AWS SQS: detect queues whose visible backlog, in-flight (not-visible) count, or oldest-message age has grown unusually relative to that queue's own history - the signature of a consumer slowing, crashing, or falling behind producer rate. Catches backlog buildup that is abnormal for the queue but still below the fixed depth/age alert thresholds, with the queue, region, and account as influencers.",
"analysis_config": {
"bucket_span": "15m",
"summary_count_field_name": "doc_count",
"detectors": [
{
"detector_description": "Anomalous visible message backlog",
"function": "high_mean",
"field_name": "metrics.amazonaws.com/AWS/SQS/ApproximateNumberOfMessagesVisible",
"by_field_name": "QueueName",
"partition_field_name": "cloud.region"
},
{
"detector_description": "Anomalous in-flight (not-visible) message count",
"function": "high_mean",
"field_name": "metrics.amazonaws.com/AWS/SQS/ApproximateNumberOfMessagesNotVisible",
"by_field_name": "QueueName",
"partition_field_name": "cloud.region"
},
{
"detector_description": "Anomalous age of oldest message (consumer falling behind)",
"function": "high_mean",
"field_name": "metrics.amazonaws.com/AWS/SQS/ApproximateAgeOfOldestMessage",
"by_field_name": "QueueName",
"partition_field_name": "cloud.region"
}
],
"influencers": [
"QueueName",
"cloud.region",
"cloud.account.id"
]
},
"analysis_limits": {
"model_memory_limit": "64mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"model_plot_config": {
"enabled": false,
"annotations_enabled": true
},
"custom_settings": {
"created_by": "ml-module-aws-sqs-otel-metrics-ml"
}
}
}
],
"datafeeds": [
{
"id": "datafeed-aws_sqs_queue_backlog_anomaly",
"job_id": "aws_sqs_queue_backlog_anomaly",
"config": {
"job_id": "aws_sqs_queue_backlog_anomaly",
"indices": [
"INDEX_PATTERN_NAME"
],
"indices_options": {
"allow_no_indices": true
},
"query": {
"bool": {
"filter": [
{
"term": {
"stat": "Average"
}
},
{
"terms": {
"MetricName": [
"ApproximateNumberOfMessagesVisible",
"ApproximateNumberOfMessagesNotVisible",
"ApproximateAgeOfOldestMessage"
]
}
},
{
"exists": {
"field": "QueueName"
}
}
]
}
},
"aggregations": {
"buckets": {
"composite": {
"size": 1000,
"sources": [
{
"date": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "900s"
}
}
},
{
"cloud.region": {
"terms": {
"field": "cloud.region"
}
}
},
{
"QueueName": {
"terms": {
"field": "QueueName"
}
}
}
]
},
"aggregations": {
"@timestamp": {
"max": {
"field": "@timestamp"
}
},
"metrics.amazonaws.com/AWS/SQS/ApproximateNumberOfMessagesVisible": {
"avg": {
"field": "metrics.amazonaws.com/AWS/SQS/ApproximateNumberOfMessagesVisible"
}
},
"metrics.amazonaws.com/AWS/SQS/ApproximateNumberOfMessagesNotVisible": {
"avg": {
"field": "metrics.amazonaws.com/AWS/SQS/ApproximateNumberOfMessagesNotVisible"
}
},
"metrics.amazonaws.com/AWS/SQS/ApproximateAgeOfOldestMessage": {
"avg": {
"field": "metrics.amazonaws.com/AWS/SQS/ApproximateAgeOfOldestMessage"
}
}
}
}
}
}
}
]
}
}
2 changes: 1 addition & 1 deletion packages/aws_sqs_otel/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: 3.6.0
name: aws_sqs_otel
title: "AWS SQS Metrics OpenTelemetry Assets"
version: 0.7.0
version: 0.8.0
source:
license: "Elastic-2.0"
description: "AWS SQS Metrics OpenTelemetry Assets"
Expand Down
Loading