Skip to content

Commit 3b49282

Browse files
gregkalapostheletterf
gregkalapos
and
theletterf
authored
[otel_collector_internal_telemetry] Add alerting rules (#18048)
* Add alerting rules * Update changelog and manifest --------- Co-authored-by: Fabrizio Ferri-Benedetti <algernon@fastmail.com>
1 parent 248957e commit 3b49282

5 files changed

Lines changed: 105 additions & 1 deletion

‎packages/otel_collector_internal_telemetry/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "1.1.0"
3+
changes:
4+
- description: Add alerting rules for failed span, metric point and log record exports.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/18048
27
- version: "1.0.0"
38
changes:
49
- description: Add ES|QL based dashboards and increase minimum Kibana version to 9.3.

‎packages/otel_collector_internal_telemetry/kibana/alerting_rule_template/otel-collector-internal-telemetry-otel-exporter-send-failed-log-records.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"id": "otel-collector-internal-telemetry-otel-exporter-send-failed-log-records",
3+
"type": "alerting_rule_template",
4+
"attributes": {
5+
"name": "[OpenTelemetry Collector Internal Telemetry Otel] Exporter send failed log records",
6+
"tags": [
7+
"OpenTelemetry",
8+
"OpenTelemetry Collector",
9+
"Internal telemetry",
10+
"Exporter"
11+
],
12+
"ruleTypeId": ".es-query",
13+
"schedule": {
14+
"interval": "5m"
15+
},
16+
"params": {
17+
"searchType": "esqlQuery",
18+
"timeWindowSize": 5,
19+
"timeWindowUnit": "m",
20+
"esqlQuery": {
21+
"esql": "// Alert when the OpenTelemetry Collector reports non-zero rate of failed log record exports.\n//\n// metrics.otelcol_exporter_send_failed_log_records is a cumulative counter; RATE() yields\n// per-bucket increases. TS is used for time-series metrics (counter resets handled by the TS pipeline).\n//\n// Scope: metrics-collectortelemetry* and collectortelemetry.otel dataset to avoid mixing with\n// unrelated metrics-* streams.\n//\n// Grouping: service.name and service.instance.id help identify which collector instance fails.\n//\n// Tuning: raise the threshold (e.g. failed_log_records_rate > N) or increase alert delay\n// to reduce noise from brief blips.\n//\nTS metrics-collectortelemetry*\n| WHERE data_stream.dataset == \"collectortelemetry.otel\"\n| STATS failed_log_records_rate = SUM(RATE(metrics.otelcol_exporter_send_failed_log_records)) BY service.name, service.instance.id\n| WHERE failed_log_records_rate > 0\n| SORT failed_log_records_rate DESC\n"
22+
},
23+
"groupBy": "row",
24+
"timeField": "@timestamp"
25+
},
26+
"alertDelay": {
27+
"active": 2
28+
}
29+
},
30+
"managed": true,
31+
"coreMigrationVersion": "8.8.0",
32+
"typeMigrationVersion": "10.1.0"
33+
}

‎packages/otel_collector_internal_telemetry/kibana/alerting_rule_template/otel-collector-internal-telemetry-otel-exporter-send-failed-metric-points.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"id": "otel-collector-internal-telemetry-otel-exporter-send-failed-metric-points",
3+
"type": "alerting_rule_template",
4+
"attributes": {
5+
"name": "[OpenTelemetry Collector Internal Telemetry Otel] Exporter send failed metric points",
6+
"tags": [
7+
"OpenTelemetry",
8+
"OpenTelemetry Collector",
9+
"Internal telemetry",
10+
"Exporter"
11+
],
12+
"ruleTypeId": ".es-query",
13+
"schedule": {
14+
"interval": "5m"
15+
},
16+
"params": {
17+
"searchType": "esqlQuery",
18+
"timeWindowSize": 5,
19+
"timeWindowUnit": "m",
20+
"esqlQuery": {
21+
"esql": "// Alert when the OpenTelemetry Collector reports non-zero rate of failed metric point exports.\n//\n// metrics.otelcol_exporter_send_failed_metric_points is a cumulative counter; RATE() yields\n// per-bucket increases. TS is used for time-series metrics (counter resets handled by the TS pipeline).\n//\n// Scope: metrics-collectortelemetry* and collectortelemetry.otel dataset to avoid mixing with\n// unrelated metrics-* streams.\n//\n// Grouping: service.name and service.instance.id help identify which collector instance fails.\n//\n// Tuning: raise the threshold (e.g. failed_metric_points_rate > N) or increase alert delay\n// to reduce noise from brief blips.\n//\nTS metrics-collectortelemetry*\n| WHERE data_stream.dataset == \"collectortelemetry.otel\"\n| STATS failed_metric_points_rate = SUM(RATE(metrics.otelcol_exporter_send_failed_metric_points)) BY service.name, service.instance.id\n| WHERE failed_metric_points_rate > 0\n| SORT failed_metric_points_rate DESC\n"
22+
},
23+
"groupBy": "row",
24+
"timeField": "@timestamp"
25+
},
26+
"alertDelay": {
27+
"active": 2
28+
}
29+
},
30+
"managed": true,
31+
"coreMigrationVersion": "8.8.0",
32+
"typeMigrationVersion": "10.1.0"
33+
}

‎packages/otel_collector_internal_telemetry/kibana/alerting_rule_template/otel-collector-internal-telemetry-otel-exporter-send-failed-spans.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"id": "otel-collector-internal-telemetry-otel-exporter-send-failed-spans",
3+
"type": "alerting_rule_template",
4+
"attributes": {
5+
"name": "[OpenTelemetry Collector Internal Telemetry Otel] Exporter send failed spans",
6+
"tags": [
7+
"OpenTelemetry",
8+
"OpenTelemetry Collector",
9+
"Internal telemetry",
10+
"Exporter"
11+
],
12+
"ruleTypeId": ".es-query",
13+
"schedule": {
14+
"interval": "5m"
15+
},
16+
"params": {
17+
"searchType": "esqlQuery",
18+
"timeWindowSize": 5,
19+
"timeWindowUnit": "m",
20+
"esqlQuery": {
21+
"esql": "// Alert when the OpenTelemetry Collector reports non-zero rate of failed span exports.\n//\n// metrics.otelcol_exporter_send_failed_spans is a cumulative counter; RATE() yields per-bucket\n// increases. TS is used for time-series metrics (counter resets handled by the TS pipeline).\n//\n// Scope: metrics-collectortelemetry* and collectortelemetry.otel dataset to avoid mixing with\n// unrelated metrics-* streams.\n//\n// Grouping: service.name and service.instance.id help identify which collector instance fails.\n// Add exporter dimension in your mapping if available (e.g. attributes.exporter).\n//\n// Tuning: raise the threshold (e.g. failed_rate > N) or require multiple consecutive runs via\n// alert delay to reduce noise from brief blips.\n//\nTS metrics-collectortelemetry*\n| WHERE data_stream.dataset == \"collectortelemetry.otel\"\n| STATS failed_spans_rate = SUM(RATE(metrics.otelcol_exporter_send_failed_spans)) BY service.name, service.instance.id\n| WHERE failed_spans_rate > 0\n| SORT failed_spans_rate DESC\n"
22+
},
23+
"groupBy": "row",
24+
"timeField": "@timestamp"
25+
},
26+
"alertDelay": {
27+
"active": 2
28+
}
29+
},
30+
"managed": true,
31+
"coreMigrationVersion": "8.8.0",
32+
"typeMigrationVersion": "10.1.0"
33+
}

‎packages/otel_collector_internal_telemetry/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 3.5.2
22
name: otel_collector_internal_telemetry
33
title: "OpenTelemetry Collector Internal Telemetry"
4-
version: 1.0.0
4+
version: 1.1.0
55
source:
66
license: "Apache-2.0"
77
description: "This package contains dashboards that visualize the internal telemetry from the OpenTelemetry Collector"

0 commit comments

Comments
 (0)