Skip to content

Commit 8d375d8

Browse files
committed
fix(grafana): fix P99 Latency NaN, 5xx filter, CPU no-data fallback
P99 Latency: - Remove outer sum() wrapping histogram_quantile (causes NaN) - Remove 'HTTP/2' from protocol regex (slash invalid in PromQL) 5xx Rate: - Add protocol label filter to match dataplane metric labels CPU Throttle: - Add 'or vector(0)' to display 0 instead of 'No data' when cAdvisor metrics are unavailable
1 parent 67dbbe2 commit 8d375d8

1 file changed

Lines changed: 4 additions & 4 deletions

File tree

deploy/observability/grafana/pingora-gateway-observability-dashboard.json

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@
298298
},
299299
"targets": [
300300
{
301-
"expr": "sum(histogram_quantile(0.99, sum(rate(nantian_gateway_dataplane_traffic_request_latency_ms_bucket{protocol=~\"HTTP|HTTPS|GRPC|GRPCS|H2C|HTTP2|HTTP/2\",response_flag=\"none\"}[$__rate_interval])) by (le)))",
301+
"expr": "histogram_quantile(0.99, sum(rate(nantian_gateway_dataplane_traffic_request_latency_ms_bucket{protocol=~\"HTTP|HTTPS|GRPC|GRPCS|H2C|HTTP2\",response_flag=\"none\"}[$__rate_interval])) by (le))",
302302
"refId": "A"
303303
}
304304
],
@@ -333,7 +333,7 @@
333333
},
334334
"targets": [
335335
{
336-
"expr": "sum(rate(nantian_gateway_dataplane_traffic_status_5xx_total[$__rate_interval]) / sum(rate(nantian_gateway_dataplane_traffic_request_events_total[$__rate_interval])) * 100)",
336+
"expr": "sum(rate(nantian_gateway_dataplane_traffic_status_5xx_total{protocol=~\"HTTP|HTTPS|GRPC|GRPCS|H2C|HTTP2\"}[$__rate_interval])) / sum(rate(nantian_gateway_dataplane_traffic_request_events_total{protocol=~\"HTTP|HTTPS|GRPC|GRPCS|H2C|HTTP2\"}[$__rate_interval])) * 100",
337337
"refId": "A"
338338
}
339339
],
@@ -403,7 +403,7 @@
403403
},
404404
"targets": [
405405
{
406-
"expr": "sum(rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval]) / rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval]))",
406+
"expr": "(sum(rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval]) / rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval]))) or vector(0)",
407407
"refId": "A"
408408
}
409409
],
@@ -1244,7 +1244,7 @@
12441244
"options": {},
12451245
"targets": [
12461246
{
1247-
"expr": "rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval]) / rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval])",
1247+
"expr": "(rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval]) / rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\",container!=\"\",pod=~\"$pod_dp\"}[$__rate_interval])) or vector(0)",
12481248
"legendFormat": "throttle",
12491249
"refId": "A"
12501250
}

0 commit comments

Comments
 (0)