From 56c6c2a9a5cd6f131c765f9efc727314c257553e Mon Sep 17 00:00:00 2001 From: dulao5 Date: Tue, 4 Aug 2026 17:58:51 +0900 Subject: [PATCH 1/3] metrics: show Disconnection Count as a rate, split undetermined out The Server > Disconnection Count panel plotted the raw cumulative tidb_server_disconnection_total counter, which grows unbounded on long-running clusters and stops being useful for spotting current connect/disconnect activity. Switch the ok/error series to irate(...[30s]) so the panel reflects disconnect frequency (and, transitively, connection churn / workload shape). Keep result="undetermined" as its own cumulative series on the right Y axis, since any nonzero value means a transaction was once left undecided and needs manual investigation, so it should stay an absolute "has this ever happened" counter rather than a rate. close pingcap/tidb#70336 Co-Authored-By: Claude Sonnet 5 --- pkg/metrics/grafana/tidb.json | 17 +++++++++++++++-- .../nextgengrafana/tidb_with_keyspace_name.json | 17 +++++++++++++++-- pkg/metrics/nextgengrafana/tidb_worker.json | 17 +++++++++++++++-- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index 7ab7d8f049bf5..9f64ae902a522 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -3093,7 +3093,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected counts", + "description": "TiDB connection disconnected rate per result (ok/error); undetermined is shown separately as a cumulative count on the right axis since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3139,6 +3139,11 @@ "alias": "total", "fill": 0, "lines": false + }, + { + "alias": "/-undetermined-total$/", + "stack": false, + "yaxis": 2 } ], "spaceLength": 10, @@ -3146,12 +3151,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)", + "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", "refId": "A", "step": 40 + }, + { + "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-undetermined-total", + "refId": "B", + "step": 40 } ], "thresholds": [], diff --git a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json index 0b1ee09b5b917..170321cd5da03 100644 --- a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json +++ b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json @@ -3259,7 +3259,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected counts", + "description": "TiDB connection disconnected rate per result (ok/error); undetermined is shown separately as a cumulative count on the right axis since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3305,6 +3305,11 @@ "alias": "total", "fill": 0, "lines": false + }, + { + "alias": "/-undetermined-total$/", + "stack": false, + "yaxis": 2 } ], "spaceLength": 10, @@ -3312,12 +3317,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (instance, result)", + "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", "refId": "A", "step": 40 + }, + { + "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-undetermined-total", + "refId": "B", + "step": 40 } ], "thresholds": [], diff --git a/pkg/metrics/nextgengrafana/tidb_worker.json b/pkg/metrics/nextgengrafana/tidb_worker.json index 2ff5d338ccd5a..39c5d4812c7c1 100644 --- a/pkg/metrics/nextgengrafana/tidb_worker.json +++ b/pkg/metrics/nextgengrafana/tidb_worker.json @@ -3259,7 +3259,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected counts", + "description": "TiDB connection disconnected rate per result (ok/error); undetermined is shown separately as a cumulative count on the right axis since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3305,6 +3305,11 @@ "alias": "total", "fill": 0, "lines": false + }, + { + "alias": "/-undetermined-total$/", + "stack": false, + "yaxis": 2 } ], "spaceLength": 10, @@ -3312,12 +3317,20 @@ "steppedLine": false, "targets": [ { - "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)", + "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", "refId": "A", "step": 40 + }, + { + "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-undetermined-total", + "refId": "B", + "step": 40 } ], "thresholds": [], From 66045cdd40178b593730a0ee810ffea42f6424c1 Mon Sep 17 00:00:00 2001 From: dulao5 Date: Fri, 7 Aug 2026 10:20:56 +0900 Subject: [PATCH 2/3] metrics: keep original Disconnection Count target, add rate as a new series Per review feedback, restore the original cumulative sum(tidb_server_disconnection_total{...}) by (instance, result) target instead of replacing it, so existing dashboard semantics and the cumulative count view are preserved. Add the ok/error irate(...[30s]) rate view and the undetermined cumulative counter as new, additional series on the right Y axis instead. Co-Authored-By: Claude Sonnet 5 --- pkg/metrics/grafana/tidb.json | 19 ++++++++++++++++--- .../tidb_with_keyspace_name.json | 19 ++++++++++++++++--- pkg/metrics/nextgengrafana/tidb_worker.json | 19 ++++++++++++++++--- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index 9f64ae902a522..ae31b468c1114 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -3093,7 +3093,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected rate per result (ok/error); undetermined is shown separately as a cumulative count on the right axis since any occurrence needs investigation", + "description": "TiDB connection disconnected counts per result (left axis, cumulative); right axis adds an ok/error disconnect rate view and a dedicated cumulative counter for result=undetermined, since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3144,6 +3144,11 @@ "alias": "/-undetermined-total$/", "stack": false, "yaxis": 2 + }, + { + "alias": "/-rate$/", + "stack": false, + "yaxis": 2 } ], "spaceLength": 10, @@ -3151,19 +3156,27 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", + "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", "refId": "A", "step": 40 }, + { + "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{result}}-rate", + "refId": "B", + "step": 40 + }, { "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-undetermined-total", - "refId": "B", + "refId": "C", "step": 40 } ], diff --git a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json index 170321cd5da03..a617635398661 100644 --- a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json +++ b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json @@ -3259,7 +3259,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected rate per result (ok/error); undetermined is shown separately as a cumulative count on the right axis since any occurrence needs investigation", + "description": "TiDB connection disconnected counts per result (left axis, cumulative); right axis adds an ok/error disconnect rate view and a dedicated cumulative counter for result=undetermined, since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3310,6 +3310,11 @@ "alias": "/-undetermined-total$/", "stack": false, "yaxis": 2 + }, + { + "alias": "/-rate$/", + "stack": false, + "yaxis": 2 } ], "spaceLength": 10, @@ -3317,19 +3322,27 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", + "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\"}) by (instance, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", "refId": "A", "step": 40 }, + { + "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{result}}-rate", + "refId": "B", + "step": 40 + }, { "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-undetermined-total", - "refId": "B", + "refId": "C", "step": 40 } ], diff --git a/pkg/metrics/nextgengrafana/tidb_worker.json b/pkg/metrics/nextgengrafana/tidb_worker.json index 39c5d4812c7c1..bf418ba761ca8 100644 --- a/pkg/metrics/nextgengrafana/tidb_worker.json +++ b/pkg/metrics/nextgengrafana/tidb_worker.json @@ -3259,7 +3259,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected rate per result (ok/error); undetermined is shown separately as a cumulative count on the right axis since any occurrence needs investigation", + "description": "TiDB connection disconnected counts per result (left axis, cumulative); right axis adds an ok/error disconnect rate view and a dedicated cumulative counter for result=undetermined, since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3310,6 +3310,11 @@ "alias": "/-undetermined-total$/", "stack": false, "yaxis": 2 + }, + { + "alias": "/-rate$/", + "stack": false, + "yaxis": 2 } ], "spaceLength": 10, @@ -3317,19 +3322,27 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", + "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by (instance, result)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{result}}", "refId": "A", "step": 40 }, + { + "expr": "sum(irate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result!=\"undetermined\"}[30s])) by (instance, result)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}-{{result}}-rate", + "refId": "B", + "step": 40 + }, { "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-undetermined-total", - "refId": "B", + "refId": "C", "step": 40 } ], From e12fb908b6a220050667eaa615a69761e3b334f3 Mon Sep 17 00:00:00 2001 From: dulao5 Date: Fri, 7 Aug 2026 12:09:20 +0900 Subject: [PATCH 3/3] metrics: drop redundant undetermined-total series in Disconnection Count The dedicated result="undetermined" query duplicated the undetermined series already produced by the original by (instance, result) cumulative target. Drop the extra query and instead route that existing series to the right Y axis via a seriesOverride alias match. Co-Authored-By: Claude Sonnet 5 --- pkg/metrics/grafana/tidb.json | 12 ++---------- .../nextgengrafana/tidb_with_keyspace_name.json | 12 ++---------- pkg/metrics/nextgengrafana/tidb_worker.json | 12 ++---------- 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/pkg/metrics/grafana/tidb.json b/pkg/metrics/grafana/tidb.json index ae31b468c1114..fd189ef9838b0 100644 --- a/pkg/metrics/grafana/tidb.json +++ b/pkg/metrics/grafana/tidb.json @@ -3093,7 +3093,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected counts per result (left axis, cumulative); right axis adds an ok/error disconnect rate view and a dedicated cumulative counter for result=undetermined, since any occurrence needs investigation", + "description": "TiDB connection disconnected counts per result (cumulative); right axis adds an ok/error disconnect rate view, and routes the undetermined cumulative counter to its own scale since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3141,7 +3141,7 @@ "lines": false }, { - "alias": "/-undetermined-total$/", + "alias": "/-undetermined$/", "stack": false, "yaxis": 2 }, @@ -3170,14 +3170,6 @@ "legendFormat": "{{instance}}-{{result}}-rate", "refId": "B", "step": 40 - }, - { - "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-undetermined-total", - "refId": "C", - "step": 40 } ], "thresholds": [], diff --git a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json index a617635398661..9c1af1a77862c 100644 --- a/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json +++ b/pkg/metrics/nextgengrafana/tidb_with_keyspace_name.json @@ -3259,7 +3259,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected counts per result (left axis, cumulative); right axis adds an ok/error disconnect rate view and a dedicated cumulative counter for result=undetermined, since any occurrence needs investigation", + "description": "TiDB connection disconnected counts per result (cumulative); right axis adds an ok/error disconnect rate view, and routes the undetermined cumulative counter to its own scale since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3307,7 +3307,7 @@ "lines": false }, { - "alias": "/-undetermined-total$/", + "alias": "/-undetermined$/", "stack": false, "yaxis": 2 }, @@ -3336,14 +3336,6 @@ "legendFormat": "{{instance}}-{{result}}-rate", "refId": "B", "step": 40 - }, - { - "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-undetermined-total", - "refId": "C", - "step": 40 } ], "thresholds": [], diff --git a/pkg/metrics/nextgengrafana/tidb_worker.json b/pkg/metrics/nextgengrafana/tidb_worker.json index bf418ba761ca8..437aed0c14a05 100644 --- a/pkg/metrics/nextgengrafana/tidb_worker.json +++ b/pkg/metrics/nextgengrafana/tidb_worker.json @@ -3259,7 +3259,7 @@ "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", - "description": "TiDB connection disconnected counts per result (left axis, cumulative); right axis adds an ok/error disconnect rate view and a dedicated cumulative counter for result=undetermined, since any occurrence needs investigation", + "description": "TiDB connection disconnected counts per result (cumulative); right axis adds an ok/error disconnect rate view, and routes the undetermined cumulative counter to its own scale since any occurrence needs investigation", "editable": true, "error": false, "fieldConfig": { @@ -3307,7 +3307,7 @@ "lines": false }, { - "alias": "/-undetermined-total$/", + "alias": "/-undetermined$/", "stack": false, "yaxis": 2 }, @@ -3336,14 +3336,6 @@ "legendFormat": "{{instance}}-{{result}}-rate", "refId": "B", "step": 40 - }, - { - "expr": "sum(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", result=\"undetermined\"}) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}-undetermined-total", - "refId": "C", - "step": 40 } ], "thresholds": [],