Merge pull request #2780 from amnonh/ethtool

scylla-os: Add ethtool section

Author: amnonh

grafana/scylla-os.template.json

@@ -585,6 +585,94 @@
                 ],
                 "title": "New row"
             },
+            {
+                "class": "row",
+                "dashversion":[">2025.4"],
+                "panels": [
+                    {
+                      "collapsed": true,
+                      "datasource": null,
+                      "id": "auto",
+                      "gridPos": {
+                        "h": 1,
+                        "w": 24
+                      },
+                      "panels": [],
+                      "title": "ethtool limitations - only applicable if ethtool is enabled on nodetool_exporter",
+                      "type": "row"
+                    }
+                ]
+            },
+           {
+                "class": "row",
+                "dashversion":[">2025.4"],
+                "panels": [
+                	{
+                        "class": "graph_panel",
+                        "span": 3,
+                        "description": "Outbound bandwidth allowance exceed events per second — the rate at which the NIC attempts to transmit above its allowed egress bandwidth limit (often indicating throttling/shaping).",
+                        "targets": [
+                            {
+                                "expr": "sum(rate(node_ethtool_bw_out_allowance_exceeded{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\"}[3m])) by ([[by]])",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "Bandwidth-Out allowance exceeded"
+                    },
+                    {
+                        "class": "graph_panel",
+                        "span": 3,
+                        "description": "Inbound bandwidth allowance exceed events per second — the rate at which the NIC receives above its allowed ingress bandwidth limit (often indicating ingress shaping/throttling).",
+                        "targets": [
+                            {
+                                "expr": "sum(rate(node_ethtool_bw_in_allowance_exceeded{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\"}[3m])) by ([[by]])",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "Bandwidth-In allowance exceeded"
+                    },
+                   {
+                        "class": "graph_panel",
+                        "span": 3,
+                        "description": "Conntrack allowance exceed events per second — the rate at which the system/NIC exceeds its allowed connection-tracking (conntrack) limit, often indicating connection pressure and potential drops or throttling.",
+                        "targets": [
+                            {
+                                "expr": "sum(rate(node_ethtool_conntrack_allowance_exceeded{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\"}[3m])) by ([[by]])",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "Conntrack allowance exceed"
+                    },
+                   {
+                        "class": "graph_panel",
+                        "span": 3,
+                        "description": "Link-local allowance exceed events per second — the rate at which the NIC exceeds its allowed link-local traffic limit (e.g., ARP/ND or other local control traffic), often indicating local-network chatter and possible throttling",
+                        "targets": [
+                            {
+                                "expr": "sum(rate(node_ethtool_linklocal_allowance_exceeded{instance=~\"[[node]]\",cluster=\"$cluster\", dc=~\"$dc\"}[3m])) by ([[by]])",
+                                "intervalFactor": 1,
+                                "legendFormat": "",
+                                "metric": "",
+                                "refId": "A",
+                                "step": 1
+                            }
+                        ],
+                        "title": "Link-local allowance exceed"
+                    }
+                ]
+            },
             {
                 "class": "monitoring_version_row"
             }

prometheus/prom_rules/prometheus.latency.rules.yml

@@ -552,6 +552,26 @@ groups:
     labels:
       by: "cluster"
       dd: "1"
+  - record: node_ethtool_bw_in_allowance_exceeded_total
+    expr: sum(node_ethtool_bw_in_allowance_exceeded) by (cluster)
+    labels:
+      by: "cluster"
+      dd: "1"
+  - record: node_ethtool_bw_out_allowance_exceeded_total
+    expr: sum(node_ethtool_bw_out_allowance_exceeded) by (cluster)
+    labels:
+      by: "cluster"
+      dd: "1"
+  - record: node_ethtool_conntrack_allowance_exceeded_total
+    expr: sum(node_ethtool_conntrack_allowance_exceeded) by (cluster)
+    labels:
+      by: "cluster"
+      dd: "1"
+  - record: node_ethtool_linklocal_allowance_exceeded_total
+    expr: sum(node_ethtool_linklocal_allowance_exceeded) by (cluster)
+    labels:
+      by: "cluster"
+      dd: "1"
   - record: wlatencyp99
     expr: scylla_storage_proxy_coordinator_write_latency_summary{quantile="0.99"}
   - record: rlatencyp99