gpu-availability-allocation-dashboard.yaml

apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaDashboard
metadata:
  name: gpu-availability-allocation-dashboard
  namespace: kubeflow-monitoring-system
  labels:
    app.kubernetes.io/part-of: kubeflow-observability
spec:
  instanceSelector:
    matchLabels:
      dashboards: "grafana"
  json: |
    {
      "__requires": [
        { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.0.0" },
        { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }
      ],
      "templating": {
        "list": [
          {
            "name": "datasource",
            "type": "datasource",
            "query": "prometheus",
            "label": "Datasource",
            "hide": 0
          }
        ]
      },
      "title": "GPU Availability & Allocation",
      "description": "Requires kube-state-metrics for GPU allocation and availability metrics.",
      "panels": [
        {
          "title": "Pending GPU workloads",
          "description": "Requires kube-state-metrics being scraped by the Prometheus datasource.",
          "type": "stat",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "count(kube_pod_status_phase{phase=\"Pending\"} * on(pod, namespace) group_left() kube_pod_container_resource_requests{resource=\"nvidia.com/gpu\"})", "legendFormat": "Pending NVIDIA GPU Pods" }
          ]
        }
      ]
    }