fix: update Prometheus datasource identifiers and query expressions in dashboards

Author: tazlin

dashboards/ai_horde_otlp/horde-app-traces.json

@@ -248,7 +248,7 @@
       "id": 31,
       "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "fixed" }, "unit": "ops" }, "overrides": [] },
       "options": { "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "area" },
-      "targets": [{ "expr": "sum(rate(traces_spanmetrics_calls_total{service_name=\"ai-horde\", span_name=~\".*generate.*\"}[5m]))", "legendFormat": "/generate", "refId": "A" }],
+      "targets": [{ "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"ai-horde\", span_name=~\".*generate.*\"}[5m]))", "legendFormat": "/generate", "refId": "A" }],
       "title": "/generate (rate)",
       "type": "stat"
     },
@@ -258,7 +258,7 @@
       "id": 32,
       "fieldConfig": { "defaults": { "color": { "fixedColor": "purple", "mode": "fixed" }, "unit": "ops" }, "overrides": [] },
       "options": { "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "area" },
-      "targets": [{ "expr": "sum(rate(traces_spanmetrics_calls_total{service_name=\"ai-horde\", span_name=~\".*pop.*\"}[5m]))", "legendFormat": "/pop", "refId": "A" }],
+      "targets": [{ "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"ai-horde\", span_name=~\".*pop.*\"}[5m]))", "legendFormat": "/pop", "refId": "A" }],
       "title": "/pop (rate)",
       "type": "stat"
     },
@@ -268,7 +268,7 @@
       "id": 33,
       "fieldConfig": { "defaults": { "color": { "fixedColor": "green", "mode": "fixed" }, "unit": "ops" }, "overrides": [] },
       "options": { "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "area" },
-      "targets": [{ "expr": "sum(rate(traces_spanmetrics_calls_total{service_name=\"ai-horde\", span_name=~\".*submit.*\"}[5m]))", "legendFormat": "/submit", "refId": "A" }],
+      "targets": [{ "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"ai-horde\", span_name=~\".*submit.*\"}[5m]))", "legendFormat": "/submit", "refId": "A" }],
       "title": "/submit (rate)",
       "type": "stat"
     },
@@ -291,9 +291,9 @@
         "tooltip": { "mode": "multi", "sort": "desc" }
       },
       "targets": [
-        { "expr": "sum(rate(traces_spanmetrics_calls_total{service_name=\"ai-horde\", span_name=~\".*generate.*\"}[$__rate_interval]))", "legendFormat": "/generate", "refId": "A" },
-        { "expr": "sum(rate(traces_spanmetrics_calls_total{service_name=\"ai-horde\", span_name=~\".*pop.*\"}[$__rate_interval]))", "legendFormat": "/pop", "refId": "B" },
-        { "expr": "sum(rate(traces_spanmetrics_calls_total{service_name=\"ai-horde\", span_name=~\".*submit.*\"}[$__rate_interval]))", "legendFormat": "/submit", "refId": "C" }
+        { "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"ai-horde\", span_name=~\".*generate.*\"}[$__rate_interval]))", "legendFormat": "/generate", "refId": "A" },
+        { "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"ai-horde\", span_name=~\".*pop.*\"}[$__rate_interval]))", "legendFormat": "/pop", "refId": "B" },
+        { "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"ai-horde\", span_name=~\".*submit.*\"}[$__rate_interval]))", "legendFormat": "/submit", "refId": "C" }
       ],
       "title": "Funnel Trend (rate)"
     }

dashboards/ai_horde_otlp/horde-operations-overview.json

@@ -348,7 +348,7 @@
       "type": "row"
     },
     {
-      "datasource": { "type": "prometheus", "uid": "mimir-app" },
+      "datasource": { "type": "prometheus", "uid": "mimir-infra" },
       "description": "Connection saturation as % of max_connections. Requires postgres_exporter (see horde_monitoring_install_postgres_alerts).",
       "fieldConfig": {
         "defaults": {
@@ -382,7 +382,7 @@
       "type": "timeseries"
     },
     {
-      "datasource": { "type": "prometheus", "uid": "mimir-app" },
+      "datasource": { "type": "prometheus", "uid": "mimir-infra" },
       "description": "Age of oldest open transaction from built-in postgres_exporter pg_stat_activity_max_tx_duration.",
       "fieldConfig": {
         "defaults": {
@@ -414,7 +414,7 @@
       "type": "timeseries"
     },
     {
-      "datasource": { "type": "prometheus", "uid": "mimir-app" },
+      "datasource": { "type": "prometheus", "uid": "mimir-infra" },
       "description": "Postgres deadlocks per 5 minutes.",
       "fieldConfig": {
         "defaults": {
@@ -616,15 +616,15 @@
       "links": [
         {
           "title": "Open in Loki Explore",
-          "url": "/explore?left=%7B%22datasource%22%3A%22loki-app%22%2C%22queries%22%3A%5B%7B%22expr%22%3A%22%7Bapp%3D%5C%22ai-horde%5C%22%7D%20%7C%20json%20%7C%20level%3D~%5C%22CRITICAL%7CERROR%5C%22%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22now-1h%22%2C%22to%22%3A%22now%22%7D%7D",
+          "url": "/explore?left=%7B%22datasource%22%3A%22loki-app%22%2C%22queries%22%3A%5B%7B%22expr%22%3A%22%7Bapp%3D%5C%22ai-horde%5C%22%2Clevel%3D~%5C%22%28%3Fi%29error%7Ccrit%28ical%29%3F%5C%22%7D%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22now-1h%22%2C%22to%22%3A%22now%22%7D%7D",
           "targetBlank": true
         }
       ],
       "options": {
         "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom", "showLegend": true },
         "tooltip": { "mode": "multi", "sort": "desc" }
       },
-      "targets": [{ "expr": "sum by (level) (count_over_time({app=\"ai-horde\"} | json | level=~\"CRITICAL|ERROR\" [5m]))", "legendFormat": "{{level}}", "refId": "A" }],
+      "targets": [{ "expr": "sum by (level) (count_over_time({app=\"ai-horde\",level=~\"(?i)error|crit(ical)?\"} [5m]))", "legendFormat": "{{level}}", "refId": "A" }],
       "title": "AI-Horde Critical/Error Log Rate (5m)",
       "type": "timeseries"
     },
@@ -643,7 +643,7 @@
         "dedupStrategy": "none",
         "sortOrder": "Descending"
       },
-      "targets": [{ "expr": "{app=\"ai-horde\"} | json | level=\"CRITICAL\"", "refId": "A" }],
+      "targets": [{ "expr": "{app=\"ai-horde\",level=~\"(?i)crit(ical)?\"}", "refId": "A" }],
       "title": "Recent CRITICAL Lines",
       "type": "logs"
     }

dashboards/ai_horde_otlp/horde-performance.json

@@ -1139,7 +1139,7 @@
   "templating": {
     "list": [
       {
-        "current": { "selected": false, "text": "", "value": "" },
+        "current": { "selected": false, "text": "AI Horde App", "value": "mimir-app" },
         "hide": 0,
         "includeAll": false,
         "label": "Data Source",