[alerts] add per-cluster Mimir rule namespaces + alert routing labels (#709)

**Context:**
After updating the key, we're now able to see the alerts in Grafana,
with examples of the following form:
```
alloy/monitoring/arc-alerts/<SOME_HASH>
...
```
I'm noticing the following issues:
1. these alerts seem to be constantly created/destroyed: *Failed to load
rules from group arc in alloy...* after clicking on such a rule in the
UI.
2. there are multiple copies. Each cluster seems to emit their own
version: `alloy/monitoring/gpu-alerts/<SOME_HASH_1, SOME_HASH_2>`, and
this seems to contest.

To address this:
- set mimir_namespace_prefix to the cluster name so clusters stop
contesting/recreating each other's rules in the shared Mimir tenant. New
name should be `<CLUSTER_NAME>/monitoring/arc-alerts/<SOME_HASH>`
- alerts: add team=pytorch-dev-infra-osdc to all rules and priority=P1
to critical rules, for the alerting-infra issue pipeline. The label
pytorch-dev-infra was not directly chosen since that would automatically
flag it with treehugger.

After this change, I'm going to add the delivery system wiring for one
of the clusters and a subset of the alerts. Afterwards, the label can be
updated if all succeeds.

```
just lint && just test
```

Author: georgehong

osdc/modules/monitoring/helm/alloy-values.yaml

@@ -114,6 +114,7 @@ alloy:
       mimir.rules.kubernetes "grafana_cloud" {
         address  = env("GCLOUD_BASE_URL")
         tenant_id = env("GCLOUD_RW_API_USER")
+        mimir_namespace_prefix = env("CLUSTER_NAME")
 
         basic_auth {
           username = env("GCLOUD_RW_API_USER")

osdc/modules/monitoring/kubernetes/alerts/arc-alerts.yaml

@@ -14,6 +14,8 @@ spec:
           for: 15m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "ARC listener stalled — jobs queued but no progress"
             description: "ARC listener has {{ $value }} assigned jobs but the assignment rate has been zero for 15 minutes. Runners may not be picking up new jobs."

osdc/modules/monitoring/kubernetes/alerts/gpu-alerts.yaml

@@ -14,6 +14,8 @@ spec:
           for: 1m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "GPU {{ $labels.gpu }} on {{ $labels.node }} has double-bit ECC errors"
             description: "GPU {{ $labels.gpu }} ({{ $labels.device }}) on node {{ $labels.node }} has new uncorrectable double-bit ECC errors. This GPU should be taken out of service."
@@ -23,6 +25,8 @@ spec:
           for: 1m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "GPU {{ $labels.gpu }} on {{ $labels.node }} has critical XID error {{ $value }}"
             description: "GPU {{ $labels.gpu }} reported XID error {{ $value }} (48=DBE, 79=GPU fallen off bus, 94/95=NVLink errors). Immediate investigation required."
@@ -32,6 +36,8 @@ spec:
           for: 1m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "GPU {{ $labels.gpu }} on {{ $labels.node }} has row remap failure"
             description: "GPU {{ $labels.gpu }} has a failed row remap, indicating irreparable memory damage. This GPU should be replaced."
@@ -41,6 +47,8 @@ spec:
           for: 5m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "GPU {{ $labels.gpu }} on {{ $labels.node }} temperature critical ({{ $value }}°C)"
             description: "GPU {{ $labels.gpu }} temperature has exceeded 95°C for 5 minutes. Sustained high temperatures cause throttling and hardware damage."

osdc/modules/monitoring/kubernetes/alerts/harbor-cache-recovery-alerts.yaml

@@ -25,6 +25,8 @@ spec:
           for: 15m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "harbor-cache-recovery cronjob is failing"
             description: >-
@@ -56,6 +58,8 @@ spec:
           for: 10m
           labels:
             severity: warning
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "harbor-cache-recovery pod was OOMKilled"
             description: >-
@@ -78,6 +82,8 @@ spec:
           for: 5m
           labels:
             severity: warning
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "harbor-cache-recovery has not completed successfully in >30m"
             description: >-

osdc/modules/monitoring/kubernetes/alerts/infrastructure-alerts.yaml

@@ -14,6 +14,8 @@ spec:
           for: 5m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Node {{ $labels.node }} is NotReady"
             description: "Node {{ $labels.node }} has been in NotReady state for more than 5 minutes."
@@ -24,6 +26,8 @@ spec:
           for: 15m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash-looping"
             description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted {{ $value }} times in the last hour."
@@ -33,6 +37,8 @@ spec:
           for: 5m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Harbor is down"
             description: "Harbor scrape target {{ $labels.instance }} has been down for more than 5 minutes. Image pulls through the proxy cache will fail."
@@ -45,6 +51,8 @@ spec:
           for: 15m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Karpenter creating nodeclaims but no nodes joining"
             description: "Karpenter has created nodeclaims in the last 15 minutes but no new nodes have joined the cluster. Nodes may be failing to register."

osdc/modules/monitoring/kubernetes/alerts/network-pressure-alerts.yaml

@@ -14,6 +14,8 @@ spec:
           for: 5m
           labels:
             severity: warning
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Node conntrack table > 80% full ({{ $labels.node }})"
             description: |
@@ -27,6 +29,8 @@ spec:
           for: 2m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Node conntrack table > 95% full — imminent connection drops ({{ $labels.node }})"
             description: |
@@ -43,6 +47,8 @@ spec:
           for: 10m
           labels:
             severity: warning
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "node_nf_conntrack_entries metric is missing — conntrack alerts inert"
             description: |

osdc/modules/monitoring/kubernetes/alerts/node-compactor-alerts.yaml

@@ -14,6 +14,8 @@ spec:
           for: 15m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Node compactor reconciliation is failing persistently"
             description: "The node-compactor in namespace {{ $labels.namespace }} has had continuous reconciliation errors for 15 minutes. Burst-absorption (untainting nodes under scheduling pressure) is offline. Check for expired SA tokens (401) or RBAC issues."

osdc/modules/monitoring/kubernetes/alerts/nodelocaldns-alerts.yaml

@@ -14,6 +14,8 @@ spec:
           for: 5m
           labels:
             severity: critical
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "NodeLocal DNSCache setup errors on {{ $labels.pod }}"
             description: |
@@ -26,6 +28,8 @@ spec:
           for: 10m
           labels:
             severity: warning
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "NodeLocal DNSCache pod restarting"
             description: |
@@ -38,6 +42,8 @@ spec:
           for: 15m
           labels:
             severity: warning
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "NodeLocal DNSCache DaemonSet has unavailable pods"
             description: |

osdc/modules/monitoring/kubernetes/alerts/zombie-cleanup-alerts.yaml

@@ -14,6 +14,8 @@ spec:
           for: 5m
           labels:
             severity: warning
+            team: pytorch-dev-infra
+            priority: P3
           annotations:
             summary: "Zombie cleanup cap reached — zombies deferred"
             description: >-