fix: improve external metrics diagnostics for CI debugging

Previous diagnostics were truncated (grep -A5 only showed first rule).
Now dumps: both external metrics responses, exact adapter PromQL via
Prometheus direct query, full externalRules from ConfigMap (using
python3 yaml parser with grep fallback), adapter container args to
verify metricsRelistInterval, and more adapter log lines.

Author: dims

kwok/scripts/validate-cluster-autoscaling.sh

@@ -123,28 +123,44 @@ verify_external_metrics() {
     if [[ "${has_data}" != "true" ]]; then
         log_error "External metric dcgm_gpu_memory_used has no data after 4 minutes"
 
-        # Diagnostic 1: Raw external metrics API response
-        log_warn "--- Raw external metrics API response ---"
+        # Diagnostic 1: Raw external metrics API responses for BOTH metrics
+        log_warn "--- External API: dcgm_gpu_memory_used ---"
         kubectl --context="${KUBE_CTX}" get --raw \
-            "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/dcgm_gpu_memory_used" 2>&1 | head -20 || true
+            "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/dcgm_gpu_memory_used" 2>&1 || true
+        log_warn "--- External API: dcgm_gpu_utilization ---"
+        kubectl --context="${KUBE_CTX}" get --raw \
+            "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/dcgm_gpu_utilization" 2>&1 || true
 
-        # Diagnostic 2: Query Prometheus directly for the underlying metric
+        # Diagnostic 2: Run the exact PromQL the adapter would execute
         local prom_svc="http://kube-prometheus-prometheus.monitoring.svc:9090"
-        log_warn "--- Prometheus direct query: DCGM_FI_DEV_FB_USED ---"
-        kubectl --context="${KUBE_CTX}" -n monitoring run prom-check --rm -i --restart=Never \
-            --image=curlimages/curl:latest -- \
-            curl -sf "${prom_svc}/api/v1/query?query=DCGM_FI_DEV_FB_USED" 2>/dev/null \
-            | jq '.data.result | length' || true
-
-        # Diagnostic 3: Check prometheus-adapter ConfigMap for externalRules
-        log_warn "--- prometheus-adapter ConfigMap (externalRules) ---"
+        log_warn "--- Prometheus: exact adapter PromQL ---"
+        kubectl --context="${KUBE_CTX}" -n monitoring run prom-diag --rm -i --restart=Never \
+            --image=curlimages/curl:latest -- sh -c "
+                echo 'Raw series count:';
+                curl -sf '${prom_svc}/api/v1/query?query=DCGM_FI_DEV_FB_USED' | head -c 500;
+                echo;
+                echo 'Adapter metricsQuery result:';
+                curl -sf '${prom_svc}/api/v1/query?query=avg(avg_over_time(DCGM_FI_DEV_FB_USED%5B2m%5D))' | head -c 500;
+                echo;
+            " 2>/dev/null || true
+
+        # Diagnostic 3: Full externalRules from ConfigMap (not truncated)
+        log_warn "--- prometheus-adapter externalRules (full) ---"
         kubectl --context="${KUBE_CTX}" -n monitoring get configmap -l app.kubernetes.io/name=prometheus-adapter \
             -o jsonpath='{.items[0].data.config\.yaml}' 2>/dev/null \
-            | grep -A5 'externalRules' || echo "No externalRules found in ConfigMap"
+            | python3 -c "import sys,yaml; cfg=yaml.safe_load(sys.stdin); print(yaml.dump(cfg.get('externalRules', 'MISSING')))" 2>/dev/null \
+            || kubectl --context="${KUBE_CTX}" -n monitoring get configmap -l app.kubernetes.io/name=prometheus-adapter \
+                -o jsonpath='{.items[0].data.config\.yaml}' 2>/dev/null | grep -A20 'externalRules' || echo "No externalRules found"
+
+        # Diagnostic 4: Adapter deployment args (verify metricsRelistInterval)
+        log_warn "--- prometheus-adapter container args ---"
+        kubectl --context="${KUBE_CTX}" -n monitoring get deployment -l app.kubernetes.io/name=prometheus-adapter \
+            -o jsonpath='{.items[0].spec.template.spec.containers[0].args}' 2>/dev/null || true
+        echo
 
-        # Diagnostic 4: prometheus-adapter logs (last 20 lines)
+        # Diagnostic 5: prometheus-adapter logs (last 30 lines)
         log_warn "--- prometheus-adapter logs (tail) ---"
-        kubectl --context="${KUBE_CTX}" -n monitoring logs deployment/prometheus-adapter --tail=20 2>/dev/null || true
+        kubectl --context="${KUBE_CTX}" -n monitoring logs deployment/prometheus-adapter --tail=30 2>/dev/null || true
 
         exit 1
     fi