opt in for keda, env specific (llm-d#967)

Signed-off-by: Mohammed Abdi <mohammed.munir.abdi@ibm.com>

Author: mamy-CS

.github/workflows/ci-e2e-openshift.yaml

@@ -656,10 +656,14 @@ jobs:
           # may be owned by another namespace's release, causing Helm ownership conflicts.
           # Fix: adopt them for our namespace so helmfile can proceed. Post-cleanup will
           # delete them, and the next user's helmfile run will recreate them fresh.
+          # Only adopt legacy helmfile-style names (release "workload-variant-autoscaler").
+          # PR-specific Helm releases use names like wva-e2e-<run_id>; those live in WVA_NAMESPACE.
+          # Re-annotating them to LLMD_NAMESPACE breaks Helm ownership and can leave the controller
+          # ServiceAccount bound to a wrong or unmanaged ClusterRole (cluster-wide list/watch denied).
           echo "Adopting shared WVA cluster-scoped resources for namespace $LLMD_NAMESPACE..."
           for kind in clusterrole clusterrolebinding; do
             kubectl get "$kind" -o json 2>/dev/null | \
-              jq -r '.items[] | select(.metadata.name | contains("workload-variant-autoscaler")) | select(.metadata.annotations["meta.helm.sh/release-namespace"] != null) | .metadata.name' 2>/dev/null | \
+              jq -r '.items[] | select(.metadata.name | contains("workload-variant-autoscaler")) | select(.metadata.name | startswith("wva-e2e-") | not) | select(.metadata.annotations["meta.helm.sh/release-namespace"] != null) | .metadata.name' 2>/dev/null | \
               while read -r name; do
                 current_ns=$(kubectl get "$kind" "$name" -o json 2>/dev/null | jq -r '.metadata.annotations["meta.helm.sh/release-namespace"] // ""')
                 if [ "$current_ns" != "$LLMD_NAMESPACE" ]; then
@@ -688,6 +692,9 @@ jobs:
           ENVIRONMENT: openshift
           INSTALL_GATEWAY_CTRLPLANE: "false"
           E2E_TESTS_ENABLED: "true"
+          # OpenShift typically lacks HPAScaleToZero; e2e forces SCALE_TO_ZERO_ENABLED off for openshift
+          # (see test/e2e/config.go). KEDA ScaledObjects support minReplicas=0 for scale-from-zero tests.
+          SCALER_BACKEND: keda
           NAMESPACE_SCOPED: "false"
           # Pass PR-specific namespaces to install script
           LLMD_NS: ${{ env.LLMD_NAMESPACE }}
@@ -793,6 +800,7 @@ jobs:
           ENVIRONMENT: openshift
           INSTALL_GATEWAY_CTRLPLANE: "false"
           E2E_TESTS_ENABLED: "true"
+          SCALER_BACKEND: keda
           NAMESPACE_SCOPED: "false"
           # Override namespaces for Model B stack
           LLMD_NS: ${{ env.LLMD_NAMESPACE_B }}

deploy/install.sh

@@ -113,13 +113,16 @@ QUEUE_SPARE_TRIGGER=${QUEUE_SPARE_TRIGGER:-""}
 
 # Scaler backend: "prometheus-adapter" (default), "keda", or "none"
 # prometheus-adapter: deploy Prometheus Adapter + patch external metrics APIService
-# keda:              deploy KEDA via Helm (or detect pre-installed) + configure ScaledObjects
+# keda:              on kubernetes assume cluster-managed KEDA (no Helm; set KEDA_HELM_INSTALL=true to install);
+#                    on kind-emulator install via Helm when needed; OpenShift is always platform-managed (no Helm)
 # none:              skip all scaler backend deployment; use when KEDA or another metrics API
 #                    is already installed on the cluster (e.g. llmd benchmark clusters)
 SCALER_BACKEND=${SCALER_BACKEND:-prometheus-adapter}
 KEDA_NAMESPACE=${KEDA_NAMESPACE:-keda-system}
 # Pin KEDA chart version for reproducible installs (only used when deploy_keda installs from helm)
 KEDA_CHART_VERSION=${KEDA_CHART_VERSION:-2.19.0}
+# kubernetes: default false (cluster-managed KEDA); set true to let this script install/upgrade KEDA via Helm
+KEDA_HELM_INSTALL=${KEDA_HELM_INSTALL:-false}
 
 # Environment-related variables
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

deploy/lib/cleanup.sh

@@ -8,6 +8,14 @@
 #
 
 undeploy_keda() {
+    if [ "$ENVIRONMENT" = "openshift" ]; then
+        log_info "OpenShift: skipping KEDA uninstall (platform-managed)"
+        return
+    fi
+    if [ "$ENVIRONMENT" = "kubernetes" ] && [ "${KEDA_HELM_INSTALL:-false}" != "true" ]; then
+        log_info "Kubernetes: skipping KEDA uninstall (cluster-managed; set KEDA_HELM_INSTALL=true if this script installed KEDA)"
+        return
+    fi
     log_info "Uninstalling KEDA..."
     helm uninstall "$KEDA_RELEASE_NAME" -n "$KEDA_NAMESPACE" 2>/dev/null || \
         log_warning "KEDA not found or already uninstalled"

deploy/lib/cli.sh

@@ -38,9 +38,11 @@ Environment Variables:
   INFRA_ONLY                   Deploy only infrastructure (default: false, same as --infra-only flag)
   SCALER_BACKEND               Scaler backend: "prometheus-adapter" (default), "keda", or "none".
                                prometheus-adapter: installs Prometheus Adapter and patches the external metrics APIService.
-                               keda: installs KEDA (or detects pre-installed) and skips Prometheus Adapter.
+                               keda: skips Prometheus Adapter; on kubernetes assumes cluster-managed KEDA (KEDA_HELM_INSTALL=true for Helm);
+                                     kind-emulator installs KEDA via Helm when needed; OpenShift is platform-managed only.
                                none: skips all scaler backend deployment. Use this on clusters that already have
                                      KEDA or another external metrics API installed (e.g. llmd benchmark clusters).
+  KEDA_HELM_INSTALL            When true with ENVIRONMENT=kubernetes, install/upgrade KEDA via Helm (default: false)
   KEDA_NAMESPACE               Namespace for KEDA (default: keda-system)
   UNDEPLOY                     Undeploy mode (default: false)
   DELETE_NAMESPACES            Delete namespaces after undeploy (default: false)

deploy/lib/infra_llmd.sh

@@ -136,6 +136,7 @@ deploy_llm_d_infrastructure() {
         if [ "$DEPLOY_WVA" == "true" ] && [ "$VLLM_SVC_ENABLED" == "true" ]; then
           helm upgrade "$WVA_RELEASE_NAME" ${WVA_PROJECT}/charts/workload-variant-autoscaler \
             -n "$WVA_NS" --reuse-values \
+            --set wva.namespaceScoped="${NAMESPACE_SCOPED:-true}" \
             --set vllmService.port="$VLLM_SVC_PORT" \
             --set vllmService.targetPort="$VLLM_SVC_PORT"
         fi
@@ -325,7 +326,9 @@ deploy_llm_d_infrastructure() {
         if [ -n "$DETECTED_POOL_GROUP" ]; then
             log_info "Detected InferencePool API group: $DETECTED_POOL_GROUP; upgrading WVA to watch it (scale-from-zero)"
             if helm upgrade "$WVA_RELEASE_NAME" ${WVA_PROJECT}/charts/workload-variant-autoscaler \
-                -n "$WVA_NS" --reuse-values --set wva.poolGroup="$DETECTED_POOL_GROUP" --wait --timeout=60s; then
+                -n "$WVA_NS" --reuse-values \
+                --set wva.namespaceScoped="${NAMESPACE_SCOPED:-true}" \
+                --set wva.poolGroup="$DETECTED_POOL_GROUP" --wait --timeout=60s; then
                 log_success "WVA upgraded with wva.poolGroup=$DETECTED_POOL_GROUP"
             else
                 log_warning "WVA upgrade with poolGroup failed - scale-from-zero may not see the InferencePool"

deploy/lib/infra_scaler_backend.sh

@@ -8,8 +8,9 @@
 
 deploy_scaler_backend() {
     # Deploy scaler backend: KEDA, Prometheus Adapter, or none.
-    # KEDA is supported on all environments. On OpenShift and CKS it is typically
-    # pre-installed on the cluster; deploy_keda will detect and skip the install.
+    # OpenShift: KEDA is never Helm-installed (platform-managed); see deploy_keda in scaler_runtime.sh.
+    # Kubernetes: deploy_keda skips Helm by default (cluster-managed); KEDA_HELM_INSTALL=true enables Helm.
+    # kind-emulator: Helm when needed; shared-cluster guard uses ClusterRole keda-operator when Helm is used.
     # Use SCALER_BACKEND=none on clusters that already have an external metrics API
     # (e.g. llmd benchmark clusters with KEDA pre-installed) to avoid conflicts.
     if [ "$SCALER_BACKEND" = "keda" ]; then

deploy/lib/scaler_runtime.sh

@@ -68,6 +68,37 @@ stop_apiservice_guard() {
 deploy_keda() {
     log_info "Deploying KEDA (scaler backend)..."
 
+    # OpenShift: KEDA is cluster-managed (OLM/operator); never Helm-install — avoids
+    # ClusterRole/release conflicts with an existing platform KEDA.
+    if [ "$ENVIRONMENT" = "openshift" ]; then
+        log_info "OpenShift: assuming platform-managed KEDA — skipping Helm install"
+        if kubectl get crd scaledobjects.keda.sh >/dev/null 2>&1; then
+            log_success "KEDA ScaledObject CRD is available on the cluster"
+        else
+            if [ "$E2E_TESTS_ENABLED" = "true" ]; then
+                log_error "OpenShift: scaledobjects.keda.sh CRD not found — install cluster KEDA before E2E (SCALER_BACKEND=keda)"
+                exit 1
+            fi
+            log_warning "KEDA ScaledObject CRD not found — ScaledObject-based scaling will not work"
+        fi
+        return
+    fi
+
+    # Kubernetes (e.g. CKS, shared clusters): assume cluster-managed KEDA; never Helm unless opted in.
+    if [ "$ENVIRONMENT" = "kubernetes" ] && [ "${KEDA_HELM_INSTALL:-false}" != "true" ]; then
+        log_info "Kubernetes: assuming cluster-managed KEDA — skipping Helm (set KEDA_HELM_INSTALL=true to install via Helm)"
+        if kubectl get crd scaledobjects.keda.sh >/dev/null 2>&1; then
+            log_success "KEDA ScaledObject CRD is available on the cluster"
+        else
+            if [ "$E2E_TESTS_ENABLED" = "true" ]; then
+                log_error "Kubernetes: scaledobjects.keda.sh CRD not found — install KEDA on the cluster or set KEDA_HELM_INSTALL=true"
+                exit 1
+            fi
+            log_warning "KEDA ScaledObject CRD not found — ScaledObject-based scaling will not work"
+        fi
+        return
+    fi
+
     # Skip install if KEDA is already fully operational on the cluster.
     # Check CRD + operator pods + external metrics APIService to avoid false positives
     # from stale CRDs left behind after a prior uninstall.
@@ -78,6 +109,16 @@ deploy_keda() {
                 return
             fi
         fi
+        # Shared clusters (e.g. CKS) often pre-install KEDA without the exact pod label / APIService
+        # shape our probe expects, but ClusterRole keda-operator already exists without Helm metadata.
+        # Helm install then fails with ownership errors — skip Helm when that pattern is present.
+        if kubectl get clusterrole keda-operator >/dev/null 2>&1; then
+            keda_cr_managed_by=$(kubectl get clusterrole keda-operator -o jsonpath='{.metadata.labels.app\.kubernetes\.io/managed-by}' 2>/dev/null || true)
+            if [ "$keda_cr_managed_by" != "Helm" ]; then
+                log_info "KEDA CRD present and ClusterRole keda-operator is not Helm-managed — skipping Helm install (pre-installed KEDA)"
+                return
+            fi
+        fi
         log_warning "KEDA ScaledObject CRD found but operator or metrics APIService not detected; proceeding with helm install"
     fi
 

test/e2e/config.go

@@ -10,7 +10,9 @@ type E2EConfig struct {
 	testconfig.SharedConfig
 
 	// Feature gates
-	ScaleToZeroEnabled bool // HPAScaleToZero feature gate
+	// ScaleToZeroEnabled: env SCALE_TO_ZERO_ENABLED — assume native HPA may use minReplicas=0
+	// ("scale-to-zero" via HPAScaleToZero). Distinct from scale-from-zero (scale up from zero replicas).
+	ScaleToZeroEnabled bool
 
 	// Timeouts (seconds unless noted)
 	PodReadyTimeout int // Wait for deployment/model pods ready
@@ -53,10 +55,10 @@ func LoadConfigFromEnv() E2EConfig {
 		PrometheusAdapterProbeSec: testconfig.GetEnvInt("E2E_PROM_ADAPTER_PROBE_SEC", 90),
 	}
 
-	// OpenShift clusters typically don't have the HPAScaleToZero feature gate
-	// enabled, so attempting to create HPAs with minReplicas=0 will fail with:
-	//   "spec.minReplicas: Invalid value: 0: must be greater than or equal to 1"
-	// Override the env var to prevent test failures on OpenShift.
+	// OpenShift clusters typically don't have the HPAScaleToZero feature gate enabled, so native HPAs
+	// cannot use minReplicas=0 ("scale-to-zero" on the HPA). Ignore SCALE_TO_ZERO_ENABLED there so e2e
+	// does not assume that path (creation fails with: minReplicas must be >= 1).
+	// Scale-from-zero (scaling workloads up from zero replicas) is separate; this block does not configure SCALER_BACKEND.
 	if cfg.Environment == "openshift" && cfg.ScaleToZeroEnabled {
 		cfg.ScaleToZeroEnabled = false
 	}

test/e2e/scale_from_zero_test.go

@@ -135,8 +135,12 @@ func cleanupScaleFromZeroResources() {
 }
 
 // Scale-from-zero test validates that the WVA controller correctly detects pending requests
-// and scales up deployments from zero replicas. Requires GIE queuing (ENABLE_EXPERIMENTAL_FLOW_CONTROL_LAYER
+// and scales up scale targets from zero replicas. Requires GIE queuing (ENABLE_EXPERIMENTAL_FLOW_CONTROL_LAYER
 // on EPP from install when E2E_TESTS_ENABLED=true) and an InferenceObjective (applied below in BeforeAll).
+// This suite needs a scaler that allows minReplicas=0 on the scaled workload: either
+// SCALE_TO_ZERO_ENABLED=true where native HPA supports it (HPAScaleToZero), or SCALER_BACKEND=keda
+// (ScaledObject). OpenShift usually lacks HPAScaleToZero; e2e config ignores SCALE_TO_ZERO_ENABLED there,
+// so use SCALER_BACKEND=keda for this Describe when running on OpenShift.
 // On platforms without the HPAScaleToZero feature gate (e.g. OpenShift), set SCALER_BACKEND=keda
 // so the test uses a KEDA ScaledObject (which supports minReplicas=0) instead of a native HPA.
 var _ = Describe("Scale-From-Zero Feature", Serial, Label("full"), Ordered, func() {
@@ -153,8 +157,9 @@ var _ = Describe("Scale-From-Zero Feature", Serial, Label("full"), Ordered, func
 		// HPAScaleToZero feature gate), SCALER_BACKEND=keda must be set so the
 		// test creates a KEDA ScaledObject instead of a native HPA.
 		if cfg.ScalerBackend != "keda" && !cfg.ScaleToZeroEnabled {
-			Skip("Scale-from-zero requires SCALER_BACKEND=\"keda\" or ENABLE_SCALE_TO_ZERO=true; " +
-				"current configuration does not support HPA minReplicas=0")
+			Skip("This suite needs minReplicas=0 on the scaler: set SCALER_BACKEND=\"keda\" " +
+				"or SCALE_TO_ZERO_ENABLED=true (ignored on OpenShift without HPAScaleToZero — use KEDA); " +
+				"current configuration does not support that scaler shape")
 		}
 
 		By("Cleaning up any existing scale-from-zero test resources")
@@ -619,8 +624,9 @@ var _ = Describe("Scale-From-Zero Feature with LeaderWorkerSet", Serial, Label("
 		// HPAScaleToZero feature gate), SCALER_BACKEND=keda must be set so the
 		// test creates a KEDA ScaledObject instead of a native HPA.
 		if cfg.ScalerBackend != "keda" && !cfg.ScaleToZeroEnabled {
-			Skip("Scale-from-zero requires SCALER_BACKEND=\"keda\" or ENABLE_SCALE_TO_ZERO=true; " +
-				"current configuration does not support HPA minReplicas=0")
+			Skip("This suite needs minReplicas=0 on the scaler: set SCALER_BACKEND=\"keda\" " +
+				"or SCALE_TO_ZERO_ENABLED=true (ignored on OpenShift without HPAScaleToZero — use KEDA); " +
+				"current configuration does not support that scaler shape")
 		}
 
 		By("Cleaning up any existing scale-from-zero test resources")
@@ -1039,8 +1045,9 @@ var _ = Describe("Scale-From-Zero Feature with LeaderWorkerSet (single-node)", S
 		// HPAScaleToZero feature gate), SCALER_BACKEND=keda must be set so the
 		// test creates a KEDA ScaledObject instead of a native HPA.
 		if cfg.ScalerBackend != "keda" && !cfg.ScaleToZeroEnabled {
-			Skip("Scale-from-zero requires SCALER_BACKEND=\"keda\" or ENABLE_SCALE_TO_ZERO=true; " +
-				"current configuration does not support HPA minReplicas=0")
+			Skip("This suite needs minReplicas=0 on the scaler: set SCALER_BACKEND=\"keda\" " +
+				"or SCALE_TO_ZERO_ENABLED=true (ignored on OpenShift without HPAScaleToZero — use KEDA); " +
+				"current configuration does not support that scaler shape")
 		}
 
 		By("Cleaning up any existing scale-from-zero test resources")