Fixes for WVA nightly E2E failing on CKS and OCP (#958)

Signed-off-by: Braulio Dumba <Braulio.Dumba@ibm.com>

Author: dumb0002

deploy/install.sh

@@ -43,6 +43,10 @@ VALUES_FILE=${VALUES_FILE:-"$WVA_PROJECT/charts/workload-variant-autoscaler/valu
 # Controller instance identifier for multi-controller isolation (optional)
 # When set, adds controller_instance label to metrics and HPA selectors
 CONTROLLER_INSTANCE=${CONTROLLER_INSTANCE:-""}
+# InferencePool API group to watch (v1 is the default, auto-detected after llm-d deploy)
+# Track if user explicitly set POOL_GROUP to avoid overriding their choice
+POOL_GROUP_USER_SET=${POOL_GROUP:+true}
+POOL_GROUP=${POOL_GROUP:-"inference.networking.k8s.io"}
 
 # llm-d Configuration
 LLM_D_OWNER=${LLM_D_OWNER:-"llm-d"}
@@ -1120,17 +1124,28 @@ deploy_llm_d_infrastructure() {
     # Align WVA with the InferencePool API group in use (scale-from-zero requires WVA to watch the same group).
     # llm-d version determines whether pools are inference.networking.k8s.io (v1) or inference.networking.x-k8s.io (v1alpha2).
     if [ "$DEPLOY_WVA" == "true" ]; then
-        detect_inference_pool_api_group
-        if [ -n "$DETECTED_POOL_GROUP" ]; then
-            log_info "Detected InferencePool API group: $DETECTED_POOL_GROUP; upgrading WVA to watch it (scale-from-zero)"
-            if helm upgrade "$WVA_RELEASE_NAME" ${WVA_PROJECT}/charts/workload-variant-autoscaler \
-                -n $WVA_NS --reuse-values --set wva.poolGroup=$DETECTED_POOL_GROUP --wait --timeout=60s; then
-                log_success "WVA upgraded with wva.poolGroup=$DETECTED_POOL_GROUP"
+        # Only auto-detect and upgrade if user didn't explicitly set POOL_GROUP
+        if [ "$POOL_GROUP_USER_SET" == "true" ]; then
+            log_info "POOL_GROUP explicitly set by user to $POOL_GROUP - skipping auto-detection to respect user's choice"
+        else
+            detect_inference_pool_api_group
+            if [ -n "$DETECTED_POOL_GROUP" ]; then
+                # Only upgrade if detected group differs from current POOL_GROUP
+                if [ "$DETECTED_POOL_GROUP" != "$POOL_GROUP" ]; then
+                    log_info "Detected InferencePool API group: $DETECTED_POOL_GROUP (differs from default $POOL_GROUP); upgrading WVA to watch it (scale-from-zero)"
+                    if helm upgrade "$WVA_RELEASE_NAME" ${WVA_PROJECT}/charts/workload-variant-autoscaler \
+                        -n $WVA_NS --reuse-values --set wva.poolGroup=$DETECTED_POOL_GROUP --wait --timeout=60s; then
+                        log_success "WVA upgraded with wva.poolGroup=$DETECTED_POOL_GROUP"
+                        POOL_GROUP=$DETECTED_POOL_GROUP
+                    else
+                        log_warning "WVA upgrade with poolGroup failed - scale-from-zero may not see the InferencePool"
+                    fi
+                else
+                    log_info "Detected InferencePool API group matches default ($POOL_GROUP) - no upgrade needed"
+                fi
             else
-                log_warning "WVA upgrade with poolGroup failed - scale-from-zero may not see the InferencePool"
+                log_warning "Could not detect InferencePool API group - using default POOL_GROUP=$POOL_GROUP (scale-from-zero may be misconfigured if cluster uses a different group)"
             fi
-        else
-            log_warning "Could not detect InferencePool API group - WVA may have empty datastore for scale-from-zero"
         fi
     fi
 

internal/datastore/datastore.go

@@ -34,7 +34,8 @@ import (
 )
 
 var (
-	errPoolNotSynced = errors.New("EndpointPool not found in datastore")
+	// ErrPoolNotSynced is returned when an EndpointPool is not found in the datastore
+	ErrPoolNotSynced = errors.New("EndpointPool not found in datastore")
 	errPoolIsNull    = errors.New("EndpointPool object is nil, does not exist")
 )
 
@@ -149,7 +150,7 @@ func (ds *datastore) PoolGet(namespacedName string) (*poolutil.EndpointPool, err
 
 	pool, exist := ds.pools.Load(namespacedName)
 	if !exist {
-		return nil, errPoolNotSynced
+		return nil, ErrPoolNotSynced
 	}
 
 	epp := pool.(*poolutil.EndpointPool)
@@ -182,7 +183,7 @@ func (ds *datastore) PoolGetFromLabels(namespace string, labels map[string]strin
 	})
 
 	if !exist {
-		return nil, errPoolNotSynced
+		return nil, ErrPoolNotSynced
 	}
 	return ep, nil
 }

internal/engines/scalefromzero/engine.go

@@ -232,18 +232,33 @@ func (e *Engine) processInactiveVariant(ctx context.Context, deployments map[str
 	// Find target EPP for metrics collection in the same namespace as the VA
 	pool, err := e.Datastore.PoolGetFromLabels(va.Namespace, labels)
 	if err != nil {
-		logger.Error(err, "Error finding target EPP", "variant", va.Name, "namespace", va.Namespace, "target VA model", va.Spec.ModelID)
+		// Only skip on "not found" errors - return other errors to surface real datastore failures
+		if errors.Is(err, datastore.ErrPoolNotSynced) {
+			logger.V(logging.DEBUG).Info("Skipping variant, target EPP not found in datastore",
+				"variant", va.Name,
+				"namespace", va.Namespace,
+				"modelID", va.Spec.ModelID)
+			return nil
+		}
+		// Unexpected error - log and return to surface the issue
+		logger.Error(err, "Unexpected error finding target EPP",
+			"variant", va.Name,
+			"namespace", va.Namespace,
+			"modelID", va.Spec.ModelID)
 		return err
 	}
 
 	// Use EPP source from registry
-	eppSource := e.Datastore.PoolGetMetricsSource(pool.Namespace + "/" + pool.Name)
+	namespacedPoolName := pool.Namespace + "/" + pool.Name
+	eppSource := e.Datastore.PoolGetMetricsSource(namespacedPoolName)
 	if eppSource == nil {
-		logger.Info("Scale-from-zero: skipping VA, EPP metrics source not found in datastore",
-			"va", va.Name,
+		// This is unexpected - pool exists but metrics source is missing
+		err := fmt.Errorf("EPP metrics source not found in registry for pool %s", namespacedPoolName)
+		logger.Error(err, "Datastore inconsistency detected",
+			"variant", va.Name,
 			"namespace", va.Namespace,
-			"pool", pool.Namespace+"/"+pool.Name)
-		return errors.New("endpointpicker metrics source not found in datastore")
+			"pool", namespacedPoolName)
+		return err
 	}
 
 	results, err := eppSource.Refresh(ctx, source.RefreshSpec{})