feat: Enable the disabling of feature gates that are by default on/active (#1728)

* Support enabling and disabling feature flags

Signed-off-by: Shmuel Kallner <kallner@il.ibm.com>

* Removed deprecated dataLayer feature flag

Signed-off-by: Shmuel Kallner <kallner@il.ibm.com>

* Updated feature flag registration with default value

Signed-off-by: Shmuel Kallner <kallner@il.ibm.com>

* Updated tests WRT new feature flag features

Signed-off-by: Shmuel Kallner <kallner@il.ibm.com>

---------

Signed-off-by: Shmuel Kallner <kallner@il.ibm.com>

Author: shmuelk

cmd/epp/runner/runner.go

@@ -602,8 +602,7 @@ func (r *Runner) parseConfigurationPhaseOne(ctx context.Context, opts *runserver
 		}
 	}
 
-	loader.RegisterFeatureGate(datalayer.ExperimentalDatalayerFeatureGate)
-	loader.RegisterFeatureGate(flowcontrol.FeatureGate)
+	loader.RegisterFeatureGate(flowcontrol.FeatureGate, false)
 
 	r.registerInTreePlugins()
 
@@ -614,11 +613,6 @@ func (r *Runner) parseConfigurationPhaseOne(ctx context.Context, opts *runserver
 
 	r.featureGates = featureGates
 
-	if r.featureGates[datalayer.ExperimentalDatalayerFeatureGate] {
-		setupLog.Info("The data layer is now enabled by default. " +
-			"Please remove the 'dataLayer' feature gate from your config.")
-	}
-
 	setupLog.Info("Data layer: ENABLED")
 
 	r.rawConfig = rawConfig

pkg/epp/config/loader/configloader.go

@@ -19,6 +19,8 @@ package loader
 import (
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/go-logr/logr"
@@ -46,7 +48,7 @@ import (
 var (
 	scheme                       = runtime.NewScheme()
 	registeredFeatureGatesMu     sync.RWMutex
-	registeredFeatureGates       = sets.New[string]()
+	registeredFeatureGates       = make(map[string]bool)
 	deprecatedSchemeGroupVersion = schema.GroupVersion{Group: "inference.networking.x-k8s.io", Version: "v1alpha1"} // TODO: deprecated should be clean up
 )
 
@@ -68,10 +70,10 @@ func init() {
 }
 
 // RegisterFeatureGate registers a feature gate name for validation purposes.
-func RegisterFeatureGate(gate string) {
+func RegisterFeatureGate(gate string, isEnabledByDefault bool) {
 	registeredFeatureGatesMu.Lock()
 	defer registeredFeatureGatesMu.Unlock()
-	registeredFeatureGates.Insert(gate)
+	registeredFeatureGates[gate] = isEnabledByDefault
 }
 
 // LoadRawConfig parses the raw configuration bytes, applies initial defaults, and extracts feature gates.
@@ -132,7 +134,11 @@ func LoadRawConfig(configBytes []byte, logger logr.Logger) (*configapi.EndpointP
 		return nil, nil, fmt.Errorf("feature gate validation failed: %w", err)
 	}
 
-	featureConfig := loadFeatureConfig(rawConfig.FeatureGates)
+	featureConfig, err := loadFeatureConfig(rawConfig.FeatureGates)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to load feature gates: %w", err)
+	}
+
 	return rawConfig, featureConfig, nil
 }
 
@@ -162,7 +168,11 @@ func InstantiateAndConfigure(
 		return nil, fmt.Errorf("scheduler config build failed: %w", err)
 	}
 
-	featureGates := loadFeatureConfig(rawConfig.FeatureGates)
+	featureGates, err := loadFeatureConfig(rawConfig.FeatureGates)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load feature gates: %w", err)
+	}
+
 	dataConfig, err := buildDataLayerConfig(rawConfig.DataLayer, handle)
 	if err != nil {
 		return nil, fmt.Errorf("data layer config build failed: %w", err)
@@ -294,17 +304,26 @@ func buildSchedulerConfig(
 	return scheduling.NewSchedulerConfig(profileHandler, profiles), nil
 }
 
-func loadFeatureConfig(gates configapi.FeatureGates) map[string]bool {
+func loadFeatureConfig(gates configapi.FeatureGates) (map[string]bool, error) {
 	registeredFeatureGatesMu.RLock()
 	defer registeredFeatureGatesMu.RUnlock()
 	config := make(map[string]bool, len(registeredFeatureGates))
-	for gate := range registeredFeatureGates {
-		config[gate] = false
+	for gate, defaultValue := range registeredFeatureGates {
+		config[gate] = defaultValue
 	}
 	for _, gate := range gates {
-		config[gate] = true
+		value := true
+		parts := strings.Split(gate, "=")
+		if len(parts) > 1 {
+			var err error
+			value, err = strconv.ParseBool(strings.TrimSpace(strings.ToLower(parts[1])))
+			if err != nil {
+				return nil, err
+			}
+		}
+		config[parts[0]] = value
 	}
-	return config
+	return config, nil
 }
 
 func buildParserRegistry(rawParserConfigs []configapi.ParserConfig, handle fwkplugin.Handle, logger logr.Logger) (*handlers.ParserRegistry, error) {

pkg/epp/config/loader/configloader_test.go

@@ -32,7 +32,6 @@ import (
 	configapi "github.com/llm-d/llm-d-router/apix/config/v1alpha1"
 	"github.com/llm-d/llm-d-router/pkg/common/observability/logging"
 	"github.com/llm-d/llm-d-router/pkg/epp/config"
-	"github.com/llm-d/llm-d-router/pkg/epp/datalayer"
 	"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol"
 	"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol/registry"
 	fwkdl "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/datalayer"
@@ -67,6 +66,8 @@ const (
 	testProfileHandler = "test-profile-handler"
 	testSourceType     = "test-source"
 	testExtractorType  = "test-extractor"
+
+	testFeatureGate = "test-feature-gate"
 )
 
 // --- Test: Phase 1 (Raw Loading & Static Defaults) ---
@@ -75,19 +76,20 @@ func TestLoadRawConfiguration(t *testing.T) {
 	t.Parallel()
 
 	// Register known feature gates for validation.
-	RegisterFeatureGate(datalayer.ExperimentalDatalayerFeatureGate)
-	RegisterFeatureGate(flowcontrol.FeatureGate)
+	RegisterFeatureGate(testFeatureGate, true)
+	RegisterFeatureGate(flowcontrol.FeatureGate, false)
 
 	queueScorerWeight := 2.0
 	kvCacheUtilizationScorerWeight := 2.0
 	prefixCacheScorerWeight := 3.0
 
 	tests := []struct {
-		name       string
-		configText string
-		want       *configapi.EndpointPickerConfig
-		wantErr    bool
-		deprecated bool
+		name         string
+		configText   string
+		want         *configapi.EndpointPickerConfig
+		wantFeatures map[string]bool
+		wantErr      bool
+		deprecated   bool
 	}{
 		{
 			name:       "Success - Full Configuration",
@@ -114,7 +116,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 					},
 				},
 				FeatureGates: configapi.FeatureGates{
-					datalayer.ExperimentalDatalayerFeatureGate,
+					testFeatureGate,
 					flowcontrol.FeatureGate,
 				},
 				FlowControl: &configapi.FlowControlConfig{
@@ -123,6 +125,10 @@ func TestLoadRawConfiguration(t *testing.T) {
 					},
 				},
 			},
+			wantFeatures: map[string]bool{
+				testFeatureGate:         true,
+				flowcontrol.FeatureGate: true,
+			},
 			wantErr:    false,
 			deprecated: false,
 		},
@@ -151,7 +157,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 					},
 				},
 				FeatureGates: configapi.FeatureGates{
-					datalayer.ExperimentalDatalayerFeatureGate,
+					testFeatureGate,
 					flowcontrol.FeatureGate,
 				},
 				FlowControl: &configapi.FlowControlConfig{
@@ -174,7 +180,13 @@ func TestLoadRawConfiguration(t *testing.T) {
 				Plugins: []configapi.PluginSpec{
 					{Name: "test1", Type: testPluginType, Parameters: json.RawMessage(`{"threshold":10}`)},
 				},
-				FeatureGates: configapi.FeatureGates{},
+				FeatureGates: configapi.FeatureGates{
+					testFeatureGate + "=false",
+				},
+			},
+			wantFeatures: map[string]bool{
+				testFeatureGate:         false,
+				flowcontrol.FeatureGate: false,
 			},
 			wantErr:    false,
 			deprecated: false,
@@ -240,6 +252,10 @@ func TestLoadRawConfiguration(t *testing.T) {
 					},
 				},
 			},
+			wantFeatures: map[string]bool{
+				testFeatureGate:         true,
+				flowcontrol.FeatureGate: false,
+			},
 			wantErr:    false,
 			deprecated: false,
 		},
@@ -322,6 +338,12 @@ func TestLoadRawConfiguration(t *testing.T) {
 			wantErr:    true,
 			deprecated: false,
 		},
+		{
+			name:       "Error - Bad Feature Gate",
+			configText: errorBadFeatureGateText,
+			wantErr:    true,
+			deprecated: false,
+		},
 	}
 
 	for _, tc := range tests {
@@ -330,7 +352,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 			writer := &strings.Builder{}
 			logger := logging.NewTestLoggerWithWriter(writer)
 
-			got, _, err := LoadRawConfig([]byte(tc.configText), logger)
+			got, featureGates, err := LoadRawConfig([]byte(tc.configText), logger)
 
 			if tc.wantErr {
 				require.Error(t, err, "Expected LoadRawConfig to fail")
@@ -340,6 +362,11 @@ func TestLoadRawConfiguration(t *testing.T) {
 			diff := cmp.Diff(tc.want, got)
 			require.Empty(t, diff, "Config mismatch (-want +got):\n%s", diff)
 
+			if tc.wantFeatures != nil {
+				diff = cmp.Diff(tc.wantFeatures, featureGates)
+				require.Empty(t, diff, "Config feature gates mismatch (-want +got):\n%s", diff)
+			}
+
 			if strings.Contains(writer.String(), "deprecated") {
 				require.True(t, tc.deprecated, "Deprecated configuration wasn't marked as deprecated")
 			} else {
@@ -355,8 +382,8 @@ func TestInstantiateAndConfigure(t *testing.T) {
 	// Not parallel because it modifies global plugin registry.
 	registerTestPlugins(t)
 
-	RegisterFeatureGate(datalayer.ExperimentalDatalayerFeatureGate)
-	RegisterFeatureGate(flowcontrol.FeatureGate)
+	RegisterFeatureGate(testFeatureGate, true)
+	RegisterFeatureGate(flowcontrol.FeatureGate, false)
 
 	tests := []struct {
 		name       string

pkg/epp/config/loader/testdata_test.go

@@ -44,7 +44,7 @@ schedulingProfiles:
     weight: 50
   - pluginRef: testPicker
 featureGates:
-- dataLayer
+- test-feature-gate
 - flowControl
 flowControl:
   saturationDetector:
@@ -76,7 +76,7 @@ schedulingProfiles:
     weight: 50
   - pluginRef: testPicker
 featureGates:
-- dataLayer
+- test-feature-gate
 - flowControl
 flowControl:
   saturationDetector:
@@ -93,6 +93,8 @@ plugins:
   type: test-plugin
   parameters:
     threshold: 10
+featureGates:
+- test-feature-gate=false
 `
 
 // successSchedulerConfigText represents a complex scheduler setup.
@@ -124,7 +126,7 @@ dataLayer:
     extractors:
     - pluginRef: testExtractor
 featureGates:
-- dataLayer
+- test-feature-gate
 - flowControl
 `
 
@@ -360,6 +362,19 @@ featureGates:
 - unknown-gate
 `
 
+// errorBadFeatureGateText includes a feature gate with an invalid value
+const errorBadFeatureGateText = `
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+- name: test1
+  type: test-plugin
+  parameters:
+    threshold: 10
+featureGates:
+- flowControl=qwerty
+`
+
 // --- Invalid Configurations (Logical/Architectural) ---
 
 // errorNoProfileNameText is missing the required profile name.
@@ -585,7 +600,7 @@ dataLayer:
     extractors:
     - pluginRef: testExtractor
 featureGates:
-- dataLayer
+- test-feature-gate
 `
 
 // errorBadSourceReferenceText has a bad DataSource plugin reference
@@ -605,7 +620,7 @@ dataLayer:
   sources:
   - pluginRef: test-one
 featureGates:
-- dataLayer
+- test-feature-gate
 - flowControl
 `
 
@@ -629,7 +644,7 @@ dataLayer:
     extractors:
     - test-one
 featureGates:
-- dataLayer
+- test-feature-gate
 - flowControl
 `
 

pkg/epp/config/loader/validation.go

@@ -19,6 +19,8 @@ package loader
 import (
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"k8s.io/apimachinery/pkg/util/sets"
 
@@ -120,8 +122,15 @@ func validateFeatureGates(gates configapi.FeatureGates) error {
 	registeredFeatureGatesMu.RLock()
 	defer registeredFeatureGatesMu.RUnlock()
 	for _, gate := range gates {
-		if !registeredFeatureGates.Has(gate) {
-			return fmt.Errorf("feature gate '%s' is unknown or unregistered", gate)
+		parts := strings.Split(gate, "=")
+		if _, ok := registeredFeatureGates[parts[0]]; !ok {
+			return fmt.Errorf("feature gate '%s' is unknown or unregistered", parts[0])
+		}
+		if len(parts) > 1 {
+			_, err := strconv.ParseBool(strings.ToLower(strings.TrimSpace(parts[1])))
+			if err != nil {
+				return fmt.Errorf("%s is not a valid value for the feature gate %s (error: %w)", parts[1], parts[0], err)
+			}
 		}
 	}
 

pkg/epp/datalayer/factory.go

@@ -22,12 +22,6 @@ import (
 	fwkdl "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/datalayer"
 )
 
-const (
-	// ExperimentalDatalayerFeatureGate is deprecated. The data layer is now enabled by default.
-	// This gate is a no-op and will be removed in a future version.
-	ExperimentalDatalayerFeatureGate = "dataLayer"
-)
-
 // PoolInfo represents the DataStore information needed for endpoints.
 type PoolInfo interface {
 	PoolGet() (*EndpointPool, error)

test/testdata/configloader_1_test.yaml

@@ -27,7 +27,7 @@ dataLayer:
     extractors:
     - pluginRef: test-extractor
 featureGates:
-- dataLayer
+- test-feature-gate
 flowControl:
   saturationDetector:
     metricsStalenessThreshold: 150ms