feat: enable the flowControl feature gate by default

Flow control becomes the default admission path. Under saturation, requests
now queue in the EPP and dispatch by priority as capacity frees, rather than
piling into each model server's local queue; queue wait is bounded by a TTL.
Sheddable (negative-priority) requests buffer under the same TTL and capacity
rules instead of being rejected immediately. Opt out with
featureGates: ["flowControl=false"].

- Flip the registered default for the flowControl gate to true and drop the
  "experimental" wording from the admission-control init logs.
- Warn when a flowControl config section is present while the gate is
  explicitly disabled; everything in it except saturationDetector (which the
  legacy admission path also uses) is silently ignored otherwise.
- Note that ENABLE_EXPERIMENTAL_FLOW_CONTROL_LAYER is now a no-op in its
  deprecation log and comment (removal tracked with #2065).
- Flip the loader tests' registered gate default and pin the hermetic
  "do not shed" cases to the explicit flowControl=false opt-out, which they
  were implicitly relying on; keep the lora-affinity case on the default
  (gate-on) path by using pod queue depths below the saturation threshold.
- Document the default, the opt-out, the metrics scrape health dependency,
  and the tuning knobs in the flow control README and metrics docs.

Part of #1187. Fixes #2104.

Signed-off-by: Luke Van Drie <lukevandrie@google.com>

Author: LukeAVanDrie

cmd/epp/runner/runner.go

@@ -146,7 +146,8 @@ import (
 const (
 	// enableExperimentalFlowControlLayer defines the environment variable used as a feature flag for the pluggable flow
 	// control layer.
-	// DEPRECATION NOTICE - this env var will be removed in the next version as we switch to configuring the EPP using FeatureGates in the config file.
+	// DEPRECATION NOTICE - the flowControl feature gate is enabled by default, so this env var is a no-op; it will be
+	// removed in a future release. Use featureGates in the config file (e.g., "flowControl=false") instead.
 	enableExperimentalFlowControlLayer = "ENABLE_EXPERIMENTAL_FLOW_CONTROL_LAYER"
 )
 
@@ -683,7 +684,7 @@ func (r *Runner) parseConfigurationPhaseOne(ctx context.Context, opts *runserver
 		}
 	}
 
-	loader.RegisterFeatureGate(flowcontrol.FeatureGate, false)
+	loader.RegisterFeatureGate(flowcontrol.FeatureGate, true)
 	loader.RegisterFeatureGate(runserver.HAPopulateNonLeaderDatastoreFeatureGate, true)
 
 	r.registerInTreePlugins()
@@ -765,7 +766,7 @@ func (r *Runner) parseConfigurationPhaseTwo(ctx context.Context, rawConfig *conf
 
 func applyDeprecatedEnvFeatureGate(envVar, featureName, featureGate string, rawConfig *configapi.EndpointPickerConfig) {
 	if _, ok := os.LookupEnv(envVar); ok {
-		setupLog.Info(fmt.Sprintf("Enabling the experimental %s using environment variables is deprecated and will be removed in next version", featureName))
+		setupLog.Info(fmt.Sprintf("The %s environment variable is deprecated and will be removed in a future release; the %s is enabled by default and is configured via featureGates in the config file", envVar, featureName))
 		if env.GetEnvBool(envVar, false, setupLog) {
 			if rawConfig.FeatureGates == nil {
 				rawConfig.FeatureGates = make(configapi.FeatureGates, 0)
@@ -890,13 +891,13 @@ func (r *Runner) initAdmissionControl(
 	endpointCandidates contracts.EndpointCandidates,
 ) (contracts.EndpointCandidates, requestcontrol.AdmissionController, contracts.PriorityBandControlPlane) {
 	if !r.featureGates[flowcontrol.FeatureGate] {
-		setupLog.Info("Experimental Flow Control layer is disabled, using legacy admission control")
+		setupLog.Info("Flow Control layer is disabled via the flowControl feature gate, using legacy admission control")
 		return endpointCandidates,
 			requestcontrol.NewLegacyAdmissionController(eppConfig.SaturationDetector, endpointCandidates),
 			nil
 	}
 	endpointCandidates = requestcontrol.NewCachedEndpointCandidates(ctx, endpointCandidates, 50*time.Millisecond)
-	setupLog.Info("Initializing experimental Flow Control layer")
+	setupLog.Info("Initializing Flow Control layer")
 	registry := fcregistry.NewFlowRegistry(eppConfig.FlowControlConfig.Registry, setupLog)
 	fc := fccontroller.NewFlowController(
 		ctx,

docs/metrics.md

@@ -204,7 +204,8 @@ are under `llm_d_epp_`; each has a deprecated `llm_d_inference_scheduler_*` twin
 
 ### Flow control
 
-Exposed when the `flowControl` feature gate is enabled.
+Exposed when the `flowControl` feature gate is enabled, which is the default; opting out via
+`featureGates: ["flowControl=false"]` removes these series.
 
 #### `flow_control_request_queue_duration_seconds`
 

pkg/epp/config/loader/configloader.go

@@ -192,6 +192,8 @@ func InstantiateAndConfigure(
 		if err != nil {
 			return nil, fmt.Errorf("failed to load flow control config: %w", err)
 		}
+	} else if flowControlSettingsConfigured(rawConfig.FlowControl) {
+		logger.Info("WARNING: the flowControl config section is set but the flowControl feature gate is disabled; its settings (other than saturationDetector) are ignored")
 	}
 
 	parserRegistry, err := buildParserRegistry(rawConfig.RequestHandler.Parsers, handle, logger)
@@ -217,6 +219,19 @@ func InstantiateAndConfigure(
 	}, nil
 }
 
+// flowControlSettingsConfigured reports whether the flowControl config section carries settings
+// beyond the saturation detector. The saturation detector is honored by the legacy admission path
+// even when the flowControl feature gate is disabled, so it alone does not indicate ignored
+// configuration.
+func flowControlSettingsConfigured(fc *configapi.FlowControlConfig) bool {
+	if fc == nil {
+		return false
+	}
+	return fc.MaxBytes != nil || fc.MaxRequests != nil || fc.DefaultRequestTTL != nil ||
+		fc.DefaultPriorityBand != nil || fc.DefaultNegativePriorityBand != nil ||
+		len(fc.PriorityBands) > 0 || fc.UsageLimitPolicyPluginRef != ""
+}
+
 func decodeRawConfig(configBytes []byte) (*configapi.EndpointPickerConfig, error) {
 	cfg := &configapi.EndpointPickerConfig{}
 	codecs := serializer.NewCodecFactory(scheme, serializer.EnableStrict)

pkg/epp/config/loader/configloader_test.go

@@ -80,7 +80,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 
 	// Register known feature gates for validation.
 	RegisterFeatureGate(testFeatureGate, true)
-	RegisterFeatureGate(flowcontrol.FeatureGate, false)
+	RegisterFeatureGate(flowcontrol.FeatureGate, true)
 
 	queueScorerWeight := 2.0
 	kvCacheUtilizationScorerWeight := 2.0
@@ -189,7 +189,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 			},
 			wantFeatures: map[string]bool{
 				testFeatureGate:         false,
-				flowcontrol.FeatureGate: false,
+				flowcontrol.FeatureGate: true,
 			},
 			wantErr:    false,
 			deprecated: false,
@@ -257,7 +257,7 @@ func TestLoadRawConfiguration(t *testing.T) {
 			},
 			wantFeatures: map[string]bool{
 				testFeatureGate:         true,
-				flowcontrol.FeatureGate: false,
+				flowcontrol.FeatureGate: true,
 			},
 			wantErr:    false,
 			deprecated: false,
@@ -444,7 +444,7 @@ func TestInstantiateAndConfigure(t *testing.T) {
 	registerTestPlugins(t)
 
 	RegisterFeatureGate(testFeatureGate, true)
-	RegisterFeatureGate(flowcontrol.FeatureGate, false)
+	RegisterFeatureGate(flowcontrol.FeatureGate, true)
 
 	tests := []struct {
 		name       string
@@ -561,7 +561,7 @@ func TestInstantiateAndConfigure(t *testing.T) {
 			},
 		},
 		{
-			name:       "Ignored - Flow Control Config Present but FeatureGate Missing",
+			name:       "Ignored - Flow Control Config Present but FeatureGate Disabled",
 			configText: successflowControlConfigDisabledText,
 			wantErr:    false,
 			validate: func(t *testing.T, handle fwkplugin.Handle, rawCfg *configapi.EndpointPickerConfig, cfg *config.Config) {
@@ -850,6 +850,27 @@ func TestInstantiateAndConfigure(t *testing.T) {
 	}
 }
 
+// TestFlowControlConfigIgnoredWarning verifies that a flowControl config section combined with a
+// disabled flowControl feature gate logs a warning that the settings are ignored.
+func TestFlowControlConfigIgnoredWarning(t *testing.T) {
+	// Not parallel because it modifies the global plugin registry.
+	registerTestPlugins(t)
+	RegisterFeatureGate(flowcontrol.FeatureGate, true)
+
+	writer := &strings.Builder{}
+	logger := logging.NewTestLoggerWithWriter(writer)
+
+	rawConfig, _, err := LoadRawConfig([]byte(successflowControlConfigDisabledText), logger)
+	require.NoError(t, err)
+
+	handle := testutils.NewTestHandle(context.Background())
+	cfg, err := InstantiateAndConfigure(rawConfig, handle, logger)
+	require.NoError(t, err)
+	require.Nil(t, cfg.FlowControlConfig, "flow control config should not be built when the gate is disabled")
+	require.Contains(t, writer.String(), "flowControl feature gate is disabled",
+		"the ignored flowControl section should be called out in the logs")
+}
+
 // TestBuildDataLayerConfigEmptySourcesWarning verifies that an empty sources list
 // logs a warning but does not return an error.
 func TestBuildDataLayerConfigEmptySourcesWarning(t *testing.T) {

pkg/epp/config/loader/testdata_test.go

@@ -274,7 +274,7 @@ schedulingProfiles:
 - name: default
   plugins:
   - pluginRef: maxScore
-featureGates: [] # Explicitly empty
+featureGates: ["flowControl=false"] # Explicit opt-out; the gate is enabled by default.
 flowControl:
   maxBytes: "1024"
 `

pkg/epp/flowcontrol/README.md

@@ -30,6 +30,33 @@ between the Routing and Scheduling layers. It decides *if* and *when* a request,
 mechanism for managing diverse SLOs, ensuring fairness among competing workloads, and maintaining system stability under
 high load.
 
+### Enabling, disabling, and tuning
+
+Flow control is enabled by default via the `flowControl` feature gate. With it on, requests queue
+under saturation and are admitted by priority instead of being rejected outright; sheddable
+(negative-priority) requests buffer until TTL or capacity limits are hit rather than shedding
+immediately on saturation. To restore the legacy saturation-only admission behavior, opt out in the
+EPP config:
+
+```yaml
+featureGates: ["flowControl=false"]
+```
+
+Setting a `flowControl:` config section while the gate is disabled logs a warning: everything in it
+except `saturationDetector` (which the legacy admission path also uses) is ignored.
+
+Operationally, dispatch decisions depend on fresh endpoint metrics: saturation detection reads the
+scraped model-server metrics, so a broken or stale scrape path degrades or halts dispatch. Keep
+metrics scrape health monitored when running with flow control on.
+
+Tuning knobs, all under the `flowControl:` config section:
+
+* Per-band `maxRequests` / `maxBytes` — lower these to shed excess load earlier instead of queueing
+  it (for example, to approximate the legacy immediate-shed behavior for sheddable traffic).
+* `defaultRequestTTL` — bounds how long a request may wait in queue before it is evicted.
+* The priority-holdback usage-limit policy — gates low-priority traffic earlier as utilization
+  rises; configure it via `usageLimitPolicyPluginRef`.
+
 ### High Level Architecture
 
 The following diagram illustrates the high-level dependency model and request flow for the system. It shows how

test/integration/epp/grpc_test.go

@@ -69,6 +69,8 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) {
 		// requiresCRDs indicates that this test case relies on specific Gateway API CRD features (like
 		// InferenceModelRewrite) which are not available in Standalone runMode without CRD.
 		requiresCRDs bool
+		// configText overrides testConfigWithVllmGRPCParser for this case if non-empty.
+		configText string
 	}{
 		// --- Standard Routing Logic ---
 		{
@@ -114,8 +116,12 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) {
 			},
 		},
 		{
-			name:     "do not shed requests by default",
-			requests: integration.ReqGRPCLLM(logger, "test2", "", integration.GenerateGRPCMethodName),
+			// With the flowControl gate on (the default), a request under full saturation queues
+			// until capacity frees instead of routing immediately, so this case pins the legacy
+			// opt-out path via flowControl=false.
+			name:       "do not shed requests under saturation with flow control opted out",
+			configText: testConfigWithVllmGRPCParser + "\nfeatureGates: [\"flowControl=false\"]\n",
+			requests:   integration.ReqGRPCLLM(logger, "test2", "", integration.GenerateGRPCMethodName),
 			pods: []PodState{
 				P(0, 6, 0.2, "foo", "bar"), // Winner (Lowest saturated)
 				P(1, 0, 0.85, "foo"),
@@ -461,7 +467,11 @@ func TestFullDuplexStreamed_GRPC_KubeInferenceObjectiveRequest(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx := t.Context()
 
-			h := NewTestHarness(ctx, t, WithStandardMode(), WithConfigText(testConfigWithVllmGRPCParser)).WithBaseResources()
+			configText := testConfigWithVllmGRPCParser
+			if tc.configText != "" {
+				configText = tc.configText
+			}
+			h := NewTestHarness(ctx, t, WithStandardMode(), WithConfigText(configText)).WithBaseResources()
 
 			h.WithPods(tc.pods).WaitForSync(len(tc.pods), modelMyModel)
 			if len(tc.pods) > 0 {

test/integration/epp/hermetic_test.go

@@ -145,10 +145,13 @@ func TestFullDuplexStreamed_KubeInferenceObjectiveRequest(t *testing.T) {
 				{
 					name:     "select lora despite higher kv cache (affinity)",
 					requests: integration.ReqLLM(logger, "test3", modelSQLLora, modelSQLLoraTarget),
+					// Queue depth is equal across pods so only KV and affinity differentiate, and
+					// stays below the default saturation queueDepthThreshold (5) so the request
+					// dispatches immediately with flow control on (the default).
 					pods: []PodState{
-						P(0, 10, 0.2, "foo", "bar"),
-						P(1, 10, 0.4, "foo", modelSQLLoraTarget), // Winner (Affinity overrides KV)
-						P(2, 10, 0.3, "foo"),
+						P(0, 4, 0.2, "foo", "bar"),
+						P(1, 4, 0.4, "foo", modelSQLLoraTarget), // Winner (Affinity overrides KV)
+						P(2, 4, 0.3, "foo"),
 					},
 					wantResponses: ExpectRouteTo("192.168.1.2:8000", modelSQLLoraTarget, "test3"),
 					wantMetrics: map[string]string{
@@ -197,8 +200,12 @@ dataLayer:
 					},
 				},
 				{
-					name:     "do not shed requests by default",
-					requests: integration.ReqLLM(logger, "test4", modelSQLLora, modelSQLLoraTarget),
+					// With the flowControl gate on (the default), a request under full saturation
+					// queues until capacity frees instead of routing immediately, so this case pins
+					// the legacy opt-out path via flowControl=false.
+					name:       "do not shed requests under saturation with flow control opted out",
+					configText: testDLConfig + "\nfeatureGates: [\"flowControl=false\"]\n",
+					requests:   integration.ReqLLM(logger, "test4", modelSQLLora, modelSQLLoraTarget),
 					pods: []PodState{
 						P(0, 6, 0.2, "foo", "bar", modelSQLLoraTarget), // Winner (Lowest saturated)
 						P(1, 0, 0.85, "foo"),