fix: correct the flow control deprecation notice and tuning guidance

ENABLE_EXPERIMENTAL_FLOW_CONTROL_LAYER=false no longer disables the layer: the
variable only ever appended the gate, so a false value falls through to the
registered default, which is now enabled. Log the observed value and the
featureGates opt-out so the operator who set it can see it is ignored.

The 5000 / 1G defaults belong to priorityBands, not to the global flowControl
caps, which default to unlimited; sizing advice that names the global knob sends
operators to the wrong field and implies a bound that is not configured.

The priority-holdback policy lowers admission ceilings, holding gated traffic in
queue. It is not a shedding knob, and listing it beside the two that do shed
reads as though it were.

Also assert the registered gate default, which nothing covered, and the silent
cases of the ignored-settings warning, which guard the saturationDetector
exclusion that keeps it from firing on every legacy-path startup.

Signed-off-by: Luke Van Drie <lukevandrie@google.com>

Author: LukeAVanDrie

cmd/epp/runner/feature_gate_test.go

@@ -127,6 +127,8 @@ featureGates:
 				wantEnabled = *tc.wantEnabled
 				require.Equal(t, wantEnabled, r.featureGates[flowcontrol.FeatureGate],
 					"the loader should honor the explicit featureGates stanza")
+			} else {
+				require.True(t, wantEnabled, "the flowControl gate is registered as enabled by default (#2104)")
 			}
 
 			ds := datastore.NewDatastore(ctx, r.setupMetricsCollection(opts))

cmd/epp/runner/runner.go

@@ -766,7 +766,7 @@ func (r *Runner) parseConfigurationPhaseTwo(ctx context.Context, rawConfig *conf
 
 func applyDeprecatedEnvFeatureGate(envVar, featureName, featureGate string, rawConfig *configapi.EndpointPickerConfig) {
 	if _, ok := os.LookupEnv(envVar); ok {
-		setupLog.Info(fmt.Sprintf("The %s environment variable is deprecated and will be removed in a future release; the %s is enabled by default and is configured via featureGates in the config file", envVar, featureName))
+		setupLog.Info(fmt.Sprintf("The %s environment variable (current value %q) is deprecated and will be removed in a future release; the %s is enabled by default, so this variable no longer disables it. To disable it, set featureGates: [%q] in the config file", envVar, os.Getenv(envVar), featureName, featureGate+"=false"))
 		if env.GetEnvBool(envVar, false, setupLog) {
 			if rawConfig.FeatureGates == nil {
 				rawConfig.FeatureGates = make(configapi.FeatureGates, 0)

docs/operations.md

@@ -21,10 +21,13 @@ The EPP acts as the routing intelligence engine. Its resource usage scales prima
 - **Inflight Requests Impact**: Memory usage increases with the number of concurrent inflight requests and the output (decode) token length.
 - **Flow Control Queues**: With flow control enabled (the default), requests that cannot dispatch
   under saturation are buffered in EPP memory, including their request bodies. The buffered volume
-  is bounded per priority band by `flowControl` `maxRequests` (default 5000) and `maxBytes`
-  (default 1G); budget for the sum of the per-band `maxBytes` limits of the priority levels your
-  traffic actually uses, on top of the inflight-request sizing above. Lower these limits (or set a
-  shorter `defaultRequestTTL`) to trade queueing for earlier shedding.
+  is bounded per priority band by `priorityBands[].maxRequests` (default 5000) and `maxBytes`
+  (default 1G), which `defaultPriorityBand` sets as a template for bands you do not list; budget for
+  the sum of the per-band `maxBytes` limits of the priority levels your traffic actually uses, on top
+  of the inflight-request sizing above. The global `flowControl.maxRequests` / `maxBytes` caps
+  default to unlimited, so set a global `maxBytes` under the container memory limit: at the per-band
+  default, a handful of bands clears the sizing guidance below before any band cap engages. Lower
+  these limits (or set a shorter `defaultRequestTTL`) to trade queueing for earlier shedding.
 - **Sizing Guidelines**:
   - For a request rate of 50 to 100 requests/second with 1k output tokens, EPP requires between **4 GiB and 6 GiB** of memory.
   - For workloads with longer output lengths (such as 5k output tokens), memory usage can reach **20+ GiB** due to the accumulation of state for concurrent inflight requests.

pkg/epp/config/loader/configloader_test.go

@@ -851,24 +851,68 @@ func TestInstantiateAndConfigure(t *testing.T) {
 }
 
 // TestFlowControlConfigIgnoredWarning verifies that a flowControl config section combined with a
-// disabled flowControl feature gate logs a warning that the settings are ignored.
+// disabled flowControl feature gate logs a warning that the settings are ignored, and that the
+// warning stays silent otherwise. The silent cases carry the weight here: ensureSaturationDetector
+// populates FlowControl for every config, so a predicate that ignored its saturationDetector
+// exclusion would warn at every legacy-path startup.
 func TestFlowControlConfigIgnoredWarning(t *testing.T) {
 	// Not parallel because it modifies the global plugin registry.
 	registerTestPlugins(t)
 	RegisterFeatureGate(flowcontrol.FeatureGate, true)
 
-	writer := &strings.Builder{}
-	logger := logging.NewTestLoggerWithWriter(writer)
+	testCases := []struct {
+		name        string
+		configText  string
+		gateEnabled bool
+		wantWarn    bool
+	}{
+		{
+			name:       "settings ignored under an explicit opt-out",
+			configText: successflowControlConfigDisabledText,
+			wantWarn:   true,
+		},
+		{
+			name:       "opt-out with no flowControl section",
+			configText: successFlowControlDisabledNoSectionText,
+		},
+		{
+			name:       "opt-out with only a saturation detector",
+			configText: successFlowControlDisabledSaturationDetectorText,
+		},
+		{
+			name:        "settings honored when the gate is on",
+			configText:  successFlowControlConfigText,
+			gateEnabled: true,
+		},
+	}
 
-	rawConfig, _, err := LoadRawConfig([]byte(successflowControlConfigDisabledText), logger)
-	require.NoError(t, err)
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			writer := &strings.Builder{}
+			logger := logging.NewTestLoggerWithWriter(writer)
 
-	handle := testutils.NewTestHandle(context.Background())
-	cfg, err := InstantiateAndConfigure(rawConfig, handle, logger)
-	require.NoError(t, err)
-	require.Nil(t, cfg.FlowControlConfig, "flow control config should not be built when the gate is disabled")
-	require.Contains(t, writer.String(), "flowControl feature gate is disabled",
-		"the ignored flowControl section should be called out in the logs")
+			rawConfig, _, err := LoadRawConfig([]byte(tc.configText), logger)
+			require.NoError(t, err)
+
+			handle := testutils.NewTestHandle(context.Background())
+			cfg, err := InstantiateAndConfigure(rawConfig, handle, logger)
+			require.NoError(t, err)
+
+			if tc.gateEnabled {
+				require.NotNil(t, cfg.FlowControlConfig, "flow control config should be built when the gate is on")
+			} else {
+				require.Nil(t, cfg.FlowControlConfig, "flow control config should not be built when the gate is disabled")
+			}
+
+			if tc.wantWarn {
+				require.Contains(t, writer.String(), "flowControl feature gate is disabled",
+					"the ignored flowControl section should be called out in the logs")
+			} else {
+				require.NotContains(t, writer.String(), "flowControl feature gate is disabled",
+					"nothing is being ignored, so the warning should stay silent")
+			}
+		})
+	}
 }
 
 // TestBuildDataLayerConfigEmptySourcesWarning verifies that an empty sources list

pkg/epp/config/loader/testdata_test.go

@@ -279,6 +279,38 @@ flowControl:
   maxBytes: "1024"
 `
 
+// successFlowControlDisabledNoSectionText is the plain legacy-path config: an explicit opt-out with
+// nothing for the loader to report as ignored.
+const successFlowControlDisabledNoSectionText = `
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+- name: maxScore
+  type: max-score-picker
+schedulingProfiles:
+- name: default
+  plugins:
+  - pluginRef: maxScore
+featureGates: ["flowControl=false"]
+`
+
+// successFlowControlDisabledSaturationDetectorText pairs an explicit opt-out with a saturation
+// detector, which the legacy admission path honors and so must not be reported as ignored.
+const successFlowControlDisabledSaturationDetectorText = `
+apiVersion: llm-d.ai/v1alpha1
+kind: EndpointPickerConfig
+plugins:
+- name: maxScore
+  type: max-score-picker
+schedulingProfiles:
+- name: default
+  plugins:
+  - pluginRef: maxScore
+featureGates: ["flowControl=false"]
+saturationDetector:
+  pluginRef: utilization-detector
+`
+
 // successComplexFlowControlConfigText tests that Flow Control configuration with custom plugins is correctly loaded.
 const successComplexFlowControlConfigText = `
 apiVersion: llm-d.ai/v1alpha1

pkg/epp/flowcontrol/README.md

@@ -56,11 +56,17 @@ threshold and monitor scrape health when running with flow control on.
 
 Tuning knobs, all under the `flowControl:` config section:
 
-* Per-band `maxRequests` / `maxBytes` — lower these to shed excess load earlier instead of queueing
-  it (for example, to approximate the legacy immediate-shed behavior for sheddable traffic).
-* `defaultRequestTTL` — bounds how long a request may wait in queue before it is evicted.
-* The priority-holdback usage-limit policy — gates low-priority traffic earlier as utilization
-  rises; configure it via `usageLimitPolicyPluginRef`.
+* Per-band `maxRequests` / `maxBytes` — the shedding knobs. Lower them to reject excess load at the
+  queue boundary instead of buffering it (for example, to approximate the legacy immediate-shed
+  behavior for sheddable traffic).
+* `defaultRequestTTL` — the queue-wait budget, and the other way a request is shed. When the pool
+  has no endpoints the queue acts as a scale-from-zero waiting room and requests hold for the full
+  budget, so keep it under the client or gateway deadline unless you want requests to survive a cold
+  start.
+* The priority-holdback usage-limit policy — a gating knob, not a shedding one. It lowers the
+  admission ceiling for low-priority traffic as utilization rises, so that traffic waits in queue
+  rather than being rejected; it sheds only by way of the two limits above. Configure it via
+  `usageLimitPolicyPluginRef`.
 
 ### High Level Architecture