Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions pkg/epp/config/loader/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,9 +292,30 @@ func ensureSaturationDetector(
}
}
}

if sd, ok := allPlugins[sdConfig.PluginRef]; ok {
if _, isFilter := sd.(fwksched.Filter); isFilter {
injectFilterIntoProfiles(cfg.SchedulingProfiles, sdConfig.PluginRef)
Comment thread
RishabhSaini marked this conversation as resolved.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking design question: this injects the detector into every profile with no opt-out. For multi-profile setups where a profile intentionally excludes saturation filtering, there's no way to decline now. Consistent with the other ensure* defaults, but unlike ensureDataLayer's InjectDefaults escape hatch.

This seems fine for the single-profile case this fixes; just flagging whether per-profile opt-out should be a follow-up.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

filed: #1826

}
}
return nil
}

func injectFilterIntoProfiles(profiles []configapi.SchedulingProfile, pluginRef string) {
for i := range profiles {
found := false
for _, p := range profiles[i].Plugins {
if p.PluginRef == pluginRef {
found = true
break
}
}
if !found {
profiles[i].Plugins = append(profiles[i].Plugins, configapi.SchedulingPlugin{PluginRef: pluginRef})
}
}
}

// ensureDataLayer additively injects the default metrics source and extractor unless opted out.
// Unlike other ensureXxx functions, it checks for explicit opt-out via InjectDefaults and avoids
// double-injection when the metrics source is already present in a user-supplied config.
Expand Down
91 changes: 91 additions & 0 deletions pkg/epp/config/loader/defaults_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,26 @@ import (
"k8s.io/utils/ptr"

configapi "github.com/llm-d/llm-d-router/apix/config/v1alpha1"
fwkdl "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/datalayer"
fwkplugin "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/plugin"
fwksched "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/scheduling"
extractormetrics "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/extractor/metrics"
sourcemetrics "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/source/metrics"
testutils "github.com/llm-d/llm-d-router/test/utils"
)

// mockFilterDetector implements both SaturationDetector and Filter, like the real utilization-detector.
type mockFilterDetector struct{ mockPlugin }

var (
_ fwksched.Filter = &mockFilterDetector{}
)

func (m *mockFilterDetector) Saturation(_ context.Context, _ []fwkdl.Endpoint) float64 { return 0 }
func (m *mockFilterDetector) Filter(_ context.Context, _ *fwksched.InferenceRequest, eps []fwksched.Endpoint) []fwksched.Endpoint {
return eps
}

// metricsPlugins returns an allPlugins map with mock stubs for both default metrics plugins.
// Providing them prevents ensureDataLayer from calling registerDefaultPlugin (which needs the
// global factory registry). The function still injects the DataLayer.Sources entries.
Expand Down Expand Up @@ -120,3 +134,80 @@ func TestEnsureDataLayer(t *testing.T) {
})

}

func TestEnsureSaturationDetector_InjectsFilter(t *testing.T) {
t.Run("detector implementing Filter is injected into profiles", func(t *testing.T) {
w := 2.0
cfg := &configapi.EndpointPickerConfig{
SchedulingProfiles: []configapi.SchedulingProfile{
{Name: "default", Plugins: []configapi.SchedulingPlugin{{PluginRef: "scorer", Weight: &w}}},
},
}
handle := testutils.NewTestHandle(context.Background())

detectorName := "my-detector"
detector := &mockFilterDetector{mockPlugin{t: fwkplugin.TypedName{Type: detectorName, Name: detectorName}}}
handle.AddPlugin(detectorName, detector)

allPlugins := handle.GetAllPluginsWithNames()
cfg.FlowControl = &configapi.FlowControlConfig{
SaturationDetector: &configapi.SaturationDetectorConfig{PluginRef: detectorName},
}

err := ensureSaturationDetector(cfg, handle, allPlugins)

require.NoError(t, err)
require.Len(t, cfg.SchedulingProfiles[0].Plugins, 2)
require.Equal(t, detectorName, cfg.SchedulingProfiles[0].Plugins[1].PluginRef)
})

t.Run("detector not implementing Filter is not injected", func(t *testing.T) {
w := 2.0
cfg := &configapi.EndpointPickerConfig{
SchedulingProfiles: []configapi.SchedulingProfile{
{Name: "default", Plugins: []configapi.SchedulingPlugin{{PluginRef: "scorer", Weight: &w}}},
},
}
handle := testutils.NewTestHandle(context.Background())

detectorName := "plain-detector"
detector := &mockSaturationDetector{mockPlugin{t: fwkplugin.TypedName{Type: detectorName, Name: detectorName}}}
handle.AddPlugin(detectorName, detector)

allPlugins := handle.GetAllPluginsWithNames()
cfg.FlowControl = &configapi.FlowControlConfig{
SaturationDetector: &configapi.SaturationDetectorConfig{PluginRef: detectorName},
}

err := ensureSaturationDetector(cfg, handle, allPlugins)

require.NoError(t, err)
require.Len(t, cfg.SchedulingProfiles[0].Plugins, 1, "non-filter detector should not be injected")
})

t.Run("detector already in profile is not duplicated", func(t *testing.T) {
detectorName := "my-detector"
cfg := &configapi.EndpointPickerConfig{
SchedulingProfiles: []configapi.SchedulingProfile{
{Name: "default", Plugins: []configapi.SchedulingPlugin{
{PluginRef: detectorName},
{PluginRef: "picker"},
}},
},
}
handle := testutils.NewTestHandle(context.Background())

detector := &mockFilterDetector{mockPlugin{t: fwkplugin.TypedName{Type: detectorName, Name: detectorName}}}
handle.AddPlugin(detectorName, detector)

allPlugins := handle.GetAllPluginsWithNames()
cfg.FlowControl = &configapi.FlowControlConfig{
SaturationDetector: &configapi.SaturationDetectorConfig{PluginRef: detectorName},
}

err := ensureSaturationDetector(cfg, handle, allPlugins)

require.NoError(t, err)
require.Len(t, cfg.SchedulingProfiles[0].Plugins, 2, "already present, no duplicate")
})
}
Loading