|
| 1 | +/* |
| 2 | +Copyright 2026 The Kubernetes Authors. |
| 3 | +
|
| 4 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +you may not use this file except in compliance with the License. |
| 6 | +You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | +Unless required by applicable law or agreed to in writing, software |
| 11 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +See the License for the specific language governing permissions and |
| 14 | +limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package endpointattribute |
| 18 | + |
| 19 | +import ( |
| 20 | + "context" |
| 21 | + "encoding/json" |
| 22 | + "errors" |
| 23 | + "fmt" |
| 24 | + |
| 25 | + "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/plugin" |
| 26 | + "github.com/llm-d/llm-d-router/pkg/epp/framework/interface/scheduling" |
| 27 | + attrmetrics "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/datalayer/attribute/metrics" |
| 28 | +) |
| 29 | + |
| 30 | +const ( |
| 31 | + // EndpointAttributeFilterType is the type of the EndpointAttributeFilter. |
| 32 | + EndpointAttributeFilterType = "endpoint-attribute-filter" |
| 33 | + |
| 34 | + onMissingPass = "Pass" |
| 35 | + onMissingFail = "Fail" |
| 36 | + |
| 37 | + algorithmThreshold = "threshold" |
| 38 | + |
| 39 | + operatorLessThan = "LessThan" |
| 40 | + operatorLessThanOrEqual = "LessThanOrEqual" |
| 41 | + operatorGreaterThan = "GreaterThan" |
| 42 | + operatorGreaterThanOrEqual = "GreaterThanOrEqual" |
| 43 | + operatorEqual = "Equal" |
| 44 | + operatorNotEqual = "NotEqual" |
| 45 | +) |
| 46 | + |
| 47 | +// thresholdParameters keeps endpoints whose attribute value compares true |
| 48 | +// against the configured value. |
| 49 | +type thresholdParameters struct { |
| 50 | + Operator string `json:"operator"` |
| 51 | + Value float64 `json:"value"` |
| 52 | +} |
| 53 | + |
| 54 | +// algorithmParameters selects the filtering algorithm. threshold is the only |
| 55 | +// algorithm currently supported; percentile, topK and range are planned as |
| 56 | +// follow-ups. |
| 57 | +type algorithmParameters struct { |
| 58 | + Type string `json:"type"` |
| 59 | + Threshold *thresholdParameters `json:"threshold,omitempty"` |
| 60 | +} |
| 61 | + |
| 62 | +type parameters struct { |
| 63 | + Attribute string `json:"attribute"` |
| 64 | + // OnMissing decides what happens to endpoints that do not have the |
| 65 | + // attribute: "Pass" keeps them (the default), "Fail" drops them. |
| 66 | + OnMissing string `json:"onMissing"` |
| 67 | + // FallbackOnEmpty returns the unfiltered candidates when every endpoint |
| 68 | + // was filtered out, so the request can still be routed somewhere. |
| 69 | + FallbackOnEmpty bool `json:"fallbackOnEmpty"` |
| 70 | + Algorithm algorithmParameters `json:"algorithm"` |
| 71 | +} |
| 72 | + |
| 73 | +// compile-time type assertion |
| 74 | +var _ scheduling.Filter = &EndpointAttributeFilter{} |
| 75 | + |
| 76 | +// EndpointAttributeFilterFactory defines the factory function for EndpointAttributeFilter. |
| 77 | +func EndpointAttributeFilterFactory(name string, rawParameters *json.Decoder, _ plugin.Handle) (plugin.Plugin, error) { |
| 78 | + var params parameters |
| 79 | + if rawParameters != nil { |
| 80 | + if err := rawParameters.Decode(¶ms); err != nil { |
| 81 | + return nil, fmt.Errorf("failed to parse the parameters of the '%s' filter - %w", EndpointAttributeFilterType, err) |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + return NewEndpointAttributeFilter(name, params) |
| 86 | +} |
| 87 | + |
| 88 | +// NewEndpointAttributeFilter validates the given parameters and returns a new |
| 89 | +// EndpointAttributeFilter with the given name. |
| 90 | +func NewEndpointAttributeFilter(name string, params parameters) (*EndpointAttributeFilter, error) { |
| 91 | + if name == "" { |
| 92 | + name = EndpointAttributeFilterType |
| 93 | + } |
| 94 | + if params.Attribute == "" { |
| 95 | + return nil, errors.New("endpoint attribute filter requires a non-empty attribute") |
| 96 | + } |
| 97 | + switch params.OnMissing { |
| 98 | + case "": |
| 99 | + params.OnMissing = onMissingPass |
| 100 | + case onMissingPass, onMissingFail: |
| 101 | + default: |
| 102 | + return nil, fmt.Errorf("endpoint attribute filter onMissing must be %q or %q, got %q", |
| 103 | + onMissingPass, onMissingFail, params.OnMissing) |
| 104 | + } |
| 105 | + if params.Algorithm.Type != algorithmThreshold { |
| 106 | + return nil, fmt.Errorf("endpoint attribute filter algorithm.type must be %q, got %q", |
| 107 | + algorithmThreshold, params.Algorithm.Type) |
| 108 | + } |
| 109 | + threshold := params.Algorithm.Threshold |
| 110 | + if threshold == nil { |
| 111 | + return nil, fmt.Errorf("endpoint attribute filter requires algorithm.threshold when algorithm.type is %q", |
| 112 | + algorithmThreshold) |
| 113 | + } |
| 114 | + switch threshold.Operator { |
| 115 | + case operatorLessThan, operatorLessThanOrEqual, operatorGreaterThan, |
| 116 | + operatorGreaterThanOrEqual, operatorEqual, operatorNotEqual: |
| 117 | + default: |
| 118 | + return nil, fmt.Errorf("endpoint attribute filter threshold.operator must be one of %q, %q, %q, %q, %q, %q, got %q", |
| 119 | + operatorLessThan, operatorLessThanOrEqual, operatorGreaterThan, |
| 120 | + operatorGreaterThanOrEqual, operatorEqual, operatorNotEqual, threshold.Operator) |
| 121 | + } |
| 122 | + |
| 123 | + return &EndpointAttributeFilter{ |
| 124 | + typedName: plugin.TypedName{Type: EndpointAttributeFilterType, Name: name}, |
| 125 | + attribute: params.Attribute, |
| 126 | + passOnMissing: params.OnMissing == onMissingPass, |
| 127 | + fallbackOnEmpty: params.FallbackOnEmpty, |
| 128 | + threshold: *threshold, |
| 129 | + }, nil |
| 130 | +} |
| 131 | + |
| 132 | +// EndpointAttributeFilter filters candidate endpoints by a single configured |
| 133 | +// numeric endpoint attribute (produced by the custom metrics extraction |
| 134 | +// layer). Endpoints whose attribute value compares true against the |
| 135 | +// configured threshold are kept. |
| 136 | +type EndpointAttributeFilter struct { |
| 137 | + typedName plugin.TypedName |
| 138 | + attribute string |
| 139 | + // passOnMissing keeps endpoints that do not have the attribute instead of |
| 140 | + // dropping them. |
| 141 | + passOnMissing bool |
| 142 | + // fallbackOnEmpty returns the unfiltered candidates when every endpoint |
| 143 | + // was filtered out. |
| 144 | + fallbackOnEmpty bool |
| 145 | + threshold thresholdParameters |
| 146 | +} |
| 147 | + |
| 148 | +// TypedName returns the typed name of the plugin. |
| 149 | +func (f *EndpointAttributeFilter) TypedName() plugin.TypedName { |
| 150 | + return f.typedName |
| 151 | +} |
| 152 | + |
| 153 | +// Consumes returns the list of data that is consumed by the plugin. |
| 154 | +func (f *EndpointAttributeFilter) Consumes() map[string]any { |
| 155 | + return map[string]any{ |
| 156 | + f.attribute: attrmetrics.ScalarMetricValue(0), |
| 157 | + } |
| 158 | +} |
| 159 | + |
| 160 | +// Filter keeps the endpoints whose attribute value satisfies the configured |
| 161 | +// threshold. Endpoints missing the attribute are kept or dropped according to |
| 162 | +// the onMissing policy. When all endpoints are filtered out and |
| 163 | +// fallbackOnEmpty is set, the original candidates are returned. |
| 164 | +func (f *EndpointAttributeFilter) Filter(_ context.Context, _ *scheduling.InferenceRequest, endpoints []scheduling.Endpoint) []scheduling.Endpoint { |
| 165 | + filtered := make([]scheduling.Endpoint, 0, len(endpoints)) |
| 166 | + for _, endpoint := range endpoints { |
| 167 | + value, ok := attrmetrics.ReadScalarMetricValue(endpoint, f.attribute) |
| 168 | + if !ok { |
| 169 | + if f.passOnMissing { |
| 170 | + filtered = append(filtered, endpoint) |
| 171 | + } |
| 172 | + continue |
| 173 | + } |
| 174 | + if f.matches(float64(value)) { |
| 175 | + filtered = append(filtered, endpoint) |
| 176 | + } |
| 177 | + } |
| 178 | + |
| 179 | + if len(filtered) == 0 && f.fallbackOnEmpty { |
| 180 | + return endpoints |
| 181 | + } |
| 182 | + return filtered |
| 183 | +} |
| 184 | + |
| 185 | +// matches reports whether the value satisfies the configured threshold. |
| 186 | +func (f *EndpointAttributeFilter) matches(value float64) bool { |
| 187 | + switch f.threshold.Operator { |
| 188 | + case operatorLessThan: |
| 189 | + return value < f.threshold.Value |
| 190 | + case operatorLessThanOrEqual: |
| 191 | + return value <= f.threshold.Value |
| 192 | + case operatorGreaterThan: |
| 193 | + return value > f.threshold.Value |
| 194 | + case operatorGreaterThanOrEqual: |
| 195 | + return value >= f.threshold.Value |
| 196 | + case operatorEqual: |
| 197 | + return value == f.threshold.Value |
| 198 | + case operatorNotEqual: |
| 199 | + return value != f.threshold.Value |
| 200 | + default: |
| 201 | + // Unreachable: the operator is validated at construction time. |
| 202 | + return false |
| 203 | + } |
| 204 | +} |
0 commit comments