Skip to content

Commit cca9eda

Browse files
asharkhan3101asharkhan3101
authored
Merge branch 'main' into ashar/hybrid-concurrency-detector
2 parents 5a4c8c1 + ed8746b commit cca9eda

46 files changed

Lines changed: 2624 additions & 299 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/pr-kind-label.yaml‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Apply kind labels from PR body
1+
name: Apply kind/area labels and milestone from PR body
22

33
on:
44
pull_request_target:
@@ -23,14 +23,53 @@ jobs:
2323
gh pr view "$PR_NUMBER" --repo "$REPO" --json labels \
2424
-q '.labels[].name | select(startswith("kind/"))' | \
2525
while IFS= read -r label; do
26-
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label"
26+
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" || echo "::warning::Failed to remove label '$label'"
2727
done
2828
2929
# Extract /kind values from PR body and apply labels
30-
printf '%s' "$PR_BODY" | tr -d '\r' | grep '^/kind ' | \
31-
sed 's|^/kind ||' | sort -u | \
30+
printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/kind[[:space:]][[:space:]]*||p' | sed 's|[[:space:]]*$||' | sort -u | \
3231
while IFS= read -r kind; do
3332
[ -z "$kind" ] && continue
3433
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "kind/$kind" || \
3534
echo "::warning::Label 'kind/$kind' not found in repository"
3635
done
36+
37+
- name: Set area labels
38+
env:
39+
GH_TOKEN: ${{ github.token }}
40+
PR_BODY: ${{ github.event.pull_request.body }}
41+
PR_NUMBER: ${{ github.event.pull_request.number }}
42+
REPO: ${{ github.repository }}
43+
run: |
44+
# Absence of a directive is a no-op. Area labels are also set by the
45+
# /area comment command and clearing here would wipe those.
46+
areas=$(printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/area[[:space:]][[:space:]]*||p' | \
47+
sed 's|[[:space:]]*$||' | sed '/^$/d' | sort -u)
48+
[ -z "$areas" ] && exit 0
49+
50+
# Remove existing area/* labels
51+
gh pr view "$PR_NUMBER" --repo "$REPO" --json labels \
52+
-q '.labels[].name | select(startswith("area/"))' | \
53+
while IFS= read -r label; do
54+
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" || echo "::warning::Failed to remove label '$label'"
55+
done
56+
57+
printf '%s\n' "$areas" | \
58+
while IFS= read -r area; do
59+
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "area/$area" || \
60+
echo "::warning::Label 'area/$area' not found in repository"
61+
done
62+
63+
- name: Set milestone
64+
env:
65+
GH_TOKEN: ${{ github.token }}
66+
PR_BODY: ${{ github.event.pull_request.body }}
67+
PR_NUMBER: ${{ github.event.pull_request.number }}
68+
REPO: ${{ github.repository }}
69+
run: |
70+
# Use the last /milestone directive in the PR body
71+
milestone=$(printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/milestone[[:space:]][[:space:]]*||p' | \
72+
sed 's|[[:space:]]*$||' | sed '/^$/d' | tail -n 1)
73+
[ -z "$milestone" ] && exit 0
74+
gh pr edit "$PR_NUMBER" --repo "$REPO" --milestone "$milestone" || \
75+
echo "::warning::Milestone '$milestone' not found in repository"

‎Makefile.coord.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ endif
116116
# Env vars forwarded into the e2e test container.
117117
E2E_ENV_VARS = COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT \
118118
E2E_GATEWAY_PORT E2E_KEEP_CLUSTER_ON_FAILURE \
119-
E2E_PRINT_COORDINATOR_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
119+
E2E_PRINT_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
120120
BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
121121
ifneq ($(filter command line environment,$(origin NAMESPACE)),)
122122
BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)

‎README.coord.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ kubectl --context kind-e2e-coordinator-tests get pods
341341
|---|---|---|
342342
| `E2E_KEEP_CLUSTER_ON_FAILURE` | `false` | Preserve the Kind cluster when the suite fails |
343343
| `E2E_GATEWAY_PORT` | `30080` | Host port mapped to the gateway NodePort |
344-
| `E2E_PRINT_COORDINATOR_LOGS` | `false` | Print coordinator pod logs during the run |
344+
| `E2E_PRINT_LOGS` | `false` | Print all pod logs (coordinator, EPPs, Envoy, workers) for every spec, not just on failure |
345345
| `CONTAINER_RUNTIME` | `docker` | Container runtime used to load images into Kind (`docker` or `podman`) |
346346
| `EPP_IMAGE` | `ghcr.io/llm-d/llm-d-router-endpoint-picker:dev` | EPP image loaded into the Kind cluster |
347347
| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` | vLLM image loaded into the Kind cluster |

‎cmd/epp/runner/feature_gate_test.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,15 @@ package runner
1818

1919
import (
2020
"context"
21+
"os"
2122
"testing"
2223

2324
"github.com/stretchr/testify/require"
2425

26+
"github.com/llm-d/llm-d-router/pkg/epp/datastore"
27+
"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol"
28+
"github.com/llm-d/llm-d-router/pkg/epp/flowcontrol/contracts"
29+
"github.com/llm-d/llm-d-router/pkg/epp/requestcontrol"
2530
runserver "github.com/llm-d/llm-d-router/pkg/epp/server"
2631
)
2732

@@ -50,3 +55,102 @@ featureGates:
5055
require.False(t, r.featureGates[runserver.HAPopulateNonLeaderDatastoreFeatureGate])
5156
})
5257
}
58+
59+
// TestFlowControlFeatureGateAdmissionControlWiring exercises the flowControl feature gate through
60+
// the production config path (parseConfigurationPhaseOne -> parseConfigurationPhaseTwo ->
61+
// initAdmissionControl) in both directions:
62+
// - gate on: the FlowControlAdmissionController is wired, the loader emits a non-nil
63+
// FlowControlConfig, and the flow registry is exposed as the priority band control plane;
64+
// - gate off: the LegacyAdmissionController is wired and no flow control config is built.
65+
//
66+
// The "no featureGates stanza" case reads the gate's registered default from the parsed
67+
// feature-gate map instead of hardcoding it, so this test keeps passing unchanged when the gate
68+
// flips to enabled-by-default (#2104) and pins that the flip actually changes the default wiring.
69+
func TestFlowControlFeatureGateAdmissionControlWiring(t *testing.T) {
70+
// The deprecated ENABLE_EXPERIMENTAL_FLOW_CONTROL_LAYER env var appends the gate to the config
71+
// during phase two; clear it so only the featureGates stanza under test drives the outcome.
72+
if v, ok := os.LookupEnv(enableExperimentalFlowControlLayer); ok {
73+
require.NoError(t, os.Unsetenv(enableExperimentalFlowControlLayer))
74+
t.Cleanup(func() { _ = os.Setenv(enableExperimentalFlowControlLayer, v) })
75+
}
76+
77+
boolPtr := func(b bool) *bool { return &b }
78+
testCases := []struct {
79+
name string
80+
configText string
81+
// wantEnabled nil means "expect whatever default the runner registered for the gate",
82+
// read programmatically from the feature gates parsed out of the stanza-less config.
83+
wantEnabled *bool
84+
}{
85+
{
86+
name: "no featureGates stanza follows the registered default",
87+
configText: `apiVersion: llm-d.ai/v1alpha1
88+
kind: EndpointPickerConfig
89+
`,
90+
wantEnabled: nil,
91+
},
92+
{
93+
name: "flowControl gate enabled wires the flow control admission controller",
94+
configText: `apiVersion: llm-d.ai/v1alpha1
95+
kind: EndpointPickerConfig
96+
featureGates:
97+
- flowControl
98+
`,
99+
wantEnabled: boolPtr(true),
100+
},
101+
{
102+
name: "flowControl=false restores the legacy admission controller",
103+
configText: `apiVersion: llm-d.ai/v1alpha1
104+
kind: EndpointPickerConfig
105+
featureGates:
106+
- flowControl=false
107+
`,
108+
wantEnabled: boolPtr(false),
109+
},
110+
}
111+
112+
for _, tc := range testCases {
113+
t.Run(tc.name, func(t *testing.T) {
114+
ctx, cancel := context.WithCancel(context.Background())
115+
defer cancel()
116+
117+
opts := runserver.NewOptions()
118+
opts.ConfigText = tc.configText
119+
opts.PoolName = "test-pool"
120+
121+
r := NewRunner()
122+
rawConfig, err := r.parseConfigurationPhaseOne(ctx, opts)
123+
require.NoError(t, err)
124+
125+
wantEnabled := r.featureGates[flowcontrol.FeatureGate] // Registered default.
126+
if tc.wantEnabled != nil {
127+
wantEnabled = *tc.wantEnabled
128+
require.Equal(t, wantEnabled, r.featureGates[flowcontrol.FeatureGate],
129+
"the loader should honor the explicit featureGates stanza")
130+
}
131+
132+
ds := datastore.NewDatastore(ctx, r.setupMetricsCollection(opts))
133+
eppConfig, err := r.parseConfigurationPhaseTwo(ctx, rawConfig, ds)
134+
require.NoError(t, err)
135+
136+
endpointCandidates := contracts.EndpointCandidates(
137+
requestcontrol.NewDatastoreEndpointCandidates(ds))
138+
_, admissionController, controlPlane :=
139+
r.initAdmissionControl(ctx, opts, eppConfig, endpointCandidates)
140+
141+
if wantEnabled {
142+
require.IsType(t, &requestcontrol.FlowControlAdmissionController{}, admissionController)
143+
require.NotNil(t, eppConfig.FlowControlConfig,
144+
"the loader should build a flow control config when the gate is on")
145+
require.NotNil(t, controlPlane,
146+
"the flow registry should be exposed as the priority band control plane")
147+
} else {
148+
require.IsType(t, &requestcontrol.LegacyAdmissionController{}, admissionController)
149+
require.Nil(t, eppConfig.FlowControlConfig,
150+
"the loader should not build a flow control config when the gate is off")
151+
require.Nil(t, controlPlane,
152+
"no priority band control plane should exist when the gate is off")
153+
}
154+
})
155+
}
156+
}

‎cmd/epp/runner/runner.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ import (
8585
"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/saturationdetector/utilization"
8686
"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/usagelimits"
8787
"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/usagelimits/priorityholdback"
88+
"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/flowcontrol/usagelimits/softreflectiveceiling"
8889
"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requestcontrol/admitter/latencyslo"
8990
"github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requestcontrol/admitter/probabilisticadmitter"
9091
reqdataprodprefix "github.com/llm-d/llm-d-router/pkg/epp/framework/plugins/requestcontrol/dataproducer/approximateprefix"
@@ -610,6 +611,7 @@ func (r *Runner) registerInTreePlugins() {
610611
fwkplugin.Register(slodeadline.SLODeadlineOrderingPolicyType, slodeadline.SLODeadlineOrderingPolicyFactory)
611612
fwkplugin.Register(usagelimits.StaticUsageLimitPolicyType, usagelimits.StaticPolicyFactory)
612613
fwkplugin.Register(priorityholdback.PolicyType, priorityholdback.PolicyFactory)
614+
fwkplugin.Register(softreflectiveceiling.PolicyType, softreflectiveceiling.Factory)
613615

614616
// Register Request level data producer plugins as defaults for their respective data keys.
615617
fwkplugin.RegisterAsDefaultProducer(reqdataprodprefix.ApproxPrefixCachePluginType, reqdataprodprefix.ApproxPrefixCacheFactory, attrprefix.PrefixCacheMatchInfoDataKey)

‎cmd/epp/runner/test_runner.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package runner
1919
import (
2020
"context"
2121
"encoding/json"
22+
"net"
2223

2324
"k8s.io/client-go/rest"
2425
ctrl "sigs.k8s.io/controller-runtime"
@@ -33,8 +34,9 @@ import (
3334
// NewTestRunnerSetup creates a setup runner dedicated for integration tests. When mockDataSource is
3435
// non-nil, its plugin type is registered as a factory that returns the provided instance, so the
3536
// YAML config can reference it by type name and the runner wires it into the endpoint factory
36-
// automatically.
37-
func NewTestRunnerSetup(ctx context.Context, cfg *rest.Config, opts *runserver.Options, mockDataSource fwkdl.DataSource) (*Runner, ctrl.Manager, datastore.Datastore, error) {
37+
// automatically. When grpcListener is non-nil the ext_proc server serves on it and
38+
// opts.GRPCPort is ignored.
39+
func NewTestRunnerSetup(ctx context.Context, cfg *rest.Config, opts *runserver.Options, mockDataSource fwkdl.DataSource, grpcListener net.Listener) (*Runner, ctrl.Manager, datastore.Datastore, error) {
3840
runner := NewRunner()
3941

4042
if mockDataSource != nil {
@@ -51,13 +53,17 @@ func NewTestRunnerSetup(ctx context.Context, cfg *rest.Config, opts *runserver.O
5153
managerOverrides := []func(*ctrl.Options){
5254
func(o *ctrl.Options) {
5355
o.Controller.SkipNameValidation = &skipNameValidation
56+
// The kernel assigns the port, so the bind cannot lose a race to
57+
// another listener the way a port number chosen in advance can.
58+
o.Metrics.BindAddress = "127.0.0.1:0"
5459
},
5560
}
5661

5762
manager, ds, err := runner.setup(ctx, cfg, opts, managerOverrides)
5863
if err != nil {
5964
return runner, manager, ds, err
6065
}
66+
runner.serverRunner.GrpcListener = grpcListener
6167

6268
// Production runs the ext_proc and health servers on a context that outlives
6369
// the manager (see Runner.runWithGracefulShutdown). Integration tests drive

‎config/charts/README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ helm install my-standalone-router ./config/charts/llm-d-router-standalone \
5252
--set router.proxy.mode=service \
5353
--set router.proxy.replicas=3
5454
```
55+
5556
---
5657

5758
### 2. Gateway Mode (`llm-d-router-gateway`)
@@ -513,17 +514,17 @@ httpRoute:
513514

514515
### Standalone Mode Configuration
515516

516-
Configures EPP to run with a proxy (Envoy proxy or Agentgateway proxy) that intercepts and routes client traffic directly to model servers (exclusive to `llm-d-router-standalone`). The proxy runs as a sidecar in the EPP pod by default, or as a separate horizontally scalable service when `router.proxy.mode=service` (Envoy only).
517+
Configures EPP to run with a proxy (Envoy proxy or Agentgateway proxy) that intercepts and routes client traffic directly to model servers (exclusive to `llm-d-router-standalone`). The proxy runs as a sidecar in the EPP pod by default, or as a separate horizontally scalable service when `router.proxy.mode=service`.
517518

518519
#### Proxy Sidecar Parameters
519520

520521
| **Parameter Name** | **Description** | **Default** |
521522
| :--- | :--- | :--- |
522523
| `router.proxy.enabled` | Enable the proxy (Envoy or Agentgateway) in front of EPP. | `false` |
523524
| `router.proxy.proxyType` | Type of proxy. Options: `[envoy, agentgateway]`. | `envoy` |
524-
| `router.proxy.mode` | Proxy deployment mode. `sidecar` runs the proxy in the EPP pod; `service` runs it as its own horizontally scalable Deployment and Service reaching EPP over the EPP Service (Envoy only). | `sidecar` |
525+
| `router.proxy.mode` | Proxy deployment mode. `sidecar` runs the proxy in the EPP pod; `service` runs it as its own horizontally scalable Deployment and Service reaching EPP over the EPP Service. | `sidecar` |
525526
| `router.proxy.replicas` | Replica count for the proxy Deployment when `mode=service`. | `2` |
526-
| `router.proxy.failOpen` | Whether the proxy passes traffic through (fail-open) when EPP is unreachable. | `true` |
527+
| `router.proxy.failOpen` | Whether the proxy passes traffic through (fail-open) when EPP is unreachable. Applies to `proxyType=envoy` only; Agentgateway exposes no fail-open setting and rejects requests (fails closed) when EPP is unreachable. | `true` |
527528
| `router.proxy.name` | Name of the sidecar container. | `""` |
528529
| `router.proxy.image` | Sidecar container image. | `""` |
529530
| `router.proxy.imagePullPolicy` | Sidecar container image pull policy. | `IfNotPresent` |

‎config/charts/llm-d-router-standalone/templates/_validations.tpl‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,6 @@ standalone validations
3333
{{- if not $proxy.enabled -}}
3434
{{- fail ".Values.router.proxy.enabled must be true when .Values.router.proxy.mode=service" -}}
3535
{{- end -}}
36-
{{- $proxyType := default "envoy" ($proxy.proxyType | default "envoy") | lower -}}
37-
{{- if ne $proxyType "envoy" -}}
38-
{{- fail (printf ".Values.router.proxy.mode=service currently supports only proxyType=envoy, got %q" $proxyType) -}}
39-
{{- end -}}
4036
{{- $hasHTTP := false -}}
4137
{{- range $servicePort := (.Values.router.extraServicePorts | default (list)) -}}
4238
{{- if eq (toString (index $servicePort "name")) "http" -}}
@@ -62,7 +58,7 @@ standalone validations
6258
{{- $listenerPort := include "llm-d-router.standaloneProxyListenerPort" . -}}
6359
{{- $flags := .Values.router.epp.flags | default dict -}}
6460
{{- if and (hasKey $flags "secure-serving") (ne (toString (index $flags "secure-serving")) "false") -}}
65-
{{- fail ".Values.router.epp.flags.secure-serving must be false when proxyType=agentgateway; standalone agentgateway uses plaintext gRPC to EPP over localhost" -}}
61+
{{- fail ".Values.router.epp.flags.secure-serving must be false when proxyType=agentgateway; standalone agentgateway uses plaintext gRPC to EPP (over loopback in sidecar mode, or the EPP Service in service mode)" -}}
6662
{{- end -}}
6763
{{- end -}}
6864
{{- end -}}

‎config/charts/routerlib/templates/_helpers.tpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ binds:
415415
policies:
416416
inferenceRouting:
417417
endpointPicker:
418-
host: {{ printf "127.0.0.1:%v" (.Values.router.epp.extProcPort | default 9002) | quote }}
418+
host: {{ printf "%s:%v" (include "llm-d-router.proxy.extProcHost" .) (.Values.router.epp.extProcPort | default 9002) | quote }}
419419
destinationMode: passthrough
420420
services:
421421
- name: {{ $serviceName | quote }}

‎config/charts/routerlib/templates/_proxy-deployment.yaml‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
{{- define "llm-d-epp.proxy-deployment" -}}
22
{{- $proxy := include "llm-d-router.proxy" . | fromYaml | default dict -}}
3+
{{- $proxyType := include "llm-d-router.proxyType" . | trim -}}
4+
{{- $proxyConfigMap := index $proxy "configMap" | default dict -}}
5+
{{- $proxyConfigMapName := index $proxyConfigMap "name" | default "" -}}
36
{{- if and $proxy.enabled (eq (include "llm-d-router.proxyMode" .) "service") -}}
47
apiVersion: apps/v1
58
kind: Deployment
@@ -62,14 +65,31 @@ spec:
6265
resources:
6366
{{- toYaml . | nindent 12 }}
6467
{{- end }}
65-
{{- with $proxy.volumeMounts }}
68+
{{- if or (and (eq $proxyType "agentgateway") $proxyConfigMapName) $proxy.volumeMounts }}
6669
volumeMounts:
70+
{{- if and (eq $proxyType "agentgateway") $proxyConfigMapName }}
71+
- name: agentgateway-config-template
72+
mountPath: /config
73+
readOnly: true
74+
{{- end }}
75+
{{- with $proxy.volumeMounts }}
6776
{{- toYaml . | nindent 12 }}
77+
{{- end }}
6878
{{- end }}
69-
{{- with $proxy.volumes }}
79+
{{- if or (and (eq $proxyType "agentgateway") $proxyConfigMapName) $proxy.volumes }}
7080
volumes:
81+
{{- if and (eq $proxyType "agentgateway") $proxyConfigMapName }}
82+
- name: agentgateway-config-template
83+
configMap:
84+
name: {{ $proxyConfigMapName }}
85+
items:
86+
- key: config.yaml
87+
path: config.yaml
88+
{{- end }}
89+
{{- with $proxy.volumes }}
7190
{{- tpl (toYaml .) $ | nindent 8 }}
7291
{{- end }}
92+
{{- end }}
7393
---
7494
{{- end -}}
7595
{{- end -}}

0 commit comments

Comments
 (0)