Skip to content

Commit a0c4afc

Browse files
roytmanroytman
committed
merge with the main branch
Signed-off-by: roytman <roytman@il.ibm.com>
2 parents 2176e3b + ed8746b commit a0c4afc

11 files changed

Lines changed: 179 additions & 51 deletions

File tree

‎.github/workflows/pr-kind-label.yaml‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Apply kind labels from PR body
1+
name: Apply kind/area labels and milestone from PR body
22

33
on:
44
pull_request_target:
@@ -23,14 +23,53 @@ jobs:
2323
gh pr view "$PR_NUMBER" --repo "$REPO" --json labels \
2424
-q '.labels[].name | select(startswith("kind/"))' | \
2525
while IFS= read -r label; do
26-
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label"
26+
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" || echo "::warning::Failed to remove label '$label'"
2727
done
2828
2929
# Extract /kind values from PR body and apply labels
30-
printf '%s' "$PR_BODY" | tr -d '\r' | grep '^/kind ' | \
31-
sed 's|^/kind ||' | sort -u | \
30+
printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/kind[[:space:]][[:space:]]*||p' | sed 's|[[:space:]]*$||' | sort -u | \
3231
while IFS= read -r kind; do
3332
[ -z "$kind" ] && continue
3433
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "kind/$kind" || \
3534
echo "::warning::Label 'kind/$kind' not found in repository"
3635
done
36+
37+
- name: Set area labels
38+
env:
39+
GH_TOKEN: ${{ github.token }}
40+
PR_BODY: ${{ github.event.pull_request.body }}
41+
PR_NUMBER: ${{ github.event.pull_request.number }}
42+
REPO: ${{ github.repository }}
43+
run: |
44+
# Absence of a directive is a no-op. Area labels are also set by the
45+
# /area comment command and clearing here would wipe those.
46+
areas=$(printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/area[[:space:]][[:space:]]*||p' | \
47+
sed 's|[[:space:]]*$||' | sed '/^$/d' | sort -u)
48+
[ -z "$areas" ] && exit 0
49+
50+
# Remove existing area/* labels
51+
gh pr view "$PR_NUMBER" --repo "$REPO" --json labels \
52+
-q '.labels[].name | select(startswith("area/"))' | \
53+
while IFS= read -r label; do
54+
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" || echo "::warning::Failed to remove label '$label'"
55+
done
56+
57+
printf '%s\n' "$areas" | \
58+
while IFS= read -r area; do
59+
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "area/$area" || \
60+
echo "::warning::Label 'area/$area' not found in repository"
61+
done
62+
63+
- name: Set milestone
64+
env:
65+
GH_TOKEN: ${{ github.token }}
66+
PR_BODY: ${{ github.event.pull_request.body }}
67+
PR_NUMBER: ${{ github.event.pull_request.number }}
68+
REPO: ${{ github.repository }}
69+
run: |
70+
# Use the last /milestone directive in the PR body
71+
milestone=$(printf '%s' "$PR_BODY" | tr -d '\r' | sed -n 's|^/milestone[[:space:]][[:space:]]*||p' | \
72+
sed 's|[[:space:]]*$||' | sed '/^$/d' | tail -n 1)
73+
[ -z "$milestone" ] && exit 0
74+
gh pr edit "$PR_NUMBER" --repo "$REPO" --milestone "$milestone" || \
75+
echo "::warning::Milestone '$milestone' not found in repository"

‎Makefile.coord.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ endif
116116
# Env vars forwarded into the e2e test container.
117117
E2E_ENV_VARS = COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT \
118118
E2E_GATEWAY_PORT E2E_KEEP_CLUSTER_ON_FAILURE \
119-
E2E_PRINT_COORDINATOR_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
119+
E2E_PRINT_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
120120
BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
121121
ifneq ($(filter command line environment,$(origin NAMESPACE)),)
122122
BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)

‎README.coord.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ kubectl --context kind-e2e-coordinator-tests get pods
341341
|---|---|---|
342342
| `E2E_KEEP_CLUSTER_ON_FAILURE` | `false` | Preserve the Kind cluster when the suite fails |
343343
| `E2E_GATEWAY_PORT` | `30080` | Host port mapped to the gateway NodePort |
344-
| `E2E_PRINT_COORDINATOR_LOGS` | `false` | Print coordinator pod logs during the run |
344+
| `E2E_PRINT_LOGS` | `false` | Print all pod logs (coordinator, EPPs, Envoy, workers) for every spec, not just on failure |
345345
| `CONTAINER_RUNTIME` | `docker` | Container runtime used to load images into Kind (`docker` or `podman`) |
346346
| `EPP_IMAGE` | `ghcr.io/llm-d/llm-d-router-endpoint-picker:dev` | EPP image loaded into the Kind cluster |
347347
| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.2` | vLLM image loaded into the Kind cluster |

‎release-notes.d/unreleased/1911.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
pr: 1911
3+
url: https://github.com/llm-d/llm-d-router/pull/1911
4+
author: capri-xiyue
5+
date: 2026-07-22
6+
---
7+
Support video input in approximate prefix match via user provided video metadata http headers.

‎release-notes.d/unreleased/1956.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
pr: 1956
3+
url: https://github.com/llm-d/llm-d-router/pull/1956
4+
author: kalantar
5+
date: 2026-07-22
6+
---
7+
Add soft-reflective-ceiling-policy: an opt-in usage-limit plugin that gates lower-priority traffic proportionally as pool saturation rises, while never gating the highest-priority band.

‎test/coordinator/e2e/coordinator/configs_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,11 @@ server:
2323
listen_addr: ":8080"
2424
read_timeout: 30s
2525
write_timeout: 120s
26-
shutdown_timeout: 25s
26+
# Recreated per spec behind a suite-lived Envoy; drain fast so a deleted
27+
# coordinator stops serving immediately instead of lingering on a stale
28+
# endpoint the gateway may still route to. 0s is avoided: it makes the
29+
# server Shutdown context expire instantly and the process exit non-zero.
30+
shutdown_timeout: 1s
2731
2832
gateway:
2933
address: "http://envoy.${NAMESPACE}.svc:8081"

‎test/coordinator/e2e/coordinator/coordinator_test.go‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -113,23 +113,14 @@ func runCoordinatorPipeline(body []byte, expectedSteps []string, expectedImages
113113
testutils.DeleteObjects(testConfig, pool, nsName)
114114
})
115115

116-
// Dump coordinator logs on failure, or always when E2E_PRINT_COORDINATOR_LOGS is
117-
// set. Registered second → runs first (LIFO), so the deployment still exists.
116+
// Dump all pod logs (coordinator, EPPs, Envoy, workers) on failure, or always
117+
// when E2E_PRINT_LOGS is set. Registered second → runs first (LIFO), so the
118+
// pods still exist.
118119
ginkgo.DeferCleanup(func() {
119-
if !ginkgo.CurrentSpecReport().Failed() && !printCoordinatorLogs {
120+
if !ginkgo.CurrentSpecReport().Failed() && !printLogs {
120121
return
121122
}
122-
args := []string{"logs", "deployment/llm-d-coordinator",
123-
"-c", "coordinator", "--namespace=" + nsName}
124-
if k8sContext != "" {
125-
args = append(args, "--context="+k8sContext)
126-
}
127-
out, err := exec.Command("kubectl", args...).CombinedOutput()
128-
if err != nil {
129-
fmt.Fprintf(ginkgo.GinkgoWriter, "\n--- coordinator logs (kubectl error: %v) ---\n%s\n---\n", err, string(out))
130-
} else {
131-
fmt.Fprintf(ginkgo.GinkgoWriter, "\n--- coordinator logs ---\n%s\n---\n", string(out))
132-
}
123+
testutils.DumpPodsAndLogs(testConfig, nsName, testutils.WithFullLogs())
133124
})
134125

135126
// Pool first so the EPP can resolve its --pool-name.

‎test/coordinator/e2e/coordinator/e2e_suite_test.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ var (
7676
testConfig *testutils.TestConfig
7777

7878
keepClusterOnFailure = env.GetEnvBool("E2E_KEEP_CLUSTER_ON_FAILURE", false, ginkgo.GinkgoLogr)
79-
printCoordinatorLogs = env.GetEnvBool("E2E_PRINT_COORDINATOR_LOGS", false, ginkgo.GinkgoLogr)
79+
printLogs = env.GetEnvBool("E2E_PRINT_LOGS", false, ginkgo.GinkgoLogr)
8080

8181
containerRuntime = env.GetEnvString("CONTAINER_RUNTIME", "docker", ginkgo.GinkgoLogr)
8282
eppImage = env.GetEnvString("EPP_IMAGE", "ghcr.io/llm-d/llm-d-router-endpoint-picker:dev", ginkgo.GinkgoLogr)
@@ -96,6 +96,7 @@ var (
9696

9797
portForwardSessions []*gexec.Session
9898
rendererObjects []string
99+
stableInfraObjects []string
99100
createdNameSpace bool
100101
)
101102

@@ -128,6 +129,10 @@ var _ = ginkgo.BeforeSuite(func() {
128129
}
129130

130131
rendererObjects = createRenderer()
132+
133+
// Coordinator and EPP Services/RBAC are created once and kept stable across
134+
// specs (see createStableInfra).
135+
createStableInfra()
131136
})
132137

133138
var _ = ginkgo.ReportAfterSuite("cleanup", func(report ginkgo.Report) {
@@ -145,6 +150,9 @@ var _ = ginkgo.ReportAfterSuite("cleanup", func(report ginkgo.Report) {
145150
if len(rendererObjects) > 0 {
146151
testutils.DeleteObjects(testConfig, rendererObjects, nsName)
147152
}
153+
if len(stableInfraObjects) > 0 {
154+
testutils.DeleteObjects(testConfig, stableInfraObjects, nsName)
155+
}
148156
for _, session := range portForwardSessions {
149157
session.Terminate()
150158
}

‎test/coordinator/e2e/coordinator/setup_test.go‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -80,17 +80,19 @@ func createCRDs() {
8080
_ = testutils.CreateObjsFromYaml(testConfig, gieCRDs, "")
8181
}
8282

83-
// createEndPointPicker creates the scheduling ConfigMap and EPP Deployment (plus
84-
// its ServiceAccount, RoleBinding, and Service) from the supplied EPP config and
85-
// waits for the EPP Deployment to become ready. Returns the created object ids
86-
// for cleanup.
83+
// createEndPointPicker creates the scheduling ConfigMap and EPP Deployment from
84+
// the supplied EPP config and waits for the EPP Deployment to become ready. Its
85+
// ServiceAccount, RoleBinding, and Service are created once by createStableInfra.
86+
// Returns the created object ids for cleanup.
8787
func createEndPointPicker(config string) []string {
8888
const cmName = "epp-config"
8989
createEPPConfigMap(cmName, config)
9090

9191
objects := make([]string, 1, 8)
9292
objects[0] = "ConfigMap/" + cmName
93-
objects = append(objects, applyManifest(eppManifest, eppSubstitutions())...)
93+
// The Service, ServiceAccount, and RoleBinding are created once by
94+
// createStableInfra; recreate only the Deployment per spec.
95+
objects = append(objects, applyManifest(eppManifest, eppSubstitutions(), "Service", "ServiceAccount", "RoleBinding")...)
9496
podsInDeploymentsReady(objects)
9597
return objects
9698
}
@@ -144,8 +146,8 @@ func createModelServers(encodeReplicas, prefillReplicas, decodeReplicas int) []s
144146
}
145147

146148
// createCoordinator builds the coordinator ConfigMap from the given pipeline
147-
// config, deploys the coordinator component (Deployment + Service + SA), and
148-
// waits for readiness.
149+
// config, deploys the coordinator Deployment, and waits for readiness. Its
150+
// Service and ServiceAccount are created once by createStableInfra.
149151
func createCoordinator(config string) []string {
150152
nsName := getNamespace()
151153
coordinatorYAML := e2eutil.SubstituteMany([]string{config}, map[string]string{
@@ -167,7 +169,9 @@ func createCoordinator(config string) []string {
167169
objects[0] = "ConfigMap/llm-d-coordinator-config"
168170

169171
docs := e2eutil.RunKustomize(coordinatorComponentDir)
170-
docs = e2eutil.FilterKinds(docs, "ConfigMap")
172+
// Service and ServiceAccount are created once by createStableInfra; recreate
173+
// only the Deployment per spec.
174+
docs = e2eutil.FilterKinds(docs, "ConfigMap", "Service", "ServiceAccount")
171175
docs = e2eutil.SubstituteMany(docs, coordinatorSubstitutions())
172176
docs = e2eutil.RemoveEmptyArgs(docs)
173177
objects = append(objects, testutils.CreateObjsFromYaml(testConfig, docs, nsName)...)
@@ -178,8 +182,11 @@ func createCoordinator(config string) []string {
178182
}
179183

180184
// waitForCoordinatorReady polls /readyz through Envoy until it returns 200,
181-
// catching Envoy's STRICT_DNS resolution lagging behind the per-test Service
182-
// (podsInDeploymentsReady already confirms the coordinator pod itself is ready).
185+
// confirming the freshly recreated coordinator pod is reachable through the
186+
// gateway before the test sends its request. The gateway Service is stable
187+
// across specs (see createStableInfra), so this waits only for the new pod to
188+
// appear behind it. (podsInDeploymentsReady already confirms the coordinator
189+
// pod itself is ready.)
183190
func waitForCoordinatorReady() {
184191
ginkgo.By("Waiting for coordinator to be reachable via gateway")
185192
gomega.Eventually(func() bool {
@@ -212,13 +219,31 @@ func createEPPConfigMap(name, content string) {
212219
}
213220
}
214221

215-
func applyManifest(path string, subs map[string]string) []string {
222+
func applyManifest(path string, subs map[string]string, excludeKinds ...string) []string {
216223
docs := testutils.ReadYaml(path)
217224
docs = e2eutil.SubstituteMany(docs, subs)
218225
docs = e2eutil.RemoveEmptyArgs(docs)
226+
docs = e2eutil.FilterKinds(docs, excludeKinds...)
219227
return testutils.CreateObjsFromYaml(testConfig, docs, getNamespace())
220228
}
221229

230+
// createStableInfra creates the coordinator and EPP Services, ServiceAccounts,
231+
// and RoleBindings once, up front. It appends each created id to
232+
// stableInfraObjects as it goes rather than returning them at the end, so a
233+
// partial failure still leaves the already-created objects tracked for suite
234+
// teardown. Envoy fronts the Services via STRICT_DNS clusters and outlives the
235+
// per-spec workload; recreating a Service each spec would rotate its ClusterIP and
236+
// force Envoy to re-resolve, so only the Deployments behind them churn per spec.
237+
func createStableInfra() {
238+
docs := e2eutil.RunKustomize(coordinatorComponentDir)
239+
docs = e2eutil.FilterKinds(docs, "ConfigMap", "Deployment")
240+
docs = e2eutil.SubstituteMany(docs, coordinatorSubstitutions())
241+
docs = e2eutil.RemoveEmptyArgs(docs)
242+
stableInfraObjects = append(stableInfraObjects, testutils.CreateObjsFromYaml(testConfig, docs, getNamespace())...)
243+
244+
stableInfraObjects = append(stableInfraObjects, applyManifest(eppManifest, eppSubstitutions(), "Deployment")...)
245+
}
246+
222247
func eppSubstitutions() map[string]string {
223248
return map[string]string{
224249
"${EPP_NAME}": eppName,

‎test/coordinator/e2e/coordinator/testdata/envoy.yaml‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,18 @@ data:
8181
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
8282
stat_prefix: http-8081
8383
preserve_external_request_id: true
84+
access_log:
85+
- name: envoy.access_loggers.stdout
86+
typed_config:
87+
"@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
88+
log_format:
89+
text_format_source:
90+
inline_string: "[envoy] %START_TIME(%s.%3f)% %REQ(:METHOD)% %REQ(:PATH)% code=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% cluster=%UPSTREAM_CLUSTER% upstream=%UPSTREAM_HOST% epp-phase=%REQ(EPP-PHASE)% id=%REQ(X-REQUEST-ID)% dur=%DURATION%\n"
91+
access_log_options:
92+
# Emit periodic access-log entries for in-flight streams so a
93+
# request that hangs (never terminates) is still recorded,
94+
# instead of only being logged on completion/disconnect.
95+
access_log_flush_interval: 5s
8496
route_config:
8597
name: vllm
8698
virtual_hosts:
@@ -119,6 +131,14 @@ data:
119131
cluster: coordinator
120132
timeout: 86400s
121133
idle_timeout: 86400s
134+
# Retry a failed upstream connection on a fresh one so a
135+
# transient connect hiccup to the coordinator ClusterIP is
136+
# ridden out instead of hanging the request. connect-failure
137+
# is safe for the non-idempotent POST: the connect never
138+
# completed, so no bytes reached the coordinator.
139+
retry_policy:
140+
retry_on: "connect-failure"
141+
num_retries: 3
122142
upgrade_configs:
123143
- upgrade_type: websocket
124144
typed_per_filter_config:
@@ -193,7 +213,10 @@ data:
193213
# the default (no EPP-Phase) route as the pipeline entry point.
194214
- name: coordinator
195215
type: STRICT_DNS
196-
connect_timeout: 1000s
216+
# Fail a stuck connect quickly (paired with the route retry_policy) so a
217+
# transient inability to open a connection to the coordinator ClusterIP is
218+
# retried on a fresh connection instead of hanging until the client times out.
219+
connect_timeout: 5s
197220
lb_policy: LEAST_REQUEST
198221
load_assignment:
199222
cluster_name: coordinator

0 commit comments

Comments
 (0)