Merge branch 'main' into fix/per-prompt-mm-features

Author: albertoperdomo2

.github/workflows/md-link-check.yml

@@ -49,7 +49,7 @@ jobs:
         uses: actions/checkout@v7
 
       - name: Run lychee link checker
-        uses: lycheeverse/lychee-action@v2.8.0
+        uses: lycheeverse/lychee-action@v2.9.0
         with:
           # Exclude release-notes.d/ — fragments are sourced verbatim from
           # PR bodies; let the original PR's link check be authoritative.

.github/workflows/pr-rebase.yaml

@@ -70,7 +70,7 @@ jobs:
 
       - name: Checkout base repo
         if: steps.check-access.outputs.skip != 'true'
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           ref: ${{ steps.pr.outputs.base-ref }}
           fetch-depth: 0

CODEOWNERS

@@ -26,8 +26,8 @@
 /apix/config/                                                             @shmuelk @llm-d/router-maintainers
 
 # Proxy sidecar
-/cmd/pd-sidecar/                                                          @shmuelk @llm-d/router-maintainers
-/pkg/sidecar/                                                             @shmuelk @llm-d/router-maintainers
+/cmd/pd-sidecar/                                                          @shmuelk @roytman @llm-d/router-maintainers
+/pkg/sidecar/                                                             @shmuelk @roytman @llm-d/router-maintainers
 
 # SLO + latency-prediction subsystem
 /pkg/epp/framework/plugins/scheduling/filter/sloheadroomtier/             @kaushikmitr @llm-d/router-maintainers
@@ -47,3 +47,7 @@
 # Request handling
 /pkg/epp/framework/interface/requesthandling/                             @zetxqx @llm-d/router-maintainers
 /pkg/epp/framework/plugins/requesthandling/                               @zetxqx @llm-d/router-maintainers
+
+# Coordinator
+/cmd/coordinator/                                                         @shmuelk @roytman @llm-d/router-maintainers
+/pkg/coordinator/                                                         @shmuelk @roytman @llm-d/router-maintainers

Dockerfile.builder

@@ -63,6 +63,11 @@ ENV ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION}
 # /usr/local/bin.
 RUN GOBIN=/usr/local/bin go install golang.org/x/vuln/cmd/govulncheck@${GOVULNCHECK_VERSION}
 
+# Install the ginkgo CLI. Build-time install ensures runtime invocations under
+# --userns=keep-id / -u <uid> can use the binary without writing to root-owned
+# /usr/local/bin.
+RUN GOBIN=/usr/local/bin go install github.com/onsi/ginkgo/v2/ginkgo@v2.28.3
+
 # Go caches are mounted as volumes at runtime for persistence across image rebuilds.
 # Directories are created with open permissions so non-root users (docker -u) can write.
 ENV GOMODCACHE=/go/pkg/mod

README.md

@@ -2,6 +2,7 @@
 [![Go Reference](https://pkg.go.dev/badge/github.com/llm-d/llm-d-router.svg)](https://pkg.go.dev/github.com/llm-d/llm-d-router)
 [![License](https://img.shields.io/github/license/llm-d/llm-d-router)](/LICENSE)
 [![Join Slack](https://img.shields.io/badge/Join_Slack-blue?logo=slack)](https://llm-d.slack.com/archives/C08SBNRRSBD)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fllm-d%2Fllm-d-router.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fllm-d%2Fllm-d-router?ref=badge_shield)
 
 # llm-d Router
 
@@ -90,3 +91,7 @@ Contributions are welcome!
 [Google Meet]:https://meet.google.com/ozx-goao-cxh
 [Meeting Notes]:https://docs.google.com/document/d/1Pf3x7ZM8nNpU56nt6CzePAOmFZ24NXDeXyaYb565Wq4
 [#sig-router]:https://llm-d.slack.com/?redir=%2Fmessages%2Fsig-router
+
+
+## License
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fllm-d%2Fllm-d-router.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fllm-d%2Fllm-d-router?ref=badge_large)

cmd/coordinator/main.go

@@ -19,7 +19,6 @@ package main
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/http"
 	"os"
 	"os/signal"
@@ -34,8 +33,8 @@ import (
 	"github.com/llm-d/llm-d-router/pkg/coordinator/config"
 	"github.com/llm-d/llm-d-router/pkg/coordinator/gateway"
 	"github.com/llm-d/llm-d-router/pkg/coordinator/pipeline"
+	"github.com/llm-d/llm-d-router/pkg/coordinator/pipeline/builder"
 	"github.com/llm-d/llm-d-router/pkg/coordinator/server"
-	"github.com/llm-d/llm-d-router/pkg/coordinator/steps"
 )
 
 func main() {
@@ -82,7 +81,7 @@ func main() {
 
 	gwClient := gateway.New(cfg.Gateway)
 
-	steps, err := buildPipeline(cfg, gwClient)
+	steps, err := builder.Build(cfg, gwClient)
 	if err != nil {
 		log.Error(err, "failed to build pipeline")
 		os.Exit(1)
@@ -128,53 +127,3 @@ func serveUntilSignal(srv *server.Server, shutdownTimeout time.Duration) error {
 	}
 	return nil
 }
-
-// validatePipeline rejects configurations that cannot work before any step runs.
-// The tokens-in format (use_openai_format=false) sends token IDs that only the
-// render step produces, so it requires a render step in the pipeline.
-func validatePipeline(p config.PipelineConfig) error {
-	if p.UseOpenAIFormat {
-		return nil
-	}
-	for _, s := range p.Steps {
-		if s.Type == steps.RenderStepName {
-			return nil
-		}
-	}
-	return fmt.Errorf("pipeline.use_openai_format=false requires a %q step (the tokens-in format sends token IDs that render produces)", steps.RenderStepName)
-}
-
-func mergeConnectorDefaults(params map[string]any, kvConnector, ecConnector string) map[string]any {
-	out := make(map[string]any, len(params))
-	for k, v := range params {
-		out[k] = v
-	}
-	if _, ok := out[steps.ParamKVConnector]; !ok && kvConnector != "" {
-		out[steps.ParamKVConnector] = kvConnector
-	}
-	if _, ok := out[steps.ParamECConnector]; !ok && ecConnector != "" {
-		out[steps.ParamECConnector] = ecConnector
-	}
-	return out
-}
-
-func buildPipeline(cfg *config.Config, gwClient *gateway.Client) ([]pipeline.Step, error) {
-	if err := validatePipeline(cfg.Pipeline); err != nil {
-		return nil, err
-	}
-
-	var steps []pipeline.Step
-	for _, stepCfg := range cfg.Pipeline.Steps {
-		params := mergeConnectorDefaults(stepCfg.Params, cfg.Pipeline.KVConnector, cfg.Pipeline.ECConnector)
-		if _, ok := params["use_openai_format"]; !ok {
-			params["use_openai_format"] = cfg.Pipeline.UseOpenAIFormat
-		}
-		step, err := pipeline.Build(stepCfg.Type, gwClient, params)
-		if err != nil {
-			return nil, err
-		}
-
-		steps = append(steps, step)
-	}
-	return steps, nil
-}

cmd/epp/runner/runner.go

@@ -355,7 +355,7 @@ func (r *Runner) setup(ctx context.Context, cfg *rest.Config, opts *runserver.Op
 		return nil, nil, err
 	}
 
-	ds, err := setupDatastore(ctx, epf, int32(opts.ModelServerMetricsPort), startCrdReconcilers,
+	ds, err := setupDatastore(ctx, epf, startCrdReconcilers,
 		gknn.Namespace, gknn.Name, opts.EndpointSelector, opts.EndpointTargetPorts)
 	if err != nil {
 		setupLog.Error(err, "Failed to setup datastore")
@@ -520,18 +520,18 @@ func NewEndpointPoolFromOptions(
 	return pool, nil
 }
 
-func setupDatastore(ctx context.Context, epFactory datalayer.EndpointFactory, modelServerMetricsPort int32,
+func setupDatastore(ctx context.Context, epFactory datalayer.EndpointFactory,
 	startCrdReconcilers bool, namespace, name string, endpointSelector labels.Selector, endpointTargetPorts []int) (datastore.Datastore, error) {
 
 	if startCrdReconcilers {
-		return datastore.NewDatastore(ctx, epFactory, modelServerMetricsPort), nil
+		return datastore.NewDatastore(ctx, epFactory), nil
 	}
 	endpointPool, err := NewEndpointPoolFromOptions(namespace, name, endpointSelector, endpointTargetPorts)
 	if err != nil {
 		setupLog.Error(err, "Failed to construct endpoint pool from options")
 		return nil, err
 	}
-	return datastore.NewDatastore(ctx, epFactory, modelServerMetricsPort).WithEndpointPool(endpointPool), nil
+	return datastore.NewDatastore(ctx, epFactory).WithEndpointPool(endpointPool), nil
 }
 
 // registerInTreePlugins registers the factory functions of all known plugins
@@ -904,7 +904,7 @@ func (r *Runner) runWithFileDiscovery(ctx context.Context, opts *runserver.Optio
 		poolName = "epp"
 	}
 	pool := datalayer.NewEndpointPool(namespace, poolName)
-	ds := datastore.NewDatastore(ctx, epf, int32(opts.ModelServerMetricsPort)).WithEndpointPool(pool)
+	ds := datastore.NewDatastore(ctx, epf).WithEndpointPool(pool)
 
 	// On bare metal / Slurm / Ray (or any deployment without the K8s Downward
 	// API), neither --pool-namespace nor the NAMESPACE env var is set, so the

deploy/environments/dev/e2e-infra/services.yaml

@@ -20,22 +20,6 @@ spec:
 ---
 apiVersion: v1
 kind: Service
-metadata:
-  name: e2e-epp-health
-spec:
-  selector:
-    app: e2e-epp
-  ports:
-  - name: health
-    protocol: TCP
-    port: 9003
-    targetPort: 9003
-    nodePort: 30081
-    appProtocol: http2
-  type: NodePort
----
-apiVersion: v1
-kind: Service
 metadata:
   name: e2e-epp-metrics
 spec:

docs/architecture.md

@@ -12,7 +12,7 @@
   - [`SchedulingProfiles` Configuration](#schedulingprofiles-configuration)
   - [Available plugins](#available-plugins)
 - [Metric Scraping](#metric-scraping)
-- [Disaggregated Encode/Prefill/Decode (E/P/D)](#disaggregated-encodeprefilldecodesepd-epd)
+- [Disaggregated Encode/Prefill/Decode (E/P/D)](#disaggregated-encodeprefilldecode-epd)
 - [InferencePool & InferenceModel Design](#inferencepool--inferencemodel-design)
   - [Current Assumptions](#current-assumptions)
 - [References](#references)
@@ -304,6 +304,6 @@ Enable chunked decode via the pd-sidecar flag:
 
 ## References
 
-- [GIE Spec](../README.md#relation-to-gie-igw)
+- [Gateway API Inference Extension](https://github.com/kubernetes-sigs/gateway-api-inference-extension)
 - [Envoy External Processing](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/ext_proc_filter)
 - [EPP Container Sizing Guide](./operations.md)

docs/communication.md

@@ -19,7 +19,7 @@ However, it is relatively new and may contain bugs. The `/v1/chat/completions` f
 - [EPP-Phase Header and Routing](#epp-phase-header-and-routing)
 - [Request Format Configuration](#request-format-configuration)
 - [Completions Requests (/v1/completions)](#completions-requests-v1completions)
-- [Text-Only Requests (no images)](#text-only-requests-no-images)
+- [Text-Only Requests (no images)](#text-only-requests-no-images-v1chatcompletions)
 - [Questions](#questions)
 
 ## Pipeline Overview