[coordinator] e2e tests: containerize and share deploy components

Signed-off-by: Revital Sur <eres@il.ibm.com>

Author: revit13

Makefile.coord.mk

@@ -1,23 +1,35 @@
 SHELL := /usr/bin/env bash
 
-# Image registry + dev-environment image tags (single source of truth).
-include versions.mk
-
-# Export all dev-env image references so the e2e suite sees them.
-export IMAGE_REGISTRY COORDINATOR_TAG VLLM_SIMULATOR_TAG EPP_TAG
-export COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE
-
 # Defaults
 TARGETOS ?= $(shell command -v go >/dev/null 2>&1 && go env GOOS || uname -s | tr '[:upper:]' '[:lower:]')
 TARGETARCH ?= $(shell command -v go >/dev/null 2>&1 && go env GOARCH || uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/; s/armv7l/arm/')
 PROJECT_NAME ?= llm-d-coordinator
 BUILDER_IMAGE_NAME ?= llm-d-coordinator-builder
 IMAGE_REGISTRY ?= ghcr.io/llm-d
 
+# Image tags
+COORDINATOR_TAG      ?= dev
+VLLM_SIMULATOR_TAG   ?= v0.10.0
+EPP_TAG              ?= dev
+
+# Full image references (derived; override only if you need a non-standard repo)
+COORDINATOR_IMAGE    ?= $(IMAGE_REGISTRY)/llm-d-coordinator:$(COORDINATOR_TAG)
+VLLM_IMAGE           ?= $(IMAGE_REGISTRY)/llm-d-inference-sim:$(VLLM_SIMULATOR_TAG)
+EPP_IMAGE            ?= $(IMAGE_REGISTRY)/llm-d-router-endpoint-picker:$(EPP_TAG)
+
+# vllm-render defaults to the same image as the other simulated vLLM roles; override
+# independently to point it at a real vLLM image (e.g. vllm/vllm-openai-cpu:v0.21.0).
+VLLM_RENDER_IMAGE    ?= $(VLLM_IMAGE)
+VLLM_RENDER_PORT     ?= 8082
+
 # Internal variable mappings for the generic image-build-% target.
 coordinator_IMAGE = $(COORDINATOR_IMAGE)
 epp_IMAGE         = $(EPP_IMAGE)
 
+# Export all dev-env image references so the e2e suite sees them.
+export IMAGE_REGISTRY COORDINATOR_TAG VLLM_SIMULATOR_TAG EPP_TAG
+export COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT
+
 BUILDER_TAG ?= dev
 BUILDER_TAG_BASE ?= $(IMAGE_REGISTRY)/$(BUILDER_IMAGE_NAME)
 export BUILDER_IMAGE ?= $(BUILDER_TAG_BASE):$(BUILDER_TAG)
@@ -56,11 +68,64 @@ endif
 
 BUILDER_RUN_FLAGS = --rm $(BUILDER_USER_FLAGS) \
 	-v $$(pwd):/app:Z -w /app \
-	-v $(GO_MOD_CACHE_VOL):/go/pkg/mod \
-	-v $(GO_BUILD_CACHE_VOL):/go/cache
+	-v $(GO_MOD_CACHE_VOL):/go/pkg/mod:z \
+	-v $(GO_BUILD_CACHE_VOL):/go/cache:z
 
 BUILDER_RUN = $(CONTAINER_RUNTIME) run $(BUILDER_RUN_FLAGS) $(BUILDER_IMAGE) sh -c
 
+# Mount the container runtime socket and set CONTAINER_HOST so podman --remote
+# inside the builder can talk to the host's container runtime.
+ifeq ($(CONTAINER_RUNTIME),podman)
+CONTAINER_SOCK ?= $(or $(shell podman info --format '{{.Host.RemoteSocket.Path}}' 2>/dev/null | sed 's|^unix://||'),/run/podman/podman.sock)
+BUILDER_SOCK_FLAGS = --security-opt label=disable \
+	-v $(CONTAINER_SOCK):$(CONTAINER_SOCK) \
+	-e CONTAINER_HOST=unix://$(CONTAINER_SOCK) \
+	-e DOCKER_HOST=unix://$(CONTAINER_SOCK) \
+	-e CONTAINER_RUNTIME=podman \
+	-e KIND_EXPERIMENTAL_PROVIDER=podman
+else
+CONTAINER_SOCK ?= /var/run/docker.sock
+ifeq ($(TARGETOS),darwin)
+DOCKER_SOCK_GID := $(shell stat -f '%g' $(CONTAINER_SOCK) 2>/dev/null)
+else
+DOCKER_SOCK_GID := $(shell stat -c '%g' $(CONTAINER_SOCK) 2>/dev/null)
+endif
+ifneq ($(DOCKER_SOCK_GID),)
+DOCKER_GROUP_PARAM := --group-add $(DOCKER_SOCK_GID)
+else
+DOCKER_GROUP_PARAM :=
+endif
+BUILDER_SOCK_FLAGS = $(DOCKER_GROUP_PARAM) \
+	-v $(CONTAINER_SOCK):$(CONTAINER_SOCK) \
+	-e DOCKER_HOST=unix://$(CONTAINER_SOCK) \
+	-e CONTAINER_RUNTIME=docker
+endif
+
+# Respect host KUBECONFIG if set; fall back to ~/.kube/config.
+HOST_KUBECONFIG ?= $(or $(KUBECONFIG),$(HOME)/.kube/config)
+
+# When K8S_CONTEXT is set, mount the host kubeconfig so the e2e suite can call
+# config.GetConfigWithContext(K8S_CONTEXT) against an existing cluster instead of
+# creating a new kind cluster.
+ifdef K8S_CONTEXT
+BUILDER_E2E_KUBECONFIG_FLAGS = -v $(HOST_KUBECONFIG):/.kube/config:ro -e KUBECONFIG=/.kube/config
+else
+BUILDER_E2E_KUBECONFIG_FLAGS =
+endif
+
+# Env vars forwarded into the e2e test container.
+E2E_ENV_VARS = COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT \
+               COORDINATOR_PORT E2E_GATEWAY_PORT E2E_KEEP_CLUSTER_ON_FAILURE \
+               E2E_PRINT_COORDINATOR_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
+BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
+ifneq ($(filter command line environment,$(origin NAMESPACE)),)
+BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)
+endif
+
+# E2e tests create their own kind cluster, need host network (for NodePort access)
+# and the container socket (for kind), but not the host kubeconfig.
+BUILDER_E2E_FLAGS = --network=host $(BUILDER_SOCK_FLAGS) $(BUILDER_E2E_ENV_FLAGS) $(BUILDER_E2E_KUBECONFIG_FLAGS)
+
 BUILDER_STAMP = build/.builder.stamp
 
 .PHONY: help
@@ -132,7 +197,9 @@ test-unit: image-build-builder
 
 .PHONY: test-e2e-coordinator
 test-e2e-coordinator: image-build-coordinator image-build-epp image-build-builder image-pull ## Run coordinator e2e tests against a new kind cluster
-	test/coordinator/scripts/run_e2e_coordinator.sh
+	@printf "\033[33;1m==== Running Coordinator End to End Tests ====\033[0m\n"
+	$(CONTAINER_RUNTIME) run $(BUILDER_RUN_FLAGS) $(BUILDER_E2E_FLAGS) \
+		$(BUILDER_IMAGE) test/coordinator/scripts/run_e2e_coordinator.sh
 
 .PHONY: build
 build: image-build-builder ## Build the coordinator binary
@@ -182,10 +249,6 @@ image-build-builder: check-container-tool ## Build builder image if missing loca
 		touch $(BUILDER_STAMP); \
 	fi
 
-.PHONY: image-build-epp
-image-build-epp: ## Clone llm-d-inference-scheduler at pinned commit and build EPP image
-	scripts/build-epp-image.sh
-
 .PHONY: image-pull
 image-pull: check-container-tool ## Pull all related images using $(CONTAINER_RUNTIME)
 	@printf "\033[33;1m==== Pulling Container images ====\033[0m\n"

README.coord.md

@@ -2,10 +2,35 @@
 
 A Go service that orchestrates multi-phase LLM inference pipelines (Encode/Prefill/Decode) across specialized worker pools. It exposes OpenAI-compatible APIs and routes requests through an Inference Gateway to disaggregated vLLM workers.
 
+## Table of Contents
+
+- [LLM-D Coordinator](#llm-d-coordinator)
+  - [Table of Contents](#table-of-contents)
+  - [Architecture](#architecture)
+  - [Quick Start](#quick-start)
+  - [Configuration](#configuration)
+    - [Server](#server)
+    - [Gateway](#gateway)
+    - [Pipeline](#pipeline)
+    - [Built-in Step Parameters](#built-in-step-parameters)
+    - [Gateway Routing](#gateway-routing)
+  - [Plugin API](#plugin-api)
+    - [Step Interface](#step-interface)
+    - [Writing a Custom Step](#writing-a-custom-step)
+    - [Registering the Step](#registering-the-step)
+    - [Dependency Injection](#dependency-injection)
+    - [RequestContext](#requestcontext)
+  - [API Endpoints](#api-endpoints)
+  - [Docker](#docker)
+  - [Development](#development)
+    - [Running Tests](#running-tests)
+      - [Unit Tests](#unit-tests)
+      - [End-to-End Tests](#end-to-end-tests)
+
 ## Architecture
 
 ```
-Client -> Coordinator -> Inference Gateway -> EPP -> vLLM Workers
+Client -> Inference Gateway -> Coordinator -> Inference Gateway -> EPP -> vLLM Workers
 ```
 
 The Coordinator processes each request through a configurable pipeline of steps:
@@ -243,9 +268,71 @@ docker run -p 8080:8080 -v $(pwd)/config/coordinator:/config/coordinator coordin
 The coordinator targets live in `Makefile.coord.mk`; pass it with `-f`:
 
 ```bash
-make -f Makefile.coord.mk build    # Build binary to bin/coordinator
-make -f Makefile.coord.mk test     # Run all tests
-make -f Makefile.coord.mk lint     # Run golangci-lint
-make -f Makefile.coord.mk tidy     # Run go mod tidy
-make -f Makefile.coord.mk clean    # Remove build artifacts
+make -f Makefile.coord.mk build   # Build binary to bin/coordinator
+make -f Makefile.coord.mk lint    # Run golangci-lint
+make -f Makefile.coord.mk tidy    # Run go mod tidy
+make -f Makefile.coord.mk clean   # Remove build artifacts
+```
+
+### Running Tests
+
+#### Unit Tests
+
+```bash
+make -f Makefile.coord.mk test   # run all tests
+```
+
+#### End-to-End Tests
+
+```bash
+make -f Makefile.coord.mk test-e2e-coordinator
+```
+
+This creates a temporary Kind cluster named `e2e-coordinator-tests`, runs the coordinator e2e suite against it, and deletes the cluster on completion.
+
+**Keeping the cluster on failure**
+
+Set `E2E_KEEP_CLUSTER_ON_FAILURE=true` to preserve the cluster when any test fails. This is useful for inspecting pod logs, events, or cluster state after a failure.
+
+```bash
+E2E_KEEP_CLUSTER_ON_FAILURE=true make -f Makefile.coord.mk test-e2e-coordinator
+```
+
+When set, a successful run still cleans up normally: the cluster is only kept if there is at least one test failure.
+
+**Accessing the cluster after a failure**
+
+E2E tests do not update the host's kubeconfig to point at the `e2e-coordinator-tests` Kind cluster. After a preserved failure, export the kubeconfig manually:
+
+```bash
+# Merge into the default kubeconfig ($HOME/.kube/config or $KUBECONFIG)
+kind export kubeconfig --name e2e-coordinator-tests
+
+# Or write to a specific file
+kind export kubeconfig --name e2e-coordinator-tests --kubeconfig /path/to/kubeconfig
 ```
+
+Then use it as normal:
+
+```bash
+kubectl --context kind-e2e-coordinator-tests get pods
+```
+
+**Environment variables**
+
+| Variable | Default | Description |
+|---|---|---|
+| `E2E_KEEP_CLUSTER_ON_FAILURE` | `false` | Preserve the Kind cluster when the suite fails |
+| `COORDINATOR_PORT` | `30081` | Host port mapped to the coordinator NodePort |
+| `E2E_GATEWAY_PORT` | `30080` | Host port mapped to the gateway NodePort |
+| `E2E_PRINT_COORDINATOR_LOGS` | `false` | Print coordinator pod logs during the run |
+| `CONTAINER_RUNTIME` | `docker` | Container runtime used to load images into Kind (`docker` or `podman`) |
+| `EPP_IMAGE` | `ghcr.io/llm-d/llm-d-router-endpoint-picker:dev` | EPP image loaded into the Kind cluster |
+| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.0` | vLLM image loaded into the Kind cluster |
+| `VLLM_RENDER_IMAGE` | same as `VLLM_IMAGE` | vLLM render image loaded into the Kind cluster |
+| `VLLM_RENDER_PORT` | `8082` | Port the vllm-render service listens on |
+| `COORDINATOR_IMAGE` | _(empty)_ | Coordinator image loaded into the Kind cluster |
+| `MODEL_NAME` | `Qwen/Qwen3-VL-2B-Instruct` | Model name used by the test pools |
+| `NAMESPACE` | `default` | Namespace to deploy test resources into |
+| `K8S_CONTEXT` | _(empty)_ | Use an existing cluster context instead of creating a Kind cluster |
+| `READY_TIMEOUT` | `10m` | How long to wait for resources to become ready |

deploy/components/vllm-encode/kustomization.yaml

@@ -3,6 +3,7 @@
 #
 # Encoder deployment for multimodal disaggregation scenarios (E/PD, E/P/D).
 # Includes a PVC for shared encoder embeddings cache.
+# Also referenced by deploy/coordinator/environments/dev/epd-pools/kustomization.yaml.
 # ------------------------------------------------------------------------------
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization

deploy/components/vllm-prefill/kustomization.yaml

@@ -2,6 +2,7 @@
 # VLLM Prefill Component
 #
 # Prefill deployment for disaggregation scenarios (P/D, E/P/D).
+# Also referenced by deploy/coordinator/environments/dev/epd-pools/kustomization.yaml.
 # ------------------------------------------------------------------------------
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization