[coordinator] e2e tests: containerize and share deploy components

.github/workflows/ci-coordinator.yaml

@@ -0,0 +1,106 @@
+name: CI - Coordinator Test
+
+# Runs coordinator unit tests and e2e tests. Path filter excludes paths not
+# exercised by the coordinator.
+on:
+  push:
+    branches:
+      - main
+      - 'release-*'
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      src: ${{ steps.filter.outputs.src }}
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+      - uses: dorny/paths-filter@v4
+        id: filter
+        with:
+          filters: |
+            src:
+              - '**/*.go'
+              - Dockerfile.*
+              - Makefile*
+              - go.mod
+              - go.sum
+              - scripts/**
+              - hack/**
+              - test/**
+              - .github/actions/e2e-runner-setup/action.yml
+              - .github/workflows/ci-coordinator.yaml
+              - '!Dockerfile.sidecar'
+              - '!cmd/pd-sidecar/**'
+              - '!pkg/sidecar/**'
+              - '!test/sidecar/**'
+              - '!pkg/generator/**'
+              - '!test/e2e/**'
+              - '!test/integration/**'
+              - '!test/profiling/**'
+              - '!Makefile'
+
+  unit-tests:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.src == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+
+      - name: Create Go cache dirs
+        id: go-cache
+        run: |
+          mkdir -p "$HOME/.cache/llm-d-gomodcache" "$HOME/.cache/llm-d-gobuildcache"
+          echo "mod=$HOME/.cache/llm-d-gomodcache" >> "$GITHUB_OUTPUT"
+          echo "build=$HOME/.cache/llm-d-gobuildcache" >> "$GITHUB_OUTPUT"
+
+      - name: Cache Go modules and build cache
+        uses: actions/cache@v6
+        with:
+          path: |
+            ${{ steps.go-cache.outputs.mod }}
+            ${{ steps.go-cache.outputs.build }}
+          key: go-cache-${{ runner.os }}-${{ hashFiles('go.sum') }}
+          restore-keys: |
+            go-cache-${{ runner.os }}-
+
+      - name: Run coordinator unit tests
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
+          GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
+        run: make -f Makefile.coord.mk test-unit
+
+  e2e-tests:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.src == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v7
+
+      - name: Set up e2e runner
+        id: e2e-setup
+        uses: ./.github/actions/e2e-runner-setup
+
+      - name: Run coordinator e2e tests
+        shell: bash
+        env:
+          GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
+          GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
+        run: make -f Makefile.coord.mk test-e2e-coordinator

.github/workflows/ci-pr-checks.yaml

@@ -42,6 +42,11 @@ jobs:
               - .github/actions/docker-build-and-push/action.yml
               - .github/actions/e2e-runner-setup/action.yml
               - .github/workflows/ci-pr-checks.yaml
+              - '!Dockerfile.coordinator'
+              - '!Makefile.coord.mk'
+              - '!cmd/coordinator/**'
+              - '!pkg/coordinator/**'
+              - '!test/coordinator/**'
             helm:
               - 'config/charts/**'
               - 'hack/verify-helm.sh'

Makefile

@@ -159,7 +159,7 @@ LDFLAGS ?= -s -w
 BASE_IMAGE ?=
 
 # test packages
-epp_TEST_PACKAGES = $$(go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/ | tr '\n' ' ')
+epp_TEST_PACKAGES = $$(go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/ | grep -v ./pkg/coordinator/ | grep -v ./cmd/coordinator | tr '\n' ' ')
 sidecar_TEST_PACKAGES = ./pkg/sidecar/...
 
 # Internal variables for generic targets

Makefile.coord.mk

@@ -1,23 +1,35 @@
 SHELL := /usr/bin/env bash
 
-# Image registry + dev-environment image tags (single source of truth).
-include versions.mk
-
-# Export all dev-env image references so the e2e suite sees them.
-export IMAGE_REGISTRY COORDINATOR_TAG VLLM_SIMULATOR_TAG EPP_TAG
-export COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE
-
 # Defaults
 TARGETOS ?= $(shell command -v go >/dev/null 2>&1 && go env GOOS || uname -s | tr '[:upper:]' '[:lower:]')
 TARGETARCH ?= $(shell command -v go >/dev/null 2>&1 && go env GOARCH || uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/; s/armv7l/arm/')
 PROJECT_NAME ?= llm-d-coordinator
 BUILDER_IMAGE_NAME ?= llm-d-coordinator-builder
 IMAGE_REGISTRY ?= ghcr.io/llm-d
 
+# Image tags
+COORDINATOR_TAG      ?= dev
+VLLM_SIMULATOR_TAG   ?= v0.10.0
+EPP_TAG              ?= dev
+
+# Full image references (derived; override only if you need a non-standard repo)
+COORDINATOR_IMAGE    ?= $(IMAGE_REGISTRY)/llm-d-coordinator:$(COORDINATOR_TAG)
+VLLM_IMAGE           ?= $(IMAGE_REGISTRY)/llm-d-inference-sim:$(VLLM_SIMULATOR_TAG)
+EPP_IMAGE            ?= $(IMAGE_REGISTRY)/llm-d-router-endpoint-picker:$(EPP_TAG)
+
+# vllm-render defaults to the same image as the other simulated vLLM roles; override
+# independently to point it at a real vLLM image (e.g. vllm/vllm-openai-cpu:v0.21.0).
+VLLM_RENDER_IMAGE    ?= $(VLLM_IMAGE)
+VLLM_RENDER_PORT     ?= 8082
+
 # Internal variable mappings for the generic image-build-% target.
 coordinator_IMAGE = $(COORDINATOR_IMAGE)
 epp_IMAGE         = $(EPP_IMAGE)
 
+# Export all dev-env image references so the e2e suite sees them.
+export IMAGE_REGISTRY COORDINATOR_TAG VLLM_SIMULATOR_TAG EPP_TAG
+export COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT
+
 BUILDER_TAG ?= dev
 BUILDER_TAG_BASE ?= $(IMAGE_REGISTRY)/$(BUILDER_IMAGE_NAME)
 export BUILDER_IMAGE ?= $(BUILDER_TAG_BASE):$(BUILDER_TAG)
@@ -39,7 +51,7 @@ LINT_NEW_ONLY ?= false
 # Optional: override the runtime base image used in container builds.
 BASE_IMAGE ?=
 
-TEST_PACKAGES = $$(go list ./... | grep -v /test/ | tr '\n' ' ')
+TEST_PACKAGES = $$(go list ./pkg/coordinator/... ./cmd/coordinator/... | tr '\n' ' ')
 
 # Common flags for running the builder container: mounts source, Go caches, and runs as current user.
 # Podman rootless requires --userns=keep-id to correctly map host UID; docker uses -u directly.
@@ -56,11 +68,64 @@ endif
 
 BUILDER_RUN_FLAGS = --rm $(BUILDER_USER_FLAGS) \
 	-v $$(pwd):/app:Z -w /app \
-	-v $(GO_MOD_CACHE_VOL):/go/pkg/mod \
-	-v $(GO_BUILD_CACHE_VOL):/go/cache
+	-v $(GO_MOD_CACHE_VOL):/go/pkg/mod:z \
+	-v $(GO_BUILD_CACHE_VOL):/go/cache:z
 
 BUILDER_RUN = $(CONTAINER_RUNTIME) run $(BUILDER_RUN_FLAGS) $(BUILDER_IMAGE) sh -c
 
+# Mount the container runtime socket and set CONTAINER_HOST so podman --remote
+# inside the builder can talk to the host's container runtime.
+ifeq ($(CONTAINER_RUNTIME),podman)
+CONTAINER_SOCK ?= $(or $(shell podman info --format '{{.Host.RemoteSocket.Path}}' 2>/dev/null | sed 's|^unix://||'),/run/podman/podman.sock)
+BUILDER_SOCK_FLAGS = --security-opt label=disable \
+	-v $(CONTAINER_SOCK):$(CONTAINER_SOCK) \
+	-e CONTAINER_HOST=unix://$(CONTAINER_SOCK) \
+	-e DOCKER_HOST=unix://$(CONTAINER_SOCK) \
+	-e CONTAINER_RUNTIME=podman \
+	-e KIND_EXPERIMENTAL_PROVIDER=podman
+else
+CONTAINER_SOCK ?= /var/run/docker.sock
+ifeq ($(TARGETOS),darwin)
+DOCKER_SOCK_GID := $(shell stat -f '%g' $(CONTAINER_SOCK) 2>/dev/null)
+else
+DOCKER_SOCK_GID := $(shell stat -c '%g' $(CONTAINER_SOCK) 2>/dev/null)
+endif
+ifneq ($(DOCKER_SOCK_GID),)
+DOCKER_GROUP_PARAM := --group-add $(DOCKER_SOCK_GID)
+else
+DOCKER_GROUP_PARAM :=
+endif
+BUILDER_SOCK_FLAGS = $(DOCKER_GROUP_PARAM) \
+	-v $(CONTAINER_SOCK):$(CONTAINER_SOCK) \
+	-e DOCKER_HOST=unix://$(CONTAINER_SOCK) \
+	-e CONTAINER_RUNTIME=docker
+endif
+
+# Respect host KUBECONFIG if set; fall back to ~/.kube/config.
+HOST_KUBECONFIG ?= $(or $(KUBECONFIG),$(HOME)/.kube/config)
+
+# When K8S_CONTEXT is set, mount the host kubeconfig so the e2e suite can call
+# config.GetConfigWithContext(K8S_CONTEXT) against an existing cluster instead of
+# creating a new kind cluster.
+ifdef K8S_CONTEXT
+BUILDER_E2E_KUBECONFIG_FLAGS = -v $(HOST_KUBECONFIG):/.kube/config:ro -e KUBECONFIG=/.kube/config
+else
+BUILDER_E2E_KUBECONFIG_FLAGS =
+endif
+
+# Env vars forwarded into the e2e test container.
+E2E_ENV_VARS = COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT \
+               E2E_GATEWAY_PORT E2E_KEEP_CLUSTER_ON_FAILURE \
+               E2E_PRINT_COORDINATOR_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
+BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
+ifneq ($(filter command line environment,$(origin NAMESPACE)),)
+BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)
+endif
+
+# E2e tests create their own kind cluster, need host network (for NodePort access)
+# and the container socket (for kind), but not the host kubeconfig.
+BUILDER_E2E_FLAGS = --network=host $(BUILDER_SOCK_FLAGS) $(BUILDER_E2E_ENV_FLAGS) $(BUILDER_E2E_KUBECONFIG_FLAGS)
+
 BUILDER_STAMP = build/.builder.stamp
 
 .PHONY: help
@@ -130,9 +195,15 @@ test-unit: image-build-builder
 	@printf "\033[33;1m==== Running Unit Tests ====\033[0m\n"
 	$(BUILDER_RUN) "go test -v -race $(TEST_PACKAGES)"
 
+.PHONY: test-e2e-coordinator-run
+test-e2e-coordinator-run: image-pull ## Ensure images are present, then run coordinator e2e tests
+	@printf "\033[33;1m==== Running Coordinator End to End Tests ====\033[0m\n"
+	$(CONTAINER_RUNTIME) run $(BUILDER_RUN_FLAGS) $(BUILDER_E2E_FLAGS) \
+		$(BUILDER_IMAGE) test/coordinator/scripts/run_e2e_coordinator.sh
+
 .PHONY: test-e2e-coordinator
-test-e2e-coordinator: image-build-coordinator image-build-epp image-build-builder image-pull ## Run coordinator e2e tests against a new kind cluster
-	test/coordinator/scripts/run_e2e_coordinator.sh
+test-e2e-coordinator: image-build-builder image-build-coordinator image-build-epp ## Build images and run coordinator e2e tests
+	$(MAKE) -f Makefile.coord.mk test-e2e-coordinator-run
 
 .PHONY: build
 build: image-build-builder ## Build the coordinator binary
@@ -182,12 +253,8 @@ image-build-builder: check-container-tool ## Build builder image if missing loca
 		touch $(BUILDER_STAMP); \
 	fi
 
-.PHONY: image-build-epp
-image-build-epp: ## Clone llm-d-inference-scheduler at pinned commit and build EPP image
-	scripts/build-epp-image.sh
-
 .PHONY: image-pull
 image-pull: check-container-tool ## Pull all related images using $(CONTAINER_RUNTIME)
 	@printf "\033[33;1m==== Pulling Container images ====\033[0m\n"
-	./scripts/pull_images.sh
+	PULL_EPP_IMAGE=false PULL_SIDECAR_IMAGE=false ./scripts/pull_images.sh
 

README.coord.md

@@ -2,10 +2,35 @@
 
 A Go service that orchestrates multi-phase LLM inference pipelines (Encode/Prefill/Decode) across specialized worker pools. It exposes OpenAI-compatible APIs and routes requests through an Inference Gateway to disaggregated vLLM workers.
 
+## Table of Contents
+
+- [LLM-D Coordinator](#llm-d-coordinator)
+  - [Table of Contents](#table-of-contents)
+  - [Architecture](#architecture)
+  - [Quick Start](#quick-start)
+  - [Configuration](#configuration)
+    - [Server](#server)
+    - [Gateway](#gateway)
+    - [Pipeline](#pipeline)
+    - [Built-in Step Parameters](#built-in-step-parameters)
+    - [Gateway Routing](#gateway-routing)
+  - [Plugin API](#plugin-api)
+    - [Step Interface](#step-interface)
+    - [Writing a Custom Step](#writing-a-custom-step)
+    - [Registering the Step](#registering-the-step)
+    - [Dependency Injection](#dependency-injection)
+    - [RequestContext](#requestcontext)
+  - [API Endpoints](#api-endpoints)
+  - [Docker](#docker)
+  - [Development](#development)
+    - [Running Tests](#running-tests)
+      - [Unit Tests](#unit-tests)
+      - [End-to-End Tests](#end-to-end-tests)
+
 ## Architecture
 
 ```
-Client -> Coordinator -> Inference Gateway -> EPP -> vLLM Workers
+Client -> Inference Gateway -> Coordinator -> Inference Gateway -> EPP -> vLLM Workers
 ```
 
 The Coordinator processes each request through a configurable pipeline of steps:
@@ -243,9 +268,70 @@ docker run -p 8080:8080 -v $(pwd)/config/coordinator:/config/coordinator coordin
 The coordinator targets live in `Makefile.coord.mk`; pass it with `-f`:
 
 ```bash
-make -f Makefile.coord.mk build    # Build binary to bin/coordinator
-make -f Makefile.coord.mk test     # Run all tests
-make -f Makefile.coord.mk lint     # Run golangci-lint
-make -f Makefile.coord.mk tidy     # Run go mod tidy
-make -f Makefile.coord.mk clean    # Remove build artifacts
+make -f Makefile.coord.mk build   # Build binary to bin/coordinator
+make -f Makefile.coord.mk lint    # Run golangci-lint
+make -f Makefile.coord.mk tidy    # Run go mod tidy
+make -f Makefile.coord.mk clean   # Remove build artifacts
+```
+
+### Running Tests
+
+#### Unit Tests
+
+```bash
+make -f Makefile.coord.mk test   # run coordinator unit tests
+```
+
+#### End-to-End Tests
+
+```bash
+make -f Makefile.coord.mk test-e2e-coordinator
+```
+
+This creates a temporary Kind cluster named `e2e-coordinator-tests`, runs the coordinator e2e suite against it, and deletes the cluster on completion.
+
+**Keeping the cluster on failure**
+
+Set `E2E_KEEP_CLUSTER_ON_FAILURE=true` to preserve the cluster when any test fails. This is useful for inspecting pod logs, events, or cluster state after a failure.
+
+```bash
+E2E_KEEP_CLUSTER_ON_FAILURE=true make -f Makefile.coord.mk test-e2e-coordinator
+```
+
+When set, a successful run still cleans up normally: the cluster is only kept if there is at least one test failure.
+
+**Accessing the cluster after a failure**
+
+E2E tests do not update the host's kubeconfig to point at the `e2e-coordinator-tests` Kind cluster. After a preserved failure, export the kubeconfig manually:
+
+```bash
+# Merge into the default kubeconfig ($HOME/.kube/config or $KUBECONFIG)
+kind export kubeconfig --name e2e-coordinator-tests
+
+# Or write to a specific file
+kind export kubeconfig --name e2e-coordinator-tests --kubeconfig /path/to/kubeconfig
 ```
+
+Then use it as normal:
+
+```bash
+kubectl --context kind-e2e-coordinator-tests get pods
+```
+
+**Environment variables**
+
+| Variable | Default | Description |
+|---|---|---|
+| `E2E_KEEP_CLUSTER_ON_FAILURE` | `false` | Preserve the Kind cluster when the suite fails |
+| `E2E_GATEWAY_PORT` | `30080` | Host port mapped to the gateway NodePort |
+| `E2E_PRINT_COORDINATOR_LOGS` | `false` | Print coordinator pod logs during the run |
+| `CONTAINER_RUNTIME` | `docker` | Container runtime used to load images into Kind (`docker` or `podman`) |
+| `EPP_IMAGE` | `ghcr.io/llm-d/llm-d-router-endpoint-picker:dev` | EPP image loaded into the Kind cluster |
+| `VLLM_IMAGE` | `ghcr.io/llm-d/llm-d-inference-sim:v0.10.0` | vLLM image loaded into the Kind cluster |
+| `VLLM_RENDER_IMAGE` | same as `VLLM_IMAGE` | vLLM render image loaded into the Kind cluster |
+| `VLLM_RENDER_PORT` | `8082` | Port the vllm-render service listens on |
+| `COORDINATOR_IMAGE` | _(empty)_ | Coordinator image loaded into the Kind cluster |
+| `MODEL_NAME` | `Qwen/Qwen3-VL-2B-Instruct` | Model name used by the test pools |
+| `NAMESPACE` | `default` | Namespace to deploy test resources into |
+| `K8S_CONTEXT` | _(empty)_ | Use an existing cluster context instead of creating a Kind cluster |
+| `READY_TIMEOUT` | `10m` | How long to wait for resources to become ready |

deploy/components/crds-gie/kustomization.yaml

@@ -3,6 +3,7 @@
 #
 # **Warning**: CRDs are cluster-level, so in a shared development environment
 # this needs to be done in a controlled and communicated manner.
+# Also used by the coordinator (deploy/coordinator/environments/dev/epd-pools).
 # ------------------------------------------------------------------------------
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization