Skip to content

Commit b418271

Browse files
revit13Copilot
revit13
and
Copilot
authored
[coordinator] e2e tests: containerize and share deploy components (#1975)
* [coordinator] e2e tests: containerize and share deploy components Signed-off-by: Revital Sur <eres@il.ibm.com> * Address review comments. Signed-off-by: Revital Sur <eres@il.ibm.com> * Address review comments and add github workflow test. Signed-off-by: Revital Sur <eres@il.ibm.com> * Minor fix. Signed-off-by: Revital Sur <eres@il.ibm.com> * Additional change. Signed-off-by: Revital Sur <eres@il.ibm.com> * Skip EPP and sidecar image pulls in coordinator e2e. Signed-off-by: Revital Sur <eres@il.ibm.com> * Add test-e2e-coordinator-run target. Signed-off-by: Revital Sur <eres@il.ibm.com> * Mirror the router's parallel-testing infrastructure. Signed-off-by: Revital Sur <eres@il.ibm.com> * Fix line error. Signed-off-by: Revital Sur <eres@il.ibm.com> * Use constants for ports. Signed-off-by: Revital Sur <eres@il.ibm.com> * Drop unused direct NodePort access to the coordinator Signed-off-by: Revital Sur <eres@il.ibm.com> * Fix comments. Signed-off-by: Revital Sur <eres@il.ibm.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Revital Sur <eres@il.ibm.com> * Address revie comments. Signed-off-by: Revital Sur <eres@il.ibm.com> * Address review comments. Signed-off-by: Revital Sur <eres@il.ibm.com> * Address review comments. Signed-off-by: Revital Sur <eres@il.ibm.com> * Update simulator tag in coord resources. Signed-off-by: Revital Sur <eres@il.ibm.com> --------- Signed-off-by: Revital Sur <eres@il.ibm.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent 6c2dd71 commit b418271

34 files changed

Lines changed: 756 additions & 19238 deletions

‎.github/workflows/ci-coordinator.yaml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: CI - Coordinator Test
2+
3+
# Runs coordinator unit tests and e2e tests. Path filter excludes paths not
4+
# exercised by the coordinator.
5+
on:
6+
push:
7+
branches:
8+
- main
9+
- 'release-*'
10+
pull_request:
11+
branches:
12+
- main
13+
14+
permissions:
15+
contents: read
16+
actions: read
17+
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
20+
cancel-in-progress: true
21+
22+
jobs:
23+
check-changes:
24+
runs-on: ubuntu-latest
25+
outputs:
26+
src: ${{ steps.filter.outputs.src }}
27+
steps:
28+
- name: Checkout source
29+
uses: actions/checkout@v7
30+
- uses: dorny/paths-filter@v4
31+
id: filter
32+
with:
33+
filters: |
34+
src:
35+
- '**/*.go'
36+
- Dockerfile.*
37+
- Makefile*
38+
- go.mod
39+
- go.sum
40+
- scripts/**
41+
- hack/**
42+
- test/**
43+
- .github/actions/e2e-runner-setup/action.yml
44+
- .github/workflows/ci-coordinator.yaml
45+
- '!Dockerfile.sidecar'
46+
- '!cmd/pd-sidecar/**'
47+
- '!pkg/sidecar/**'
48+
- '!test/sidecar/**'
49+
- '!pkg/generator/**'
50+
- '!test/e2e/**'
51+
- '!test/integration/**'
52+
- '!test/profiling/**'
53+
- '!Makefile'
54+
55+
unit-tests:
56+
needs: check-changes
57+
if: ${{ needs.check-changes.outputs.src == 'true' }}
58+
runs-on: ubuntu-latest
59+
timeout-minutes: 30
60+
steps:
61+
- name: Checkout source
62+
uses: actions/checkout@v7
63+
64+
- name: Create Go cache dirs
65+
id: go-cache
66+
run: |
67+
mkdir -p "$HOME/.cache/llm-d-gomodcache" "$HOME/.cache/llm-d-gobuildcache"
68+
echo "mod=$HOME/.cache/llm-d-gomodcache" >> "$GITHUB_OUTPUT"
69+
echo "build=$HOME/.cache/llm-d-gobuildcache" >> "$GITHUB_OUTPUT"
70+
71+
- name: Cache Go modules and build cache
72+
uses: actions/cache@v6
73+
with:
74+
path: |
75+
${{ steps.go-cache.outputs.mod }}
76+
${{ steps.go-cache.outputs.build }}
77+
key: go-cache-${{ runner.os }}-${{ hashFiles('go.sum') }}
78+
restore-keys: |
79+
go-cache-${{ runner.os }}-
80+
81+
- name: Run coordinator unit tests
82+
shell: bash
83+
env:
84+
GO_MOD_CACHE_VOL: ${{ steps.go-cache.outputs.mod }}
85+
GO_BUILD_CACHE_VOL: ${{ steps.go-cache.outputs.build }}
86+
run: make -f Makefile.coord.mk test-unit
87+
88+
e2e-tests:
89+
needs: check-changes
90+
if: ${{ needs.check-changes.outputs.src == 'true' }}
91+
runs-on: ubuntu-latest
92+
timeout-minutes: 60
93+
steps:
94+
- name: Checkout source
95+
uses: actions/checkout@v7
96+
97+
- name: Set up e2e runner
98+
id: e2e-setup
99+
uses: ./.github/actions/e2e-runner-setup
100+
101+
- name: Run coordinator e2e tests
102+
shell: bash
103+
env:
104+
GO_MOD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-mod-cache }}
105+
GO_BUILD_CACHE_VOL: ${{ steps.e2e-setup.outputs.go-build-cache }}
106+
run: make -f Makefile.coord.mk test-e2e-coordinator

‎.github/workflows/ci-pr-checks.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ jobs:
4545
- .github/actions/docker-build-and-push/action.yml
4646
- .github/actions/e2e-runner-setup/action.yml
4747
- .github/workflows/ci-pr-checks.yaml
48+
- '!Dockerfile.coordinator'
49+
- '!Makefile.coord.mk'
50+
- '!cmd/coordinator/**'
51+
- '!pkg/coordinator/**'
52+
- '!test/coordinator/**'
4853
helm:
4954
- 'config/charts/**'
5055
- 'hack/verify-helm.sh'

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ LDFLAGS ?= -s -w
159159
BASE_IMAGE ?=
160160

161161
# test packages
162-
epp_TEST_PACKAGES = $$(go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/ | tr '\n' ' ')
162+
epp_TEST_PACKAGES = $$(go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/ | grep -v ./pkg/coordinator/ | grep -v ./cmd/coordinator | tr '\n' ' ')
163163
sidecar_TEST_PACKAGES = ./pkg/sidecar/...
164164

165165
# Internal variables for generic targets

‎Makefile.coord.mk‎

Lines changed: 84 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,35 @@
11
SHELL := /usr/bin/env bash
22

3-
# Image registry + dev-environment image tags (single source of truth).
4-
include versions.mk
5-
6-
# Export all dev-env image references so the e2e suite sees them.
7-
export IMAGE_REGISTRY COORDINATOR_TAG VLLM_SIMULATOR_TAG EPP_TAG
8-
export COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE
9-
103
# Defaults
114
TARGETOS ?= $(shell command -v go >/dev/null 2>&1 && go env GOOS || uname -s | tr '[:upper:]' '[:lower:]')
125
TARGETARCH ?= $(shell command -v go >/dev/null 2>&1 && go env GOARCH || uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/; s/armv7l/arm/')
136
PROJECT_NAME ?= llm-d-coordinator
147
BUILDER_IMAGE_NAME ?= llm-d-coordinator-builder
158
IMAGE_REGISTRY ?= ghcr.io/llm-d
169

10+
# Image tags
11+
COORDINATOR_TAG ?= dev
12+
VLLM_SIMULATOR_TAG ?= v0.10.2
13+
EPP_TAG ?= dev
14+
15+
# Full image references (derived; override only if you need a non-standard repo)
16+
COORDINATOR_IMAGE ?= $(IMAGE_REGISTRY)/llm-d-coordinator:$(COORDINATOR_TAG)
17+
VLLM_IMAGE ?= $(IMAGE_REGISTRY)/llm-d-inference-sim:$(VLLM_SIMULATOR_TAG)
18+
EPP_IMAGE ?= $(IMAGE_REGISTRY)/llm-d-router-endpoint-picker:$(EPP_TAG)
19+
20+
# vllm-render defaults to the same image as the other simulated vLLM roles; override
21+
# independently to point it at a real vLLM image (e.g. vllm/vllm-openai-cpu:v0.21.0).
22+
VLLM_RENDER_IMAGE ?= $(VLLM_IMAGE)
23+
VLLM_RENDER_PORT ?= 8082
24+
1725
# Internal variable mappings for the generic image-build-% target.
1826
coordinator_IMAGE = $(COORDINATOR_IMAGE)
1927
epp_IMAGE = $(EPP_IMAGE)
2028

29+
# Export all dev-env image references so the e2e suite sees them.
30+
export IMAGE_REGISTRY COORDINATOR_TAG VLLM_SIMULATOR_TAG EPP_TAG
31+
export COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT
32+
2133
BUILDER_TAG ?= dev
2234
BUILDER_TAG_BASE ?= $(IMAGE_REGISTRY)/$(BUILDER_IMAGE_NAME)
2335
export BUILDER_IMAGE ?= $(BUILDER_TAG_BASE):$(BUILDER_TAG)
@@ -39,7 +51,7 @@ LINT_NEW_ONLY ?= false
3951
# Optional: override the runtime base image used in container builds.
4052
BASE_IMAGE ?=
4153

42-
TEST_PACKAGES = $$(go list ./... | grep -v /test/ | tr '\n' ' ')
54+
TEST_PACKAGES = $$(go list ./pkg/coordinator/... ./cmd/coordinator/... | tr '\n' ' ')
4355

4456
# Common flags for running the builder container: mounts source, Go caches, and runs as current user.
4557
# Podman rootless requires --userns=keep-id to correctly map host UID; docker uses -u directly.
@@ -56,11 +68,64 @@ endif
5668

5769
BUILDER_RUN_FLAGS = --rm $(BUILDER_USER_FLAGS) \
5870
-v $$(pwd):/app:Z -w /app \
59-
-v $(GO_MOD_CACHE_VOL):/go/pkg/mod \
60-
-v $(GO_BUILD_CACHE_VOL):/go/cache
71+
-v $(GO_MOD_CACHE_VOL):/go/pkg/mod:z \
72+
-v $(GO_BUILD_CACHE_VOL):/go/cache:z
6173

6274
BUILDER_RUN = $(CONTAINER_RUNTIME) run $(BUILDER_RUN_FLAGS) $(BUILDER_IMAGE) sh -c
6375

76+
# Mount the container runtime socket and set CONTAINER_HOST so podman --remote
77+
# inside the builder can talk to the host's container runtime.
78+
ifeq ($(CONTAINER_RUNTIME),podman)
79+
CONTAINER_SOCK ?= $(or $(shell podman info --format '{{.Host.RemoteSocket.Path}}' 2>/dev/null | sed 's|^unix://||'),/run/podman/podman.sock)
80+
BUILDER_SOCK_FLAGS = --security-opt label=disable \
81+
-v $(CONTAINER_SOCK):$(CONTAINER_SOCK) \
82+
-e CONTAINER_HOST=unix://$(CONTAINER_SOCK) \
83+
-e DOCKER_HOST=unix://$(CONTAINER_SOCK) \
84+
-e CONTAINER_RUNTIME=podman \
85+
-e KIND_EXPERIMENTAL_PROVIDER=podman
86+
else
87+
CONTAINER_SOCK ?= /var/run/docker.sock
88+
ifeq ($(TARGETOS),darwin)
89+
DOCKER_SOCK_GID := $(shell stat -f '%g' $(CONTAINER_SOCK) 2>/dev/null)
90+
else
91+
DOCKER_SOCK_GID := $(shell stat -c '%g' $(CONTAINER_SOCK) 2>/dev/null)
92+
endif
93+
ifneq ($(DOCKER_SOCK_GID),)
94+
DOCKER_GROUP_PARAM := --group-add $(DOCKER_SOCK_GID)
95+
else
96+
DOCKER_GROUP_PARAM :=
97+
endif
98+
BUILDER_SOCK_FLAGS = $(DOCKER_GROUP_PARAM) \
99+
-v $(CONTAINER_SOCK):$(CONTAINER_SOCK) \
100+
-e DOCKER_HOST=unix://$(CONTAINER_SOCK) \
101+
-e CONTAINER_RUNTIME=docker
102+
endif
103+
104+
# Respect host KUBECONFIG if set; fall back to ~/.kube/config.
105+
HOST_KUBECONFIG ?= $(or $(KUBECONFIG),$(HOME)/.kube/config)
106+
107+
# When K8S_CONTEXT is set, mount the host kubeconfig so the e2e suite can call
108+
# config.GetConfigWithContext(K8S_CONTEXT) against an existing cluster instead of
109+
# creating a new kind cluster.
110+
ifdef K8S_CONTEXT
111+
BUILDER_E2E_KUBECONFIG_FLAGS = -v $(HOST_KUBECONFIG):/.kube/config:ro -e KUBECONFIG=/.kube/config
112+
else
113+
BUILDER_E2E_KUBECONFIG_FLAGS =
114+
endif
115+
116+
# Env vars forwarded into the e2e test container.
117+
E2E_ENV_VARS = COORDINATOR_IMAGE VLLM_IMAGE EPP_IMAGE VLLM_RENDER_IMAGE VLLM_RENDER_PORT \
118+
E2E_GATEWAY_PORT E2E_KEEP_CLUSTER_ON_FAILURE \
119+
E2E_PRINT_COORDINATOR_LOGS K8S_CONTEXT READY_TIMEOUT MODEL_NAME
120+
BUILDER_E2E_ENV_FLAGS = $(foreach v,$(E2E_ENV_VARS),$(if $($(v)),-e '$(v)=$($(v))'))
121+
ifneq ($(filter command line environment,$(origin NAMESPACE)),)
122+
BUILDER_E2E_ENV_FLAGS += -e NAMESPACE=$(NAMESPACE)
123+
endif
124+
125+
# E2e tests create their own kind cluster, need host network (for NodePort access)
126+
# and the container socket (for kind), but not the host kubeconfig.
127+
BUILDER_E2E_FLAGS = --network=host $(BUILDER_SOCK_FLAGS) $(BUILDER_E2E_ENV_FLAGS) $(BUILDER_E2E_KUBECONFIG_FLAGS)
128+
64129
BUILDER_STAMP = build/.builder.stamp
65130

66131
.PHONY: help
@@ -130,9 +195,15 @@ test-unit: image-build-builder
130195
@printf "\033[33;1m==== Running Unit Tests ====\033[0m\n"
131196
$(BUILDER_RUN) "go test -v -race $(TEST_PACKAGES)"
132197

198+
.PHONY: test-e2e-coordinator-run
199+
test-e2e-coordinator-run: image-pull ## Ensure images are present, then run coordinator e2e tests
200+
@printf "\033[33;1m==== Running Coordinator End to End Tests ====\033[0m\n"
201+
$(CONTAINER_RUNTIME) run $(BUILDER_RUN_FLAGS) $(BUILDER_E2E_FLAGS) \
202+
$(BUILDER_IMAGE) test/coordinator/scripts/run_e2e_coordinator.sh
203+
133204
.PHONY: test-e2e-coordinator
134-
test-e2e-coordinator: image-build-coordinator image-build-epp image-build-builder image-pull ## Run coordinator e2e tests against a new kind cluster
135-
test/coordinator/scripts/run_e2e_coordinator.sh
205+
test-e2e-coordinator: image-build-builder image-build-coordinator image-build-epp ## Build images and run coordinator e2e tests
206+
$(MAKE) -f Makefile.coord.mk test-e2e-coordinator-run
136207

137208
.PHONY: build
138209
build: image-build-builder ## Build the coordinator binary
@@ -182,12 +253,8 @@ image-build-builder: check-container-tool ## Build builder image if missing loca
182253
touch $(BUILDER_STAMP); \
183254
fi
184255

185-
.PHONY: image-build-epp
186-
image-build-epp: ## Clone llm-d-inference-scheduler at pinned commit and build EPP image
187-
scripts/build-epp-image.sh
188-
189256
.PHONY: image-pull
190257
image-pull: check-container-tool ## Pull all related images using $(CONTAINER_RUNTIME)
191258
@printf "\033[33;1m==== Pulling Container images ====\033[0m\n"
192-
./scripts/pull_images.sh
259+
PULL_EPP_IMAGE=false PULL_SIDECAR_IMAGE=false ./scripts/pull_images.sh
193260

0 commit comments

Comments
 (0)