Merge remote-tracking branch 'upstream/main' into odh-main

Author: pierDipi

.github/workflows/check-typos.yaml

@@ -13,5 +13,5 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Check typos
-        uses: crate-ci/typos@v1.39.2
+        uses: crate-ci/typos@v1.40.0
 

.github/workflows/ci-pr-checks.yaml

@@ -9,12 +9,28 @@ on:
       - main
 
 jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      docs: ${{ steps.filter.outputs.docs }}
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            docs:
+              - 'README.md'
+              - 'docs/**'
   lint-and-test:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.docs == 'false' }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source
         uses: actions/checkout@v6
-        
+
       - name: Sanity check repo contents
         run: ls -la
 

.github/workflows/ci-signed-commits.yaml

@@ -0,0 +1,21 @@
+name: Check Signed Commits in PR
+
+on:
+  pull_request_target:
+
+jobs:
+  check-signed-commits:
+    name: Check signed commits in PR
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write # Required to post comments on PRs
+    steps:
+      - name: Check signed commits in PR
+        uses: 1Password/check-signed-commits-action@v1 # Use the action
+        with:
+          comment: |
+            🚨 Unsigned commits detected! Please sign your commits.
+
+            For instructions on how to set up GPG/SSH signing and verify your commits,
+            please see [GitHub Documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification).

Makefile

@@ -4,8 +4,8 @@ include Makefile.tools.mk
 SHELL := /usr/bin/env bash
 
 # Defaults
-TARGETOS ?= $(shell go env GOOS)
-TARGETARCH ?= $(shell go env GOARCH)
+TARGETOS ?= $(shell command -v go >/dev/null 2>&1 && go env GOOS || uname -s | tr '[:upper:]' '[:lower:]')
+TARGETARCH ?= $(shell command -v go >/dev/null 2>&1 && go env GOARCH || uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/; s/armv7l/arm/')
 PROJECT_NAME ?= llm-d-inference-scheduler
 SIDECAR_IMAGE_NAME ?= llm-d-routing-sidecar
 VLLM_SIMULATOR_IMAGE_NAME ?= llm-d-inference-sim
@@ -69,31 +69,29 @@ PYTHON_VERSION := 3.12
 PYTHON_CONFIG ?= $(shell command -v python$(PYTHON_VERSION)-config || command -v python3-config)
 ifeq ($(PYTHON_CONFIG),)
 	ifeq ($(TARGETOS),darwin)
-		# macOS: Find Homebrew's python-config script for the most reliable flags.
-		BREW_PREFIX := $(shell command -v brew >/dev/null 2>&1 && brew --prefix python@$(PYTHON_VERSION) 2>/dev/null)
-		PYTHON_CONFIG ?= $(BREW_PREFIX)/bin/python$(PYTHON_VERSION)-config
-		PYTHON_ERROR := "Could not execute 'python$(PYTHON_VERSION)-config'. Please ensure Python is installed correctly with: 'brew install python@$(PYTHON_VERSION)' or install python3.12 manually."
+        # macOS: Find Homebrew's python-config script for the most reliable flags.
+        BREW_PREFIX := $(shell command -v brew >/dev/null 2>&1 && brew --prefix python@$(PYTHON_VERSION) 2>/dev/null)
+        PYTHON_CONFIG ?= $(BREW_PREFIX)/bin/python$(PYTHON_VERSION)-config
+        PYTHON_ERROR := "Could not execute 'python$(PYTHON_VERSION)-config'. Please ensure Python is installed correctly with: 'brew install python@$(PYTHON_VERSION)' or install python3.12 manually."
 	else ifeq ($(TARGETOS),linux)
-		# Linux: Use standard system tools to find flags.
-		PYTHON_ERROR := "Python $(PYTHON_VERSION) development headers not found. Please install with: 'sudo apt install python$(PYTHON_VERSION)-dev' or 'sudo dnf install python$(PYTHON_VERSION)-devel'"
+        # Linux: Use standard system tools to find flags.
+        PYTHON_ERROR := "Python $(PYTHON_VERSION) development headers not found. Please install with: 'sudo apt install python$(PYTHON_VERSION)-dev' or 'sudo dnf install python$(PYTHON_VERSION)-devel'"
 	else
-		PYTHON_ERROR := "you should set up PYTHON_CONFIG variable manually"
+        PYTHON_ERROR := "you should set up PYTHON_CONFIG variable manually"
 	endif
 endif
 
-# Check if PYTHON_CONFIG is found before using it
-ifneq ($(PYTHON_CONFIG),)
-ifneq ($(shell $(PYTHON_CONFIG) --cflags 2>/dev/null),)
-	PYTHON_CFLAGS := $(shell $(PYTHON_CONFIG) --cflags)
-	# Use --ldflags --embed to get all necessary flags for linking
-	PYTHON_LDFLAGS := $(shell $(PYTHON_CONFIG) --ldflags --embed)
-else
-	$(error ${PYTHON_ERROR})
+ifeq ($(PYTHON_CONFIG),)
+    $(error ${PYTHON_ERROR})
 endif
-else
-	$(error ${PYTHON_ERROR})
+
+ifeq ($(shell $(PYTHON_CONFIG) --cflags 2>/dev/null),)
+    $(error Python configuration tool cannot provide cflags)
 endif
 
+PYTHON_CFLAGS := $(shell $(PYTHON_CONFIG) --cflags)
+PYTHON_LDFLAGS := $(shell $(PYTHON_CONFIG) --ldflags --embed)
+
 # CGO flags with all dependencies
 CGO_CFLAGS := $(PYTHON_CFLAGS) '-I$(shell pwd)/lib'
 CGO_LDFLAGS := $(PYTHON_LDFLAGS) $(PYTHON_LIBS) '-L$(shell pwd)/lib' -ltokenizers -ldl -lm
@@ -156,12 +154,12 @@ test: test-unit test-e2e ## Run unit tests and e2e tests
 test-unit: test-unit-epp test-unit-sidecar
 
 .PHONY: test-unit-%
-test-unit-%: download-tokenizer install-dependencies ## Run unit tests
+test-unit-%: download-tokenizer check-dependencies ## Run unit tests
 	@printf "\033[33;1m==== Running Unit Tests ====\033[0m\n"
 	CGO_CFLAGS=${$*_CGO_CFLAGS} CGO_LDFLAGS=${$*_CGO_LDFLAGS} go test $($*_LDFLAGS) -v $$($($*_TEST_FILES) | tr '\n' ' ')
 
 .PHONY: test-integration
-test-integration: download-tokenizer install-dependencies ## Run integration tests
+test-integration: download-tokenizer check-dependencies ## Run integration tests
 	@printf "\033[33;1m==== Running Integration Tests ====\033[0m\n"
 	go test -ldflags="$(LDFLAGS)" -v -tags=integration_tests ./test/integration/
 
@@ -187,9 +185,9 @@ lint: check-golangci-lint check-typos ## Run lint
 build: build-epp build-sidecar ## Build the project
 
 .PHONY: build-%
-build-%: check-go install-dependencies download-tokenizer ## Build the project
+build-%: check-go download-tokenizer ## Build the project
 	@printf "\033[33;1m==== Building ====\033[0m\n"
-	go build $($*_LDFLAGS) -o bin/$($*_NAME) cmd/$($*_NAME)/main.go
+	CGO_CFLAGS=${$*_CGO_CFLAGS} CGO_LDFLAGS=${$*_CGO_LDFLAGS} go build $($*_LDFLAGS) -o bin/$($*_NAME) cmd/$($*_NAME)/main.go
 
 ##@ Container Build/Push
 
@@ -465,26 +463,55 @@ clean-env-dev-kubernetes: check-kubectl check-kustomize check-envsubst
 
 ##@ Dependencies
 
+.PHONY: check-dependencies
+check-dependencies: ## Check if development dependencies are installed
+	@if [ "$(TARGETOS)" = "linux" ]; then \
+	  if [ -x "$$(command -v apt)" ]; then \
+	    if ! dpkg -s libzmq3-dev >/dev/null 2>&1 || ! dpkg -s g++ >/dev/null 2>&1 || ! dpkg -s python$(PYTHON_VERSION)-dev >/dev/null 2>&1; then \
+	      echo "ERROR: Missing dependencies. Please run 'sudo make install-dependencies'"; \
+	      exit 1; \
+	    fi; \
+	  elif [ -x "$$(command -v dnf)" ]; then \
+	    if ! rpm -q zeromq-devel >/dev/null 2>&1 || ! rpm -q gcc-c++ >/dev/null 2>&1 || ! rpm -q python$(PYTHON_VERSION)-devel >/dev/null 2>&1; then \
+	      echo "ERROR: Missing dependencies. Please run 'sudo make install-dependencies'"; \
+	      exit 1; \
+	    fi; \
+	  else \
+	    echo "WARNING: Unsupported Linux package manager. Cannot verify dependencies."; \
+	  fi; \
+	elif [ "$(TARGETOS)" = "darwin" ]; then \
+	  if [ -x "$$(command -v brew)" ]; then \
+	    if ! brew list zeromq pkg-config >/dev/null 2>&1; then \
+	      echo "ERROR: Missing dependencies. Please run 'make install-dependencies'"; \
+	      exit 1; \
+	    fi; \
+	  else \
+	    echo "ERROR: Homebrew is not installed and is required. Install it from https://brew.sh/"; \
+	    exit 1; \
+	  fi; \
+	fi
+	@echo "✅ All dependencies are installed."
+
 .PHONY: install-dependencies
 install-dependencies: ## Install development dependencies based on OS/ARCH
 	@echo "Checking and installing development dependencies..."
 	@if [ "$(TARGETOS)" = "linux" ]; then \
 	  if [ -x "$$(command -v apt)" ]; then \
-	    if ! dpkg -s libzmq3-dev >/dev/null 2>&1 || ! dpkg -s g++ >/dev/null 2>&1; then \
+	    if ! dpkg -s libzmq3-dev >/dev/null 2>&1 || ! dpkg -s g++ >/dev/null 2>&1 || ! dpkg -s python$(PYTHON_VERSION)-dev >/dev/null 2>&1; then \
 	      echo "Installing dependencies with apt..."; \
-	      apt-get update && apt-get install -y libzmq3-dev g++; \
+	      apt-get update && apt-get install -y libzmq3-dev g++ python$(PYTHON_VERSION)-dev; \
 	    else \
-	      echo "✅ ZMQ and g++ are already installed."; \
+	      echo "✅ ZMQ, g++, and Python dev headers are already installed."; \
 	    fi; \
 	  elif [ -x "$$(command -v dnf)" ]; then \
-	    if ! dnf -q list installed zeromq-devel >/dev/null 2>&1 || ! dnf -q list installed gcc-c++ >/dev/null 2>&1; then \
+	    if ! rpm -q zeromq-devel >/dev/null 2>&1 || ! rpm -q gcc-c++ >/dev/null 2>&1 || ! rpm -q python$(PYTHON_VERSION)-devel >/dev/null 2>&1; then \
 	      echo "Installing dependencies with dnf..."; \
-	      dnf install -y zeromq-devel gcc-c++; \
+	      dnf install -y zeromq-devel gcc-c++ python$(PYTHON_VERSION)-devel; \
 	    else \
-	      echo "✅ ZMQ and gcc-c++ are already installed."; \
+	      echo "✅ ZMQ, gcc-c++, and Python dev headers are already installed."; \
 	    fi; \
 	  else \
-	    echo "Unsupported Linux package manager. Install libzmq and g++/gcc-c++ manually."; \
+	    echo "ERROR: Unsupported Linux package manager. Install libzmq, g++/gcc-c++, and python-devel manually."; \
 	    exit 1; \
 	  fi; \
 	elif [ "$(TARGETOS)" = "darwin" ]; then \
@@ -496,10 +523,10 @@ install-dependencies: ## Install development dependencies based on OS/ARCH
 	      echo "✅ ZeroMQ and pkgconf are already installed."; \
 	    fi; \
 	  else \
-	    echo "Homebrew is not installed and is required to install zeromq. Install it from https://brew.sh/"; \
+	    echo "ERROR: Homebrew is not installed and is required to install zeromq. Install it from https://brew.sh/"; \
 	    exit 1; \
 	  fi; \
 	else \
-	  echo "Unsupported OS: $(TARGETOS). Install development dependencies manually."; \
+	  echo "ERROR: Unsupported OS: $(TARGETOS). Install development dependencies manually."; \
 	  exit 1; \
 	fi

go.mod

@@ -26,7 +26,7 @@ require (
 	k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d
 	sigs.k8s.io/controller-runtime v0.22.4
 	sigs.k8s.io/gateway-api v1.4.0
-	sigs.k8s.io/gateway-api-inference-extension v1.2.0-rc.1
+	sigs.k8s.io/gateway-api-inference-extension v1.2.1
 )
 
 require (

go.sum

@@ -181,8 +181,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/llm-d/llm-d-kv-cache-manager v0.4.0-rc2 h1:l2Sm8W6SRg4TAme4RsndwZ++5+4aQvDI4vnf8TKrhww=
-github.com/llm-d/llm-d-kv-cache-manager v0.4.0-rc2/go.mod h1:ZlK7MCuz5D/weLeHyNKEmVF/eJZDyYn3XyRowTihq9o=
 github.com/llm-d/llm-d-kv-cache-manager v0.4.0 h1:MBWVpDW0PWsqNJEEAW1esrJW+Xavb0a7w14tCJWWyRY=
 github.com/llm-d/llm-d-kv-cache-manager v0.4.0/go.mod h1:ZlK7MCuz5D/weLeHyNKEmVF/eJZDyYn3XyRowTihq9o=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
@@ -411,8 +409,8 @@ sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327U
 sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8=
 sigs.k8s.io/gateway-api v1.4.0 h1:ZwlNM6zOHq0h3WUX2gfByPs2yAEsy/EenYJB78jpQfQ=
 sigs.k8s.io/gateway-api v1.4.0/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk=
-sigs.k8s.io/gateway-api-inference-extension v1.2.0-rc.1 h1:46O32J2MN4kqumN1rypbuI1N3AjDdRSJ8DrNP7p4WFA=
-sigs.k8s.io/gateway-api-inference-extension v1.2.0-rc.1/go.mod h1:/HWeqxuOMjFM56YwJ2Spt3qceK7Spz4hk6ZfXYgE9a8=
+sigs.k8s.io/gateway-api-inference-extension v1.2.1 h1:kQjnFWW8YLCN42EZxDNxTuDE0xHkPkoyaEVpQ5sNCBQ=
+sigs.k8s.io/gateway-api-inference-extension v1.2.1/go.mod h1:/HWeqxuOMjFM56YwJ2Spt3qceK7Spz4hk6ZfXYgE9a8=
 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
 sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=

pkg/sidecar/proxy/chat_completions.go

@@ -57,7 +57,7 @@ func (s *Server) chatCompletionsHandler(w http.ResponseWriter, r *http.Request)
 	if len(prefillHostPort) == 0 {
 		s.logger.V(4).Info("skip disaggregated prefill")
 
-		if s.forwardDataParallel && !s.dataParallelHandler(w, r) {
+		if !s.forwardDataParallel || !s.dataParallelHandler(w, r) {
 			s.decoderProxy.ServeHTTP(w, r)
 		}
 		return

pkg/sidecar/proxy/connector_lmcache.go

@@ -84,7 +84,8 @@ func (s *Server) runLMCacheProtocol(w http.ResponseWriter, r *http.Request, pref
 	// Forward original request to local decoder
 
 	r.Body = io.NopCloser(strings.NewReader(string(original)))
-	if s.forwardDataParallel && !s.dataParallelHandler(w, r) {
+	if !s.forwardDataParallel || !s.dataParallelHandler(w, r) {
+		s.logger.V(4).Info("sending request to decoder", "to", s.decoderURL.Host)
 		s.decoderProxy.ServeHTTP(w, r)
 	}
 }

pkg/sidecar/proxy/connector_nixlv2.go

@@ -168,7 +168,8 @@ func (s *Server) runNIXLProtocolV2(w http.ResponseWriter, r *http.Request, prefi
 	// 2. Forward to local decoder.
 
 	s.logger.V(5).Info("sending request to decoder", "body", string(dbody))
-	if s.forwardDataParallel && !s.dataParallelHandler(w, dreq) {
+	if !s.forwardDataParallel || !s.dataParallelHandler(w, dreq) {
+		s.logger.V(4).Info("sending request to decoder", "to", s.decoderURL.Host)
 		s.decoderProxy.ServeHTTP(w, dreq)
 	}
 }

pkg/sidecar/proxy/data_parallel.go

@@ -53,11 +53,11 @@ func (s *Server) startDataParallel(ctx context.Context, cert *tls.Certificate, g
 		decoderPort := strconv.Itoa(baseDecoderPort + idx + 1)
 		rankPort := strconv.Itoa(basePort + idx + 1)
 		hostPort := net.JoinHostPort(podIP, rankPort)
-		rankURL, err := url.Parse(s.decoderURL.Scheme + "://localhost:" + decoderPort)
+		decoderURL, err := url.Parse(s.decoderURL.Scheme + "://localhost:" + decoderPort)
 		if err != nil {
 			return err
 		}
-		handler := s.createDecoderProxyHandler(rankURL, s.config.DecoderInsecureSkipVerify)
+		handler := s.createDecoderProxyHandler(decoderURL, s.config.DecoderInsecureSkipVerify)
 		s.dataParallelProxies[hostPort] = handler
 	}
 
@@ -77,6 +77,7 @@ func (s *Server) startDataParallel(ctx context.Context, cert *tls.Certificate, g
 			clone.forwardDataParallel = false
 			// Configure handlers
 			clone.handler = clone.createRoutes()
+			clone.setConnector()
 
 			return clone.startHTTP(ctx, cert)
 		})