update: add comments for download-tokenizer and remove as dependecy to

build

Signed-off-by: Wen Zhou <wenzhou@redhat.com>

Author: zdtsw

Makefile

@@ -80,7 +80,8 @@ BUILD_REF ?= $(shell git describe --abbrev=0 2>/dev/null)
 # go source files
 SRC = $(shell find . -type f -name '*.go')
 
-# Tokenizer & Linking
+# DEPRECATED: Tokenizer & Linking flags (only used for legacy embedded tokenizer builds)
+# UDS builds don't need these flags - ZMQ linking is handled automatically via pkg-config
 LDFLAGS ?= -extldflags '-L$(LOCALLIB)'
 CGO_ENABLED=1
 
@@ -111,7 +112,8 @@ endif
 PYTHON_CFLAGS := $(shell $(PYTHON_CONFIG) --cflags)
 PYTHON_LDFLAGS := $(shell $(PYTHON_CONFIG) --ldflags --embed)
 
-# CGO flags with all dependencies
+# DEPRECATED: CGO flags with Python and tokenizers (only used for legacy embedded tokenizer builds)
+# The default UDS build only needs ZMQ
 CGO_CFLAGS := $(PYTHON_CFLAGS) '-I$(shell pwd)/lib'
 CGO_LDFLAGS := $(PYTHON_LDFLAGS) $(PYTHON_LIBS) '-L$(shell pwd)/lib' -ltokenizers -ldl -lm
 
@@ -121,11 +123,12 @@ epp_IMAGE = $(EPP_IMAGE)
 sidecar_IMAGE = $(SIDECAR_IMAGE)
 epp_NAME = epp
 sidecar_NAME = $(SIDECAR_NAME)
-epp_LDFLAGS = -ldflags="$(LDFLAGS)"
+# Empty flags for UDS-only builds (ZMQ linking handled via pkg-config)
+epp_LDFLAGS =
 sidecar_LDFLAGS =
-epp_CGO_CFLAGS = "${CGO_CFLAGS}"
+epp_CGO_CFLAGS =
 sidecar_CGO_CFLAGS =
-epp_CGO_LDFLAGS = "${CGO_LDFLAGS}"
+epp_CGO_LDFLAGS =
 sidecar_CGO_LDFLAGS =
 epp_TEST_FILES = go list ./... | grep -v /test/ | grep -v ./pkg/sidecar/
 sidecar_TEST_FILES = go list ./pkg/sidecar/...
@@ -171,14 +174,14 @@ test: test-unit test-e2e ## Run all tests (unit and e2e)
 test-unit: test-unit-epp test-unit-sidecar ## Run unit tests
 
 .PHONY: test-unit-%
-test-unit-%: download-tokenizer install-python-deps check-dependencies ## Run unit tests
+test-unit-%: check-dependencies ## Run unit tests
 	@printf "\033[33;1m==== Running Unit Tests ====\033[0m\n"
 	@KV_CACHE_PKG=$$(go list -m -f '{{.Dir}}/pkg/preprocessing/chat_completions' github.com/llm-d/llm-d-kv-cache 2>/dev/null || echo ""); \
 	PYTHONPATH="$$KV_CACHE_PKG:$(VENV_DIR)/lib/python$(PYTHON_VERSION)/site-packages" \
 	CGO_CFLAGS=${$*_CGO_CFLAGS} CGO_LDFLAGS=${$*_CGO_LDFLAGS} go test $($*_LDFLAGS) -v $$($($*_TEST_FILES) | tr '\n' ' ')
 
 .PHONY: test-filter
-test-filter: download-tokenizer install-python-deps check-dependencies ## Run filtered unit tests (usage: make test-filter PATTERN=TestName TYPE=epp)
+test-filter: check-dependencies ## Run filtered unit tests (usage: make test-filter PATTERN=TestName TYPE=epp)
 	@if [ -z "$(PATTERN)" ]; then \
 		echo "ERROR: PATTERN is required. Usage: make test-filter PATTERN=TestName [TYPE=epp|sidecar]"; \
 		exit 1; \
@@ -197,7 +200,7 @@ test-filter: download-tokenizer install-python-deps check-dependencies ## Run fi
 	fi
 
 .PHONY: test-integration
-test-integration: download-tokenizer check-dependencies ## Run integration tests
+test-integration: check-dependencies ## Run integration tests
 	@printf "\033[33;1m==== Running Integration Tests ====\033[0m\n"
 	go test -ldflags="$(LDFLAGS)" -v -tags=integration_tests ./test/integration/
 
@@ -218,7 +221,7 @@ post-deploy-test: ## Run post deployment tests
 build: build-epp build-sidecar ## Build the project for both epp and sidecar
 
 .PHONY: build-%
-build-%: check-go download-tokenizer ## Build the project
+build-%: check-go ## Build the project
 	@printf "\033[33;1m==== Building ====\033[0m\n"
 	CGO_CFLAGS=${$*_CGO_CFLAGS} CGO_LDFLAGS=${$*_CGO_LDFLAGS} go build $($*_LDFLAGS) -o bin/$($*_NAME) cmd/$($*_NAME)/main.go
 

Makefile.tools.mk

@@ -22,8 +22,9 @@ VLLM_VERSION ?= 0.14.0
 
 ## Python Configuration
 PYTHON_VERSION ?= 3.12
-# Extract RELEASE_VERSION from Dockerfile
-TOKENIZER_VERSION ?= $(shell grep '^ARG RELEASE_VERSION=' Dockerfile.epp | cut -d'=' -f2)
+# DEPRECATED: Hardcoded tokenizer version for legacy embedded tokenizer support
+# Embedded tokenizers are deprecated. Use UDS tokenizer instead (see DEVELOPMENT.md)
+TOKENIZER_VERSION ?= v1.22.1
 
 # Python executable for creating venv
 PYTHON_EXE := $(shell command -v python$(PYTHON_VERSION) || command -v python3)
@@ -143,14 +144,18 @@ install-dependencies: ## Install development dependencies based on OS/ARCH
 	  exit 1; \
 	fi
 
+## DEPRECATED: download-tokenizer target
+## This target downloads embedded tokenizer bindings (daulet/tokenizers) which are deprecated.
+## The project now uses UDS (Unix Domain Socket) tokenizers via a separate sidecar container.
 .PHONY: download-tokenizer
 download-tokenizer: $(TOKENIZER_LIB)
 $(TOKENIZER_LIB): | $(LOCALLIB)
-	## Download the HuggingFace tokenizer bindings.
-	@echo "Downloading HuggingFace tokenizer bindings for version $(TOKENIZER_VERSION)..."
+	## Download the HuggingFace tokenizer bindings (DEPRECATED - use UDS tokenizer instead).
+	@echo "WARNING: Downloading deprecated embedded tokenizer bindings for version $(TOKENIZER_VERSION)..."
+	@echo "WARNING: Embedded tokenizers are deprecated. Use UDS tokenizer instead (see DEVELOPMENT.md)"
 	@curl -L https://github.com/daulet/tokenizers/releases/download/$(TOKENIZER_VERSION)/libtokenizers.$(TARGETOS)-$(TOKENIZER_ARCH).tar.gz | tar -xz -C $(LOCALLIB)
 	@ranlib $(LOCALLIB)/*.a
-	@echo "Tokenizer bindings downloaded successfully."
+	@echo "Tokenizer bindings downloaded successfully (DEPRECATED)."
 
 .PHONY: detect-python
 detect-python: ## Detects Python and prints the configuration.