Merge pull request #130 from zdtsw-forking/prep_release_0.5_3

[sync]: upstream main change + update konflux dockerfile to match the change

Author: openshift-merge-bot[bot]

.github/workflows/ci-pr-checks.yaml

@@ -52,21 +52,15 @@ jobs:
           go-version: "${{ env.GO_VERSION }}"
           cache-dependency-path: ./go.sum
 
-      - name: Configure CGO for Python
+      - name: Configure CGO for ZMQ
         run: |
-          PYTHON_INCLUDE=$(python3 -c "import sysconfig; print(sysconfig.get_path('include'))")
-          echo "CPATH=${PYTHON_INCLUDE}:${CPATH}" >> $GITHUB_ENV
           echo "CGO_ENABLED=1" >> $GITHUB_ENV
-          echo "CGO_CFLAGS=$(python3-config --cflags --embed)" >> $GITHUB_ENV
-          echo "CGO_LDFLAGS=$(python3-config --ldflags --embed)" >> $GITHUB_ENV
-
-      - name: Set PKG_CONFIG_PATH
-        run: echo "PKG_CONFIG_PATH=/usr/lib/pkgconfig" >> $GITHUB_ENV
+          echo "PKG_CONFIG_PATH=/usr/lib/pkgconfig" >> $GITHUB_ENV
 
       - name: Install dependencies
         run: |
           go mod tidy
-          sudo -E env "PATH=$PATH" make install-dependencies install-python-deps
+          sudo -E env "PATH=$PATH" make install-dependencies
 
       - name: Run lint checks
         uses: golangci/golangci-lint-action@v9
@@ -76,9 +70,6 @@ jobs:
           skip-cache: true
         env:
           CGO_ENABLED: ${{ env.CGO_ENABLED }}
-          CGO_CFLAGS: ${{ env.CGO_CFLAGS }}
-          CGO_LDFLAGS: ${{ env.CGO_LDFLAGS }}
-          CPATH: ${{ env.CPATH }}
           PKG_CONFIG_PATH: ${{ env.PKG_CONFIG_PATH }}
 
       - name: Run make build

DEVELOPMENT.md

@@ -6,38 +6,37 @@ Documentation for developing the inference scheduler.
 
 - [Make] `v4`+
 - [Golang] `v1.24`+
-- [Python] `v3.12`
 - [Docker] (or [Podman])
 - [Kubernetes in Docker (KIND)]
 - [Kustomize]
+- [ZeroMQ]
 
 [Make]:https://www.gnu.org/software/make/
 [Golang]:https://go.dev/
-[Python]:https://www.python.org/
 [Docker]:https://www.docker.com/
 [Podman]:https://podman.io/
 [Kubernetes in Docker (KIND)]:https://github.com/kubernetes-sigs/kind
 [Kustomize]:https://kubectl.docs.kubernetes.io/installation/kustomize/
+[ZeroMQ]:https://zeromq.org/
 
-### Python Version Configuration
+> [!NOTE]
+> **Python is NOT required** as of v0.5.1. Tokenization is handled by a separate UDS (Unix Domain Socket) tokenizer sidecar container. Previous versions (< v0.5.1) used embedded Python tokenizers with daulet/tokenizers bindings, but these are now deprecated.
 
-The project uses Python 3.12 by default, but this can be configured:
+## Tokenization Architecture
 
-**For local development:**
-`PYTHON_VERSION` in the Makefile set which Python version is used.
+The project uses **UDS (Unix Domain Socket)** tokenization. Tokenization is handled by a separate UDS tokenizer sidecar container, not by the EPP container itself. Previous embedded tokenizer approaches (daulet/tokenizers, direct Python/vLLM linking) are deprecated and no longer used.
 
-**For Docker builds:**
-The Python version is parameterized in the Dockerfile via the `PYTHON_VERSION` build argument, which defaults to 3.12. To build with a different Python version:
+**Building the UDS tokenizer image:**
 
 ```bash
-PYTHON_VERSION=3.13 make image-build
-
-# Or directly with Docker
-docker build --build-arg PYTHON_VERSION=3.13 -f Dockerfile.epp .
+make image-build-uds-tokenizer
 ```
 
-**For CI/CD:**
-Workflow uses Python 3.12 by default. The version can be set by modifying the `python-version` input in workflow file.
+The image is tagged as `ghcr.io/llm-d/llm-d-uds-tokenizer:dev` by default. Override with:
+
+```bash
+UDS_TOKENIZER_TAG=v1.0.0 make image-build-uds-tokenizer
+```
 
 ## Kind Development Environment
 

Dockerfile.epp

@@ -1,9 +1,23 @@
 ## Minimal runtime Dockerfile (microdnf-only, no torch, wrapper in site-packages)
-# Go dependencies stage: download go modules and extract kv-cache
-FROM quay.io/projectquay/golang:1.25 AS go-deps
+## Simplified EPP Dockerfile - UDS tokenizer only (no vLLM, no embedded tokenizer)
+## This build uses the default kv-cache pool (UDS-only, no embedded_tokenizers build tag)
+## Tokenization is handled by a separate UDS tokenizer sidecar container
+##
+## CGO is still required for ZMQ (kvevents) but Python/vLLM dependencies are removed
+# Go build stage
+FROM quay.io/projectquay/golang:1.25 AS go-builder
+
+ARG TARGETOS
+ARG TARGETARCH
 
 WORKDIR /workspace
 
+# Install ZMQ development libraries (required for CGO)
+# The builder is based on UBI8, so we need epel-release-8
+RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' && \
+    dnf install -y zeromq-devel pkgconfig && \
+    dnf clean all
+
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
@@ -14,98 +28,26 @@ COPY pkg/ pkg/
 
 RUN go mod download
 
-# Copy Python wrapper and requirements from llm-d-kv-cache dependency
-# Extract version dynamically and copy to a known location
-RUN KV_CACHE_PKG=$(go list -m -f '{{.Dir}}' github.com/llm-d/llm-d-kv-cache) && \
-    mkdir -p /workspace/kv-cache && \
-    cp -r $KV_CACHE_PKG/* /workspace/kv-cache
-
-FROM python:3.12-slim AS python-builder
-
-ARG TARGETARCH
-
-COPY --from=go-deps /workspace/kv-cache /workspace/kv-cache
-WORKDIR /workspace/kv-cache
-
-# Create venv and install vLLM based on architecture using pre-built wheels
-RUN python3.12 -m venv /workspace/kv-cache/build/venv && \
-    . /workspace/kv-cache/build/venv/bin/activate && \
-    pip install --upgrade pip && \
-    VLLM_VERSION="0.14.0" && \
-    if [ "$TARGETARCH" = "arm64" ]; then \
-        pip install https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cpu-cp38-abi3-manylinux_2_35_aarch64.whl; \
-    elif [ "$TARGETARCH" = "amd64" ]; then \
-        pip install https://github.com/vllm-project/vllm/releases/download/v${VLLM_VERSION}/vllm-${VLLM_VERSION}+cpu-cp38-abi3-manylinux_2_35_x86_64.whl --extra-index-url https://download.pytorch.org/whl/cpu; \
-    else \
-        echo "ERROR: Unsupported architecture: $TARGETARCH. Only arm64 and amd64 are supported." && exit 1; \
-    fi
-
-# Go build stage
-FROM quay.io/projectquay/golang:1.25 AS go-builder
-
-ARG TARGETOS
-ARG TARGETARCH
-ARG PYTHON_VERSION=3.12
-ENV PYTHON=python${PYTHON_VERSION}
-
-# Install build tools
-# The builder is based on UBI8, so we need epel-release-8.
-# ${PYTHON}-devel needed for CGO compilation (Python headers and ${PYTHON}-config for linker flags)
-RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' && \
-    dnf install -y gcc-c++ libstdc++ libstdc++-devel clang zeromq-devel pkgconfig ${PYTHON}-devel ${PYTHON}-pip git && \
-    dnf clean all
-
-COPY --from=go-deps /workspace /workspace
-COPY --from=go-deps /go/pkg/mod /go/pkg/mod
-
-WORKDIR /workspace
-
-COPY Makefile* ./
-
-COPY --from=python-builder /workspace/kv-cache/pkg/preprocessing/chat_completions /workspace/kv-cache/pkg/preprocessing/chat_completions
-RUN make setup-venv
-COPY --from=python-builder /workspace/kv-cache/build/venv/lib/python3.12/site-packages /workspace/build/venv/lib/python3.12/site-packages
-
-ENV PYTHONPATH=/workspace/kv-cache/pkg/preprocessing/chat_completions:/workspace/build/venv/lib/python3.12/site-packages
-RUN python3.12 -c "import tokenizer_wrapper"  # verify tokenizer_wrapper is correctly installed
-
-ARG RELEASE_VERSION=v1.22.1
-RUN TOKENIZER_VERSION=${RELEASE_VERSION} make build-epp
+# Build EPP with CGO for ZMQ only (no Python, no embedded tokenizer)
+# The default kv-cache build uses UDS tokenizer (//go:build !embedded_tokenizers)
+RUN CGO_ENABLED=1 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o bin/epp cmd/epp/main.go
 
 # Runtime stage
 # Use ubi9 as a minimal base image to package the manager binary
 # Refer to https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5 for more details
 FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7
-ARG PYTHON_VERSION=3.12
-WORKDIR /
-COPY --from=go-builder /workspace/bin/epp /app/epp
 
-USER root
+WORKDIR /
 
-ENV PYTHON=python${PYTHON_VERSION}
-# Install zeromq runtime library and Python runtime needed by the manager.
-# The final image is UBI9, so we need epel-release-9.
-# Using microdnf for minimal image size
+# Install ZMQ runtime library only (no Python needed)
 RUN curl -L -o /tmp/epel-release.rpm https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     rpm -i /tmp/epel-release.rpm && \
     rm /tmp/epel-release.rpm && \
-    microdnf install -y --setopt=install_weak_deps=0 zeromq ${PYTHON} ${PYTHON}-libs ${PYTHON}-pip && \
+    microdnf install -y --setopt=install_weak_deps=0 zeromq && \
     microdnf clean all && \
-    rm -rf /var/cache/yum /var/lib/yum && \
-    # Note: ${PYTHON} package does not automatically create python3/python symlinks - they must be created manually
-    ln -sf /usr/bin/${PYTHON} /usr/bin/python3 && \
-    ln -sf /usr/bin/${PYTHON} /usr/bin/python
+    rm -rf /var/cache/yum /var/lib/yum
 
-# Copy Python kv-cache package and site-packages from the python-builder stage
-COPY --from=python-builder /workspace/kv-cache /workspace/kv-cache
-ENV PYTHONPATH=/workspace/kv-cache/pkg/preprocessing/chat_completions:/workspace/kv-cache/build/venv/lib/python3.12/site-packages
-RUN ${PYTHON} -c "import tokenizer_wrapper"  # verify tokenizer_wrapper is correctly installed
-
-ENV HF_HOME="/tmp/.cache"
-# used by kv-cache-manager
-ENV LOCAL_TOKENIZER_DIR="/tmp/.cache"
-# Create cache directory and set permissions for non-root user
-RUN mkdir -p /tmp/.cache && chown -R 65532:65532 ${HF_HOME}
+COPY --from=go-builder /workspace/bin/epp /app/epp
 
 USER 65532:65532
 

Dockerfile.epp.konflux

@@ -1,20 +1,24 @@
 ## Minimal runtime Dockerfile (microdnf-only, no torch, wrapper in site-packages)
-# Build Stage: using Go 1.24 image
-FROM registry.access.redhat.com/ubi9/go-toolset:1.24 AS builder
+## Simplified EPP Dockerfile - UDS tokenizer only (no vLLM, no embedded tokenizer)
+## This build uses the default kv-cache pool (UDS-only, no embedded_tokenizers build tag)
+## Tokenization is handled by a separate UDS tokenizer sidecar container
+##
+## CGO is still required for ZMQ (kvevents) but Python/vLLM dependencies are removed
+# Go build stage
+FROM registry.access.redhat.com/ubi9/go-toolset:1.25.7 AS go-builder
 
 ARG TARGETOS
 ARG TARGETARCH
 
 USER root
 
-# Install build tools
-# The builder is based on UBI8, so we need epel-release-8.
+WORKDIR /workspace
+
+# Install ZMQ development libraries (required for CGO)
+# The builder is based on UBI8, so we need epel-release-9
 RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' && \
-    dnf install -y gcc-c++ libstdc++ libstdc++-devel clang zeromq-devel pkgconfig python3.12-devel python3.12-pip git && \
+    dnf install -y zeromq-devel pkgconfig && \
     dnf clean all
-# python3.12-devel needed for CGO compilation (Python headers and python3.12-config for linker flags)
-
-WORKDIR /workspace
 
 # Copy the Go Modules manifests
 COPY go.mod go.mod
@@ -26,79 +30,26 @@ COPY pkg/ pkg/
 
 RUN go mod download
 
-# Copy Python wrapper and requirements from kv-cache-manager dependency
-# Extract version dynamically and copy to a known location
-RUN KVCACHE_MANAGER_VERSION=$(go list -m -f '{{.Version}}' github.com/llm-d/llm-d-kv-cache-manager) && \
-    mkdir -p /workspace/kv-cache-manager-wrapper && \
-    cp $(go env GOMODCACHE)/github.com/llm-d/llm-d-kv-cache-manager@${KVCACHE_MANAGER_VERSION}/pkg/preprocessing/chat_completions/render_jinja_template_wrapper.py \
-       /workspace/kv-cache-manager-wrapper/ && \
-    cp $(go env GOMODCACHE)/github.com/llm-d/llm-d-kv-cache-manager@${KVCACHE_MANAGER_VERSION}/pkg/preprocessing/chat_completions/requirements.txt \
-       /workspace/kv-cache-manager-wrapper/requirements.txt
-
-# HuggingFace tokenizer bindings (static lib)
-RUN mkdir -p lib
-# Ensure that the RELEASE_VERSION matches the one used in the imported llm-d-kv-cache-manager version
-ARG RELEASE_VERSION=v1.22.1
-RUN curl -L https://github.com/daulet/tokenizers/releases/download/${RELEASE_VERSION}/libtokenizers.${TARGETOS}-${TARGETARCH}.tar.gz | tar -xz -C lib
-RUN ranlib lib/*.a
-
-# Build
-# the GOARCH has not a default value to allow the binary be built according to the host where the command
-# was called. For example, if we call make image-build in a local env which has the Apple Silicon M1 SO
-# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
-# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-ENV CGO_ENABLED=1
-ENV GOOS=${TARGETOS:-linux}
-ENV GOARCH=${TARGETARCH}
-ENV PYTHON=python3.12
-ENV PYTHONPATH=/usr/lib64/python3.12/site-packages:/usr/lib/python3.12/site-packages
-
-ARG COMMIT_SHA=unknown
-ARG BUILD_REF
-RUN export CGO_CFLAGS="$(python3.12-config --cflags) -I/workspace/lib" && \
-    export CGO_LDFLAGS="$(python3.12-config --ldflags --embed) -L/workspace/lib -ltokenizers -ldl -lm" && \
-    go build -a -o bin/epp -ldflags="-extldflags '-L$(pwd)/lib' -X sigs.k8s.io/gateway-api-inference-extension/version.CommitSHA=${COMMIT_SHA} -X sigs.k8s.io/gateway-api-inference-extension/version.BuildRef=${BUILD_REF}" cmd/epp/main.go
+# Build EPP with CGO for ZMQ only (no Python, no embedded tokenizer)
+# The default kv-cache build uses UDS tokenizer (//go:build !embedded_tokenizers)
+RUN CGO_ENABLED=1 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o bin/epp cmd/epp/main.go
 
 # Runtime stage
 # Use ubi9 as a minimal base image to package the manager binary
 # Refer to https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5 for more details
-FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.7
+
 WORKDIR /
-COPY --from=builder /workspace/bin/epp /app/epp
 
-USER root
-# Install zeromq runtime library and Python runtime needed by the manager.
-# The final image is UBI9, so we need epel-release-9.
-# Using microdnf for minimal image size
+# Install ZMQ runtime library only (no Python needed)
 RUN curl -L -o /tmp/epel-release.rpm https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     rpm -i /tmp/epel-release.rpm && \
     rm /tmp/epel-release.rpm && \
-    microdnf install -y --setopt=install_weak_deps=0 zeromq python3.12 python3.12-libs python3.12-pip && \
+    microdnf install -y --setopt=install_weak_deps=0 zeromq && \
     microdnf clean all && \
-    rm -rf /var/cache/yum /var/lib/yum && \
-    ln -sf /usr/bin/python3.12 /usr/bin/python3 && \
-    ln -sf /usr/bin/python3.12 /usr/bin/python
-# Note: python3.12 package does not automatically create python3/python symlinks - they must be created manually
-
-# Install wrapper as a module in site-packages
-RUN mkdir -p /usr/local/lib/python3.12/site-packages/
-COPY --from=builder /workspace/kv-cache-manager-wrapper/render_jinja_template_wrapper.py /usr/local/lib/python3.12/site-packages/
-
-# Python deps (no cache, single target) – filter out torch
-ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
-COPY --from=builder /workspace/kv-cache-manager-wrapper/requirements.txt /tmp/requirements.txt
-RUN sed '/^torch\b/d' /tmp/requirements.txt > /tmp/requirements.notorch.txt && \
-    python3.12 -m pip install --no-cache-dir --upgrade pip setuptools wheel && \
-    python3.12 -m pip install --no-cache-dir --target /usr/local/lib/python3.12/site-packages -r /tmp/requirements.notorch.txt && \
-    python3.12 -m pip install --no-cache-dir --target /usr/local/lib/python3.12/site-packages PyYAML && \
-    rm /tmp/requirements.txt /tmp/requirements.notorch.txt && \
-    rm -rf /root/.cache/pip
+    rm -rf /var/cache/yum /var/lib/yum
 
-# Python env
-ENV PYTHONPATH="/usr/local/lib/python3.12/site-packages:/usr/lib/python3.12/site-packages"
-ENV PYTHON=python3.12
-ENV PATH=/usr/bin:/usr/local/bin:$PATH
-ENV HF_HOME="/tmp/.cache"
+COPY --from=go-builder /workspace/bin/epp /app/epp
 
 USER 65532:65532
 

Dockerfile.sidecar.konflux

@@ -34,6 +34,6 @@ RUN go build -a -tags strictfipsruntime -o bin/pd-sidecar \
 FROM registry.access.redhat.com/ubi9/ubi-micro:9.7
 WORKDIR /
 COPY --from=builder /workspace/bin/pd-sidecar /app/pd-sidecar
-USER 1001:1001
+USER 65532:65532
 
 ENTRYPOINT ["/app/pd-sidecar"]