fix: deprecate embedded tokenizers

.github/workflows/ci-lint.yaml

@@ -26,7 +26,7 @@ jobs:
           mkdir -p apt-archives
           sudo cp -a apt-archives /var/cache/apt/archives # workaround for apt-get cache https://github.com/actions/cache/issues/324#issuecomment-1816908646
           sudo apt-get update
-          sudo apt-get install -y libzmq3-dev pkg-config python3.12-dev python3.12-venv clang-format
+          sudo apt-get install -y libzmq3-dev pkg-config clang-format
           cp -a /var/cache/apt/archives/*.deb apt-archives 2>/dev/null || true
 
       - name: Extract Go version from go.mod
@@ -38,25 +38,8 @@ jobs:
           go-version: "${{ env.GO_VERSION }}"
           cache-dependency-path: ./go.sum
 
-      - name: Cache Python (pip) dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pip
-            build/venv
-          # This key is based ONLY on the requirements file
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/.github/workflows/ci-lint.yaml') }}-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      #  Set up the Python virtual environment (includes Python config verification)
-      - name: Run setup-venv
-        run: make setup-venv
-
       - name: Install dependencies
-        run: |
-          go mod download
-          make install-python-deps
+        run: go mod download
 
       - name: Run pre-commit hooks
         id: pre-commit
@@ -70,18 +53,8 @@ jobs:
           version: v2.1.6
           args: "--help"
 
-      - name: Run precommit checks with CGo environment
-        run: |
-          # Export the CGO flags required for the linter to parse CGo files.
-          # This duplicates some logic from the Makefile but is necessary for the CI step.
-          CGO_CFLAGS="$(python3.12-config --cflags)"
-          CGO_LDFLAGS="$(python3.12-config --ldflags --embed) -ldl -lm"
-          export CGO_CFLAGS
-          export CGO_LDFLAGS
-          export CGO_ENABLED=1
-
-          # Now run the linting command from the Makefile
-          make precommit
+      - name: Run precommit checks
+        run: make precommit
 
       - name: Run C/C++/CUDA formatting check
         run: make clang

.gitignore

@@ -76,13 +76,6 @@ Thumbs.db
 # cuFile deployment config (contains deployment-specific RDMA NIC IPs)
 cufile.json
 
-# cgo build output (if any)
-*.a
-*.o
-*.cgo1.go
-*.cgo2.c
-_cgo_*
-
 # Gopls and tools
 .gopls/
 /hack/tools

Dockerfile

@@ -12,19 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM python:3.12-slim AS python-builder
-
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-
-WORKDIR /workspace
-
-RUN apt-get update && apt-get install -y --no-install-recommends build-essential
-
-COPY Makefile Makefile
-COPY pkg/preprocessing/chat_completions/ pkg/preprocessing/chat_completions/
-RUN TARGETOS=${TARGETOS} TARGETARCH=${TARGETARCH} make install-python-deps
-
 # Build Stage: using Go 1.24.1 image
 FROM quay.io/projectquay/golang:1.24 AS builder
 ARG TARGETOS
@@ -34,11 +21,10 @@ WORKDIR /workspace
 
 # Install system-level dependencies first. This layer is very stable.
 USER root
-# Install EPEL repository directly and then ZeroMQ, as epel-release is not in default repos.
-# Install all necessary dependencies including Python 3.12 for chat-completions templating.
+# Install EPEL repository directly and then ZeroMQ.
 # The builder is based on UBI8, so we need epel-release-8.
 RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' && \
-    dnf install -y gcc-c++ libstdc++ libstdc++-devel clang zeromq-devel pkgconfig python3.12-devel python3.12-pip && \
+    dnf install -y zeromq-devel pkgconfig && \
     dnf clean all
 
 # Copy the Go Modules manifests
@@ -51,16 +37,6 @@ RUN go mod download
 # Copy the source code.
 COPY . .
 
-# Copy this project's own Python source code into the final image
-COPY --from=python-builder /workspace/pkg/preprocessing/chat_completions /workspace/pkg/preprocessing/chat_completions
-RUN make setup-venv
-COPY --from=python-builder /workspace/build/venv/lib/python3.12/site-packages /workspace/build/venv/lib/python3.12/site-packages
-
-# Set the PYTHONPATH. This mirrors the Makefile's export, ensuring both this project's
-# Python code and the installed libraries (site-packages) are found at runtime.
-ENV PYTHONPATH=/workspace/pkg/preprocessing/chat_completions:/workspace/build/venv/lib/python3.12/site-packages
-RUN python3.12 -c "import tokenizer_wrapper"
-
 RUN make build
 
 # Use distroless as minimal base image to package the manager binary
@@ -71,18 +47,9 @@ WORKDIR /
 # The final image is UBI9, so we need epel-release-9.
 USER root
 RUN dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' && \
-    dnf install -y zeromq libxcrypt-compat python3.12 python3.12-pip && \
+    dnf install -y zeromq && \
     dnf clean all
 
-# Copy this project's own Python source code into the final image
-COPY --from=python-builder /workspace/pkg/preprocessing/chat_completions /app/pkg/preprocessing/chat_completions
-COPY --from=python-builder /workspace/build/venv/lib/python3.12/site-packages /workspace/build/venv/lib/python3.12/site-packages
-
-# Set the PYTHONPATH. This mirrors the Makefile's export, ensuring both this project's
-# Python code and the installed libraries (site-packages) are found at runtime.
-ENV PYTHONPATH=/app/pkg/preprocessing/chat_completions:/workspace/build/venv/lib/python3.12/site-packages
-RUN python3.12 -c "import tokenizer_wrapper"
-
 # Copy the compiled Go application
 COPY --from=builder /workspace/bin/llm-d-kv-cache /app/kv-cache-manager
 USER 65532:65532