[integration_test] drop dev-machine assumptions from the vllm e2e harness

- delete the immature test-vllm-e2e workflow (runner/model/venv/uv are all
  dev-machine specific); e2e CI will land as a separate PR
- KVCM_E2E_MODEL / KVCM_E2E_PYTHON become required env vars with clear
  errors instead of local-path defaults
- remove the unused cosine-similarity fallback: every scenario is verified
  bit-exact
- tag the e2e targets manual (GPU + vLLM venv + model required) and add an
  explicit :e2e_tests suite
- document all environment variables in the README

Author: lpdink

.github/workflows/test-vllm-e2e.yml

@@ -14,130 +14,155 @@ py_library(
 
 py_test(
     name = "test_basic",
+    timeout = "eternal",
     srcs = ["test_basic.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
 
 py_test(
     name = "test_concurrent",
+    timeout = "eternal",
     srcs = ["test_concurrent.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
 
 py_test(
     name = "test_tp",
+    timeout = "eternal",
     srcs = ["test_tp.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
 
 py_test(
     name = "test_full_hit",
+    timeout = "eternal",
     srcs = ["test_full_hit.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
 
 py_test(
     name = "test_partial_hit",
+    timeout = "eternal",
     srcs = ["test_partial_hit.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
 
 py_test(
     name = "test_load_failure",
+    timeout = "eternal",
     srcs = ["test_load_failure.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
 
 py_test(
     name = "test_multi_turn",
+    timeout = "eternal",
     srcs = ["test_multi_turn.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
 
 # Meta-test: injects an off-by-one into the connector's token translation and
 # asserts the KV verification FAILS -- proof the harness is not vacuous.
 py_test(
     name = "test_mutation",
+    timeout = "eternal",
     srcs = ["test_mutation.py"],
     data = [
         "//kv_cache_manager:kv_cache_manager_bin",
     ],
     imports = ["."],
     tags = [
-        "no-remote-exec",
-        "gpu",  # requires 1+ GPU
         "exclusive",  # GPU tests must run serially to avoid CUDA OOM contention
+        "gpu",  # requires 1+ GPU
+        "manual",  # needs a GPU machine + vLLM venv + model; see README.md
+        "no-remote-exec",
     ],
-    timeout = "eternal",
     deps = [":e2e_lib"],
 )
+
+# Wildcards skip "manual" targets; run the whole suite explicitly with
+#   bazelisk test //integration_test/vllm_e2e:e2e_tests
+test_suite(
+    name = "e2e_tests",
+    tags = ["manual"],  # keep wildcard builds from expanding into the GPU tests
+    tests = [
+        ":test_basic",
+        ":test_concurrent",
+        ":test_full_hit",
+        ":test_load_failure",
+        ":test_multi_turn",
+        ":test_mutation",
+        ":test_partial_hit",
+        ":test_tp",
+    ],
+)

integration_test/vllm_e2e/BUILD

@@ -29,8 +29,8 @@ block-table mapping:
 2. **Phase 2** — same prompts + suffix: connector reports an external match and
    loads from KVCM. The loaded blocks are captured (**loaded** captures).
 3. The driver (`e2e_lib.py`) matches loaded captures against references by
-   token content and compares per layer: bit-exact preferred, cosine
-   similarity > 99.99% as fallback.
+   token content and compares per layer, requiring bit-exact equality (the
+   transfer is a verbatim byte round trip; all scenarios achieve it).
 
 ## Model coverage
 
@@ -56,9 +56,18 @@ Hybrid specifics verified:
 | `test_basic` | 1 | 1 | Minimal save -> load round trip |
 | `test_concurrent` | 1 | 4 | Concurrent requests: ReqState tracking, per-request block attribution |
 | `test_tp` | 2 | 2 | TP coordination; for full-attention models also `preferred_block_size=32` != vLLM block size (16), forcing real cross-block translation |
+| `test_partial_hit` | 1 | 1 | Phase 2 extends the prompt mid-block: partial external hit |
+| `test_full_hit` | 1 | 1 | Phase 2 resends the identical prompt: full-prompt hit is capped so >= 1 token is recomputed |
+| `test_multi_turn` | 1 | 1 | Growing conversation: each turn loads the previous turns' blocks and saves new ones |
+| `test_load_failure` | 1 | 1 | Storage files deleted between phases: load fails, retry loop must not spin, request still completes |
+| `test_mutation` | 1 | 1 | Meta-test: injected off-by-one in the slot translation must make verification FAIL (proves the harness is not vacuous) |
 
 ## Running
 
+These targets are tagged `manual`: they need a GPU machine with a prepared
+vLLM venv and a local model, so `bazelisk test //integration_test/...` skips
+them and they must be requested explicitly (see below).
+
 Build prerequisites (from the repo root):
 
 ```bash
@@ -75,19 +84,27 @@ from the wheel's `METADATA`).
 Run (tagged `exclusive`, so they execute serially):
 
 ```bash
-bazelisk test //integration_test/vllm_e2e/... \
+bazelisk test //integration_test/vllm_e2e:e2e_tests \
   --cache_test_results=no --test_output=errors \
   --test_env=KVCM_E2E_PYTHON=/path/to/vllm-venv/bin/python \
   --test_env=KVCM_E2E_MODEL=/path/to/model \
   --per_file_copt='external/jsoncpp_git/.*@-Wno-error'
 ```
 
-Environment variables:
+## Environment variables
+
+All environment variables used by the e2e harness:
+
+| Variable | Required | Meaning |
+|---|---|---|
+| `KVCM_E2E_MODEL` | yes | Path to a local HF model directory (`config.json` + weights). Full-attention coverage needs a plain attention model (e.g. Qwen2.5-7B-Instruct); hybrid coverage needs a mamba/linear + attention model (e.g. Qwen3.5-4B). Hybrid models are auto-detected from `config.json`. |
+| `KVCM_E2E_PYTHON` | yes | Python interpreter of a venv with vLLM >= 0.26.0 and both KVCM wheels (`kvcm_py_client`, `kvcm_vllm_connector`) installed. |
+| `KVCM_E2E_CAPTURE_DIR` | internal | Set by the driver for the vLLM subprocess; tells `VerifyingConnector` where to write `.pt` captures. Do not set manually. |
 
-| Variable | Meaning |
-|---|---|
-| `KVCM_E2E_PYTHON` | Python interpreter with vLLM + both KVCM wheels installed |
-| `KVCM_E2E_MODEL` | Model path; hybrid models are auto-detected from `config.json` |
+The driver also sets vLLM knobs for the spawned server (`VLLM_KV_CACHE_LAYOUT=NHD`,
+`VLLM_ATTENTION_BACKEND=FLASH_ATTN`, `VLLM_USE_FLASHINFER_SAMPLER=0`,
+`FLASHINFER_DISABLE_VERSION_CHECK=1`) via `env.setdefault`, so a value you
+export yourself wins.
 
 ## Debugging