ADR-0007 level-3: GPU image variant, rehearsed on kind 7/7 + baseline 6/6

Dockerfile parameterized via build args: default unchanged (musl
static on distroless/static), GPU variant builds the gnu target with
feature gpu-nvidia on distroless/cc-debian12 -- nvml-wrapper dlopens
libnvidia-ml.so at runtime, which a static musl binary cannot do, so
the GPU build must be dynamically linked (verified: interpreter
present in the builder stage). NVIDIA driver libs are injected by the
node container runtime, never baked in.

Rehearsal script grows GPU_VARIANT=1: same cluster, same asserts,
but the image is the exact level-3 GPU build with REPORTER_GPU=nvidia
set. kind has no GPU, so the double gate must fail open -- assert 7
requires the NVML init-failure warning on every reporter. Also fixes
the helm image tag to derive from IMG (was hardcoded, broke any
variant tag with ErrImageNeverPull).

Evidence (local runs/, gitignored by design -- kind rehearsal evidence
stays off-repo, GPU-session evidence is what gets committed):
GPU variant 7/7 (20260722T162142), baseline re-run 6/6 (20260722T162312).

Author: MicheleCampi

Dockerfile

@@ -1,12 +1,23 @@
 # syntax=docker/dockerfile:1.7
+# ---- Build args ------------------------------------------------------------
+# Default: static musl binary on distroless/static (CPU-only path, unchanged).
+# GPU variant (level-3 sessions): nvml-wrapper dlopens libnvidia-ml.so at
+# runtime, which a static musl binary cannot do. Build with:
+#   --build-arg RUST_TARGET=x86_64-unknown-linux-gnu \
+#   --build-arg CARGO_FEATURES=gpu-nvidia \
+#   --build-arg RUNTIME_IMAGE=gcr.io/distroless/cc-debian12:nonroot
+# (distroless/cc = glibc + dynamic loader; NVIDIA driver libs are injected
+# on the node by the NVIDIA container runtime, never baked into the image.)
+ARG RUNTIME_IMAGE=gcr.io/distroless/static:nonroot
 # ---- Builder ---------------------------------------------------------------
 # Pinned to the dev toolchain (1.95) for reproducible builds.
-# Static musl target -> single self-contained binary, no shared libs at runtime.
 FROM rust:1.95-bookworm AS builder
+ARG RUST_TARGET=x86_64-unknown-linux-musl
+ARG CARGO_FEATURES=""
 RUN apt-get update \
     && apt-get install -y --no-install-recommends musl-tools \
     && rm -rf /var/lib/apt/lists/* \
-    && rustup target add x86_64-unknown-linux-musl
+    && rustup target add "${RUST_TARGET}"
 WORKDIR /build
 # Manifests + sources together: the manifest uses target auto-discovery,
 # so `cargo fetch` needs the targets present. Heavy compile cost stays
@@ -15,23 +26,25 @@ COPY Cargo.toml Cargo.lock ./
 COPY src ./src
 RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
     cargo fetch --locked
-# CC points at musl-gcc so ring's C code compiles for the musl target.
+# CC points at musl-gcc so ring's C code compiles for the musl target
+# (harmless for the gnu target, which never reads this variable).
 # The LINKER is intentionally NOT overridden: rustc's default linker emits a
 # correct static-pie binary, whereas forcing musl-gcc as linker breaks
 # static-pie and produces a bogus INTERP (rust-lang/rust#95926).
 ENV CC_x86_64_unknown_linux_musl=musl-gcc
 RUN --mount=type=cache,target=/build/target,sharing=locked \
     --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
     cargo build --release --locked \
-        --target x86_64-unknown-linux-musl \
+        --target "${RUST_TARGET}" \
+        ${CARGO_FEATURES:+--features "${CARGO_FEATURES}"} \
         --bin vllm-coldstart-operator \
         --bin reporter \
-    && cp target/x86_64-unknown-linux-musl/release/vllm-coldstart-operator /vllm-coldstart-operator \
-    && cp target/x86_64-unknown-linux-musl/release/reporter /reporter \
+    && cp "target/${RUST_TARGET}/release/vllm-coldstart-operator" /vllm-coldstart-operator \
+    && cp "target/${RUST_TARGET}/release/reporter" /reporter \
     && strip /vllm-coldstart-operator /reporter
 # ---- Runtime ---------------------------------------------------------------
-# distroless static: no shell, no libc, nonroot (uid 65532) by default.
-FROM gcr.io/distroless/static:nonroot AS runtime
+# Default distroless static: no shell, no libc, nonroot (uid 65532).
+FROM ${RUNTIME_IMAGE} AS runtime
 LABEL org.opencontainers.image.source="https://github.com/MicheleCampi/vllm-coldstart-operator"
 LABEL org.opencontainers.image.description="Kubernetes operator for vLLM cold-start lifecycle management"
 LABEL org.opencontainers.image.licenses="Apache-2.0"

hack/rehearsal/adr0007-real-source.sh

@@ -9,11 +9,24 @@
 #   5) gpuUtilization/gpuMemoryUsedBytes ABSENT on every worker (NVML not
 #      wired = key deleted by merge-patch null, not zero)
 #   6) reporter logs free of panics
+# GPU_VARIANT=1: same asserts, but the image is the level-3 GPU build
+# (gnu target + gpu-nvidia feature + distroless/cc) with REPORTER_GPU=nvidia
+# set. kind has no GPU, so the double gate must fail open: adds assert
+#   7) every reporter logs the NVML init failure warning
 set -euo pipefail
 cd "$(dirname "$0")/../.."
 CLUSTER="${CLUSTER:-adr0007rs}"
 CTX="kind-${CLUSTER}"
-IMG="vllm-coldstart-operator:adr0007rs"
+GPU_VARIANT="${GPU_VARIANT:-}"
+if [ -n "$GPU_VARIANT" ]; then
+  IMG="vllm-coldstart-operator:adr0007rs-gpu"
+  BUILD_ARGS=(--build-arg RUST_TARGET=x86_64-unknown-linux-gnu
+              --build-arg CARGO_FEATURES=gpu-nvidia
+              --build-arg RUNTIME_IMAGE=gcr.io/distroless/cc-debian12:nonroot)
+else
+  IMG="vllm-coldstart-operator:adr0007rs"
+  BUILD_ARGS=()
+fi
 TS="$(date -u +%Y%m%dT%H%M%S)"
 RUN_DIR="hack/rehearsal/runs/${TS}-adr0007-real-source"
 mkdir -p "$RUN_DIR"
@@ -36,7 +49,7 @@ nodes:
 KINDCFG
 
 echo "==> build + load operator image"
-docker build -t "$IMG" .
+docker build "${BUILD_ARGS[@]}" -t "$IMG" .
 kind load docker-image "$IMG" --name "$CLUSTER"
 
 echo "==> fixtures: fake vLLM /metrics with advancing counters"
@@ -82,15 +95,16 @@ echo "==> CRDs + chart install (reporter enabled, REAL source, per-node targets)
 kubectl --context "$CTX" apply --server-side -f deploy/crd.yaml
 helm --kube-context "$CTX" install adr7rs chart/ \
   --set image.repository=vllm-coldstart-operator \
-  --set image.tag=adr0007rs \
+  --set "image.tag=${IMG##*:}" \
   --set image.pullPolicy=Never \
   --set reporter.enabled=true \
   --set-string 'reporter.extraEnv[0].name=REPORTER_SCRAPE_TARGETS_NODE_ADR0007RS_WORKER' \
   --set-string 'reporter.extraEnv[0].value=http://fixture-a.default.svc.cluster.local:9090/metrics' \
   --set-string 'reporter.extraEnv[1].name=REPORTER_SCRAPE_TARGETS_NODE_ADR0007RS_WORKER2' \
   --set-string 'reporter.extraEnv[1].value=http://fixture-b.default.svc.cluster.local:9090/metrics' \
   --set-string 'reporter.extraEnv[2].name=REPORTER_SCRAPE_TARGETS_NODE_ADR0007RS_WORKER3' \
-  --set-string 'reporter.extraEnv[2].value=http://no-such-service.default.svc.cluster.local:9090/metrics'
+  --set-string 'reporter.extraEnv[2].value=http://no-such-service.default.svc.cluster.local:9090/metrics' \
+  ${GPU_VARIANT:+--set-string 'reporter.extraEnv[3].name=REPORTER_GPU' --set-string 'reporter.extraEnv[3].value=nvidia'}
 
 kubectl --context "$CTX" rollout status deployment/adr7rs-vllm-coldstart-operator --timeout=120s
 kubectl --context "$CTX" rollout status daemonset/adr7rs-vllm-coldstart-operator-reporter --timeout=120s
@@ -134,6 +148,15 @@ for p in $(kubectl --context "$CTX" get pods -l app.kubernetes.io/component=repo
 done
 ok=0; [ "$PANICS" -eq 0 ] || ok=1; verdict "reporter logs free of panics" $ok
 
+if [ -n "$GPU_VARIANT" ]; then
+  MISSING=0
+  for f in "$RUN_DIR"/reporter-*.log; do
+    grep -qi "NVML init failed" "$f" || MISSING=$((MISSING + 1))
+  done
+  ok=0; [ "$MISSING" -eq 0 ] || ok=1
+  verdict "GPU variant: NVML fail-open warning on every reporter" $ok
+fi
+
 echo "==> verdict: ${PASS} pass / ${FAIL} fail (evidence: ${RUN_DIR})"
 cat "$RUN_DIR/verdict.txt"
 [ "$FAIL" -eq 0 ]