Merge branch 'main' into ci/azure-nightly-inference

Author: njhensley

.github/actions/aicr-build/build-validator-images.sh

@@ -45,11 +45,11 @@ for phase in ${PHASES//,/ }; do
     exit 1
   fi
   docker build -t "ko.local/aicr-validators/${phase}:latest" -f - . <<DOCKERFILE
-FROM gcr.io/distroless/static-debian12:nonroot
+FROM nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69
 COPY dist/validator/${phase} /${phase}
 COPY validators/${phase}/testdata /app/testdata
 WORKDIR /app
-USER nonroot
+USER nvs
 ENTRYPOINT ["/${phase}"]
 DOCKERFILE
   timeout 600 kind load docker-image "ko.local/aicr-validators/${phase}:latest" --name "${KIND_CLUSTER_NAME}" || {

.github/actions/e2e/action.yml

@@ -87,12 +87,12 @@ runs:
           done
           for phase in deployment performance conformance; do
             docker build -t "localhost:5001/aicr-validators/${phase}:latest" -f - . <<DOCKERFILE
-        FROM gcr.io/distroless/static-debian12:nonroot
+        FROM nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69
         COPY dist/validator/${phase} /${phase}
         COPY validators/${phase}/testdata /app/testdata
         COPY dist/validator/chainsaw /usr/local/bin/chainsaw
         WORKDIR /app
-        USER nonroot
+        USER nvs
         ENTRYPOINT ["/${phase}"]
         DOCKERFILE
             docker push "localhost:5001/aicr-validators/${phase}:latest"

.github/workflows/vuln-scan-images.yaml

@@ -92,7 +92,8 @@ jobs:
         env:
           GOFLAGS: -mod=vendor
           KO_DOCKER_REPO: ghcr.io/nvidia/aicrd
-          KO_DEFAULTBASEIMAGE: gcr.io/distroless/static:nonroot
+          # Base image inherited from .ko.yaml (nvcr.io/nvidia/distroless/static),
+          # same as the aicr build above.
         run: |
           set -euo pipefail
           ko build ./cmd/aicrd \

.goreleaser.yaml

@@ -166,10 +166,11 @@ kos:
     build: aicr
     repositories:
       - ghcr.io/nvidia/aicr
-    # No base_image: ko uses its default static, distroless base
-    # (cgr.dev/chainguard/static). The aicr CLI/agent is pure Go and discovers
-    # the GPU SKU driver-free via NFD/PCI, so it no longer needs nvidia-smi or
-    # the CUDA base image (and its CVE surface).
+    # NVIDIA's static distroless base (replaces ko's chainguard default). The
+    # aicr CLI/agent is pure Go and discovers the GPU SKU driver-free via
+    # NFD/PCI, so it no longer needs nvidia-smi or the CUDA base image (and its
+    # CVE surface).
+    base_image: nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69
     platforms:
       - linux/amd64
       - linux/arm64
@@ -185,10 +186,11 @@ kos:
     build: aicrd
     repositories:
       - ghcr.io/nvidia/aicrd
-    # No base_image: aicrd intentionally uses ko's default static, distroless
-    # base (cgr.dev/chainguard/static), same as aicr above. It runs as the
-    # nonroot user 65532 by default, preserving the security posture of the
-    # previous gcr.io/distroless/static:nonroot base (also UID 65532).
+    # NVIDIA's static distroless base, same as aicr above. ko stamps its default
+    # nonroot user (UID 65532) numerically, preserving the unprivileged security
+    # posture of the previous gcr.io/distroless/static:nonroot base (also UID
+    # 65532); the base image's own nvs (UID 1000) account is unused here.
+    base_image: nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69
     platforms:
       - linux/amd64
       - linux/arm64

.ko.yaml

@@ -12,7 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# No defaultBaseImage: the aicr CLI/agent is a pure-Go binary that discovers
-# the GPU SKU driver-free via NFD/PCI, so it no longer needs nvidia-smi or the
-# CUDA base (and its recurring CVE surface). With no override, ko uses its
-# default static, distroless base image (cgr.dev/chainguard/static).
+# The aicr CLI/agent is a pure-Go binary that discovers the GPU SKU driver-free
+# via NFD/PCI, so it needs neither nvidia-smi nor the CUDA base (and its
+# recurring CVE surface). It runs on NVIDIA's own static distroless base rather
+# than ko's default (cgr.dev/chainguard/static). ko still stamps its default
+# nonroot user (UID 65532) numerically, so it runs unprivileged regardless of
+# the base image's own nvs (UID 1000) account. This defaultBaseImage applies to
+# `ko build` CLI invocations (e.g. the vuln-scan workflow); goreleaser's kos
+# entries pin the same image explicitly via base_image.
+defaultBaseImage: nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69

.settings.yaml

@@ -173,8 +173,9 @@ testing:
   gpu_operator_chart_version: 'v25.10.1'
   # Base image for the CI smoke-test snapshot-agent image. The agent binary is
   # static Go and detects GPUs driver-free via NFD/PCI, so it no longer needs a
-  # CUDA base / nvidia-smi. Matches ko's default static base for the aicr image.
-  snapshot_agent_base_image: 'cgr.dev/chainguard/static:latest'
+  # CUDA base / nvidia-smi. Matches the NVIDIA static distroless base used for
+  # the aicr image (see .ko.yaml).
+  snapshot_agent_base_image: 'nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69'
 
   # Component test harness configuration
   # Used by tools/component-test/ scripts to validate individual components

RELEASING.md

@@ -172,16 +172,16 @@ Published to GitHub Container Registry (`ghcr.io/nvidia/`):
 
 | Image | Base | Description |
 |-------|------|-------------|
-| `aicr` | `nvcr.io/nvidia/cuda:13.1.0-runtime-ubuntu24.04` | CLI with CUDA runtime |
-| `aicrd` | `gcr.io/distroless/static:nonroot` | Minimal API server |
+| `aicr` | `nvcr.io/nvidia/distroless/static:v4.0.0` | Pure-Go CLI/agent (driver-free GPU discovery) |
+| `aicrd` | `nvcr.io/nvidia/distroless/static:v4.0.0` | Minimal API server |
 
 Published to GitHub Container Registry (`ghcr.io/nvidia/aicr-validators/`):
 
 | Image | Base | Description |
 |-------|------|-------------|
-| `deployment` | `gcr.io/distroless/static-debian12:nonroot` | Deployment validator |
-| `performance` | `gcr.io/distroless/static-debian12:nonroot` | Performance validator |
-| `conformance` | `gcr.io/distroless/static-debian12:nonroot` | Conformance validator |
+| `deployment` | `nvcr.io/nvidia/distroless/static:v4.0.0` | Deployment validator |
+| `performance` | `nvcr.io/nvidia/distroless/static:v4.0.0` | Performance validator |
+| `conformance` | `nvcr.io/nvidia/distroless/static:v4.0.0` | Conformance validator |
 | `aiperf-bench` | `python:3.12-slim` | AIPerf benchmark runner |
 
 Stable releases promote `vX.Y.Z` and `latest`; prereleases promote their

cmd/gate/Dockerfile

@@ -46,7 +46,7 @@ RUN CGO_ENABLED=0 go build -trimpath \
     -ldflags="-s -w -extldflags '-static' -X main.version=${VERSION}" \
     -o /out/gate ./cmd/gate
 
-FROM gcr.io/distroless/static-debian12:nonroot
+FROM nvcr.io/nvidia/distroless/static:v4.0.0@sha256:d90158b69e250d2018f32622b5c622925202ee97224a990a54b63811cb1e3d69
 
 COPY --from=builder /out/gate /gate
 COPY --from=chainsaw /ko-app/chainsaw /usr/local/bin/chainsaw
@@ -56,5 +56,7 @@ COPY --from=chainsaw /ko-app/chainsaw /usr/local/bin/chainsaw
 # points here; see cmd/gate/main.go (defaultChainsawConfig).
 COPY cmd/gate/chainsaw-config.yaml /etc/chainsaw/config.yaml
 
-USER nonroot
+# nvs (UID 1000) is the sole nonroot user shipped in the NVIDIA distroless
+# static base; it has no "nonroot"/65532 user like the previous distroless bases.
+USER nvs
 ENTRYPOINT ["/gate"]

demos/cuj1-slinky-slurm.md

@@ -31,6 +31,7 @@ Pick the row that matches your cluster. Each resolves to a slurm leaf with at le
 
 | Cloud    | Command                                                                                                      | Leaf overlay                                               |
 | -------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------- |
+| **AKS**  | `aicr recipe --service aks --accelerator h100 --intent training --os ubuntu --platform slurm -o recipe.yaml` | `h100-aks-ubuntu-training-slurm`                           |
 | **EKS**  | `aicr recipe --service eks --accelerator h100 --intent training --os ubuntu --platform slurm -o recipe.yaml` | `h100-eks-ubuntu-training-slurm`                           |
 | **GKE**  | `aicr recipe --service gke --accelerator h100 --intent training --os cos --platform slurm -o recipe.yaml`    | `h100-gke-cos-training-slurm`                              |
 | **Kind** | `aicr recipe --service kind --accelerator h100 --intent training --platform slurm -o recipe.yaml`            | `h100-kind-training-slurm` (CPU-only NodeSet; no GPU GRES) |
@@ -112,6 +113,27 @@ GKE system nodes should **not** carry custom taints (konnectivity and other mana
 
 Optional: `--accelerated-node-toleration nvidia.com/gpu=present:NoSchedule` (harmless if that taint is absent).
 
+### AKS (system + cpu + gpu pools; CriticalAddonsOnly + GPU taint)
+
+Example layout: 3× `system` (`CriticalAddonsOnly=true:NoSchedule`), 1× `cpuworker1` (untainted), 2× `gpuworker1` (`nvidia.com/gpu=present:NoSchedule`). Operator on **system**; controller / login / restapi on **cpuworker1**; slurmd on **gpuworker1**.
+
+```shell
+aicr bundle \
+  --recipe recipe.yaml \
+  --deployer helm \
+  --system-node-selector agentpool=system \
+  --system-node-toleration CriticalAddonsOnly=true:NoSchedule \
+  --accelerated-node-selector agentpool=gpuworker1 \
+  --accelerated-node-toleration nvidia.com/gpu=present:NoSchedule \
+  --set slinkyslurm:nodesets.slinky.replicas=2 \
+  --set-json 'slinkyslurm:controller.podSpec={"nodeSelector":{"agentpool":"cpuworker1"}}' \
+  --set-json 'slinkyslurm:restapi.podSpec={"nodeSelector":{"agentpool":"cpuworker1"}}' \
+  --set-json 'slinkyslurm:loginsets.slinky.podSpec={"nodeSelector":{"agentpool":"cpuworker1"}}' \
+  --output bundle
+```
+
+AKS ships `managed-csi` as the default StorageClass; omit `--storage-class` unless you need a non-default class.
+
 ### Kind (CPU-only smoke / CI)
 
 No GPU pools or taints; omit accelerated flags unless your Kind config adds them.

docs/integrator/recipe-development.md

@@ -121,7 +121,7 @@ For example, `--platform slurm` leaves inline three `componentRefs`:
 
 - `slinky-slurm-operator-crds` — SchedMD Slinky CRDs
 - `slinky-slurm-operator` — the operator and admission webhook
-- `slinky-slurm` — the Slinky-managed Slurm cluster instance (Controller / LoginSet / NodeSet / RestApi), with leaf-specific `overrides` (e.g. H100 GRES wiring on the `nodesets.slinky` map)
+- `slinky-slurm` — the Slinky-managed Slurm cluster instance (Controller / LoginSet / NodeSet / RestApi), with leaf-specific `overrides` (e.g. GPU GRES wiring on `nodesets.slinky` and `controller.extraConfMap`)
 
 This is the same shape `dynamo-platform` uses across the `*-inference-dynamo` leaves. See `recipes/overlays/h100-eks-ubuntu-training-slurm.yaml` for the full example.