Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changes/unreleased/added-20260810-112731.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
kind: Added
body: |-
Install gpu-sharing as an OCI subchart
1 change: 1 addition & 0 deletions .github/workflows/on-pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ jobs:

- name: Build helm chart
run: |
helm dependency build ./deployments/kai-scheduler
helm package ./deployments/kai-scheduler -d ./charts --app-version $PACKAGE_VERSION --version $PACKAGE_VERSION
cp charts/kai-scheduler-$PACKAGE_VERSION.tgz /mnt/images/

Expand Down
1 change: 1 addition & 0 deletions .github/workflows/on-release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ jobs:
- name: Build helm chart
run: |
sed -i 's#registry/local/kai-scheduler#${{ env.DOCKER_REGISTRY }}#' deployments/kai-scheduler/values.yaml
helm dependency build ./deployments/kai-scheduler
helm package ./deployments/kai-scheduler -d ./charts --app-version $PACKAGE_VERSION --version $PACKAGE_VERSION
cp charts/kai-scheduler-$PACKAGE_VERSION.tgz /mnt/images/

Expand Down
1 change: 1 addition & 0 deletions .github/workflows/push-artifacts.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ jobs:
- name: Build helm chart
run: |
sed -i 's#registry/local/kai-scheduler#${{ env.DOCKER_REGISTRY }}#' deployments/kai-scheduler/values.yaml
helm dependency build ./deployments/kai-scheduler
helm package ./deployments/kai-scheduler -d ./charts --app-version $PACKAGE_VERSION --version $PACKAGE_VERSION

- name: Push Helm Chart
Expand Down
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,13 @@ K8S_COPYRIGHTED_MANIFEST_FILES := deployments/kai-scheduler/crds/kai.scheduler_t
lint: fmt-go vet-go lint-go
.PHONY: lint

.PHONY: chart-deps
chart-deps:
@echo "Fetching Helm chart dependencies: kai-scheduler"
helm dependency build ./deployments/kai-scheduler

.PHONY: test-chart
test-chart:
test-chart: chart-deps
@echo "Running tests for Helm chart: kai-scheduler"
docker run -t --rm -v ./deployments/kai-scheduler:/apps helmunittest/helm-unittest:3.17.2-0.8.1 . -f 'tests/**/*_test.yaml'

Expand Down
6 changes: 6 additions & 0 deletions deployments/kai-scheduler/Chart.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
dependencies:
- name: gpu-sharing
repository: oci://ghcr.io/kai-scheduler/gpu-sharing
version: v0.1.0-alpha.1
digest: sha256:c3d7f76c6d2b2fd7b256b7450fb435a82693e8acef1265d10a8d5270fcb028db
generated: "2026-08-02T20:53:54.374149+03:00"
8 changes: 7 additions & 1 deletion deployments/kai-scheduler/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,10 @@ name: kai-scheduler
description: KAI Scheduler by NVIDIA
type: application
version: 0.0.0
appVersion: 0.0.0
appVersion: 0.0.0

dependencies:
- name: gpu-sharing
version: v0.1.0-alpha.1
repository: oci://ghcr.io/kai-scheduler/gpu-sharing
condition: global.nvFractions.set
14 changes: 9 additions & 5 deletions deployments/kai-scheduler/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,14 @@ spec:
{{- if .Values.podgrouper.queueLabelKey }}
queueLabelKey: {{ .Values.podgrouper.queueLabelKey | quote }}
{{- end }}
{{- /* GpuSharingMode precedence: an explicit (non-empty) mode wins; otherwise a
legacy gpuSharing value is passed through (operator derives the mode); when
neither is set, default to NonMemoryEnforced. */}}
{{- if .Values.global.gpuSharingMode }}
{{- /* GpuSharingMode precedence: nvFractions.set forces NvFractions (validate.yaml has
already rejected a conflicting explicit mode, so this only guards against a legacy
gpuSharing value downgrading it); then an explicit (non-empty) mode; then a legacy
gpuSharing value is passed through (operator derives the mode); when none is set,
default to NonMemoryEnforced. */}}
{{- if .Values.global.nvFractions.set }}
gpuSharingMode: NvFractions
{{- else if .Values.global.gpuSharingMode }}
gpuSharingMode: {{ .Values.global.gpuSharingMode }}
{{- else if not (hasKey .Values.global "gpuSharing") }}
gpuSharingMode: NonMemoryEnforced
Expand Down Expand Up @@ -264,7 +268,7 @@ spec:
maxUnavailable: {{ .Values.admission.podDisruptionBudget.maxUnavailable }}
{{- end }}
{{- end }}
{{- if and (not .Values.global.gpuSharingMode) (hasKey .Values.global "gpuSharing") }}
{{- if and (not .Values.global.nvFractions.set) (not .Values.global.gpuSharingMode) (hasKey .Values.global "gpuSharing") }}
gpuSharing: {{ .Values.global.gpuSharing }}
{{- end }}
blockNvidiaVisibleDevices: {{ .Values.global.blockNvidiaVisibleDevices | default false }}
Expand Down
9 changes: 9 additions & 0 deletions deployments/kai-scheduler/templates/validate.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{{/*
Copyright 2025 NVIDIA CORPORATION
SPDX-License-Identifier: Apache-2.0
*/}}
{{- /* Renders no resource. Aborts the render when nvFractions.set asks for the gpu-sharing
operator while gpuSharingMode explicitly selects a different mode. */}}
{{- if and .Values.global.nvFractions.set .Values.global.gpuSharingMode (ne .Values.global.gpuSharingMode "NvFractions") }}
{{- fail (printf "global.nvFractions.set=true requires global.gpuSharingMode to be empty or \"NvFractions\", got %q" .Values.global.gpuSharingMode) }}
{{- end }}
108 changes: 108 additions & 0 deletions deployments/kai-scheduler/tests/nv_fractions_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
# Copyright 2025 NVIDIA CORPORATION
# SPDX-License-Identifier: Apache-2.0

suite: test nvFractions subchart gating and validation

tests:
- it: rejects nvFractions.set=true with a conflicting gpuSharingMode
templates:
- templates/validate.yaml
set:
global.nvFractions.set: true
global.gpuSharingMode: HamiCore
asserts:
- failedTemplate:
errorMessage: 'global.nvFractions.set=true requires global.gpuSharingMode to be empty or "NvFractions", got "HamiCore"'

- it: allows nvFractions.set=true with an explicit NvFractions mode
templates:
- templates/validate.yaml
set:
global.nvFractions.set: true
global.gpuSharingMode: NvFractions
asserts:
- notFailedTemplate: {}

- it: allows nvFractions.set=true with an empty gpuSharingMode
templates:
- templates/validate.yaml
set:
global.nvFractions.set: true
asserts:
- notFailedTemplate: {}

- it: allows gpuSharingMode NvFractions while the subchart is disabled
templates:
- templates/validate.yaml
set:
global.gpuSharingMode: NvFractions
asserts:
- notFailedTemplate: {}

- it: nvFractions.set=true selects NvFractions and omits legacy gpuSharing
templates:
- kai-config.yaml
set:
kaiConfigDeployer.enabled: false
kaiConfig.render: true
global.nvFractions.set: true
asserts:
- equal:
path: spec.global.gpuSharingMode
value: NvFractions
- notExists:
path: spec.admission.gpuSharing

- it: nvFractions.set=true is not downgraded by a legacy gpuSharing value
templates:
- kai-config.yaml
set:
kaiConfigDeployer.enabled: false
kaiConfig.render: true
global.nvFractions.set: true
global.gpuSharing: false
asserts:
- equal:
path: spec.global.gpuSharingMode
value: NvFractions
- notExists:
path: spec.admission.gpuSharing

- it: nvFractions.set=true with an explicit NvFractions mode agrees
templates:
- kai-config.yaml
set:
kaiConfigDeployer.enabled: false
kaiConfig.render: true
global.nvFractions.set: true
global.gpuSharingMode: NvFractions
asserts:
- equal:
path: spec.global.gpuSharingMode
value: NvFractions

- it: default is unchanged when nvFractions.set is false
templates:
- kai-config.yaml
set:
kaiConfigDeployer.enabled: false
kaiConfig.render: true
asserts:
- equal:
path: spec.global.gpuSharingMode
value: NonMemoryEnforced

# No mirror-image test asserting the deployment is absent when nvFractions.set is false:
# helm-unittest treats a conditioned-out subchart template as "not exists or not selected"
# and errors instead of rendering zero documents. The off state is covered by Helm's
# condition: semantics and by the NonMemoryEnforced default asserted above.
- it: renders the gpu-sharing operator deployment when nvFractions.set is true
templates:
- charts/gpu-sharing/templates/deployment.yaml
set:
global.nvFractions.set: true
asserts:
- hasDocuments:
count: 1
- isKind:
of: Deployment
8 changes: 8 additions & 0 deletions deployments/kai-scheduler/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ global:
# Deprecated: gpuSharing is honored only for upgrades when gpuSharingMode is empty
# (true -> NonMemoryEnforced, false -> Disabled). Prefer gpuSharingMode.
# gpuSharing: false
# nvFractions.set installs the gpu-sharing operator
# (oci://ghcr.io/kai-scheduler/gpu-sharing) as a subchart and selects the NvFractions
# GPU sharing mode. Setting it together with a global.gpuSharingMode other than
# NvFractions is rejected at render time.
# KAI's global.imagePullSecrets/nodeSelector/affinity/tolerations are NOT propagated to
# the subchart; override them under the top-level `gpu-sharing:` key.
nvFractions:
set: false
blockNvidiaVisibleDevices: false
clusterAutoscaling: false
nodeSelector: {}
Expand Down
26 changes: 26 additions & 0 deletions docs/operator/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,32 @@ The KAI operator reports installation health on `Config.status.conditions`. The

When `global.gpuSharingMode` is `NvFractions`, the KAI operator also checks the cluster-scoped `GpuSharingConfig` resource from the GPU-sharing operator, as running fractional GPU pods depends on it.

### Installing the GPU-sharing operator

The GPU-sharing operator is required only for `NvFractions`; every other `gpuSharingMode` runs without it. It also raises the cluster's requirements: [NVIDIA GPU Operator](https://github.com/NVIDIA/gpu-operator) **v26.7.1 or newer** and an **`r615` or newer driver (CUDA 13.4)** on the GPU nodes. `r615` is not the GPU Operator default, so it must be requested explicitly via `driver.version`; on an older driver the `GpuSharingConfig` `Ready` condition reports `GPUDriverVersionUnsupported` and the node-level daemons are not rolled out. See the [gpu-sharing prerequisites](https://github.com/kai-scheduler/gpu-sharing#prerequisites) for the full list, including containerd 2.0+ with NRI enabled.

Set `global.nvFractions.set=true` to install the [gpu-sharing](https://github.com/kai-scheduler/gpu-sharing) operator as a subchart and select the `NvFractions` mode in one step:

```bash
helm upgrade -i kai-scheduler oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler \
-n kai-scheduler --create-namespace --set global.nvFractions.set=true
```

Leaving `global.gpuSharingMode` empty is enough — `nvFractions.set` selects `NvFractions`. Setting it to any other mode at the same time fails the install at render time. To run `NvFractions` against a GPU-sharing operator installed out-of-band, leave `global.nvFractions.set` at its `false` default and set `global.gpuSharingMode: NvFractions` instead.

KAI's `global.imagePullSecrets`, `global.nodeSelector`, `global.affinity` and `global.tolerations` are **not** propagated to the subchart, because the gpu-sharing chart reads those keys at its own top level rather than under `global`. Set them explicitly under the `gpu-sharing:` key — relevant for private-registry and air-gapped installs:

```yaml
global:
nvFractions:
set: true
gpu-sharing:
imagePullSecrets:
- name: my-registry-secret
nodeSelector:
nvidia.com/gpu.present: "true"
```

If all KAI services are deployed and available, but the GPU-sharing operator reports a dependency failure, the KAI `Config` can look like this:

```yaml
Expand Down
1 change: 1 addition & 0 deletions hack/run-e2e-upgrade-kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then
fi

# Package the new helm chart
helm dependency build ./deployments/kai-scheduler
helm package ./deployments/kai-scheduler -d ./charts --app-version $PACKAGE_VERSION --version $PACKAGE_VERSION
export UPGRADE_CHART_PATH=${REPO_ROOT}/charts/kai-scheduler-$PACKAGE_VERSION.tgz

Expand Down
1 change: 1 addition & 0 deletions hack/setup-e2e-cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ if [ "$LOCAL_IMAGES_BUILD" = "true" ]; then
fi

# Package helm chart
helm dependency build ./deployments/kai-scheduler
helm package ./deployments/kai-scheduler -d ./charts --app-version $PACKAGE_VERSION --version $PACKAGE_VERSION
if [ "$SKIP_KAI_INSTALL" = "true" ]; then
echo "Skipping KAI install; packaged chart kept at ./charts/kai-scheduler-$PACKAGE_VERSION.tgz"
Expand Down
Loading