feat(build): publish FIPS-enabled images and add global.fips helm option (#1882)

Signed-off-by: gshaibi <gshaibi@nvidia.com>

Author: gshaibi

.github/workflows/push-artifacts.yaml

@@ -64,3 +64,41 @@ jobs:
       - name: Push Helm Chart
         run: |
           helm push ./charts/kai-scheduler-$PACKAGE_VERSION.tgz oci://${{ env.DOCKER_REGISTRY }}
+
+  build-and-push-fips:
+    name: Build & Push FIPS
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Extract package version from tag
+        run: |
+          PACKAGE_VERSION=${GITHUB_REF_NAME}
+          echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV
+          echo $PACKAGE_VERSION
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: '1.26.3'
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Docker build & push FIPS images
+        run: make build FIPS=1 DOCKER_BUILD_PLATFORM=linux/amd64,linux/arm64 DOCKER_REPO_BASE=${{ env.DOCKER_REGISTRY }} VERSION=$PACKAGE_VERSION DOCKER_BUILDX_ADDITIONAL_ARGS=--push
+
+      - name: Verify binaries are FIPS-enabled
+        run: |
+          for f in bin/*-amd64 bin/*-arm64; do
+            go version -m "$f" | grep -q 'GOFIPS140=v1.0.0' || { echo "$f is not FIPS-enabled"; exit 1; }
+          done

CHANGELOG.md

@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+### Added
+- Publish FIPS-enabled image variants (`<version>-fips`) for every release, built with the Go toolchain's native FIPS 140-3 mode (`GOFIPS140`), and added a `global.fips` Helm value (default `false`) that appends `-fips` to every resolved image tag ([guide](docs/fips/README.md)). [#1867](https://github.com/kai-scheduler/KAI-Scheduler/issues/1867)
+
 ### Fixed
 - Scoped the operator's informer cache for Pods, Leases and EndpointSlices to the KAI namespace and stripped managed fields from cached objects. Since v0.15.0 the operator cached every such object in the cluster, so its memory grew with cluster size and exceeded the default 256Mi limit on large clusters. [#1780](https://github.com/kai-scheduler/KAI-Scheduler/issues/1780)
 - Block NaN value for fraction in the pod admission [#1798](https://github.com/kai-scheduler/KAI-Scheduler/issues/1798) [davidLif](https://github.com/davidLif)
@@ -22,6 +25,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ## [v0.16.1] - 2026-06-28
 
 ### Added
+- Added `global.resourceReservation.createNamespace` Helm value (default `true`) to allow disabling creation of the resource-reservation namespace, for embedding KAI in a parent chart that creates the namespace itself.
+- Added `global.resourceReservation.createServiceAccount` Helm value (default `true`) to allow disabling creation of the resource-reservation ServiceAccount, for embedding KAI in a parent chart that creates the ServiceAccount itself.
 - Added `defaultPriorityClasses.enabled` Helm value (default `true`) for installations that manage KAI PriorityClasses externally.
 
 ### Changed

build/makefile/base.mk

@@ -14,6 +14,11 @@ CRD_UPGRADER_DOCKERFILE_PATH=./deployments/crd-upgrader/Dockerfile
 DOCKER_TAG?=0.0.0
 VERSION?=${DOCKER_TAG}
 
+FIPS?=0
+ifeq ($(FIPS), 1)
+override VERSION := ${VERSION}-fips
+endif
+
 DOCKER_REPO_BASE?=registry/local/kai-scheduler
 DOCKER_REPO_FULL?=${DOCKER_REPO_BASE}/${SERVICE_NAME}
 DOCKER_IMAGE_NAME?=${DOCKER_REPO_FULL}:${VERSION}

build/makefile/golang.mk

@@ -28,6 +28,9 @@ GOLANGCI_LINT_VERSION=v2.11.3
 ## Tool Versions
 CGO_ENABLED?=1
 
+## FIPS
+GOFIPS140_VERSION?=v1.0.0
+
 ## Version Variables
 BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
 GIT_COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
@@ -43,6 +46,9 @@ LDFLAGS := -X '$(VERSION_PKG).buildDate=$(BUILD_DATE)' \
 
 ### GO
 DOCKER_GO_BASE_COMMAND=${DOCKER_COMMAND} -e CGO_ENABLED=${CGO_ENABLED} -e GO111MODULE=on ${DOCKER_GO_CACHING_VOLUME_AND_ENV}
+ifeq ($(FIPS), 1)
+DOCKER_GO_BASE_COMMAND += -e GOFIPS140=${GOFIPS140_VERSION}
+endif
 
 GO_ENV_ARCH_AMD=-e GOOS=linux -e GOARCH=amd64 -e CC=x86_64-linux-gnu-gcc -e CXX=x86_64-linux-gnu-g++
 GO_ENV_ARCH_ARM=-e GOOS=linux -e GOARCH=arm64 -e CC=aarch64-linux-gnu-gcc -e CXX=aarch64-linux-gnu-g++

deployments/kai-scheduler/templates/_helpers.tpl

@@ -1,3 +1,15 @@
+{{/*
+Resolves a component image tag: explicit tag, then global.tag, then the chart
+version. When global.fips is set, appends "-fips" to whatever tag resolves so
+the FIPS image variants are used. Usage:
+  {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.<svc>.image.tag) }}
+*/}}
+{{- define "kai-scheduler.imageTag" -}}
+{{- $tag := .tag | default .root.Values.global.tag | default .root.Chart.AppVersion -}}
+{{- if .root.Values.global.fips -}}{{- $tag = printf "%s-fips" $tag -}}{{- end -}}
+{{- $tag -}}
+{{- end -}}
+
 {{/*
 Renders the kai-config Config CR. Used by the kai-config-deployer hook
 ConfigMap so the operator's input config can be applied via kubectl
@@ -65,7 +77,7 @@ spec:
       image:
         name: {{ .Values.binder.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.binder.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.binder.image.tag) }}
         pullPolicy: {{ .Values.binder.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.binder.resources }}
       resources:
@@ -92,7 +104,7 @@ spec:
       image:
         name: {{ .Values.binder.resourceReservationImage.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.binder.resourceReservationImage.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.binder.resourceReservationImage.tag) }}
         pullPolicy: {{ .Values.binder.resourceReservationImage.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.binder.resourceReservationPodResources }}
       podResources:
@@ -112,7 +124,7 @@ spec:
       image:
         name: {{ .Values.podgrouper.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.podgrouper.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.podgrouper.image.tag) }}
         pullPolicy: {{ .Values.podgrouper.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.podgrouper.resources }}
       resources:
@@ -129,7 +141,7 @@ spec:
       image:
         name: {{ .Values.podgroupcontroller.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.podgroupcontroller.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.podgroupcontroller.image.tag) }}
         pullPolicy: {{ .Values.podgroupcontroller.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.podgroupcontroller.resources }}
       resources:
@@ -146,7 +158,7 @@ spec:
       image:
         name: {{ .Values.queuecontroller.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.queuecontroller.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.queuecontroller.image.tag) }}
         pullPolicy: {{ .Values.queuecontroller.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.queuecontroller.resources }}
       resources:
@@ -163,7 +175,7 @@ spec:
       image:
         name: {{ .Values.admission.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.admission.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.admission.image.tag) }}
         pullPolicy: {{ .Values.admission.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.admission.resources }}
       resources:
@@ -200,7 +212,7 @@ spec:
       image:
         name: {{ .Values.nodescaleadjuster.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.nodescaleadjuster.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.nodescaleadjuster.image.tag) }}
         pullPolicy: {{ .Values.nodescaleadjuster.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.nodescaleadjuster.resources }}
       resources:
@@ -215,7 +227,7 @@ spec:
       scalingPodImage:
         name: {{ .Values.nodescaleadjuster.scalingPodImage.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.nodescaleadjuster.scalingPodImage.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.nodescaleadjuster.scalingPodImage.tag) }}
         pullPolicy: {{ .Values.nodescaleadjuster.scalingPodImage.pullPolicy | default .Values.global.imagePullPolicy }}
 
   scheduler:
@@ -224,7 +236,7 @@ spec:
       image:
         name: {{ .Values.scheduler.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.scheduler.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.scheduler.image.tag) }}
         pullPolicy: {{ .Values.scheduler.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.scheduler.resources }}
       resources:
@@ -255,7 +267,7 @@ spec:
       image:
         name: {{ .Values.numaPlacementExporter.image.name }}
         repository: {{ .Values.global.registry }}
-        tag: {{ .Values.numaPlacementExporter.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        tag: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.numaPlacementExporter.image.tag) }}
         pullPolicy: {{ .Values.numaPlacementExporter.image.pullPolicy | default .Values.global.imagePullPolicy }}
       {{- if .Values.numaPlacementExporter.resources }}
       resources:

deployments/kai-scheduler/templates/hooks/post/kai-config-deployer/job.yaml

@@ -48,7 +48,7 @@ spec:
       {{- end }}
       containers:
       - name: deployer
-        image: "{{ .Values.kaiConfigDeployer.image.registry | default .Values.global.registry }}/{{ .Values.kaiConfigDeployer.image.name }}:{{ .Values.kaiConfigDeployer.image.tag | default .Values.global.tag | default .Chart.AppVersion }}"
+        image: "{{ .Values.kaiConfigDeployer.image.registry | default .Values.global.registry }}/{{ .Values.kaiConfigDeployer.image.name }}:{{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.kaiConfigDeployer.image.tag) }}"
         imagePullPolicy: {{ .Values.kaiConfigDeployer.image.pullPolicy }}
         {{- with .Values.kaiConfigDeployer.resources }}
         resources:

deployments/kai-scheduler/templates/hooks/post/post-delete-job.yaml

@@ -46,7 +46,7 @@ spec:
       {{- end }}
       containers:
         - name: deleter
-          image: "{{ .Values.global.registry }}/{{ .Values.postCleanup.image.name }}:{{ .Values.postCleanup.image.tag | default .Values.global.tag | default .Chart.AppVersion }}"
+          image: "{{ .Values.global.registry }}/{{ .Values.postCleanup.image.name }}:{{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.postCleanup.image.tag) }}"
           imagePullPolicy: {{ .Values.postCleanup.image.pullPolicy }}
           {{- with .Values.postCleanup.resources }}
           resources:

deployments/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml

@@ -39,7 +39,7 @@ spec:
       {{- end }}
       containers:
       - name: upgrader
-        image: "{{ .Values.crdupgrader.image.registry | default .Values.global.registry }}/{{ .Values.crdupgrader.image.name }}:{{ .Values.crdupgrader.image.tag | default .Values.global.tag | default .Chart.AppVersion }}"
+        image: "{{ .Values.crdupgrader.image.registry | default .Values.global.registry }}/{{ .Values.crdupgrader.image.name }}:{{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.crdupgrader.image.tag) }}"
         imagePullPolicy: {{ .Values.crdupgrader.image.pullPolicy }}
         {{- with .Values.crdupgrader.resources }}
         resources:

deployments/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml

@@ -40,7 +40,7 @@ spec:
       {{- end }}
       containers:
       - name: migration
-        image: "{{ .Values.global.registry }}/{{ .Values.topologyMigration.image.name }}:{{ .Values.topologyMigration.image.tag | default .Values.global.tag | default .Chart.AppVersion }}"
+        image: "{{ .Values.global.registry }}/{{ .Values.topologyMigration.image.name }}:{{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.topologyMigration.image.tag) }}"
         imagePullPolicy: {{ .Values.topologyMigration.image.pullPolicy }}
         {{- with .Values.topologyMigration.resources }}
         resources:

deployments/kai-scheduler/templates/services/operator.yaml

@@ -21,7 +21,7 @@ spec:
       serviceAccountName: kai-operator
       containers:
       - name: operator
-        image: {{ .Values.global.registry }}/{{ .Values.operator.image.name }}:{{ .Values.operator.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+        image: {{ .Values.global.registry }}/{{ .Values.operator.image.name }}:{{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.operator.image.tag) }}
         imagePullPolicy: {{ .Values.operator.image.pullPolicy }}
         {{- with .Values.operator.resources }}
         resources:
@@ -52,7 +52,7 @@ spec:
         - name: MS_REPOSITORY
           value: {{ .Values.global.registry }}
         - name: MS_TAG
-          value: {{ .Values.operator.image.tag | default .Values.global.tag | default .Chart.AppVersion }}
+          value: {{ include "kai-scheduler.imageTag" (dict "root" $ "tag" .Values.operator.image.tag) }}
         ports:
         - containerPort: 8080
           name: metrics