Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions ai-ml/gke-slurm/metrics-autoscaling/autoscaling-metrics.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# [START gke_ai_ml_gke_slurm_metrics_autoscaling_keda]

apiVersion: monitoring.googleapis.com/v1
kind: PodMonitoring
metadata:
name: slurm-metrics-exporter
namespace: slurm
spec:
selector:
matchLabels:
app: slurmctld
endpoints:
- port: 6817
path: /metrics/partitions
interval: 5s
---
apiVersion: keda.sh/v1alpha1
kind: ClusterTriggerAuthentication
metadata:
name: google-workload-identity-auth
namespace: slurm
spec:
podIdentity:
provider: gcp
---
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: slurm-worker-cpu-nodes-scaler
namespace: slurm
spec:
scaleTargetRef:
apiVersion: slinky.slurm.net/v1beta1
kind: NodeSet
name: slurm-worker-cpu-nodes
minReplicaCount: 1
maxReplicaCount: 10
pollingInterval: 5
cooldownPeriod: 30
triggers:
- type: prometheus
metadata:
serverAddress: https://monitoring.googleapis.com/v1/projects/PROJECT_ID/location/global/prometheus
query: |
sum(slurm_partition_jobs_pending{project_id="PROJECT_ID",partition="cpu-partition",cluster="CLUSTER_NAME"})+sum(slurm_partition_nodes_alloc{project_id="PROJECT_ID",partition="cpu-partition",cluster="CLUSTER_NAME"})
threshold: "1.0"
authenticationRef:
kind: ClusterTriggerAuthentication
name: google-workload-identity-auth

# [END gke_ai_ml_gke_slurm_metrics_autoscaling_keda]
50 changes: 50 additions & 0 deletions ai-ml/gke-slurm/metrics-autoscaling/pv-pvc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# [START gke_ai_ml_gke_slurm_metrics_autoscaling_pv_pvc]

apiVersion: v1
kind: PersistentVolume
metadata:
name: PV_NAME
spec:
storageClassName: ""
capacity:
storage: 1Ti
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Retain
volumeMode: Filesystem
csi:
driver: filestore.csi.storage.gke.io
volumeHandle: "modeInstance/INSTANCE_ZONE/INSTANCE_NAME/SHARE_NAME"
volumeAttributes:
ip: INSTANCE_IP
volume: SHARE_NAME
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: PVC_NAME
namespace: slurm
spec:
accessModes:
- ReadWriteMany
storageClassName: ""
resources:
requests:
storage: 1Ti
volumeName: PV_NAME

# [END gke_ai_ml_gke_slurm_metrics_autoscaling_pv_pvc]
147 changes: 147 additions & 0 deletions ai-ml/gke-slurm/metrics-autoscaling/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# [START gke_ai_ml_gke_slurm_metrics_autoscaling_values]

imagePullPolicy: IfNotPresent

controller:
slurmctld:
image:
repository: gcr.io/gke-release/slinky/slurmctld
tag: TAG
reconfigure:
image:
repository: gcr.io/gke-release/slinky/slurmctld
tag: TAG
logfile:
image:
repository: docker.io/library/alpine
tag: latest
persistence:
enabled: true
existingClaim: ""
storageClassName: null
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 4Gi
extraConf: null

metadata:
labels:
app: slurmctld
extraConfMap:
SlurmctldDebug: verbose
MetricsType: metrics/openmetrics
metrics:
enabled: true
serviceMonitor:
enabled: false

restapi:
replicas: 1
slurmrestd:
image:
repository: gcr.io/gke-release/slinky/slurmrestd
tag: TAG
podSpec:
nodeSelector:
kubernetes.io/os: linux

loginsets:
slinky:
enabled: true
replicas: 1
podSpec:
tolerations:
- key: "login-pool"
operator: "Equal"
value: "login-pods"
effect: "NoSchedule"
volumes:
- name: home-vol
persistentVolumeClaim:
claimName: PVC_NAME
login:
image:
repository: gcr.io/gke-release/slinky/login
tag: TAG
volumeMounts:
- name: home-vol
mountPath: /home

nodesets:
slinky:
enabled: false
cpu-nodes:
enabled: true
replicas: 1
updateStrategy:
type: RollingUpdate
slurmd:
args: [" -N ${POD_NAME#slurm-worker-}"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image:
repository: gcr.io/gke-release/slinky/slurmd
tag: TAG
imagePullPolicy: IfNotPresent
volumeMounts:
- name: home-vol
mountPath: /home
logfile:
image:
repository: docker.io/library/alpine
tag: latest
partition:
enabled: true
useResourceLimits: true
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 25%
podSpec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
tolerations:
- key: "slurm-pool"
operator: "Equal"
value: "cpu-nodes"
effect: "NoSchedule"
nodeSelector:
kubernetes.io/os: linux
volumes:
- name: home-vol
persistentVolumeClaim:
claimName: PVC_NAME
taintKubeNodes: false

partitions:
all:
enabled: false
cpu-partition:
enabled: true
nodesets:
- cpu-nodes
configMap:
State: UP
Default: "YES"
MaxTime: UNLIMITED

# [END gke_ai_ml_gke_slurm_metrics_autoscaling_values]
Loading