Skip to content

Commit 5c31e6f

Browse files
shengnuoshengnuo
authored
Merge pull request NVIDIA#492 from shengnuo/empty-dir-size-limit
Allow configurable sizeLimit for shared memory (emptyDir) Volumes
2 parents 117e676 + 2d6ae9d commit 5c31e6f

24 files changed

Lines changed: 292 additions & 187 deletions

‎api/apps/v1alpha1/nemo_customizer_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
corev1 "k8s.io/api/core/v1"
3030
networkingv1 "k8s.io/api/networking/v1"
3131
rbacv1 "k8s.io/api/rbac/v1"
32+
"k8s.io/apimachinery/pkg/api/resource"
3233
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3334
"k8s.io/apimachinery/pkg/util/intstr"
3435
"k8s.io/utils/ptr"
@@ -150,6 +151,8 @@ type TrainingConfig struct {
150151
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
151152
// PodAffinity for the training jobs
152153
PodAffinity *corev1.PodAffinity `json:"podAffinity,omitempty"`
154+
// SharedMemorySizeLimit sets the max size of the shared memory volume (emptyDir) used by the training jobs for fast model runtime I/O.
155+
SharedMemorySizeLimit *resource.Quantity `json:"sharedMemorySizeLimit,omitempty"`
153156
// Resources for the training jobs
154157
Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
155158
}

‎api/apps/v1alpha1/nimservice_types.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
corev1 "k8s.io/api/core/v1"
2727
networkingv1 "k8s.io/api/networking/v1"
2828
rbacv1 "k8s.io/api/rbac/v1"
29+
"k8s.io/apimachinery/pkg/api/resource"
2930
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3031
"k8s.io/apimachinery/pkg/util/intstr"
3132
"k8s.io/utils/ptr"
@@ -135,6 +136,8 @@ type NIMServiceList struct {
135136
// NIMServiceStorage defines the attributes of various storage targets used to store the model.
136137
type NIMServiceStorage struct {
137138
NIMCache NIMCacheVolSpec `json:"nimCache,omitempty"`
139+
// SharedMemorySizeLimit sets the max size of the shared memory volume (emptyDir) used by NIMs for fast model runtime I/O.
140+
SharedMemorySizeLimit *resource.Quantity `json:"sharedMemorySizeLimit,omitempty"`
138141
// PersistentVolumeClaim is the pvc volume used for caching NIM
139142
PVC PersistentVolumeClaim `json:"pvc,omitempty"`
140143
// HostPath is the host path volume for caching NIM
@@ -449,7 +452,8 @@ func (n *NIMService) GetVolumes(modelPVC PersistentVolumeClaim) []corev1.Volume
449452
Name: "dshm",
450453
VolumeSource: corev1.VolumeSource{
451454
EmptyDir: &corev1.EmptyDirVolumeSource{
452-
Medium: corev1.StorageMediumMemory,
455+
Medium: corev1.StorageMediumMemory,
456+
SizeLimit: n.Spec.Storage.SharedMemorySizeLimit,
453457
},
454458
},
455459
},

‎api/apps/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bundle/manifests/apps.nvidia.com_nemocustomizers.yaml‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ spec:
2525
name: v1alpha1
2626
schema:
2727
openAPIV3Schema:
28-
description: NemoCustomizer is the Schema for the NemoCustomizer API
28+
description: NemoCustomizer is the Schema for the NemoCustomizer API.
2929
properties:
3030
apiVersion:
3131
description: |-
@@ -45,7 +45,7 @@ spec:
4545
metadata:
4646
type: object
4747
spec:
48-
description: NemoCustomizerSpec defines the desired state of NemoCustomizer
48+
description: NemoCustomizerSpec defines the desired state of NemoCustomizer.
4949
properties:
5050
annotations:
5151
additionalProperties:
@@ -254,7 +254,7 @@ spec:
254254
type: object
255255
type: array
256256
expose:
257-
description: ExposeV1 defines attributes to expose the service
257+
description: ExposeV1 defines attributes to expose the service.
258258
properties:
259259
ingress:
260260
description: IngressV1 defines attributes for ingress
@@ -275,7 +275,7 @@ spec:
275275
paths:
276276
items:
277277
description: IngressPath defines attributes for ingress
278-
paths
278+
paths.
279279
properties:
280280
path:
281281
default: /
@@ -296,7 +296,7 @@ spec:
296296
rule: (has(self.spec) && has(self.enabled) && self.enabled)
297297
|| !has(self.enabled) || !self.enabled
298298
service:
299-
description: Service defines attributes to create a service
299+
description: Service defines attributes to create a service.
300300
properties:
301301
annotations:
302302
additionalProperties:
@@ -323,7 +323,7 @@ spec:
323323
format: int64
324324
type: integer
325325
image:
326-
description: Image defines image attributes
326+
description: Image defines image attributes.
327327
properties:
328328
pullPolicy:
329329
type: string
@@ -344,7 +344,7 @@ spec:
344344
type: string
345345
type: object
346346
metrics:
347-
description: Metrics defines attributes to setup metrics collection
347+
description: Metrics defines attributes to setup metrics collection.
348348
properties:
349349
enabled:
350350
type: boolean
@@ -1145,7 +1145,7 @@ spec:
11451145
type: string
11461146
scale:
11471147
description: Autoscaling defines attributes to automatically scale
1148-
the service based on metrics
1148+
the service based on metrics.
11491149
properties:
11501150
annotations:
11511151
additionalProperties:
@@ -1155,7 +1155,7 @@ spec:
11551155
type: boolean
11561156
hpa:
11571157
description: HorizontalPodAutoscalerSpec defines the parameters
1158-
required to setup HPA
1158+
required to setup HPA.
11591159
properties:
11601160
behavior:
11611161
description: |-
@@ -2525,6 +2525,15 @@ spec:
25252525
RunAIQueue is the Run.AI's scheduler queue to be used for training jobs.
25262526
Used only if the scheduler is set to runai.
25272527
type: string
2528+
sharedMemorySizeLimit:
2529+
anyOf:
2530+
- type: integer
2531+
- type: string
2532+
description: SharedMemorySizeLimit sets the max size of the shared
2533+
memory volume (emptyDir) used by the training jobs for fast
2534+
model runtime I/O.
2535+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
2536+
x-kubernetes-int-or-string: true
25282537
timeout:
25292538
description: Timeout for the training job to complete
25302539
type: integer
@@ -2655,7 +2664,7 @@ spec:
26552664
- wandb
26562665
type: object
26572666
status:
2658-
description: NemoCustomizerStatus defines the observed state of NemoCustomizer
2667+
description: NemoCustomizerStatus defines the observed state of NemoCustomizer.
26592668
properties:
26602669
availableReplicas:
26612670
format: int32

‎bundle/manifests/apps.nvidia.com_nemodatastores.yaml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ spec:
2525
name: v1alpha1
2626
schema:
2727
openAPIV3Schema:
28-
description: NemoDatastore is the Schema for the NemoDatastore API
28+
description: NemoDatastore is the Schema for the NemoDatastore API.
2929
properties:
3030
apiVersion:
3131
description: |-
@@ -45,7 +45,7 @@ spec:
4545
metadata:
4646
type: object
4747
spec:
48-
description: NemoDatastoreSpec defines the desired state of NemoDatastore
48+
description: NemoDatastoreSpec defines the desired state of NemoDatastore.
4949
properties:
5050
annotations:
5151
additionalProperties:
@@ -234,7 +234,7 @@ spec:
234234
type: object
235235
type: array
236236
expose:
237-
description: ExposeV1 defines attributes to expose the service
237+
description: ExposeV1 defines attributes to expose the service.
238238
properties:
239239
ingress:
240240
description: IngressV1 defines attributes for ingress
@@ -255,7 +255,7 @@ spec:
255255
paths:
256256
items:
257257
description: IngressPath defines attributes for ingress
258-
paths
258+
paths.
259259
properties:
260260
path:
261261
default: /
@@ -276,7 +276,7 @@ spec:
276276
rule: (has(self.spec) && has(self.enabled) && self.enabled)
277277
|| !has(self.enabled) || !self.enabled
278278
service:
279-
description: Service defines attributes to create a service
279+
description: Service defines attributes to create a service.
280280
properties:
281281
annotations:
282282
additionalProperties:
@@ -303,7 +303,7 @@ spec:
303303
format: int64
304304
type: integer
305305
image:
306-
description: Image defines image attributes
306+
description: Image defines image attributes.
307307
properties:
308308
pullPolicy:
309309
type: string
@@ -324,7 +324,7 @@ spec:
324324
type: string
325325
type: object
326326
metrics:
327-
description: Metrics defines attributes to setup metrics collection
327+
description: Metrics defines attributes to setup metrics collection.
328328
properties:
329329
enabled:
330330
type: boolean
@@ -857,7 +857,7 @@ spec:
857857
type: string
858858
scale:
859859
description: Autoscaling defines attributes to automatically scale
860-
the service based on metrics
860+
the service based on metrics.
861861
properties:
862862
annotations:
863863
additionalProperties:
@@ -867,7 +867,7 @@ spec:
867867
type: boolean
868868
hpa:
869869
description: HorizontalPodAutoscalerSpec defines the parameters
870-
required to setup HPA
870+
required to setup HPA.
871871
properties:
872872
behavior:
873873
description: |-
@@ -1529,7 +1529,7 @@ spec:
15291529
- secrets
15301530
type: object
15311531
status:
1532-
description: NemoDatastoreStatus defines the observed state of NemoDatastore
1532+
description: NemoDatastoreStatus defines the observed state of NemoDatastore.
15331533
properties:
15341534
availableReplicas:
15351535
format: int32

‎bundle/manifests/apps.nvidia.com_nemoentitystores.yaml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ spec:
2525
name: v1alpha1
2626
schema:
2727
openAPIV3Schema:
28-
description: NemoEntitystore is the Schema for the NemoEntitystore API
28+
description: NemoEntitystore is the Schema for the NemoEntitystore API.
2929
properties:
3030
apiVersion:
3131
description: |-
@@ -45,7 +45,7 @@ spec:
4545
metadata:
4646
type: object
4747
spec:
48-
description: NemoEntitystoreSpec defines the desired state of NemoEntitystore
48+
description: NemoEntitystoreSpec defines the desired state of NemoEntitystore.
4949
properties:
5050
annotations:
5151
additionalProperties:
@@ -246,7 +246,7 @@ spec:
246246
type: object
247247
type: array
248248
expose:
249-
description: ExposeV1 defines attributes to expose the service
249+
description: ExposeV1 defines attributes to expose the service.
250250
properties:
251251
ingress:
252252
description: IngressV1 defines attributes for ingress
@@ -267,7 +267,7 @@ spec:
267267
paths:
268268
items:
269269
description: IngressPath defines attributes for ingress
270-
paths
270+
paths.
271271
properties:
272272
path:
273273
default: /
@@ -288,7 +288,7 @@ spec:
288288
rule: (has(self.spec) && has(self.enabled) && self.enabled)
289289
|| !has(self.enabled) || !self.enabled
290290
service:
291-
description: Service defines attributes to create a service
291+
description: Service defines attributes to create a service.
292292
properties:
293293
annotations:
294294
additionalProperties:
@@ -315,7 +315,7 @@ spec:
315315
format: int64
316316
type: integer
317317
image:
318-
description: Image defines image attributes
318+
description: Image defines image attributes.
319319
properties:
320320
pullPolicy:
321321
type: string
@@ -336,7 +336,7 @@ spec:
336336
type: string
337337
type: object
338338
metrics:
339-
description: Metrics defines attributes to setup metrics collection
339+
description: Metrics defines attributes to setup metrics collection.
340340
properties:
341341
enabled:
342342
type: boolean
@@ -795,7 +795,7 @@ spec:
795795
type: string
796796
scale:
797797
description: Autoscaling defines attributes to automatically scale
798-
the service based on metrics
798+
the service based on metrics.
799799
properties:
800800
annotations:
801801
additionalProperties:
@@ -805,7 +805,7 @@ spec:
805805
type: boolean
806806
hpa:
807807
description: HorizontalPodAutoscalerSpec defines the parameters
808-
required to setup HPA
808+
required to setup HPA.
809809
properties:
810810
behavior:
811811
description: |-
@@ -1446,7 +1446,7 @@ spec:
14461446
- image
14471447
type: object
14481448
status:
1449-
description: NemoEntitystoreStatus defines the observed state of NemoEntitystore
1449+
description: NemoEntitystoreStatus defines the observed state of NemoEntitystore.
14501450
properties:
14511451
availableReplicas:
14521452
format: int32

0 commit comments

Comments
 (0)