Skip to content

Commit 5812e5d

Browse files
shivamerlashivamerla
committed
Allow PVC to be updated with following supported mutable fields
* Annotations for custom provisioning (import, clone volumes) * Size for Online volume expansion * Avoid redundant code for reconciling PVC * Add unit tests Signed-off-by: Shiva Krishna, Merla <smerla@nvidia.com>
1 parent b630bd8 commit 5812e5d

40 files changed

Lines changed: 688 additions & 290 deletions

‎api/apps/v1alpha1/common_types.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,3 +217,22 @@ type NGCSecret struct {
217217
// +kubebuilder:default:="NGC_API_KEY"
218218
Key string `json:"key"`
219219
}
220+
221+
// PersistentVolumeClaim defines the attributes of PVC used as a source for caching NIM model.
222+
type PersistentVolumeClaim struct {
223+
// Create indicates to create a new PVC
224+
Create *bool `json:"create,omitempty"`
225+
// Name is the name of the PVC
226+
Name string `json:"name,omitempty"`
227+
// StorageClass to be used for PVC creation. Leave it as empty if the PVC is already created or
228+
// a default storage class is set in the cluster.
229+
StorageClass string `json:"storageClass,omitempty"`
230+
// Size of the NIM cache in Gi, used during PVC creation
231+
Size string `json:"size,omitempty"`
232+
// VolumeAccessMode is the volume access mode of the PVC
233+
VolumeAccessMode corev1.PersistentVolumeAccessMode `json:"volumeAccessMode,omitempty"`
234+
// SubPath is the path inside the PVC that should be mounted
235+
SubPath string `json:"subPath,omitempty"`
236+
// Annotations for the PVC
237+
Annotations map[string]string `json:"annotations,omitempty"`
238+
}

‎api/apps/v1alpha1/nemo_customizer_types.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1060,6 +1060,46 @@ func (n *NemoCustomizer) GetSecretParams(secretMapData map[string]string) *rende
10601060
return params
10611061
}
10621062

1063+
// GetPVCName returns the name to be used for the PVC based on the custom spec
1064+
// Prefers pvc.Name if explicitly set by the user in the NeMo Customizer instance.
1065+
func (n *NemoCustomizer) GetPVCName() string {
1066+
pvcName := fmt.Sprintf("%s-pvc", n.GetName())
1067+
if n.Spec.Training.ModelPVC.Name != "" {
1068+
pvcName = n.Spec.Training.ModelPVC.Name
1069+
}
1070+
return pvcName
1071+
}
1072+
1073+
// GetPVCParams returns parameters to render a PersistentVolumeClaim from templates.
1074+
func (n *NemoCustomizer) GetPVCParams() *rendertypes.PVCParams {
1075+
params := &rendertypes.PVCParams{}
1076+
1077+
// Set metadata
1078+
params.Enabled = ptr.Deref(n.Spec.Training.ModelPVC.Create, false)
1079+
params.Name = n.GetPVCName()
1080+
params.Namespace = n.GetNamespace()
1081+
params.Labels = n.GetServiceLabels()
1082+
params.Annotations = n.GetPVCAnnotations()
1083+
1084+
// PVC-specific config
1085+
params.AccessMode = n.Spec.Training.ModelPVC.VolumeAccessMode
1086+
params.Storage = n.Spec.Training.ModelPVC.Size
1087+
params.StorageClassName = n.Spec.Training.ModelPVC.StorageClass
1088+
1089+
return params
1090+
}
1091+
1092+
// GetPVCAnnotations return custom PVC annotations setup by the user
1093+
func (n *NemoCustomizer) GetPVCAnnotations() map[string]string {
1094+
// Get global customizer annotations
1095+
pvcAnnotations := n.GetAnnotations()
1096+
1097+
if n.Spec.Training.ModelPVC.Annotations != nil {
1098+
return utils.MergeMaps(pvcAnnotations, n.Spec.Training.ModelPVC.Annotations)
1099+
}
1100+
return pvcAnnotations
1101+
}
1102+
10631103
func init() {
10641104
SchemeBuilder.Register(&NemoCustomizer{}, &NemoCustomizerList{})
10651105
}

‎api/apps/v1alpha1/nemo_datastore_types.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1068,6 +1068,35 @@ func (n *NemoDatastore) GetServiceMonitorParams() *rendertypes.ServiceMonitorPar
10681068
return params
10691069
}
10701070

1071+
// GetPVCParams returns parameters to render a PersistentVolumeClaim from templates.
1072+
func (n *NemoDatastore) GetPVCParams() *rendertypes.PVCParams {
1073+
params := &rendertypes.PVCParams{}
1074+
1075+
// Set metadata
1076+
params.Enabled = ptr.Deref(n.Spec.PVC.Create, false)
1077+
params.Name = n.GetPVCName()
1078+
params.Namespace = n.GetNamespace()
1079+
params.Labels = n.GetServiceLabels()
1080+
params.Annotations = n.GetPVCAnnotations()
1081+
1082+
// PVC-specific config
1083+
params.AccessMode = n.Spec.PVC.VolumeAccessMode
1084+
params.Storage = n.Spec.PVC.Size
1085+
params.StorageClassName = n.Spec.PVC.StorageClass
1086+
1087+
return params
1088+
}
1089+
1090+
func (n *NemoDatastore) GetPVCAnnotations() map[string]string {
1091+
// Get global service annotations
1092+
pvcAnnotations := n.GetServiceAnnotations()
1093+
1094+
if n.Spec.PVC.Annotations != nil {
1095+
return utils.MergeMaps(pvcAnnotations, n.Spec.PVC.Annotations)
1096+
}
1097+
return pvcAnnotations
1098+
}
1099+
10711100
func (n *NemoDatastore) GetIngressAnnotations() map[string]string {
10721101
NemoDatastoreAnnotations := n.GetNemoDatastoreAnnotations()
10731102

‎api/apps/v1alpha1/nemo_guardrails_types.go‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,10 +144,10 @@ type NemoGuardrailList struct {
144144

145145
// GetPVCName returns the name to be used for the PVC based on the custom spec
146146
// Prefers pvc.Name if explicitly set by the user in the NemoGuardrail instance.
147-
func (n *NemoGuardrail) GetPVCName(pvc PersistentVolumeClaim) string {
147+
func (n *NemoGuardrail) GetPVCName() string {
148148
pvcName := fmt.Sprintf("%s-pvc", n.GetName())
149-
if pvc.Name != "" {
150-
pvcName = pvc.Name
149+
if n.Spec.ConfigStore.PVC != nil && n.Spec.ConfigStore.PVC.Name != "" {
150+
pvcName = n.Spec.ConfigStore.PVC.Name
151151
}
152152
return pvcName
153153
}
@@ -763,6 +763,39 @@ func (n *NemoGuardrail) GetServiceMonitorParams() *rendertypes.ServiceMonitorPar
763763
return params
764764
}
765765

766+
// GetPVCParams returns parameters to render a PersistentVolumeClaim from templates.
767+
func (n *NemoGuardrail) GetPVCParams() *rendertypes.PVCParams {
768+
params := &rendertypes.PVCParams{}
769+
770+
if n.Spec.ConfigStore.PVC == nil {
771+
return nil
772+
}
773+
774+
// Set metadata
775+
params.Enabled = ptr.Deref(n.Spec.ConfigStore.PVC.Create, false)
776+
params.Name = n.GetPVCName()
777+
params.Namespace = n.GetNamespace()
778+
params.Labels = n.GetServiceLabels()
779+
params.Annotations = n.GetPVCAnnotations()
780+
781+
// PVC-specific config
782+
params.AccessMode = n.Spec.ConfigStore.PVC.VolumeAccessMode
783+
params.Storage = n.Spec.ConfigStore.PVC.Size
784+
params.StorageClassName = n.Spec.ConfigStore.PVC.StorageClass
785+
786+
return params
787+
}
788+
789+
func (n *NemoGuardrail) GetPVCAnnotations() map[string]string {
790+
// Get global service annotations
791+
pvcAnnotations := n.GetServiceAnnotations()
792+
793+
if n.Spec.ConfigStore.PVC.Annotations != nil {
794+
return utils.MergeMaps(pvcAnnotations, n.Spec.ConfigStore.PVC.Annotations)
795+
}
796+
return pvcAnnotations
797+
}
798+
766799
func (n *NemoGuardrail) GetIngressAnnotations() map[string]string {
767800
NemoGuardrailAnnotations := n.GetNemoGuardrailAnnotations()
768801

‎api/apps/v1alpha1/nimcache_types.go‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"k8s.io/utils/ptr"
2626

2727
"github.com/NVIDIA/k8s-nim-operator/internal/k8sutil"
28+
rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types"
2829
)
2930

3031
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
@@ -132,23 +133,6 @@ type NIMCacheStorage struct {
132133
HostPath *string `json:"hostPath,omitempty"`
133134
}
134135

135-
// PersistentVolumeClaim defines the attributes of PVC used as a source for caching NIM model.
136-
type PersistentVolumeClaim struct {
137-
// Create indicates to create a new PVC
138-
Create *bool `json:"create,omitempty"`
139-
// Name is the name of the PVC
140-
Name string `json:"name,omitempty"`
141-
// StorageClass to be used for PVC creation. Leave it as empty if the PVC is already created or
142-
// a default storage class is set in the cluster.
143-
StorageClass string `json:"storageClass,omitempty"`
144-
// Size of the NIM cache in Gi, used during PVC creation
145-
Size string `json:"size,omitempty"`
146-
// VolumeAccessMode is the volume access mode of the PVC
147-
VolumeAccessMode corev1.PersistentVolumeAccessMode `json:"volumeAccessMode,omitempty"`
148-
// SubPath is the path inside the PVC that should be mounted
149-
SubPath string `json:"subPath,omitempty"`
150-
}
151-
152136
// NIMCacheStatus defines the observed state of NIMCache.
153137
type NIMCacheStatus struct {
154138
State string `json:"state,omitempty"`
@@ -247,10 +231,10 @@ type NIMCache struct {
247231

248232
// GetPVCName returns the name to be used for the PVC based on the custom spec
249233
// Prefers pvc.Name if explicitly set by the user in the NIMCache instance.
250-
func (n *NIMCache) GetPVCName(pvc PersistentVolumeClaim) string {
234+
func (n *NIMCache) GetPVCName() string {
251235
pvcName := fmt.Sprintf("%s-pvc", n.GetName())
252-
if pvc.Name != "" {
253-
pvcName = pvc.Name
236+
if n.Spec.Storage.PVC.Name != "" {
237+
pvcName = n.Spec.Storage.PVC.Name
254238
}
255239
return pvcName
256240
}
@@ -351,6 +335,24 @@ func (n *NIMCache) GetInitContainers() []corev1.Container {
351335
return []corev1.Container{}
352336
}
353337

338+
// GetPVCParams returns parameters to render a PersistentVolumeClaim from templates.
339+
func (n *NIMCache) GetPVCParams() *rendertypes.PVCParams {
340+
params := &rendertypes.PVCParams{}
341+
342+
// Set metadata
343+
params.Enabled = ptr.Deref(n.Spec.Storage.PVC.Create, false)
344+
params.Name = n.GetPVCName()
345+
params.Namespace = n.GetNamespace()
346+
params.Annotations = n.Spec.Storage.PVC.Annotations
347+
348+
// PVC-specific config
349+
params.AccessMode = n.Spec.Storage.PVC.VolumeAccessMode
350+
params.Storage = n.Spec.Storage.PVC.Size
351+
params.StorageClassName = n.Spec.Storage.PVC.StorageClass
352+
353+
return params
354+
}
355+
354356
// +kubebuilder:object:root=true
355357

356358
// NIMCacheList contains a list of NIMCache.

‎api/apps/v1alpha1/nimservice_types.go‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -147,10 +147,10 @@ type NIMServiceStorage struct {
147147

148148
// GetPVCName returns the name to be used for the PVC based on the custom spec
149149
// Prefers pvc.Name if explicitly set by the user in the NIMService instance.
150-
func (n *NIMService) GetPVCName(pvc PersistentVolumeClaim) string {
150+
func (n *NIMService) GetPVCName() string {
151151
pvcName := fmt.Sprintf("%s-pvc", n.GetName())
152-
if pvc.Name != "" {
153-
pvcName = pvc.Name
152+
if n.Spec.Storage.PVC.Name != "" {
153+
pvcName = n.Spec.Storage.PVC.Name
154154
}
155155
return pvcName
156156
}
@@ -463,7 +463,7 @@ func (n *NIMService) GetVolumesMounts() []corev1.Volume {
463463
}
464464

465465
// GetVolumes returns volumes for the NIMService container.
466-
func (n *NIMService) GetVolumes(modelPVC PersistentVolumeClaim) []corev1.Volume {
466+
func (n *NIMService) GetVolumes(pvcName string) []corev1.Volume {
467467
// TODO: Fetch actual PVC name from associated NIMCache obj
468468
volumes := []corev1.Volume{
469469
{
@@ -479,7 +479,7 @@ func (n *NIMService) GetVolumes(modelPVC PersistentVolumeClaim) []corev1.Volume
479479
Name: "model-store",
480480
VolumeSource: corev1.VolumeSource{
481481
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
482-
ClaimName: modelPVC.Name,
482+
ClaimName: pvcName,
483483
ReadOnly: n.GetStorageReadOnly(),
484484
},
485485
},
@@ -494,12 +494,12 @@ func (n *NIMService) GetVolumes(modelPVC PersistentVolumeClaim) []corev1.Volume
494494
}
495495

496496
// GetVolumeMounts returns volumes for the NIMService container.
497-
func (n *NIMService) GetVolumeMounts(modelPVC PersistentVolumeClaim) []corev1.VolumeMount {
497+
func (n *NIMService) GetVolumeMounts(pvcSubPath string) []corev1.VolumeMount {
498498
volumeMounts := []corev1.VolumeMount{
499499
{
500500
Name: "model-store",
501501
MountPath: "/model-store",
502-
SubPath: modelPVC.SubPath,
502+
SubPath: pvcSubPath,
503503
},
504504
{
505505
Name: "dshm",
@@ -953,6 +953,35 @@ func (n *NIMService) GetServiceMonitorParams() *rendertypes.ServiceMonitorParams
953953
return params
954954
}
955955

956+
// GetPVCParams returns parameters to render a PersistentVolumeClaim from templates.
957+
func (n *NIMService) GetPVCParams() *rendertypes.PVCParams {
958+
params := &rendertypes.PVCParams{}
959+
960+
// Set metadata
961+
params.Enabled = ptr.Deref(n.Spec.Storage.PVC.Create, false)
962+
params.Name = n.GetPVCName()
963+
params.Namespace = n.GetNamespace()
964+
params.Labels = n.GetServiceLabels()
965+
params.Annotations = n.GetPVCAnnotations()
966+
967+
// PVC-specific config
968+
params.AccessMode = n.Spec.Storage.PVC.VolumeAccessMode
969+
params.Storage = n.Spec.Storage.PVC.Size
970+
params.StorageClassName = n.Spec.Storage.PVC.StorageClass
971+
972+
return params
973+
}
974+
975+
func (n *NIMService) GetPVCAnnotations() map[string]string {
976+
// Get global service annotations
977+
pvcAnnotations := n.GetNIMServiceAnnotations()
978+
979+
if n.Spec.Storage.PVC.Annotations != nil {
980+
return utils.MergeMaps(pvcAnnotations, n.Spec.Storage.PVC.Annotations)
981+
}
982+
return pvcAnnotations
983+
}
984+
956985
func (n *NIMService) GetIngressAnnotations() map[string]string {
957986
nimServiceAnnotations := n.GetNIMServiceAnnotations()
958987

‎api/apps/v1alpha1/nimservice_types_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,14 @@ import (
2727
func TestGetVolumes(t *testing.T) {
2828
tests := []struct {
2929
name string
30-
modelPVC PersistentVolumeClaim
30+
modelPVC string
3131
desired []corev1.Volume
3232
nimService *NIMService
3333
}{
3434
{
3535
name: "Storage read only is nil",
3636
nimService: &NIMService{Spec: NIMServiceSpec{Storage: NIMServiceStorage{}}},
37-
modelPVC: PersistentVolumeClaim{Name: "test-pvc"},
37+
modelPVC: "test-pvc",
3838
desired: []corev1.Volume{
3939
{
4040
Name: "dshm",
@@ -58,7 +58,7 @@ func TestGetVolumes(t *testing.T) {
5858
{
5959
name: "Storage read only is false",
6060
nimService: &NIMService{Spec: NIMServiceSpec{Storage: NIMServiceStorage{ReadOnly: &[]bool{false}[0]}}},
61-
modelPVC: PersistentVolumeClaim{Name: "test-pvc"},
61+
modelPVC: "test-pvc",
6262
desired: []corev1.Volume{
6363
{
6464
Name: "dshm",
@@ -82,7 +82,7 @@ func TestGetVolumes(t *testing.T) {
8282
{
8383
name: "Storage read only is true",
8484
nimService: &NIMService{Spec: NIMServiceSpec{Storage: NIMServiceStorage{ReadOnly: &[]bool{true}[0]}}},
85-
modelPVC: PersistentVolumeClaim{Name: "test-pvc"},
85+
modelPVC: "test-pvc",
8686
desired: []corev1.Volume{
8787
{
8888
Name: "dshm",

‎api/apps/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bundle/manifests/apps.nvidia.com_nemocustomizers.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1974,6 +1974,11 @@ spec:
19741974
description: ModelPVC is the persistent storage for models used
19751975
for finetuning
19761976
properties:
1977+
annotations:
1978+
additionalProperties:
1979+
type: string
1980+
description: Annotations for the PVC
1981+
type: object
19771982
create:
19781983
description: Create indicates to create a new PVC
19791984
type: boolean

‎bundle/manifests/apps.nvidia.com_nemodatastores.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -788,6 +788,11 @@ spec:
788788
pvc:
789789
description: PVC defines the PersistentVolumeClaim for the datastore
790790
properties:
791+
annotations:
792+
additionalProperties:
793+
type: string
794+
description: Annotations for the PVC
795+
type: object
791796
create:
792797
description: Create indicates to create a new PVC
793798
type: boolean

0 commit comments

Comments
 (0)