Skip to content

Commit 3346bbd

Browse files
shengnuoshengnuo
committed
Add objectStore and ExternalDatabase fields to datastore .spec
Signed-off-by: Sheng Lin <[email protected]>
1 parent 32ff019 commit 3346bbd

File tree

7 files changed

+539
-278
lines changed

7 files changed

+539
-278
lines changed

‎api/apps/v1alpha1/nemo_datastore_types.go‎

Lines changed: 113 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"fmt"
2121
"maps"
2222
"os"
23+
"strconv"
2324

2425
rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types"
2526
utils "github.com/NVIDIA/k8s-nim-operator/internal/utils"
@@ -79,7 +80,22 @@ type NemoDatastoreSpec struct {
7980
GroupID *int64 `json:"groupID,omitempty"`
8081
RuntimeClass string `json:"runtimeClass,omitempty"`
8182

82-
DataStoreParams NemoDatastoreParams `json:"dataStoreParams"`
83+
// ObjectStore specifies the location and credentials for accessing the external Object Storage
84+
ObjectStoreConfig ObjectStoreConfig `json:"objectStoreConfig"` // e.g. minio
85+
// ExternalDatabase contains external PostgreSQL configuration
86+
DatabaseConfig DatabaseConfig `json:"databaseConfig"` // e.g. postgres
87+
// secrets contains the pre-requisite secrets that must be created before deploying the datastore CR
88+
Secrets Secrets `json:"secrets"`
89+
// PVC defines the PersistentVolumeClaim for the datastore
90+
PVC *PersistentVolumeClaim `json:"pvc,omitempty"`
91+
}
92+
93+
type Secrets struct {
94+
GiteaAdminSecret string `json:"giteaAdminSecret"`
95+
LfsJwtSecret string `json:"lfsJwtSecret"`
96+
DataStoreInitSecret string `json:"datastoreInitSecret"`
97+
DataStoreConfigSecret string `json:"datastoreConfigSecret"` // config_environment.sh
98+
DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"`
8399
}
84100

85101
// NemoDatastoreStatus defines the observed state of NemoDatastore
@@ -89,21 +105,32 @@ type NemoDatastoreStatus struct {
89105
State string `json:"state,omitempty"`
90106
}
91107

92-
type NemoDatastoreParams struct {
93-
DBSecret string `json:"dbSecret"`
94-
GiteaAdminSecret string `json:"giteaAdminSecret"`
108+
type ObjectStoreConfig struct { // e.g. Minio, s3
109+
// ObjectStoreCredentials stores the configuration to retrieve the object store credentials
110+
Credentials ObjectStoreCredentials `json:"credentials"`
95111

96-
ObjectStoreSecret string `json:"objStoreSecret"`
97-
DataStoreSettingsSecret string `json:"datastoreSettingsSecret"`
98-
LfsJwtSecret string `json:"lfsJwtSecret"`
112+
// +kubebuilder:default:=true
113+
ServeDirect bool `json:"serveDirect,omitempty"`
99114

100-
DataStoreInitSecret string `json:"datastoreInitSecret"`
101-
DataStoreConfigSecret string `json:"datastoreConfigSecret"`
102-
DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"`
115+
// endpoint is the fully qualidfied object store endpoint
116+
Endpoint string `json:"endpoint"`
117+
// BucketName is the bucket where LFS files will be stored
118+
BucketName string `json:"bucketName"`
119+
// Region is the region where bucket is hosted
120+
Region string `json:"region"`
121+
// SSL enable ssl for object store transport
122+
SSL bool `json:"ssl"`
123+
}
103124

104-
SshEnabled bool `json:"sshEnabled"`
125+
type ObjectStoreCredentials struct {
126+
// User is the non-root username for a NEMO Service in the object store.
127+
User string `json:"user"`
105128

106-
PVC *PersistentVolumeClaim `json:"pvc,omitempty"`
129+
// SecretName is the name of the secret which has the object credentials for a NEMO service user.
130+
SecretName string `json:"secretName"`
131+
132+
// PasswordKey is the name of the key in the `CredentialsSecret` secret for the object store credentials.
133+
PasswordKey string `json:"passwordKey"`
107134
}
108135

109136
// +genclient
@@ -134,9 +161,8 @@ type NemoDatastoreList struct {
134161
// Prefers pvc.Name if explicitly set by the user in the NemoDatastore instance
135162
func (n *NemoDatastore) GetPVCName() string {
136163
pvcName := fmt.Sprintf("%s-pvc", n.GetName())
137-
dsParam := n.Spec.DataStoreParams
138-
if dsParam.PVC != nil && dsParam.PVC.Name != "" {
139-
pvcName = dsParam.PVC.Name
164+
if n.Spec.PVC != nil && n.Spec.PVC.Name != "" {
165+
pvcName = n.Spec.PVC.Name
140166
}
141167
return pvcName
142168
}
@@ -197,23 +223,16 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
197223
Value: "/data/gitea/git",
198224
},
199225
{
200-
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
201-
ValueFrom: &corev1.EnvVarSource{
202-
SecretKeyRef: &corev1.SecretKeySelector{
203-
Key: "objectStoreKey",
204-
LocalObjectReference: corev1.LocalObjectReference{
205-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
206-
},
207-
},
208-
},
226+
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
227+
Value: n.Spec.ObjectStoreConfig.Credentials.User,
209228
},
210229
{
211230
Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY",
212231
ValueFrom: &corev1.EnvVarSource{
213232
SecretKeyRef: &corev1.SecretKeySelector{
214-
Key: "objectStoreSecret",
233+
Key: n.Spec.ObjectStoreConfig.Credentials.PasswordKey,
215234
LocalObjectReference: corev1.LocalObjectReference{
216-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
235+
Name: n.Spec.ObjectStoreConfig.Credentials.SecretName,
217236
},
218237
},
219238
},
@@ -224,7 +243,7 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
224243
SecretKeyRef: &corev1.SecretKeySelector{
225244
Key: "jwtSecret",
226245
LocalObjectReference: corev1.LocalObjectReference{
227-
Name: n.Spec.DataStoreParams.LfsJwtSecret,
246+
Name: n.Spec.Secrets.LfsJwtSecret,
228247
},
229248
},
230249
},
@@ -233,9 +252,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
233252
Name: "GITEA__DATABASE__PASSWD",
234253
ValueFrom: &corev1.EnvVarSource{
235254
SecretKeyRef: &corev1.SecretKeySelector{
236-
Key: "postgresPassword",
255+
Key: n.Spec.DatabaseConfig.Credentials.PasswordKey,
237256
LocalObjectReference: corev1.LocalObjectReference{
238-
Name: n.Spec.DataStoreParams.DBSecret,
257+
Name: n.Spec.DatabaseConfig.Credentials.SecretName,
239258
},
240259
},
241260
},
@@ -245,6 +264,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
245264
}
246265

247266
func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
267+
objStoreSetting := n.Spec.ObjectStoreConfig
268+
dbSetting := n.Spec.DatabaseConfig
269+
248270
envVars := []corev1.EnvVar{
249271
{
250272
Name: "GITEA_APP_INI",
@@ -271,23 +293,16 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
271293
Value: "/data/gitea/git",
272294
},
273295
{
274-
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
275-
ValueFrom: &corev1.EnvVarSource{
276-
SecretKeyRef: &corev1.SecretKeySelector{
277-
Key: "objectStoreKey",
278-
LocalObjectReference: corev1.LocalObjectReference{
279-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
280-
},
281-
},
282-
},
296+
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
297+
Value: objStoreSetting.Credentials.User,
283298
},
284299
{
285300
Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY",
286301
ValueFrom: &corev1.EnvVarSource{
287302
SecretKeyRef: &corev1.SecretKeySelector{
288-
Key: "objectStoreSecret",
303+
Key: objStoreSetting.Credentials.PasswordKey,
289304
LocalObjectReference: corev1.LocalObjectReference{
290-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
305+
Name: objStoreSetting.Credentials.SecretName,
291306
},
292307
},
293308
},
@@ -298,7 +313,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
298313
SecretKeyRef: &corev1.SecretKeySelector{
299314
Key: "jwtSecret",
300315
LocalObjectReference: corev1.LocalObjectReference{
301-
Name: n.Spec.DataStoreParams.LfsJwtSecret,
316+
Name: n.Spec.Secrets.LfsJwtSecret,
302317
},
303318
},
304319
},
@@ -307,9 +322,9 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
307322
Name: "GITEA__DATABASE__PASSWD",
308323
ValueFrom: &corev1.EnvVarSource{
309324
SecretKeyRef: &corev1.SecretKeySelector{
310-
Key: "postgresPassword",
325+
Key: dbSetting.Credentials.PasswordKey,
311326
LocalObjectReference: corev1.LocalObjectReference{
312-
Name: n.Spec.DataStoreParams.DBSecret,
327+
Name: dbSetting.Credentials.SecretName,
313328
},
314329
},
315330
},
@@ -320,7 +335,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
320335
SecretKeyRef: &corev1.SecretKeySelector{
321336
Key: "GITEA_ADMIN_USERNAME",
322337
LocalObjectReference: corev1.LocalObjectReference{
323-
Name: n.Spec.DataStoreParams.GiteaAdminSecret,
338+
Name: n.Spec.Secrets.GiteaAdminSecret,
324339
},
325340
},
326341
},
@@ -331,11 +346,55 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
331346
SecretKeyRef: &corev1.SecretKeySelector{
332347
Key: "GITEA_ADMIN_PASSWORD",
333348
LocalObjectReference: corev1.LocalObjectReference{
334-
Name: n.Spec.DataStoreParams.GiteaAdminSecret,
349+
Name: n.Spec.Secrets.GiteaAdminSecret,
335350
},
336351
},
337352
},
338353
},
354+
{
355+
Name: "GITEA__LFS__SERVE_DIRECT",
356+
Value: strconv.FormatBool(objStoreSetting.ServeDirect),
357+
},
358+
{
359+
Name: "GITEA__LFS__STORAGE_TYPE",
360+
Value: "minio",
361+
},
362+
{
363+
Name: "GITEA__LFS__MINIO_ENDPOINT",
364+
Value: objStoreSetting.Endpoint,
365+
},
366+
{
367+
Name: "GITEA__LFS__MINIO_BUCKET",
368+
Value: objStoreSetting.BucketName,
369+
},
370+
{
371+
Name: "GITEA__LFS__MINIO_LOCATION",
372+
Value: objStoreSetting.Region,
373+
},
374+
{
375+
Name: "GITEA__LFS__MINIO_LOCATION",
376+
Value: objStoreSetting.Region,
377+
},
378+
{
379+
Name: "GITEA__LFS__MINIO_USE_SSL",
380+
Value: strconv.FormatBool(objStoreSetting.SSL),
381+
},
382+
{
383+
Name: "GITEA__DATABASE__SSL_MODE",
384+
Value: "disable",
385+
},
386+
{
387+
Name: "GITEA__DATABASE__NAME",
388+
Value: dbSetting.DatabaseName,
389+
},
390+
{
391+
Name: "GITEA__DATABASE__HOST",
392+
Value: fmt.Sprintf("%s:%d", dbSetting.Host, dbSetting.Port),
393+
},
394+
{
395+
Name: "GITEA__DATABASE__USER",
396+
Value: dbSetting.Credentials.User,
397+
},
339398
}
340399
return envVars
341400
}
@@ -369,7 +428,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
369428
Name: "init",
370429
VolumeSource: corev1.VolumeSource{
371430
Secret: &corev1.SecretVolumeSource{
372-
SecretName: n.Spec.DataStoreParams.DataStoreInitSecret,
431+
SecretName: n.Spec.Secrets.DataStoreInitSecret,
373432
DefaultMode: &initMode,
374433
},
375434
},
@@ -378,7 +437,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
378437
Name: "config",
379438
VolumeSource: corev1.VolumeSource{
380439
Secret: &corev1.SecretVolumeSource{
381-
SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret,
440+
SecretName: n.Spec.Secrets.DataStoreConfigSecret,
382441
DefaultMode: &initMode,
383442
},
384443
},
@@ -387,7 +446,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
387446
Name: "inline-config-sources",
388447
VolumeSource: corev1.VolumeSource{
389448
Secret: &corev1.SecretVolumeSource{
390-
SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret,
449+
SecretName: n.Spec.Secrets.DataStoreInlineConfigSecret,
391450
DefaultMode: &configMode,
392451
},
393452
},
@@ -400,7 +459,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
400459
},
401460
}
402461

403-
if n.Spec.DataStoreParams.PVC != nil {
462+
if n.Spec.PVC != nil {
404463
volumes = append(volumes, corev1.Volume{
405464
Name: "data",
406465
VolumeSource: corev1.VolumeSource{
@@ -421,25 +480,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
421480
}
422481

423482
func (n *NemoDatastore) ShouldCreatePersistentStorage() bool {
424-
return n.Spec.DataStoreParams.PVC != nil && n.Spec.DataStoreParams.PVC.Create != nil && *n.Spec.DataStoreParams.PVC.Create
425-
}
426-
427-
// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance
428-
func (n *NemoDatastore) GetEnvFrom() []corev1.EnvFromSource {
429-
return []corev1.EnvFromSource{}
430-
}
431-
432-
// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance
433-
func (n *NemoDatastore) GetInitAppIniEnvFrom() []corev1.EnvFromSource {
434-
return []corev1.EnvFromSource{
435-
{
436-
SecretRef: &corev1.SecretEnvSource{
437-
LocalObjectReference: corev1.LocalObjectReference{
438-
Name: n.Spec.DataStoreParams.DataStoreSettingsSecret,
439-
},
440-
},
441-
},
442-
}
483+
return n.Spec.PVC != nil && n.Spec.PVC.Create != nil && *n.Spec.PVC.Create
443484
}
444485

445486
// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance
@@ -633,8 +674,8 @@ func (n *NemoDatastore) GetVolumeMounts() []corev1.VolumeMount {
633674
Name: "data",
634675
}
635676

636-
if n.Spec.DataStoreParams.PVC != nil {
637-
dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath
677+
if n.Spec.PVC != nil {
678+
dataMount.SubPath = n.Spec.PVC.SubPath
638679
}
639680
mounts = append(mounts, dataMount)
640681
return mounts
@@ -664,8 +705,8 @@ func (n *NemoDatastore) GetVolumeMountsInitContainer() []corev1.VolumeMount {
664705
Name: "data",
665706
}
666707

667-
if n.Spec.DataStoreParams.PVC != nil {
668-
dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath
708+
if n.Spec.PVC != nil {
709+
dataMount.SubPath = n.Spec.PVC.SubPath
669710
}
670711
mounts = append(mounts, dataMount)
671712
return mounts
@@ -682,7 +723,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container {
682723
},
683724
VolumeMounts: n.GetVolumeMountsInitContainer(),
684725
Env: n.GetInitContainerEnv(),
685-
EnvFrom: n.GetInitAppIniEnvFrom(),
686726
},
687727
{
688728
Name: "init-app-ini",
@@ -693,7 +733,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container {
693733
},
694734
VolumeMounts: n.GetVolumeMountsInitContainer(),
695735
Env: n.GetInitContainerEnv(),
696-
EnvFrom: n.GetInitAppIniEnvFrom(),
697736
},
698737
{
699738
Name: "configure-datastore",
@@ -707,7 +746,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container {
707746
},
708747
VolumeMounts: n.GetVolumeMountsInitContainer(),
709748
Env: n.GetInitContainerEnv(),
710-
EnvFrom: n.GetInitAppIniEnvFrom(),
711749
SecurityContext: &corev1.SecurityContext{
712750
RunAsUser: n.GetUserID(),
713751
},

0 commit comments

Comments
 (0)