Skip to content

Commit 29b27f6

Browse files
shengnuoshengnuo
committed
Add objectStore and ExternalDatabase fields to datastore .spec
Signed-off-by: Sheng Lin <shelin@nvidia.com>
1 parent 32ff019 commit 29b27f6

File tree

7 files changed

+572
-270
lines changed

7 files changed

+572
-270
lines changed

‎api/apps/v1alpha1/nemo_datastore_types.go‎

Lines changed: 130 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"fmt"
2121
"maps"
2222
"os"
23+
"strconv"
2324

2425
rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types"
2526
utils "github.com/NVIDIA/k8s-nim-operator/internal/utils"
@@ -79,7 +80,22 @@ type NemoDatastoreSpec struct {
7980
GroupID *int64 `json:"groupID,omitempty"`
8081
RuntimeClass string `json:"runtimeClass,omitempty"`
8182

82-
DataStoreParams NemoDatastoreParams `json:"dataStoreParams"`
83+
// ObjectStore specifies the location and credentials for accessing the external Object Storage
84+
ObjectStoreConfig ObjectStoreConfig `json:"objectStoreConfig"` // e.g. minio
85+
// ExternalDatabase contains external PostgreSQL configuration
86+
DatabaseConfig DatabaseConfig `json:"databaseConfig"` // e.g. postgres
87+
// secrets contains the pre-requisite secrets that must be created before deploying the datastore CR
88+
Secrets Secrets `json:"secrets"`
89+
// PVC defines the PersistentVolumeClaim for the datastore
90+
PVC *PersistentVolumeClaim `json:"pvc,omitempty"`
91+
}
92+
93+
type Secrets struct {
94+
GiteaAdminSecret string `json:"giteaAdminSecret"`
95+
LfsJwtSecret string `json:"lfsJwtSecret"`
96+
DataStoreInitSecret string `json:"datastoreInitSecret"`
97+
DataStoreConfigSecret string `json:"datastoreConfigSecret"` // config_environment.sh
98+
DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"`
8399
}
84100

85101
// NemoDatastoreStatus defines the observed state of NemoDatastore
@@ -89,21 +105,49 @@ type NemoDatastoreStatus struct {
89105
State string `json:"state,omitempty"`
90106
}
91107

92-
type NemoDatastoreParams struct {
93-
DBSecret string `json:"dbSecret"`
94-
GiteaAdminSecret string `json:"giteaAdminSecret"`
108+
type ObjectStoreConfig struct { // e.g. Minio, s3
109+
// ObjectStoreCredentials stores the configuration to retrieve the object store credentials
110+
Credentials ObjectStoreCredentials `json:"credentials"`
95111

96-
ObjectStoreSecret string `json:"objStoreSecret"`
97-
DataStoreSettingsSecret string `json:"datastoreSettingsSecret"`
98-
LfsJwtSecret string `json:"lfsJwtSecret"`
112+
// +kubebuilder:default:=true
113+
ServeDirect bool `json:"serveDirect,omitempty"`
99114

100-
DataStoreInitSecret string `json:"datastoreInitSecret"`
101-
DataStoreConfigSecret string `json:"datastoreConfigSecret"`
102-
DataStoreInlineConfigSecret string `json:"datastoreInlineConfigSecret"`
115+
// endpoint is the fully qualidfied object store endpoint
116+
Endpoint string `json:"endpoint"`
117+
// BucketName is the bucket where LFS files will be stored
118+
BucketName string `json:"bucketName"`
119+
// Region is the region where bucket is hosted
120+
Region string `json:"region"`
121+
// SSL enable ssl for object store transport
122+
SSL bool `json:"ssl"`
123+
}
103124

104-
SshEnabled bool `json:"sshEnabled"`
125+
type ObjectStoreCredentials struct {
126+
// User is the non-root username for a NEMO Service in the object store.
127+
User string `json:"user"`
105128

106-
PVC *PersistentVolumeClaim `json:"pvc,omitempty"`
129+
// SecretName is the name of the secret which has the object credentials for a NEMO service user.
130+
SecretName string `json:"secretName"`
131+
132+
// PasswordKey is the name of the key in the `CredentialsSecret` secret for the object store credentials.
133+
PasswordKey string `json:"passwordKey"`
134+
}
135+
136+
type ExternalDatabase struct {
137+
// SSLMode for communicating to the database
138+
SSLMode string `json:"sslMode"`
139+
// Host is the database host
140+
Host string `json:"host"`
141+
// Port is the database port number
142+
Port int `json:"port"`
143+
// User is the Non-root username for Datastore service
144+
User string `json:"user"`
145+
// Data is the database name
146+
Database string `json:"database"`
147+
// DatabaseSecret is the name of an existing secret resource containing the database credentials
148+
DatabaseSecret string `json:"databaseSecret"`
149+
// DatabaseSecretKey is the name of an existing secret key containing the database credentials
150+
DatabaseSecretKey string `json:"databaseSecretKey"`
107151
}
108152

109153
// +genclient
@@ -134,9 +178,8 @@ type NemoDatastoreList struct {
134178
// Prefers pvc.Name if explicitly set by the user in the NemoDatastore instance
135179
func (n *NemoDatastore) GetPVCName() string {
136180
pvcName := fmt.Sprintf("%s-pvc", n.GetName())
137-
dsParam := n.Spec.DataStoreParams
138-
if dsParam.PVC != nil && dsParam.PVC.Name != "" {
139-
pvcName = dsParam.PVC.Name
181+
if n.Spec.PVC != nil && n.Spec.PVC.Name != "" {
182+
pvcName = n.Spec.PVC.Name
140183
}
141184
return pvcName
142185
}
@@ -197,23 +240,16 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
197240
Value: "/data/gitea/git",
198241
},
199242
{
200-
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
201-
ValueFrom: &corev1.EnvVarSource{
202-
SecretKeyRef: &corev1.SecretKeySelector{
203-
Key: "objectStoreKey",
204-
LocalObjectReference: corev1.LocalObjectReference{
205-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
206-
},
207-
},
208-
},
243+
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
244+
Value: n.Spec.ObjectStoreConfig.Credentials.User,
209245
},
210246
{
211247
Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY",
212248
ValueFrom: &corev1.EnvVarSource{
213249
SecretKeyRef: &corev1.SecretKeySelector{
214-
Key: "objectStoreSecret",
250+
Key: n.Spec.ObjectStoreConfig.Credentials.PasswordKey,
215251
LocalObjectReference: corev1.LocalObjectReference{
216-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
252+
Name: n.Spec.ObjectStoreConfig.Credentials.SecretName,
217253
},
218254
},
219255
},
@@ -224,7 +260,7 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
224260
SecretKeyRef: &corev1.SecretKeySelector{
225261
Key: "jwtSecret",
226262
LocalObjectReference: corev1.LocalObjectReference{
227-
Name: n.Spec.DataStoreParams.LfsJwtSecret,
263+
Name: n.Spec.Secrets.LfsJwtSecret,
228264
},
229265
},
230266
},
@@ -233,9 +269,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
233269
Name: "GITEA__DATABASE__PASSWD",
234270
ValueFrom: &corev1.EnvVarSource{
235271
SecretKeyRef: &corev1.SecretKeySelector{
236-
Key: "postgresPassword",
272+
Key: n.Spec.DatabaseConfig.Credentials.PasswordKey,
237273
LocalObjectReference: corev1.LocalObjectReference{
238-
Name: n.Spec.DataStoreParams.DBSecret,
274+
Name: n.Spec.DatabaseConfig.Credentials.SecretName,
239275
},
240276
},
241277
},
@@ -245,6 +281,9 @@ func (n *NemoDatastore) GetStandardEnv() []corev1.EnvVar {
245281
}
246282

247283
func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
284+
objStoreSetting := n.Spec.ObjectStoreConfig
285+
dbSetting := n.Spec.DatabaseConfig
286+
248287
envVars := []corev1.EnvVar{
249288
{
250289
Name: "GITEA_APP_INI",
@@ -271,23 +310,16 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
271310
Value: "/data/gitea/git",
272311
},
273312
{
274-
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
275-
ValueFrom: &corev1.EnvVarSource{
276-
SecretKeyRef: &corev1.SecretKeySelector{
277-
Key: "objectStoreKey",
278-
LocalObjectReference: corev1.LocalObjectReference{
279-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
280-
},
281-
},
282-
},
313+
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
314+
Value: objStoreSetting.Credentials.User,
283315
},
284316
{
285317
Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY",
286318
ValueFrom: &corev1.EnvVarSource{
287319
SecretKeyRef: &corev1.SecretKeySelector{
288-
Key: "objectStoreSecret",
320+
Key: objStoreSetting.Credentials.PasswordKey,
289321
LocalObjectReference: corev1.LocalObjectReference{
290-
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
322+
Name: objStoreSetting.Credentials.SecretName,
291323
},
292324
},
293325
},
@@ -298,7 +330,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
298330
SecretKeyRef: &corev1.SecretKeySelector{
299331
Key: "jwtSecret",
300332
LocalObjectReference: corev1.LocalObjectReference{
301-
Name: n.Spec.DataStoreParams.LfsJwtSecret,
333+
Name: n.Spec.Secrets.LfsJwtSecret,
302334
},
303335
},
304336
},
@@ -307,9 +339,9 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
307339
Name: "GITEA__DATABASE__PASSWD",
308340
ValueFrom: &corev1.EnvVarSource{
309341
SecretKeyRef: &corev1.SecretKeySelector{
310-
Key: "postgresPassword",
342+
Key: dbSetting.Credentials.PasswordKey,
311343
LocalObjectReference: corev1.LocalObjectReference{
312-
Name: n.Spec.DataStoreParams.DBSecret,
344+
Name: dbSetting.Credentials.SecretName,
313345
},
314346
},
315347
},
@@ -320,7 +352,7 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
320352
SecretKeyRef: &corev1.SecretKeySelector{
321353
Key: "GITEA_ADMIN_USERNAME",
322354
LocalObjectReference: corev1.LocalObjectReference{
323-
Name: n.Spec.DataStoreParams.GiteaAdminSecret,
355+
Name: n.Spec.Secrets.GiteaAdminSecret,
324356
},
325357
},
326358
},
@@ -331,11 +363,55 @@ func (n *NemoDatastore) GetInitContainerEnv() []corev1.EnvVar {
331363
SecretKeyRef: &corev1.SecretKeySelector{
332364
Key: "GITEA_ADMIN_PASSWORD",
333365
LocalObjectReference: corev1.LocalObjectReference{
334-
Name: n.Spec.DataStoreParams.GiteaAdminSecret,
366+
Name: n.Spec.Secrets.GiteaAdminSecret,
335367
},
336368
},
337369
},
338370
},
371+
{
372+
Name: "GITEA__LFS__SERVE_DIRECT",
373+
Value: strconv.FormatBool(objStoreSetting.ServeDirect),
374+
},
375+
{
376+
Name: "GITEA__LFS__STORAGE_TYPE",
377+
Value: "minio",
378+
},
379+
{
380+
Name: "GITEA__LFS__MINIO_ENDPOINT",
381+
Value: objStoreSetting.Endpoint,
382+
},
383+
{
384+
Name: "GITEA__LFS__MINIO_BUCKET",
385+
Value: objStoreSetting.BucketName,
386+
},
387+
{
388+
Name: "GITEA__LFS__MINIO_LOCATION",
389+
Value: objStoreSetting.Region,
390+
},
391+
{
392+
Name: "GITEA__LFS__MINIO_LOCATION",
393+
Value: objStoreSetting.Region,
394+
},
395+
{
396+
Name: "GITEA__LFS__MINIO_USE_SSL",
397+
Value: strconv.FormatBool(objStoreSetting.SSL),
398+
},
399+
{
400+
Name: "GITEA__DATABASE__SSL_MODE",
401+
Value: "disable",
402+
},
403+
{
404+
Name: "GITEA__DATABASE__NAME",
405+
Value: dbSetting.DatabaseName,
406+
},
407+
{
408+
Name: "GITEA__DATABASE__HOST",
409+
Value: fmt.Sprintf("%s:%d", dbSetting.Host, dbSetting.Port),
410+
},
411+
{
412+
Name: "GITEA__DATABASE__USER",
413+
Value: dbSetting.Credentials.User,
414+
},
339415
}
340416
return envVars
341417
}
@@ -369,7 +445,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
369445
Name: "init",
370446
VolumeSource: corev1.VolumeSource{
371447
Secret: &corev1.SecretVolumeSource{
372-
SecretName: n.Spec.DataStoreParams.DataStoreInitSecret,
448+
SecretName: n.Spec.Secrets.DataStoreInitSecret,
373449
DefaultMode: &initMode,
374450
},
375451
},
@@ -378,7 +454,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
378454
Name: "config",
379455
VolumeSource: corev1.VolumeSource{
380456
Secret: &corev1.SecretVolumeSource{
381-
SecretName: n.Spec.DataStoreParams.DataStoreConfigSecret,
457+
SecretName: n.Spec.Secrets.DataStoreConfigSecret,
382458
DefaultMode: &initMode,
383459
},
384460
},
@@ -387,7 +463,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
387463
Name: "inline-config-sources",
388464
VolumeSource: corev1.VolumeSource{
389465
Secret: &corev1.SecretVolumeSource{
390-
SecretName: n.Spec.DataStoreParams.DataStoreInlineConfigSecret,
466+
SecretName: n.Spec.Secrets.DataStoreInlineConfigSecret,
391467
DefaultMode: &configMode,
392468
},
393469
},
@@ -400,7 +476,7 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
400476
},
401477
}
402478

403-
if n.Spec.DataStoreParams.PVC != nil {
479+
if n.Spec.PVC != nil {
404480
volumes = append(volumes, corev1.Volume{
405481
Name: "data",
406482
VolumeSource: corev1.VolumeSource{
@@ -421,27 +497,14 @@ func (n *NemoDatastore) GetVolumes() []corev1.Volume {
421497
}
422498

423499
func (n *NemoDatastore) ShouldCreatePersistentStorage() bool {
424-
return n.Spec.DataStoreParams.PVC != nil && n.Spec.DataStoreParams.PVC.Create != nil && *n.Spec.DataStoreParams.PVC.Create
500+
return n.Spec.PVC != nil && n.Spec.PVC.Create != nil && *n.Spec.PVC.Create
425501
}
426502

427503
// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance
428504
func (n *NemoDatastore) GetEnvFrom() []corev1.EnvFromSource {
429505
return []corev1.EnvFromSource{}
430506
}
431507

432-
// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance
433-
func (n *NemoDatastore) GetInitAppIniEnvFrom() []corev1.EnvFromSource {
434-
return []corev1.EnvFromSource{
435-
{
436-
SecretRef: &corev1.SecretEnvSource{
437-
LocalObjectReference: corev1.LocalObjectReference{
438-
Name: n.Spec.DataStoreParams.DataStoreSettingsSecret,
439-
},
440-
},
441-
},
442-
}
443-
}
444-
445508
// GetStandardAnnotations returns default annotations to apply to the NemoDatastore instance
446509
func (n *NemoDatastore) GetStandardAnnotations() map[string]string {
447510
standardAnnotations := map[string]string{
@@ -633,8 +696,8 @@ func (n *NemoDatastore) GetVolumeMounts() []corev1.VolumeMount {
633696
Name: "data",
634697
}
635698

636-
if n.Spec.DataStoreParams.PVC != nil {
637-
dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath
699+
if n.Spec.PVC != nil {
700+
dataMount.SubPath = n.Spec.PVC.SubPath
638701
}
639702
mounts = append(mounts, dataMount)
640703
return mounts
@@ -664,8 +727,8 @@ func (n *NemoDatastore) GetVolumeMountsInitContainer() []corev1.VolumeMount {
664727
Name: "data",
665728
}
666729

667-
if n.Spec.DataStoreParams.PVC != nil {
668-
dataMount.SubPath = n.Spec.DataStoreParams.PVC.SubPath
730+
if n.Spec.PVC != nil {
731+
dataMount.SubPath = n.Spec.PVC.SubPath
669732
}
670733
mounts = append(mounts, dataMount)
671734
return mounts
@@ -682,7 +745,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container {
682745
},
683746
VolumeMounts: n.GetVolumeMountsInitContainer(),
684747
Env: n.GetInitContainerEnv(),
685-
EnvFrom: n.GetInitAppIniEnvFrom(),
686748
},
687749
{
688750
Name: "init-app-ini",
@@ -693,7 +755,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container {
693755
},
694756
VolumeMounts: n.GetVolumeMountsInitContainer(),
695757
Env: n.GetInitContainerEnv(),
696-
EnvFrom: n.GetInitAppIniEnvFrom(),
697758
},
698759
{
699760
Name: "configure-datastore",
@@ -707,7 +768,6 @@ func (n *NemoDatastore) GetInitContainers() []corev1.Container {
707768
},
708769
VolumeMounts: n.GetVolumeMountsInitContainer(),
709770
Env: n.GetInitContainerEnv(),
710-
EnvFrom: n.GetInitAppIniEnvFrom(),
711771
SecurityContext: &corev1.SecurityContext{
712772
RunAsUser: n.GetUserID(),
713773
},

0 commit comments

Comments
 (0)