Skip to content

Commit 2a729ed

Browse files
shengnuoshivamerla
shengnuo
authored and
shivamerla
committed
Update for NeMo Evaluator 25.03
Signed-off-by: Sheng Lin <shelin@nvidia.com>
1 parent c8ea490 commit 2a729ed

8 files changed

Lines changed: 280 additions & 8 deletions

File tree

‎api/apps/v1alpha1/nemo_common_types.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,24 @@ type ArgoWorkflows struct {
2323
ServiceAccount string `json:"serviceAccount"`
2424
}
2525

26-
// VectorDB defines configuration for connecting to external VectorDB
26+
// VectorDB defines the configuration for connecting to the external VectorDB
2727
type VectorDB struct {
2828
// +kubebuilder:validation:MinLength=1
2929
Endpoint string `json:"endpoint"`
3030
}
3131

32-
// Datastore defines configuration for connecting to NeMo Datastore service
32+
// Datastore defines the configuration for connecting to the NeMo Datastore service
3333
type Datastore struct {
3434
// +kubebuilder:validation:MinLength=1
3535
Endpoint string `json:"endpoint"`
3636
}
3737

38+
// Entitystore defines the configuration for connecting to the NeMo EntityStore service
39+
type Entitystore struct {
40+
// +kubebuilder:validation:MinLength=1
41+
Endpoint string `json:"endpoint"`
42+
}
43+
3844
// DatabaseConfig is the external database configuration
3945
type DatabaseConfig struct {
4046
// Host is the hostname of the database.

‎api/apps/v1alpha1/nemo_evaluator_types.go‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ type NemoEvaluatorSpec struct {
8787
VectorDB VectorDB `json:"vectorDB"`
8888
// Datastore stores the datastore endpoint.
8989
Datastore Datastore `json:"datastore"`
90-
90+
// Entitystore stores the entitystore endpoint.
91+
Entitystore Entitystore `json:"entitystore"`
9192
// OpenTelemetry Settings
9293
// +kubebuilder:validation:Optional
9394
OpenTelemetry OTelSpec `json:"otel,omitempty"`
@@ -112,6 +113,26 @@ type NemoEvaluatorSpec struct {
112113

113114
// EnableValidation indicates that the validation jobs to be enabled
114115
EnableValidation *bool `json:"enableValidation,omitempty"`
116+
117+
// EvaluationImages defines the external images used for evaluation
118+
EvaluationImages EvaluationImages `json:"evaluationImages"`
119+
}
120+
121+
type EvaluationImages struct {
122+
// +kubebuilder:validation:MinLength=1
123+
BigcodeEvalHarness string `json:"bigcodeEvalHarness"`
124+
// +kubebuilder:validation:MinLength=1
125+
LmEvalHarness string `json:"lmEvalHarness"`
126+
// +kubebuilder:validation:MinLength=1
127+
SimilarityMetrics string `json:"similarityMetrics"`
128+
// +kubebuilder:validation:MinLength=1
129+
LlmAsJudge string `json:"llmAsJudge"`
130+
// +kubebuilder:validation:MinLength=1
131+
MtBench string `json:"mtBench"`
132+
// +kubebuilder:validation:MinLength=1
133+
Retriever string `json:"retriever"`
134+
// +kubebuilder:validation:MinLength=1
135+
Rag string `json:"rag"`
115136
}
116137

117138
// NemoEvaluatorStatus defines the observed state of NemoEvaluator
@@ -145,6 +166,39 @@ type NemoEvaluatorList struct {
145166
Items []NemoEvaluator `json:"items"`
146167
}
147168

169+
func (ei EvaluationImages) GetEvaluationImageEnv() []corev1.EnvVar {
170+
return []corev1.EnvVar{
171+
{
172+
Name: "BIGCODE_EVALUATION_HARNESS",
173+
Value: ei.BigcodeEvalHarness,
174+
},
175+
{
176+
Name: "LM_EVAL_HARNESS",
177+
Value: ei.LmEvalHarness,
178+
},
179+
{
180+
Name: "SIMILARITY_METRICS",
181+
Value: ei.SimilarityMetrics,
182+
},
183+
{
184+
Name: "LLM_AS_A_JUDGE",
185+
Value: ei.LlmAsJudge,
186+
},
187+
{
188+
Name: "MT_BENCH",
189+
Value: ei.MtBench,
190+
},
191+
{
192+
Name: "RETRIEVER",
193+
Value: ei.Retriever,
194+
},
195+
{
196+
Name: "RAG",
197+
Value: ei.Rag,
198+
},
199+
}
200+
}
201+
148202
// GetStandardSelectorLabels returns the standard selector labels for the NemoEvaluator deployment
149203
func (n *NemoEvaluator) GetStandardSelectorLabels() map[string]string {
150204
return map[string]string{
@@ -202,9 +256,13 @@ func (n *NemoEvaluator) GetStandardEnv() []corev1.EnvVar {
202256
Value: n.Spec.ArgoWorkflows.ServiceAccount,
203257
},
204258
{
205-
Name: "DATA_STORE_HOST",
259+
Name: "DATA_STORE_URL",
206260
Value: n.Spec.Datastore.Endpoint,
207261
},
262+
{
263+
Name: "ENTITY_STORE_URL",
264+
Value: n.Spec.Entitystore.Endpoint,
265+
},
208266
{
209267
Name: "EVAL_CONTAINER",
210268
Value: n.GetImage(),
@@ -231,6 +289,9 @@ func (n *NemoEvaluator) GetStandardEnv() []corev1.EnvVar {
231289
// Append the environment variables for Postgres
232290
envVars = append(envVars, n.GetPostgresEnv()...)
233291

292+
// Append the environment variables for EvaluationImages
293+
envVars = append(envVars, n.Spec.EvaluationImages.GetEvaluationImageEnv()...)
294+
234295
// Append the environment variables for OTel
235296
if n.IsOtelEnabled() {
236297
envVars = append(envVars, n.GetOtelEnv()...)
@@ -916,13 +977,16 @@ func (n *NemoEvaluator) GetInitContainers() []corev1.Container {
916977
Value: n.GetImage(),
917978
},
918979
{
919-
Name: "DATA_STORE_HOST",
980+
Name: "DATA_STORE_URL",
920981
Value: n.Spec.Datastore.Endpoint,
921982
},
922983
}
923984
// Append the environment variables for Postgres
924985
envVars = append(envVars, n.GetPostgresEnv()...)
925986

987+
// Append the environment variables for EvaluationImages
988+
envVars = append(envVars, n.Spec.EvaluationImages.GetEvaluationImageEnv()...)
989+
926990
return []corev1.Container{
927991
{
928992
Name: "wait-for-postgres",

‎api/apps/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 32 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bundle/manifests/apps.nvidia.com_nemoevaluators.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,15 @@ spec:
150150
description: EnableValidation indicates that the validation jobs to
151151
be enabled
152152
type: boolean
153+
entitystore:
154+
description: Entitystore stores the entitystore endpoint.
155+
properties:
156+
endpoint:
157+
minLength: 1
158+
type: string
159+
required:
160+
- endpoint
161+
type: object
153162
env:
154163
items:
155164
description: EnvVar represents an environment variable present in
@@ -275,6 +284,40 @@ spec:
275284
- INFO
276285
- DEBUG
277286
type: string
287+
evaluationImages:
288+
description: EvaluationImages defines the external images used for
289+
evaluation
290+
properties:
291+
bigcodeEvalHarness:
292+
minLength: 1
293+
type: string
294+
llmAsJudge:
295+
minLength: 1
296+
type: string
297+
lmEvalHarness:
298+
minLength: 1
299+
type: string
300+
mtBench:
301+
minLength: 1
302+
type: string
303+
rag:
304+
minLength: 1
305+
type: string
306+
retriever:
307+
minLength: 1
308+
type: string
309+
similarityMetrics:
310+
minLength: 1
311+
type: string
312+
required:
313+
- bigcodeEvalHarness
314+
- llmAsJudge
315+
- lmEvalHarness
316+
- mtBench
317+
- rag
318+
- retriever
319+
- similarityMetrics
320+
type: object
278321
expose:
279322
description: Expose defines attributes to expose the service
280323
properties:
@@ -2281,6 +2324,8 @@ spec:
22812324
- argoWorkflows
22822325
- databaseConfig
22832326
- datastore
2327+
- entitystore
2328+
- evaluationImages
22842329
- image
22852330
- vectorDB
22862331
type: object

‎config/crd/bases/apps.nvidia.com_nemoevaluators.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,15 @@ spec:
150150
description: EnableValidation indicates that the validation jobs to
151151
be enabled
152152
type: boolean
153+
entitystore:
154+
description: Entitystore stores the entitystore endpoint.
155+
properties:
156+
endpoint:
157+
minLength: 1
158+
type: string
159+
required:
160+
- endpoint
161+
type: object
153162
env:
154163
items:
155164
description: EnvVar represents an environment variable present in
@@ -275,6 +284,40 @@ spec:
275284
- INFO
276285
- DEBUG
277286
type: string
287+
evaluationImages:
288+
description: EvaluationImages defines the external images used for
289+
evaluation
290+
properties:
291+
bigcodeEvalHarness:
292+
minLength: 1
293+
type: string
294+
llmAsJudge:
295+
minLength: 1
296+
type: string
297+
lmEvalHarness:
298+
minLength: 1
299+
type: string
300+
mtBench:
301+
minLength: 1
302+
type: string
303+
rag:
304+
minLength: 1
305+
type: string
306+
retriever:
307+
minLength: 1
308+
type: string
309+
similarityMetrics:
310+
minLength: 1
311+
type: string
312+
required:
313+
- bigcodeEvalHarness
314+
- llmAsJudge
315+
- lmEvalHarness
316+
- mtBench
317+
- rag
318+
- retriever
319+
- similarityMetrics
320+
type: object
278321
expose:
279322
description: Expose defines attributes to expose the service
280323
properties:
@@ -2281,6 +2324,8 @@ spec:
22812324
- argoWorkflows
22822325
- databaseConfig
22832326
- datastore
2327+
- entitystore
2328+
- evaluationImages
22842329
- image
22852330
- vectorDB
22862331
type: object

‎config/samples/apps_v1alpha1_nemoevaluator.yaml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,14 @@ metadata:
77
name: nemoevaluator-sample
88
namespace: nemo
99
spec:
10+
evaluationImages:
11+
bigcodeEvalHarness: "nvcr.io/nvidia/nemo-microservices/eval-tool-benchmark-bigcode:0.12.5"
12+
lmEvalHarness: "nvcr.io/nvidia/nemo-microservices/eval-tool-benchmark-lm-eval-harness:0.12.5"
13+
similarityMetrics: "nvcr.io/nvidia/nemo-microservices/eval-tool-benchmark-custom-eval:0.12.5"
14+
llmAsJudge: "nvcr.io/nvidia/nemo-microservices/eval-tool-benchmark-llm-as-a-judge:0.12.5"
15+
mtBench: "nvcr.io/nvidia/nemo-microservices/eval-tool-benchmark-llm-as-a-judge:0.12.5"
16+
retriever: "nvcr.io/nvidia/nemo-microservices/eval-tool-benchmark-retriever:0.12.5"
17+
rag: "nvcr.io/nvidia/nemo-microservices/eval-tool-benchmark-rag:0.12.5"
1018
image:
1119
repository: nvcr.io/nvidia/nemo-microservices/evaluator
1220
tag: "25.03"
@@ -23,7 +31,9 @@ spec:
2331
vectorDB:
2432
endpoint: http://milvus.nemo.svc.cluster.local:19530
2533
datastore:
26-
endpoint: http://datastore-sample.nemo.svc.cluster.local:3000/v1/hf
34+
endpoint: http://nemodatastore-sample.nemo.svc.cluster.local:3000/v1/hf
35+
entitystore:
36+
endpoint: http://nemoentitystore-sample.nemo.svc.cluster.local:8000
2737
databaseConfig:
2838
host: evaluator-pg-postgresql.nemo.svc.cluster.local
2939
port: 5432

0 commit comments

Comments
 (0)