Skip to content

Commit 0b15e9e

Browse files
penguine-ippenguine-ip
authored
Merge pull request #2066 from A-Vamshi/main
Add name and comments to save_as and loading goldens
2 parents e538925 + 0eb575f commit 0b15e9e

8 files changed

Lines changed: 164 additions & 51 deletions

File tree

‎deepeval/dataset/dataset.py‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -458,6 +458,8 @@ def add_goldens_from_csv_file(
458458
tools_called_col_delimiter: str = ";",
459459
expected_tools_col_name: Optional[str] = "expected_tools",
460460
expected_tools_col_delimiter: str = ";",
461+
comments_key_name: str = "comments",
462+
name_key_name: str = "name",
461463
source_file_col_name: Optional[str] = None,
462464
additional_metadata_col_name: Optional[str] = None,
463465
scenario_col_name: Optional[str] = "scenario",
@@ -526,6 +528,8 @@ def get_column_data(df: pd.DataFrame, col_name: str, default=None):
526528
df, expected_tools_col_name, default=""
527529
)
528530
]
531+
comments = get_column_data(df, comments_key_name)
532+
name = get_column_data(df, name_key_name)
529533
source_files = get_column_data(df, source_file_col_name)
530534
additional_metadatas = [
531535
ast.literal_eval(metadata) if metadata else None
@@ -546,6 +550,8 @@ def get_column_data(df: pd.DataFrame, col_name: str, default=None):
546550
retrieval_context,
547551
tools_called,
548552
expected_tools,
553+
comments,
554+
name,
549555
source_file,
550556
additional_metadata,
551557
scenario,
@@ -560,6 +566,8 @@ def get_column_data(df: pd.DataFrame, col_name: str, default=None):
560566
retrieval_contexts,
561567
tools_called,
562568
expected_tools,
569+
comments,
570+
name,
563571
source_files,
564572
additional_metadatas,
565573
scenarios,
@@ -569,14 +577,16 @@ def get_column_data(df: pd.DataFrame, col_name: str, default=None):
569577
):
570578
if scenario:
571579
self._multi_turn = True
572-
parsed_turns = parse_turns(turns)
580+
parsed_turns = parse_turns(turns) if turns else []
573581
self.goldens.append(
574582
ConversationalGolden(
575583
scenario=scenario,
576584
turns=parsed_turns,
577585
expected_outcome=expected_outcome,
578586
user_description=user_description,
579587
context=context,
588+
comments=comments,
589+
name=name,
580590
)
581591
)
582592
else:
@@ -592,6 +602,8 @@ def get_column_data(df: pd.DataFrame, col_name: str, default=None):
592602
expected_tools=expected_tools,
593603
additional_metadata=additional_metadata,
594604
source_file=source_file,
605+
comments=comments,
606+
name=name,
595607
)
596608
)
597609

@@ -605,6 +617,8 @@ def add_goldens_from_json_file(
605617
retrieval_context_key_name: Optional[str] = "retrieval_context",
606618
tools_called_key_name: Optional[str] = "tools_called",
607619
expected_tools_key_name: Optional[str] = "expected_tools",
620+
comments_key_name: str = "comments",
621+
name_key_name: str = "name",
608622
source_file_key_name: Optional[str] = "source_file",
609623
additional_metadata_key_name: Optional[str] = "additional_metadata",
610624
scenario_key_name: Optional[str] = "scenario",
@@ -628,7 +642,8 @@ def add_goldens_from_json_file(
628642
expected_outcome = json_obj.get(expected_outcome_key_name)
629643
user_description = json_obj.get(user_description_key_name)
630644
context = json_obj.get(context_key_name)
631-
645+
comments = json_obj.get(comments_key_name)
646+
name = json_obj.get(name_key_name)
632647
parsed_turns = parse_turns(turns) if turns else []
633648

634649
self._multi_turn = True
@@ -639,6 +654,8 @@ def add_goldens_from_json_file(
639654
expected_outcome=expected_outcome,
640655
user_description=user_description,
641656
context=context,
657+
comments=comments,
658+
name=name,
642659
)
643660
)
644661
else:
@@ -649,6 +666,8 @@ def add_goldens_from_json_file(
649666
retrieval_context = json_obj.get(retrieval_context_key_name)
650667
tools_called = json_obj.get(tools_called_key_name)
651668
expected_tools = json_obj.get(expected_tools_key_name)
669+
comments = json_obj.get(comments_key_name)
670+
name = json_obj.get(name_key_name)
652671
source_file = json_obj.get(source_file_key_name)
653672
additional_metadata = json_obj.get(additional_metadata_key_name)
654673

@@ -663,6 +682,8 @@ def add_goldens_from_json_file(
663682
tools_called=tools_called,
664683
expected_tools=expected_tools,
665684
additional_metadata=additional_metadata,
685+
comments=comments,
686+
name=name,
666687
source_file=source_file,
667688
)
668689
)
@@ -928,6 +949,8 @@ def save_as(
928949
expected_outcome=golden.expected_outcome,
929950
user_description=golden.user_description,
930951
context=golden.context,
952+
name=golden.name,
953+
comments=golden.comments,
931954
)
932955
for golden in self.goldens
933956
]
@@ -939,6 +962,8 @@ def save_as(
939962
actual_output=golden.actual_output,
940963
retrieval_context=golden.retrieval_context,
941964
context=golden.context,
965+
name=golden.name,
966+
comments=golden.comments,
942967
source_file=golden.source_file,
943968
)
944969
for golden in self.goldens
@@ -981,6 +1006,8 @@ def save_as(
9811006
"expected_outcome": golden.expected_outcome,
9821007
"user_description": golden.user_description,
9831008
"context": golden.context,
1009+
"name": golden.name,
1010+
"comments": golden.comments,
9841011
}
9851012
for golden in goldens
9861013
]
@@ -992,6 +1019,8 @@ def save_as(
9921019
"expected_output": golden.expected_output,
9931020
"retrieval_context": golden.retrieval_context,
9941021
"context": golden.context,
1022+
"name": golden.name,
1023+
"comments": golden.comments,
9951024
"source_file": golden.source_file,
9961025
}
9971026
for golden in goldens
@@ -1010,6 +1039,8 @@ def save_as(
10101039
"expected_outcome",
10111040
"user_description",
10121041
"context",
1042+
"name",
1043+
"comments",
10131044
]
10141045
)
10151046
for golden in goldens:
@@ -1030,6 +1061,8 @@ def save_as(
10301061
golden.expected_outcome,
10311062
golden.user_description,
10321063
context,
1064+
golden.name,
1065+
golden.comments,
10331066
]
10341067
)
10351068
else:
@@ -1040,6 +1073,8 @@ def save_as(
10401073
"expected_output",
10411074
"retrieval_context",
10421075
"context",
1076+
"name",
1077+
"comments",
10431078
"source_file",
10441079
]
10451080
)
@@ -1061,6 +1096,8 @@ def save_as(
10611096
golden.expected_output,
10621097
retrieval_context,
10631098
context,
1099+
golden.name,
1100+
golden.comments,
10641101
golden.source_file,
10651102
]
10661103
)

‎docs/docs/evaluation-datasets.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ Depending on the type of goldens you supply, datasets are either **single-turn**
4040
If you don't already have an `EvaluationDataset`, a great starting point is to simply write down the prompts you're currently using to manually eyeball your LLM outputs. You can also do this on Confident AI, which integrates 100% with `deepeval`:
4141

4242
<VideoDisplayer
43-
src="https://confident-docs.s3.us-east-1.amazonaws.com/dataset-editor:dataset-annotation.mp4"
43+
src="https://confident-docs.s3.us-east-1.amazonaws.com/datasets:create-4k.mp4"
4444
confidentUrl="/docs/dataset-editor/annotate-datasets"
4545
label="Learn Dataset Annotation on Confident AI"
4646
/>

‎docs/docs/metrics-introduction.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ evaluate(
149149
If you're logged into [Confident AI](https://confident-ai.com) before running an evaluation (`deepeval login` or `deepeval view` in the CLI), you'll also get entire testing reports on the platform:
150150

151151
<VideoDisplayer
152-
src="https://confident-docs.s3.us-east-1.amazonaws.com/evaluation:testing-report.mp4"
152+
src="https://confident-docs.s3.us-east-1.amazonaws.com/evaluation:single-turn-e2e-report.mp4"
153153
confidentUrl="/docs/llm-evaluation/dashboards/testing-reports"
154154
label="Run Evaluations on Confident AI"
155155
/>

0 commit comments

Comments
 (0)