Skip to content

fix: add deepseek-v4 models, fix calculate_cost, and improve error ha… #2736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Comui520 wants to merge 1 commit into
base: main
Choose a base branch
from
+62 −6
Open

fix: add deepseek-v4 models, fix calculate_cost, and improve error ha… #2736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions deepeval/models/llms/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1059,6 +1059,22 @@ def make_model_data(**kwargs: Any) -> ModelDataFactory:
input_price=1.00 / 1e6,
output_price=2.00 / 1e6,
),
"deepseek-v4-flash": make_model_data(
supports_log_probs=False,
supports_multimodal=False,
supports_structured_outputs=True,
supports_json=True,
input_price=None,
output_price=None,
),
"deepseek-v4-pro": make_model_data(
supports_log_probs=False,
supports_multimodal=False,
supports_structured_outputs=True,
supports_json=True,
input_price=None,
output_price=None,
),
}
)

Expand Down
23 changes: 21 additions & 2 deletions deepeval/synthesizer/chunking/context_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
DeepEvalBaseEmbeddingModel,
DeepEvalBaseLLM,
)
from deepeval.errors import DeepEvalError
from deepeval.utils import update_pbar, add_pbar, remove_pbars
from deepeval.config.settings import get_settings

Expand Down Expand Up @@ -209,6 +210,7 @@ def generate_contexts(
update_pbar(progress, pbar_id, remove=False)

# process each doc end-to-end (sync), with per-doc error logging
docs_with_errors: List[str] = []
for path, chunker in source_file_to_chunker_map.items():
collection = None
try:
Expand Down Expand Up @@ -267,6 +269,7 @@ def generate_contexts(
source_files.extend([path] * len(ctxs_for_doc))

except Exception as exc:
docs_with_errors.append(path)
# record and continue with other docs
show_trace = bool(get_settings().DEEPEVAL_LOG_STACK_TRACES)
exc_info = (
Expand Down Expand Up @@ -306,6 +309,12 @@ def generate_contexts(
"Not enough chunks in smallest document",
)

if docs_with_errors and not contexts:
raise DeepEvalError(
f"Context generation failed for all {len(docs_with_errors)} "
f"document(s). Check the logs above for per-document errors."
)

return contexts, source_files, scores

finally:
Expand Down Expand Up @@ -432,8 +441,10 @@ async def pipeline(path: str, chunker: DocumentChunker):
results = await asyncio.gather(*tasks, return_exceptions=True)

# Collect results, surface any errors after cleanup
docs_with_errors: List[str] = []
for path, res in zip(paths, results):
if isinstance(res, Exception):
docs_with_errors.append(path)
logger.error(
"Document pipeline failed for %s",
path,
Expand Down Expand Up @@ -463,6 +474,12 @@ async def pipeline(path: str, chunker: DocumentChunker):
"Not enough chunks in smallest document",
)

if docs_with_errors and not contexts:
raise DeepEvalError(
f"Context generation failed for all {len(docs_with_errors)} "
f"document(s). Check the logs above for per-document errors."
)

return contexts, source_files, scores

finally:
Expand Down Expand Up @@ -837,7 +854,8 @@ def evaluate_chunk(self, chunk) -> float:
prompt = FilterTemplate.evaluate_context(chunk)
if self.using_native_model:
res, cost = self.model.generate(prompt, schema=ContextScore)
self.total_cost += cost
if cost is not None:
self.total_cost += cost
return (res.clarity + res.depth + res.structure + res.relevance) / 4
else:
try:
Expand All @@ -862,7 +880,8 @@ async def a_evaluate_chunk(self, chunk) -> float:
prompt = FilterTemplate.evaluate_context(chunk)
if self.using_native_model:
res, cost = await self.model.a_generate(prompt, schema=ContextScore)
self.total_cost += cost
if cost is not None:
self.total_cost += cost
return (res.clarity + res.depth + res.structure + res.relevance) / 4
else:

Expand Down
29 changes: 25 additions & 4 deletions deepeval/synthesizer/synthesizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import os
from contextlib import nullcontext

from deepeval.errors import DeepEvalError
from deepeval.utils import get_or_create_event_loop
from deepeval.synthesizer.chunking.context_generator import ContextGenerator
from deepeval.metrics.utils import (
Expand Down Expand Up @@ -486,6 +487,11 @@ def generate_goldens_from_docs(
pbar_id=pbar_id,
)
)
if not contexts:
raise DeepEvalError(
"No contexts were generated from the provided documents. "
"This may be caused by an incompatible model or invalid document format."
)
if self.synthesis_cost:
self.synthesis_cost += context_generator.total_cost
print_synthesizer_status(
Expand Down Expand Up @@ -605,6 +611,11 @@ async def a_generate_goldens_from_docs(
pbar_id=pbar_id,
)
)
if not contexts:
raise DeepEvalError(
"No contexts were generated from the provided documents. "
"This may be caused by an incompatible model or invalid document format."
)
if self.synthesis_cost:
self.synthesis_cost += context_generator.total_cost
print_synthesizer_status(
Expand Down Expand Up @@ -1686,7 +1697,7 @@ def _generate_schema(
) -> BaseModel:
if is_native_model(model):
res, cost = model.generate(prompt, schema)
if self.synthesis_cost is not None:
if cost is not None and self.synthesis_cost is not None:
self.synthesis_cost += cost
return res
else:
Expand All @@ -1712,7 +1723,7 @@ async def _a_generate_schema(
) -> BaseModel:
if is_native_model(model):
res, cost = await model.a_generate(prompt, schema)
if self.synthesis_cost is not None:
if cost is not None and self.synthesis_cost is not None:
self.synthesis_cost += cost
return res
else:
Expand All @@ -1733,7 +1744,7 @@ async def _a_generate_schema(
def _generate(self, prompt: str) -> str:
if self.using_native_model:
res, cost = self.model.generate(prompt)
if self.synthesis_cost is not None:
if cost is not None and self.synthesis_cost is not None:
self.synthesis_cost += cost
return res
else:
Expand All @@ -1747,7 +1758,7 @@ def _generate(self, prompt: str) -> str:
async def _a_generate(self, prompt: str) -> str:
if self.using_native_model:
res, cost = await self.model.a_generate(prompt)
if self.synthesis_cost is not None:
if cost is not None and self.synthesis_cost is not None:
self.synthesis_cost += cost
return res
else:
Expand Down Expand Up @@ -2119,6 +2130,11 @@ def generate_conversational_goldens_from_docs(
pbar_id=pbar_id,
)
)
if not contexts:
raise DeepEvalError(
"No contexts were generated from the provided documents. "
"This may be caused by an incompatible model or invalid document format."
)
if self.synthesis_cost:
self.synthesis_cost += context_generator.total_cost
print_synthesizer_status(
Expand Down Expand Up @@ -2236,6 +2252,11 @@ async def a_generate_conversational_goldens_from_docs(
pbar_id=pbar_id,
)
)
if not contexts:
raise DeepEvalError(
"No contexts were generated from the provided documents. "
"This may be caused by an incompatible model or invalid document format."
)
if self.synthesis_cost:
self.synthesis_cost += context_generator.total_cost
print_synthesizer_status(
Expand Down
Loading