diff --git a/deepeval/models/llms/constants.py b/deepeval/models/llms/constants.py index d91d12299..5ef8d9dec 100644 --- a/deepeval/models/llms/constants.py +++ b/deepeval/models/llms/constants.py @@ -1059,6 +1059,22 @@ def make_model_data(**kwargs: Any) -> ModelDataFactory: input_price=1.00 / 1e6, output_price=2.00 / 1e6, ), + "deepseek-v4-flash": make_model_data( + supports_log_probs=False, + supports_multimodal=False, + supports_structured_outputs=True, + supports_json=True, + input_price=None, + output_price=None, + ), + "deepseek-v4-pro": make_model_data( + supports_log_probs=False, + supports_multimodal=False, + supports_structured_outputs=True, + supports_json=True, + input_price=None, + output_price=None, + ), } ) diff --git a/deepeval/synthesizer/chunking/context_generator.py b/deepeval/synthesizer/chunking/context_generator.py index 6ca56d734..2f7d74b99 100644 --- a/deepeval/synthesizer/chunking/context_generator.py +++ b/deepeval/synthesizer/chunking/context_generator.py @@ -27,6 +27,7 @@ DeepEvalBaseEmbeddingModel, DeepEvalBaseLLM, ) +from deepeval.errors import DeepEvalError from deepeval.utils import update_pbar, add_pbar, remove_pbars from deepeval.config.settings import get_settings @@ -209,6 +210,7 @@ def generate_contexts( update_pbar(progress, pbar_id, remove=False) # process each doc end-to-end (sync), with per-doc error logging + docs_with_errors: List[str] = [] for path, chunker in source_file_to_chunker_map.items(): collection = None try: @@ -267,6 +269,7 @@ def generate_contexts( source_files.extend([path] * len(ctxs_for_doc)) except Exception as exc: + docs_with_errors.append(path) # record and continue with other docs show_trace = bool(get_settings().DEEPEVAL_LOG_STACK_TRACES) exc_info = ( @@ -306,6 +309,12 @@ def generate_contexts( "Not enough chunks in smallest document", ) + if docs_with_errors and not contexts: + raise DeepEvalError( + f"Context generation failed for all {len(docs_with_errors)} " + f"document(s). Check the logs above for per-document errors." + ) + return contexts, source_files, scores finally: @@ -432,8 +441,10 @@ async def pipeline(path: str, chunker: DocumentChunker): results = await asyncio.gather(*tasks, return_exceptions=True) # Collect results, surface any errors after cleanup + docs_with_errors: List[str] = [] for path, res in zip(paths, results): if isinstance(res, Exception): + docs_with_errors.append(path) logger.error( "Document pipeline failed for %s", path, @@ -463,6 +474,12 @@ async def pipeline(path: str, chunker: DocumentChunker): "Not enough chunks in smallest document", ) + if docs_with_errors and not contexts: + raise DeepEvalError( + f"Context generation failed for all {len(docs_with_errors)} " + f"document(s). Check the logs above for per-document errors." + ) + return contexts, source_files, scores finally: @@ -837,7 +854,8 @@ def evaluate_chunk(self, chunk) -> float: prompt = FilterTemplate.evaluate_context(chunk) if self.using_native_model: res, cost = self.model.generate(prompt, schema=ContextScore) - self.total_cost += cost + if cost is not None: + self.total_cost += cost return (res.clarity + res.depth + res.structure + res.relevance) / 4 else: try: @@ -862,7 +880,8 @@ async def a_evaluate_chunk(self, chunk) -> float: prompt = FilterTemplate.evaluate_context(chunk) if self.using_native_model: res, cost = await self.model.a_generate(prompt, schema=ContextScore) - self.total_cost += cost + if cost is not None: + self.total_cost += cost return (res.clarity + res.depth + res.structure + res.relevance) / 4 else: diff --git a/deepeval/synthesizer/synthesizer.py b/deepeval/synthesizer/synthesizer.py index ffe448592..ec671b510 100644 --- a/deepeval/synthesizer/synthesizer.py +++ b/deepeval/synthesizer/synthesizer.py @@ -15,6 +15,7 @@ import os from contextlib import nullcontext +from deepeval.errors import DeepEvalError from deepeval.utils import get_or_create_event_loop from deepeval.synthesizer.chunking.context_generator import ContextGenerator from deepeval.metrics.utils import ( @@ -486,6 +487,11 @@ def generate_goldens_from_docs( pbar_id=pbar_id, ) ) + if not contexts: + raise DeepEvalError( + "No contexts were generated from the provided documents. " + "This may be caused by an incompatible model or invalid document format." + ) if self.synthesis_cost: self.synthesis_cost += context_generator.total_cost print_synthesizer_status( @@ -605,6 +611,11 @@ async def a_generate_goldens_from_docs( pbar_id=pbar_id, ) ) + if not contexts: + raise DeepEvalError( + "No contexts were generated from the provided documents. " + "This may be caused by an incompatible model or invalid document format." + ) if self.synthesis_cost: self.synthesis_cost += context_generator.total_cost print_synthesizer_status( @@ -1686,7 +1697,7 @@ def _generate_schema( ) -> BaseModel: if is_native_model(model): res, cost = model.generate(prompt, schema) - if self.synthesis_cost is not None: + if cost is not None and self.synthesis_cost is not None: self.synthesis_cost += cost return res else: @@ -1712,7 +1723,7 @@ async def _a_generate_schema( ) -> BaseModel: if is_native_model(model): res, cost = await model.a_generate(prompt, schema) - if self.synthesis_cost is not None: + if cost is not None and self.synthesis_cost is not None: self.synthesis_cost += cost return res else: @@ -1733,7 +1744,7 @@ async def _a_generate_schema( def _generate(self, prompt: str) -> str: if self.using_native_model: res, cost = self.model.generate(prompt) - if self.synthesis_cost is not None: + if cost is not None and self.synthesis_cost is not None: self.synthesis_cost += cost return res else: @@ -1747,7 +1758,7 @@ def _generate(self, prompt: str) -> str: async def _a_generate(self, prompt: str) -> str: if self.using_native_model: res, cost = await self.model.a_generate(prompt) - if self.synthesis_cost is not None: + if cost is not None and self.synthesis_cost is not None: self.synthesis_cost += cost return res else: @@ -2119,6 +2130,11 @@ def generate_conversational_goldens_from_docs( pbar_id=pbar_id, ) ) + if not contexts: + raise DeepEvalError( + "No contexts were generated from the provided documents. " + "This may be caused by an incompatible model or invalid document format." + ) if self.synthesis_cost: self.synthesis_cost += context_generator.total_cost print_synthesizer_status( @@ -2236,6 +2252,11 @@ async def a_generate_conversational_goldens_from_docs( pbar_id=pbar_id, ) ) + if not contexts: + raise DeepEvalError( + "No contexts were generated from the provided documents. " + "This may be caused by an incompatible model or invalid document format." + ) if self.synthesis_cost: self.synthesis_cost += context_generator.total_cost print_synthesizer_status(