diff --git a/Makefile b/Makefile index 0cdf1e0d..6ff0b2f1 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ test-integration: $(PYTEST_INTEGRATION) --maxfail=1 test-integration-verbose: - uv run pytest tests/integration -v -rA --durations=10 + $(PYTEST_INTEGRATION) -s -rA --durations=10 --tb=long test-integration-openai: $(PYTEST_INTEGRATION) -m openai diff --git a/deepfabric/generator.py b/deepfabric/generator.py index 3a125aa8..8cc8297e 100644 --- a/deepfabric/generator.py +++ b/deepfabric/generator.py @@ -1461,10 +1461,10 @@ async def _run_generation_loop_async( # noqa: PLR0912, PLR0915 topic_model_type: str | None = None, ) -> AsyncGenerator[dict | HFDataset, None]: """Run the main generation loop yielding progress events.""" - # Verify topic paths cover all expected samples + # Verify topic paths cover all expected samples (only when a topic model is used) expected_prompts = num_steps * batch_size actual_paths = len(topic_paths) if topic_paths else 0 - if actual_paths < expected_prompts: + if topic_paths and actual_paths < expected_prompts: logger.warning( "Topic paths (%d) < expected samples (%d). Steps beyond path %d will produce 0 samples.", actual_paths, diff --git a/pyproject.toml b/pyproject.toml index a00b860b..e7575273 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,8 @@ dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", "pytest-mock>=3.10.0", + "pytest-rerunfailures>=14.0", + "pytest-asyncio>=0.23.0", "pytest-httpx>=0.30.0", "requests-mock>=1.11.0", "ruff>=0.1.0", @@ -74,12 +76,14 @@ filterwarnings = [ "ignore::ImportWarning", "ignore::pydantic.PydanticDeprecatedSince20", ] +asyncio_mode = "auto" markers = [ "asyncio: mark tests that require an asyncio event loop", "openai: mark tests requiring OpenAI API access", "gemini: mark tests requiring Gemini API access", "huggingface: mark tests requiring HuggingFace Hub access", "spin: mark tests for Spin service (mocked HTTP)", + "flaky: mark tests that may fail due to external LLM service variability", ] [tool.ruff] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 10787d33..bf34c167 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,9 +1,18 @@ """Shared fixtures and markers for integration tests.""" +import logging import os import pytest +# Suppress httpx async client cleanup noise. +# LLMClient doesn't expose aclose(), so httpx connections linger after +# pytest-asyncio tears down the event loop, producing harmless +# "Event loop is closed" ERROR logs. +logging.getLogger("asyncio").addFilter( + lambda record: "Event loop is closed" not in record.getMessage() +) + # Skip markers for conditional test execution based on API key availability requires_openai = pytest.mark.skipif( not os.getenv("OPENAI_API_KEY"), @@ -40,3 +49,63 @@ def gemini_config(): "model_name": os.getenv("GEMINI_TEST_MODEL", "gemini-2.0-flash"), "temperature": 0.2, } + + +def assert_generation_result(result, generator, min_samples=1, context=""): + """Assert generation produced enough samples, with diagnostics on failure. + + Args: + result: HuggingFace Dataset returned by create_data. + generator: DataSetGenerator instance for reading failure diagnostics. + min_samples: Minimum expected sample count. + context: Test context for the failure message. + """ + if len(result) >= min_samples: + return + + lines = [ + f"Generation produced {len(result)} samples, expected >= {min_samples}.", + ] + if context: + lines.append(f"Context: {context}") + + lines.append(f"Total failures: {len(generator.failed_samples)}") + + failure_counts = {k: len(v) for k, v in generator.failure_analysis.items() if v} + if failure_counts: + lines.append("Failure breakdown:") + for ftype, count in sorted(failure_counts.items()): + lines.append(f" {ftype}: {count}") + + for i, f in enumerate(generator.failed_samples[:5], 1): + error_msg = f.get("error", str(f)) if isinstance(f, dict) else str(f) + lines.append(f" [{i}] {error_msg[:300]}") + + pytest.fail("\n".join(lines)) + + +def assert_topic_build_result(paths, model, min_paths=1, context=""): + """Assert topic model build produced enough paths, with diagnostics on failure. + + Args: + paths: List of paths from get_all_paths(). + model: Tree or Graph instance for reading failed_generations. + min_paths: Minimum expected path count. + context: Test context for the failure message. + """ + if len(paths) >= min_paths: + return + + lines = [ + f"Topic build produced {len(paths)} paths, expected >= {min_paths}.", + ] + if context: + lines.append(f"Context: {context}") + + failed_gens = getattr(model, "failed_generations", []) + if failed_gens: + lines.append(f"Failed generations ({len(failed_gens)}):") + for i, fg in enumerate(failed_gens[:5], 1): + lines.append(f" [{i}] {fg}") + + pytest.fail("\n".join(lines)) diff --git a/tests/integration/test_generator_integration.py b/tests/integration/test_generator_integration.py index e2531553..34226a4b 100644 --- a/tests/integration/test_generator_integration.py +++ b/tests/integration/test_generator_integration.py @@ -6,7 +6,12 @@ from deepfabric import DataSetGenerator, Graph -from .conftest import requires_gemini, requires_openai +from .conftest import ( + assert_generation_result, + assert_topic_build_result, + requires_gemini, + requires_openai, +) @pytest.fixture @@ -48,14 +53,18 @@ async def build_graph(): pass asyncio.run(build_graph()) + assert_topic_build_result( + graph.get_all_paths(), graph, min_paths=1, context="small_topic_graph fixture" + ) return graph +@requires_openai class TestDataSetGeneratorOpenAI: """Integration tests for DataSetGenerator with OpenAI provider.""" - @requires_openai @pytest.mark.openai + @pytest.mark.flaky(reruns=2, reruns_delay=5) def test_basic_generation(self, openai_generator): """Test basic dataset generation without topic model.""" result = openai_generator.create_data( @@ -63,12 +72,13 @@ def test_basic_generation(self, openai_generator): batch_size=2, ) - # Result should be a HuggingFace Dataset assert result is not None - assert len(result) >= 1 + assert_generation_result( + result, openai_generator, context="basic generation (no topic model)" + ) - @requires_openai @pytest.mark.openai + @pytest.mark.flaky(reruns=2, reruns_delay=5) def test_generation_with_topic_model(self, openai_generator, small_topic_graph): """Test dataset generation with a topic graph.""" result = openai_generator.create_data( @@ -78,26 +88,22 @@ def test_generation_with_topic_model(self, openai_generator, small_topic_graph): ) assert result is not None - assert len(result) >= 1 + assert_generation_result(result, openai_generator, context="generation with topic model") - @requires_openai @pytest.mark.openai - def test_async_generation(self, openai_generator): + @pytest.mark.flaky(reruns=2, reruns_delay=5) + async def test_async_generation(self, openai_generator): """Test async dataset generation.""" - - async def run_async(): - return await openai_generator.create_data_async( - num_steps=1, - batch_size=2, - ) - - result = asyncio.run(run_async()) + result = await openai_generator.create_data_async( + num_steps=1, + batch_size=2, + ) assert result is not None - assert len(result) >= 1 + assert_generation_result(result, openai_generator, context="async generation") - @requires_openai @pytest.mark.openai + @pytest.mark.flaky(reruns=2, reruns_delay=5) def test_generation_with_cot(self, openai_config): """Test generation with cot conversation type.""" generator = DataSetGenerator( @@ -115,40 +121,33 @@ def test_generation_with_cot(self, openai_config): ) assert result is not None - assert len(result) >= 1 + assert_generation_result(result, generator, context="cot generation") +@requires_gemini class TestDataSetGeneratorGemini: """Integration tests for DataSetGenerator with Gemini provider.""" - @requires_gemini @pytest.mark.gemini - def test_basic_generation(self, gemini_generator): + @pytest.mark.flaky(reruns=2, reruns_delay=5) + async def test_basic_generation(self, gemini_generator): """Test basic dataset generation with Gemini.""" - - async def run_async(): - return await gemini_generator.create_data_async( - num_steps=1, - batch_size=2, - ) - - result = asyncio.run(run_async()) + result = await gemini_generator.create_data_async( + num_steps=1, + batch_size=2, + ) assert result is not None - assert len(result) >= 1 + assert_generation_result(result, gemini_generator, context="Gemini basic generation") - @requires_gemini @pytest.mark.gemini - def test_generation_saves_to_file(self, tmp_path, gemini_generator): + @pytest.mark.flaky(reruns=2, reruns_delay=5) + async def test_generation_saves_to_file(self, tmp_path, gemini_generator): """Test that generated data can be saved.""" - - async def run_async(): - return await gemini_generator.create_data_async( - num_steps=1, - batch_size=2, - ) - - asyncio.run(run_async()) + await gemini_generator.create_data_async( + num_steps=1, + batch_size=2, + ) # Save dataset out_path = tmp_path / "dataset.jsonl" diff --git a/tests/integration/test_graph_integration.py b/tests/integration/test_graph_integration.py index 311a12b0..c1335286 100644 --- a/tests/integration/test_graph_integration.py +++ b/tests/integration/test_graph_integration.py @@ -1,21 +1,21 @@ """Integration tests for Graph with real API calls.""" -import asyncio import json import pytest # pyright: ignore[reportMissingImports] from deepfabric import Graph -from .conftest import requires_gemini, requires_openai +from .conftest import assert_topic_build_result, requires_gemini, requires_openai +@requires_openai class TestGraphOpenAI: """Integration tests for Graph with OpenAI provider.""" - @requires_openai @pytest.mark.openai - def test_graph_builds_basic(self, openai_config): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_graph_builds_basic(self, openai_config): """Test basic graph building with OpenAI.""" degree = 2 depth = 1 @@ -30,10 +30,7 @@ def test_graph_builds_basic(self, openai_config): depth=depth, ) - async def run_build(): - return [event async for event in graph.build_async()] - - events = asyncio.run(run_build()) + events = [event async for event in graph.build_async()] # Verify build completion completes = [e for e in events if e.get("event") == "build_complete"] @@ -41,12 +38,12 @@ async def run_build(): # Verify paths were built paths = graph.get_all_paths() - assert len(paths) >= 1 + assert_topic_build_result(paths, graph, min_paths=1, context="graph basic build (OpenAI)") assert all(p[0] == topic for p in paths) - @requires_openai @pytest.mark.openai - def test_graph_save_and_load_roundtrip(self, tmp_path, openai_config): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_graph_save_and_load_roundtrip(self, tmp_path, openai_config): """Test saving and loading a graph with OpenAI.""" degree = 2 depth = 1 @@ -61,11 +58,8 @@ def test_graph_save_and_load_roundtrip(self, tmp_path, openai_config): depth=depth, ) - async def build_graph(): - async for _ in graph.build_async(): - pass - - asyncio.run(build_graph()) + async for _ in graph.build_async(): + pass # Save to file out_path = tmp_path / "graph.json" @@ -90,12 +84,13 @@ async def build_graph(): assert new_graph.get_all_paths() == graph.get_all_paths() +@requires_gemini class TestGraphGemini: """Integration tests for Graph with Gemini provider.""" - @requires_gemini @pytest.mark.gemini - def test_graph_builds_basic(self, gemini_config): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_graph_builds_basic(self, gemini_config): """Test basic graph building with Gemini.""" degree = 2 depth = 1 @@ -110,10 +105,7 @@ def test_graph_builds_basic(self, gemini_config): depth=depth, ) - async def run_build(): - return [event async for event in graph.build_async()] - - events = asyncio.run(run_build()) + events = [event async for event in graph.build_async()] # Verify build completion completes = [e for e in events if e.get("event") == "build_complete"] @@ -121,12 +113,12 @@ async def run_build(): # Verify paths were built paths = graph.get_all_paths() - assert len(paths) >= 1 + assert_topic_build_result(paths, graph, min_paths=1, context="graph basic build (Gemini)") assert all(p[0] == topic for p in paths) - @requires_gemini @pytest.mark.gemini - def test_graph_no_cycles(self, gemini_config): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_graph_no_cycles(self, gemini_config): """Test that generated graphs have no cycles.""" graph = Graph( topic_prompt="Software Engineering", @@ -137,14 +129,11 @@ def test_graph_no_cycles(self, gemini_config): depth=2, ) - async def build_graph(): - async for _ in graph.build_async(): - pass - - asyncio.run(build_graph()) + async for _ in graph.build_async(): + pass # Verify no cycles assert not graph.has_cycle() # Verify we got paths paths = graph.get_all_paths() - assert len(paths) >= 1 + assert_topic_build_result(paths, graph, min_paths=1, context="graph no cycles (Gemini)") diff --git a/tests/integration/test_llm_client_integration.py b/tests/integration/test_llm_client_integration.py index 290f6ccc..e85d9a8b 100644 --- a/tests/integration/test_llm_client_integration.py +++ b/tests/integration/test_llm_client_integration.py @@ -1,7 +1,5 @@ """Integration tests for LLMClient with real API calls.""" -import asyncio - import pytest # pyright: ignore[reportMissingImports] from deepfabric.llm.client import LLMClient @@ -28,114 +26,118 @@ def gemini_client(gemini_config): ) +@requires_openai class TestLLMClientOpenAI: """Integration tests for LLMClient with OpenAI provider.""" - @requires_openai @pytest.mark.openai + @pytest.mark.flaky(reruns=1, reruns_delay=3) def test_basic_structured_output(self, openai_client): """Test basic structured output generation with OpenAI.""" prompt = "Generate a short greeting conversation between two people." result = openai_client.generate(prompt, ChatTranscript) - assert isinstance(result, ChatTranscript) - assert len(result.messages) >= 1 + assert isinstance(result, ChatTranscript), ( + f"Expected ChatTranscript, got {type(result).__name__}: {result}" + ) + assert len(result.messages) >= 1, ( + f"Expected >= 1 messages, got {len(result.messages)}: {result}" + ) assert all(isinstance(m, ChatMessage) for m in result.messages) - @requires_openai @pytest.mark.openai - def test_async_structured_output(self, openai_client): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_async_structured_output(self, openai_client): """Test async structured output generation with OpenAI.""" - - async def run_async(): - prompt = "List 3 subtopics about machine learning." - return await openai_client.generate_async(prompt, TopicList) - - result = asyncio.run(run_async()) - - assert isinstance(result, TopicList) - assert len(result.subtopics) >= 1 + prompt = "List 3 subtopics about machine learning." + result = await openai_client.generate_async(prompt, TopicList) + + assert isinstance(result, TopicList), ( + f"Expected TopicList, got {type(result).__name__}: {result}" + ) + assert len(result.subtopics) >= 1, ( + f"Expected >= 1 subtopics, got {len(result.subtopics)}: {result}" + ) assert all(isinstance(s, str) for s in result.subtopics) - @requires_openai @pytest.mark.openai - def test_async_streaming(self, openai_client): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_async_streaming(self, openai_client): """Test async streaming generation with OpenAI.""" + prompt = "Generate a brief Q&A about Python programming." + chunks = [] + final_result = None - async def run_stream(): - prompt = "Generate a brief Q&A about Python programming." - chunks = [] - final_result = None - - # generate_async_stream yields tuples: (chunk, None) or (None, result) - async for chunk, result in openai_client.generate_async_stream(prompt, ChatTranscript): - if chunk is not None: - chunks.append(chunk) - if result is not None: - final_result = result - - return chunks, final_result - - chunks, result = asyncio.run(run_stream()) + # generate_async_stream yields tuples: (chunk, None) or (None, result) + async for chunk, result in openai_client.generate_async_stream(prompt, ChatTranscript): + if chunk is not None: + chunks.append(chunk) + if result is not None: + final_result = result # Should have received streaming chunks - assert len(chunks) >= 1 + assert len(chunks) >= 1, f"Expected streaming chunks, got {len(chunks)}" # Final result should be valid - assert isinstance(result, ChatTranscript) - assert len(result.messages) >= 1 + assert isinstance(final_result, ChatTranscript), ( + f"Expected ChatTranscript, got {type(final_result).__name__}: {final_result}" + ) + assert len(final_result.messages) >= 1, ( + f"Expected >= 1 messages, got {len(final_result.messages)}: {final_result}" + ) +@requires_gemini class TestLLMClientGemini: """Integration tests for LLMClient with Gemini provider. Note: Gemini only supports async generation, so all tests use generate_async(). """ - @requires_gemini @pytest.mark.gemini - def test_basic_structured_output(self, gemini_client): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_basic_structured_output(self, gemini_client): """Test basic structured output generation with Gemini.""" - - async def run_async(): - prompt = "Generate a short greeting conversation between two people." - return await gemini_client.generate_async(prompt, ChatTranscript) - - result = asyncio.run(run_async()) - - assert isinstance(result, ChatTranscript) - assert len(result.messages) >= 1 + prompt = "Generate a short greeting conversation between two people." + result = await gemini_client.generate_async(prompt, ChatTranscript) + + assert isinstance(result, ChatTranscript), ( + f"Expected ChatTranscript, got {type(result).__name__}: {result}" + ) + assert len(result.messages) >= 1, ( + f"Expected >= 1 messages, got {len(result.messages)}: {result}" + ) assert all(isinstance(m, ChatMessage) for m in result.messages) - @requires_gemini @pytest.mark.gemini - def test_async_topic_list(self, gemini_client): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_async_topic_list(self, gemini_client): """Test async structured output generation with Gemini.""" - - async def run_async(): - prompt = "List 3 subtopics about data science." - return await gemini_client.generate_async(prompt, TopicList) - - result = asyncio.run(run_async()) - - assert isinstance(result, TopicList) - assert len(result.subtopics) >= 1 + prompt = "List 3 subtopics about data science." + result = await gemini_client.generate_async(prompt, TopicList) + + assert isinstance(result, TopicList), ( + f"Expected TopicList, got {type(result).__name__}: {result}" + ) + assert len(result.subtopics) >= 1, ( + f"Expected >= 1 subtopics, got {len(result.subtopics)}: {result}" + ) assert all(isinstance(s, str) for s in result.subtopics) - @requires_gemini @pytest.mark.gemini - def test_gemini_schema_handling(self, gemini_client): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_gemini_schema_handling(self, gemini_client): """Test that Gemini correctly handles schema conversion. Gemini has specific requirements around JSON schemas (no additionalProperties, specific array constraints). This test verifies the schema conversion works. """ - - async def run_async(): - # TopicList has min_length constraint which tests array handling - prompt = "List exactly 2 subtopics about cloud computing." - return await gemini_client.generate_async(prompt, TopicList) - - result = asyncio.run(run_async()) - - assert isinstance(result, TopicList) - assert len(result.subtopics) >= 2 # noqa: PLR2004 + # TopicList has min_length constraint which tests array handling + prompt = "List exactly 2 subtopics about cloud computing." + result = await gemini_client.generate_async(prompt, TopicList) + + assert isinstance(result, TopicList), ( + f"Expected TopicList, got {type(result).__name__}: {result}" + ) + assert len(result.subtopics) >= 2, ( # noqa: PLR2004 + f"Expected >= 2 subtopics, got {len(result.subtopics)}: {result}" + ) diff --git a/tests/integration/test_tree_integration.py b/tests/integration/test_tree_integration.py index e6e19ae3..a329578d 100644 --- a/tests/integration/test_tree_integration.py +++ b/tests/integration/test_tree_integration.py @@ -1,4 +1,3 @@ -import asyncio import json import pytest # pyright: ignore[reportMissingImports] @@ -6,19 +5,15 @@ from deepfabric import Tree, topic_manager from deepfabric.utils import read_topic_tree_from_jsonl -from .conftest import requires_openai +from .conftest import assert_topic_build_result, requires_openai +@requires_openai class TestTreeIntegration: """Integration tests for the Tree class, requiring real LLM calls.""" - # Centralize test configuration - TEST_PROVIDER = "openai" - TEST_MODEL = "gpt-4o-mini" - TEST_TEMPERATURE = 0.2 - @pytest.fixture - def tree_builder(self): + def tree_builder(self, openai_config): """Fixture to provide a factory for creating Tree instances.""" def _builder( @@ -33,9 +28,9 @@ def _builder( topic_system_prompt=system_prompt, degree=degree, depth=depth, - provider=self.TEST_PROVIDER, - model_name=self.TEST_MODEL, - temperature=self.TEST_TEMPERATURE, + provider=openai_config["provider"], + model_name=openai_config["model_name"], + temperature=openai_config["temperature"], ) return _builder @@ -44,15 +39,15 @@ async def _run_build(self, tree: Tree): """Helper to run the async build and collect all events.""" return [event async for event in tree.build_async()] - @requires_openai @pytest.mark.openai - def test_tree_builds_basic(self, tree_builder): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_tree_builds_basic(self, tree_builder): degree = 2 depth = 1 topic = "AI Ethics" tree = tree_builder(topic_prompt=topic, degree=degree, depth=depth) - events = asyncio.run(self._run_build(tree)) + events = await self._run_build(tree) # Verify build completion event completes = [e for e in events if e.get("event") == "build_complete"] @@ -61,39 +56,41 @@ def test_tree_builds_basic(self, tree_builder): # Verify paths were built as expected paths = tree.get_all_paths() - assert len(paths) == degree**depth + assert_topic_build_result(paths, tree, min_paths=degree**depth, context="tree basic build") assert all(len(p) == depth + 1 for p in paths) assert all(p[0] == topic for p in paths) - @requires_openai @pytest.mark.openai - def test_tree_builds_with_deeper_recursion(self, tree_builder): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_tree_builds_with_deeper_recursion(self, tree_builder): # Single branch but deeper depth to exercise recursion with real LLM degree = 1 depth = 2 # two LLM calls topic = "Quantum Computing" tree = tree_builder(topic_prompt=topic, degree=degree, depth=depth) - events = asyncio.run(self._run_build(tree)) + events = await self._run_build(tree) last_event = events[-1] if events else None assert last_event is not None assert last_event.get("event") == "build_complete" assert last_event["total_paths"] == degree**depth paths = tree.get_all_paths() - assert len(paths) == degree**depth + assert_topic_build_result( + paths, tree, min_paths=degree**depth, context="tree deeper recursion" + ) assert all(len(p) == depth + 1 for p in paths) - @requires_openai @pytest.mark.openai - def test_tree_generates_and_saves_jsonl_structure(self, tmp_path, tree_builder): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_tree_generates_and_saves_jsonl_structure(self, tmp_path, tree_builder): # Generate a small tree, save to JSONL, and validate file structure degree = 2 depth = 1 root_topic = "Software Engineering" tree = tree_builder(topic_prompt=root_topic, degree=degree, depth=depth) - asyncio.run(self._run_build(tree)) + await self._run_build(tree) # Save to a temp file out_path = tmp_path / "topics.jsonl" @@ -111,16 +108,16 @@ def test_tree_generates_and_saves_jsonl_structure(self, tmp_path, tree_builder): assert all(obj["path"][0] == root_topic for obj in items) assert all(list(obj.keys()) == ["path"] for obj in items) - @requires_openai @pytest.mark.openai - def test_tree_save_and_load_round_trip(self, tmp_path, tree_builder): + @pytest.mark.flaky(reruns=1, reruns_delay=3) + async def test_tree_save_and_load_round_trip(self, tmp_path, tree_builder): degree = 2 depth = 1 root = "RoundTrip Root" # Build original tree tree = tree_builder(topic_prompt=root, degree=degree, depth=depth) - asyncio.run(self._run_build(tree)) + await self._run_build(tree) # Save to JSONL out = tmp_path / "topics.jsonl" @@ -138,8 +135,8 @@ def test_tree_save_and_load_round_trip(self, tmp_path, tree_builder): assert all(len(p) == depth + 1 for p in rebuilt.get_all_paths()) assert all(p[0] == root for p in rebuilt.get_all_paths()) - @requires_openai @pytest.mark.openai + @pytest.mark.flaky(reruns=1, reruns_delay=3) def test_tree_tui_streaming_and_events(self, monkeypatch, tree_builder): # Use topic_manager + TUI to drive build and capture streaming chunks @@ -149,6 +146,7 @@ def __init__(self): self.finished = [] self.failures = 0 self.chunks = [] + self.progress_advances = 0 # Methods used by topic_manager def start_building( @@ -159,6 +157,12 @@ def start_building( def add_failure(self) -> None: self.failures += 1 + def advance_simple_progress(self) -> None: + self.progress_advances += 1 + + def stop_live(self) -> None: + pass + def finish_building(self, total_paths: int, failed_generations: int) -> None: self.finished.append((total_paths, failed_generations)) @@ -183,6 +187,7 @@ def on_stream_chunk(self, source: str, chunk: str, metadata: dict) -> None: # n assert len(fake_tui.finished) == 1 assert fake_tui.finished[0][0] == degree**depth assert fake_tui.failures == 0 + assert fake_tui.progress_advances == degree**depth # Streaming should have produced at least one chunk assert isinstance(fake_tui.chunks, list) assert len(fake_tui.chunks) >= 1 diff --git a/uv.lock b/uv.lock index d47dbede..572de4d6 100644 --- a/uv.lock +++ b/uv.lock @@ -214,6 +214,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, ] +[[package]] +name = "backports-asyncio-runner" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, +] + [[package]] name = "backrefs" version = "5.9" @@ -483,7 +492,7 @@ wheels = [ [[package]] name = "deepfabric" -version = "4.10.0" +version = "4.10.1" source = { editable = "." } dependencies = [ { name = "anthropic" }, @@ -514,9 +523,11 @@ dev = [ { name = "bandit" }, { name = "mermaid-py" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-httpx" }, { name = "pytest-mock" }, + { name = "pytest-rerunfailures" }, { name = "requests-mock" }, { name = "ruff" }, ] @@ -556,9 +567,11 @@ requires-dist = [ { name = "protobuf", specifier = ">=3.20.0" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" }, { name = "pytest-httpx", marker = "extra == 'dev'", specifier = ">=0.30.0" }, { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.10.0" }, + { name = "pytest-rerunfailures", marker = "extra == 'dev'", specifier = ">=14.0" }, { name = "pyyaml", specifier = ">=6.0.1" }, { name = "requests-mock", marker = "extra == 'dev'", specifier = ">=1.11.0" }, { name = "rich", specifier = ">=13.0.0" }, @@ -2435,6 +2448,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, +] + [[package]] name = "pytest-cov" version = "6.2.1" @@ -2474,6 +2501,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/05/77b60e520511c53d1c1ca75f1930c7dd8e971d0c4379b7f4b3f9644685ba/pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0", size = 9923, upload-time = "2025-05-26T13:58:43.487Z" }, ] +[[package]] +name = "pytest-rerunfailures" +version = "16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/04/71e9520551fc8fe2cf5c1a1842e4e600265b0815f2016b7c27ec85688682/pytest_rerunfailures-16.1.tar.gz", hash = "sha256:c38b266db8a808953ebd71ac25c381cb1981a78ff9340a14bcb9f1b9bff1899e", size = 30889, upload-time = "2025-10-10T07:06:01.238Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/54/60eabb34445e3db3d3d874dc1dfa72751bfec3265bd611cb13c8b290adea/pytest_rerunfailures-16.1-py3-none-any.whl", hash = "sha256:5d11b12c0ca9a1665b5054052fcc1084f8deadd9328962745ef6b04e26382e86", size = 14093, upload-time = "2025-10-10T07:06:00.019Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"