diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000..977667d0 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,172 @@ +name: Integration Tests + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + workflow_dispatch: # Allow manual trigger + +permissions: + models: read + contents: read + +jobs: + integration-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 # Prevent hanging tests + + strategy: + matrix: + python-version: ['3.11'] + fail-fast: false # Continue testing other versions if one fails + + steps: + - uses: actions/checkout@v5 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install dependencies + run: | + uv sync --all-extras + + - name: Verify GitHub Models access + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + curl -s "https://models.github.ai/inference/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $GITHUB_TOKEN" \ + -d '{ + "messages": [{"role": "user", "content": "Hello"}], + "model": "openai/gpt-4o-mini", + "max_tokens": 10 + }' || echo "GitHub Models API test failed - tests may be skipped" + + - name: Run tree integration tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEEPFABRIC_TESTING: "true" + ANONYMIZED_TELEMETRY: "False" + run: | + uv run pytest tests/integration/test_tree_integration.py -v \ + --tb=short \ + --durations=10 + + - name: Run graph integration tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEEPFABRIC_TESTING: "true" + ANONYMIZED_TELEMETRY: "False" + run: | + uv run pytest tests/integration/test_graph_integration.py -v \ + --tb=short \ + --durations=10 + + - name: Run generator integration tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEEPFABRIC_TESTING: "true" + ANONYMIZED_TELEMETRY: "False" + run: | + uv run pytest tests/integration/test_generator_integration.py -v \ + --tb=short \ + --durations=10 + + - name: Run pipeline integration tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEEPFABRIC_TESTING: "true" + ANONYMIZED_TELEMETRY: "False" + run: | + uv run pytest tests/integration/test_pipeline_integration.py -v \ + --tb=short \ + --durations=10 + + - name: Run CLI integration tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEEPFABRIC_TESTING: "true" + ANONYMIZED_TELEMETRY: "False" + run: | + uv run pytest tests/integration/test_cli_integration.py -v \ + --tb=short \ + --durations=10 + + - name: Generate integration test report + if: always() + run: | + echo "## Integration Test Summary" > integration_report.md + echo "- **Python Version**: ${{ matrix.python-version }}" >> integration_report.md + echo "- **Timestamp**: $(date)" >> integration_report.md + echo "- **Branch**: ${{ github.ref }}" >> integration_report.md + echo "- **Commit**: ${{ github.sha }}" >> integration_report.md + + # Count test files + echo "- **Test Files**: $(find tests/integration -name 'test_*.py' | wc -l)" >> integration_report.md + + # Show which tests were run + echo "" >> integration_report.md + echo "### Test Coverage" >> integration_report.md + echo "- Tree Generation Tests" >> integration_report.md + echo "- Graph Generation Tests" >> integration_report.md + echo "- Dataset Generator Tests" >> integration_report.md + echo "- End-to-End Pipeline Tests" >> integration_report.md + echo "- CLI Integration Tests" >> integration_report.md + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-results-${{ matrix.python-version }} + path: | + integration_report.md + .pytest_cache/ + retention-days: 7 + + - name: Check for test failures + if: failure() + run: | + echo "❌ Integration tests failed!" + echo "Please check the logs above for details." + echo "Common issues:" + echo "- GitHub Models API rate limits" + echo "- Network connectivity" + echo "- Test timeouts" + exit 1 + + integration-test-summary: + needs: integration-tests + runs-on: ubuntu-latest + if: always() + + steps: + - name: Integration Test Summary + run: | + echo "## Integration Test Results" + echo "Status: ${{ needs.integration-tests.result }}" + + if [ "${{ needs.integration-tests.result }}" == "success" ]; then + echo "✅ All integration tests passed!" + echo "The following components have been validated:" + echo "- GitHub Models provider integration" + echo "- Tree and Graph generation" + echo "- Dataset generation pipeline" + echo "- CLI functionality" + echo "- End-to-end workflows" + else + echo "❌ Integration tests failed or were skipped" + echo "This may be due to:" + echo "- Missing GITHUB_TOKEN (tests are skipped automatically)" + echo "- GitHub Models API issues" + echo "- Test timeouts or network issues" + fi \ No newline at end of file diff --git a/deepfabric/llm/client.py b/deepfabric/llm/client.py index fd3e361a..46443897 100644 --- a/deepfabric/llm/client.py +++ b/deepfabric/llm/client.py @@ -36,7 +36,7 @@ def make_outlines_model(provider: str, model_name: str, **kwargs) -> Any: """Create an Outlines model for the specified provider and model. Args: - provider: Provider name (openai, anthropic, gemini, ollama) + provider: Provider name (openai, anthropic, gemini, ollama, github) model_name: Model identifier **kwargs: Additional parameters passed to the client @@ -79,10 +79,28 @@ def make_outlines_model(provider: str, model_name: str, **kwargs) -> Any: if provider == "ollama": # Use OpenAI-compatible endpoint for Ollama base_url = kwargs.get("base_url", "http://localhost:11434/v1") + client_kwargs = kwargs.copy() + client_kwargs.pop("base_url", None) client = openai.OpenAI( base_url=base_url, api_key="ollama", # Dummy key for Ollama - **{k: v for k, v in kwargs.items() if k != "base_url"}, + **client_kwargs, + ) + return outlines.from_openai(client, model_name) + + if provider == "github": + # Use GitHub Models API with OpenAI-compatible interface + api_key = os.getenv("GITHUB_TOKEN") or os.getenv("MODELS_TOKEN") + if not api_key: + _raise_api_key_error("GITHUB_TOKEN or MODELS_TOKEN") + + base_url = kwargs.get("base_url", "https://models.github.ai/inference") + client_kwargs = kwargs.copy() + client_kwargs.pop("base_url", None) + client = openai.OpenAI( + base_url=base_url, + api_key=api_key, + **client_kwargs, ) return outlines.from_openai(client, model_name) diff --git a/examples/README.md b/examples/README.md index fed09d68..2aa4ff38 100644 --- a/examples/README.md +++ b/examples/README.md @@ -140,7 +140,7 @@ dataset.save("output.jsonl") config = DeepFabricConfig.from_yaml("config.yaml") # Use configuration parameters -tree_params = config.get_tree_params() +tree_params = config.get_topic_tree_params() engine_params = config.get_engine_params() dataset_config = config.get_dataset_config() ``` diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..7a39dfc6 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests for DeepFabric.""" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..282b6703 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,56 @@ +import os +import tempfile + +from pathlib import Path + +import pytest + + +@pytest.fixture +def fixtures_dir(): + """Return path to the fixtures directory.""" + return Path(__file__).parent / "fixtures" + + +@pytest.fixture +def test_config_path(fixtures_dir): + """Return path to test configuration file.""" + return fixtures_dir / "test_config.yaml" + + +@pytest.fixture +def graph_config_path(fixtures_dir): + """Return path to graph configuration file.""" + return fixtures_dir / "graph_config.yaml" + + +@pytest.fixture +def sample_tree_path(fixtures_dir): + """Return path to sample tree file.""" + return fixtures_dir / "sample_tree.jsonl" + + +@pytest.fixture +def temp_output_dir(): + """Create a temporary directory for test outputs.""" + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + + +@pytest.fixture(autouse=True) +def github_token_check(): + """Skip integration tests if GITHUB_TOKEN or MODELS_TOKEN is not available.""" + if not (os.environ.get("GITHUB_TOKEN") or os.environ.get("MODELS_TOKEN")): + pytest.skip("GITHUB_TOKEN or MODELS_TOKEN not available - skipping integration test") + + +@pytest.fixture +def minimal_test_config(): + """Provide minimal configuration for fast testing.""" + return { + "provider": "github", + "model_name": "openai/gpt-4o-mini", + "temperature": 0.1, + "max_retries": 1, + "request_timeout": 30, + } diff --git a/tests/integration/fixtures/graph_config.yaml b/tests/integration/fixtures/graph_config.yaml new file mode 100644 index 00000000..d0609b61 --- /dev/null +++ b/tests/integration/fixtures/graph_config.yaml @@ -0,0 +1,31 @@ +dataset_system_prompt: "You are a helpful AI assistant focused on providing clear, accurate, and concise responses." + +topic_graph: + args: + topic_prompt: "Basic Python programming concepts" + topic_system_prompt: "You are an expert Python educator creating educational content." + degree: 2 + depth: 2 + provider: "github" + model_name: "openai/gpt-4o-mini" + temperature: 0.1 + save_as: "test_graph.json" + +data_engine: + instructions: "Create simple Python programming questions and answers suitable for beginners." + args: + generation_system_prompt: "You are a Python programming tutor. Create clear, educational content with practical examples." + provider: "github" + model_name: "openai/gpt-4o-mini" + temperature: 0.1 + max_retries: 2 + default_batch_size: 2 + default_num_examples: 1 + request_timeout: 30 + sys_msg: true + +dataset: + creation: + num_steps: 5 + batch_size: 2 + save_as: "test_dataset.jsonl" diff --git a/tests/integration/fixtures/test_config.yaml b/tests/integration/fixtures/test_config.yaml new file mode 100644 index 00000000..ea265e2c --- /dev/null +++ b/tests/integration/fixtures/test_config.yaml @@ -0,0 +1,31 @@ +dataset_system_prompt: "You are a helpful AI assistant focused on providing clear, accurate, and concise responses." + +topic_tree: + args: + topic_prompt: "Basic Python programming concepts" + topic_system_prompt: "You are an expert Python educator creating educational content." + degree: 2 + depth: 2 + provider: "github" + model_name: "openai/gpt-4o-mini" + temperature: 0.1 + save_as: "test_tree.jsonl" + +data_engine: + instructions: "Create simple Python programming questions and answers suitable for beginners." + args: + generation_system_prompt: "You are a Python programming tutor. Create clear, educational content with practical examples." + provider: "github" + model_name: "openai/gpt-4o-mini" + temperature: 0.1 + max_retries: 2 + default_batch_size: 2 + default_num_examples: 1 + request_timeout: 30 + sys_msg: true + +dataset: + creation: + num_steps: 5 + batch_size: 2 + save_as: "test_dataset.jsonl" \ No newline at end of file diff --git a/tests/integration/test_cli_integration.py b/tests/integration/test_cli_integration.py new file mode 100644 index 00000000..a067d1c5 --- /dev/null +++ b/tests/integration/test_cli_integration.py @@ -0,0 +1,344 @@ +""" +Integration tests for CLI functionality. +""" + +import json +import os +import subprocess +import tempfile + +from pathlib import Path + +import pytest +import yaml + + +def run_cli_command(*args, **kwargs): + """Helper function to run deepfabric CLI commands with common settings. + + Args: + *args: Command arguments (after 'deepfabric') + **kwargs: Additional arguments for subprocess.run + + Returns: + subprocess.CompletedProcess result + """ + cmd = ["uv", "run", "deepfabric"] + list(args) + + # Default settings + defaults = { + "check": False, + "capture_output": True, + "text": True, + "cwd": Path(__file__).parent.parent.parent, + } + + # Override with any provided kwargs + defaults.update(kwargs) + + return subprocess.run(cmd, **defaults) # noqa: PLW1510, S603 + + +class TestCLIIntegration: + """Integration tests for CLI commands.""" + + def test_cli_generate_with_config(self, test_config_path, temp_output_dir): + """Test CLI generate command with config file.""" + # Create a temporary config with updated paths + temp_config = temp_output_dir / "cli_test_config.yaml" + + # Load original config and update paths using YAML parser + with open(test_config_path) as f: + config_data = yaml.safe_load(f) + + # Update paths in config data structure + config_data["topic_tree"]["save_as"] = str(temp_output_dir / "cli_tree.jsonl") + config_data["dataset"]["save_as"] = str(temp_output_dir / "cli_dataset.jsonl") + + # Write updated config + with open(temp_config, "w") as f: + yaml.safe_dump(config_data, f, default_flow_style=False) + + # Run CLI command + result = run_cli_command("generate", str(temp_config)) + + # Check command succeeded + assert result.returncode == 0, f"CLI failed with stderr: {result.stderr}" + + # Verify outputs were created + tree_file = temp_output_dir / "cli_tree.jsonl" + dataset_file = temp_output_dir / "cli_dataset.jsonl" + + assert tree_file.exists(), "Tree file was not created" + assert dataset_file.exists(), "Dataset file was not created" + + # Verify file contents + with open(dataset_file) as f: + lines = f.readlines() + assert len(lines) > 0, "Dataset file is empty" + + # Verify JSONL format + for line in lines: + data = json.loads(line) + assert "messages" in data + + def test_cli_generate_with_overrides(self, test_config_path, temp_output_dir): + """Test CLI generate with parameter overrides.""" + tree_path = temp_output_dir / "override_tree.jsonl" + dataset_path = temp_output_dir / "override_dataset.jsonl" + + result = run_cli_command( + "generate", + str(test_config_path), + "--save-tree", + str(tree_path), + "--dataset-save-as", + str(dataset_path), + "--num-steps", + "2", + "--batch-size", + "1", + "--temperature", + "0.1", + ) + + assert result.returncode == 0, f"CLI failed: {result.stderr}" + assert tree_path.exists() + assert dataset_path.exists() + + def test_cli_validate_config(self, test_config_path): + """Test CLI config validation.""" + result = run_cli_command("validate", str(test_config_path)) + + assert result.returncode == 0, f"Validation failed: {result.stderr}" + assert "valid" in result.stdout.lower() + + def test_cli_info_command(self): + """Test CLI info command.""" + result = run_cli_command("info") + + assert result.returncode == 0 + assert "DeepFabric" in result.stdout + assert "generate" in result.stdout # Should list commands + + def test_cli_tree_mode(self, temp_output_dir): + """Test CLI with tree mode explicitly.""" + + tree_path = temp_output_dir / "cli_tree_mode.jsonl" + dataset_path = temp_output_dir / "cli_tree_dataset.jsonl" + + result = run_cli_command( + "generate", + "--mode", + "tree", + "--topic-prompt", + "Simple programming concepts", + "--generation-system-prompt", + "You are a programming tutor.", + "--provider", + "github", + "--model", + "openai/gpt-4o-mini", + "--degree", + "2", + "--depth", + "1", + "--num-steps", + "2", + "--batch-size", + "1", + "--temperature", + "0.1", + "--save-tree", + str(tree_path), + "--dataset-save-as", + str(dataset_path), + ) + + assert result.returncode == 0, f"CLI tree mode failed: {result.stderr}" + assert tree_path.exists() + assert dataset_path.exists() + + def test_cli_graph_mode(self, temp_output_dir): + """Test CLI with graph mode.""" + + graph_path = temp_output_dir / "cli_graph.json" + dataset_path = temp_output_dir / "cli_graph_dataset.jsonl" + + result = run_cli_command( + "generate", + "--mode", + "graph", + "--topic-prompt", + "Data science fundamentals", + "--generation-system-prompt", + "You are a data science educator.", + "--provider", + "github", + "--model", + "openai/gpt-4o-mini", + "--degree", + "2", + "--depth", + "1", + "--num-steps", + "2", + "--batch-size", + "1", + "--temperature", + "0.1", + "--save-graph", + str(graph_path), + "--dataset-save-as", + str(dataset_path), + ) + + assert result.returncode == 0, f"CLI graph mode failed: {result.stderr}" + assert graph_path.exists() + assert dataset_path.exists() + + def test_cli_load_existing_tree(self, sample_tree_path, temp_output_dir): + """Test CLI loading existing tree file.""" + dataset_path = temp_output_dir / "loaded_tree_dataset.jsonl" + + result = run_cli_command( + "generate", + "--load-tree", + str(sample_tree_path), + "--generation-system-prompt", + "You are an educational assistant.", + "--provider", + "github", + "--model", + "openai/gpt-4o-mini", + "--temperature", + "0.1", + "--num-steps", + "2", + "--batch-size", + "1", + "--dataset-save-as", + str(dataset_path), + ) + + # Skip if no token available + if result.returncode != 0 and "TOKEN" in result.stderr: + pytest.skip("GitHub token not available") + + assert result.returncode == 0, f"CLI load tree failed: {result.stderr}" + assert dataset_path.exists() + + def test_cli_error_handling(self): + """Test CLI error handling for invalid inputs.""" + # Test with non-existent config file + result = run_cli_command("generate", "nonexistent.yaml") + + assert result.returncode != 0 + assert "not found" in result.stderr.lower() or "does not exist" in result.stderr.lower() + + # Test validation with invalid config + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write("invalid: yaml: content:") + invalid_config = f.name + + try: + result = run_cli_command("validate", invalid_config) + + assert result.returncode != 0 + finally: + os.unlink(invalid_config) + + def test_cli_visualize_command(self, temp_output_dir): + """Test CLI visualize command.""" + + # First create a graph + graph_path = temp_output_dir / "viz_graph.json" + + result = run_cli_command( + "generate", + "--mode", + "graph", + "--topic-prompt", + "Visualization test", + "--provider", + "github", + "--model", + "openai/gpt-4o-mini", + "--degree", + "2", + "--depth", + "1", + "--temperature", + "0.1", + "--save-graph", + str(graph_path), + "--dataset-save-as", + str(temp_output_dir / "viz_dataset.jsonl"), + "--num-steps", + "1", + "--batch-size", + "1", + ) + + if result.returncode != 0: + pytest.skip(f"Graph generation failed: {result.stderr}") + + # Now visualize it + viz_output = temp_output_dir / "test_viz" + + result = run_cli_command("visualize", str(graph_path), "--output", str(viz_output)) + + assert result.returncode == 0, f"Visualize failed: {result.stderr}" + + # Check SVG file was created + svg_file = Path(f"{viz_output}.svg") + assert svg_file.exists(), "SVG file was not created" + + def test_cli_help_commands(self): + """Test CLI help functionality.""" + # Test main help + result = run_cli_command("--help") + + assert result.returncode == 0 + assert "generate" in result.stdout + assert "validate" in result.stdout + + # Test generate help + result = run_cli_command("generate", "--help") + + assert result.returncode == 0 + assert "--provider" in result.stdout + assert "--model" in result.stdout + + def test_cli_with_different_providers_fallback(self, temp_output_dir): + """Test CLI gracefully handles missing provider configurations.""" + # Test with missing provider (should use config defaults) + result = run_cli_command( + "generate", + "--topic-prompt", + "Fallback test", + "--generation-system-prompt", + "Test system prompt", + "--provider", + "nonexistent_provider", + "--model", + "test/model", + "--degree", + "2", + "--depth", + "1", + "--num-steps", + "1", + "--batch-size", + "1", + "--dataset-save-as", + str(temp_output_dir / "fallback_dataset.jsonl"), + ) + + # Should fail with appropriate error message + assert result.returncode != 0 + assert ( + "unsupported" in result.stderr.lower() + or "error" in result.stderr.lower() + or "provider" in result.stderr.lower() + ) diff --git a/tests/integration/test_generator_integration.py b/tests/integration/test_generator_integration.py new file mode 100644 index 00000000..cff3c3e2 --- /dev/null +++ b/tests/integration/test_generator_integration.py @@ -0,0 +1,274 @@ +""" +Integration tests for DataSetGenerator. +""" + +import pytest + +from deepfabric import Dataset, DataSetGenerator, Tree +from deepfabric.exceptions import DataSetGeneratorError + +MESSAGE_LIMIT = 2 + + +class TestDataSetGeneratorIntegration: + """Integration tests for DataSetGenerator functionality.""" + + def test_generator_creation_basic(self, minimal_test_config): + """Test basic generator creation with minimal parameters.""" + generator = DataSetGenerator( + instructions="Create simple Python Q&A pairs", + generation_system_prompt="You are a Python tutor.", + **minimal_test_config, + ) + + assert generator.config.instructions == "Create simple Python Q&A pairs" + assert generator.config.generation_system_prompt == "You are a Python tutor." + assert isinstance(generator.dataset, Dataset) + + def test_generator_with_tree(self, minimal_test_config): + """Test generator creating data from a tree.""" + # Create a simple tree + tree = Tree( + topic_prompt="Python basics", + topic_system_prompt="You are a Python educator.", + degree=2, + depth=1, # Keep small + **minimal_test_config, + ) + + # Build the tree + list(tree.build()) + + # Create generator + generator = DataSetGenerator( + instructions="Create beginner Python questions and answers", + generation_system_prompt="You are a Python programming tutor.", + **minimal_test_config, + ) + + # Generate small dataset + dataset = generator.create_data( + num_steps=2, + batch_size=1, + topic_model=tree, + ) + + # Verify dataset structure + assert isinstance(dataset, Dataset) + assert len(dataset.samples) > 0 + + # Verify sample format + for sample in dataset.samples: + assert "messages" in sample + messages = sample["messages"] + assert len(messages) >= MESSAGE_LIMIT # Should have user and assistant + assert any(msg["role"] == "user" for msg in messages) + assert any(msg["role"] == "assistant" for msg in messages) + + def test_generator_conversation_types(self, minimal_test_config): + """Test different conversation types.""" + # Create minimal tree + tree = Tree( + topic_prompt="Math concepts", + degree=2, + depth=1, + **minimal_test_config, + ) + list(tree.build()) + + # Test different conversation types + conversation_types = ["basic", "cot_freetext", "cot_structured"] + + for conv_type in conversation_types: + generator = DataSetGenerator( + instructions="Create educational content", + generation_system_prompt="You are an educator.", + conversation_type=conv_type, + **minimal_test_config, + ) + + dataset = generator.create_data( + num_steps=1, + batch_size=1, + topic_model=tree, + ) + + assert len(dataset.samples) > 0 # type: ignore + + def test_generator_with_system_message(self, minimal_test_config): + """Test generator with sys_msg parameter.""" + tree = Tree( + topic_prompt="Science topics", + degree=2, + depth=1, + **minimal_test_config, + ) + list(tree.build()) + + # Test with system message + generator_with_sys = DataSetGenerator( + instructions="Create science Q&A", + generation_system_prompt="You are a science teacher.", + dataset_system_prompt="You are a helpful science tutor.", + sys_msg=True, + **minimal_test_config, + ) + + dataset_with_sys = generator_with_sys.create_data( + num_steps=1, + batch_size=1, + topic_model=tree, + ) + + # Verify system message is included + assert len(dataset_with_sys.samples) > 0 # type: ignore + sample = dataset_with_sys.samples[0] # type: ignore + messages = sample["messages"] + system_messages = [msg for msg in messages if msg["role"] == "system"] + assert len(system_messages) > 0 + + # Test without system message + generator_no_sys = DataSetGenerator( + instructions="Create science Q&A", + generation_system_prompt="You are a science teacher.", + sys_msg=False, + **minimal_test_config, + ) + + dataset_no_sys = generator_no_sys.create_data( + num_steps=1, + batch_size=1, + topic_model=tree, + ) + + # Verify no system message + sample = dataset_no_sys.samples[0] # type: ignore + messages = sample["messages"] + system_messages = [msg for msg in messages if msg["role"] == "system"] + assert len(system_messages) == 0 + + def test_generator_error_handling(self, minimal_test_config): + """Test generator error handling.""" + generator = DataSetGenerator( + instructions="Test", + generation_system_prompt="Test", + **minimal_test_config, + ) + + # Test with invalid num_steps + tree = Tree(topic_prompt="Test", degree=2, depth=1, **minimal_test_config) + list(tree.build()) + + with pytest.raises(DataSetGeneratorError): + generator.create_data(num_steps=0, batch_size=1, topic_model=tree) + + def test_generator_with_retries(self, minimal_test_config): + """Test generator retry mechanism.""" + config = minimal_test_config.copy() + config["max_retries"] = 2 + + generator = DataSetGenerator( + instructions="Create content", + generation_system_prompt="You are helpful.", + **config, + ) + + tree = Tree(topic_prompt="Simple topic", degree=2, depth=1, **minimal_test_config) + list(tree.build()) + + # Should handle retries gracefully + dataset = generator.create_data( + num_steps=1, + batch_size=1, + topic_model=tree, + ) + + assert isinstance(dataset, Dataset) + + def test_generator_with_github_provider(self): + """Test generator specifically with GitHub provider.""" + + # Create tree + tree = Tree( + topic_prompt="Programming fundamentals", + topic_system_prompt="You are a programming instructor.", + provider="github", + model_name="openai/gpt-4o-mini", + degree=2, + depth=1, + temperature=0.1, + ) + list(tree.build()) + + # Create generator + generator = DataSetGenerator( + instructions="Create programming Q&A pairs for beginners", + generation_system_prompt="You are a helpful programming tutor.", + provider="github", + model_name="openai/gpt-4o-mini", + temperature=0.1, + max_retries=1, + ) + + # Generate data + dataset = generator.create_data( + num_steps=1, + batch_size=1, + topic_model=tree, + ) + + assert len(dataset.samples) > 0 # type: ignore + + def test_generator_batch_processing(self, minimal_test_config): + """Test generator batch processing.""" + tree = Tree( + topic_prompt="General knowledge", + degree=2, + depth=1, + **minimal_test_config, + ) + list(tree.build()) + + generator = DataSetGenerator( + instructions="Create general knowledge questions", + generation_system_prompt="You are knowledgeable.", + **minimal_test_config, + ) + + # Test with batch size > 1 + dataset = generator.create_data( + num_steps=1, + batch_size=2, # Process 2 samples at once + topic_model=tree, + ) + + assert len(dataset.samples) >= MESSAGE_LIMIT # type: ignore + + def test_generator_dataset_validation(self, minimal_test_config): + """Test that generated datasets are properly validated.""" + tree = Tree( + topic_prompt="Test validation", + degree=2, + depth=1, + **minimal_test_config, + ) + list(tree.build()) + + generator = DataSetGenerator( + instructions="Create simple content", + generation_system_prompt="You create valid responses.", + **minimal_test_config, + ) + + dataset = generator.create_data( + num_steps=1, + batch_size=1, + topic_model=tree, + ) + + # Verify all samples pass validation + for sample in dataset.samples: # type: ignore + assert Dataset.validate_sample(sample) + + # Check for failed samples + assert len(generator.failed_samples) == 0 or generator.failed_samples is None diff --git a/tests/integration/test_graph_integration.py b/tests/integration/test_graph_integration.py new file mode 100644 index 00000000..d908dd2d --- /dev/null +++ b/tests/integration/test_graph_integration.py @@ -0,0 +1,221 @@ +""" +Integration tests for Graph generation. +""" + +import json + +import pytest + +from deepfabric import Graph + +MESSAGE_LIMIT = 2 + + +class TestGraphIntegration: + """Integration tests for Graph generation functionality.""" + + def test_graph_creation_basic(self, minimal_test_config): + """Test basic graph creation with minimal parameters.""" + graph = Graph( + topic_prompt="Basic Python concepts", + topic_system_prompt="You are a Python educator.", + degree=2, + depth=2, + **minimal_test_config, + ) + + assert graph.topic_prompt == "Basic Python concepts" + assert graph.degree == MESSAGE_LIMIT + assert graph.depth == MESSAGE_LIMIT + + def test_graph_build_small(self, minimal_test_config): + """Test building a small graph and validate structure.""" + graph = Graph( + topic_prompt="Python data structures", + topic_system_prompt="You are a Python expert creating educational topics about data structures.", + degree=2, + depth=2, + **minimal_test_config, + ) + + # Build the graph + events = list(graph.build()) + + # Check that we get events + assert len(events) > 0 + + # Check final event + final_event = events[-1] + assert final_event["event"] == "build_complete" + + # Graph should have nodes + assert graph.root is not None # type: ignore + assert len(graph.nodes) > 1 # Should have root + additional nodes + + def test_graph_save_load(self, minimal_test_config, temp_output_dir): + """Test graph persistence to JSON format.""" + graph = Graph( + topic_prompt="Python functions", + topic_system_prompt="You are a Python expert.", + degree=2, + depth=1, # Keep small for speed + **minimal_test_config, + ) + + # Build the graph + list(graph.build()) + + # Save to file + save_path = temp_output_dir / "test_graph.json" + graph.save(str(save_path)) + + # Verify file exists and has content + assert save_path.exists() + + with open(save_path) as f: + data = json.load(f) + assert "nodes" in data + assert "root_id" in data + # Graph metadata like degree and depth might not be saved + # Just verify the essential structure is present + + def test_graph_from_json(self, minimal_test_config, temp_output_dir): + """Test loading graph from JSON file.""" + # Create and save a graph + original_graph = Graph( + topic_prompt="Python basics", + degree=2, + depth=1, + **minimal_test_config, + ) + + list(original_graph.build()) + save_path = temp_output_dir / "test_graph.json" + original_graph.save(str(save_path)) + + # Load graph from JSON + loaded_graph = Graph.from_json( + str(save_path), + { + "topic_prompt": "Python basics", + "degree": 2, + "depth": 1, + **minimal_test_config, + }, + ) + + # Verify loaded graph has same structure + assert loaded_graph.degree == original_graph.degree + assert loaded_graph.depth == original_graph.depth + assert len(loaded_graph.nodes) == len(original_graph.nodes) + + def test_graph_get_all_paths(self, minimal_test_config): + """Test getting all paths from graph.""" + graph = Graph( + topic_prompt="Python control flow", + degree=2, + depth=2, + **minimal_test_config, + ) + + # Build the graph + list(graph.build()) + + # Get all paths through the graph + paths = graph.get_all_paths() + + # Verify paths structure + assert paths is not None + assert len(paths) > 0 + + # Each path should contain topics + for path in paths: + assert len(path) > 0 + assert all(isinstance(topic, str) for topic in path) + + def test_graph_visualization(self, minimal_test_config, temp_output_dir): + """Test graph visualization generation.""" + graph = Graph( + topic_prompt="Python modules", + degree=2, + depth=1, + **minimal_test_config, + ) + + # Build the graph + list(graph.build()) + + # Create visualization + output_path = temp_output_dir / "test_graph_viz" + graph.visualize(str(output_path)) + + # Check SVG file was created + svg_path = temp_output_dir / "test_graph_viz.svg" + assert svg_path.exists() + + # Verify SVG content + with open(svg_path) as f: + content = f.read() + assert "" in content + + def test_graph_with_github_provider(self): + """Test graph creation specifically with GitHub provider.""" + + graph = Graph( + topic_prompt="Web development basics", + topic_system_prompt="You are a web development expert creating educational content.", + provider="github", + model_name="openai/gpt-4o-mini", + degree=2, + depth=1, # Keep minimal for speed + temperature=0.1, + ) + + # Build should succeed + events = list(graph.build()) + assert len(events) > 0 + + final_event = events[-1] + assert final_event["event"] == "build_complete" + + def test_graph_node_relationships(self, minimal_test_config): + """Test that graph nodes have proper parent-child relationships.""" + graph = Graph( + topic_prompt="Database concepts", + degree=2, + depth=2, + **minimal_test_config, + ) + + # Build the graph + list(graph.build()) + + # Check root node exists + assert graph.root is not None # type: ignore + + # Verify relationships + nodes_with_children = [node for node in graph.nodes.values() if node.children] + nodes_with_parents = [node for node in graph.nodes.values() if node.parents] + + # Should have some nodes with children (non-leaf nodes) + assert len(nodes_with_children) > 0 + + # Should have some nodes with parents (non-root nodes) + assert len(nodes_with_parents) > 0 + + def test_graph_error_handling(self, minimal_test_config): + """Test graph handles API errors gracefully.""" + # Use completely invalid provider to trigger error + config = minimal_test_config.copy() + config["provider"] = "nonexistent_provider" + config["model_name"] = "invalid_model" + + # Creating graph with invalid provider should raise error + with pytest.raises(Exception): # noqa: B017 + Graph( + topic_prompt="Test", + degree=2, + depth=1, + **config, + ) diff --git a/tests/integration/test_pipeline_integration.py b/tests/integration/test_pipeline_integration.py new file mode 100644 index 00000000..01dd4f00 --- /dev/null +++ b/tests/integration/test_pipeline_integration.py @@ -0,0 +1,375 @@ +""" +End-to-end pipeline integration tests. +""" + +import json + +from deepfabric import DataSetGenerator, DeepFabricConfig, Tree + +MESSAGE_LIMIT = 2 +SAMPLE_LIMIT = 3 +CONTENT_LENGTH_THRESHOLD = 50 + + +class TestPipelineIntegration: + """End-to-end pipeline integration tests.""" + + def test_tree_to_dataset_pipeline(self, minimal_test_config, temp_output_dir): + """Test complete pipeline from tree creation to dataset generation.""" + # Step 1: Create and build tree + tree = Tree( + topic_prompt="Basic mathematics", + topic_system_prompt="You are a mathematics educator creating educational topics.", + degree=2, + depth=2, + **minimal_test_config, + ) + + events = list(tree.build()) + assert events[-1]["event"] == "build_complete" + + # Step 2: Save tree + tree_path = temp_output_dir / "pipeline_tree.jsonl" + tree.save(str(tree_path)) + assert tree_path.exists() + + # Step 3: Create generator + generator = DataSetGenerator( + instructions="Create mathematics questions and detailed step-by-step solutions.", + generation_system_prompt="You are a mathematics tutor providing clear explanations.", + **minimal_test_config, + ) + + # Step 4: Generate dataset + dataset = generator.create_data( + num_steps=3, + batch_size=1, + topic_model=tree, + ) + + # Step 5: Save dataset + dataset_path = temp_output_dir / "pipeline_dataset.jsonl" + dataset.save(str(dataset_path)) # type: ignore + assert dataset_path.exists() + + # Step 6: Verify pipeline outputs + assert len(dataset.samples) >= SAMPLE_LIMIT # type: ignore + + # Verify dataset content quality + for sample in dataset.samples: # type: ignore + assert "messages" in sample + messages = sample["messages"] + assert len(messages) >= MESSAGE_LIMIT # At least user and assistant + + # Check for user and assistant messages + roles = [msg["role"] for msg in messages] + assert "user" in roles + assert "assistant" in roles + + def test_config_based_pipeline(self, test_config_path, temp_output_dir): + """Test pipeline using YAML configuration.""" + # Load config + config = DeepFabricConfig.from_yaml(str(test_config_path)) + + # Override paths to use temp directory + tree_path = temp_output_dir / "config_tree.jsonl" + dataset_path = temp_output_dir / "config_dataset.jsonl" + + # Step 1: Create tree from config + tree_params = config.get_topic_tree_params() # type: ignore + tree = Tree(**tree_params) + list(tree.build()) + tree.save(str(tree_path)) + + # Step 2: Create generator from config + engine_params = config.get_engine_params() + generator = DataSetGenerator(**engine_params) + + # Step 3: Create dataset + dataset_config = config.get_dataset_config() + dataset = generator.create_data( + num_steps=dataset_config["creation"]["num_steps"], + batch_size=dataset_config["creation"]["batch_size"], + topic_model=tree, + ) + + # Step 4: Save dataset + dataset.save(str(dataset_path)) # type: ignore + + # Verify outputs + assert tree_path.exists() + assert dataset_path.exists() + assert len(dataset.samples) > 0 # type: ignore + + def test_error_recovery_pipeline(self, minimal_test_config, temp_output_dir): + """Test pipeline error recovery and partial results.""" + # Create tree + tree = Tree( + topic_prompt="Complex topics", + degree=2, + depth=1, + **minimal_test_config, + ) + list(tree.build()) + + # Create generator with limited retries + config = minimal_test_config.copy() + config["max_retries"] = 1 + + generator = DataSetGenerator( + instructions="Create content", + generation_system_prompt="You are helpful.", + **config, + ) + + # Generate with potential failures + dataset = generator.create_data( + num_steps=3, + batch_size=1, + topic_model=tree, + ) + + # Should have some results even if some fail + assert isinstance(dataset, type(dataset)) # Should not crash + + # Save what we have + dataset_path = temp_output_dir / "partial_dataset.jsonl" + dataset.save(str(dataset_path)) # type: ignore + assert dataset_path.exists() + + def test_validation_pipeline(self, minimal_test_config): + """Test pipeline data validation throughout.""" + # Create tree with validation + tree = Tree( + topic_prompt="Validated content", + degree=2, + depth=1, + **minimal_test_config, + ) + + list(tree.build()) + + # Verify tree structure + paths = tree.get_all_paths() + assert len(paths) > 0 + for path in paths: + assert len(path) > 0 + assert all(isinstance(topic, str) and len(topic.strip()) > 0 for topic in path) + + # Create generator + generator = DataSetGenerator( + instructions="Create valid, structured content", + generation_system_prompt="You create well-formatted responses.", + **minimal_test_config, + ) + + # Generate and validate dataset + dataset = generator.create_data( + num_steps=2, + batch_size=1, + topic_model=tree, + ) + + # Validate all samples + for sample in dataset.samples: # type: ignore + assert dataset.validate_sample(sample), f"Invalid sample: {sample}" # type: ignore + + def test_different_conversation_types_pipeline(self, minimal_test_config): + """Test pipeline with different conversation types.""" + conversation_types = ["basic", "cot_freetext", "cot_structured"] + + for conv_type in conversation_types: + # Create tree + tree = Tree( + topic_prompt=f"Topics for {conv_type}", + degree=2, + depth=1, + **minimal_test_config, + ) + list(tree.build()) + + # Create generator with specific conversation type + generator = DataSetGenerator( + instructions=f"Create {conv_type} style content", + generation_system_prompt="You adapt your style to requirements.", + conversation_type=conv_type, + **minimal_test_config, + ) + + # Generate dataset + dataset = generator.create_data( + num_steps=1, + batch_size=1, + topic_model=tree, + ) + + assert len(dataset.samples) > 0 # type: ignore + + # Verify conversation structure based on type + sample = dataset.samples[0] # type: ignore + messages = sample["messages"] + + if conv_type.startswith("cot"): + # CoT should have more detailed responses + assistant_messages = [msg for msg in messages if msg["role"] == "assistant"] + assert len(assistant_messages) > 0 + # CoT responses should be longer + for msg in assistant_messages: + assert len(msg["content"]) > CONTENT_LENGTH_THRESHOLD # Basic length check + + def test_graph_to_dataset_pipeline(self, minimal_test_config, temp_output_dir): + """Test pipeline using graph instead of tree.""" + # Import Graph + from deepfabric import Graph # noqa: PLC0415 + + # Create graph + graph = Graph( + topic_prompt="Software engineering", + topic_system_prompt="You are a software engineering expert.", + degree=2, + depth=2, + **minimal_test_config, + ) + + list(graph.build()) + + # Save graph + graph_path = temp_output_dir / "pipeline_graph.json" + graph.save(str(graph_path)) + + # Generate dataset using graph as topic model + generator = DataSetGenerator( + instructions="Create software engineering content", + generation_system_prompt="You are a software engineering instructor.", + **minimal_test_config, + ) + + dataset = generator.create_data( + num_steps=2, + batch_size=1, + topic_model=graph, # Use graph directly as topic model + ) + + # Save dataset + dataset_path = temp_output_dir / "graph_dataset.jsonl" + dataset.save(str(dataset_path)) # type: ignore + + # Verify outputs + assert graph_path.exists() + assert dataset_path.exists() + assert len(dataset.samples) >= MESSAGE_LIMIT # type: ignore + + def test_pipeline_with_github_provider(self, temp_output_dir): + """Test complete pipeline with GitHub provider.""" + + github_config = { + "provider": "github", + "model_name": "openai/gpt-4o-mini", + "temperature": 0.1, + "max_retries": 1, + "request_timeout": 60, # Longer timeout for real API + } + + # Create tree + tree = Tree( + topic_prompt="Environmental science basics", + topic_system_prompt="You are an environmental science educator.", + degree=2, + depth=1, + **github_config, + ) + + events = list(tree.build()) + assert events[-1]["event"] == "build_complete" + + # Generate dataset + generator = DataSetGenerator( + instructions="Create environmental science educational content with real-world examples.", + generation_system_prompt="You are an environmental science teacher creating engaging content.", + **github_config, + ) + + dataset = generator.create_data( + num_steps=2, + batch_size=1, + topic_model=tree, + ) + + # Save outputs + tree_path = temp_output_dir / "github_tree.jsonl" + dataset_path = temp_output_dir / "github_dataset.jsonl" + + tree.save(str(tree_path)) + dataset.save(str(dataset_path)) # type: ignore + + # Verify quality with real API + assert len(dataset.samples) >= MESSAGE_LIMIT # type: ignore + for sample in dataset.samples: # type: ignore + messages = sample["messages"] + # Real API should produce higher quality content + for msg in messages: + assert len(msg["content"]) > CONTENT_LENGTH_THRESHOLD # Reasonable content length + assert msg["content"].strip() # Not empty + + def test_pipeline_statistics_and_metadata(self, minimal_test_config, temp_output_dir): + """Test pipeline generates proper statistics and metadata.""" + # Create tree + tree = Tree( + topic_prompt="Statistics topics", + degree=2, + depth=1, + **minimal_test_config, + ) + list(tree.build()) + + # Generate dataset + generator = DataSetGenerator( + instructions="Create statistics content", + generation_system_prompt="You teach statistics.", + **minimal_test_config, + ) + + dataset = generator.create_data( + num_steps=3, + batch_size=1, + topic_model=tree, + ) + + # Get statistics + stats = dataset.get_statistics() # type: ignore + + # Verify statistics structure + assert "total_samples" in stats + assert "total_messages" in stats + assert "avg_messages_per_sample" in stats + assert stats["total_samples"] == len(dataset.samples) # type: ignore + assert stats["total_messages"] > 0 + + # Create metadata + metadata = { + "tree_config": { + "degree": tree.degree, + "depth": tree.depth, + "total_paths": len(tree.get_all_paths()), + }, + "generator_config": { + "provider": generator.config.provider, + "model": generator.config.model_name, + "temperature": generator.config.temperature, + }, + "dataset_stats": stats, + "failed_samples": len(generator.failed_samples) if generator.failed_samples else 0, + } + + # Save metadata + metadata_path = temp_output_dir / "pipeline_metadata.json" + with open(metadata_path, "w") as f: + json.dump(metadata, f, indent=2) + + assert metadata_path.exists() + + # Verify metadata content + with open(metadata_path) as f: + loaded_metadata = json.load(f) + assert loaded_metadata["dataset_stats"]["total_samples"] > 0 + assert loaded_metadata["tree_config"]["total_paths"] > 0 diff --git a/tests/integration/test_tree_integration.py b/tests/integration/test_tree_integration.py new file mode 100644 index 00000000..46d20b16 --- /dev/null +++ b/tests/integration/test_tree_integration.py @@ -0,0 +1,166 @@ +""" +Integration tests for Tree generation. +""" + +import pytest + +from deepfabric import Tree +from deepfabric.exceptions import TreeError + +MESSAGE_LIMIT = 2 +TOTAL_PATHS = 4 +PATHS_DEPTH = 3 +TREE_DEPTH_LIMIT = 2 +TREE_DEGREE_LIMIT = 2 + + +class TestTreeIntegration: + """Integration tests for Tree generation functionality.""" + + def test_tree_creation_basic(self, minimal_test_config): + """Test basic tree creation with minimal parameters.""" + tree = Tree( + topic_prompt="Basic Python concepts", + topic_system_prompt="You are a Python educator.", + degree=2, + depth=2, + **minimal_test_config, + ) + + assert tree.topic_prompt == "Basic Python concepts" + assert tree.degree == TREE_DEGREE_LIMIT + assert tree.depth == TREE_DEPTH_LIMIT + + def test_tree_build_small(self, minimal_test_config): + """Test building a small tree and validate structure.""" + tree = Tree( + topic_prompt="Python data types", + topic_system_prompt="You are a Python expert creating educational topics.", + degree=2, + depth=2, + **minimal_test_config, + ) + + # Build the tree + events = list(tree.build()) + + # Check that we get events + assert len(events) > 0 + + # Check final event + final_event = events[-1] + assert final_event["event"] == "build_complete" + assert final_event["total_paths"] == TOTAL_PATHS # 2^2 = 4 paths + + # Verify tree has paths + paths = tree.get_all_paths() + assert len(paths) == TOTAL_PATHS + + # Verify each path has correct depth + for path in paths: + assert len(path) == PATHS_DEPTH # root + 2 levels + + def test_tree_save_load(self, minimal_test_config, temp_output_dir): + """Test tree persistence to JSONL format.""" + tree = Tree( + topic_prompt="Python functions", + topic_system_prompt="You are a Python expert.", + degree=2, + depth=1, # Keep small for speed + **minimal_test_config, + ) + + # Build the tree + list(tree.build()) + + # Save to file + save_path = temp_output_dir / "test_tree.jsonl" + tree.save(str(save_path)) + + # Verify file exists and has content + assert save_path.exists() + with open(save_path) as f: + lines = f.readlines() + assert len(lines) > 0 + + # Check JSONL format + import json # noqa: PLC0415 + + for line in lines: + data = json.loads(line) + # The format has changed to use 'path' instead of 'topic_path' + assert "path" in data + assert isinstance(data["path"], list) + + def test_tree_validation_errors(self, minimal_test_config): + """Test tree validation for invalid configurations.""" + # Test invalid degree + with pytest.raises((TreeError, ValueError)): + Tree( + topic_prompt="Test", + degree=0, # Invalid + depth=2, + **minimal_test_config, + ) + + # Test invalid depth + with pytest.raises((TreeError, ValueError)): + Tree( + topic_prompt="Test", + degree=2, + depth=0, # Invalid + **minimal_test_config, + ) + + def test_tree_path_calculation(self, minimal_test_config): + """Test tree path calculation is correct.""" + tree = Tree( + topic_prompt="Test topic", + degree=3, + depth=2, + **minimal_test_config, + ) + + # Build tree + list(tree.build()) + + # Verify path count + paths = tree.get_all_paths() + expected_paths = 3**2 # degree^depth + assert len(paths) == expected_paths + + def test_tree_with_github_provider(self): + """Test tree creation specifically with GitHub provider.""" + + tree = Tree( + topic_prompt="Machine learning basics", + topic_system_prompt="You are an ML expert creating educational content.", + provider="github", + model_name="openai/gpt-4o-mini", + degree=2, + depth=1, # Keep minimal for speed + temperature=0.1, + ) + + # Build should succeed + events = list(tree.build()) + assert len(events) > 0 + + final_event = events[-1] + assert final_event["event"] == "build_complete" + + def test_tree_error_handling(self, minimal_test_config): + """Test tree handles API errors gracefully.""" + # Use completely invalid provider to trigger error + config = minimal_test_config.copy() + config["provider"] = "nonexistent_provider" + config["model_name"] = "invalid_model" + + # Creating tree with invalid provider should raise error + with pytest.raises(Exception): # noqa: B017 + Tree( + topic_prompt="Test", + degree=2, + depth=1, + **config, + )