Integration Testing

.github/workflows/integration-tests.yml

@@ -0,0 +1,177 @@
+name: Integration Tests
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:  # Allow manual trigger
+
+permissions:
+  models: read
+  contents: read
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30  # Prevent hanging tests
+
+    strategy:
+      matrix:
+        python-version: ['3.11']
+      fail-fast: false  # Continue testing other versions if one fails
+
+    steps:
+    - uses: actions/checkout@v5
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"
+
+    - name: Install dependencies
+      run: |
+        uv sync --all-extras
+
+    - name: Verify GitHub Models access
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        curl -s "https://models.github.ai/inference/chat/completions" \
+          -H "Content-Type: application/json" \
+          -H "Authorization: Bearer $GITHUB_TOKEN" \
+          -d '{
+            "messages": [{"role": "user", "content": "Hello"}],
+            "model": "openai/gpt-4o-mini",
+            "max_tokens": 10
+          }' || echo "GitHub Models API test failed - tests may be skipped"
+
+    - name: Run tree integration tests
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEEPFABRIC_TESTING: "true"
+        ANONYMIZED_TELEMETRY: "False"
+      run: |
+        uv run pytest tests/integration/test_tree_integration.py -v \
+          --tb=short \
+          --durations=10 \
+          --timeout=300
+
+    - name: Run graph integration tests
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEEPFABRIC_TESTING: "true"
+        ANONYMIZED_TELEMETRY: "False"
+      run: |
+        uv run pytest tests/integration/test_graph_integration.py -v \
+          --tb=short \
+          --durations=10 \
+          --timeout=300
+
+    - name: Run generator integration tests
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEEPFABRIC_TESTING: "true"
+        ANONYMIZED_TELEMETRY: "False"
+      run: |
+        uv run pytest tests/integration/test_generator_integration.py -v \
+          --tb=short \
+          --durations=10 \
+          --timeout=300
+
+    - name: Run pipeline integration tests
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEEPFABRIC_TESTING: "true"
+        ANONYMIZED_TELEMETRY: "False"
+      run: |
+        uv run pytest tests/integration/test_pipeline_integration.py -v \
+          --tb=short \
+          --durations=10 \
+          --timeout=300
+
+    - name: Run CLI integration tests
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEEPFABRIC_TESTING: "true"
+        ANONYMIZED_TELEMETRY: "False"
+      run: |
+        uv run pytest tests/integration/test_cli_integration.py -v \
+          --tb=short \
+          --durations=10 \
+          --timeout=300
+
+    - name: Generate integration test report
+      if: always()
+      run: |
+        echo "## Integration Test Summary" > integration_report.md
+        echo "- **Python Version**: ${{ matrix.python-version }}" >> integration_report.md
+        echo "- **Timestamp**: $(date)" >> integration_report.md
+        echo "- **Branch**: ${{ github.ref }}" >> integration_report.md
+        echo "- **Commit**: ${{ github.sha }}" >> integration_report.md
+
+        # Count test files
+        echo "- **Test Files**: $(find tests/integration -name 'test_*.py' | wc -l)" >> integration_report.md
+
+        # Show which tests were run
+        echo "" >> integration_report.md
+        echo "### Test Coverage" >> integration_report.md
+        echo "- Tree Generation Tests" >> integration_report.md
+        echo "- Graph Generation Tests" >> integration_report.md
+        echo "- Dataset Generator Tests" >> integration_report.md
+        echo "- End-to-End Pipeline Tests" >> integration_report.md
+        echo "- CLI Integration Tests" >> integration_report.md
+
+    - name: Upload test artifacts
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: integration-test-results-${{ matrix.python-version }}
+        path: |
+          integration_report.md
+          .pytest_cache/
+        retention-days: 7
+
+    - name: Check for test failures
+      if: failure()
+      run: |
+        echo "❌ Integration tests failed!"
+        echo "Please check the logs above for details."
+        echo "Common issues:"
+        echo "- GitHub Models API rate limits"
+        echo "- Network connectivity"
+        echo "- Test timeouts"
+        exit 1
+
+  integration-test-summary:
+    needs: integration-tests
+    runs-on: ubuntu-latest
+    if: always()
+
+    steps:
+    - name: Integration Test Summary
+      run: |
+        echo "## Integration Test Results"
+        echo "Status: ${{ needs.integration-tests.result }}"
+
+        if [ "${{ needs.integration-tests.result }}" == "success" ]; then
+          echo "✅ All integration tests passed!"
+          echo "The following components have been validated:"
+          echo "- GitHub Models provider integration"
+          echo "- Tree and Graph generation"
+          echo "- Dataset generation pipeline"
+          echo "- CLI functionality"
+          echo "- End-to-end workflows"
+        else
+          echo "❌ Integration tests failed or were skipped"
+          echo "This may be due to:"
+          echo "- Missing GITHUB_TOKEN (tests are skipped automatically)"
+          echo "- GitHub Models API issues"
+          echo "- Test timeouts or network issues"
+        fi

deepfabric/llm/client.py

@@ -36,7 +36,7 @@ def make_outlines_model(provider: str, model_name: str, **kwargs) -> Any:
     """Create an Outlines model for the specified provider and model.
 
     Args:
-        provider: Provider name (openai, anthropic, gemini, ollama)
+        provider: Provider name (openai, anthropic, gemini, ollama, github)
         model_name: Model identifier
         **kwargs: Additional parameters passed to the client
 
@@ -86,6 +86,20 @@ def make_outlines_model(provider: str, model_name: str, **kwargs) -> Any:
             )
             return outlines.from_openai(client, model_name)
 
+        if provider == "github":
+            # Use GitHub Models API with OpenAI-compatible interface
+            api_key = os.getenv("GITHUB_TOKEN") or os.getenv("MODELS_TOKEN")
+            if not api_key:
+                _raise_api_key_error("GITHUB_TOKEN or MODELS_TOKEN")
+
+            base_url = kwargs.get("base_url", "https://models.github.ai/inference")
+            client = openai.OpenAI(
+                base_url=base_url,
+                api_key=api_key,
+                **{k: v for k, v in kwargs.items() if k != "base_url"},
+            )
+            return outlines.from_openai(client, model_name)
+
         _raise_unsupported_provider_error(provider)
 
     except DataSetGeneratorError:

tests/integration/__init__.py

@@ -0,0 +1 @@
+"""Integration tests for DeepFabric."""

tests/integration/conftest.py

@@ -0,0 +1,56 @@
+import os
+import tempfile
+
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def fixtures_dir():
+    """Return path to the fixtures directory."""
+    return Path(__file__).parent / "fixtures"
+
+
+@pytest.fixture
+def test_config_path(fixtures_dir):
+    """Return path to test configuration file."""
+    return fixtures_dir / "test_config.yaml"
+
+
+@pytest.fixture
+def graph_config_path(fixtures_dir):
+    """Return path to graph configuration file."""
+    return fixtures_dir / "graph_config.yaml"
+
+
+@pytest.fixture
+def sample_tree_path(fixtures_dir):
+    """Return path to sample tree file."""
+    return fixtures_dir / "sample_tree.jsonl"
+
+
+@pytest.fixture
+def temp_output_dir():
+    """Create a temporary directory for test outputs."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        yield Path(tmpdir)
+
+
+@pytest.fixture(autouse=True)
+def github_token_check():
+    """Skip integration tests if GITHUB_TOKEN or MODELS_TOKEN is not available."""
+    if not (os.environ.get("GITHUB_TOKEN") or os.environ.get("MODELS_TOKEN")):
+        pytest.skip("GITHUB_TOKEN or MODELS_TOKEN not available - skipping integration test")
+
+
+@pytest.fixture
+def minimal_test_config():
+    """Provide minimal configuration for fast testing."""
+    return {
+        "provider": "github",
+        "model_name": "openai/gpt-4o-mini",
+        "temperature": 0.1,
+        "max_retries": 1,
+        "request_timeout": 30,
+    }

tests/integration/fixtures/graph_config.yaml

@@ -0,0 +1,31 @@
+dataset_system_prompt: "You are a helpful AI assistant focused on providing clear, accurate, and concise responses."
+
+topic_graph:
+  args:
+    topic_prompt: "Basic Python programming concepts"
+    topic_system_prompt: "You are an expert Python educator creating educational content."
+    degree: 2
+    depth: 2
+    provider: "github"
+    model_name: "openai/gpt-4o-mini"
+    temperature: 0.1
+  save_as: "test_graph.json"
+
+data_engine:
+  instructions: "Create simple Python programming questions and answers suitable for beginners."
+  args:
+    generation_system_prompt: "You are a Python programming tutor. Create clear, educational content with practical examples."
+    provider: "github"
+    model_name: "openai/gpt-4o-mini"
+    temperature: 0.1
+    max_retries: 2
+    default_batch_size: 2
+    default_num_examples: 1
+    request_timeout: 30
+    sys_msg: true
+
+dataset:
+  creation:
+    num_steps: 5
+    batch_size: 2
+  save_as: "test_dataset.jsonl"

tests/integration/fixtures/test_config.yaml

@@ -0,0 +1,31 @@
+dataset_system_prompt: "You are a helpful AI assistant focused on providing clear, accurate, and concise responses."
+
+topic_tree:
+  args:
+    topic_prompt: "Basic Python programming concepts"
+    topic_system_prompt: "You are an expert Python educator creating educational content."
+    degree: 2
+    depth: 2
+    provider: "github"
+    model_name: "openai/gpt-4o-mini"
+    temperature: 0.1
+  save_as: "test_tree.jsonl"
+
+data_engine:
+  instructions: "Create simple Python programming questions and answers suitable for beginners."
+  args:
+    generation_system_prompt: "You are a Python programming tutor. Create clear, educational content with practical examples."
+    provider: "github"
+    model_name: "openai/gpt-4o-mini"
+    temperature: 0.1
+    max_retries: 2
+    default_batch_size: 2
+    default_num_examples: 1
+    request_timeout: 30
+    sys_msg: true
+
+dataset:
+  creation:
+    num_steps: 5
+    batch_size: 2
+  save_as: "test_dataset.jsonl"