test_simulation.py

"""
Simulation test to demonstrate deepeval-multirun library usage.

This script simulates real usage scenarios with mocked metrics to avoid API calls.
"""

import time
from unittest.mock import Mock, patch
from deepeval.test_case import LLMTestCase
from deepeval.metrics.base_metric import BaseMetric
from deepeval_multirun import (
    MultiRunEvaluator,
    multirun_assert_test,
    ResultsAnalyzer,
    ConfidenceLevel,
)


# Create mock metric classes that simulate real behavior
class MockAnswerRelevancyMetric(BaseMetric):
    """Mock metric that simulates answer relevancy evaluation."""

    def __init__(self, threshold=0.7):
        self.threshold = threshold
        self.score = 0.0

    def measure(self, test_case):
        # Simulate evaluation - this would normally call an LLM
        pass

    def is_successful(self):
        return self.score >= self.threshold


class MockFaithfulnessMetric(BaseMetric):
    """Mock metric that simulates faithfulness evaluation."""

    def __init__(self, threshold=0.7):
        self.threshold = threshold
        self.score = 0.0

    def measure(self, test_case):
        # Simulate evaluation - this would normally call an LLM
        pass

    def is_successful(self):
        return self.score >= self.threshold


def print_section(title):
    """Print a formatted section header."""
    print("\n" + "=" * 70)
    print(f"  {title}")
    print("=" * 70)


def scenario_1_all_pass():
    """Scenario 1: All runs pass - High confidence PASS."""
    print_section("SCENARIO 1: All Runs Pass (High Confidence)")

    test_case = LLMTestCase(
        input="What is 2+2?",
        actual_output="2+2 equals 4",
        expected_output="4",
    )

    # Mock the assert_test to always pass
    with patch("deepeval_multirun.core.assert_test") as mock_assert:
        mock_assert.return_value = None  # No exception = pass

        evaluator = MultiRunEvaluator(
            num_runs=5,
            pass_threshold=3,
            enable_logging=False,
            rate_limit_delay=0.1,  # Reduced delay for testing
        )

        metric = MockAnswerRelevancyMetric(threshold=0.7)

        print("\nRunning evaluation...")
        start = time.time()
        result = evaluator.evaluate_test_case(test_case, [metric], "test_001")
        elapsed = time.time() - start

        print(f"\n✅ Result: {'PASS' if result.final_pass else 'FAIL'}")
        print(f"   Confidence: {result.confidence_level.value.upper()}")
        print(f"   Pass Count: {result.pass_count}/{result.total_runs} metrics")
        print(f"   Execution Time: {elapsed:.2f}s")
        print(f"   Individual Runs: {len(result.individual_results)} runs executed")

        # Show breakdown
        for metric_name, breakdown in result.metrics_breakdown.items():
            print(f"\n   {metric_name} Breakdown:")
            print(f"      Passes: {breakdown['pass_count']}/5 runs")
            print(f"      Pass Rate: {breakdown['pass_rate']:.0%}")
            print(f"      Avg Score: {breakdown['average_raw_score']:.2f}")
            print(f"      Confidence: {breakdown['confidence_level']}")


def scenario_2_partial_pass():
    """Scenario 2: Some runs fail - Low confidence result."""
    print_section("SCENARIO 2: Partial Pass (Low Confidence)")

    test_case = LLMTestCase(
        input="Explain quantum computing",
        actual_output="Quantum computers use quantum mechanics",
        expected_output="Advanced computing using quantum mechanics",
    )

    # Mock alternating pass/fail results
    pass_fail_sequence = [
        None,
        AssertionError("Failed"),
        None,
        AssertionError("Failed"),
        None,
    ]

    with patch("deepeval_multirun.core.assert_test") as mock_assert:
        mock_assert.side_effect = pass_fail_sequence

        evaluator = MultiRunEvaluator(
            num_runs=5, pass_threshold=3, enable_logging=False, rate_limit_delay=0.05
        )

        metric = MockAnswerRelevancyMetric(threshold=0.7)

        print("\nRunning evaluation with inconsistent results...")
        result = evaluator.evaluate_test_case(test_case, [metric], "test_002")

        print(f"\n⚠️  Result: {'PASS' if result.final_pass else 'FAIL'}")
        print(f"   Confidence: {result.confidence_level.value.upper()} (needs review)")
        print(f"   Pass Count: {result.pass_count}/{result.total_runs} metrics")

        for metric_name, breakdown in result.metrics_breakdown.items():
            print(f"\n   {metric_name} Breakdown:")
            print(f"      Passes: {breakdown['pass_count']}/5 runs (3 required)")
            print(f"      Pass Rate: {breakdown['pass_rate']:.0%}")
            print(f"      ⚠️  Low confidence - inconsistent results")


def scenario_3_multiple_metrics():
    """Scenario 3: Multiple metrics evaluation."""
    print_section("SCENARIO 3: Multiple Metrics Evaluation")

    test_case = LLMTestCase(
        input="What is Python?",
        actual_output="Python is a programming language",
        expected_output="Programming language",
        retrieval_context=["Python is a high-level programming language"],
    )

    with patch("deepeval_multirun.core.assert_test") as mock_assert:
        mock_assert.return_value = None

        evaluator = MultiRunEvaluator(
            num_runs=3, pass_threshold=2, enable_logging=False, rate_limit_delay=0.05
        )

        # Create multiple metrics with proper class names
        class MockMetric1(BaseMetric):
            pass

        class MockMetric2(BaseMetric):
            pass

        metric1 = Mock(spec=MockMetric1)
        metric1.__class__ = MockMetric1
        metric2 = Mock(spec=MockMetric2)
        metric2.__class__ = MockMetric2

        print("\nEvaluating with 2 metrics, 3 runs each...")
        result = evaluator.evaluate_test_case(test_case, [metric1, metric2], "test_003")

        print(f"\n✅ Result: {'PASS' if result.final_pass else 'FAIL'}")
        print(f"   Confidence: {result.confidence_level.value.upper()}")
        print(f"   Metrics Passing: {result.pass_count}/{result.total_runs}")
        print(
            f"   Total Evaluations: {len(result.individual_results)} (3 runs × 2 metrics)"
        )

        print("\n   Per-Metric Results:")
        for metric_name, breakdown in result.metrics_breakdown.items():
            status = "✅ PASS" if breakdown["final_pass"] else "❌ FAIL"
            print(f"      {metric_name}: {status} ({breakdown['pass_count']}/3 runs)")


def scenario_4_batch_analysis():
    """Scenario 4: Batch evaluation with results analysis."""
    print_section("SCENARIO 4: Batch Evaluation & Analysis")

    test_cases = [
        LLMTestCase(
            input=f"Test question {i}",
            actual_output=f"Answer {i}",
            expected_output=f"Expected {i}",
        )
        for i in range(1, 4)
    ]

    # Simulate different outcomes
    mock_results_sequence = [
        [None] * 5,  # Test 1: all pass
        [None, AssertionError(), None, AssertionError(), None],  # Test 2: partial
        [AssertionError()] * 5,  # Test 3: all fail
    ]

    results = []

    with patch("deepeval_multirun.core.assert_test") as mock_assert:
        evaluator = MultiRunEvaluator(
            num_runs=5, pass_threshold=3, enable_logging=False, rate_limit_delay=0.05
        )

        metric = MockAnswerRelevancyMetric(threshold=0.7)

        print(f"\nEvaluating {len(test_cases)} test cases...")

        for i, (test_case, mock_sequence) in enumerate(
            zip(test_cases, mock_results_sequence), 1
        ):
            mock_assert.side_effect = mock_sequence
            result = evaluator.evaluate_test_case(test_case, [metric], f"test_{i:03d}")
            results.append(result)

            status = "✅" if result.final_pass else "❌"
            confidence = (
                "🟢" if result.confidence_level == ConfidenceLevel.HIGH else "🟡"
            )
            print(f"   Test {i}: {status} {confidence} {result.confidence_level.value}")

    # Analyze results
    print("\n📊 Overall Statistics:")
    stats = ResultsAnalyzer.calculate_overall_stats(results)
    print(f"   Total: {stats['total_test_cases']}")
    print(f"   Passed: {stats['passed_cases']} ✅")
    print(f"   Failed: {stats['failed_cases']} ❌")
    print(f"   Pass Rate: {stats['overall_pass_rate']:.0%}")
    print(f"   High Confidence: {stats['high_confidence_cases']}")
    print(f"   Low Confidence: {stats['low_confidence_cases']}")

    # Low confidence cases
    low_conf = ResultsAnalyzer.get_low_confidence_cases(results)
    if low_conf:
        print(f"\n⚠️  Cases Needing Review: {len(low_conf)}")
        for r in low_conf:
            print(f"      {r.test_case_id} - {r.confidence_level.value}")


def scenario_5_wrapper_function():
    """Scenario 5: Using the multirun_assert_test wrapper."""
    print_section("SCENARIO 5: Using multirun_assert_test Wrapper")

    test_case = LLMTestCase(
        input="Simple test",
        actual_output="Simple answer",
        expected_output="Expected answer",
    )

    with patch("deepeval_multirun.core.assert_test") as mock_assert:
        with patch("deepeval_multirun.core.time.sleep"):
            mock_assert.return_value = None

            metric = MockAnswerRelevancyMetric(threshold=0.7)

            print(
                "\nCalling multirun_assert_test (drop-in replacement for assert_test)..."
            )
            print("This will run 5 evaluations by default...")

            try:
                # This will use environment config or defaults
                multirun_assert_test(test_case, [metric])
                print("\n✅ Test PASSED - All runs successful!")
            except AssertionError as e:
                print(f"\n❌ Test FAILED - {e}")


def main():
    """Run all simulation scenarios."""
    print("\n" + "=" * 70)
    print("  DEEPEVAL-MULTIRUN SIMULATION TEST")
    print("  Demonstrating library usage without API calls")
    print("=" * 70)

    scenarios = [
        scenario_1_all_pass,
        scenario_2_partial_pass,
        scenario_3_multiple_metrics,
        scenario_4_batch_analysis,
        scenario_5_wrapper_function,
    ]

    for i, scenario in enumerate(scenarios, 1):
        try:
            scenario()
            print(f"\n✅ Scenario {i} completed successfully")
        except Exception as e:
            print(f"\n❌ Scenario {i} failed: {e}")
            import traceback

            traceback.print_exc()

    print("\n" + "=" * 70)
    print("  SIMULATION COMPLETE")
    print("=" * 70)
    print("\n💡 Key Features Demonstrated:")
    print("   ✓ Multi-run evaluation for consistency")
    print("   ✓ Confidence level assessment")
    print("   ✓ Multiple metrics support")
    print("   ✓ Batch processing and analysis")
    print("   ✓ Drop-in replacement for assert_test")
    print("\n🎯 Use Cases:")
    print("   • Reduce flakiness in LLM evaluations")
    print("   • Identify test cases needing human review")
    print("   • Track evaluation confidence over time")
    print("   • Aggregate metrics across multiple runs")


if __name__ == "__main__":
    main()