benchmark_example.py

"""
Example: Using the Accuracy Benchmarking Suite

Demonstrates how to benchmark compressed models.
"""

import sys
import os
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

import torch
from utils.huggingface_loader import load_gpt2_model
from benchmarks.accuracy_benchmark import AccuracyBenchmark


def benchmark_gpt2_example():
    """
    Example: Benchmark original vs compressed GPT-2 model.
    """
    print("=" * 80)
    print("Accuracy Benchmarking Example - GPT-2")
    print("=" * 80)
    
    # Load model
    print("\n[1] Loading GPT-2 model...")
    model, tokenizer, config = load_gpt2_model('gpt2', task='causal-lm')
    print(f"  Model: {config['model_name']} ({config['num_parameters']:,} parameters)")
    
    # For this example, we'll simulate a compressed model
    # In practice, this would be the output of the compression pipeline
    print("\n[2] Note: Using same model for both original and 'compressed' (demo only)")
    print("  In real usage, compressed_model would be from pipeline.compress()")
    
    original_model = model
    compressed_model = model  # In reality, this would be compressed
    
    # Create benchmark
    benchmark = AccuracyBenchmark()
    
    # Test texts
    test_texts = [
        "The quick brown fox jumps over the lazy dog.",
        "Artificial intelligence is transforming technology.",
        "Machine learning models can be compressed efficiently.",
        "Neural networks are powerful tools for pattern recognition."
    ]
    
    print("\n[3] Running comprehensive benchmark...")
    print(f"  Test samples: {len(test_texts)}")
    
    # Run comprehensive comparison
    results = benchmark.compare_models(
        original_model,
        compressed_model,
        tokenizer,
        test_texts,
        model_type='causal-lm'
    )
    
    # Print summary
    print("\n" + "=" * 80)
    print("BENCHMARK RESULTS")
    print("=" * 80)
    
    print("\n[Similarity Metrics]")
    similarity = results['comparison']['similarity']
    print(f"  Cosine Similarity: {similarity.get('cosine_similarity', 'N/A'):.6f}")
    print(f"  MSE: {similarity.get('mse', 'N/A'):.6e}")
    print(f"  L1 Difference: {similarity.get('l1_difference', 'N/A'):.6e}")
    print(f"  L2 Difference: {similarity.get('l2_difference', 'N/A'):.6e}")
    print(f"  Max Absolute Error: {similarity.get('max_absolute_error', 'N/A'):.6e}")
    
    print("\n[Performance Metrics]")
    print(f"  Speedup: {results['comparison']['speedup']:.2f}x")
    print(f"  Original Speed: {results['original']['speed']['avg_time_ms']:.2f} ms/sample")
    print(f"  Compressed Speed: {results['compressed']['speed']['avg_time_ms']:.2f} ms/sample")
    
    # Save results
    output_path = "benchmark_results_gpt2.json"
    benchmark.results = results
    benchmark.save_results(output_path)
    
    print("\n" + "=" * 80)
    print("[OK] Benchmark complete!")
    print(f"Results saved to: {output_path}")
    print("=" * 80)


def benchmark_perplexity_example():
    """
    Example: Measure perplexity on language model.
    """
    print("\n\n" + "=" * 80)
    print("Perplexity Measurement Example")
    print("=" * 80)
    
    # Load model
    print("\n[1] Loading GPT-2 model...")
    model, tokenizer, config = load_gpt2_model('gpt2', task='causal-lm')
    
    # Create benchmark
    benchmark = AccuracyBenchmark()
    
    # Test texts
    test_texts = [
        "The capital of France is Paris.",
        "Water freezes at zero degrees Celsius.",
        "The Earth orbits around the Sun.",
    ]
    
    print("\n[2] Measuring perplexity...")
    results = benchmark.measure_perplexity(
        model,
        tokenizer,
        test_texts,
        model_name="gpt2"
    )
    
    print("\n[Results]")
    print(f"  Perplexity: {results['perplexity']:.2f}")
    print(f"  Average Loss: {results['avg_loss']:.4f}")
    print(f"  Total Tokens: {results['total_tokens']}")
    
    print("\n[Interpretation]")
    if results['perplexity'] < 50:
        print("  Excellent - Model is very confident")
    elif results['perplexity'] < 100:
        print("  Good - Model has reasonable confidence")
    elif results['perplexity'] < 200:
        print("  Fair - Model is somewhat uncertain")
    else:
        print("  Poor - Model is very uncertain")


def benchmark_speed_example():
    """
    Example: Measure inference speed.
    """
    print("\n\n" + "=" * 80)
    print("Inference Speed Measurement Example")
    print("=" * 80)
    
    # Load model
    print("\n[1] Loading GPT-2 model...")
    model, tokenizer, config = load_gpt2_model('gpt2', task='causal-lm')
    
    # Create sample input
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token
    
    sample_text = "The future of artificial intelligence is"
    sample_inputs = tokenizer(sample_text, return_tensors='pt')['input_ids']
    
    # Create benchmark
    benchmark = AccuracyBenchmark()
    
    print("\n[2] Measuring inference speed (100 runs)...")
    results = benchmark.measure_inference_speed(
        model,
        sample_inputs,
        num_runs=100,
        model_name="gpt2"
    )
    
    print("\n[Results]")
    print(f"  Average Time: {results['avg_time_ms']:.2f} ms")
    print(f"  Min Time: {results['min_time_ms']:.2f} ms")
    print(f"  Max Time: {results['max_time_ms']:.2f} ms")
    print(f"  Throughput: {results['throughput_samples_per_sec']:.2f} samples/sec")


if __name__ == "__main__":
    import argparse
    
    parser = argparse.ArgumentParser(description="Accuracy Benchmarking Examples")
    parser.add_argument(
        '--example',
        type=str,
        choices=['comprehensive', 'perplexity', 'speed', 'all'],
        default='comprehensive',
        help='Which example to run'
    )
    
    args = parser.parse_args()
    
    try:
        if args.example == 'comprehensive' or args.example == 'all':
            benchmark_gpt2_example()
        
        if args.example == 'perplexity' or args.example == 'all':
            benchmark_perplexity_example()
        
        if args.example == 'speed' or args.example == 'all':
            benchmark_speed_example()
        
        print("\n" + "=" * 80)
        print("[OK] All benchmark examples completed!")
        print("=" * 80)
        
    except KeyboardInterrupt:
        print("\n\n[WARNING] Interrupted by user")
    except Exception as e:
        print(f"\n\n[ERROR] Error: {e}")
        import traceback
        traceback.print_exc()