|
| 1 | +""" |
| 2 | +A/B Testing Demo for prompt-vcs. |
| 3 | +
|
| 4 | +This example demonstrates how to use the A/B testing features to compare |
| 5 | +different versions of prompts. |
| 6 | +
|
| 7 | +Run this after: |
| 8 | +1. pip install -e . |
| 9 | +2. pvcs init |
| 10 | +""" |
| 11 | + |
| 12 | +from prompt_vcs import p, ab_test, ABTestManager, ABTestConfig, ABTestVariant |
| 13 | + |
| 14 | + |
| 15 | +def demo_context_manager_mode(): |
| 16 | + """ |
| 17 | + Demonstrate A/B testing using context manager. |
| 18 | + |
| 19 | + This is the most flexible approach, giving you full control over |
| 20 | + the experiment flow. |
| 21 | + """ |
| 22 | + print("=" * 50) |
| 23 | + print(" A/B Testing: Context Manager Mode") |
| 24 | + print("=" * 50 + "\n") |
| 25 | + |
| 26 | + # 1. Create an experiment |
| 27 | + manager = ABTestManager.get_instance() |
| 28 | + |
| 29 | + config = ABTestConfig( |
| 30 | + name="greeting_experiment", |
| 31 | + prompt_id="user_greeting", |
| 32 | + description="Testing different greeting styles", |
| 33 | + variants=[ |
| 34 | + ABTestVariant("v1", weight=1.0, description="Formal greeting"), |
| 35 | + ABTestVariant("v2", weight=1.0, description="Casual greeting"), |
| 36 | + ], |
| 37 | + ) |
| 38 | + manager.create_experiment(config) |
| 39 | + print(f"Created experiment: {config.name}") |
| 40 | + print(f" Prompt ID: {config.prompt_id}") |
| 41 | + print(f" Variants: v1 (50%), v2 (50%)") |
| 42 | + print() |
| 43 | + |
| 44 | + # 2. Run the experiment multiple times |
| 45 | + print("Running 10 experiments...\n") |
| 46 | + |
| 47 | + for i in range(10): |
| 48 | + user_id = f"user_{i}" # Use user_id for consistent bucketing |
| 49 | + |
| 50 | + with manager.experiment("greeting_experiment", user_id=user_id) as exp: |
| 51 | + # Get the prompt (automatically selects variant) |
| 52 | + prompt = exp.get_prompt(name="Alice") |
| 53 | + |
| 54 | + # Simulate LLM response (in real usage, call your LLM here) |
| 55 | + response = f"AI response to: {prompt[:30]}..." |
| 56 | + |
| 57 | + # Simulate quality score (in real usage, evaluate the response) |
| 58 | + import random |
| 59 | + score = random.uniform(0.6, 1.0) |
| 60 | + |
| 61 | + # Record the result |
| 62 | + exp.record(output=response, score=score) |
| 63 | + |
| 64 | + print(f" User {i}: {exp.variant.version} → score={score:.2f}") |
| 65 | + |
| 66 | + print() |
| 67 | + |
| 68 | + # 3. Analyze results |
| 69 | + result = manager.analyze("greeting_experiment") |
| 70 | + print("Analysis Results:") |
| 71 | + print(result.summary()) |
| 72 | + print() |
| 73 | + |
| 74 | + |
| 75 | +def demo_decorator_mode(): |
| 76 | + """ |
| 77 | + Demonstrate A/B testing using the @ab_test decorator. |
| 78 | + |
| 79 | + This is a simpler approach for basic use cases. |
| 80 | + """ |
| 81 | + print("=" * 50) |
| 82 | + print(" A/B Testing: Decorator Mode") |
| 83 | + print("=" * 50 + "\n") |
| 84 | + |
| 85 | + # Define a function with A/B testing |
| 86 | + @ab_test("farewell_experiment", prompt_id="farewell", variants=["v1", "v2"]) |
| 87 | + def get_farewell(name: str) -> str: |
| 88 | + return p("farewell", "再见，{name}！", name=name) |
| 89 | + |
| 90 | + # Run the experiment |
| 91 | + print("Running 5 experiments...\n") |
| 92 | + |
| 93 | + for i in range(5): |
| 94 | + result = get_farewell(name="Bob") |
| 95 | + print(f" Run {i+1}: {result}") |
| 96 | + |
| 97 | + # Record result (the decorator returns a special object with record method) |
| 98 | + import random |
| 99 | + result.record(output="AI says goodbye", score=random.uniform(0.7, 1.0)) |
| 100 | + |
| 101 | + # Analyze |
| 102 | + manager = ABTestManager.get_instance() |
| 103 | + analysis = manager.analyze("farewell_experiment") |
| 104 | + print("\nAnalysis Results:") |
| 105 | + print(analysis.summary()) |
| 106 | + print() |
| 107 | + |
| 108 | + |
| 109 | +def demo_cli_workflow(): |
| 110 | + """ |
| 111 | + Demonstrate the CLI workflow for A/B testing. |
| 112 | + |
| 113 | + These commands can be run from the terminal. |
| 114 | + """ |
| 115 | + print("=" * 50) |
| 116 | + print(" A/B Testing: CLI Workflow") |
| 117 | + print("=" * 50 + "\n") |
| 118 | + |
| 119 | + print("CLI commands available:\n") |
| 120 | + print(" # Create an experiment") |
| 121 | + print(" pvcs ab create greeting_test user_greeting --variants v1,v2\n") |
| 122 | + |
| 123 | + print(" # List all experiments") |
| 124 | + print(" pvcs ab list\n") |
| 125 | + |
| 126 | + print(" # View experiment status") |
| 127 | + print(" pvcs ab status greeting_test\n") |
| 128 | + |
| 129 | + print(" # Manually record a result") |
| 130 | + print(" pvcs ab record greeting_test v1 --score 0.8\n") |
| 131 | + |
| 132 | + print(" # Analyze results") |
| 133 | + print(" pvcs ab analyze greeting_test\n") |
| 134 | + |
| 135 | + print(" # Clear experiment records") |
| 136 | + print(" pvcs ab clear greeting_test --yes\n") |
| 137 | + |
| 138 | + |
| 139 | +def main(): |
| 140 | + print("\n" + "=" * 50) |
| 141 | + print(" prompt-vcs A/B Testing Demo") |
| 142 | + print("=" * 50 + "\n") |
| 143 | + |
| 144 | + demo_context_manager_mode() |
| 145 | + demo_decorator_mode() |
| 146 | + demo_cli_workflow() |
| 147 | + |
| 148 | + print("=" * 50) |
| 149 | + print(" Demo Complete!") |
| 150 | + print("=" * 50 + "\n") |
| 151 | + |
| 152 | + |
| 153 | +if __name__ == "__main__": |
| 154 | + main() |
0 commit comments