style_validator.py

#!/usr/bin/env python3
"""
Style Validator - LLM-based style drift measurement tool

Compares original text vs modified text and uses an LLM to:
- Rate stylistic similarity (0-100 score)
- Explain what drifted (sentence length, tone, vocabulary, etc.)
- Compare against author profile characteristics if provided

Usage:
    python style_validator.py --original original.txt --modified modified.txt
    python style_validator.py --original original.txt --modified modified.txt --profile author.json
    python style_validator.py --original original.txt --modified modified.txt --output report.json
"""

import argparse
import json
import os
import sys
from dataclasses import dataclass, asdict
from datetime import datetime
from pathlib import Path
from typing import Optional

try:
    import anthropic
except ImportError:
    print("Error: anthropic package not installed. Run: pip install anthropic")
    anthropic = None


# ---------- DATA STRUCTURES ----------

@dataclass
class DriftDimension:
    """A specific dimension of style drift."""
    name: str
    score: int  # 0-100, where 100 = identical to original
    description: str
    severity: str  # "none", "minor", "moderate", "significant", "severe"


@dataclass
class StyleValidationResult:
    """Complete result of a style validation comparison."""
    overall_score: int  # 0-100, where 100 = perfectly preserved style
    overall_assessment: str
    dimensions: list  # List of DriftDimension
    recommendations: list  # List of improvement suggestions
    profile_comparison: Optional[dict]  # Comparison against author profile if provided
    metadata: dict


# ---------- PROMPTS ----------

VALIDATION_SYSTEM_PROMPT = """You are an expert literary analyst specializing in authorial voice and style preservation. Your task is to compare an original text with a modified version and measure how well the original style has been preserved.

You must analyze multiple dimensions of writing style:
1. SENTENCE STRUCTURE: Length distribution, complexity, variety
2. TONE: Emotional register, formality level, narrative distance
3. VOCABULARY: Word choice, register, domain-specific terms
4. RHYTHM: Pacing, flow, paragraph structure
5. VOICE: Authorial presence, perspective consistency, personality
6. LITERARY DEVICES: Use of metaphor, imagery, figurative language

Provide objective, detailed analysis with specific examples from both texts."""


VALIDATION_USER_PROMPT = """Compare the following original and modified texts to measure style preservation.

## ORIGINAL TEXT:
{original_text}

## MODIFIED TEXT:
{modified_text}

## YOUR TASK:
Analyze how well the modified text preserves the style of the original. Provide your response in the following JSON format:

```json
{{
    "overall_score": <0-100, where 100 means style is perfectly preserved>,
    "overall_assessment": "<2-3 sentence summary of style preservation quality>",
    "dimensions": [
        {{
            "name": "sentence_structure",
            "score": <0-100>,
            "description": "<specific observations about sentence length, complexity, variety>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "tone",
            "score": <0-100>,
            "description": "<specific observations about emotional register, formality>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "vocabulary",
            "score": <0-100>,
            "description": "<specific observations about word choice, register>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "rhythm",
            "score": <0-100>,
            "description": "<specific observations about pacing, flow>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "voice",
            "score": <0-100>,
            "description": "<specific observations about authorial presence>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "literary_devices",
            "score": <0-100>,
            "description": "<specific observations about metaphor, imagery>",
            "severity": "<none|minor|moderate|significant|severe>"
        }}
    ],
    "recommendations": [
        "<specific suggestion for improving style preservation>",
        "<another suggestion>"
    ]
}}
```

IMPORTANT:
- A score of 100 means the dimension is identical between original and modified
- A score of 0 means complete deviation
- Provide specific examples from both texts to justify your scores
- Be precise and objective in your analysis

Respond with ONLY the JSON object, no additional text."""


VALIDATION_WITH_PROFILE_PROMPT = """Compare the following original and modified texts to measure style preservation, with reference to the author's style profile.

## AUTHOR STYLE PROFILE:
{profile_summary}

## ORIGINAL TEXT:
{original_text}

## MODIFIED TEXT:
{modified_text}

## YOUR TASK:
Analyze how well the modified text preserves the style of the original, and how well it matches the author's known style profile. Provide your response in the following JSON format:

```json
{{
    "overall_score": <0-100, where 100 means style is perfectly preserved>,
    "overall_assessment": "<2-3 sentence summary of style preservation quality>",
    "dimensions": [
        {{
            "name": "sentence_structure",
            "score": <0-100>,
            "description": "<specific observations about sentence length, complexity, variety>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "tone",
            "score": <0-100>,
            "description": "<specific observations about emotional register, formality>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "vocabulary",
            "score": <0-100>,
            "description": "<specific observations about word choice, register>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "rhythm",
            "score": <0-100>,
            "description": "<specific observations about pacing, flow>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "voice",
            "score": <0-100>,
            "description": "<specific observations about authorial presence>",
            "severity": "<none|minor|moderate|significant|severe>"
        }},
        {{
            "name": "literary_devices",
            "score": <0-100>,
            "description": "<specific observations about metaphor, imagery>",
            "severity": "<none|minor|moderate|significant|severe>"
        }}
    ],
    "profile_comparison": {{
        "matches_profile": <true|false>,
        "profile_alignment_score": <0-100>,
        "profile_observations": [
            "<observation about how modified text aligns or deviates from profile>",
            "<another observation>"
        ],
        "do_list_compliance": "<assessment of whether the modified text follows the profile's 'do' guidelines>",
        "avoid_list_violations": "<any violations of the profile's 'avoid' guidelines>"
    }},
    "recommendations": [
        "<specific suggestion for improving style preservation>",
        "<another suggestion>"
    ]
}}
```

IMPORTANT:
- A score of 100 means the dimension is identical between original and modified
- A score of 0 means complete deviation
- Reference the author profile when evaluating style preservation
- Provide specific examples from both texts to justify your scores
- Be precise and objective in your analysis

Respond with ONLY the JSON object, no additional text."""


# ---------- UTILITY FUNCTIONS ----------

def create_client() -> anthropic.Anthropic:
    """Create Anthropic client using environment variable."""
    if not anthropic:
        print("Error: anthropic package not installed")
        sys.exit(1)

    api_key = os.environ.get('ANTHROPIC_API_KEY')
    if not api_key:
        print("Error: ANTHROPIC_API_KEY environment variable not set")
        print("Set it with: export ANTHROPIC_API_KEY='your-api-key'")
        sys.exit(1)

    return anthropic.Anthropic(api_key=api_key)


def load_text_file(path: str) -> str:
    """Load text content from a file."""
    file_path = Path(path)
    if not file_path.exists():
        print(f"Error: File not found: {path}")
        sys.exit(1)

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            return f.read()
    except Exception as e:
        print(f"Error reading file {path}: {e}")
        sys.exit(1)


def load_author_profile(path: str) -> dict:
    """Load and validate an author profile JSON file."""
    file_path = Path(path)
    if not file_path.exists():
        print(f"Error: Profile file not found: {path}")
        sys.exit(1)

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            profile = json.load(f)
    except json.JSONDecodeError as e:
        print(f"Error parsing profile JSON: {e}")
        sys.exit(1)
    except Exception as e:
        print(f"Error reading profile file: {e}")
        sys.exit(1)

    # Basic validation
    if 'overall_style' not in profile:
        print("Warning: Profile missing 'overall_style' field")

    return profile


def summarize_profile(profile: dict) -> str:
    """Create a concise summary of an author profile for the LLM prompt."""
    summary_parts = []

    # Overall style
    if 'overall_style' in profile:
        summary_parts.append(f"OVERALL STYLE:\n{profile['overall_style']}")

    # Metadata
    if 'metadata' in profile:
        meta = profile['metadata']
        if 'author_name' in meta:
            summary_parts.insert(0, f"AUTHOR: {meta['author_name']}")

    # Do list
    if 'do_list' in profile and profile['do_list']:
        dos = "\n".join(f"  - {item}" for item in profile['do_list'][:10])
        summary_parts.append(f"STYLE ELEMENTS TO EMULATE:\n{dos}")

    # Avoid list
    if 'avoid_list' in profile and profile['avoid_list']:
        avoids = "\n".join(f"  - {item}" for item in profile['avoid_list'][:10])
        summary_parts.append(f"ELEMENTS TO AVOID:\n{avoids}")

    # Vocabulary highlights
    if 'vocabulary' in profile:
        vocab = profile['vocabulary']
        if 'signature_phrases' in vocab and vocab['signature_phrases']:
            phrases = [p.get('phrase', p) if isinstance(p, dict) else p
                      for p in vocab['signature_phrases'][:5]]
            summary_parts.append(f"SIGNATURE PHRASES: {', '.join(phrases)}")

    # Narrative techniques
    if 'narrative_techniques' in profile:
        tech = profile['narrative_techniques']
        tech_items = []
        if 'point_of_view' in tech:
            tech_items.append(f"POV: {tech['point_of_view']}")
        if 'tense' in tech:
            tech_items.append(f"Tense: {tech['tense']}")
        if 'pacing' in tech:
            tech_items.append(f"Pacing: {tech['pacing']}")
        if tech_items:
            summary_parts.append(f"NARRATIVE TECHNIQUES: {', '.join(tech_items)}")

    return "\n\n".join(summary_parts)


def parse_llm_response(response_text: str) -> dict:
    """Parse the LLM's JSON response, handling potential formatting issues."""
    # Try to extract JSON from the response
    text = response_text.strip()

    # Remove markdown code blocks if present
    if text.startswith('```'):
        # Find the end of the first line (might be ```json)
        first_newline = text.find('\n')
        if first_newline != -1:
            text = text[first_newline + 1:]
        # Remove closing ```
        if text.endswith('```'):
            text = text[:-3]
        text = text.strip()

    try:
        return json.loads(text)
    except json.JSONDecodeError as e:
        print(f"Warning: Failed to parse LLM response as JSON: {e}")
        print(f"Response preview: {text[:500]}...")
        # Return a minimal valid structure
        return {
            "overall_score": 0,
            "overall_assessment": f"Error parsing response: {str(e)}",
            "dimensions": [],
            "recommendations": ["Unable to parse LLM response. Please try again."],
            "parse_error": True,
            "raw_response": text[:2000]
        }


# ---------- MAIN VALIDATION FUNCTION ----------

def validate_style(
    original_text: str,
    modified_text: str,
    profile: Optional[dict] = None,
    model: str = "claude-sonnet-4-5-20250929",
    verbose: bool = False
) -> StyleValidationResult:
    """
    Compare original and modified text to measure style drift.

    Args:
        original_text: The original text content
        modified_text: The modified text content
        profile: Optional author profile dict for additional comparison
        model: The Anthropic model to use
        verbose: Whether to print progress information

    Returns:
        StyleValidationResult with scores and analysis
    """
    client = create_client()

    if verbose:
        print(f"Analyzing style drift using {model}...")
        print(f"Original text length: {len(original_text)} characters")
        print(f"Modified text length: {len(modified_text)} characters")
        if profile:
            author = profile.get('metadata', {}).get('author_name', 'Unknown')
            print(f"Comparing against profile: {author}")

    # Build the prompt
    if profile:
        profile_summary = summarize_profile(profile)
        user_prompt = VALIDATION_WITH_PROFILE_PROMPT.format(
            profile_summary=profile_summary,
            original_text=original_text[:15000],  # Limit to avoid token limits
            modified_text=modified_text[:15000]
        )
    else:
        user_prompt = VALIDATION_USER_PROMPT.format(
            original_text=original_text[:15000],
            modified_text=modified_text[:15000]
        )

    # Call the LLM
    try:
        response = client.messages.create(
            model=model,
            max_tokens=4000,
            system=VALIDATION_SYSTEM_PROMPT,
            messages=[{"role": "user", "content": user_prompt}]
        )

        response_text = ""
        for block in response.content:
            if hasattr(block, 'text'):
                response_text += block.text

        if verbose:
            print("Received response from LLM")

    except Exception as e:
        print(f"Error calling LLM: {e}")
        return StyleValidationResult(
            overall_score=0,
            overall_assessment=f"Error during analysis: {str(e)}",
            dimensions=[],
            recommendations=[],
            profile_comparison=None,
            metadata={
                "error": str(e),
                "timestamp": datetime.now().isoformat()
            }
        )

    # Parse the response
    result_dict = parse_llm_response(response_text)

    # Build dimension objects
    dimensions = []
    for dim in result_dict.get('dimensions', []):
        dimensions.append(DriftDimension(
            name=dim.get('name', 'unknown'),
            score=dim.get('score', 0),
            description=dim.get('description', ''),
            severity=dim.get('severity', 'unknown')
        ))

    # Build result
    result = StyleValidationResult(
        overall_score=result_dict.get('overall_score', 0),
        overall_assessment=result_dict.get('overall_assessment', ''),
        dimensions=dimensions,
        recommendations=result_dict.get('recommendations', []),
        profile_comparison=result_dict.get('profile_comparison'),
        metadata={
            "model": model,
            "timestamp": datetime.now().isoformat(),
            "original_length": len(original_text),
            "modified_length": len(modified_text),
            "profile_used": profile is not None,
            "parse_error": result_dict.get('parse_error', False)
        }
    )

    return result


def result_to_dict(result: StyleValidationResult) -> dict:
    """Convert a StyleValidationResult to a JSON-serializable dict."""
    return {
        "overall_score": result.overall_score,
        "overall_assessment": result.overall_assessment,
        "dimensions": [asdict(dim) for dim in result.dimensions],
        "recommendations": result.recommendations,
        "profile_comparison": result.profile_comparison,
        "metadata": result.metadata
    }


def print_result(result: StyleValidationResult):
    """Print a formatted version of the validation result."""
    print("\n" + "=" * 60)
    print("STYLE VALIDATION RESULT")
    print("=" * 60)

    # Overall score with visual indicator
    score = result.overall_score
    if score >= 90:
        indicator = "[EXCELLENT]"
    elif score >= 75:
        indicator = "[GOOD]"
    elif score >= 50:
        indicator = "[MODERATE]"
    elif score >= 25:
        indicator = "[POOR]"
    else:
        indicator = "[SEVERE DRIFT]"

    print(f"\nOVERALL SCORE: {score}/100 {indicator}")
    print(f"\n{result.overall_assessment}")

    # Dimensions
    if result.dimensions:
        print("\n" + "-" * 60)
        print("DIMENSION SCORES:")
        print("-" * 60)
        for dim in result.dimensions:
            severity_indicator = {
                "none": "",
                "minor": "*",
                "moderate": "**",
                "significant": "***",
                "severe": "****"
            }.get(dim.severity, "")

            print(f"\n  {dim.name.upper()}: {dim.score}/100 {severity_indicator}")
            print(f"    {dim.description}")

    # Profile comparison
    if result.profile_comparison:
        print("\n" + "-" * 60)
        print("PROFILE COMPARISON:")
        print("-" * 60)
        pc = result.profile_comparison
        print(f"  Matches Profile: {pc.get('matches_profile', 'N/A')}")
        print(f"  Alignment Score: {pc.get('profile_alignment_score', 'N/A')}/100")
        if pc.get('profile_observations'):
            print("  Observations:")
            for obs in pc['profile_observations']:
                print(f"    - {obs}")
        if pc.get('do_list_compliance'):
            print(f"  Do List Compliance: {pc['do_list_compliance']}")
        if pc.get('avoid_list_violations'):
            print(f"  Avoid List Violations: {pc['avoid_list_violations']}")

    # Recommendations
    if result.recommendations:
        print("\n" + "-" * 60)
        print("RECOMMENDATIONS:")
        print("-" * 60)
        for i, rec in enumerate(result.recommendations, 1):
            print(f"  {i}. {rec}")

    print("\n" + "=" * 60)


# ---------- CLI ----------

def main():
    """Main entry point with CLI argument parsing."""
    parser = argparse.ArgumentParser(
        description='Style Validator - LLM-based style drift measurement',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  %(prog)s --original original.txt --modified modified.txt
  %(prog)s --original original.txt --modified modified.txt --profile author.json
  %(prog)s --original original.txt --modified modified.txt --output report.json

Environment variables:
  ANTHROPIC_API_KEY    Your Anthropic API key (required)

Score interpretation:
  90-100: Excellent - Style nearly perfectly preserved
  75-89:  Good - Minor stylistic differences
  50-74:  Moderate - Noticeable drift in some areas
  25-49:  Poor - Significant style changes
  0-24:   Severe - Almost entirely different style
        """
    )

    parser.add_argument(
        '--original', '-o',
        required=True,
        help='Path to the original text file'
    )

    parser.add_argument(
        '--modified', '-m',
        required=True,
        help='Path to the modified text file'
    )

    parser.add_argument(
        '--profile', '-p',
        help='Optional path to author profile JSON file for additional comparison'
    )

    parser.add_argument(
        '--output',
        help='Path to save JSON output (default: print to console)'
    )

    parser.add_argument(
        '--model',
        default='claude-sonnet-4-5-20250929',
        help='Anthropic model to use (default: claude-sonnet-4-5-20250929)'
    )

    parser.add_argument(
        '--verbose', '-v',
        action='store_true',
        help='Enable verbose output'
    )

    parser.add_argument(
        '--json',
        action='store_true',
        help='Output only JSON (no formatted output)'
    )

    args = parser.parse_args()

    # Load files
    if args.verbose:
        print(f"Loading original text: {args.original}")
    original_text = load_text_file(args.original)

    if args.verbose:
        print(f"Loading modified text: {args.modified}")
    modified_text = load_text_file(args.modified)

    # Load profile if provided
    profile = None
    if args.profile:
        if args.verbose:
            print(f"Loading author profile: {args.profile}")
        profile = load_author_profile(args.profile)

    # Run validation
    result = validate_style(
        original_text=original_text,
        modified_text=modified_text,
        profile=profile,
        model=args.model,
        verbose=args.verbose
    )

    # Output results
    result_dict = result_to_dict(result)

    if args.output:
        # Save to file
        output_path = Path(args.output)
        try:
            with open(output_path, 'w', encoding='utf-8') as f:
                json.dump(result_dict, f, indent=2, ensure_ascii=False)
            print(f"Results saved to: {args.output}")
        except Exception as e:
            print(f"Error saving output: {e}")
            sys.exit(1)

        if not args.json:
            print_result(result)

    elif args.json:
        # JSON only to stdout
        print(json.dumps(result_dict, indent=2, ensure_ascii=False))
    else:
        # Formatted output to console
        print_result(result)

        # Also print metadata
        print(f"\nMetadata:")
        print(f"  Model: {result.metadata.get('model')}")
        print(f"  Timestamp: {result.metadata.get('timestamp')}")
        print(f"  Original length: {result.metadata.get('original_length')} chars")
        print(f"  Modified length: {result.metadata.get('modified_length')} chars")

    # Exit with non-zero if severe drift
    if result.overall_score < 25:
        sys.exit(2)  # Severe drift
    elif result.overall_score < 50:
        sys.exit(1)  # Poor score

    sys.exit(0)


if __name__ == "__main__":
    main()