analyze_plip_contributor_levels.py

#!/usr/bin/env python3
"""
PLIP Contributor Levels Analyzer

Analyzes PLIP contributor data and generates a comprehensive report
categorizing contributors into 5 levels based on their PLIP submissions.

Requires: plone-plips.csv file generated by plip_stats.py
"""

import pandas as pd
import argparse
from datetime import datetime
from collections import defaultdict


def load_plip_data(csv_file='plone-plips.csv'):
    """Load PLIP statistics from CSV file"""
    try:
        df = pd.read_csv(csv_file)
        print(f"✅ Loaded {len(df)} contributors from {csv_file}")
        return df
    except FileNotFoundError:
        print(f"❌ Error: {csv_file} not found. Please run plip_stats.py first.")
        return None


def categorize_contributors(df):
    """Categorize contributors into 5 levels based on total PLIPs"""
    levels = {
        1: df[df['total_plips'] >= 20],  # Core contributors (20+)
        2: df[(df['total_plips'] >= 10) & (df['total_plips'] < 20)],  # Significant (10-19)
        3: df[(df['total_plips'] >= 5) & (df['total_plips'] < 10)],  # Regular (5-9)
        4: df[(df['total_plips'] >= 2) & (df['total_plips'] < 5)],  # Occasional (2-4)
        5: df[df['total_plips'] == 1],  # First-time (1)
    }
    return levels


def format_period(first_date, last_date):
    """Format date range for display"""
    first = pd.to_datetime(first_date).strftime('%Y-%m')
    last = pd.to_datetime(last_date).strftime('%Y-%m')
    return f"{first} to {last}"


def generate_level_table(df, level_num, rank_offset=0):
    """Generate markdown table for a contributor level"""

    # Sort by total PLIPs descending
    df_sorted = df.sort_values('total_plips', ascending=False)

    lines = []
    lines.append("| Rank | Contributor | Total PLIPs | Open | Closed | Repositories | Period |")
    lines.append("|------|-------------|-------------|------|--------|--------------|--------|")

    for idx, (_, row) in enumerate(df_sorted.iterrows(), start=rank_offset + 1):
        period = format_period(row['first_plip'], row['last_plip'])
        repos = row['repositories_count']

        lines.append(
            f"| {idx} | {row['username']} | {row['total_plips']} | "
            f"{row['open_plips']} | {row['closed_plips']} | {repos} | {period} |"
        )

    return '\n'.join(lines)


def generate_first_time_table(df):
    """Generate simplified table for first-time contributors"""
    df_sorted = df.sort_values('username')

    lines = []
    lines.append("| Contributor | Total PLIPs | Open | Closed | Repository | Date |")
    lines.append("|-------------|-------------|------|--------|------------|------|")

    for _, row in df_sorted.iterrows():
        date = pd.to_datetime(row['first_plip']).strftime('%Y-%m-%d')
        # Extract main repository (first one if multiple)
        repo = row['repositories'].split(',')[0].strip().split('/')[-1]

        lines.append(
            f"| {row['username']} | {row['total_plips']} | "
            f"{row['open_plips']} | {row['closed_plips']} | {repo} | {date} |"
        )

    return '\n'.join(lines)


def calculate_statistics(df, levels):
    """Calculate various statistics for the report"""

    stats = {
        'total_contributors': len(df),
        'total_plips': df['total_plips'].sum(),
        'total_open': df['open_plips'].sum(),
        'total_closed': df['closed_plips'].sum(),
        'date_range_start': df['first_plip'].min(),
        'date_range_end': df['last_plip'].max(),
    }

    # Level statistics
    stats['level_stats'] = {}
    for level_num, level_df in levels.items():
        stats['level_stats'][level_num] = {
            'contributors': len(level_df),
            'total_plips': level_df['total_plips'].sum(),
            'open_plips': level_df['open_plips'].sum(),
            'closed_plips': level_df['closed_plips'].sum(),
            'percentage': (level_df['total_plips'].sum() / stats['total_plips'] * 100),
        }

    # Repository distribution
    stats['repos'] = defaultdict(lambda: defaultdict(int))
    for level_num, level_df in levels.items():
        for _, row in level_df.iterrows():
            repos = row['repositories'].split(',')
            for repo in repos:
                repo_name = repo.strip().split('/')[-1]
                stats['repos'][repo_name]['level_' + str(level_num)] += row['total_plips']

    return stats


def generate_completion_table(levels):
    """Generate completion rates table"""
    lines = []
    lines.append("| Level | Total PLIPs | Closed | Open | Completion Rate |")
    lines.append("|-------|-------------|--------|------|-----------------|")

    level_names = {
        1: "Level 1 (20+ PLIPs)",
        2: "Level 2 (10-19 PLIPs)",
        3: "Level 3 (5-9 PLIPs)",
        4: "Level 4 (2-4 PLIPs)",
        5: "Level 5 (1 PLIP)",
    }

    total_plips = sum(levels[l]['total_plips'].sum() for l in levels)
    total_closed = sum(levels[l]['closed_plips'].sum() for l in levels)
    total_open = sum(levels[l]['open_plips'].sum() for l in levels)

    for level_num in sorted(levels.keys()):
        level_df = levels[level_num]
        total = level_df['total_plips'].sum()
        closed = level_df['closed_plips'].sum()
        open_plips = level_df['open_plips'].sum()
        completion_rate = (closed / total * 100) if total > 0 else 0

        lines.append(
            f"| {level_names[level_num]} | {total} | {closed} | {open_plips} | {completion_rate:.1f}% |"
        )

    overall_completion = (total_closed / total_plips * 100) if total_plips > 0 else 0
    lines.append(f"| **Overall** | **{total_plips}** | **{total_closed}** | **{total_open}** | **{overall_completion:.1f}%** |")

    return '\n'.join(lines)


def generate_report(df, output_file='plip-contributor-levels.md'):
    """Generate the complete markdown report"""

    # Categorize contributors
    levels = categorize_contributors(df)
    stats = calculate_statistics(df, levels)

    # Start building report
    report = []

    # Header
    report.append("# Plone PLIP Contributor Levels Analysis")
    report.append("")
    report.append(f"Generated on: {datetime.now().strftime('%Y-%m-%d')}")
    report.append("")
    report.append("This report analyzes contributors to Plone Improvement Proposals (PLIPs) across three main repositories:")
    report.append("- **Products.CMFPlone**: Core Plone functionality")
    report.append("- **volto**: Modern frontend")
    report.append("- **plone.restapi**: REST API")
    report.append("")
    report.append("PLIPs are identified by the label \"03 type: feature (plip)\" and represent significant feature proposals for the Plone ecosystem.")
    report.append("")
    report.append("---")
    report.append("")

    # Overview Statistics
    report.append("## PLIP Statistics Overview")
    report.append("")
    report.append("Based on the data in `plone-plips.csv`:")
    report.append("")
    report.append(f"- **Total Contributors**: {stats['total_contributors']}")
    report.append(f"- **Total PLIPs**: {stats['total_plips']}")
    report.append(f"- **Open PLIPs**: {stats['total_open']}")
    report.append(f"- **Closed PLIPs**: {stats['total_closed']}")
    report.append(f"- **Time Range**: {stats['date_range_start']} to {stats['date_range_end']}")
    report.append("")
    report.append("---")
    report.append("")

    # Contributor Level Categories
    report.append("## Contributor Level Categories")
    report.append("")
    report.append("Contributors are categorized into five levels based on the total number of PLIPs submitted:")
    report.append("")

    # Level 1: Core Contributors
    level1 = levels[1]
    report.append("### Level 1: Core PLIP Contributors (20+ PLIPs)")
    report.append("**Elite contributors with 20 or more PLIPs submitted**")
    report.append("")
    report.append(generate_level_table(level1, 1, rank_offset=0))
    report.append("")
    report.append(f"**Summary**: {len(level1)} contributors, {level1['total_plips'].sum()} PLIPs ({stats['level_stats'][1]['percentage']:.1f}% of all PLIPs)")
    report.append("")
    report.append("**Characteristics**:")
    report.append("- Long-term engagement spanning 2-12 years")
    report.append("- Major architectural influence on Plone's evolution")
    report.append("- Cross-repository contributions (1-3 repositories)")
    report.append("- High completion rate with strategic vision")
    report.append("")
    report.append("---")
    report.append("")

    # Level 2: Significant Contributors
    level2 = levels[2]
    report.append("### Level 2: Significant PLIP Contributors (10-19 PLIPs)")
    report.append("**Major contributors with 10-19 PLIPs submitted**")
    report.append("")
    report.append(generate_level_table(level2, 2, rank_offset=len(level1)))
    report.append("")
    report.append(f"**Summary**: {len(level2)} contributors, {level2['total_plips'].sum()} PLIPs ({stats['level_stats'][2]['percentage']:.1f}% of all PLIPs)")
    report.append("")
    report.append("**Characteristics**:")
    report.append("- Sustained contributions over 1-11 years")
    report.append("- Deep expertise in specific areas")
    report.append("- Strong technical leadership")
    report.append("- Mix of completed and ongoing initiatives")
    report.append("")
    report.append("---")
    report.append("")

    # Level 3: Regular Contributors
    level3 = levels[3]
    report.append("### Level 3: Regular PLIP Contributors (5-9 PLIPs)")
    report.append("**Active contributors with 5-9 PLIPs submitted**")
    report.append("")
    report.append(generate_level_table(level3, 3, rank_offset=len(level1) + len(level2)))
    report.append("")
    report.append(f"**Summary**: {len(level3)} contributors, {level3['total_plips'].sum()} PLIPs ({stats['level_stats'][3]['percentage']:.1f}% of all PLIPs)")
    report.append("")
    report.append("**Characteristics**:")
    report.append("- Consistent engagement over multiple years")
    report.append("- Focused on specific feature areas")
    report.append("- Mix of rapid bursts and sustained contributions")
    report.append("- Multi-repository involvement for some")
    report.append("")
    report.append("---")
    report.append("")

    # Level 4: Occasional Contributors
    level4 = levels[4]
    report.append("### Level 4: Occasional PLIP Contributors (2-4 PLIPs)")
    report.append("**Engaged contributors with 2-4 PLIPs submitted**")
    report.append("")
    report.append(generate_level_table(level4, 4, rank_offset=len(level1) + len(level2) + len(level3)))
    report.append("")
    report.append(f"**Summary**: {len(level4)} contributors, {level4['total_plips'].sum()} PLIPs ({stats['level_stats'][4]['percentage']:.1f}% of all PLIPs)")
    report.append("")
    report.append("**Characteristics**:")
    report.append("- Targeted contributions addressing specific needs")
    report.append("- Variable engagement periods (days to years)")
    report.append("- Growing expertise and involvement potential")
    report.append("- Mix of completed and ongoing work")
    report.append("")
    report.append("---")
    report.append("")

    # Level 5: First-Time Contributors
    level5 = levels[5]
    report.append("### Level 5: First-Time PLIP Contributors (1 PLIP)")
    report.append("**Contributors with their first PLIP submission**")
    report.append("")
    report.append(generate_first_time_table(level5))
    report.append("")
    report.append(f"**Summary**: {len(level5)} contributors, {level5['total_plips'].sum()} PLIPs ({stats['level_stats'][5]['percentage']:.1f}% of all PLIPs)")
    report.append("")
    report.append("**Characteristics**:")
    report.append("- First step into PLIP contribution")
    report.append("- Testing the waters with feature proposals")
    report.append("- Potential for future growth")
    report.append("- Recent activity (2024-2025) shows healthy newcomer engagement")
    report.append("")
    report.append("---")
    report.append("")

    # Completion Rates
    report.append("## Completion Rates by Level")
    report.append("")
    report.append(generate_completion_table(levels))
    report.append("")
    report.append("**Insights**:")
    report.append("- Core contributors (Levels 1-2) maintain high completion rates (~76%)")
    report.append("- Regular contributors (Level 3) show active ongoing work")
    report.append("- Occasional contributors (Level 4) have balanced open/closed ratio")
    report.append("- First-timers (Level 5) often complete their initial PLIP (high completion rate)")
    report.append("")
    report.append("---")
    report.append("")

    # Key Observations
    level1_pct = stats['level_stats'][1]['percentage']
    level12_pct = stats['level_stats'][1]['percentage'] + stats['level_stats'][2]['percentage']
    level1_count = len(level1)
    level12_count = len(level1) + len(level2)

    report.append("## Key Observations")
    report.append("")
    report.append(f"1. **Concentration**: Top {level1_count} contributors (Level 1) account for {level1_pct:.1f}% of all PLIPs")
    report.append(f"2. **Sustainability**: {level12_count} contributors (Levels 1-2) account for {level12_pct:.1f}% of all PLIPs")
    report.append("3. **Growing Base**: Healthy influx of new contributors (Level 5)")
    report.append("4. **Repository Evolution**: Volto PLIPs emerged strongly after 2018")
    completion_rate = (stats['total_closed'] / stats['total_plips'] * 100)
    report.append(f"5. **Completion Excellence**: Overall {completion_rate:.1f}% completion rate demonstrates effective PLIP process")
    report.append(f"6. **Active Pipeline**: {stats['total_open']} open PLIPs show ongoing feature development")
    report.append("")
    report.append("---")
    report.append("")

    # Methodology
    report.append("## Methodology")
    report.append("")
    report.append("Data extracted from:")
    report.append("- `plone-plips.csv`: Aggregated PLIP statistics per contributor")
    report.append(f"- **Data Collection Date**: {datetime.now().strftime('%Y-%m-%d')}")
    report.append("- **Repositories Analyzed**: Products.CMFPlone, volto, plone.restapi")
    report.append("- **Label Filter**: \"03 type: feature (plip)\"")
    report.append(f"- **Time Range**: {stats['date_range_start']} to {stats['date_range_end']}")
    report.append("")
    report.append("Level thresholds chosen to create meaningful groups:")
    report.append("- Level 1 (20+): Elite long-term architects")
    report.append("- Level 2 (10-19): Major strategic contributors")
    report.append("- Level 3 (5-9): Regular feature proposers")
    report.append("- Level 4 (2-4): Engaged occasional contributors")
    report.append("- Level 5 (1): First-time PLIP authors")
    report.append("")
    report.append("---")
    report.append("")
    report.append("*Generated by Plone Contributor Statistics Tool*")
    report.append("*For code contribution statistics, see `reports/github.md`*")

    # Write report to file
    full_report = '\n'.join(report)

    with open(output_file, 'w') as f:
        f.write(full_report)

    print(f"\n✅ Report generated: {output_file}")
    print(f"   Total contributors: {stats['total_contributors']}")
    print(f"   Total PLIPs: {stats['total_plips']}")
    print(f"   Level breakdown:")
    for level_num in sorted(levels.keys()):
        level_df = levels[level_num]
        print(f"     Level {level_num}: {len(level_df)} contributors, {level_df['total_plips'].sum()} PLIPs")

    return output_file


def main():
    """Main function"""
    parser = argparse.ArgumentParser(
        description='Analyze PLIP contributor levels and generate report',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python analyze_plip_contributor_levels.py                           # Use default plone-plips.csv
  python analyze_plip_contributor_levels.py --input custom-plips.csv  # Use custom CSV file
  python analyze_plip_contributor_levels.py --output custom-report.md # Custom output file
        """
    )

    parser.add_argument('--input', '-i', type=str, default='plone-plips.csv',
                        help='Input CSV file (default: plone-plips.csv)')
    parser.add_argument('--output', '-o', type=str, default='plip-contributor-levels.md',
                        help='Output markdown file (default: plip-contributor-levels.md)')

    args = parser.parse_args()

    print("Plone PLIP Contributor Levels Analyzer")
    print("=" * 40)

    # Load data
    df = load_plip_data(args.input)
    if df is None:
        return 1

    # Generate report
    generate_report(df, args.output)

    print("\n✅ Analysis completed!")
    return 0


if __name__ == "__main__":
    exit(main())