app.py

"""Flask web application for financial chart analysis with local LLM."""

# Fix UTF-8 encoding for Windows PowerShell
import sys
import io
if sys.stdout.encoding != 'utf-8':
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
if sys.stderr.encoding != 'utf-8':
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')

from flask import Flask, render_template, request, jsonify, redirect, url_for, session
from flask_login import login_required, current_user
from data_fetcher import FinancialDataFetcher, normalize_crypto_symbol
from chart_generator import ChartGenerator
from pattern_recognizer import PatternRecognizer
from llm_analyzer import LLMAnalyzer
from config import Config
from datetime import datetime, timedelta
import json
import traceback
import pandas as pd
import os

# Phase 2: Database and Authentication
try:
    from models import db, User, Watchlist, Alert, Portfolio, Transaction, OptionsPosition, AnalysisHistory, MLPattern, MLPrediction, PortfolioSnapshot, PortfolioAccount, Dividend, DiscussionThread, ThreadReply, ThreadVote, CopyTradingFollow
    from db_config import init_database
    from auth import init_auth, get_auth_routes, require_api_auth
    from monitoring_service import init_monitoring_service, get_monitoring_service
    from ml_pattern_detector import MLPatternDetector
    PHASE2_ENABLED = True
except ImportError as e:
    print(f"⚠ Phase 2 features not available: {e}")
    print("ℹ Run 'install_phase2.bat' to enable database and authentication")
    PHASE2_ENABLED = False
    # Define dummy decorator for when auth is not available
    def require_api_auth(f):
        """Dummy auth decorator when Phase 2 is disabled"""
        return f

# Phase 3: Advanced Trading Intelligence
try:
    from options_analyzer import OptionsAnalyzer
    from trading_time_analyzer import TradingTimeAnalyzer
    from sentiment_analyzer import SentimentAnalyzer
    from risk_analyzer import RiskAnalyzer
    PHASE3_ENABLED = True
except ImportError as e:
    print(f"⚠ Phase 3 features not available: {e}")
    print("ℹ Run 'install_phase3.bat' to enable advanced analysis")
    PHASE3_ENABLED = False

# Phase 4: Portfolio Management & Real-Time Intelligence
try:
    from volatility_monitor import VolatilityMonitor
    from portfolio_analyzer import PortfolioAnalyzer
    from smart_alerts import SmartAlertsEngine
    from alert_suggestions import AlertSuggestionEngine
    from news_fetcher import NewsFetcher
    from correlation_analyzer import CorrelationAnalyzer
    from trade_journal import TradeJournal
    from politician_trades import PoliticianTradeTracker
    PHASE4_ENABLED = True
except ImportError as e:
    print(f"⚠ Phase 4 features not available: {e}")
    print("ℹ Phase 4 requires Phase 2 database and Phase 3 analyzers")
    PHASE4_ENABLED = False


from werkzeug.middleware.proxy_fix import ProxyFix

def _get_current_user_id():
    """Safely get current user ID without requiring login_manager"""
    uid = session.get('user_id')
    if uid:
        return uid
    try:
        if hasattr(app, 'login_manager') and current_user.is_authenticated:
            return current_user.id
    except Exception:
        pass
    return None

app = Flask(__name__)
app.config.from_object(Config)
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)

# Add logging for debugging
import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

# Phase 2: Initialize database and authentication
if PHASE2_ENABLED:
    try:
        # Initialize database
        database = init_database(app)
        logger.info("✓ Database initialized")
        
        # Initialize authentication
        login_manager, google_oauth = init_auth(app)
        logger.info("✓ Authentication system initialized")
        
        # Initialize ML pattern detector
        ml_detector = MLPatternDetector()
        logger.info("✓ ML Pattern Detector initialized")
        
        # Initialize monitoring service with longer interval to reduce API load
        monitoring_svc = init_monitoring_service(app, check_interval=900)  # 15 minutes
        logger.info("✓ Real-time Monitoring Service initialized (15min interval)")
        
        # Register authentication routes
        auth_routes = get_auth_routes(google_oauth)
        
        @app.route('/login')
        def login():
            return auth_routes['login']()
        
        @app.route('/authorize')
        def authorize():
            return auth_routes['authorize']()
        
        @app.route('/logout')
        def logout():
            return auth_routes['logout']()
        
        logger.info("✓ Authentication routes registered")
        
    except Exception as e:
        logger.error(f"✗ Failed to initialize Phase 2 features: {e}")
        PHASE2_ENABLED = False
else:
    logger.info("ℹ Running in Phase 1 mode (no authentication)")

# Phase 3: Initialize advanced analyzers
if PHASE3_ENABLED:
    try:
        options_analyzer = OptionsAnalyzer()
        logger.info("✓ Options Analyzer initialized")
        
        trading_time_analyzer = TradingTimeAnalyzer()
        logger.info("✓ Trading Time Analyzer initialized")
        
        sentiment_analyzer = SentimentAnalyzer()
        logger.info("✓ Sentiment Analyzer initialized")
        
        risk_analyzer = RiskAnalyzer()
        logger.info("✓ Risk Analyzer initialized")
        
    except Exception as e:
        logger.error(f"✗ Failed to initialize Phase 3 features: {e}")
        PHASE3_ENABLED = False
else:
    logger.info("ℹ Phase 3 features not available")

# Phase 4: Initialize portfolio management & volatility monitoring
if PHASE4_ENABLED and PHASE2_ENABLED:
    try:
        volatility_monitor = VolatilityMonitor()
        logger.info("✓ Volatility Monitor initialized")
        
        portfolio_analyzer = PortfolioAnalyzer()
        logger.info("✓ Portfolio Analyzer initialized")
        
        smart_alerts = SmartAlertsEngine()
        logger.info("✓ Smart Alerts Engine initialized")
        
        # Initialize alert suggestions engine with other analyzers
        pattern_rec = PatternRecognizer() if 'PatternRecognizer' in dir() else None
        sentiment_an = sentiment_analyzer if PHASE3_ENABLED else None
        alert_suggestions = AlertSuggestionEngine(
            pattern_recognizer=pattern_rec,
            sentiment_analyzer=sentiment_an,
            volatility_monitor=volatility_monitor,
            portfolio_analyzer=portfolio_analyzer
        )
        logger.info("✓ AI Alert Suggestion Engine initialized")
        
        # Initialize news fetcher
        news_fetcher = NewsFetcher()
        logger.info("✓ News Fetcher initialized")
        
        # Initialize correlation analyzer
        correlation_analyzer = CorrelationAnalyzer()
        logger.info("✓ Correlation Analyzer initialized")
        
    except Exception as e:
        logger.error(f"✗ Failed to initialize Phase 4 features: {e}")
        PHASE4_ENABLED = False
else:
    if not PHASE2_ENABLED:
        logger.info("ℹ Phase 4 requires Phase 2 (database)")
    else:
        logger.info("ℹ Phase 4 features not available")

# Initialize components
try:
    data_fetcher = FinancialDataFetcher()
    logger.info("✓ FinancialDataFetcher initialized")
except Exception as e:
    logger.error(f"✗ Failed to initialize FinancialDataFetcher: {e}")
    raise

try:
    chart_generator = ChartGenerator()
    logger.info("✓ ChartGenerator initialized")
except Exception as e:
    logger.error(f"✗ Failed to initialize ChartGenerator: {e}")
    raise

try:
    pattern_recognizer = PatternRecognizer()
    logger.info("✓ PatternRecognizer initialized")
except Exception as e:
    logger.error(f"✗ Failed to initialize PatternRecognizer: {e}")
    raise

try:
    llm_analyzer = LLMAnalyzer()
    logger.info("✓ LLMAnalyzer initialized")
except Exception as e:
    logger.error(f"✗ Failed to initialize LLMAnalyzer: {e}")
    raise

# Initialize Trade Journal after LLM Analyzer (Feature #5)
if PHASE4_ENABLED and PHASE2_ENABLED:
    try:
        trade_journal = TradeJournal(llm_analyzer)
        logger.info("✓ Trade Journal initialized")
    except Exception as e:
        logger.error(f"✗ Failed to initialize Trade Journal: {e}")


@app.before_request
def log_request():
    """Log all incoming requests."""
    logger.info(f"→ {request.method} {request.path}")


@app.after_request
def log_response(response):
    """Log all responses."""
    logger.info(f"← {request.method} {request.path} → {response.status_code}")
    return response


@app.route('/')
def index():
    """Render the main dashboard."""
    logger.debug("Rendering dashboard.html as main page")
    if PHASE2_ENABLED:
        # Require login for Phase 2
        if not current_user.is_authenticated:
            return redirect(url_for('login'))
    return render_template('dashboard.html')


@app.route('/api/test/yfinance', methods=['GET'])
def test_yfinance():
    """Test endpoint to verify Yahoo Finance connectivity."""
    try:
        symbol = request.args.get('symbol', 'AAPL')
        logger.info(f"Testing Yahoo Finance with symbol: {symbol}")
        
        # Try to fetch minimal data
        stock_data = data_fetcher.fetch_stock_data(symbol, period='5d', interval='1d')
        
        if stock_data is None:
            return jsonify({
                'status': 'failed',
                'error': 'No data returned from Yahoo Finance',
                'symbol': symbol,
                'message': 'Yahoo Finance API may be down or rate limiting'
            }), 503
        
        if stock_data.empty:
            return jsonify({
                'status': 'failed',
                'error': 'Empty data returned',
                'symbol': symbol,
                'message': 'Symbol may be invalid or data not available'
            }), 404
        
        return jsonify({
            'status': 'success',
            'symbol': symbol,
            'rows': len(stock_data),
            'columns': list(stock_data.columns),
            'latest_price': float(stock_data['Close'].iloc[-1]) if 'Close' in stock_data.columns else None,
            'message': 'Yahoo Finance connection working'
        }), 200
        
    except Exception as e:
        logger.error(f"Error testing Yahoo Finance: {e}", exc_info=True)
        return jsonify({
            'status': 'error',
            'error': str(e),
            'message': 'Exception during Yahoo Finance test'
        }), 500


@app.route('/dashboard')
def dashboard():
    """Render the enhanced dashboard with watchlist, alerts, and comparison features."""
    logger.debug("Rendering dashboard.html")
    if PHASE2_ENABLED:
        # Require login for Phase 2
        if not current_user.is_authenticated:
            return redirect(url_for('login'))
    return render_template('dashboard.html')


@app.route('/simple')
def simple():
    """Render the simple/classic interface."""
    logger.debug("Rendering index.html (simple interface)")
    return render_template('index.html')


@app.route('/portfolio')
@login_required
def portfolio():
    """Render the portfolio analytics dashboard (Phase 4)."""
    logger.debug("Rendering portfolio.html")
    if not PHASE4_ENABLED:
        return "Portfolio features are not enabled", 503
    return render_template('portfolio.html')


@app.route('/copytrading')
@login_required
def copytrading():
    """Render the copy trading research page."""
    logger.debug("Rendering copytrading.html")
    return render_template('copytrading.html')


@app.route('/admin')
@login_required
def admin_page():
    """Render the admin dashboard (admin only)."""
    if not current_user.is_admin():
        return redirect(url_for('portfolio'))
    return render_template('admin.html')


@app.route('/community')
@login_required
def community_page():
    """Render the community discussion page."""
    return render_template('community.html')


# ===================== ADMIN API =====================

@app.route('/api/admin/users', methods=['GET'])
@require_api_auth
def admin_list_users():
    """List all users (admin only)."""
    if not current_user.is_admin():
        return jsonify({'error': 'Unauthorized'}), 403
    users = User.query.order_by(User.created_at.desc()).all()
    return jsonify({'users': [u.to_dict() for u in users]})


@app.route('/api/admin/users/<int:user_id>/role', methods=['PUT'])
@require_api_auth
def admin_change_role(user_id):
    """Change a user's role (admin only)."""
    if not current_user.is_admin():
        return jsonify({'error': 'Unauthorized'}), 403
    data = request.get_json()
    role = data.get('role')
    if role not in ('user', 'moderator', 'admin'):
        return jsonify({'error': 'Invalid role'}), 400
    user = User.query.get_or_404(user_id)
    if user.id == current_user.id:
        return jsonify({'error': 'Cannot change your own role'}), 400
    user.role = role
    db.session.commit()
    return jsonify({'success': True, 'user': user.to_dict()})


@app.route('/api/admin/users/<int:user_id>/active', methods=['PUT'])
@require_api_auth
def admin_toggle_active(user_id):
    """Enable/disable a user account (admin only)."""
    if not current_user.is_admin():
        return jsonify({'error': 'Unauthorized'}), 403
    user = User.query.get_or_404(user_id)
    if user.id == current_user.id:
        return jsonify({'error': 'Cannot disable your own account'}), 400
    data = request.get_json()
    user.is_active = data.get('is_active', True)
    db.session.commit()
    return jsonify({'success': True, 'user': user.to_dict()})


@app.route('/api/admin/stats', methods=['GET'])
@require_api_auth
def admin_stats():
    """Get admin dashboard stats."""
    if not current_user.is_admin():
        return jsonify({'error': 'Unauthorized'}), 403
    total_users = User.query.count()
    active_users = User.query.filter_by(is_active=True).count()
    total_threads = DiscussionThread.query.count()
    total_holdings = Portfolio.query.count()
    return jsonify({
        'total_users': total_users,
        'active_users': active_users,
        'total_threads': total_threads,
        'total_holdings': total_holdings,
    })


# ===================== COMMUNITY API =====================

@app.route('/api/community/threads', methods=['GET'])
@require_api_auth
def list_threads():
    """List discussion threads with optional category filter."""
    category = request.args.get('category', 'all')
    page = int(request.args.get('page', 1))
    per_page = int(request.args.get('per_page', 20))
    
    query = DiscussionThread.query
    if category != 'all':
        query = query.filter_by(category=category)
    query = query.order_by(DiscussionThread.pinned.desc(), DiscussionThread.created_at.desc())
    pagination = query.paginate(page=page, per_page=per_page, error_out=False)
    
    threads = []
    for t in pagination.items:
        td = t.to_dict()
        td['reply_count'] = ThreadReply.query.filter_by(thread_id=t.id).count()
        threads.append(td)
    
    return jsonify({
        'threads': threads,
        'total': pagination.total,
        'pages': pagination.pages,
        'current_page': page,
    })


@app.route('/api/community/threads', methods=['POST'])
@require_api_auth
def create_thread():
    """Create a new discussion thread."""
    data = request.get_json()
    title = (data.get('title') or '').strip()
    body = (data.get('body') or '').strip()
    if not title or not body:
        return jsonify({'error': 'Title and body are required'}), 400
    
    thread = DiscussionThread(
        user_id=current_user.id,
        title=title[:200],
        body=body[:10000],
        symbol=(data.get('symbol') or '').upper()[:10] or None,
        category=data.get('category', 'general'),
    )
    db.session.add(thread)
    db.session.commit()
    return jsonify({'success': True, 'thread': thread.to_dict()}), 201


@app.route('/api/community/threads/<int:thread_id>', methods=['GET'])
@require_api_auth
def get_thread(thread_id):
    """Get a single thread with its replies."""
    thread = DiscussionThread.query.get_or_404(thread_id)
    thread.views = (thread.views or 0) + 1
    db.session.commit()
    
    td = thread.to_dict()
    replies = ThreadReply.query.filter_by(thread_id=thread_id)\
        .order_by(ThreadReply.created_at.asc()).all()
    td['replies'] = [r.to_dict() for r in replies]
    
    # Include current user's votes
    votes = ThreadVote.query.filter_by(user_id=current_user.id, thread_id=thread_id).all()
    user_votes = {}
    for v in votes:
        key = f'reply_{v.reply_id}' if v.reply_id else 'thread'
        user_votes[key] = v.vote
    td['user_votes'] = user_votes
    
    return jsonify(td)


@app.route('/api/community/threads/<int:thread_id>/replies', methods=['POST'])
@require_api_auth
def add_reply(thread_id):
    """Add a reply to a thread."""
    thread = DiscussionThread.query.get_or_404(thread_id)
    if thread.locked:
        return jsonify({'error': 'Thread is locked'}), 403
    
    data = request.get_json()
    body = (data.get('body') or '').strip()
    if not body:
        return jsonify({'error': 'Reply body is required'}), 400
    
    reply = ThreadReply(
        thread_id=thread_id,
        user_id=current_user.id,
        body=body[:10000],
    )
    db.session.add(reply)
    thread.updated_at = datetime.utcnow()
    db.session.commit()
    return jsonify({'success': True, 'reply': reply.to_dict()}), 201


@app.route('/api/community/threads/<int:thread_id>/vote', methods=['POST'])
@require_api_auth
def vote_thread(thread_id):
    """Upvote/downvote a thread."""
    thread = DiscussionThread.query.get_or_404(thread_id)
    data = request.get_json()
    vote_val = data.get('vote', 1)
    if vote_val not in (1, -1):
        return jsonify({'error': 'Vote must be 1 or -1'}), 400
    
    existing = ThreadVote.query.filter_by(
        user_id=current_user.id, thread_id=thread_id, reply_id=None
    ).first()
    
    if existing:
        if existing.vote == vote_val:
            # Remove vote
            thread.upvotes = (thread.upvotes or 0) - vote_val
            db.session.delete(existing)
        else:
            # Change vote
            thread.upvotes = (thread.upvotes or 0) + (vote_val - existing.vote)
            existing.vote = vote_val
    else:
        thread.upvotes = (thread.upvotes or 0) + vote_val
        db.session.add(ThreadVote(
            user_id=current_user.id, thread_id=thread_id, reply_id=None, vote=vote_val
        ))
    
    db.session.commit()
    return jsonify({'success': True, 'upvotes': thread.upvotes})


@app.route('/api/community/replies/<int:reply_id>/vote', methods=['POST'])
@require_api_auth
def vote_reply(reply_id):
    """Upvote/downvote a reply."""
    reply = ThreadReply.query.get_or_404(reply_id)
    data = request.get_json()
    vote_val = data.get('vote', 1)
    if vote_val not in (1, -1):
        return jsonify({'error': 'Vote must be 1 or -1'}), 400
    
    existing = ThreadVote.query.filter_by(
        user_id=current_user.id, thread_id=reply.thread_id, reply_id=reply_id
    ).first()
    
    if existing:
        if existing.vote == vote_val:
            reply.upvotes = (reply.upvotes or 0) - vote_val
            db.session.delete(existing)
        else:
            reply.upvotes = (reply.upvotes or 0) + (vote_val - existing.vote)
            existing.vote = vote_val
    else:
        reply.upvotes = (reply.upvotes or 0) + vote_val
        db.session.add(ThreadVote(
            user_id=current_user.id, thread_id=reply.thread_id, reply_id=reply_id, vote=vote_val
        ))
    
    db.session.commit()
    return jsonify({'success': True, 'upvotes': reply.upvotes})


@app.route('/api/community/threads/<int:thread_id>', methods=['DELETE'])
@require_api_auth
def delete_thread(thread_id):
    """Delete a thread (author or moderator)."""
    thread = DiscussionThread.query.get_or_404(thread_id)
    if thread.user_id != current_user.id and not current_user.is_moderator():
        return jsonify({'error': 'Unauthorized'}), 403
    db.session.delete(thread)
    db.session.commit()
    return jsonify({'success': True})


@app.route('/api/community/threads/<int:thread_id>/lock', methods=['PUT'])
@require_api_auth
def lock_thread(thread_id):
    """Lock/unlock a thread (moderator only)."""
    if not current_user.is_moderator():
        return jsonify({'error': 'Unauthorized'}), 403
    thread = DiscussionThread.query.get_or_404(thread_id)
    data = request.get_json()
    thread.locked = data.get('locked', True)
    db.session.commit()
    return jsonify({'success': True, 'locked': thread.locked})


@app.route('/api/community/threads/<int:thread_id>/pin', methods=['PUT'])
@require_api_auth
def pin_thread(thread_id):
    """Pin/unpin a thread (moderator only)."""
    if not current_user.is_moderator():
        return jsonify({'error': 'Unauthorized'}), 403
    thread = DiscussionThread.query.get_or_404(thread_id)
    data = request.get_json()
    thread.pinned = data.get('pinned', True)
    db.session.commit()
    return jsonify({'success': True, 'pinned': thread.pinned})


@app.route('/api/community/online', methods=['GET'])
@require_api_auth
def online_users():
    """List recently active users (active in last 5 minutes)."""
    cutoff = datetime.utcnow() - timedelta(minutes=5)
    users = User.query.filter(User.last_active >= cutoff).all()
    return jsonify({'online': [{'id': u.id, 'name': u.name, 'picture_url': u.picture_url} for u in users]})


# ===================== COPY TRADING MEMBER API =====================

@app.route('/api/copytrading/members', methods=['GET'])
@require_api_auth
def list_copy_trading_members():
    """List users who have opted in to member copy trading."""
    members = User.query.filter_by(copy_trading_enabled=True, is_active=True).all()
    result = []
    for m in members:
        holdings_count = Portfolio.query.filter_by(user_id=m.id).count()
        follower_count = CopyTradingFollow.query.filter_by(leader_id=m.id).count()
        is_following = CopyTradingFollow.query.filter_by(
            follower_id=current_user.id, leader_id=m.id
        ).first() is not None
        result.append({
            'id': m.id,
            'name': m.name,
            'picture_url': m.picture_url,
            'bio': m.bio,
            'holdings_count': holdings_count,
            'follower_count': follower_count,
            'is_following': is_following,
            'member_since': m.created_at.isoformat() if m.created_at else None,
        })
    return jsonify({'members': result})


@app.route('/api/copytrading/opt-in', methods=['POST'])
@require_api_auth
def toggle_copy_trading():
    """Toggle copy trading opt-in for current user."""
    data = request.get_json()
    current_user.copy_trading_enabled = data.get('enabled', False)
    if data.get('bio') is not None:
        current_user.bio = (data['bio'] or '')[:500]
    db.session.commit()
    return jsonify({'success': True, 'enabled': current_user.copy_trading_enabled})


@app.route('/api/copytrading/follow/<int:leader_id>', methods=['POST'])
@require_api_auth
def follow_member(leader_id):
    """Follow a member for copy trading."""
    if leader_id == current_user.id:
        return jsonify({'error': 'Cannot follow yourself'}), 400
    leader = User.query.get_or_404(leader_id)
    if not leader.copy_trading_enabled:
        return jsonify({'error': 'This user has not enabled copy trading'}), 400
    existing = CopyTradingFollow.query.filter_by(
        follower_id=current_user.id, leader_id=leader_id
    ).first()
    if existing:
        return jsonify({'error': 'Already following'}), 400
    db.session.add(CopyTradingFollow(follower_id=current_user.id, leader_id=leader_id))
    db.session.commit()
    return jsonify({'success': True})


@app.route('/api/copytrading/unfollow/<int:leader_id>', methods=['POST'])
@require_api_auth
def unfollow_member(leader_id):
    """Unfollow a copy trading member."""
    follow = CopyTradingFollow.query.filter_by(
        follower_id=current_user.id, leader_id=leader_id
    ).first()
    if follow:
        db.session.delete(follow)
        db.session.commit()
    return jsonify({'success': True})


@app.route('/api/copytrading/status', methods=['GET'])
@require_api_auth
def copy_trading_status():
    """Get current user's copy trading status."""
    return jsonify({
        'enabled': current_user.copy_trading_enabled or False,
        'bio': current_user.bio or '',
        'follower_count': CopyTradingFollow.query.filter_by(leader_id=current_user.id).count(),
        'following_count': CopyTradingFollow.query.filter_by(follower_id=current_user.id).count(),
    })


@app.route('/api/user/heartbeat', methods=['POST'])
@require_api_auth
def user_heartbeat():
    """Update user's last active timestamp."""
    current_user.last_active = datetime.utcnow()
    db.session.commit()
    return jsonify({'success': True})


@app.route('/api/user/preferences', methods=['GET'])
@require_api_auth
def get_preferences():
    """Get current user's preferences."""
    prefs = current_user.preferences or {}
    return jsonify({'preferences': prefs})


@app.route('/api/user/preferences', methods=['PUT'])
@require_api_auth
def save_preferences():
    """Save current user's preferences."""
    data = request.get_json()
    if not isinstance(data, dict):
        return jsonify({'error': 'Invalid data'}), 400
    # Only allow known safe keys
    allowed = {'darkMode', 'autoRefreshWatchlist', 'notificationsEnabled',
                'defaultPeriod', 'defaultChartType'}
    prefs = {k: v for k, v in data.items() if k in allowed}
    current_user.preferences = {**(current_user.preferences or {}), **prefs}
    db.session.commit()
    return jsonify({'success': True, 'preferences': current_user.preferences})


@app.route('/api/analyze', methods=['POST'])
def analyze_stock():
    """Analyze a stock symbol and return comprehensive results."""
    logger.info("Starting stock analysis...")
    try:
        data = request.get_json()
        if data is None:
            logger.error("No JSON data in request")
            return jsonify({'error': 'No JSON data provided'}), 400
        
        symbol = data.get('symbol', 'AAPL').upper()
        logger.info(f"Analyzing symbol: {symbol}")
        period = data.get('period', '6mo')
        interval = data.get('interval', '1d')
        chart_type = data.get('chart_type', 'candlestick')
        
        # Fetch stock data
        logger.info(f"Fetching data for {symbol} (period={period}, interval={interval})")
        stock_data = data_fetcher.fetch_stock_data(symbol, period, interval)
        
        if stock_data is None:
            logger.error(f"No data returned for {symbol} - check Yahoo Finance API status")
            return jsonify({
                'error': f'Unable to fetch data for {symbol}. Yahoo Finance may be rate limiting or the symbol may be invalid.',
                'symbol': symbol,
                'suggestion': 'Try again in a few moments or verify the symbol is correct.'
            }), 503
        
        if stock_data.empty:
            logger.error(f"Empty dataframe for {symbol}")
            return jsonify({
                'error': f'No data available for {symbol}',
                'symbol': symbol,
                'suggestion': 'Please verify the symbol is correct.'
            }), 404
        
        logger.info(f"Data fetched: {len(stock_data)} rows")
        
        # Calculate indicators
        logger.debug("Calculating technical indicators...")
        try:
            stock_data = pattern_recognizer.calculate_indicators(stock_data)
        except Exception as e:
            logger.error(f"Error calculating indicators: {e}")
            return jsonify({'error': f'Error calculating indicators: {str(e)}'}), 500
        
        # Generate chart
        logger.debug(f"Generating {chart_type} chart...")
        try:
            if chart_type == 'line':
                chart_base64 = chart_generator.generate_line_chart(
                    stock_data, symbol
                )
            elif chart_type == 'volume':
                chart_base64 = chart_generator.generate_volume_chart(
                    stock_data, symbol
                )
            else:  # Default to candlestick
                chart_base64 = chart_generator.generate_candlestick_chart(
                    stock_data, symbol
                )
        except Exception as e:
            logger.error(f"Error generating chart: {e}")
            # Continue without chart
            chart_base64 = ""
        
        # Detect patterns
        logger.debug("Detecting patterns...")
        try:
            candlestick_patterns = pattern_recognizer.detect_candlestick_patterns(stock_data)
            support_resistance = pattern_recognizer.detect_support_resistance(stock_data)
            trend = pattern_recognizer.detect_trend(stock_data)
            signals = pattern_recognizer.generate_signals(stock_data)
        except Exception as e:
            logger.error(f"Error detecting patterns: {e}")
            candlestick_patterns = []
            support_resistance = {'support': [], 'resistance': []}
            trend = 'unknown'
            signals = {}
        
        # Get latest values
        try:
            latest = stock_data.iloc[-1]
            prev = stock_data.iloc[-2] if len(stock_data) > 1 else latest
        except Exception as e:
            logger.error(f"Error accessing stock data: {e}")
            return jsonify({'error': f'Error processing stock data: {str(e)}'}), 500
        
        # Prepare indicators summary
        try:
            indicators = {
                'RSI': round(float(latest.get('RSI', 0)), 2) if pd.notna(latest.get('RSI')) else 0,
                'MACD': round(float(latest.get('MACD', 0)), 4) if pd.notna(latest.get('MACD')) else 0,
                'MACD_Signal': round(float(latest.get('MACD_Signal', 0)), 4) if pd.notna(latest.get('MACD_Signal')) else 0,
                'SMA_20': round(float(latest.get('SMA_20', 0)), 2) if pd.notna(latest.get('SMA_20')) else 0,
                'SMA_50': round(float(latest.get('SMA_50', 0)), 2) if pd.notna(latest.get('SMA_50')) else 0,
                'BB_High': round(float(latest.get('BB_High', 0)), 2) if pd.notna(latest.get('BB_High')) else 0,
                'BB_Low': round(float(latest.get('BB_Low', 0)), 2) if pd.notna(latest.get('BB_Low')) else 0,
            }
        except Exception as e:
            logger.error(f"Error preparing indicators: {e}")
            indicators = {'RSI': 0, 'MACD': 0, 'MACD_Signal': 0, 'SMA_20': 0, 'SMA_50': 0, 'BB_High': 0, 'BB_Low': 0}
        
        # LLM analysis
        logger.debug("Running LLM analysis...")
        try:
            llm_analysis = llm_analyzer.analyze_chart(
                chart_base64,
                symbol,
                indicators,
                candlestick_patterns,
                context=f"Current trend: {trend}"
            )
        except Exception as e:
            logger.error(f"Error in LLM analysis: {e}")
            llm_analysis = f"Error performing AI analysis: {str(e)}"
        
        # ML Pattern Detection (Phase 2)
        ml_patterns = []
        ml_prediction = None
        if PHASE2_ENABLED and ml_detector:
            logger.debug("Detecting ML patterns...")
            try:
                ml_patterns = ml_detector.detect_patterns(stock_data, symbol)
                logger.info(f"Detected {len(ml_patterns)} ML patterns")
            except Exception as e:
                logger.error(f"Error detecting ML patterns: {e}")
            
            logger.debug("Generating ML prediction...")
            try:
                ml_prediction = ml_detector.make_prediction(stock_data, symbol)
                if ml_prediction:
                    logger.info(f"ML Prediction: {ml_prediction['predicted_direction']} ({ml_prediction['confidence']*100:.1f}% confidence)")
            except Exception as e:
                logger.error(f"Error making ML prediction: {e}")
        
        # Phase 3: Advanced Analysis
        sentiment_analysis = None
        risk_analysis = None
        timing_analysis = None
        if PHASE3_ENABLED:
            logger.debug("Running Phase 3 analysis...")
            try:
                sentiment_analysis = sentiment_analyzer.analyze_sentiment(stock_data, symbol)
                logger.info(f"Sentiment: {sentiment_analysis.get('sentiment_label', 'N/A')}")
            except Exception as e:
                logger.error(f"Error in sentiment analysis: {e}")
            
            try:
                risk_analysis = risk_analyzer.comprehensive_risk_analysis(stock_data, symbol)
                logger.info(f"Risk Score: {risk_analysis.get('overall_risk_score', 'N/A')}")
            except Exception as e:
                logger.error(f"Error in risk analysis: {e}")
            
            try:
                timing_analysis = {
                    'entry': trading_time_analyzer.analyze_entry_points(stock_data, symbol),
                    'exit': trading_time_analyzer.analyze_exit_points(stock_data, symbol),
                    'volume': trading_time_analyzer.analyze_volume_profile(stock_data, symbol)
                }
                logger.info(f"Entry Score: {timing_analysis['entry'].get('entry_score', 'N/A')}")
            except Exception as e:
                logger.error(f"Error in timing analysis: {e}")
        
        # Company info
        logger.debug("Fetching company info...")
        try:
            company_info = data_fetcher.get_company_info(symbol)
        except Exception as e:
            logger.error(f"Error fetching company info: {e}")
            company_info = {'name': symbol, 'sector': 'N/A', 'industry': 'N/A', 'market_cap': 'N/A', 'description': 'N/A'}
        
        # Prepare response
        try:
            response = {
                'symbol': symbol,
                'company_info': company_info,
                'current_price': round(float(latest['Close']), 2),
                'price_change': round(float(latest['Close'] - prev['Close']), 2),
                'price_change_pct': round(float((latest['Close'] - prev['Close']) / prev['Close'] * 100), 2) if float(prev['Close']) != 0 else 0,
                'volume': int(latest['Volume']) if not pd.isna(latest['Volume']) else 0,
                'trend': trend,
                'chart': f'data:image/png;base64,{chart_base64}' if chart_base64 else '',
                'indicators': indicators,
                'signals': signals,
                'support_resistance': {
                    'support': [round(float(x), 2) for x in support_resistance['support'][-3:]] if support_resistance['support'] else [],
                    'resistance': [round(float(x), 2) for x in support_resistance['resistance'][-3:]] if support_resistance['resistance'] else []
                },
                'patterns': candlestick_patterns,
                'ml_patterns': ml_patterns,
                'ml_prediction': ml_prediction,
                'sentiment': sentiment_analysis,
                'risk': risk_analysis,
                'timing': timing_analysis,
                'llm_analysis': llm_analysis
            }
        except Exception as e:
            logger.error(f"Error preparing response: {e}")
            logger.error(traceback.format_exc())
            return jsonify({'error': f'Error preparing response: {str(e)}'}), 500
        
        logger.info(f"✓ Analysis complete for {symbol}")
        return jsonify(response)
        
    except Exception as e:
        logger.error(f"Error analyzing stock: {e}")
        logger.error(traceback.format_exc())
        return jsonify({'error': str(e), 'traceback': traceback.format_exc()}), 500


@app.route('/api/technical-chart', methods=['POST'])
def generate_technical_chart():
    """Generate technical chart with customizable indicators."""
    try:
        data = request.get_json()
        if data is None:
            return jsonify({'error': 'No JSON data provided'}), 400
        
        symbol = data.get('symbol', 'AAPL').upper()
        period = data.get('period', '6mo')
        interval = data.get('interval', '1d')
        indicators = data.get('indicators', [])  # ['rsi', 'macd', 'bb', 'ma']
        
        logger.info(f"Generating technical chart for {symbol} with indicators: {indicators}")
        
        # Fetch stock data
        stock_data = data_fetcher.fetch_stock_data(symbol, period, interval)
        
        if stock_data is None or stock_data.empty:
            return jsonify({'error': f'No data found for {symbol}'}), 404
        
        # Generate technical chart
        chart_base64 = chart_generator.generate_technical_chart(
            stock_data, symbol, indicators
        )
        
        # Get current stats
        latest = stock_data.iloc[-1]
        current_price = float(latest['Close'])
        prev_close = float(stock_data.iloc[-2]['Close']) if len(stock_data) > 1 else current_price
        price_change = current_price - prev_close
        price_change_pct = (price_change / prev_close * 100) if prev_close != 0 else 0
        
        return jsonify({
            'symbol': symbol,
            'chart': f'data:image/png;base64,{chart_base64}',
            'current_price': round(current_price, 2),
            'price_change': round(price_change, 2),
            'price_change_pct': round(price_change_pct, 2),
            'indicators': indicators
        })
        
    except Exception as e:
        logger.error(f"Error generating technical chart: {e}")
        logger.error(traceback.format_exc())
        return jsonify({'error': str(e)}), 500


@app.route('/health', methods=['GET'])
@app.route('/api/health', methods=['GET'])
def health_check():
    """Simple health check endpoint."""
    git_rev = os.environ.get('GIT_COMMIT', 'unknown')
    logger.debug("Health check requested")
    return jsonify({
        'status': 'healthy',
        'message': 'Server is running',
        'git_commit': git_rev,
        'routes_registered': [str(rule) for rule in app.url_map.iter_rules()]
    })


@app.route('/api/import-data', methods=['POST'])
def import_data():
    """One-time data import from SQLite export JSON. Remove after use."""
    import_key = os.environ.get('IMPORT_KEY', '')
    provided_key = request.headers.get('X-Import-Key', '')
    if not import_key or provided_key != import_key:
        return jsonify({'error': 'Unauthorized'}), 401

    if not PHASE2_ENABLED:
        return jsonify({'error': 'Database not initialized'}), 503

    from models import db
    from sqlalchemy import text

    data = request.get_json()
    if not data:
        return jsonify({'error': 'No JSON data provided'}), 400

    results = {}
    # Import order matters for foreign keys
    table_order = [
        'users', 'user_sessions', 'watchlist', 'portfolio_accounts', 'portfolio',
        'transactions', 'options_positions', 'alerts', 'market_conditions',
        'portfolio_snapshots', 'alert_suggestions', 'dividends',
        'discussion_threads', 'thread_replies', 'thread_votes', 'copy_trading_follows',
    ]

    for table in table_order:
        if table not in data:
            continue
        rows = data[table]
        if not rows:
            continue

        imported = 0
        for row in rows:
            cols = list(row.keys())
            col_list = ', '.join(f'"{c}"' for c in cols)
            placeholders = ', '.join(f':{c}' for c in cols)
            sql = text(f'INSERT INTO "{table}" ({col_list}) VALUES ({placeholders}) ON CONFLICT DO NOTHING')
            try:
                db.session.execute(sql, row)
                imported += 1
            except Exception as e:
                db.session.rollback()
                logger.warning(f"Import error in {table}: {e}")
                continue

        # Reset sequence
        try:
            db.session.execute(text(
                f"SELECT setval('{table}_id_seq', COALESCE((SELECT MAX(id) FROM \"{table}\"), 0) + 1, false)"
            ))
        except Exception:
            pass

        db.session.commit()
        results[table] = f"{imported}/{len(rows)}"

    return jsonify({'results': results})


@app.route('/api/version', methods=['GET'])
def version():
    """Return the deployed commit SHA for deployment verification."""
    return jsonify({'commit': os.environ.get('GIT_COMMIT', 'unknown')})


@app.route('/api/status', methods=['GET'])
def check_status():
    """Check system status."""
    ollama_status = llm_analyzer.check_ollama_status()
    return jsonify({
        'status': 'ok',
        'ollama': ollama_status
    })


@app.route('/api/pattern-info/<pattern_name>', methods=['GET'])
def get_pattern_info(pattern_name):
    """Get information about a specific pattern."""
    try:
        explanation = llm_analyzer.get_pattern_explanation(pattern_name)
        return jsonify({
            'pattern': pattern_name,
            'explanation': explanation
        })
    except Exception as e:
        return jsonify({'error': str(e)}), 500


@app.route('/api/compare', methods=['POST'])
def compare_stocks():
    """Compare multiple stocks with optional chart."""
    try:
        data = request.get_json()
        symbols = data.get('symbols', [])
        period = data.get('period', '6mo')
        normalize = data.get('normalize', True)
        include_chart = data.get('include_chart', False)
        
        if not symbols:
            return jsonify({'error': 'No symbols provided'}), 400
        
        if len(symbols) > 10:
            return jsonify({'error': 'Maximum 10 symbols allowed'}), 400
        
        results = {}
        data_dict = {}
        
        for symbol in symbols:
            symbol_upper = symbol.upper()
            stock_data = data_fetcher.fetch_stock_data(symbol_upper, period)
            
            if stock_data is not None and not stock_data.empty:
                data_dict[symbol_upper] = stock_data
                
                latest = stock_data.iloc[-1]
                first = stock_data.iloc[0]
                
                # Calculate volatility safely
                volatility = 0
                if 'Returns' in stock_data.columns:
                    vol_value = stock_data['Returns'].std() * 100
                    volatility = round(float(vol_value), 2) if pd.notna(vol_value) else 0
                
                # Calculate return safely
                return_pct = 0
                if pd.notna(latest['Close']) and pd.notna(first['Close']) and first['Close'] != 0:
                    return_pct = round(float((latest['Close'] - first['Close']) / first['Close'] * 100), 2)
                
                results[symbol_upper] = {
                    'current_price': round(float(latest['Close']), 2) if pd.notna(latest['Close']) else 0,
                    'return_pct': return_pct,
                    'volatility': volatility,
                    'volume': int(latest['Volume']) if pd.notna(latest['Volume']) else 0,
                    'high': round(float(stock_data['High'].max()), 2) if pd.notna(stock_data['High'].max()) else 0,
                    'low': round(float(stock_data['Low'].min()), 2) if pd.notna(stock_data['Low'].min()) else 0
                }
        
        response = {'symbols': results}
        
        # Generate comparison chart if requested
        if include_chart and data_dict:
            try:
                chart_base64 = chart_generator.generate_comparison_chart(
                    data_dict, 
                    normalize=normalize
                )
                response['chart'] = f'data:image/png;base64,{chart_base64}'
            except Exception as e:
                logger.error(f"Error generating comparison chart: {e}")
                response['chart'] = None
        
        return jsonify(response)
        
    except Exception as e:
        logger.error(f"Error in compare_stocks: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/watchlist', methods=['GET', 'POST', 'DELETE'])
def manage_watchlist():
    """Manage watchlist."""
    if not PHASE2_ENABLED:
        # Phase 1: Use localStorage
        if request.method == 'GET':
            return jsonify({'message': 'Use localStorage on client', 'symbols': []})
        elif request.method == 'POST':
            data = request.get_json()
            return jsonify({'message': 'Use localStorage on client', 'success': True})
        elif request.method == 'DELETE':
            return jsonify({'message': 'Use localStorage on client', 'success': True})
    
    # Phase 2: Database-backed watchlist
    if not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        if request.method == 'GET':
            watchlist = Watchlist.query.filter_by(user_id=current_user.id).all()
            
            # If watchlist is empty, auto-populate with portfolio symbols
            if not watchlist and PHASE2_ENABLED:
                portfolio_positions = Portfolio.query.filter_by(user_id=current_user.id).all()
                if portfolio_positions:
                    for pos in portfolio_positions:
                        item = Watchlist(
                            user_id=current_user.id,
                            symbol=pos.symbol,
                            notes='Auto-added from portfolio'
                        )
                        db.session.add(item)
                    
                    # Also add common market indices
                    for symbol in ['SPY', 'QQQ', 'DIA']:
                        if not Watchlist.query.filter_by(user_id=current_user.id, symbol=symbol).first():
                            item = Watchlist(
                                user_id=current_user.id,
                                symbol=symbol,
                                notes='Market index'
                            )
                            db.session.add(item)
                    
                    db.session.commit()
                    watchlist = Watchlist.query.filter_by(user_id=current_user.id).all()
                    logger.info(f"Auto-populated watchlist with {len(watchlist)} symbols for user {current_user.id}")
            
            return jsonify({'symbols': [item.to_dict() for item in watchlist]})
        
        elif request.method == 'POST':
            data = request.get_json()
            symbol = data.get('symbol', '').upper()
            notes = data.get('notes', '')
            
            if not symbol:
                return jsonify({'error': 'No symbol provided'}), 400
            
            # Check if already exists
            existing = Watchlist.query.filter_by(
                user_id=current_user.id,
                symbol=symbol
            ).first()
            
            if existing:
                return jsonify({'error': 'Symbol already in watchlist'}), 400
            
            # Add to watchlist
            item = Watchlist(
                user_id=current_user.id,
                symbol=symbol,
                notes=notes
            )
            db.session.add(item)
            db.session.commit()
            
            return jsonify({'message': f'Added {symbol} to watchlist', 'success': True})
        
        elif request.method == 'DELETE':
            data = request.get_json()
            symbol = data.get('symbol', '').upper()
            
            if not symbol:
                return jsonify({'error': 'No symbol provided'}), 400
            
            item = Watchlist.query.filter_by(
                user_id=current_user.id,
                symbol=symbol
            ).first()
            
            if item:
                db.session.delete(item)
                db.session.commit()
                return jsonify({'message': f'Removed {symbol} from watchlist', 'success': True})
            
            return jsonify({'error': 'Symbol not found in watchlist'}), 404
    
    except Exception as e:
        logger.error(f"Error managing watchlist: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/alerts', methods=['GET', 'POST', 'DELETE'])
def manage_alerts():
    """Manage price alerts."""
    if not PHASE2_ENABLED:
        # Phase 1: Use localStorage
        if request.method == 'GET':
            return jsonify({'message': 'Use localStorage on client', 'alerts': []})
        elif request.method == 'POST':
            return jsonify({'message': 'Use localStorage on client', 'success': True})
        elif request.method == 'DELETE':
            return jsonify({'message': 'Use localStorage on client', 'success': True})
    
    # Phase 2: Database-backed alerts
    if not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        if request.method == 'GET':
            alerts = Alert.query.filter_by(user_id=current_user.id, enabled=True).all()
            return jsonify({'alerts': [alert.to_dict() for alert in alerts]})
        
        elif request.method == 'POST':
            data = request.get_json()
            symbol = data.get('symbol', '').upper()
            alert_type = data.get('type')  # 'above' or 'below'
            target_price = data.get('targetPrice')
            
            if not all([symbol, alert_type, target_price]):
                return jsonify({'error': 'Missing required fields'}), 400
            
            # Create alert
            alert = Alert(
                user_id=current_user.id,
                symbol=symbol,
                alert_type=alert_type,
                target_price=target_price,
                enabled=True
            )
            db.session.add(alert)
            db.session.commit()
            
            return jsonify({
                'message': f'Alert created for {symbol}',
                'success': True,
                'alert': alert.to_dict()
            })
        
        elif request.method == 'DELETE':
            data = request.get_json()
            alert_id = data.get('id')
            
            if not alert_id:
                return jsonify({'error': 'No alert ID provided'}), 400
            
            alert = Alert.query.filter_by(
                id=alert_id,
                user_id=current_user.id
            ).first()
            
            if alert:
                db.session.delete(alert)
                db.session.commit()
                return jsonify({'message': 'Alert deleted', 'success': True})
            
            return jsonify({'error': 'Alert not found'}), 404
    
    except Exception as e:
        logger.error(f"Error managing alerts: {e}")
        return jsonify({'error': str(e)}), 500

# Portfolio Account Management
@app.route('/api/portfolio/accounts', methods=['GET'])
def list_portfolio_accounts():
    """List all portfolio accounts for the user."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        accounts = PortfolioAccount.query.filter_by(user_id=current_user.id).order_by(PortfolioAccount.created_at).all()
        
        # Calculate totals per account
        result = []
        for acc in accounts:
            holdings = Portfolio.query.filter_by(user_id=current_user.id, account_id=acc.id).all()
            total_value = sum(float(h.quantity) * float(h.current_price or h.average_cost) for h in holdings)
            total_cost = sum(float(h.quantity) * float(h.average_cost) for h in holdings)
            cash = float(acc.cash_balance) if acc.cash_balance else 0
            result.append({
                **acc.to_dict(),
                'total_value': round(total_value, 2),
                'total_cost': round(total_cost, 2),
                'gain_loss': round(total_value - total_cost, 2),
                'holdings_count': len(holdings),
                'total_account_value': round(total_value + cash, 2),
                'cash_pct': round((cash / (total_value + cash) * 100) if (total_value + cash) > 0 else 0, 1)
            })
        
        # Also get unassigned holdings count
        unassigned = Portfolio.query.filter_by(user_id=current_user.id, account_id=None).count()
        
        return jsonify({'accounts': result, 'unassigned_count': unassigned})
    except Exception as e:
        logger.error(f"Error listing accounts: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/accounts', methods=['POST'])
def create_portfolio_account():
    """Create a new portfolio account."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        data = request.get_json()
        name = data.get('name', '').strip()
        investment_style = data.get('investment_style', 'moderate')
        description = data.get('description', '').strip()
        
        if not name:
            return jsonify({'error': 'Account name is required'}), 400
        
        valid_styles = ['aggressive', 'moderate', 'conservative', 'balanced']
        if investment_style not in valid_styles:
            return jsonify({'error': f'Invalid style. Must be one of: {", ".join(valid_styles)}'}), 400
        
        existing = PortfolioAccount.query.filter_by(user_id=current_user.id, name=name).first()
        if existing:
            return jsonify({'error': 'Account with this name already exists'}), 400
        
        account = PortfolioAccount(
            user_id=current_user.id,
            name=name,
            investment_style=investment_style,
            description=description,
            cash_balance=float(data.get('cash_balance', 0))
        )
        db.session.add(account)
        db.session.commit()
        
        return jsonify({'account': account.to_dict(), 'message': 'Account created'}), 201
    except Exception as e:
        db.session.rollback()
        logger.error(f"Error creating account: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/accounts/<int:account_id>', methods=['PUT'])
def update_portfolio_account(account_id):
    """Update portfolio account settings."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        account = PortfolioAccount.query.filter_by(id=account_id, user_id=current_user.id).first()
        if not account:
            return jsonify({'error': 'Account not found'}), 404
        
        data = request.get_json()
        if 'name' in data and data['name'].strip():
            account.name = data['name'].strip()
        if 'investment_style' in data:
            valid_styles = ['aggressive', 'moderate', 'conservative', 'balanced']
            if data['investment_style'] in valid_styles:
                account.investment_style = data['investment_style']
        if 'description' in data:
            account.description = data['description'].strip()
        if 'cash_balance' in data:
            account.cash_balance = max(0, float(data['cash_balance']))
        
        db.session.commit()
        return jsonify({'account': account.to_dict(), 'message': 'Account updated'})
    except Exception as e:
        db.session.rollback()
        logger.error(f"Error updating account: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/accounts/<int:account_id>', methods=['DELETE'])
def delete_portfolio_account(account_id):
    """Delete a portfolio account (moves holdings to unassigned)."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        account = PortfolioAccount.query.filter_by(id=account_id, user_id=current_user.id).first()
        if not account:
            return jsonify({'error': 'Account not found'}), 404
        
        # Unassign holdings instead of deleting them
        Portfolio.query.filter_by(account_id=account_id).update({'account_id': None})
        Transaction.query.filter_by(account_id=account_id).update({'account_id': None})
        
        db.session.delete(account)
        db.session.commit()
        return jsonify({'message': 'Account deleted, holdings moved to unassigned'})
    except Exception as e:
        db.session.rollback()
        logger.error(f"Error deleting account: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/accounts/<int:account_id>/cash', methods=['PUT'])
def update_account_cash(account_id):
    """Update cash balance for an account."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        account = PortfolioAccount.query.filter_by(id=account_id, user_id=current_user.id).first()
        if not account:
            return jsonify({'error': 'Account not found'}), 404
        
        data = request.get_json()
        account.cash_balance = max(0, float(data.get('cash_balance', 0)))
        db.session.commit()
        return jsonify({'message': 'Cash balance updated', 'cash_balance': float(account.cash_balance)})
    except Exception as e:
        db.session.rollback()
        logger.error(f"Error updating cash: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/cash-optimization', methods=['GET'])
def get_cash_optimization():
    """Get cash optimization suggestions based on account style and holdings."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        account_id = request.args.get('account_id', type=int)
        if not account_id:
            return jsonify({'error': 'account_id is required'}), 400
        
        account = PortfolioAccount.query.filter_by(id=account_id, user_id=current_user.id).first()
        if not account:
            return jsonify({'error': 'Account not found'}), 404
        
        cash = float(account.cash_balance) if account.cash_balance else 0
        if cash <= 0:
            return jsonify({'suggestions': [], 'message': 'No free cash to optimize'})
        
        holdings = Portfolio.query.filter_by(user_id=current_user.id, account_id=account_id).all()
        invested_value = sum(float(h.quantity) * float(h.current_price or h.average_cost) for h in holdings)
        total_account = invested_value + cash
        cash_pct = (cash / total_account * 100) if total_account > 0 else 100
        
        held_symbols = {h.symbol for h in holdings}
        style = account.investment_style or 'moderate'
        
        # Suggestions based on investment style
        suggestions = []
        
        # Style-specific ETF suggestions
        style_etfs = {
            'aggressive': [
                {'symbol': 'QQQ', 'name': 'Invesco QQQ Trust (Nasdaq 100)', 'reason': 'High-growth tech exposure for aggressive portfolios', 'allocation': 30},
                {'symbol': 'ARKK', 'name': 'ARK Innovation ETF', 'reason': 'Disruptive innovation exposure', 'allocation': 15},
                {'symbol': 'SOXX', 'name': 'iShares Semiconductor ETF', 'reason': 'Semiconductor sector growth play', 'allocation': 15},
                {'symbol': 'VGT', 'name': 'Vanguard Info Tech ETF', 'reason': 'Broad tech sector allocation', 'allocation': 20},
                {'symbol': 'TQQQ', 'name': 'ProShares UltraPro QQQ', 'reason': '3x leveraged Nasdaq for high-risk plays', 'allocation': 10},
            ],
            'moderate': [
                {'symbol': 'VOO', 'name': 'Vanguard S&P 500 ETF', 'reason': 'Core S&P 500 index exposure', 'allocation': 35},
                {'symbol': 'QQQ', 'name': 'Invesco QQQ Trust', 'reason': 'Growth tilt via Nasdaq 100', 'allocation': 20},
                {'symbol': 'SCHD', 'name': 'Schwab US Dividend Equity', 'reason': 'Reliable dividend income', 'allocation': 20},
                {'symbol': 'BND', 'name': 'Vanguard Total Bond Market', 'reason': 'Fixed income ballast', 'allocation': 15},
                {'symbol': 'VNQ', 'name': 'Vanguard Real Estate ETF', 'reason': 'Real estate diversification', 'allocation': 10},
            ],
            'conservative': [
                {'symbol': 'BND', 'name': 'Vanguard Total Bond Market', 'reason': 'Core bond holding for stability', 'allocation': 30},
                {'symbol': 'SCHD', 'name': 'Schwab US Dividend Equity', 'reason': 'High-quality dividend stocks', 'allocation': 25},
                {'symbol': 'VIG', 'name': 'Vanguard Dividend Appreciation', 'reason': 'Dividend growth for income', 'allocation': 20},
                {'symbol': 'SHY', 'name': 'iShares 1-3 Year Treasury', 'reason': 'Short-term treasury safety', 'allocation': 15},
                {'symbol': 'VTIP', 'name': 'Vanguard Short-Term TIPS', 'reason': 'Inflation protection', 'allocation': 10},
            ],
            'balanced': [
                {'symbol': 'VTI', 'name': 'Vanguard Total Stock Market', 'reason': 'Broad US equity exposure', 'allocation': 30},
                {'symbol': 'VXUS', 'name': 'Vanguard Total International', 'reason': 'International diversification', 'allocation': 20},
                {'symbol': 'BND', 'name': 'Vanguard Total Bond Market', 'reason': 'Bond allocation for balance', 'allocation': 20},
                {'symbol': 'SCHD', 'name': 'Schwab US Dividend Equity', 'reason': 'Dividend income stream', 'allocation': 15},
                {'symbol': 'GLD', 'name': 'SPDR Gold Shares', 'reason': 'Gold as inflation hedge', 'allocation': 15},
            ]
        }
        
        etfs = style_etfs.get(style, style_etfs['moderate'])
        
        for etf in etfs:
            already_held = etf['symbol'] in held_symbols
            dollar_amount = round(cash * etf['allocation'] / 100, 2)
            suggestions.append({
                'symbol': etf['symbol'],
                'name': etf['name'],
                'reason': etf['reason'],
                'allocation_pct': etf['allocation'],
                'dollar_amount': dollar_amount,
                'already_held': already_held,
                'action': 'Add to position' if already_held else 'New position'
            })
        
        # Cash analysis
        if cash_pct > 30:
            urgency = 'high'
            message = f'{cash_pct:.0f}% of your account is in cash — significant drag on returns. Consider deploying capital.'
        elif cash_pct > 15:
            urgency = 'medium'
            message = f'{cash_pct:.0f}% cash allocation is above typical targets. Consider partially investing.'
        else:
            urgency = 'low'
            message = f'{cash_pct:.0f}% cash is a reasonable reserve. Optional to optimize further.'
        
        return jsonify({
            'account_name': account.name,
            'style': style,
            'cash_balance': cash,
            'invested_value': round(invested_value, 2),
            'total_account_value': round(total_account, 2),
            'cash_pct': round(cash_pct, 1),
            'urgency': urgency,
            'message': message,
            'suggestions': suggestions
        })
    except Exception as e:
        logger.error(f"Error getting cash optimization: {e}")
        return jsonify({'error': str(e)}), 500

# Dividend tracking endpoints
@app.route('/api/portfolio/dividends', methods=['GET'])
def get_dividends():
    """Get dividend history with optional filters."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    try:
        query = Dividend.query.filter_by(user_id=current_user.id)
        account_id = request.args.get('account_id')
        if account_id and account_id.isdigit():
            query = query.filter_by(account_id=int(account_id))
        elif account_id == 'unassigned':
            query = query.filter(Dividend.account_id.is_(None))
        symbol = request.args.get('symbol')
        if symbol:
            query = query.filter_by(symbol=symbol.upper())
        dividends = query.order_by(Dividend.pay_date.desc(), Dividend.recorded_at.desc()).all()
        return jsonify([d.to_dict() for d in dividends])
    except Exception as e:
        logger.error(f"Error getting dividends: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/dividends', methods=['POST'])
def record_dividend():
    """Record a new dividend payment."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    try:
        data = request.get_json()
        if not data or not data.get('symbol') or not data.get('total_amount'):
            return jsonify({'error': 'symbol and total_amount are required'}), 400
        symbol = data['symbol'].upper().strip()
        total_amount = float(data['total_amount'])
        shares = float(data.get('shares', 0))
        amount_per_share = float(data.get('amount_per_share', 0))
        if shares > 0 and amount_per_share == 0 and total_amount > 0:
            amount_per_share = total_amount / shares
        elif amount_per_share > 0 and shares > 0 and total_amount == 0:
            total_amount = amount_per_share * shares
        dividend = Dividend(
            user_id=current_user.id,
            account_id=int(data['account_id']) if data.get('account_id') else None,
            symbol=symbol,
            amount_per_share=amount_per_share,
            shares=shares,
            total_amount=total_amount,
            ex_date=datetime.strptime(data['ex_date'], '%Y-%m-%d').date() if data.get('ex_date') else None,
            pay_date=datetime.strptime(data['pay_date'], '%Y-%m-%d').date() if data.get('pay_date') else None,
            reinvested=data.get('reinvested', False),
            notes=data.get('notes', '')
        )
        db.session.add(dividend)
        db.session.commit()
        return jsonify(dividend.to_dict()), 201
    except Exception as e:
        db.session.rollback()
        logger.error(f"Error recording dividend: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/dividends/<int:dividend_id>', methods=['DELETE'])
def delete_dividend(dividend_id):
    """Delete a dividend record."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    try:
        dividend = Dividend.query.filter_by(id=dividend_id, user_id=current_user.id).first()
        if not dividend:
            return jsonify({'error': 'Dividend not found'}), 404
        db.session.delete(dividend)
        db.session.commit()
        return jsonify({'success': True})
    except Exception as e:
        db.session.rollback()
        logger.error(f"Error deleting dividend: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/dividends/summary', methods=['GET'])
def get_dividend_summary():
    """Get aggregate dividend summary."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    try:
        from sqlalchemy import func as sqlfunc
        account_id = request.args.get('account_id')
        base_query = Dividend.query.filter_by(user_id=current_user.id)
        if account_id and account_id.isdigit():
            base_query = base_query.filter_by(account_id=int(account_id))
        elif account_id == 'unassigned':
            base_query = base_query.filter(Dividend.account_id.is_(None))

        # Total dividend income
        total_income = db.session.query(sqlfunc.sum(Dividend.total_amount)).filter(
            Dividend.user_id == current_user.id
        )
        if account_id and account_id.isdigit():
            total_income = total_income.filter(Dividend.account_id == int(account_id))
        elif account_id == 'unassigned':
            total_income = total_income.filter(Dividend.account_id.is_(None))
        total_income = total_income.scalar() or 0

        # Per-symbol breakdown
        symbol_query = db.session.query(
            Dividend.symbol,
            sqlfunc.sum(Dividend.total_amount),
            sqlfunc.count(Dividend.id)
        ).filter(Dividend.user_id == current_user.id)
        if account_id and account_id.isdigit():
            symbol_query = symbol_query.filter(Dividend.account_id == int(account_id))
        elif account_id == 'unassigned':
            symbol_query = symbol_query.filter(Dividend.account_id.is_(None))
        symbol_rows = symbol_query.group_by(Dividend.symbol).all()

        by_symbol = [{'symbol': row[0], 'total': round(float(row[1]), 2), 'count': row[2]} for row in symbol_rows]

        # YTD income
        year_start = datetime(datetime.now().year, 1, 1).date()
        ytd_query = db.session.query(sqlfunc.sum(Dividend.total_amount)).filter(
            Dividend.user_id == current_user.id,
            Dividend.pay_date >= year_start
        )
        if account_id and account_id.isdigit():
            ytd_query = ytd_query.filter(Dividend.account_id == int(account_id))
        elif account_id == 'unassigned':
            ytd_query = ytd_query.filter(Dividend.account_id.is_(None))
        ytd_income = ytd_query.scalar() or 0

        # Monthly average (last 12 months)
        year_ago = (datetime.now() - timedelta(days=365)).date()
        recent_query = db.session.query(sqlfunc.sum(Dividend.total_amount)).filter(
            Dividend.user_id == current_user.id,
            Dividend.pay_date >= year_ago
        )
        if account_id and account_id.isdigit():
            recent_query = recent_query.filter(Dividend.account_id == int(account_id))
        elif account_id == 'unassigned':
            recent_query = recent_query.filter(Dividend.account_id.is_(None))
        recent_total = recent_query.scalar() or 0
        monthly_avg = round(float(recent_total) / 12, 2)

        return jsonify({
            'total_income': round(float(total_income), 2),
            'ytd_income': round(float(ytd_income), 2),
            'monthly_avg': monthly_avg,
            'by_symbol': sorted(by_symbol, key=lambda x: x['total'], reverse=True),
            'payment_count': base_query.count()
        })
    except Exception as e:
        logger.error(f"Error getting dividend summary: {e}")
        return jsonify({'error': str(e)}), 500

# Phase 2: Portfolio endpoints
@app.route('/api/portfolio/list', methods=['GET'])
def list_portfolio_holdings():
    """Get list of portfolio holdings (Phase 2)."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        # Filter by account if specified
        account_id = request.args.get('account_id')
        query = Portfolio.query.filter_by(user_id=current_user.id)
        if account_id:
            if account_id == 'unassigned':
                query = query.filter_by(account_id=None)
            else:
                query = query.filter_by(account_id=int(account_id))
        
        holdings = query.all()
        
        # Update current prices
        for holding in holdings:
            stock_data = data_fetcher.fetch_stock_data(holding.symbol, '1d')
            if stock_data is not None and not stock_data.empty:
                holding.current_price = float(stock_data.iloc[-1]['Close'])
        
        db.session.commit()
        
        return jsonify({
            'holdings': [h.to_dict() for h in holdings],
            'total_value': sum(float(h.quantity) * float(h.current_price) for h in holdings if h.current_price),
            'total_cost': sum(float(h.quantity) * float(h.average_cost) for h in holdings)
        })
        
    except Exception as e:
        logger.error(f"Error listing portfolio: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/portfolio', methods=['POST', 'DELETE'])
def manage_portfolio():
    """Add or remove portfolio holdings (Phase 2)."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        if request.method == 'POST':
            # Add new position
            data = request.get_json()
            symbol = data.get('symbol', '').upper()
            asset_type = data.get('asset_type', 'stock')
            quantity = float(data.get('quantity', 0))
            price = float(data.get('price', 0))
            purchase_date = data.get('purchase_date')  # Get purchase date if provided
            account_id = data.get('account_id')  # Optional portfolio account
            
            if not all([symbol, quantity > 0, price > 0]):
                return jsonify({'error': 'Missing or invalid fields'}), 400
            
            # Normalize crypto symbols (e.g., AVAX → AVAX-USD for yfinance)
            original_symbol = symbol
            symbol = normalize_crypto_symbol(symbol, asset_type)
            if symbol != original_symbol:
                logger.info(f"Normalized crypto symbol: {original_symbol} → {symbol}")
            
            # Parse purchase date if provided
            parsed_date = None
            if purchase_date:
                try:
                    from datetime import datetime
                    parsed_date = datetime.fromisoformat(purchase_date.replace('Z', '+00:00'))
                except:
                    parsed_date = datetime.utcnow()
            else:
                parsed_date = datetime.utcnow()
            
            # Validate account_id if provided
            if account_id:
                account_id = int(account_id)
                account = PortfolioAccount.query.filter_by(id=account_id, user_id=current_user.id).first()
                if not account:
                    return jsonify({'error': 'Invalid account'}), 400
            
            # Check if position exists (scoped to account)
            existing = Portfolio.query.filter_by(
                user_id=current_user.id,
                symbol=symbol,
                asset_type=asset_type,
                account_id=account_id
            ).first()
            
            if existing:
                # Update average cost - cast DB Decimal values to float to avoid type mismatch
                logger.info(f"Updating existing {symbol}: qty type={type(existing.quantity).__name__}, avg_cost type={type(existing.average_cost).__name__}")
                existing_quantity = float(existing.quantity)
                existing_avg_cost = float(existing.average_cost)
                total_cost = (existing_quantity * existing_avg_cost) + (quantity * price)
                total_quantity = existing_quantity + quantity
                logger.info(f"Computed: total_cost={total_cost}, total_quantity={total_quantity}")
                existing.average_cost = total_cost / total_quantity
                existing.quantity = total_quantity
                # Keep original purchase date for existing positions
            else:
                # Create new position
                position = Portfolio(
                    user_id=current_user.id,
                    account_id=account_id,
                    symbol=symbol,
                    asset_type=asset_type,
                    quantity=quantity,
                    average_cost=price,
                    purchase_date=parsed_date
                )
                db.session.add(position)
            
            # Record transaction with the purchase date
            transaction = Transaction(
                user_id=current_user.id,
                account_id=account_id,
                symbol=symbol,
                asset_type=asset_type,
                transaction_type='buy',
                quantity=quantity,
                price=price,
                transaction_date=parsed_date
            )
            db.session.add(transaction)
            db.session.commit()
            
            resp = {'message': f'Added {quantity} shares of {symbol}', 'success': True}
            if symbol != original_symbol:
                resp['corrected_symbol'] = symbol
                resp['message'] = f'Added {quantity} shares of {symbol} (normalized from {original_symbol})'
            return jsonify(resp), 201
        
        elif request.method == 'DELETE':
            # Remove position
            data = request.get_json()
            position_id = data.get('id')
            
            if not position_id:
                return jsonify({'error': 'No position ID provided'}), 400
            
            position = Portfolio.query.filter_by(
                id=position_id,
                user_id=current_user.id
            ).first()
            
            if position:
                db.session.delete(position)
                db.session.commit()
                return jsonify({'message': 'Position removed', 'success': True})
            
            return jsonify({'error': 'Position not found'}), 404
    
    except Exception as e:
        logger.error(f"Error managing portfolio: {e}")
        logger.error(traceback.format_exc())
        return jsonify({'error': str(e), 'traceback': traceback.format_exc()}), 500


@app.route('/api/transactions', methods=['GET'])
def get_transactions():
    """Get transaction history."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        symbol = request.args.get('symbol')
        
        query = Transaction.query.filter_by(user_id=current_user.id)
        if symbol:
            query = query.filter_by(symbol=symbol.upper())
        
        transactions = query.order_by(Transaction.transaction_date.desc()).limit(100).all()
        
        return jsonify({
            'transactions': [t.to_dict() for t in transactions]
        })
    
    except Exception as e:
        logger.error(f"Error fetching transactions: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/options', methods=['GET', 'POST'])
def manage_options():
    """Manage options positions."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        if request.method == 'GET':
            positions = OptionsPosition.query.filter_by(
                user_id=current_user.id,
                status='open'
            ).all()
            
            return jsonify({
                'positions': [p.to_dict() for p in positions]
            })
        
        elif request.method == 'POST':
            data = request.get_json()
            
            position = OptionsPosition(
                user_id=current_user.id,
                underlying_symbol=data.get('symbol', '').upper(),
                option_type=data.get('option_type'),
                strike_price=float(data.get('strike_price')),
                expiration_date=data.get('expiration_date'),
                quantity=int(data.get('quantity')),
                premium_paid=float(data.get('premium_paid'))
            )
            
            db.session.add(position)
            db.session.commit()
            
            return jsonify({
                'message': 'Options position added',
                'success': True,
                'position': position.to_dict()
            })
    
    except Exception as e:
        logger.error(f"Error managing options: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/options/<int:position_id>', methods=['DELETE'])
def delete_option_position(position_id):
    """Delete an options position."""
    if not PHASE2_ENABLED or not current_user.is_authenticated:
        return jsonify({'error': 'Authentication required'}), 401
    
    try:
        position = OptionsPosition.query.filter_by(
            id=position_id,
            user_id=current_user.id
        ).first()
        
        if not position:
            return jsonify({'error': 'Position not found'}), 404
        
        db.session.delete(position)
        db.session.commit()
        
        logger.info(f"Deleted options position {position.underlying_symbol} (ID: {position_id}) for user {current_user.id}")
        return jsonify({'message': 'Position deleted successfully'}), 200
    
    except Exception as e:
        logger.error(f"Error deleting options position: {e}")
        db.session.rollback()
        return jsonify({'error': str(e)}), 500


@app.route('/api/ml-patterns', methods=['GET', 'POST'])
def manage_ml_patterns():
    """Get or generate ML detected patterns."""
    if not PHASE2_ENABLED:
        return jsonify({'message': 'ML features not available', 'patterns': []})
    
    try:
        if request.method == 'GET':
            # Get existing patterns from database
            symbol = request.args.get('symbol')
            
            query = MLPattern.query
            if symbol:
                query = query.filter_by(symbol=symbol.upper())
            
            patterns = query.order_by(MLPattern.detected_at.desc()).limit(50).all()
            
            return jsonify({
                'patterns': [p.to_dict() for p in patterns]
            })
        
        elif request.method == 'POST':
            # Generate new patterns for a symbol
            data = request.get_json()
            symbol = data.get('symbol', '').upper()
            period = data.get('period', '6mo')
            
            if not symbol:
                return jsonify({'error': 'No symbol provided'}), 400
            
            # Fetch stock data
            stock_data = data_fetcher.fetch_stock_data(symbol, period)
            
            if stock_data is None or stock_data.empty:
                return jsonify({'error': f'No data for {symbol}'}), 404
            
            # Detect patterns
            patterns = ml_detector.detect_patterns(stock_data, symbol)
            
            # Save to database
            saved_patterns = []
            for pattern_data in patterns:
                pattern = MLPattern(
                    symbol=pattern_data['symbol'],
                    pattern_type=pattern_data['pattern_type'],
                    confidence=pattern_data['confidence'],
                    prediction=pattern_data['prediction'],
                    time_horizon=pattern_data['time_horizon'],
                    pattern_data=pattern_data['pattern_data'],
                    price_at_detection=pattern_data['price_at_detection']
                )
                db.session.add(pattern)
                saved_patterns.append(pattern_data)
            
            db.session.commit()
            
            return jsonify({
                'message': f'Detected {len(patterns)} patterns for {symbol}',
                'patterns': saved_patterns
            })
    
    except Exception as e:
        logger.error(f"Error managing ML patterns: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/ml-predictions', methods=['GET', 'POST'])
def manage_ml_predictions():
    """Get or generate ML predictions."""
    if not PHASE2_ENABLED:
        return jsonify({'message': 'ML features not available', 'predictions': []})
    
    try:
        if request.method == 'GET':
            # Get existing predictions
            symbol = request.args.get('symbol')
            
            query = MLPrediction.query
            if symbol:
                query = query.filter_by(symbol=symbol.upper())
            
            predictions = query.order_by(MLPrediction.created_at.desc()).limit(50).all()
            
            return jsonify({
                'predictions': [p.to_dict() for p in predictions]
            })
        
        elif request.method == 'POST':
            # Generate new prediction
            data = request.get_json()
            symbol = data.get('symbol', '').upper()
            period = data.get('period', '6mo')
            horizon_days = data.get('horizon_days', 5)
            
            if not symbol:
                return jsonify({'error': 'No symbol provided'}), 400
            
            # Fetch stock data
            stock_data = data_fetcher.fetch_stock_data(symbol, period)
            
            if stock_data is None or stock_data.empty:
                return jsonify({'error': f'No data for {symbol}'}), 404
            
            # Make prediction
            prediction_data = ml_detector.make_prediction(stock_data, symbol, horizon_days)
            
            if not prediction_data:
                return jsonify({'error': 'Could not generate prediction'}), 500
            
            # Save to database
            prediction = MLPrediction(
                symbol=prediction_data['symbol'],
                prediction_type=prediction_data['prediction_type'],
                predicted_direction=prediction_data['predicted_direction'],
                predicted_price=prediction_data['predicted_price'],
                confidence=prediction_data['confidence'],
                time_horizon=prediction_data['time_horizon'],
                target_date=prediction_data['target_date'],
                model_version=prediction_data['model_version']
            )
            
            db.session.add(prediction)
            db.session.commit()
            
            return jsonify({
                'message': f'Generated prediction for {symbol}',
                'prediction': prediction_data
            })
    
    except Exception as e:
        logger.error(f"Error managing ML predictions: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/monitoring/status', methods=['GET'])
def monitoring_status():
    """Get monitoring service status."""
    if not PHASE2_ENABLED:
        return jsonify({'message': 'Monitoring not available', 'running': False})
    
    try:
        service = get_monitoring_service()
        if service:
            stats = service.get_monitoring_stats()
            return jsonify(stats)
        else:
            return jsonify({'message': 'Monitoring service not initialized', 'running': False})
    
    except Exception as e:
        logger.error(f"Error getting monitoring status: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/monitoring/check/<symbol>', methods=['GET'])
def check_symbol_price(symbol):
    """Manually check a symbol's current price."""
    if not PHASE2_ENABLED:
        return jsonify({'error': 'Monitoring not available'}), 503
    
    try:
        service = get_monitoring_service()
        if service:
            result = service.check_symbol(symbol.upper())
            return jsonify(result)
        else:
            return jsonify({'error': 'Monitoring service not initialized'}), 503
    
    except Exception as e:
        logger.error(f"Error checking symbol {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/ml-patterns', methods=['GET'])
def get_ml_patterns():
    """Get ML detected patterns."""
    if not PHASE2_ENABLED:
        return jsonify({'message': 'ML features not available', 'patterns': []})
    
    try:
        symbol = request.args.get('symbol')
        
        query = MLPattern.query
        if symbol:
            query = query.filter_by(symbol=symbol.upper())
        
        patterns = query.order_by(MLPattern.detected_at.desc()).limit(50).all()
        
        return jsonify({
            'patterns': [p.to_dict() for p in patterns]
        })
    
    except Exception as e:
        logger.error(f"Error fetching ML patterns: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/ml-predictions', methods=['GET'])
def get_ml_predictions():
    """Get ML predictions."""
    if not PHASE2_ENABLED:
        return jsonify({'message': 'ML features not available', 'predictions': []})
    
    try:
        symbol = request.args.get('symbol')
        
        query = MLPrediction.query
        if symbol:
            query = query.filter_by(symbol=symbol.upper())
        
        predictions = query.order_by(MLPrediction.created_at.desc()).limit(50).all()
        
        return jsonify({
            'predictions': [p.to_dict() for p in predictions]
        })
    
    except Exception as e:
        logger.error(f"Error fetching ML predictions: {e}")
        return jsonify({'error': str(e)}), 500


# ==========================================
# PHASE 3: ADVANCED TRADING INTELLIGENCE API
# ==========================================

@app.route('/api/options/<symbol>', methods=['GET'])
def get_options_analysis(symbol):
    """Get comprehensive options analysis for a symbol."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        analysis = options_analyzer.analyze_options_comprehensive(symbol)
        
        return jsonify(analysis)
    
    except Exception as e:
        logger.error(f"Error in options analysis for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/options/<symbol>/chain', methods=['GET'])
def get_options_chain(symbol):
    """Get options chain data."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        expiration = request.args.get('expiration')
        chain = options_analyzer.get_options_chain(symbol, expiration)
        
        return jsonify(chain)
    
    except Exception as e:
        logger.error(f"Error fetching options chain for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/options/<symbol>/max-pain', methods=['GET'])
def get_max_pain(symbol):
    """Calculate max pain for options."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        expiration = request.args.get('expiration')
        max_pain = options_analyzer.calculate_max_pain(symbol, expiration)
        
        return jsonify(max_pain)
    
    except Exception as e:
        logger.error(f"Error calculating max pain for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/timing/<symbol>/entry', methods=['GET'])
def get_entry_timing(symbol):
    """Get optimal entry point analysis."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        period = request.args.get('period', '6mo')
        
        stock_data = data_fetcher.fetch_stock_data(symbol, period, '1d')
        if stock_data is None or stock_data.empty:
            return jsonify({'error': 'No data available'}), 404
        
        stock_data = pattern_recognizer.calculate_indicators(stock_data)
        entry_analysis = trading_time_analyzer.analyze_entry_points(stock_data, symbol)
        
        return jsonify(entry_analysis)
    
    except Exception as e:
        logger.error(f"Error analyzing entry timing for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/timing/<symbol>/exit', methods=['GET'])
def get_exit_timing(symbol):
    """Get optimal exit point analysis."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        period = request.args.get('period', '6mo')
        entry_price = request.args.get('entry_price', type=float)
        
        stock_data = data_fetcher.fetch_stock_data(symbol, period, '1d')
        if stock_data is None or stock_data.empty:
            return jsonify({'error': 'No data available'}), 404
        
        stock_data = pattern_recognizer.calculate_indicators(stock_data)
        exit_analysis = trading_time_analyzer.analyze_exit_points(stock_data, symbol, entry_price)
        
        return jsonify(exit_analysis)
    
    except Exception as e:
        logger.error(f"Error analyzing exit timing for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/timing/<symbol>/volume', methods=['GET'])
def get_volume_analysis(symbol):
    """Get volume profile analysis."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        period = request.args.get('period', '6mo')
        
        stock_data = data_fetcher.fetch_stock_data(symbol, period, '1d')
        if stock_data is None or stock_data.empty:
            return jsonify({'error': 'No data available'}), 404
        
        volume_analysis = trading_time_analyzer.analyze_volume_profile(stock_data, symbol)
        
        return jsonify(volume_analysis)
    
    except Exception as e:
        logger.error(f"Error analyzing volume for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/sentiment/<symbol>', methods=['GET'])
def get_sentiment_analysis(symbol):
    """Get comprehensive sentiment analysis."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        period = request.args.get('period', '6mo')
        
        stock_data = data_fetcher.fetch_stock_data(symbol, period, '1d')
        if stock_data is None or stock_data.empty:
            return jsonify({'error': 'No data available'}), 404
        
        stock_data = pattern_recognizer.calculate_indicators(stock_data)
        sentiment = sentiment_analyzer.analyze_sentiment(stock_data, symbol)
        
        return jsonify(sentiment)
    
    except Exception as e:
        logger.error(f"Error analyzing sentiment for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/risk/<symbol>', methods=['GET'])
def get_risk_analysis(symbol):
    """Get comprehensive risk analysis."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        symbol = symbol.upper()
        period = request.args.get('period', '6mo')
        position_size = request.args.get('position_size', type=int)
        portfolio_value = request.args.get('portfolio_value', type=float)
        
        stock_data = data_fetcher.fetch_stock_data(symbol, period, '1d')
        if stock_data is None or stock_data.empty:
            return jsonify({'error': 'No data available'}), 404
        
        risk = risk_analyzer.comprehensive_risk_analysis(
            stock_data, symbol, position_size, portfolio_value
        )
        
        return jsonify(risk)
    
    except Exception as e:
        logger.error(f"Error analyzing risk for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/risk/position-sizing', methods=['POST'])
def calculate_position_sizing():
    """Calculate optimal position size."""
    if not PHASE3_ENABLED:
        return jsonify({'message': 'Phase 3 features not available'}), 501
    
    try:
        data = request.get_json()
        
        account_value = data.get('account_value')
        risk_per_trade_pct = data.get('risk_per_trade_pct', 2)
        entry_price = data.get('entry_price')
        stop_loss_price = data.get('stop_loss_price')
        
        if not all([account_value, entry_price, stop_loss_price]):
            return jsonify({'error': 'Missing required parameters'}), 400
        
        position_sizing = risk_analyzer.calculate_position_sizing(
            account_value, risk_per_trade_pct, entry_price, stop_loss_price
        )
        
        return jsonify(position_sizing)
    
    except Exception as e:
        logger.error(f"Error calculating position sizing: {e}")
        return jsonify({'error': str(e)}), 500

# ============================================================================
# Phase 4: Portfolio Management & Market Intelligence API Endpoints
# ============================================================================

@app.route('/api/market/vix', methods=['GET'])
def get_vix():
    """Get current VIX data with interpretation"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        vix_data = volatility_monitor.get_vix_data()
        
        if not vix_data:
            return jsonify({'error': 'VIX data not available'}), 503
        
        return jsonify(vix_data)
    
    except Exception as e:
        logger.error(f"Error fetching VIX data: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/market/volatility-indices', methods=['GET'])
def get_volatility_indices():
    """Get all major volatility indices (VIX, VXN, RVX)"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        indices = volatility_monitor.get_all_volatility_indices()
        return jsonify(indices)
    
    except Exception as e:
        logger.error(f"Error fetching volatility indices: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/market/snapshot', methods=['GET'])
def get_market_snapshot():
    """Get quick market overview"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        snapshot = volatility_monitor.get_market_snapshot()
        
        if not snapshot:
            return jsonify({'error': 'Market snapshot not available'}), 503
        
        return jsonify(snapshot)
    
    except Exception as e:
        logger.error(f"Error fetching market snapshot: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/market/fear-greed', methods=['GET'])
def get_fear_greed():
    """Get Fear & Greed Index"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        fear_greed = volatility_monitor.get_fear_greed_index()
        return jsonify(fear_greed)
    
    except Exception as e:
        logger.error(f"Error calculating fear & greed: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/market/volatile-stocks', methods=['GET'])
def get_volatile_stocks():
    """Get top volatile stocks with comprehensive metrics"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        limit = request.args.get('limit', 50, type=int)
        limit = min(limit, 100)  # Cap at 100
        
        volatile_stocks = volatility_monitor.get_top_volatile_stocks(limit=limit)
        
        return jsonify({
            'count': len(volatile_stocks),
            'stocks': volatile_stocks,
            'timestamp': datetime.now().isoformat()
        })
    
    except Exception as e:
        logger.error(f"Error fetching volatile stocks: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/market/fastest-movers', methods=['GET'])
def get_fastest_movers():
    """Get fastest moving stocks by daily % change"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        limit = request.args.get('limit', 25, type=int)
        limit = min(limit, 50)
        
        fastest = volatility_monitor.get_fastest_movers(limit=limit)
        
        return jsonify({
            'count': len(fastest),
            'movers': fastest,
            'timestamp': datetime.now().isoformat()
        })
    
    except Exception as e:
        logger.error(f"Error fetching fastest movers: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/market/volume-leaders', methods=['GET'])
def get_volume_leaders():
    """Get stocks with highest volume surges"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        limit = request.args.get('limit', 25, type=int)
        limit = min(limit, 50)
        
        volume_leaders = volatility_monitor.get_volume_leaders(limit=limit)
        
        return jsonify({
            'count': len(volume_leaders),
            'leaders': volume_leaders,
            'timestamp': datetime.now().isoformat()
        })
    
    except Exception as e:
        logger.error(f"Error fetching volume leaders: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/market/momentum-stocks', methods=['GET'])
def get_momentum_stocks():
    """Get stocks with strongest momentum"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        limit = request.args.get('limit', 25, type=int)
        limit = min(limit, 50)
        
        momentum_stocks = volatility_monitor.get_high_momentum_stocks(limit=limit)
        
        return jsonify({
            'count': len(momentum_stocks),
            'stocks': momentum_stocks,
            'timestamp': datetime.now().isoformat()
        })
    
    except Exception as e:
        logger.error(f"Error fetching momentum stocks: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio', methods=['GET'])
@require_api_auth
def get_portfolio():
    """Get complete portfolio analysis"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        analysis = portfolio_analyzer.analyze_portfolio(user_id)
        
        if not analysis:
            return jsonify({'error': 'Could not analyze portfolio'}), 500
        
        return jsonify(analysis)
    
    except Exception as e:
        logger.error(f"Error analyzing portfolio: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/holding/<int:holding_id>', methods=['GET'])
@require_api_auth
def get_holding_analysis(holding_id):
    """Get detailed analysis of specific holding"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        holding_type = request.args.get('type', 'stock')
        analysis = portfolio_analyzer.analyze_holding(holding_id, holding_type)
        
        if not analysis:
            return jsonify({'error': 'Holding not found'}), 404
        
        logger.info(f"Holding {holding_id} analysis: {analysis.get('symbol')} - Recommendation: {analysis.get('recommendation', {}).get('action', 'N/A')}")
        return jsonify(analysis)
    
    except Exception as e:
        logger.error(f"Error analyzing holding {holding_id}: {e}")
        logger.error(traceback.format_exc())
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/holding/<int:holding_id>', methods=['DELETE'])
@require_api_auth
def delete_holding(holding_id):
    """Delete a portfolio holding"""
    if not PHASE2_ENABLED:
        return jsonify({'error': 'Phase 2 not enabled'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        
        holding = Portfolio.query.filter_by(id=holding_id, user_id=user_id).first()
        if not holding:
            return jsonify({'error': 'Holding not found'}), 404
        
        db.session.delete(holding)
        db.session.commit()
        
        logger.info(f"Deleted holding {holding.symbol} (ID: {holding_id}) for user {user_id}")
        return jsonify({'message': 'Holding deleted successfully'}), 200
    
    except Exception as e:
        logger.error(f"Error deleting holding {holding_id}: {e}")
        db.session.rollback()
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/holding/<int:holding_id>', methods=['PUT'])
@require_api_auth
def update_holding(holding_id):
    """Update a portfolio holding (e.g., cost basis)"""
    if not PHASE2_ENABLED:
        return jsonify({'error': 'Phase 2 not enabled'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        data = request.get_json()
        
        holding = Portfolio.query.filter_by(id=holding_id, user_id=user_id).first()
        if not holding:
            return jsonify({'error': 'Holding not found'}), 404
        
        if 'cost_basis' in data:
            holding.average_cost = float(data['cost_basis'])
        
        if 'purchase_date' in data:
            purchase_date = data['purchase_date']
            if purchase_date:
                try:
                    from datetime import datetime
                    holding.purchase_date = datetime.fromisoformat(purchase_date.replace('Z', '+00:00'))
                except:
                    pass
        
        if 'account_id' in data:
            new_account_id = data['account_id']
            if new_account_id is not None:
                account = PortfolioAccount.query.filter_by(id=new_account_id, user_id=user_id).first()
                if not account:
                    return jsonify({'error': 'Account not found'}), 404
            holding.account_id = new_account_id
        
        db.session.commit()
        
        logger.info(f"Updated holding {holding.symbol} (ID: {holding_id}) for user {user_id}")
        return jsonify({'message': 'Holding updated successfully'}), 200
    
    except Exception as e:
        logger.error(f"Error updating holding {holding_id}: {e}")
        db.session.rollback()
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/transaction', methods=['POST'])
@require_api_auth
def record_portfolio_transaction():
    """Record a sell transaction or position adjustment"""
    if not PHASE2_ENABLED:
        return jsonify({'error': 'Phase 2 not enabled'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        data = request.get_json()
        
        holding_id = data.get('holding_id')
        transaction_type = data.get('transaction_type', 'sell')
        quantity = data.get('quantity')
        price = float(data.get('price', 0))
        sell_all = data.get('sell_all', False)
        
        holding = Portfolio.query.filter_by(id=holding_id, user_id=user_id).first()
        if not holding:
            return jsonify({'error': 'Holding not found'}), 404
        
        # Record transaction in Transaction table
        transaction = Transaction(
            user_id=user_id,
            symbol=holding.symbol,
            asset_type='stock',
            transaction_type=transaction_type,
            quantity=quantity if quantity else holding.quantity,
            price=price,
            transaction_date=datetime.utcnow()
        )
        db.session.add(transaction)
        
        # Update or delete holding
        holding_quantity = float(holding.quantity)
        sell_quantity = float(quantity) if quantity else holding_quantity
        
        if sell_all or sell_quantity >= holding_quantity:
            # Selling entire position
            db.session.delete(holding)
            logger.info(f"Sold all of {holding.symbol} at ${price} for user {user_id}")
        elif quantity:
            # Partial sell
            holding.quantity = holding_quantity - sell_quantity
            logger.info(f"Sold {sell_quantity} shares of {holding.symbol} at ${price} for user {user_id}")
        
        db.session.commit()
        
        return jsonify({'message': 'Transaction recorded successfully'}), 200
    
    except Exception as e:
        logger.error(f"Error recording transaction: {e}")
        db.session.rollback()
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/rebalance', methods=['GET'])
@require_api_auth
def get_rebalancing_suggestions():
    """Get portfolio rebalancing suggestions"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        suggestions = portfolio_analyzer.get_rebalancing_suggestions(user_id)
        
        return jsonify({'suggestions': suggestions})
    
    except Exception as e:
        logger.error(f"Error generating rebalancing suggestions: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/portfolio/history', methods=['GET'])
@require_api_auth
def get_portfolio_history():
    """Get portfolio value history for charting"""
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        period = request.args.get('period', '1m')
        
        period_map = {
            '1d': 1, '1w': 7, '1m': 30, '6m': 180,
            '1y': 365, '3y': 1095, '5y': 1825, 'max': 9999
        }
        days = period_map.get(period, 30)
        
        from datetime import datetime, timedelta
        since = datetime.utcnow() - timedelta(days=days)
        
        # Always build history from real price data
        history = _build_portfolio_history(user_id, days)
        
        # Fall back to snapshots only if price history is empty
        if not history:
            snapshots = PortfolioSnapshot.query.filter(
                PortfolioSnapshot.user_id == user_id,
                PortfolioSnapshot.timestamp >= since
            ).order_by(PortfolioSnapshot.timestamp.asc()).all()
            if snapshots:
                history = [s.to_dict() for s in snapshots]
        
        # Save a snapshot while we're here
        try:
            portfolio_analyzer.save_portfolio_snapshot(user_id)
        except Exception:
            pass
        
        return jsonify({'history': history, 'period': period})
    
    except Exception as e:
        logger.error(f"Error fetching portfolio history: {e}")
        return jsonify({'error': str(e)}), 500

def _build_portfolio_history(user_id, days):
    """Build portfolio value history from transactions when no snapshots exist"""
    from datetime import datetime, timedelta
    import yfinance as yf
    from data_fetcher import normalize_crypto_symbol
    
    # Build position state over time
    holdings = Portfolio.query.filter_by(user_id=user_id).all()
    if not holdings:
        return []
    
    # Normalize symbols (e.g., crypto AVAX -> AVAX-USD)
    symbol_map = {}
    for h in holdings:
        yf_sym = normalize_crypto_symbol(h.symbol, h.asset_type)
        symbol_map[h.symbol] = yf_sym
    symbols = list(set(symbol_map.values()))
    
    end = datetime.now()
    period_start = end - timedelta(days=days)
    
    # Find earliest purchase date across all holdings
    earliest = None
    for h in holdings:
        if h.purchase_date:
            pd_date = h.purchase_date if isinstance(h.purchase_date, datetime) else datetime.combine(h.purchase_date, datetime.min.time())
            if earliest is None or pd_date < earliest:
                earliest = pd_date
    
    # For 'max' (9999 days), use earliest purchase date; otherwise respect the period
    if days >= 9999 and earliest:
        start = earliest
    else:
        start = period_start
    
    # Fetch historical prices for all symbols
    price_data = {}
    for symbol in symbols:
        try:
            ticker = yf.Ticker(symbol)
            hist = ticker.history(start=start, end=end)
            if not hist.empty:
                price_data[symbol] = hist['Close']
        except Exception:
            continue
    
    if not price_data:
        return []
    
    # Get common dates
    import pandas as pd
    all_dates = set()
    for series in price_data.values():
        all_dates.update(series.index.date)
    dates = sorted(all_dates)
    
    # Build cost basis per symbol from transactions
    positions = {}
    for h in holdings:
        yf_sym = symbol_map[h.symbol]
        if yf_sym in positions:
            positions[yf_sym]['quantity'] += float(h.quantity)
            positions[yf_sym]['cost_basis'] += float(h.quantity) * float(h.average_cost)
        else:
            positions[yf_sym] = {
                'quantity': float(h.quantity),
                'cost_basis': float(h.quantity) * float(h.average_cost)
            }
    
    # Calculate portfolio value for each date
    history = []
    for date in dates:
        total_value = 0
        total_cost = sum(p['cost_basis'] for p in positions.values())
        for symbol, pos in positions.items():
            if symbol in price_data:
                series = price_data[symbol]
                # Find closest price on or before this date
                mask = series.index.date <= date
                if mask.any():
                    price = float(series[mask].iloc[-1])
                    total_value += price * pos['quantity']
        
        if total_value > 0:
            pnl = total_value - total_cost
            pnl_pct = (pnl / total_cost * 100) if total_cost > 0 else 0
            history.append({
                'timestamp': datetime.combine(date, datetime.min.time()).isoformat(),
                'total_value': round(total_value, 2),
                'total_cost_basis': round(total_cost, 2),
                'total_pnl': round(pnl, 2),
                'total_pnl_pct': round(pnl_pct, 4)
            })
    
    return history

@app.route('/api/alerts', methods=['GET'])
@require_api_auth
def get_alerts():
    """Get all active alerts for user"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        
        from models import Alert
        alerts = Alert.query.filter_by(
            user_id=user_id,
            status='active'
        ).all()
        
        return jsonify({'alerts': [alert.to_dict() for alert in alerts]})
    
    except Exception as e:
        logger.error(f"Error fetching alerts: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/alerts/triggered', methods=['GET'])
@require_api_auth
def get_triggered_alerts():
    """Get recently triggered alerts"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        triggered = smart_alerts.get_triggered_alerts(user_id)
        
        return jsonify({'alerts': triggered})
    
    except Exception as e:
        logger.error(f"Error fetching triggered alerts: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/alerts', methods=['POST'])
@require_api_auth
def create_alert():
    """Create a new smart alert"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        data = request.get_json()
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        
        symbol = data.get('symbol')
        alert_type = data.get('alert_type')
        condition = data.get('condition')
        priority = data.get('priority', 'medium')
        
        if not all([symbol, alert_type, condition]):
            return jsonify({'error': 'Missing required parameters'}), 400
        
        alert = smart_alerts.create_alert(
            user_id=user_id,
            symbol=symbol,
            alert_type=alert_type,
            condition=condition,
            priority=priority
        )
        
        if alert:
            return jsonify(alert.to_dict()), 201
        else:
            return jsonify({'error': 'Failed to create alert'}), 500
    
    except Exception as e:
        logger.error(f"Error creating alert: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/alerts/<int:alert_id>', methods=['DELETE'])
@require_api_auth
def delete_alert(alert_id):
    """Delete an alert"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        success = smart_alerts.delete_alert(alert_id)
        
        if success:
            return jsonify({'message': 'Alert deleted'}), 200
        else:
            return jsonify({'error': 'Alert not found'}), 404
    
    except Exception as e:
        logger.error(f"Error deleting alert: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/alerts/<int:alert_id>/dismiss', methods=['PUT'])
@require_api_auth
def dismiss_alert(alert_id):
    """Dismiss/acknowledge an alert"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Phase 4 not enabled'}), 503
    
    try:
        success = smart_alerts.dismiss_alert(alert_id)
        
        if success:
            return jsonify({'message': 'Alert dismissed'}), 200
        else:
            return jsonify({'error': 'Alert not found'}), 404
    
    except Exception as e:
        logger.error(f"Error dismissing alert: {e}")
        return jsonify({'error': str(e)}), 500

# ===========================
# PHASE 5: AI ALERT SUGGESTIONS
# ===========================

@app.route('/api/alert-suggestions', methods=['GET'])
@require_api_auth
def get_alert_suggestions():
    """Get pending AI-generated alert suggestions"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Alert suggestions not available'}), 503
    
    try:
        # Get existing active alerts to filter out matching suggestions
        existing_alerts = []
        try:
            active_alerts = Alert.query.filter_by(
                user_id=current_user.id,
                status='active',
                enabled=True
            ).all()
            existing_alerts = [a.to_dict() for a in active_alerts]
            
            # Clean up suggestions that match existing alerts
            alert_suggestions.cleanup_matching_alerts(existing_alerts)
        except Exception as e:
            logger.warning(f"Could not filter existing alerts: {e}")
        
        suggestions = alert_suggestions.get_pending_suggestions(limit=20)
        return jsonify({
            'suggestions': [s.to_dict() for s in suggestions],
            'count': len(suggestions)
        }), 200
    
    except Exception as e:
        logger.error(f"Error fetching alert suggestions: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/alert-suggestions/generate', methods=['POST'])
@require_api_auth
def generate_alert_suggestions():
    """Generate new alert suggestions for watchlist and portfolio"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Alert suggestions not available'}), 503
    
    try:
        data = request.get_json() or {}
        symbols = data.get('symbols', [])
        
        # Get portfolio holdings if available
        portfolio_holdings = []
        if PHASE2_ENABLED:
            holdings = Portfolio.query.filter_by(user_id=current_user.id).all()
            portfolio_holdings = [h.to_dict() for h in holdings]
            
            # Add portfolio symbols to watchlist
            for h in holdings:
                if h.symbol not in symbols:
                    symbols.append(h.symbol)
        
        # Get existing active alerts to avoid duplicates
        existing_alerts = []
        try:
            active_alerts = Alert.query.filter_by(
                user_id=current_user.id,
                status='active',
                enabled=True
            ).all()
            existing_alerts = [a.to_dict() for a in active_alerts]
        except Exception as e:
            logger.warning(f"Could not fetch existing alerts: {e}")
        
        # Generate suggestions
        new_suggestions = alert_suggestions.generate_suggestions(symbols, portfolio_holdings)
        
        # Save to database (filtering out ones matching existing alerts)
        saved_count = alert_suggestions.save_suggestions(new_suggestions, existing_alerts)
        
        return jsonify({
            'message': f'Generated {len(new_suggestions)} suggestions, saved {saved_count} new ones',
            'suggestions': new_suggestions
        }), 200
    
    except Exception as e:
        logger.error(f"Error generating alert suggestions: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/alert-suggestions/<int:suggestion_id>/accept', methods=['POST'])
@require_api_auth
def accept_alert_suggestion(suggestion_id):
    """Accept a suggestion and create an alert"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Alert suggestions not available'}), 503
    
    try:
        logger.info(f"User {current_user.id} accepting suggestion {suggestion_id}")
        
        # Accept the suggestion and get alert data
        alert_data = alert_suggestions.accept_suggestion(suggestion_id)
        
        if not alert_data:
            logger.error(f"Suggestion {suggestion_id} not found or already actioned")
            return jsonify({'error': 'Suggestion not found'}), 404
        
        logger.info(f"Creating alert from suggestion {suggestion_id}: {alert_data}")
        
        # Create condition dict for smart_alerts
        condition = {
            'metric': 'price',
            'operator': '>' if alert_data['type'] == 'high' else '<',
            'value': float(alert_data['price'])
        }
        
        # Create the actual alert
        alert = smart_alerts.create_alert(
            user_id=current_user.id,
            symbol=alert_data['symbol'],
            alert_type='price',
            condition=condition,
            priority='high'
        )
        
        if alert:
            logger.info(f"Alert created successfully: ID={alert.id} for {alert.symbol}")
            return jsonify({
                'message': 'Alert created successfully',
                'alert': alert.to_dict()
            }), 201
        else:
            logger.error("smart_alerts.create_alert returned None")
            return jsonify({'error': 'Failed to create alert - check server logs'}), 500
    
    except Exception as e:
        logger.error(f"Error accepting suggestion {suggestion_id}: {e}")
        logger.error(traceback.format_exc())
        return jsonify({'error': str(e)}), 500

@app.route('/api/alert-suggestions/<int:suggestion_id>/dismiss', methods=['POST'])
@require_api_auth
def dismiss_alert_suggestion(suggestion_id):
    """Dismiss a suggestion"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Alert suggestions not available'}), 503
    
    try:
        success = alert_suggestions.dismiss_suggestion(suggestion_id)
        
        if success:
            return jsonify({'message': 'Suggestion dismissed'}), 200
        else:
            return jsonify({'error': 'Suggestion not found'}), 404
    
    except Exception as e:
        logger.error(f"Error dismissing suggestion: {e}")
        return jsonify({'error': str(e)}), 500

# ============================================================================
# Politician Trading Data API Endpoints
# ============================================================================

@app.route('/api/politician-trades', methods=['GET'])
@require_api_auth
def get_politician_trades():
    """Get recent politician trades"""
    try:
        days = request.args.get('days', 30, type=int)
        politician_tracker = PoliticianTradeTracker()
        trades = politician_tracker.get_recent_trades(days=days)
        
        return jsonify({
            'trades': trades,
            'count': len(trades)
        }), 200
    except Exception as e:
        logger.error(f"Error fetching politician trades: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/politician-trades/trending', methods=['GET'])
@require_api_auth
def get_trending_politician_stocks():
    """Get trending stocks among politicians"""
    try:
        politician_tracker = PoliticianTradeTracker()
        trending = politician_tracker.get_trending_symbols()
        
        return jsonify({
            'trending': trending
        }), 200
    except Exception as e:
        logger.error(f"Error fetching trending stocks: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/politician-trades/symbol/<symbol>', methods=['GET'])
@require_api_auth
def get_politician_trades_by_symbol(symbol):
    """Get politician trades for a specific symbol"""
    try:
        politician_tracker = PoliticianTradeTracker()
        trades = politician_tracker.search_by_symbol(symbol)
        
        return jsonify({
            'symbol': symbol,
            'trades': trades,
            'count': len(trades)
        }), 200
    except Exception as e:
        logger.error(f"Error fetching trades for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/politician-trades/performance', methods=['GET'])
@require_api_auth
def get_politician_performance():
    """Get performance metrics by politician"""
    try:
        politician_name = request.args.get('politician')
        politician_tracker = PoliticianTradeTracker()
        performance = politician_tracker.get_politician_performance(politician_name)
        
        return jsonify({
            'politicians': performance
        }), 200
    except Exception as e:
        logger.error(f"Error fetching politician performance: {e}")
        return jsonify({'error': str(e)}), 500

# ============================================================================
# Feature #3: News & Events Feed API Endpoints
# ============================================================================

@app.route('/api/news/market', methods=['GET'])
@require_api_auth
def get_market_news():
    """Get general market news"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'News feed not available'}), 503
    
    try:
        limit = request.args.get('limit', 20, type=int)
        news = news_fetcher.get_market_news(limit=limit)
        
        return jsonify({
            'news': news,
            'count': len(news)
        }), 200
    
    except Exception as e:
        logger.error(f"Error fetching market news: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/news/symbol/<symbol>', methods=['GET'])
@require_api_auth
def get_symbol_news(symbol):
    """Get news for a specific symbol"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'News feed not available'}), 503
    
    try:
        limit = request.args.get('limit', 10, type=int)
        news = news_fetcher.get_symbol_news(symbol.upper(), limit=limit)
        
        return jsonify({
            'symbol': symbol.upper(),
            'news': news,
            'count': len(news)
        }), 200
    
    except Exception as e:
        logger.error(f"Error fetching news for {symbol}: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/news/earnings', methods=['GET'])
@require_api_auth
def get_earnings_calendar():
    """Get upcoming earnings calendar"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Earnings calendar not available'}), 503
    
    try:
        days_ahead = request.args.get('days', 7, type=int)
        earnings = news_fetcher.get_earnings_calendar(days_ahead=days_ahead)
        
        return jsonify({
            'earnings': earnings,
            'count': len(earnings),
            'days_ahead': days_ahead
        }), 200
    
    except Exception as e:
        logger.error(f"Error fetching earnings calendar: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/news/trending', methods=['GET'])
@require_api_auth
def get_trending_tickers():
    """Get trending tickers with news"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Trending tickers not available'}), 503
    
    try:
        limit = request.args.get('limit', 10, type=int)
        trending = news_fetcher.get_trending_tickers(limit=limit)
        
        return jsonify({
            'trending': trending,
            'count': len(trending)
        }), 200
    
    except Exception as e:
        logger.error(f"Error fetching trending tickers: {e}")
        return jsonify({'error': str(e)}), 500

@app.route('/api/news/market-summary', methods=['GET'])
@require_api_auth
def get_market_summary():
    """Get market summary with news sentiment"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Market summary not available'}), 503
    
    try:
        summary = news_fetcher.get_market_summary()
        return jsonify(summary), 200
    
    except Exception as e:
        logger.error(f"Error fetching market summary: {e}")
        return jsonify({'error': str(e)}), 500


# ============================================================================
# Correlation & Heat Map API Endpoints (Feature #4)
# ============================================================================

@app.route('/api/correlation/matrix', methods=['GET'])
@require_api_auth
def get_correlation_matrix():
    """Get portfolio correlation matrix"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Correlation analysis not available'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            logger.warning("Correlation matrix requested without authentication")
            return jsonify({'error': 'Authentication required'}), 401
        
        period = request.args.get('period', '3mo')
        logger.debug(f"Fetching correlation matrix for user {user_id}, period={period}")
        result = correlation_analyzer.get_portfolio_correlation_matrix(user_id, period)
        
        if 'error' in result:
            logger.info(f"Correlation matrix error for user {user_id}: {result['error']}")
            result.setdefault('empty', True)
            result.setdefault('message', result['error'])
            return jsonify(result), 200
        
        return jsonify(result), 200
    
    except Exception as e:
        logger.error(f"Error fetching correlation matrix: {e}", exc_info=True)
        return jsonify({'error': str(e), 'empty': True, 'message': 'Correlation analysis temporarily unavailable'}), 200

@app.route('/api/correlation/diversification', methods=['GET'])
@require_api_auth
def get_diversification():
    """Get portfolio diversification metrics"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Diversification analysis not available'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            logger.warning("Diversification metrics requested without authentication")
            return jsonify({'error': 'Authentication required'}), 401
        
        logger.debug(f"Fetching diversification metrics for user {user_id}")
        result = correlation_analyzer.get_diversification_metrics(user_id)
        
        if 'error' in result:
            logger.info(f"Diversification metrics error for user {user_id}: {result['error']}")
            result.setdefault('empty', True)
            result.setdefault('message', result['error'])
            return jsonify(result), 200
        
        return jsonify(result), 200
    
    except Exception as e:
        logger.error(f"Error fetching diversification metrics: {e}", exc_info=True)
        return jsonify({'error': str(e), 'empty': True, 'message': 'Diversification analysis temporarily unavailable'}), 200

@app.route('/api/correlation/time-series', methods=['GET'])
@require_api_auth
def get_correlation_time_series():
    """Get correlation over different time periods"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Correlation time series not available'}), 503
    
    try:
        symbols = request.args.get('symbols', '').split(',')
        if len(symbols) != 2:
            return jsonify({'error': 'Provide exactly 2 symbols separated by comma'}), 400
        
        result = correlation_analyzer.get_correlation_over_time(symbols)
        
        if 'error' in result:
            return jsonify(result), 400
        
        return jsonify(result), 200
    
    except Exception as e:
        logger.error(f"Error fetching correlation time series: {e}")
        return jsonify({'error': str(e)}), 500


# ============================================================================
# Trade Journal & Analytics API Endpoints (Feature #5)
# ============================================================================

@app.route('/api/journal/history', methods=['GET'])
@require_api_auth
def get_journal_history():
    """Get trade history with summary metrics"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Trade journal not available'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            logger.warning("Journal history requested without authentication")
            return jsonify({'error': 'Authentication required'}), 401
        
        days = int(request.args.get('days', 90))
        logger.debug(f"Fetching trade history for user {user_id}, days={days}")
        result = trade_journal.get_trade_history(user_id, days)
        
        if 'error' in result:
            logger.info(f"Trade history error for user {user_id}: {result['error']}")
            # Return 200 for empty trades (not an error condition)
            if 'No trades found' in result['error'] or 'No transaction' in result['error']:
                return jsonify({
                    'error': result['error'],
                    'empty': True,
                    'message': 'Your trade history will appear here once you record trades',
                    'trades': []
                }), 200
            return jsonify(result), 400
        
        return jsonify(result), 200
    
    except Exception as e:
        logger.error(f"Error fetching trade history: {e}", exc_info=True)
        return jsonify({'error': str(e)}), 500

@app.route('/api/journal/performance', methods=['GET'])
@require_api_auth
def get_journal_performance():
    """Get trading performance analysis"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Trade journal not available'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            logger.warning("Journal performance requested without authentication")
            return jsonify({'error': 'Authentication required'}), 401
        
        days = int(request.args.get('days', 90))
        logger.debug(f"Fetching journal performance for user {user_id}, days={days}")
        result = trade_journal.analyze_performance(user_id, days)
        
        if 'error' in result:
            logger.info(f"Journal performance error for user {user_id}: {result['error']}")
            # Return 200 for empty trades (not an error condition)
            if 'No trades found' in result['error'] or 'No transaction' in result['error']:
                return jsonify({
                    'error': result['error'],
                    'empty': True,
                    'message': 'Record your trades to see performance analytics'
                }), 200
            return jsonify(result), 400
        
        return jsonify(result), 200
    
    except Exception as e:
        logger.error(f"Error analyzing performance: {e}", exc_info=True)
        return jsonify({'error': str(e)}), 500

@app.route('/api/journal/insights', methods=['GET'])
@require_api_auth
def get_journal_insights():
    """Get AI-powered trading insights"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Trade journal not available'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            logger.warning("Journal insights requested without authentication")
            return jsonify({'error': 'Authentication required'}), 401
        
        days = int(request.args.get('days', 90))
        logger.debug(f"Generating AI insights for user {user_id}, days={days}")
        result = trade_journal.get_ai_insights(user_id, days)
        
        if 'error' in result:
            logger.info(f"Journal insights error for user {user_id}: {result['error']}")
            # Return 200 for empty trades (not an error condition)
            if 'No trades found' in result['error'] or 'No transaction' in result['error']:
                return jsonify({
                    'error': result['error'],
                    'empty': True,
                    'message': 'Record your trades to get AI-powered insights'
                }), 200
            return jsonify(result), 400
        
        return jsonify(result), 200
    
    except Exception as e:
        logger.error(f"Error generating AI insights: {e}", exc_info=True)
        return jsonify({'error': str(e)}), 500

@app.route('/api/journal/note', methods=['POST'])
@require_api_auth
def save_trade_note():
    """Add or update note for a transaction"""
    if not PHASE4_ENABLED:
        return jsonify({'error': 'Trade journal not available'}), 503
    
    try:
        user_id = _get_current_user_id()
        if not user_id:
            return jsonify({'error': 'Authentication required'}), 401
        
        data = request.get_json()
        transaction_id = data.get('transaction_id')
        note = data.get('note', '')
        
        if not transaction_id:
            return jsonify({'error': 'transaction_id required'}), 400
        
        result = trade_journal.add_trade_note(transaction_id, user_id, note)
        
        if 'error' in result:
            return jsonify(result), 400
        
        return jsonify(result), 200
    
    except Exception as e:
        logger.error(f"Error saving trade note: {e}")
        return jsonify({'error': str(e)}), 500


@app.route('/api/export', methods=['GET'])
@require_api_auth
def export_user_data():
    """Export current user's financial data as JSON or CSV."""
    if not PHASE2_ENABLED:
        return jsonify({'error': 'Database not available'}), 503

    fmt = request.args.get('format', 'json').lower()
    if fmt not in ('json', 'csv'):
        return jsonify({'error': 'Invalid format. Use json or csv'}), 400

    user_id = _get_current_user_id()
    if not user_id:
        return jsonify({'error': 'Authentication required'}), 401

    from flask import make_response
    import csv
    import io as _io

    watchlist = [w.to_dict() for w in Watchlist.query.filter_by(user_id=user_id).all()]
    portfolio = [p.to_dict() for p in Portfolio.query.filter_by(user_id=user_id).all()]
    transactions = [t.to_dict() for t in Transaction.query.filter_by(user_id=user_id).all()]
    options = [o.to_dict() for o in OptionsPosition.query.filter_by(user_id=user_id).all()]
    alerts = [a.to_dict() for a in Alert.query.filter_by(user_id=user_id).all()]
    dividends = [d.to_dict() for d in Dividend.query.filter_by(user_id=user_id).all()]

    timestamp = datetime.utcnow().strftime('%Y%m%d_%H%M%S')

    if fmt == 'json':
        payload = {
            'exported_at': datetime.utcnow().isoformat(),
            'watchlist': watchlist,
            'portfolio': portfolio,
            'transactions': transactions,
            'options_positions': options,
            'alerts': alerts,
            'dividends': dividends,
        }
        resp = make_response(json.dumps(payload, indent=2))
        resp.headers['Content-Type'] = 'application/json'
        resp.headers['Content-Disposition'] = f'attachment; filename=tradertools_export_{timestamp}.json'
        return resp

    # CSV: one sheet per section, separated by blank lines
    buf = _io.StringIO()
    writer = csv.writer(buf)

    sections = [
        ('Watchlist', watchlist),
        ('Portfolio', portfolio),
        ('Transactions', transactions),
        ('Options Positions', options),
        ('Alerts', alerts),
        ('Dividends', dividends),
    ]

    for name, rows in sections:
        writer.writerow([f'=== {name} ==='])
        if rows:
            writer.writerow(list(rows[0].keys()))
            for row in rows:
                writer.writerow(list(row.values()))
        else:
            writer.writerow(['No data'])
        writer.writerow([])

    resp = make_response(buf.getvalue())
    resp.headers['Content-Type'] = 'text/csv'
    resp.headers['Content-Disposition'] = f'attachment; filename=tradertools_export_{timestamp}.csv'
    return resp


@app.errorhandler(500)
def handle_500(e):
    """Clear stale sessions and redirect on internal server errors"""
    try:
        session.clear()
        if db:
            db.session.rollback()
    except Exception:
        pass
    return redirect(url_for('login') if PHASE2_ENABLED else url_for('index'))


@app.errorhandler(Exception)
def handle_unhandled_exception(e):
    """Catch-all for unhandled exceptions to avoid raw 500 pages"""
    logger.error(f"Unhandled exception: {e}", exc_info=True)
    try:
        session.clear()
        if db:
            db.session.rollback()
    except Exception:
        pass
    from werkzeug.exceptions import HTTPException
    if isinstance(e, HTTPException):
        return e
    return redirect(url_for('login') if PHASE2_ENABLED else url_for('index'))


if __name__ == '__main__':
    print("="*70)
    print("🚀 Financial Chart Analyzer with Local LLM")
    print("="*70)
    print(f"📊 Main Dashboard: http://localhost:{app.config['FLASK_PORT']}")
    print(f"💼 Portfolio: http://localhost:{app.config['FLASK_PORT']}/portfolio")
    print(f"🎯 Copy Trading: http://localhost:{app.config['FLASK_PORT']}/copytrading")
    print(f"🤖 LLM Model: {app.config['OLLAMA_MODEL']}")
    
    if PHASE2_ENABLED:
        print("="*70)
        print("✅ PHASE 2 FEATURES ENABLED")
        print("="*70)
        print("🔐 Authentication: Google OAuth")
        print("💾 Database: SQLite (local)")
        print("📈 Portfolio Tracking: Enabled")
        print("🧠 ML Patterns: Enabled")
    
    if PHASE3_ENABLED:
        print("="*70)
        print("✅ PHASE 3 FEATURES ENABLED")
        print("="*70)
        print("📊 Options Analysis with Greeks")
        print("⏰ Trading Time Intelligence")
        print("💭 Multi-Source Sentiment Analysis")
        print("⚠️  Advanced Risk Assessment")
    
    if PHASE4_ENABLED:
        print("="*70)
        print("✅ PHASE 4 FEATURES ENABLED")
        print("="*70)
        print("📈 Portfolio Analytics & P&L Tracking")
        print("🔔 Smart Alerts System")
        print("📉 VIX & Volatility Monitoring")
        print("💰 Position-Specific Recommendations")
    
    if not PHASE2_ENABLED:
        print("="*70)
        print("ℹ️  PHASE 1 MODE (No Authentication)")
        print("="*70)
        print("   Run 'install_phase2.bat' to enable Phase 2 features")
    elif PHASE2_ENABLED and not PHASE3_ENABLED:
        print("")
        print("⚠️  Phase 3 not enabled - run 'install_phase3.bat'")
    elif PHASE3_ENABLED and not PHASE4_ENABLED:
        print("")
        print("⚠️  Phase 4 not enabled - modules may not be installed")
    else:
        print("")
        print("✅ ALL FEATURES ENABLED - Full Trading Intelligence Platform")
    
    if PHASE2_ENABLED and not os.path.exists('.env'):
        print("")
        print("⚠️  SETUP REQUIRED:")
        print("   1. Create .env file with Google OAuth credentials")
        print("   2. See PHASE2_SETUP.md for instructions")
    
    print("="*70)
    
    app.run(
        host='0.0.0.0',
        port=app.config['FLASK_PORT'],
        debug=app.config['FLASK_DEBUG']
    )