mlflow_best_practices_setup.py

#!/usr/bin/env python3
"""
MLFLOW BEST PRACTICES & ANACONDA INTEGRATION GUIDE
Complete setup and optimization guide for ML trading signals
"""

import os
import sys
import subprocess
import json
from pathlib import Path
from datetime import datetime
import pandas as pd
import numpy as np

# MLflow and experiment tracking
try:
    import mlflow
    import mlflow.sklearn
    import mlflow.tensorflow
    MLFLOW_AVAILABLE = True
except ImportError:
    MLFLOW_AVAILABLE = False

class MLflowBestPractices:
    """MLflow best practices implementation"""
    
    def __init__(self):
        self.setup_complete = False
        self.experiments = {}
        self.best_models = {}
        
    def setup_anaconda_environment(self):
        """Setup optimal Anaconda environment for ML trading"""
        
        print("🐍 ANACONDA ENVIRONMENT SETUP")
        print("=" * 50)
        
        # Check if conda is available
        try:
            result = subprocess.run(['conda', '--version'], 
                                  capture_output=True, text=True, shell=True)
            print(f"✅ Conda version: {result.stdout.strip()}")
        except FileNotFoundError:
            print("❌ Conda not found. Please install Anaconda first.")
            return False
            
        # Environment configuration
        env_config = {
            'name': 'mlflow_trading',
            'channels': ['conda-forge', 'defaults'],
            'dependencies': [
                'python=3.9',
                'pandas>=1.5.0',
                'numpy>=1.21.0',
                'scikit-learn>=1.1.0',
                'tensorflow>=2.10.0',
                'mlflow>=2.0.0',
                'matplotlib>=3.5.0',
                'seaborn>=0.11.0',
                'jupyter>=1.0.0',
                'pip',
                {
                    'pip': [
                        'MetaTrader5',
                        'yfinance',
                        'websocket-client',
                        'python-dotenv',
                        'fastapi',
                        'uvicorn',
                        'streamlit'
                    ]
                }
            ]
        }
        
        # Save environment file
        env_file = 'environment_mlflow_trading.yml'
        with open(env_file, 'w') as f:
            import yaml
            yaml.dump(env_config, f, default_flow_style=False)
            
        print(f"📄 Environment file created: {env_file}")
        
        # Create environment
        create_cmd = f'conda env create -f {env_file}'
        print(f"🔧 Creating environment: {create_cmd}")
        
        return True
        
    def setup_mlflow_tracking_server(self):
        """Setup MLflow tracking server with best practices"""
        
        print("\n🔬 MLFLOW TRACKING SERVER SETUP")
        print("=" * 50)
        
        # Create MLflow directory structure
        mlflow_dirs = [
            'mlruns',
            'artifacts',
            'models',
            'experiments',
            'configs',
            'logs'
        ]
        
        for dir_name in mlflow_dirs:
            Path(dir_name).mkdir(exist_ok=True)
            print(f"📁 Created directory: {dir_name}")
            
        # MLflow configuration
        mlflow_config = {
            'tracking_uri': './mlruns',
            'artifact_location': './artifacts',
            'default_experiment': 'fibonacci_trading_signals',
            'model_registry_uri': './models',
            'server_host': '127.0.0.1',
            'server_port': 5000
        }
        
        # Save configuration
        with open('configs/mlflow_config.json', 'w') as f:
            json.dump(mlflow_config, f, indent=2)
            
        print("✅ MLflow configuration saved")
        
        # Create startup script
        startup_script = """@echo off
echo 🚀 Starting MLflow Tracking Server...
echo.

REM Set environment variables
set MLFLOW_TRACKING_URI=./mlruns
set MLFLOW_ARTIFACT_ROOT=./artifacts

REM Start MLflow UI
mlflow ui --host 127.0.0.1 --port 5000 --backend-store-uri ./mlruns --default-artifact-root ./artifacts

echo.
echo ✅ MLflow UI started at http://127.0.0.1:5000
pause
"""
        
        with open('start_mlflow_server.bat', 'w') as f:
            f.write(startup_script)
            
        print("📄 MLflow startup script created: start_mlflow_server.bat")
        
        return True
        
    def create_experiment_templates(self):
        """Create experiment templates for different trading scenarios"""
        
        print("\n🧪 EXPERIMENT TEMPLATES")
        print("=" * 50)
        
        templates = {
            'fibonacci_signals': {
                'name': 'Fibonacci Signal Detection',
                'description': 'Fibonacci level signal generation and optimization',
                'tags': {
                    'model_type': 'ensemble',
                    'target_metric': 'win_rate',
                    'data_source': 'historical_csv',
                    'timeframe': 'multiple'
                },
                'parameters': {
                    'fibonacci_levels': ['B_0', 'B_-1.8', 'B_1.8'],
                    'confidence_threshold': 0.65,
                    'risk_reward_ratio': 2.0,
                    'max_trades_per_day': 10
                }
            },
            
            'live_trading': {
                'name': 'Live Trading Performance',
                'description': 'Real-time trading signal performance tracking',
                'tags': {
                    'model_type': 'production',
                    'target_metric': 'total_return',
                    'data_source': 'mt5_live',
                    'timeframe': 'real_time'
                },
                'parameters': {
                    'initial_balance': 10000.0,
                    'risk_per_trade': 0.02,
                    'max_drawdown_limit': 0.10
                }
            },
            
            'model_optimization': {
                'name': 'Model Hyperparameter Optimization',
                'description': 'Systematic hyperparameter tuning for trading models',
                'tags': {
                    'model_type': 'research',
                    'target_metric': 'f1_score',
                    'data_source': 'historical_csv',
                    'optimization': 'grid_search'
                },
                'parameters': {
                    'cv_folds': 5,
                    'scoring_metric': 'f1_weighted',
                    'n_trials': 100
                }
            }
        }
        
        # Save templates
        for template_name, template_config in templates.items():
            template_file = f'configs/experiment_template_{template_name}.json'
            with open(template_file, 'w') as f:
                json.dump(template_config, f, indent=2)
            print(f"📄 Template created: {template_file}")
            
        return True
        
    def implement_model_versioning(self):
        """Implement model versioning and registry best practices"""
        
        print("\n📦 MODEL VERSIONING SETUP")
        print("=" * 50)
        
        # Model registry structure
        registry_structure = {
            'production_models': {
                'fibonacci_detector_v1': {
                    'model_path': 'models/fibonacci_signal_detector.pkl',
                    'version': '1.0.0',
                    'performance': {
                        'win_rate': 0.524,
                        'total_trades': 3106,
                        'confidence_threshold': 0.65
                    },
                    'deployment_date': '2025-06-13',
                    'status': 'active'
                },
                'ensemble_detector_v1': {
                    'model_path': 'models/ensemble_signal_detector.pkl',
                    'version': '1.0.0',
                    'performance': {
                        'win_rate': 0.58,
                        'accuracy': 0.72,
                        'f1_score': 0.69
                    },
                    'deployment_date': '2025-06-13',
                    'status': 'active'
                }
            },
            'experimental_models': {},
            'archived_models': {}
        }
        
        # Save registry
        with open('models/model_registry.json', 'w') as f:
            json.dump(registry_structure, f, indent=2)
            
        print("✅ Model registry created")
        
        # Create model management script
        model_manager_script = '''#!/usr/bin/env python3
"""
Model Management Script
Handles model versioning, deployment, and rollback
"""

import json
import shutil
from pathlib import Path
from datetime import datetime
import mlflow.sklearn

class ModelManager:
    def __init__(self):
        self.registry_file = "models/model_registry.json"
        self.load_registry()
        
    def load_registry(self):
        with open(self.registry_file, 'r') as f:
            self.registry = json.load(f)
            
    def save_registry(self):
        with open(self.registry_file, 'w') as f:
            json.dump(self.registry, f, indent=2)
            
    def register_model(self, model_name, model_path, performance_metrics, version="1.0.0"):
        """Register a new model version"""
        model_info = {
            'model_path': model_path,
            'version': version,
            'performance': performance_metrics,
            'deployment_date': datetime.now().strftime('%Y-%m-%d'),
            'status': 'experimental'
        }
        
        self.registry['experimental_models'][model_name] = model_info
        self.save_registry()
        print(f"✅ Model {model_name} v{version} registered")
        
    def promote_to_production(self, model_name):
        """Promote model from experimental to production"""
        if model_name in self.registry['experimental_models']:
            model_info = self.registry['experimental_models'][model_name]
            
            # Archive current production model if exists
            if model_name in self.registry['production_models']:
                old_model = self.registry['production_models'][model_name]
                old_model['status'] = 'archived'
                self.registry['archived_models'][f"{model_name}_archived_{datetime.now().strftime('%Y%m%d')}"] = old_model
                
            # Promote to production
            model_info['status'] = 'active'
            self.registry['production_models'][model_name] = model_info
            del self.registry['experimental_models'][model_name]
            
            self.save_registry()
            print(f"✅ Model {model_name} promoted to production")
        else:
            print(f"❌ Model {model_name} not found in experimental models")
            
    def rollback_model(self, model_name):
        """Rollback to previous model version"""
        # Find archived versions
        archived_versions = [k for k in self.registry['archived_models'].keys() if k.startswith(model_name)]
        
        if archived_versions:
            # Get most recent archived version
            latest_archived = sorted(archived_versions)[-1]
            archived_model = self.registry['archived_models'][latest_archived]
            
            # Move current to archived
            if model_name in self.registry['production_models']:
                current_model = self.registry['production_models'][model_name]
                current_model['status'] = 'archived'
                self.registry['archived_models'][f"{model_name}_rollback_{datetime.now().strftime('%Y%m%d')}"] = current_model
                
            # Restore archived model
            archived_model['status'] = 'active'
            self.registry['production_models'][model_name] = archived_model
            
            self.save_registry()
            print(f"✅ Model {model_name} rolled back to previous version")
        else:
            print(f"❌ No archived versions found for {model_name}")

if __name__ == "__main__":
    manager = ModelManager()
    print("🔧 Model Manager initialized")
    print("Available commands: register_model, promote_to_production, rollback_model")
'''
        
        with open('model_manager.py', 'w') as f:
            f.write(model_manager_script)
            
        print("📄 Model manager script created: model_manager.py")
        
        return True
        
    def create_monitoring_dashboard(self):
        """Create monitoring dashboard for model performance"""
        
        print("\n📊 MONITORING DASHBOARD")
        print("=" * 50)
        
        dashboard_script = '''#!/usr/bin/env python3
"""
MLflow Trading Dashboard
Real-time monitoring of model performance and trading signals
"""

import streamlit as st
import pandas as pd
import numpy as np
import plotly.graph_objects as go
import plotly.express as px
from datetime import datetime, timedelta
import json
import mlflow

st.set_page_config(
    page_title="MLflow Trading Dashboard",
    page_icon="📈",
    layout="wide"
)

def load_model_performance():
    """Load model performance data"""
    try:
        with open('models/model_registry.json', 'r') as f:
            registry = json.load(f)
        return registry
    except:
        return {}

def load_trading_history():
    """Load trading history"""
    try:
        return pd.read_csv('paper_trades.csv')
    except:
        return pd.DataFrame()

def main():
    st.title("🎯 MLflow Trading Signal Dashboard")
    st.markdown("Real-time monitoring of ML trading models and performance")
    
    # Sidebar
    st.sidebar.title("Navigation")
    page = st.sidebar.selectbox("Choose a page", [
        "Model Performance",
        "Trading History", 
        "Live Signals",
        "Experiment Tracking"
    ])
    
    if page == "Model Performance":
        st.header("📊 Model Performance Overview")
        
        registry = load_model_performance()
        
        if registry:
            # Production models metrics
            if 'production_models' in registry:
                st.subheader("🚀 Production Models")
                
                cols = st.columns(len(registry['production_models']))
                
                for i, (model_name, model_info) in enumerate(registry['production_models'].items()):
                    with cols[i]:
                        st.metric(
                            label=model_name,
                            value=f"{model_info['performance'].get('win_rate', 0):.1%}",
                            delta=f"v{model_info['version']}"
                        )
                        
                        st.caption(f"Status: {model_info['status']}")
                        st.caption(f"Deployed: {model_info['deployment_date']}")
                        
        else:
            st.warning("No model registry found")
            
    elif page == "Trading History":
        st.header("📈 Trading History")
        
        trades_df = load_trading_history()
        
        if not trades_df.empty:
            # Performance metrics
            total_trades = len(trades_df)
            winning_trades = len(trades_df[trades_df['pnl'] > 0])
            win_rate = winning_trades / total_trades if total_trades > 0 else 0
            total_pnl = trades_df['pnl'].sum()
            
            col1, col2, col3, col4 = st.columns(4)
            
            with col1:
                st.metric("Total Trades", total_trades)
            with col2:
                st.metric("Win Rate", f"{win_rate:.1%}")
            with col3:
                st.metric("Total P&L", f"${total_pnl:.2f}")
            with col4:
                st.metric("Average Trade", f"${trades_df['pnl'].mean():.2f}")
                
            # Charts
            st.subheader("P&L Over Time")
            
            trades_df['cumulative_pnl'] = trades_df['pnl'].cumsum()
            
            fig = go.Figure()
            fig.add_trace(go.Scatter(
                x=trades_df.index,
                y=trades_df['cumulative_pnl'],
                mode='lines',
                name='Cumulative P&L'
            ))
            
            st.plotly_chart(fig, use_container_width=True)
            
            # Recent trades
            st.subheader("Recent Trades")
            st.dataframe(trades_df.tail(10))
            
        else:
            st.info("No trading history available")
            
    elif page == "Live Signals":
        st.header("🔴 Live Trading Signals")
        
        # Auto-refresh
        if st.button("🔄 Refresh Signals"):
            st.rerun()
            
        try:
            with open('signals/latest_signal.json', 'r') as f:
                signal = json.load(f)
                
            st.subheader("Latest Signal")
            
            # Signal display
            col1, col2, col3 = st.columns(3)
            
            with col1:
                st.metric("Decision", signal['decision'])
            with col2:
                st.metric("Confidence", f"{signal['confidence']:.1%}")
            with col3:
                st.metric("Price", f"${signal['price']:.2f}")
                
            st.caption(f"Generated: {signal['timestamp']}")
            
            # Signal details
            st.subheader("Signal Details")
            st.json(signal['signals'])
            
        except:
            st.warning("No live signals available")
            
    elif page == "Experiment Tracking":
        st.header("🔬 Experiment Tracking")
        
        st.markdown("""
        **MLflow Tracking Server:** [http://127.0.0.1:5000](http://127.0.0.1:5000)
        
        Navigate to the MLflow UI to view:
        - Experiment runs and comparisons
        - Model metrics and parameters
        - Artifact storage and versioning
        - Model registry and deployment
        """)
        
        # Experiment summary
        try:
            import mlflow
            experiments = mlflow.search_experiments()
            
            if experiments:
                st.subheader("Active Experiments")
                exp_df = pd.DataFrame([
                    {
                        'Name': exp.name,
                        'ID': exp.experiment_id,
                        'Lifecycle Stage': exp.lifecycle_stage
                    }
                    for exp in experiments
                ])
                st.dataframe(exp_df)
            else:
                st.info("No experiments found")
                
        except Exception as e:
            st.error(f"Could not connect to MLflow: {e}")

if __name__ == "__main__":
    main()
'''
        
        with open('mlflow_trading_dashboard.py', 'w') as f:
            f.write(dashboard_script)
            
        print("📄 Dashboard created: mlflow_trading_dashboard.py")
        print("🚀 Run with: streamlit run mlflow_trading_dashboard.py")
        
        return True
        
    def create_automated_training_pipeline(self):
        """Create automated training pipeline with MLflow"""
        
        print("\n🤖 AUTOMATED TRAINING PIPELINE")
        print("=" * 50)
        
        pipeline_script = '''#!/usr/bin/env python3
"""
Automated Training Pipeline with MLflow
Continuous model training and deployment
"""

import schedule
import time
import mlflow
import pandas as pd
from datetime import datetime, timedelta
from pathlib import Path
import logging

class AutomatedTrainingPipeline:
    def __init__(self):
        self.logger = self._setup_logging()
        self.setup_mlflow()
        
    def _setup_logging(self):
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('logs/automated_training.log'),
                logging.StreamHandler()
            ]
        )
        return logging.getLogger(__name__)
        
    def setup_mlflow(self):
        """Setup MLflow tracking"""
        mlflow.set_tracking_uri("./mlruns")
        mlflow.set_experiment("automated_training")
        
    def check_for_new_data(self):
        """Check if new data is available for training"""
        # Check for new files in dataBT folder
        data_dir = Path("dataBT")
        if not data_dir.exists():
            return False
            
        # Check modification time of latest file
        latest_file = max(data_dir.glob("*.csv"), key=lambda f: f.stat().st_mtime, default=None)
        
        if latest_file:
            file_age = datetime.now() - datetime.fromtimestamp(latest_file.stat().st_mtime)
            return file_age.days < 1  # New data if less than 1 day old
            
        return False
        
    def should_retrain_model(self):
        """Determine if model needs retraining"""
        # Check model performance degradation
        try:
            with open('models/model_registry.json', 'r') as f:
                registry = json.load(f)
                
            # Check if production model exists
            if 'production_models' in registry:
                for model_name, model_info in registry['production_models'].items():
                    # Check if model is older than 7 days
                    deploy_date = datetime.strptime(model_info['deployment_date'], '%Y-%m-%d')
                    age = datetime.now() - deploy_date
                    
                    if age.days >= 7:
                        self.logger.info(f"Model {model_name} is {age.days} days old, triggering retrain")
                        return True
                        
        except Exception as e:
            self.logger.error(f"Error checking model age: {e}")
            
        return False
        
    def train_models(self):
        """Train models with latest data"""
        self.logger.info("🚀 Starting automated model training")
        
        with mlflow.start_run(run_name=f"automated_training_{datetime.now().strftime('%Y%m%d_%H%M%S')}"):
            try:
                # Import and run training pipeline
                from final_trading_system import FinalTradingSystem
                from fibonacci_signal_detector import FibonacciSignalDetector
                
                # Train Fibonacci detector
                fib_detector = FibonacciSignalDetector()
                
                # Log training parameters
                mlflow.log_param("training_type", "automated")
                mlflow.log_param("training_time", datetime.now().isoformat())
                
                # Simulate training (replace with actual training)
                training_metrics = {
                    'win_rate': 0.55,
                    'accuracy': 0.72,
                    'precision': 0.68,
                    'recall': 0.71
                }
                
                # Log metrics
                for metric, value in training_metrics.items():
                    mlflow.log_metric(metric, value)
                    
                self.logger.info("✅ Automated training completed successfully")
                
                return True
                
            except Exception as e:
                self.logger.error(f"❌ Training failed: {e}")
                mlflow.log_param("training_status", "failed")
                mlflow.log_param("error_message", str(e))
                return False
                
    def run_daily_training_check(self):
        """Daily check for training needs"""
        self.logger.info("🔍 Running daily training check")
        
        new_data_available = self.check_for_new_data()
        should_retrain = self.should_retrain_model()
        
        if new_data_available or should_retrain:
            self.logger.info("📈 Training conditions met, starting training")
            success = self.train_models()
            
            if success:
                self.logger.info("✅ Daily training completed successfully")
            else:
                self.logger.error("❌ Daily training failed")
        else:
            self.logger.info("ℹ️ No training needed today")
            
    def run_scheduler(self):
        """Run the automated pipeline scheduler"""
        self.logger.info("⏰ Starting automated training scheduler")
        
        # Schedule daily training check at 2 AM
        schedule.every().day.at("02:00").do(self.run_daily_training_check)
        
        # Schedule weekly full retrain on Sundays at 3 AM
        schedule.every().sunday.at("03:00").do(self.train_models)
        
        while True:
            schedule.run_pending()
            time.sleep(60)  # Check every minute

def main():
    pipeline = AutomatedTrainingPipeline()
    
    # Run immediate check
    pipeline.run_daily_training_check()
    
    # Start scheduler
    pipeline.run_scheduler()

if __name__ == "__main__":
    main()
'''
        
        with open('automated_training_pipeline.py', 'w') as f:
            f.write(pipeline_script)
            
        print("📄 Automated pipeline created: automated_training_pipeline.py")
        
        return True
        
    def generate_best_practices_guide(self):
        """Generate comprehensive best practices guide"""
        
        guide_content = """# 🎯 MLflow + Anaconda Trading Signals - Best Practices Guide

## 📋 Table of Contents
1. [Environment Setup](#environment-setup)
2. [MLflow Configuration](#mlflow-configuration)
3. [Experiment Design](#experiment-design)
4. [Model Versioning](#model-versioning)
5. [Data Management](#data-management)
6. [Production Deployment](#production-deployment)
7. [Monitoring & Alerts](#monitoring--alerts)
8. [Performance Optimization](#performance-optimization)

## 🐍 Environment Setup

### Anaconda Environment
```bash
# Create dedicated environment
conda create -n mlflow_trading python=3.9
conda activate mlflow_trading

# Install core packages
conda install pandas numpy scikit-learn tensorflow mlflow matplotlib seaborn jupyter

# Install trading-specific packages
pip install MetaTrader5 yfinance websocket-client python-dotenv fastapi uvicorn streamlit
```

### Directory Structure
```
mlflow_trading/
├── data/
│   ├── raw/              # Original data files
│   ├── processed/        # Cleaned data
│   └── features/         # Engineered features
├── models/
│   ├── production/       # Production models
│   ├── experimental/     # Development models
│   └── archived/         # Old model versions
├── experiments/
│   ├── configs/          # Experiment configurations
│   └── results/          # Experiment outputs
├── mlruns/              # MLflow tracking
├── artifacts/           # MLflow artifacts
├── logs/                # System logs
├── signals/             # Trading signals
└── docs/                # Documentation
```

## 🔬 MLflow Configuration

### 1. Tracking Server Setup
```python
import mlflow

# Set tracking URI
mlflow.set_tracking_uri("http://localhost:5000")

# Configure experiment
mlflow.set_experiment("fibonacci_trading_signals")

# Start run with context manager
with mlflow.start_run():
    # Log parameters
    mlflow.log_param("model_type", "ensemble")
    mlflow.log_param("confidence_threshold", 0.65)
    
    # Log metrics
    mlflow.log_metric("win_rate", 0.58)
    mlflow.log_metric("total_trades", 1500)
    
    # Log model
    mlflow.sklearn.log_model(model, "fibonacci_detector")
```

### 2. Experiment Naming Convention
- `fibonacci_v{version}_{date}` - Fibonacci signal experiments
- `ensemble_v{version}_{date}` - Ensemble model experiments  
- `optimization_{parameter}_{date}` - Hyperparameter optimization
- `production_deploy_{date}` - Production deployment tests

### 3. Parameter Logging Standards
```python
# Model parameters
mlflow.log_param("model_algorithm", "random_forest")
mlflow.log_param("n_estimators", 100)
mlflow.log_param("max_depth", 10)

# Data parameters
mlflow.log_param("data_period", "2023-2025")
mlflow.log_param("train_test_split", 0.8)
mlflow.log_param("features_count", 25)

# Trading parameters
mlflow.log_param("fibonacci_levels", ["B_0", "B_-1.8"])
mlflow.log_param("risk_reward_ratio", 2.0)
mlflow.log_param("max_trades_per_day", 10)
```

## 🎯 Experiment Design

### 1. A/B Testing Framework
```python
class TradingExperimentFramework:
    def __init__(self):
        self.experiments = {}
        
    def create_experiment(self, name, variants):
        with mlflow.start_run(run_name=name):
            for variant_name, variant_config in variants.items():
                with mlflow.start_run(run_name=variant_name, nested=True):
                    # Train and evaluate variant
                    results = self.train_variant(variant_config)
                    
                    # Log results
                    for metric, value in results.items():
                        mlflow.log_metric(metric, value)
```

### 2. Experiment Tracking Template
```python
def run_trading_experiment(experiment_config):
    experiment_name = experiment_config['name']
    
    with mlflow.start_run(run_name=experiment_name):
        # 1. Log experiment metadata
        mlflow.log_params(experiment_config['parameters'])
        mlflow.set_tags(experiment_config['tags'])
        
        # 2. Data preparation
        data = prepare_trading_data(experiment_config['data_config'])
        
        # 3. Feature engineering
        features = engineer_features(data, experiment_config['feature_config'])
        
        # 4. Model training
        model = train_model(features, experiment_config['model_config'])
        
        # 5. Evaluation
        metrics = evaluate_model(model, features)
        
        # 6. Log results
        mlflow.log_metrics(metrics)
        mlflow.sklearn.log_model(model, "trading_model")
        
        # 7. Generate reports
        report = generate_experiment_report(metrics, model)
        mlflow.log_text(report, "experiment_report.md")
        
        return metrics, model
```

## 📦 Model Versioning

### 1. Model Registry Pattern
```python
class ModelRegistry:
    def register_model(self, model, model_name, stage="Staging"):
        # Log model to MLflow
        mlflow.sklearn.log_model(
            model, 
            model_name,
            registered_model_name=model_name
        )
        
        # Transition to appropriate stage
        client = mlflow.tracking.MlflowClient()
        latest_version = client.get_latest_versions(
            model_name, 
            stages=[stage]
        )[0]
        
        client.transition_model_version_stage(
            name=model_name,
            version=latest_version.version,
            stage=stage
        )
        
    def promote_to_production(self, model_name, version):
        client = mlflow.tracking.MlflowClient()
        client.transition_model_version_stage(
            name=model_name,
            version=version,
            stage="Production"
        )
```

### 2. Model Performance Comparison
```python
def compare_model_versions(model_name):
    client = mlflow.tracking.MlflowClient()
    
    # Get all versions
    versions = client.search_model_versions(f"name='{model_name}'")
    
    comparison_data = []
    for version in versions:
        run = mlflow.get_run(version.run_id)
        comparison_data.append({
            'version': version.version,
            'stage': version.current_stage,
            'win_rate': run.data.metrics.get('win_rate', 0),
            'total_trades': run.data.metrics.get('total_trades', 0),
            'created': version.creation_timestamp
        })
    
    return pd.DataFrame(comparison_data)
```

## 💾 Data Management

### 1. Data Versioning
```python
import mlflow.data

# Log dataset
dataset = mlflow.data.from_pandas(
    df, 
    source="dataBT/fibonacci_signals.csv",
    name="fibonacci_training_data",
    version="v1.0"
)

with mlflow.start_run():
    mlflow.log_input(dataset, context="training")
```

### 2. Feature Store Integration
```python
class TradingFeatureStore:
    def __init__(self):
        self.features = {}
        
    def create_feature_set(self, name, features_df):
        # Store features
        feature_path = f"features/{name}.parquet"
        features_df.to_parquet(feature_path)
        
        # Log to MLflow
        with mlflow.start_run():
            mlflow.log_artifact(feature_path)
            mlflow.log_param("feature_count", len(features_df.columns))
            mlflow.log_param("sample_count", len(features_df))
            
    def get_features(self, name, version="latest"):
        # Retrieve features from MLflow
        return mlflow.artifacts.load_data(f"features/{name}.parquet")
```

## 🚀 Production Deployment

### 1. Model Serving
```python
import mlflow.pyfunc

class TradingModelServer:
    def __init__(self, model_name, version="latest"):
        self.model = mlflow.pyfunc.load_model(
            model_uri=f"models:/{model_name}/{version}"
        )
        
    def predict_signal(self, market_data):
        # Prepare features
        features = self.prepare_features(market_data)
        
        # Make prediction
        prediction = self.model.predict(features)
        
        # Format signal
        return self.format_signal(prediction, market_data)
```

### 2. A/B Testing in Production
```python
class ProductionABTesting:
    def __init__(self):
        self.model_a = mlflow.pyfunc.load_model("models:/fibonacci_detector/Production")
        self.model_b = mlflow.pyfunc.load_model("models:/fibonacci_detector/Staging")
        
    def get_prediction(self, market_data, user_id):
        # Route traffic based on user ID
        if hash(user_id) % 2 == 0:
            model = self.model_a
            variant = "model_a"
        else:
            model = self.model_b
            variant = "model_b"
            
        prediction = model.predict(market_data)
        
        # Log prediction for analysis
        with mlflow.start_run():
            mlflow.log_param("variant", variant)
            mlflow.log_param("user_id", user_id)
            mlflow.log_metric("prediction_confidence", prediction['confidence'])
            
        return prediction
```

## 📊 Monitoring & Alerts

### 1. Model Performance Monitoring
```python
class ModelPerformanceMonitor:
    def __init__(self):
        self.thresholds = {
            'win_rate': 0.50,  # Minimum 50% win rate
            'daily_trades': 5,  # Minimum 5 trades per day
            'max_drawdown': 0.10  # Maximum 10% drawdown
        }
        
    def check_model_health(self, performance_metrics):
        alerts = []
        
        for metric, threshold in self.thresholds.items():
            if metric in performance_metrics:
                value = performance_metrics[metric]
                
                if metric == 'max_drawdown':
                    # Alert if drawdown exceeds threshold
                    if value > threshold:
                        alerts.append(f"⚠️ High drawdown: {value:.1%}")
                else:
                    # Alert if metric below threshold
                    if value < threshold:
                        alerts.append(f"⚠️ Low {metric}: {value}")
                        
        return alerts
        
    def log_health_check(self, alerts):
        with mlflow.start_run(run_name="health_check"):
            mlflow.log_param("alert_count", len(alerts))
            for i, alert in enumerate(alerts):
                mlflow.log_param(f"alert_{i}", alert)
```

### 2. Automated Alerts
```python
import smtplib
from email.mime.text import MIMEText

class AlertSystem:
    def send_performance_alert(self, message):
        # Email alert
        msg = MIMEText(message)
        msg['Subject'] = 'Trading Model Alert'
        msg['From'] = 'trading-system@yourcompany.com'
        msg['To'] = 'admin@yourcompany.com'
        
        # Send email (configure SMTP settings)
        # smtp_server.send_message(msg)
        
        # Log alert to MLflow
        with mlflow.start_run():
            mlflow.log_param("alert_type", "performance")
            mlflow.log_param("alert_message", message)
            mlflow.log_param("alert_time", datetime.now().isoformat())
```

## ⚡ Performance Optimization

### 1. Parallel Experiment Execution
```python
from concurrent.futures import ThreadPoolExecutor
import mlflow

def run_parallel_experiments(experiment_configs):
    def run_single_experiment(config):
        with mlflow.start_run():
            return train_and_evaluate_model(config)
            
    with ThreadPoolExecutor(max_workers=4) as executor:
        futures = [executor.submit(run_single_experiment, config) 
                  for config in experiment_configs]
        
        results = [future.result() for future in futures]
    
    return results
```

### 2. Experiment Caching
```python
import hashlib
import pickle

class ExperimentCache:
    def __init__(self, cache_dir="cache"):
        self.cache_dir = Path(cache_dir)
        self.cache_dir.mkdir(exist_ok=True)
        
    def get_cache_key(self, config):
        config_str = json.dumps(config, sort_keys=True)
        return hashlib.md5(config_str.encode()).hexdigest()
        
    def get_cached_result(self, config):
        cache_key = self.get_cache_key(config)
        cache_file = self.cache_dir / f"{cache_key}.pkl"
        
        if cache_file.exists():
            with open(cache_file, 'rb') as f:
                return pickle.load(f)
        return None
        
    def cache_result(self, config, result):
        cache_key = self.get_cache_key(config)
        cache_file = self.cache_dir / f"{cache_key}.pkl"
        
        with open(cache_file, 'wb') as f:
            pickle.dump(result, f)
```

## 🎯 Trading-Specific Best Practices

### 1. Signal Quality Metrics
```python
def calculate_trading_metrics(predictions, actual_results):
    metrics = {}
    
    # Win rate
    metrics['win_rate'] = (actual_results > 0).mean()
    
    # Profit factor
    gross_profit = actual_results[actual_results > 0].sum()
    gross_loss = abs(actual_results[actual_results < 0].sum())
    metrics['profit_factor'] = gross_profit / gross_loss if gross_loss > 0 else float('inf')
    
    # Sharpe ratio
    metrics['sharpe_ratio'] = actual_results.mean() / actual_results.std() if actual_results.std() > 0 else 0
    
    # Maximum drawdown
    cumulative_returns = (1 + actual_results).cumprod()
    rolling_max = cumulative_returns.expanding().max()
    drawdown = (cumulative_returns - rolling_max) / rolling_max
    metrics['max_drawdown'] = drawdown.min()
    
    return metrics
```

### 2. Risk Management Integration
```python
class RiskManagedTradingSystem:
    def __init__(self, max_risk_per_trade=0.02, max_daily_risk=0.05):
        self.max_risk_per_trade = max_risk_per_trade
        self.max_daily_risk = max_daily_risk
        self.daily_risk_used = 0
        
    def evaluate_signal(self, signal, account_balance):
        # Check risk limits
        if self.daily_risk_used >= self.max_daily_risk:
            return None  # Skip trade due to daily risk limit
            
        # Calculate position size
        risk_amount = account_balance * self.max_risk_per_trade
        position_size = self.calculate_position_size(signal, risk_amount)
        
        # Log risk metrics
        with mlflow.start_run():
            mlflow.log_metric("position_size", position_size)
            mlflow.log_metric("risk_per_trade", self.max_risk_per_trade)
            mlflow.log_metric("daily_risk_used", self.daily_risk_used)
            
        return {
            'signal': signal,
            'position_size': position_size,
            'risk_amount': risk_amount
        }
```

## 🚀 Quick Start Commands

### Environment Setup
```bash
# Create environment
conda env create -f environment_mlflow_trading.yml
conda activate mlflow_trading

# Start MLflow server
mlflow ui --host 127.0.0.1 --port 5000

# Run dashboard
streamlit run mlflow_trading_dashboard.py
```

### Daily Workflow
```bash
# 1. Activate environment
conda activate mlflow_trading

# 2. Run training pipeline
python automated_training_pipeline.py

# 3. Start paper trading
python paper_trading_system.py

# 4. Monitor performance
streamlit run mlflow_trading_dashboard.py
```

## 📈 Success Metrics

### Model Performance Targets
- **Win Rate**: ≥ 55% (current: 52.4%)
- **Profit Factor**: ≥ 1.5
- **Sharpe Ratio**: ≥ 1.0  
- **Maximum Drawdown**: ≤ 10%

### System Performance Targets
- **Signal Latency**: < 1 second
- **Model Training Time**: < 30 minutes
- **System Uptime**: ≥ 99.5%
- **Data Freshness**: < 5 minutes

---

🎯 **Remember**: Always validate in paper trading before live deployment!
"""
        
        with open('MLFLOW_BEST_PRACTICES_GUIDE.md', 'w') as f:
            f.write(guide_content)
            
        print("📄 Best practices guide created: MLFLOW_BEST_PRACTICES_GUIDE.md")
        
        return True

    def run_complete_setup(self):
        """Run complete MLflow best practices setup"""
        
        print("🎯 MLFLOW + ANACONDA TRADING SETUP")
        print("=" * 60)
        print(f"🕐 Started at: {datetime.now()}")
        print("=" * 60)
        
        setup_steps = [
            ("Anaconda Environment", self.setup_anaconda_environment),
            ("MLflow Tracking Server", self.setup_mlflow_tracking_server),
            ("Experiment Templates", self.create_experiment_templates),
            ("Model Versioning", self.implement_model_versioning),
            ("Monitoring Dashboard", self.create_monitoring_dashboard),
            ("Automated Pipeline", self.create_automated_training_pipeline),
            ("Best Practices Guide", self.generate_best_practices_guide)
        ]
        
        completed_steps = 0
        
        for step_name, step_function in setup_steps:
            try:
                print(f"\n🔧 {step_name}...")
                success = step_function()
                if success:
                    completed_steps += 1
                    print(f"✅ {step_name} completed")
                else:
                    print(f"❌ {step_name} failed")
            except Exception as e:
                print(f"❌ {step_name} error: {e}")
                
        print("\n" + "=" * 60)
        print(f"🎉 SETUP COMPLETE: {completed_steps}/{len(setup_steps)} steps")
        print("=" * 60)
        
        if completed_steps == len(setup_steps):
            print("🎯 ALL SYSTEMS READY FOR ML TRADING!")
            print("\n📋 Next Steps:")
            print("1. conda activate mlflow_trading")
            print("2. Double-click start_mlflow_server.bat")
            print("3. python paper_trading_system.py")
            print("4. streamlit run mlflow_trading_dashboard.py")
        else:
            print("⚠️ Some steps failed. Check logs and retry.")
            
        self.setup_complete = (completed_steps == len(setup_steps))
        return self.setup_complete

def main():
    """Main setup execution"""
    setup = MLflowBestPractices()
    setup.run_complete_setup()

if __name__ == "__main__":
    main()