setup.py

#!/usr/bin/env python3
"""
Setup script for Big Data Text Analysis Project
Handles installation, configuration, and validation
"""

import os
import sys
import subprocess
import logging
from pathlib import Path

def setup_logging():
    """Setup basic logging"""
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s'
    )
    return logging.getLogger(__name__)

def check_python_version():
    """Check if Python version is compatible"""
    if sys.version_info < (3, 8):
        print("❌ Python 3.8 or higher is required")
        sys.exit(1)
    print(f"✅ Python {sys.version_info.major}.{sys.version_info.minor} detected")

def create_virtual_environment():
    """Create virtual environment if it doesn't exist"""
    venv_path = Path("myenv")
    
    if venv_path.exists():
        print("✅ Virtual environment already exists")
        return True
    
    print("📦 Creating virtual environment...")
    try:
        subprocess.run([sys.executable, "-m", "venv", "myenv"], check=True)
        print("✅ Virtual environment created successfully")
        return True
    except subprocess.CalledProcessError:
        print("❌ Failed to create virtual environment")
        return False

def install_dependencies():
    """Install required dependencies"""
    print("📦 Installing dependencies...")
    
    # Determine the pip command based on OS
    if os.name == 'nt':  # Windows
        pip_cmd = "myenv\\Scripts\\pip"
        python_cmd = "myenv\\Scripts\\python"
    else:  # Unix/Linux/Mac
        pip_cmd = "myenv/bin/pip"
        python_cmd = "myenv/bin/python"
    
    try:
        # Try to upgrade pip, but don't fail if it doesn't work
        try:
            print("📦 Upgrading pip...")
            subprocess.run([pip_cmd, "install", "--upgrade", "pip"], 
                         check=True, capture_output=True, text=True)
            print("✅ Pip upgraded successfully")
        except subprocess.CalledProcessError as e:
            print("⚠️  Pip upgrade failed, continuing with current version...")
            # On Windows, try the alternative upgrade method
            if os.name == 'nt':
                try:
                    subprocess.run([python_cmd, "-m", "pip", "install", "--upgrade", "pip"], 
                                 check=True, capture_output=True, text=True)
                    print("✅ Pip upgraded using alternative method")
                except subprocess.CalledProcessError:
                    print("⚠️  Pip upgrade failed, using current version")
        
        # Install requirements
        if Path("requirements.txt").exists():
            print("📦 Installing packages from requirements.txt...")
            subprocess.run([pip_cmd, "install", "-r", "requirements.txt"], check=True)
            print("✅ Dependencies installed successfully")
        else:
            print("⚠️  requirements.txt not found, installing basic packages...")
            basic_packages = [
                "scikit-learn>=1.3.0",
                "spacy>=3.6.0",
                "numpy>=1.24.0",
                "pandas>=2.0.0"
            ]
            for package in basic_packages:
                print(f"📦 Installing {package}...")
                subprocess.run([pip_cmd, "install", package], check=True)
            print("✅ Basic packages installed")
        
        # Install spaCy model
        print("📦 Installing spaCy English model...")
        try:
            subprocess.run([python_cmd, "-m", "spacy", "download", "en_core_web_sm"], 
                         check=True, capture_output=True, text=True)
            print("✅ spaCy model installed")
        except subprocess.CalledProcessError as e:
            print("⚠️  spaCy model installation failed, trying alternative method...")
            # Try installing spacy first if it's not already installed
            subprocess.run([pip_cmd, "install", "spacy"], check=True)
            subprocess.run([python_cmd, "-m", "spacy", "download", "en_core_web_sm"], 
                         check=True, capture_output=True, text=True)
            print("✅ spaCy model installed with alternative method")
        
        return True
        
    except subprocess.CalledProcessError as e:
        print(f"❌ Failed to install dependencies: {e}")
        return False

def create_directories():
    """Create necessary directories"""
    directories = ["input", "output", "results"]
    
    print("📁 Creating directories...")
    for directory in directories:
        Path(directory).mkdir(exist_ok=True)
        print(f"✅ Created directory: {directory}")

def run_tests():
    """Run the test suite"""
    print("🧪 Running tests...")
    
    if os.name == 'nt':  # Windows
        python_cmd = "myenv\\Scripts\\python"
    else:  # Unix/Linux/Mac
        python_cmd = "myenv/bin/python"
    
    try:
        result = subprocess.run([python_cmd, "test_analysis.py"], capture_output=True, text=True)
        
        if result.returncode == 0:
            print("✅ All tests passed!")
            print(result.stdout)
        else:
            print("⚠️  Some tests failed:")
            print(result.stderr)
            
        return result.returncode == 0
        
    except subprocess.CalledProcessError:
        print("❌ Failed to run tests")
        return False

def create_sample_data():
    """Create sample input files for testing"""
    sample_text = """
    This is a sample text for testing the EKU student handbook analysis pipeline.
    The university provides various academic services to students.
    Students must follow the academic policies and conduct guidelines.
    The campus offers many resources for student success.
    """
    
    sample_file = Path("input/sample_input.txt")
    sample_file.parent.mkdir(exist_ok=True)
    
    with open(sample_file, 'w') as f:
        f.write(sample_text)
    
    print(f"✅ Created sample input file: {sample_file}")

def print_usage_instructions():
    """Print usage instructions"""
    print("\n" + "="*60)
    print("🎉 SETUP COMPLETE!")
    print("="*60)
    print("\n📋 Usage Instructions:")
    print("\n1. Activate virtual environment:")
    if os.name == 'nt':  # Windows
        print("   myenv\\Scripts\\activate")
    else:  # Unix/Linux/Mac
        print("   source myenv/bin/activate")
    
    print("\n2. Run the full pipeline:")
    print("   ./run_pipeline.sh")
    
    print("\n3. Run individual analyses:")
    print("   # Word count")
    print("   cat input/sample_input.txt | python preprocess.py | python mapper.py | python reducer.py")
    print("\n   # TF-IDF")
    print("   cat input/sample_input.txt | python preprocess_tfidf.py | python mapper_tfidf.py doc1 | python reducer_tfidf.py")
    print("\n   # LDA")
    print("   cat input/sample_input.txt | python mapper_lda.py | python reducer_lda.py")
    
    print("\n4. Run tests:")
    print("   python test_analysis.py")
    
    print("\n📁 Project structure:")
    print("   - input/     : Place your input files here")
    print("   - output/    : Intermediate outputs")
    print("   - results/   : Final analysis results")
    
    print("\n🔧 Configuration:")
    print("   - Edit config.py to customize parameters")
    print("   - Modify utils.py for custom preprocessing")
    
    print("\n📚 Documentation:")
    print("   - README.md contains detailed usage instructions")
    print("   - All Python files include docstrings and type hints")
    
    print("\n" + "="*60)

def main():
    """Main setup function"""
    logger = setup_logging()
    
    print("🚀 Big Data Text Analysis Project Setup")
    print("="*50)
    
    # Check Python version
    check_python_version()
    
    # Create virtual environment
    if not create_virtual_environment():
        sys.exit(1)
    
    # Install dependencies
    if not install_dependencies():
        sys.exit(1)
    
    # Create directories
    create_directories()
    
    # Create sample data
    create_sample_data()
    
    # Run tests
    tests_passed = run_tests()
    
    # Print instructions
    print_usage_instructions()
    
    if tests_passed:
        print("✅ Setup completed successfully!")
        sys.exit(0)
    else:
        print("⚠️  Setup completed with test warnings")
        sys.exit(0)

if __name__ == "__main__":
    main()