milestone1.py

import re
import ast
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import os
import pickle
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.preprocessing import LabelEncoder, MultiLabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.ensemble import RandomForestRegressor, VotingRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

def save_object(obj, filename):
    with open(filename, 'wb') as f:
        pickle.dump(obj, f)

def load_data():
    """Load and return the initial datasets."""
    info_base = pd.read_csv('Milestone 1/info_base_games.csv')
    gamalytic = pd.read_csv('Milestone 1/gamalytic_steam_games.csv')
    dlcs = pd.read_csv('Milestone 1/dlcs.csv')
    demos = pd.read_csv('Milestone 1/demos.csv')
    return info_base, gamalytic, dlcs, demos

def clean_info_base(info_base):
    """Clean the info_base dataframe by removing duplicates."""
    print(f"Initial unique appids: {info_base['appid'].nunique()}")
    print(f"Duplicate appids: {info_base['appid'].duplicated().sum()}")
    
    index = info_base[info_base["appid"].duplicated()].index
    print(f"Duplicate indices: {index}")
    print(info_base.iloc[index])
    
    info_base.drop_duplicates(subset=['appid'], inplace=True)
    print(f"Remaining duplicates: {info_base['appid'].duplicated().sum()}")
    return info_base

def prepare_demos_data(demos):
    """Prepare the demos dataframe by cleaning and validating."""
    demos_df = demos.drop(columns=demos.columns[0])
    
    if 'full_game_appid' in demos_df.columns:
        print(f"Unique full_game_appids: {demos_df['full_game_appid'].nunique()}")
    else:
        print("Column 'full_game_appid' not found in DataFrame")
        print("Available columns:", demos_df.columns.tolist())
    return demos_df

def merge_datasets(info_base, gamalytic, dlcs, demos_df):
    """Merge all datasets into a single dataframe."""
    df = info_base.merge(gamalytic, left_on='appid', right_on='steamId', how='inner')
    
    # Add DLC flag
    dlc_flag = dlcs[['base_appid']].drop_duplicates()
    dlc_flag['has_dlc'] = 1
    df = df.merge(dlc_flag, left_on='appid', right_on='base_appid', how='left')
    df['has_dlc'] = df['has_dlc'].fillna(0).astype(int)
    
    # Add demo flag
    demo_flag = demos_df[['full_game_appid']].drop_duplicates()
    demo_flag['has_demo'] = 1
    df = df.merge(demo_flag, left_on='appid', right_on='full_game_appid', how='left')
    df['has_demo'] = df['has_demo'].fillna(0).astype(int)
    
    return df

def clean_merged_data(df):
    """Clean the merged dataframe by removing unnecessary columns and handling missing values."""
    df.drop(columns=['steamId', 'base_appid', 'full_game_appid'], inplace=True)
    
    # Convert numeric columns
    df['steam_achievements'] = df['steam_achievements'].astype(int)
    df['steam_trading_cards'] = df['steam_trading_cards'].astype(int)
    df['workshop_support'] = df['workshop_support'].astype(int)
    
    # Analyze the percentage of missing values in each column
    null_counts = df.isnull().sum()
    print("\nPercentage of missing values in each column:")
    print(null_counts[null_counts > 0] / len(df))
    # 0% of games have aiContent
    # <5% of games have a metacritic
    # <50% of games have a achievements_total

    # Too many missing values to keep these columns

    # Analyze percentage of games with no demo or dlc
    print("\nPercentage of games with no demo or dlc:")
    print(df['has_demo'].value_counts() / len(df))
    print(df['has_dlc'].value_counts() / len(df))
    # 0% of games have a demo
    # 0% of games have a dlc

    # Too many missing values to keep these columns
    
    # Analyze percentage of unique values in each column
    print("\nPercentage of unique values in each column:")
    print(df.nunique() / len(df))
    # appid is 100% unique
    # name is ~100% unique and is text
    # appid and name are not needed

    # Remove appid and name
    # Remove unnecessary columns
    df.drop(columns=['aiContent', 'metacritic', 'achievements_total', 'has_dlc', 'has_demo', 'name', 'appid'], inplace=True)

    # <1% have missing genres
    # Handle missing values
    df['genres'] = df['genres'].fillna("Unknown")
    
    # Add free game flag
    df['game_is_free'] = (df['price'] == 0).astype(int)
    
    return df

def encode_features(df):
    """Encode categorical features using appropriate encoders."""
    # Encode platforms
    df['platfroms_split'] = df['supported_platforms'].fillna('').apply(lambda x: ast.literal_eval(x))
    mlb_platforms = MultiLabelBinarizer()
    platforms_encoded = pd.DataFrame(mlb_platforms.fit_transform(df['platfroms_split']),
                                   columns=[f"Platform_{p}" for p in mlb_platforms.classes_],
                                   index=df.index)
    df = pd.concat([df.drop(columns=['supported_platforms', 'platfroms_split']), platforms_encoded], axis=1)
    
    # Encode genres
    df['genres_split'] = df['genres'].fillna('').apply(lambda x: [genre.strip() for genre in x.split(',')])
    mlb_genres = MultiLabelBinarizer()
    genres_encoded = pd.DataFrame(mlb_genres.fit_transform(df['genres_split']),
                                columns=[f"Genre_{g}" for g in mlb_genres.classes_],
                                index=df.index)
    df = pd.concat([df.drop(columns=['genres', 'genres_split']), genres_encoded], axis=1)
    
    # Encode publisher class
    le_publisher = LabelEncoder()
    df['publisherClass'] = le_publisher.fit_transform(df['publisherClass'])
    
    return df, mlb_platforms, mlb_genres, le_publisher

def try_parse_date(x):
    """Parse various date formats into datetime objects."""
    for fmt in ("%b %d, %Y", "%b-%y", "%Y"):
        try:
            return pd.to_datetime(x, format=fmt)
        except (ValueError, TypeError):
            continue
    match = re.match(r"(Q[1-4])\s*(\d{4})", x) or re.match(r"(\d{4})\s*(Q[1-4])", x)
    if match:
        parts = match.groups()
        q, y = (parts[0], parts[1]) if 'Q' in parts[0] else (parts[1], parts[0])
        month = {'Q1': 1, 'Q2': 4, 'Q3': 7, 'Q4': 10}[q]
        return pd.Timestamp(year=int(y), month=month, day=1)
    else:
        timestamp_dict = {
            "To be announced": pd.Timestamp(year=2025, month=12, day=31),
            "Coming soon": pd.Timestamp(year=2025, month=12, day=31),
        }
        return timestamp_dict.get(x, pd.NaT)

def process_dates(df):
    """Process and extract features from release dates."""
    df['release_date'] = df['release_date'].apply(try_parse_date)
    df['release_date_weekday'] = df['release_date'].dt.strftime('%w').astype(int)
    df['release_date_month'] = df['release_date'].dt.month.astype(int)
    df['release_date_year'] = df['release_date'].dt.year.astype(int)
    df.drop(columns=['release_date'], inplace=True)
    return df

def perform_eda(df):
    """Perform exploratory data analysis and create visualizations."""
    plt.figure(figsize=(20, 15))
    sns.heatmap(df.corr(), cmap='coolwarm', annot=False)
    plt.title('Correlation Heatmap')
    plt.show()

    plt.figure(figsize=(8, 6))
    sns.histplot(df['reviewScore'], bins=100, kde=True)
    plt.title('Distribution of Review Score')
    plt.show()

    plt.figure(figsize=(8, 6))
    sns.scatterplot(y=df['copiesSold'], x=df['reviewScore'])
    plt.yscale('log')
    plt.title('Copies Sold vs Review Score')
    plt.show()

def safe_target_transform(series):
    """Transform the target variable safely by capping outliers and applying log transformation."""
    cap = series.quantile(0.999)
    clipped = series.clip(upper=cap)
    return np.log1p(clipped)

def train_models(X_train, X_test, y_train, y_test):
    """Train and evaluate multiple regression models."""
    # Random Forest
    rf = RandomForestRegressor(
        n_estimators=300,
        max_depth=7,
        min_samples_leaf=5,
        random_state=42
    )

    # XGBoost
    xgb = XGBRegressor(
        n_estimators=300,
        learning_rate=0.05,
        max_depth=5,
        reg_alpha=0.5,
        reg_lambda=0.5,
        random_state=42
    )

    # Polynomial Regression Pipeline
    # poly_reg = Pipeline([
    #     ('poly', PolynomialFeatures(degree=2)),
    #     ('linear', LinearRegression())
    # ])

    # LGBM
    lgbm = LGBMRegressor(
        n_estimators=300,
        learning_rate=0.05,
        max_depth=5,
        reg_alpha=0.5,
        reg_lambda=0.5,
        random_state=42
    )

    models = {
        "Random Forest": rf,
        "XGBoost": xgb,
        "LGBM": lgbm,
        "Ensemble": VotingRegressor([
            ('rf', rf), 
            ('xgb', xgb),
            ('lgbm', lgbm)
            #('poly', poly_reg)
        ])
    }

    results = []
    for name, model in models.items():
        model.fit(X_train, y_train)
        train_pred = model.predict(X_train)
        test_pred = model.predict(X_test)

        train_rmse = np.sqrt(mean_squared_error(np.expm1(y_train), np.expm1(train_pred)))
        test_rmse = np.sqrt(mean_squared_error(np.expm1(y_test), np.expm1(test_pred)))
        train_r2 = r2_score(y_train, train_pred)
        test_r2 = r2_score(y_test, test_pred)

        results.append({
            'Model': name,
            'Train RMSE': f"{train_rmse:,.2f}",
            'Test RMSE': f"{test_rmse:,.2f}",
            'Train R2': f"{train_r2:.4f}",
            'Test R2': f"{test_r2:.4f}"
        })

    return pd.DataFrame(results), models

def plot_model_results(results_df, models, X_test, y_test):
    """Create visualizations of model results."""
    print("\n-----Model performance ---")
    print(results_df.to_markdown(index=False))

    plt.figure(figsize=(10, 6))
    pd.Series(models['XGBoost'].feature_importances_, index=X_test.columns)\
        .sort_values(ascending=False)\
        .head(15)\
        .plot.barh(color='darkblue')
    plt.title('Top 15 important features (XGBoost)')
    plt.xlabel('importance score')
    plt.tight_layout()
    plt.show()

    plt.figure(figsize=(12, 5))
    for i, (name, model) in enumerate(models.items()):
        plt.subplot(2, 2, i+1)
        
        # Get predictions from the model
        y_pred = model.predict(X_test)
        
        # Convert to numpy arrays for sorting
        y_test_np = y_test.values
        y_pred_np = y_pred
        
        # Sort both true values and predictions for better visualization
        sorted_indices = np.argsort(y_test_np)
        y_test_sorted = y_test_np[sorted_indices]
        y_pred_sorted = y_pred_np[sorted_indices]
        
        # Plot the scatter points
        plt.scatter(y_test_sorted, y_pred_sorted, alpha=0.5)
        
        # Add a line of best fit
        #plt.plot(y_test_sorted, y_pred_sorted, 'r-', linewidth=2)
        
        # Add a perfect prediction line
        plt.plot([min(y_test_sorted), max(y_test_sorted)], 
                [min(y_test_sorted), max(y_test_sorted)], 
                'k--', alpha=0.5)
        
        plt.xlabel('True Values')
        plt.ylabel('Predicted Values')
        plt.title(f'{name} Prediction vs Actual')

    
    plt.tight_layout()
    plt.show()

    plt.figure(figsize=(12, 5))
    for i, (name, model) in enumerate(models.items()):
        plt.subplot(2, 2, i+1)
        errors = y_test - model.predict(X_test)
        plt.hist(errors, bins=50)
        plt.xlabel('Prediction Errors')
        plt.title(f'{name} Error Distribution')
    plt.tight_layout()
    plt.show()

def main():
    """Main function to execute the entire pipeline."""
    # Load and clean data
    info_base, gamalytic, dlcs, demos = load_data()
    info_base = clean_info_base(info_base)
    demos_df = prepare_demos_data(demos)
    
    # Merge and process data
    df = merge_datasets(info_base, gamalytic, dlcs, demos_df)
    df = clean_merged_data(df)
    df, mlb_platforms, mlb_genres, le_publisher = encode_features(df)
    df = process_dates(df)
    
    # Rename columns to remove special characters for lgbm compatibility
    df = df.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))

    # Save intermediate results
    df.to_csv("df.csv", index=False)
    
    # Perform EDA
    perform_eda(df)
    
    # Prepare data for modeling
    target = 'reviewScore'
    df[target] = safe_target_transform(df[target])
    
    y = df[target]
    X = pd.get_dummies(df.drop(columns=[target]), drop_first=True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    
    # Train and evaluate models
    results_df, models = train_models(X_train, X_test, y_train, y_test)
    plot_model_results(results_df, models, X_test, y_test)

    # Create directories for saving
    models_dir = "output/models/regressors"
    preprocessors_dir = "output/preprocessors"
    os.makedirs(models_dir, exist_ok=True)
    os.makedirs(preprocessors_dir, exist_ok=True)

    # Save models
    for name, model in models.items():
        save_object(model, f'{models_dir}/{name.lower().replace(" ", "_")}_model.pkl')

    # Save preprocessors
    save_object(mlb_platforms, f'{preprocessors_dir}/mlb_platforms_milestone1.pkl')
    save_object(mlb_genres, f'{preprocessors_dir}/mlb_genres_milestone1.pkl')
    save_object(le_publisher, f'{preprocessors_dir}/le_publisher_milestone1.pkl')
    
    print("\nSuccessfully saved models and preprocessors.")

if __name__ == "__main__":
    main()