milestone2.py

import re
import ast
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.preprocessing import LabelEncoder, MultiLabelBinarizer, StandardScaler, RobustScaler
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from imblearn.over_sampling import SMOTE
from sklearn.feature_selection import SelectKBest, f_classif
import time
import pickle
import os

def save_object(obj, filename):
    with open(filename, 'wb') as f:
        pickle.dump(obj, f)

def load_object(filename):
    with open(filename, 'rb') as f:
        return pickle.load(f)
def load_data():
    info_base = pd.read_csv('Milestone 2/info_base_games.csv')
    gamalytic = pd.read_csv('Milestone 2/ms2_gamalytic_steam_games.csv')
    dlcs = pd.read_csv('Milestone 2/dlcs.csv')
    demos = pd.read_csv('Milestone 2/demos.csv')
    return info_base, gamalytic, dlcs, demos

def clean_info_base(info_base):
    """Clean the info_base dataframe by removing duplicates."""
    print(f"Initial unique appids: {info_base['appid'].nunique()}")
    print(f"Duplicate appids: {info_base['appid'].duplicated().sum()}")

    index = info_base[info_base["appid"].duplicated()].index
    print(f"Duplicate indices: {index}")
    print(info_base.iloc[index])

    info_base.drop_duplicates(subset=['appid'], inplace=True)
    print(f"Remaining duplicates: {info_base['appid'].duplicated().sum()}")
    return info_base

def prepare_demos_data(demos):
    """Prepare the demos dataframe by cleaning and validating."""
    demos_df = demos.drop(columns=demos.columns[0])

    if 'full_game_appid' in demos_df.columns:
        print(f"Unique full_game_appids: {demos_df['full_game_appid'].nunique()}")
    else:
        print("Column 'full_game_appid' not found in DataFrame")
        print("Available columns:", demos_df.columns.tolist())
    return demos_df

def merge_datasets(info_base, gamalytic, dlcs, demos_df):
    """Merge all datasets into a single dataframe."""
    df = info_base.merge(gamalytic, left_on='appid', right_on='steamId', how='inner')

    # Add DLC flag
    dlc_flag = dlcs[['base_appid']].drop_duplicates()
    dlc_flag['has_dlc'] = 1
    df = df.merge(dlc_flag, left_on='appid', right_on='base_appid', how='left')
    df['has_dlc'] = df['has_dlc'].fillna(0).astype(int)

    # Add demo flag
    demo_flag = demos_df[['full_game_appid']].drop_duplicates()
    demo_flag['has_demo'] = 1
    df = df.merge(demo_flag, left_on='appid', right_on='full_game_appid', how='left')
    df['has_demo'] = df['has_demo'].fillna(0).astype(int)

    return df

def clean_merged_data(df):
    """Clean the merged dataframe by removing unnecessary columns and handling missing values."""
    df.drop(columns=['steamId', 'base_appid', 'full_game_appid'], inplace=True)

    # Convert numeric columns
    df['steam_achievements'] = df['steam_achievements'].astype(int)
    df['steam_trading_cards'] = df['steam_trading_cards'].astype(int)
    df['workshop_support'] = df['workshop_support'].astype(int)

    # Analyze the percentage of missing values in each column
    null_counts = df.isnull().sum()
    print("\nPercentage of missing values in each column:")
    print(null_counts[null_counts > 0] / len(df))

    # Analyze percentage of games with no demo or dlc
    print("\nPercentage of games with no demo or dlc:")
    print(df['has_demo'].value_counts() / len(df))
    print(df['has_dlc'].value_counts() / len(df))

    # Analyze percentage of unique values in each column
    print("\nPercentage of unique values in each column:")
    print(df.nunique() / len(df))

    # Remove unnecessary columns
    df.drop(columns=['aiContent', 'metacritic', 'achievements_total', 'has_dlc', 'has_demo', 'name', 'appid'], inplace=True)

    # Handle missing genres
    df['genres'] = df['genres'].fillna("Unknown")

    # Add free game flag
    df['game_is_free'] = (df['price'] == 0).astype(int)

    return df

def encode_features(df):
    """Encode categorical features using appropriate encoders."""
    # Encode platforms
    df['platforms_split'] = df['supported_platforms'].fillna('').apply(lambda x: ast.literal_eval(x))
    mlb_platforms = MultiLabelBinarizer()
    platforms_encoded = pd.DataFrame(mlb_platforms.fit_transform(df['platforms_split']),
                                   columns=[f"Platform_{p}" for p in mlb_platforms.classes_],
                                   index=df.index)
    df = pd.concat([df.drop(columns=['supported_platforms', 'platforms_split']), platforms_encoded], axis=1)

    # Encode genres
    df['genres_split'] = df['genres'].fillna('').apply(lambda x: [genre.strip() for genre in x.split(',')])
    mlb_genres= MultiLabelBinarizer()
    genres_encoded = pd.DataFrame(mlb_genres.fit_transform(df['genres_split']),
                                columns=[f"Genre_{g}" for g in mlb_genres.classes_],
                                index=df.index)
    df = pd.concat([df.drop(columns=['genres', 'genres_split']), genres_encoded], axis=1)

    
    le_publisher = LabelEncoder()
    df['publisherClass'] = le_publisher.fit_transform(df['publisherClass'])
    
    le_target = LabelEncoder()
    df['reviewScore'] = le_target.fit_transform(df['reviewScore'])

    return df, le_publisher, le_target, mlb_platforms, mlb_genres

def try_parse_date(x):
    for fmt in ("%b %d, %Y", "%b-%y", "%Y"):
        try:
            return pd.to_datetime(x, format=fmt)
        except (ValueError, TypeError):
            continue
    match = re.match(r"(Q[1-4])\s*(\d{4})", x) or re.match(r"(\d{4})\s*(Q[1-4])", x)
    if match:
        parts = match.groups()
        q, y = (parts[0], parts[1]) if 'Q' in parts[0] else (parts[1], parts[0])
        month = {'Q1': 1, 'Q2': 4, 'Q3': 7, 'Q4': 10}[q]
        return pd.Timestamp(year=int(y), month=month, day=1)
    else:
        timestamp_dict = {
            "To be announced": pd.Timestamp(year=2025, month=12, day=31),
            "Coming soon": pd.Timestamp(year=2025, month=12, day=31),
        }
        return timestamp_dict.get(x, pd.NaT)

def process_dates(df):
    """Process and extract features from release dates."""
    df['release_date'] = df['release_date'].apply(try_parse_date)
    df['release_date_weekday'] = df['release_date'].dt.strftime('%w').astype(int)
    df['release_date_month'] = df['release_date'].dt.month.astype(int)
    df['release_date_year'] = df['release_date'].dt.year.astype(int)
    df.drop(columns=['release_date'], inplace=True)
    return df

def perform_eda(df):
    """Perform exploratory data analysis and create visualizations."""
    plt.figure(figsize=(20, 15))
    sns.heatmap(df.corr(), cmap='coolwarm', annot=False)
    plt.title('Correlation Heatmap')
    plt.show()

    plt.figure(figsize=(8, 6))
    sns.histplot(df['reviewScore'], bins=100, kde=True)
    plt.title('Distribution of Review Score')
    plt.show()

    plt.figure(figsize=(8, 6))
    sns.scatterplot(y=df['copiesSold'], x=df['reviewScore'])
    plt.yscale('log')
    plt.title('Copies Sold vs Review Score')
    plt.show()

def tune_hyperparameters(X_train, X_test, y_train, y_test):
    results = []
    # XGBoost tuning
    for depth in [3, 5, 7, 9, 12, 15]:
        model = XGBClassifier(
            max_depth=depth,
            learning_rate=0.05,
            n_estimators=300,
            random_state=42,
            eval_metric='mlogloss'
        )
        model.fit(X_train, y_train)
        acc = accuracy_score(y_test, model.predict(X_test))
        results.append({'Model': 'XGBoost', 'Hyperparameter': 'max_depth', 'Value': depth, 'Accuracy': acc})

    for lr in [0.01, 0.03, 0.05, 0.1, 0.2, 0.3]:
        model = XGBClassifier(
            max_depth=5,
            learning_rate=lr,
            n_estimators=300,
            random_state=42,
            eval_metric='mlogloss'
        )
        model.fit(X_train, y_train)
        acc = accuracy_score(y_test, model.predict(X_test))
        results.append({'Model': 'XGBoost', 'Hyperparameter': 'learning_rate', 'Value': lr, 'Accuracy': acc})

    # Random Forest tuning
    for depth in [5, 8, 10, 15, 20, 25, 30]:
        model = RandomForestClassifier(n_estimators=300, max_depth=depth, random_state=42)
        model.fit(X_train, y_train)
        acc = accuracy_score(y_test, model.predict(X_test))
        results.append({'Model': 'RandomForest', 'Hyperparameter': 'max_depth', 'Value': depth, 'Accuracy': acc})

    for n_est in [50, 100, 200, 300, 500, 700]:
        model = RandomForestClassifier(n_estimators=n_est, max_depth=10, random_state=42)
        model.fit(X_train, y_train)
        acc = accuracy_score(y_test, model.predict(X_test))
        results.append({'Model': 'RandomForest', 'Hyperparameter': 'n_estimators', 'Value': n_est, 'Accuracy': acc})

    # LGBM tuning
    for leaves in [10, 20, 31, 50, 75, 100]:
        model = LGBMClassifier(num_leaves=leaves, learning_rate=0.05, random_state=42)
        model.fit(X_train, y_train)
        acc = accuracy_score(y_test, model.predict(X_test))
        results.append({'Model': 'LGBM', 'Hyperparameter': 'num_leaves', 'Value': leaves, 'Accuracy': acc})

    for lr in [0.01, 0.03, 0.05, 0.1, 0.2, 0.3]:
        model = LGBMClassifier(num_leaves=31, learning_rate=lr, random_state=42)
        model.fit(X_train, y_train)
        acc = accuracy_score(y_test, model.predict(X_test))
        results.append({'Model': 'LGBM', 'Hyperparameter': 'learning_rate', 'Value': lr, 'Accuracy': acc})

    return pd.DataFrame(results)

def get_best_params(tuning_results):
    best_params = {}

    for model in tuning_results['Model'].unique():
        best_params[model] = {}
        model_df = tuning_results[tuning_results['Model'] == model]

        for hyperparam in model_df['Hyperparameter'].unique():
            hyperparam_df = model_df[model_df['Hyperparameter'] == hyperparam]
            best_row = hyperparam_df.loc[hyperparam_df['Accuracy'].idxmax()]
            best_params[model][hyperparam] = best_row['Value']

    return best_params

def train_models(X_train, X_test, y_train, y_test, best_params=None):
    if best_params is None:
        best_params = {}

    xgb = XGBClassifier(
        n_estimators=300,
        learning_rate=float(best_params.get('XGBoost', {}).get('learning_rate', 0.05)),
        max_depth=int(best_params.get('XGBoost', {}).get('max_depth', 5)),
        reg_alpha=0.5,
        reg_lambda=0.5,
        random_state=42,
        eval_metric='mlogloss'
    )

    rf = RandomForestClassifier(
        n_estimators=int(best_params.get('RandomForest', {}).get('n_estimators', 300)),
        max_depth=int(best_params.get('RandomForest', {}).get('max_depth', 10)),
        min_samples_leaf=5,
        random_state=42
    )

    lgbm = LGBMClassifier(
        n_estimators=300,
        learning_rate=float(best_params.get('LGBM', {}).get('learning_rate', 0.05)),
        num_leaves=int(best_params.get('LGBM', {}).get('num_leaves', 31)),
        max_depth=10, 
        min_child_samples=20,
        reg_alpha=0.1,
        reg_lambda=0.1,
        random_state=42
    )

    models = {
        "Random Forest Classifier": rf,
        "XGBoost Classifier": xgb,
        "LGBM Classifier": lgbm,
        "Ensemble Classifier": VotingClassifier([
            ('rf', rf),
            ('xgb', xgb),
            ('lgbm', lgbm)
        ])
    }

    results = []
    class_reports = {}
    conf_matrices = {}

    for name, model in models.items():
        print(f"Training {name} with tuned paramaters...")
        start_train = time.time()
        model.fit(X_train, y_train)
        train_time = time.time() - start_train

        start_test = time.time()
        y_pred = model.predict(X_test)
        test_time = time.time() - start_test

        accuracy = accuracy_score(y_test, y_pred)
        results.append({
            'Model': name,
            'Accuracy': f"{accuracy:.4f}",
            'Train Time (s)': f"{train_time:.2f}",
            'Test Time (s)': f"{test_time:.2f}"
        })

        class_reports[name] = classification_report(y_test, y_pred, zero_division=0)
        conf_matrices[name] = confusion_matrix(y_test, y_pred)

    for name in models.keys():
        print(f"--- {name} Classification Report ---")
        print(class_reports[name])
        print(f"--- {name} Confusion Matrix ---")
        print(conf_matrices[name])
        print("\n")

    return pd.DataFrame(results), models

def handle_outliers(df, columns, method='winsorize', threshold=0.05):
    """Handle outliers without removing them.
    
    Methods:
        - winsorize: Caps outliers at specified percentiles
        - log: Applies log transformation to reduce impact of outliers
        - robust_scale: Uses robust scaling that is less sensitive to outliers
    """
    df_processed = df.copy()
    
    for col in columns:
        if df_processed[col].dtype in [np.float64, np.int64]:
            if method == 'winsorize':
                # Cap values at percentiles
                lower_limit = df_processed[col].quantile(threshold)
                upper_limit = df_processed[col].quantile(1 - threshold)
                df_processed.loc[df_processed[col] < lower_limit, col] = lower_limit
                df_processed.loc[df_processed[col] > upper_limit, col] = upper_limit
                print(f"Winsorized column {col} at {threshold} and {1-threshold} percentiles")
            
            elif method == 'log':
                # Apply log transformation (adding 1 to handle zeros)
                if (df_processed[col] <= 0).any():
                    min_val = abs(df_processed[col].min()) + 1 if df_processed[col].min() < 0 else 0
                    df_processed[col] = np.log1p(df_processed[col] + min_val)
                else:
                    df_processed[col] = np.log1p(df_processed[col])
                print(f"Applied log transformation to column {col}")

    return df_processed

def handle_class_imbalance(X, y):
    """Handle class imbalance using SMOTE."""
    print("Class distribution before upsampling:")
    before_counts = y.value_counts().sort_index()
    print("Raw counts:")
    print(before_counts)
    print("Proportions:")
    print(before_counts / len(y))
    
    # Find minimum samples in any class
    min_samples = before_counts.min()
    
    # If smallest class has very few samples, use a smaller k_neighbors value
    k_neighbors = min(5, min_samples - 1) if min_samples > 1 else 1
    
    if min_samples <= 1:
        print("Warning: Some classes have only 1 sample. Using random oversampling instead of SMOTE.")
        from imblearn.over_sampling import RandomOverSampler
        sampler = RandomOverSampler(random_state=42)
    else:
        print(f"Using SMOTE with k_neighbors={k_neighbors}")
        sampler = SMOTE(random_state=42, k_neighbors=k_neighbors)
    
    X_resampled, y_resampled = sampler.fit_resample(X, y)
    
    print("\nClass distribution after upsampling:")
    after_counts = pd.Series(y_resampled).value_counts().sort_index()
    print("Raw counts:")
    print(after_counts)
    print("Proportions:")
    print(after_counts / len(y_resampled))
    
    return X_resampled, y_resampled

def plot_results(results_df):
    """Plot training time, testing time, and accuracy for each model."""
    # Convert relevant columns to numeric, handling potential errors
    results_df['Train Time (s)'] = pd.to_numeric(results_df['Train Time (s)'], errors='coerce')
    results_df['Test Time (s)'] = pd.to_numeric(results_df['Test Time (s)'], errors='coerce')
    results_df['Accuracy'] = pd.to_numeric(results_df['Accuracy'], errors='coerce')

    # Drop rows where conversion might have failed (resulting in NaN)
    results_df.dropna(subset=['Train Time (s)', 'Test Time (s)', 'Accuracy'], inplace=True)

    plt.figure(figsize=(15, 5))

    plt.subplot(1, 3, 1)
    sns.barplot(x='Model', y='Train Time (s)', data=results_df)
    plt.title('Train Time per Model')
    plt.xticks(rotation=45, ha='right')
    plt.ylabel('Train Time (s)')
    plt.xlabel('Model')

    plt.subplot(1, 3, 2)
    sns.barplot(x='Model', y='Test Time (s)', data=results_df)
    plt.title('Test Time per Model')
    plt.xticks(rotation=45, ha='right')
    plt.ylabel('Test Time (s)')
    plt.xlabel('Model')

    plt.subplot(1, 3, 3)
    sns.barplot(x='Model', y='Accuracy', data=results_df)
    plt.title('Accuracy per Model')
    plt.xticks(rotation=45, ha='right')
    plt.ylabel('Accuracy')
    plt.xlabel('Model')

    plt.tight_layout()
    plt.show()

def main():
    info_base, gamalytic, dlcs, demos = load_data()
    info_base = clean_info_base(info_base)
    demos_df = prepare_demos_data(demos)
    df = merge_datasets(info_base, gamalytic, dlcs, demos_df)
    df = clean_merged_data(df)
    df, le_publisher, le_target, mlb_platforms, mlb_genres = encode_features(df)
    df = process_dates(df)
    
    # Handle outliers in numerical columns without removing data
    numerical_cols = df.select_dtypes(include=[np.float64, np.int64]).columns.tolist()
    numerical_cols = [col for col in numerical_cols if col != 'reviewScore']  # Don't transform target
    df = handle_outliers(df, numerical_cols, method='winsorize', threshold=0.05)
    
    perform_eda(df)

    X = pd.get_dummies(df.drop(columns=['reviewScore']), drop_first=True)
    y = df['reviewScore']
    
    # Feature scaling
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)
    X_scaled = pd.DataFrame(X_scaled, columns=X.columns)

    # Feature selection using ANOVA
    k_best = 20  
    selector = SelectKBest(score_func=f_classif, k=k_best)
    X_selected = selector.fit_transform(X_scaled, y)
    selected_features = X.columns[selector.get_support()]
    print(f"Selected top {k_best} features: {selected_features.tolist()}")
    X_selected = pd.DataFrame(X_selected, columns=selected_features)
    
    # Handle class imbalance
    X_resampled, y_resampled = handle_class_imbalance(X_selected, y)

    X_train, X_test, y_train, y_test = train_test_split(X_resampled, y_resampled, test_size=0.2, random_state=42)
    tuning_results = tune_hyperparameters(X_train, X_test, y_train, y_test)

    best_params = get_best_params(tuning_results)

    results, models = train_models(X_train, X_test, y_train, y_test, best_params=best_params)

    print("\n--- Hyperparameter Tuning Summary ---")
    print(tuning_results)
    print("\nSummary of Model Performances:")
    print(results)
    plot_results(results)
    
    # Create directories for saving
    models_dir = "output/models/classifiers"
    preprocessors_dir = "output/preprocessors"
    os.makedirs(models_dir, exist_ok=True)
    os.makedirs(preprocessors_dir, exist_ok=True)

    for name, model in models.items():
        save_object(model, f'{models_dir}/{name.lower().replace(" ", "_")}_model.pkl')

    save_object(le_publisher, f'{preprocessors_dir}/le_publisher_milestone2.pkl')
    save_object(le_target, f'{preprocessors_dir}/le_target_milestone2.pkl')
    save_object(mlb_platforms, f'{preprocessors_dir}/mlb_platforms_milestone2.pkl')
    save_object(mlb_genres, f'{preprocessors_dir}/mlb_genres_milestone2.pkl')
    save_object(scaler, f'{preprocessors_dir}/standard_scaler_milestone2.pkl')
    save_object(selector, f'{preprocessors_dir}/select_k_best_milestone2.pkl')
    
    print("\nSuccessfully saved models and preprocessors to new paths.")

if __name__ == "__main__":
    main()