evaluate.py

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import (
    classification_report, roc_auc_score,
    confusion_matrix, precision_recall_curve
)
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split


def evaluate_lstm(model, X_test, y_test):
    loss, acc = model.evaluate(X_test, y_test)
    print("Accuracy:", acc)

    y_prob = model.predict(X_test).flatten()
    auc = roc_auc_score(y_test, y_prob)
    print("ROC-AUC:", auc)

    preds = (y_prob > 0.5).astype(int)
    print("\nClassification Report:")
    print(classification_report(y_test, preds))

    return y_prob, auc


def evaluate_autoencoder(autoencoder, X_test, y_test):
    recon = autoencoder.predict(X_test)
    mse = np.mean(np.square(X_test - recon), axis=(1, 2))

    # Threshold based on normal data
    normal_mse = mse[y_test == 0]
    threshold = normal_mse.mean() + 1.0 * normal_mse.std()
    preds = (mse > threshold).astype(int)

    print("\nClassification Report:")
    print(classification_report(y_test, preds))
    print("Confusion Matrix:")
    print(confusion_matrix(y_test, preds))

    auc = roc_auc_score(y_test, mse)
    print("ROC-AUC:", auc)

    # PR curve
    precision, recall, _ = precision_recall_curve(y_test, mse)
    plt.figure()
    plt.plot(recall, precision)
    plt.xlabel("Recall")
    plt.ylabel("Precision")
    plt.title("Precision-Recall Curve")
    plt.savefig("results/pr_curve.png")
    plt.close()

    # Save results
    pd.DataFrame({
        "accuracy": [None],
        "roc_auc": [auc],
        "threshold": [threshold]
    }).to_csv("results/autoencoder_metrics.csv", index=False)

    pd.DataFrame({
        "y_true": y_test,
        "y_pred": preds,
        "mse": mse
    }).to_csv("results/autoencoder_predictions.csv", index=False)

    return mse, auc


def evaluate_hybrid(lstm_model, transformer_model, autoencoder, X_test, y_test, use_meta_learner=True):
    import numpy as np
    from sklearn.metrics import roc_auc_score, classification_report, confusion_matrix
    from sklearn.linear_model import LogisticRegression
    from sklearn.model_selection import train_test_split

    print(">>> ENTERED HYBRID FUNCTION")

    # Get scores from all three models
    lstm_probs = lstm_model.predict(X_test).flatten()
    transformer_probs = transformer_model.predict(X_test).flatten()

    recon = autoencoder.predict(X_test)
    mse = np.mean(np.square(X_test - recon), axis=(1, 2))
    denom = mse.max() - mse.min()
    mse_norm = (mse - mse.min()) / denom if denom != 0 else np.zeros_like(mse)

    if use_meta_learner:
        # Three component meta-learner
        features = np.column_stack([lstm_probs, transformer_probs, mse_norm])

        X_meta_train, X_meta_test, y_meta_train, y_meta_test = train_test_split(
            features, y_test, test_size=0.5, random_state=42, stratify=y_test
        )

        meta = LogisticRegression()
        meta.fit(X_meta_train, y_meta_train)

        hybrid_score = meta.predict_proba(X_meta_test)[:, 1]
        y_eval = y_meta_test

        print("Meta-learner weights (lstm, transformer, mse):", meta.coef_)

    else:
        hybrid_score = 0.4 * lstm_probs + 0.4 * transformer_probs + 0.2 * mse_norm
        y_eval = y_test

    threshold = np.percentile(hybrid_score, 50)
    preds = (hybrid_score > threshold).astype(int)

    print("\n--- HYBRID RESULTS ---")
    print(classification_report(y_eval, preds))
    print("Confusion Matrix:")
    print(confusion_matrix(y_eval, preds))

    auc = roc_auc_score(y_eval, hybrid_score)
    print("Hybrid ROC-AUC:", auc)

    return auc