BasisFunctionRegression.py

import numpy as np
import pandas as pd
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_val_score, KFold
import time
import model_evaluation

# this program trains and tests a Polynomial Basis Function Regression model and prints its evalutaion metrics

def optimized_degree_poly_reg_model(degree_values, cv_folds, train_X_df, train_y_df, test_X_df):
    #k fold cross validation
    cv_scores = []
    
    for k in degree_values:
        #Adjust features for polynomial regression
        poly = PolynomialFeatures(degree = k, include_bias=False)

        #train input data
        train_X_df_poly = poly.fit_transform(train_X_df)

        #train model with polynomial features and OLS with optimized weights
        polynomial_regression_model = LinearRegression()

        score = cross_val_score(polynomial_regression_model, train_X_df_poly, train_y_df.values, cv=cv_folds, scoring='neg_root_mean_squared_error')
        rmse_scores = -score
        cv_scores.append(rmse_scores.mean())
        print(f"{k}\t{rmse_scores.mean():.4f}\t\t{rmse_scores.std():.4f}")

    #final model
    optimal_k = degree_values[np.argmin(cv_scores)]
    print(f"\nOptimal k: {optimal_k}")
    print(f"Best CV RMSE: {np.min(cv_scores)}")
        
    optimal_poly = PolynomialFeatures(degree = optimal_k, include_bias=False)
    train_X_df_poly = optimal_poly.fit_transform(train_X_df)
    optimized_model = LinearRegression()
    optimized_model.fit(train_X_df_poly, train_y_df)

    test_X_df_poly = optimal_poly.transform(test_X_df)

    return test_X_df_poly, optimized_model


def main():

    #load dataset
    train_df = pd.read_csv("train.csv")
    test_df = pd.read_csv("test.csv")

    #train data
    train_X_df = train_df.drop("TARGET", axis=1)
    train_y_df = train_df[["TARGET"]]

    #test data
    test_X_df = test_df.drop("TARGET", axis=1)
    test_y_df = test_df[["TARGET"]]

    #needed for k fold cross validation
    #possible degrees are of low value to attempt to avoid overfitting
    degree_values = list(range(1,6))
    cv_folds = KFold(n_splits=5, shuffle=True, random_state=42)

    start = time.time()
    
    #train model and adjust features of test dataset
    test_X_df_poly, polynomial_regression_model = optimized_degree_poly_reg_model(degree_values, cv_folds, train_X_df, train_y_df, test_X_df)

    end = time.time()

    print(f"Training Time: {end-start:.2f} seconds")

    #test model
    polynomial_y_prediction = polynomial_regression_model.predict(test_X_df_poly)

    #poly_y_pred = PolyRegression(k, train_X_df, train_y_df, test_X_df, test_y_df)
    model_evaluation.print_evaluation(test_y_df, polynomial_y_prediction)

    print("BFR END")

if __name__ == "__main__":
    main()