ml_lab1_label2.py

# -*- coding: utf-8 -*-
"""ML-lab1-label2-190429G.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1oHESNO8yjvMGsZIQ4d4QgwIcM0btEI6_
"""

from google.colab import drive
drive.mount('/content/drive')

"""Import necssary libraries and modules"""

#import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split

from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor

from sklearn.metrics import mean_squared_error, r2_score

"""Import training, validation and testing datasets"""

# file paths for the datasets
train_path = '/content/drive/MyDrive/Colab Notebooks/ML/Lab 1/train.csv'
valid_path = '/content/drive/MyDrive/Colab Notebooks/ML/Lab 1/valid.csv'
test_path = '/content/drive/MyDrive/Colab Notebooks/ML/Lab 1/test.csv'

# Load the train dataset
train_data = pd.read_csv(train_path)
# Load the valid dataset
valid_data = pd.read_csv(valid_path)
# Load the test dataset
test_data = pd.read_csv(test_path)

"""Visualize original training data"""

train_data.head()

"""Process the data to remove null values for labels and estimate missing values in features

Drop the columns where there are null values for the lables in the training dataset
"""

# Check for null values in train dataset
train_null_counts = train_data.isnull().sum()
print("train null counts : \n {}".format(train_null_counts))

# Drop rows with null values in the final four columns (target labels) for train dataset
train_data = train_data.dropna(subset=train_data.columns[-4:], how='any')

"""Fill the null values in the features with their means in the train, valid and test datasets."""

# Fill null values with mean in train dataset
train_data = train_data.fillna(train_data.mean())

# Fill null values with mean in valid dataset
valid_data = valid_data.fillna(valid_data.mean())

# Fill null values with mean in test dataset
test_data = test_data.fillna(test_data.mean())

"""Visualize processed training data"""

train_data.head()

"""Separate features and labels in the train, valid and test datasets"""

# Separate features and labels in train dataset
train_features = train_data.iloc[:, :-4]
train_labels = train_data.iloc[:, -4:]

# Separate features and labels in valid dataset
valid_features = valid_data.iloc[:, :-4]
valid_labels = valid_data.iloc[:, -4:]

# Separate features and labels in test dataset
test_features = test_data.iloc[:, :-4]
test_labels = test_data.iloc[:, -4:]

"""Extract the second label in the train, valid and test datasets"""

# get the second label of the train dataset
train_label2 = train_labels.iloc[:,1]

# get the second label of the valid dataset
valid_label2 = valid_labels.iloc[:,1]

# get the second label of the test dataset
test_label2 = test_labels.iloc[:,1]

"""# Predicting Label 2 without Feature Engineering

Predict label 2 without feature engineering steps and techniques

Make copies of the features and labels of the datasets to be used in the models without feature engineering
"""

# Make a copy features and labels in train dataset
train_features_copy = train_features.copy()
train_labels_copy = train_labels.copy()

# Make a copy features and labels in valid dataset
valid_features_copy = valid_features.copy()
valid_labels_copy = valid_labels.copy()

# Make a copy features and labels in test dataset
test_features_copy = test_features.copy()
test_labels_copy = test_labels.copy()

"""Make copies of the label 2 of the datasets to be used in the models without feature engineering"""

# Make a copy of the second label of the train dataset
train_label2_copy = train_label2.copy()

# Make a copy of the second label of the valid dataset
valid_label2_copy = valid_label2.copy()

# Make a copy of the second label of the test dataset
test_label2_copy = test_label2.copy()

"""Standardize the features of all datasets"""

# Standardize the features
scaler = StandardScaler()
train_features_copy = scaler.fit_transform(train_features_copy)
valid_features_copy = scaler.transform(valid_features_copy)
test_features_copy = scaler.transform(test_features_copy)

"""Use the raw scaled features to train the best model which is KNN Regressor"""

best_model = KNeighborsRegressor()

best_model.fit(train_features_copy, train_label2_copy)

"""Used the trained model on all features to predict the valid and get metrics"""

# Predict on the train data
y_pred_base_train = best_model.predict(train_features_copy)

# Calculate metrics for classification evaluation
mse = mean_squared_error(train_label2_copy, y_pred_base_train)
r2s = r2_score(train_label2_copy, y_pred_base_train)

print(f"Metrics for KNeighborsRegressor on train data:")
print(f"Mean Squared Error: {mse:.2f}")
print(f"R2 Score: {r2s:.2f}")
print("\n")

# Predict on the validation data
y_pred_base_valid = best_model.predict(valid_features_copy)

# Calculate metrics for classification evaluation on validation data
mse = mean_squared_error(valid_label2_copy, y_pred_base_valid)
r2s = r2_score(valid_label2_copy, y_pred_base_valid)

print(f"Metrics for KNeighborsRegressor on valid data:")
print(f"Mean Squared Error: {mse:.2f}")
print(f"R2 Score: {r2s:.2f}")
print("\n")

"""Predict the label 2 on test data"""

# Predict on the test data
y_pred_base_test = best_model.predict(test_features_copy)

"""# Predicting Label 2 with Feature Engineering

Predict label 2 with feature engineering steps and techniques

## Feature Engineering

Use feature selection based on correlation matrix and feature extraction based on PCA

### Feature Selection

Visualize the distribution of the training label 2
"""

# Plotting the distribution of train_label2
labels, counts = np.unique(train_label2, return_counts=True)

plt.figure(figsize=(10, 6))
plt.xticks(labels)
plt.bar(labels, counts)
plt.xlabel('Target Label 2')
plt.ylabel('Frequency')
plt.title('Distribution of Target Label 2')
plt.show()

"""Calculate the correlation matrix of the training data features"""

#Calculate the correlation matrix
correlation_matrix = train_features.corr()

mask = np.triu(np.ones_like(correlation_matrix))

# Create a heatmap of the correlation matrix using seaborn
plt.figure(figsize=(12, 12))
sns.heatmap(correlation_matrix, cmap='coolwarm', center=0, mask=mask)
plt.title("Correlation Matrix")
plt.show()

"""Identify the features that are highly correlated with each other using the traning dataset"""

# Set the threshold for correlation
correlation_threshold = 0.9

highly_correlated = set()

# Find highly correlated features
for i in range(len(correlation_matrix.columns)):
    for j in range(i):
        if abs(correlation_matrix.iloc[i, j]) > correlation_threshold:
            colname = correlation_matrix.columns[i]
            highly_correlated.add(colname)

print(highly_correlated)

"""Remove the previously identified highly correlated features from all the datasets"""

# Remove highly correlated features
train_features = train_features.drop(columns=highly_correlated)
valid_features = valid_features.drop(columns=highly_correlated)
test_features = test_features.drop(columns=highly_correlated)

"""Display the resulting feature shapes of the datasets"""

# Display the filtered train feature count
print("Filtered train features: {}".format(train_features.shape))

# Display the filtered valid feature count
print("Filtered valid features: {}".format(valid_features.shape))

# Display the filtered test feature count
print("Filtered test features: {}".format(test_features.shape))

"""Identify the features that are highly correlated with the label using the traning dataset"""

# Calculate the correlation matrix between features and train_label2
correlation_with_target = train_features.corrwith(train_label2)

# Set the correlation threshold
correlation_threshold = 0.05

# Select features that meet the correlation threshold
highly_correlated_features = correlation_with_target[correlation_with_target.abs() > correlation_threshold]

print(highly_correlated_features)

"""Extract the features that are only highly correlated with the label from all datasets"""

# Drop the features with low correlated in train data
train_features = train_features[highly_correlated_features.index]

# Drop the features with low correlated in valid data
valid_features = valid_features[highly_correlated_features.index]

# Drop the features with low correlated in test data
test_features = test_features[highly_correlated_features.index]

"""Display the resulting feature shapes of the datasets"""

# Display the filtered train feature count
print("Filtered train features: {}".format(train_features.shape))

# Display the filtered valid feature count
print("Filtered valid features: {}".format(valid_features.shape))

# Display the filtered test feature count
print("Filtered test features: {}".format(test_features.shape))

"""Standardize the features of all datasets"""

# Standardize the features
scaler = StandardScaler()
standardized_train_features = scaler.fit_transform(train_features)
standardized_valid_features = scaler.transform(valid_features)
standardized_test_features = scaler.transform(test_features)

"""### Feature Extraction

Extract can combine the features that are highly significant in predicting the label using Principal Componenet Analysis(PCA)

Extract the features that can explain the variance of the label to 99%

Display the resulting explained variances of each principal component
"""

variance_threshold = 0.99

# Apply PCA with the determined number of components
pca = PCA(n_components=variance_threshold, svd_solver='full')

pca_train_result = pca.fit_transform(standardized_train_features)
pca_valid_result = pca.transform(standardized_valid_features)
pca_test_result = pca.transform(standardized_test_features)

# Explained variance ratio after dimensionality reduction
explained_variance_ratio_reduced = pca.explained_variance_ratio_
print("Explained Variance Ratio after Dimensionality Reduction:", explained_variance_ratio_reduced)

# Plot explained variance ratio
plt.figure(figsize=(18, 10))
plt.bar(range(1, pca_train_result.shape[1] + 1), explained_variance_ratio_reduced)
plt.xlabel('Principal Component')
plt.ylabel('Explained Variance Ratio')
plt.title('Explained Variance Ratio per Principal Component (Reduced)')
plt.show()

# Display the reduced train feature matrix
print("Reduced Train feature matrix shape: {}".format(pca_train_result.shape))
# Display the reduced valid feature matrix
print("Reduced valid feature matrix shape: {}".format(pca_valid_result.shape))
# Display the reduced test feature matrix
print("Reduced test feature matrix shape: {}".format(pca_test_result.shape))

"""## Model Selection

Select the model that best predicts the valid and test datasets based on root mean squared error and r2 score
"""

# Define a list of classification models
classification_models = [
    # ('Linear Regression', LinearRegression()),
    ('K Neighbors', KNeighborsRegressor()),
    # ('Decision Tree', DecisionTreeRegressor()),
    # ('Random Forest', RandomForestRegressor()),
    # ('XGBoost', XGBRegressor())
]

# KNN Regressor is the best model

# Number of features used in PCA
num_features = pca_train_result.shape[1]
print(f"Number of features: {num_features}\n")

# Train and evaluate each classification model
for model_name, model in classification_models:
    # Train the model on the training data
    model.fit(pca_train_result, train_label2)

    # Predict on the train data
    y_pred_train = model.predict(pca_train_result)

    # Calculate metrics for classification evaluation
    mse = mean_squared_error(train_label2, y_pred_train)
    r2s = r2_score(train_label2, y_pred_train)

    print(f"Metrics for {model_name} on train data:")
    print(f"Mean Squared Error: {mse:.2f}")
    print(f"R2 Score: {r2s:.2f}")
    print("\n")

    # Predict on the validation data
    y_pred_valid = model.predict(pca_valid_result)

    # Calculate metrics for classification evaluation on validation data
    mse = mean_squared_error(valid_label2, y_pred_valid)
    r2s = r2_score(valid_label2, y_pred_valid)

    print(f"Metrics for {model_name} on validation data:")
    print(f"Mean Squared Error: {mse:.2f}")
    print(f"R2 Score: {r2s:.2f}")
    print("\n")

    # Predict on the test data
    y_pred_test = model.predict(pca_test_result)

"""# Generate Output CSV

Define method to create the csv file
"""

# define method to create the dataframe and save it as a csv file
def create_csv(features, pred_before_fe, pred_after_fe, destination):
  feature_count = features.shape[1]

  header_row = [f"new_feature_{i}" for i in range(1,feature_count+1)]

  df = pd.DataFrame(features, columns  = header_row)

  df.insert(loc=0, column='Predicted labels before feature engineering', value=pred_before_fe)
  df.insert(loc=1, column='Predicted labels after feature engineering', value=pred_after_fe)
  df.insert(loc=2, column='No of new features', value=np.repeat(feature_count, features.shape[0]))

  df.to_csv(destination, index=False)

"""Create CSV file"""

destination = '/content/drive/MyDrive/Colab Notebooks/ML/Lab 1/190429G_label_2.csv'

# create the csv output file
create_csv(pca_test_result, y_pred_base_test, y_pred_test, destination)