Updated model and added new dataset to improve results.

Author: JesusAnaya

.gitignore

@@ -26,6 +26,7 @@ share/python-wheels/
 *.egg
 MANIFEST
 dataset/
+dataset2/
 save/
 
 # PyInstaller

calculate_dataset_std_mean.py

@@ -0,0 +1,25 @@
+import torch
+from dataset_loader import get_full_dataset_loader
+
+
+def batch_mean_and_sd(loader):
+    
+    cnt = 0
+    fst_moment = torch.empty(3)
+    snd_moment = torch.empty(3)
+
+    for images, _ in loader:
+        b, c, h, w = images.shape
+        nb_pixels = b * h * w
+        sum_ = torch.sum(images, dim=[0, 2, 3])
+        sum_of_square = torch.sum(images ** 2, dim=[0, 2, 3])
+        fst_moment = (cnt * fst_moment + sum_) / (cnt + nb_pixels)
+        snd_moment = (cnt * snd_moment + sum_of_square) / (cnt + nb_pixels)
+        cnt += nb_pixels
+
+    mean, std = fst_moment, torch.sqrt(snd_moment - fst_moment ** 2)        
+    return mean,std
+  
+mean, std = batch_mean_and_sd(get_full_dataset_loader(dataset_type='udacity'))
+
+print(f"mean {mean}, and std: {std}")

config.py

@@ -4,14 +4,13 @@
 
 @dataclass
 class Config(object):
+    dataset_type = "udacity"
     batch_size = 50
     num_workers = 8
     shuffle = True
     train_split_size = 0.8
     test_split_size = 0.2
     resize = (66, 200)
-    mean = [0.3568, 0.3770, 0.3691]
-    std = [0.2121, 0.2040, 0.1968]
     epochs_count = 60
     learning_rate = 1e-4
     weight_decay = 1e-5
@@ -25,5 +24,8 @@ class Config(object):
     device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
     test_interval = 1
 
+    scheduler_step_size = 10  # Decrease the learning rate every 10 epochs
+    scheduler_gamma = 0.1     # Multiply the learning rate by 0.1 when decreasing
+
 
 config = Config()

dataset_loader.py

@@ -11,7 +11,7 @@
 
 
 transform_img = transforms.Compose([
-    transforms.Resize(config.resize),
+    transforms.Resize(config.resize, antialias=True),
     transforms.ToTensor(),
 ])
 
@@ -35,23 +35,80 @@ def __getitem__(self, idx):
         img_name = os.path.join(os.path.join(self.root_dir, "data"), self.dataframe.iloc[idx, 0])
         image = Image.open(img_name)
         width, height = image.size
-        area = (0, 90, width, height)
+        area = (100, 125, width, height)
         cropped_img = image.crop(area)
 
         y = np.radians(self.dataframe.iloc[idx, 1])
         if self.transform:
             cropped_img = self.transform(cropped_img)
 
-        return cropped_img.float(), np.float(y)
+        return cropped_img, float(y)
+    
+    @staticmethod
+    def get_mean():
+        return [0.3568, 0.3770, 0.3691]
 
+    @staticmethod
+    def get_std():
+        return [0.2121, 0.2040, 0.1968]
+
+
+class UdacityDataset(Dataset):
+    def __init__(self, csv_file="interpolated.csv", root_dir="dataset2", transform=None):
+        self.transform = transform
+        self.dataset_folder = root_dir
+        self.data = pd.read_csv(os.path.join(root_dir, csv_file))
+
+        # Filter for center_camera images only
+        self.data = self.data[self.data['frame_id'] == 'center_camera']
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+
+        img_name = os.path.join(self.dataset_folder, self.data.iloc[idx]['filename'])
+        image = Image.open(img_name)
+        width, height = image.size
+        area = (0, 180, width, height)
+        cropped_img = image.crop(area)
+
+        angle = np.radians(self.data.iloc[idx]['angle'])
+
+        if self.transform:
+            image = self.transform(cropped_img)
+
+        return image, float(angle)
+    
+    @staticmethod
+    def get_mean():
+        return [0.2957, 0.3153, 0.3688]
+
+    @staticmethod
+    def get_std():
+        return [0.2556, 0.2609, 0.2822]
+
+
+def get_data_subsets_loaders(dataset_type='sully') -> (DataLoader, DataLoader):
+    dataset_class = None
+    
+    if dataset_type == 'sully':
+        dataset_class = SullyChenDataset
+    elif dataset_type == 'udacity':
+        dataset_class = UdacityDataset
+    else:
+        raise ValueError("Invalid dataset type")
 
-def get_data_subsets_loaders() -> (DataLoader, DataLoader):    
     transform_img = transforms.Compose([
         transforms.ToTensor(),
-        transforms.Resize(config.resize),
-        transforms.Normalize(config.mean, config.std)
+        transforms.Resize(config.resize, antialias=True),
+        transforms.Normalize(dataset_class.get_mean(), dataset_class.get_std())
     ])
-    dataset = SullyChenDataset(transform=transform_img)
+
+    dataset = dataset_class(transform=transform_img)
+
     train_set, val_set = random_split(dataset, [config.train_split_size, config.test_split_size])
 
     train_subset_loader = DataLoader(
@@ -69,8 +126,12 @@ def get_data_subsets_loaders() -> (DataLoader, DataLoader):
     return train_subset_loader, val_subset_loader
 
 
-def get_full_dataset() -> DataLoader:
-    dataset = SullyChenDataset(transform=transform_img)
+def get_full_dataset_loader(dataset_type='sully') -> DataLoader:
+    if dataset_type == 'sully':
+        dataset = SullyChenDataset(transform=transform_img)
+    elif dataset_type == 'udacity':
+        dataset = UdacityDataset(transform=transform_img)
+
     full_dataset_loader = DataLoader(
         dataset,
         batch_size=1,
@@ -80,5 +141,8 @@ def get_full_dataset() -> DataLoader:
     return full_dataset_loader
 
 
-def get_inference_dataset() -> DataLoader:
-    return SullyChenDataset(transform=transform_img)
+def get_inference_dataset(dataset_type='sully') -> DataLoader:
+    if dataset_type == 'sully':
+        return SullyChenDataset(transform=transform_img)
+    elif dataset_type == 'udacity':
+        return UdacityDataset(transform=transform_img)

inference_run_dataset.py

@@ -21,7 +21,8 @@ def main():
     dataset_iterator = iter(dataset)
 
     model = NvidiaModel()
-    model.load_state_dict(torch.load("./save/model.pt"))
+    model.load_state_dict(torch.load("./save/model.pt", map_location=torch.device(config.device)))
+    model.to(config.device)
     model.eval()
 
     steering_wheel_1 = cv2.imread('./steering_wheel_tesla.jpg', 0)
@@ -33,15 +34,17 @@ def main():
 
     while cv2.waitKey(20) != ord('q'):
         transformed_image, image, target = next(dataset_iterator)
+        transformed_image = transformed_image.to(config.device)
+
         batch_t = torch.unsqueeze(transformed_image, 0)
 
         # Predictions
         with torch.no_grad():
             y_predict = model(batch_t)
 
         # Converting prediction to degrees
-        pred_degrees = np.degrees(y_predict[0].item() * 2)
-        target_degrees = np.degrees(target.item() * 2)
+        pred_degrees = np.degrees(y_predict[0].item())
+        target_degrees = np.degrees(target)
 
         print(f"Predicted Steering angle: {pred_degrees}")
         print(f"Steering angle: {pred_degrees} (pred)\t {target_degrees} (actual)")

inference_run_video.py

@@ -8,7 +8,7 @@
 
 transform_img = transforms.Compose([
     transforms.ToTensor(),
-    transforms.Resize(config.resize),
+    transforms.Resize(config.resize, antialias=True),
     transforms.Normalize(config.mean, config.std)
 ])
 
@@ -22,14 +22,15 @@ def angel_to_steer(degrees, cols, rows, smoothed_angle):
 def crop_down(image):
     h = image.shape[0]
     w = image.shape[1]
-    y = 150
-    x = 60
-    return image[60:int(y+h), int(x):int(x+(w-(x+90)))]
+    y = 350
+    x = 90
+    return image[190:int(y+h), int(x):int(x+(w-(x+150)))]
 
 
 def main():
     model = NvidiaModel()
-    model.load_state_dict(torch.load("./save/model.pt"))
+    model.load_state_dict(torch.load("./save/model.pt", map_location=torch.device(config.device)))
+    model.to(config.device)
     model.eval()
 
     steering_wheel_1 = cv2.imread('./steering_wheel_tesla.jpg', 0)
@@ -49,19 +50,19 @@ def main():
             break
 
         image_cropped = crop_down(image)
-        frame = transform_img(cv2.cvtColor(image_cropped, cv2.COLOR_BGR2RGB)).double()
+        frame = transform_img(cv2.cvtColor(image_cropped, cv2.COLOR_BGR2RGB)).to(config.device)
         batch_t = torch.unsqueeze(frame, 0)
 
         # Predictions
         with torch.no_grad():
             y_predict = model(batch_t)
 
         # Converting prediction to degrees
-        pred_degrees = np.degrees(y_predict[0].item() * 2)
+        pred_degrees = np.degrees(y_predict.item())
 
         print(f"Predicted Steering angle: {pred_degrees}")
         print(f"Steering angle: {pred_degrees} (pred)")
-        cv2.imshow("frame", image_cropped)
+        cv2.imshow("frame", image)
 
         # make smooth angle transitions by turning the steering wheel based on the difference of the current angle
         # and the predicted angle

loss_comparison.png

@@ -1,12 +1,9 @@
-import torch
 import torch.nn as nn
-import scipy
 
 
 class NvidiaModel(nn.Module):
     def __init__(self):
         super().__init__()
-        self.scale_factor = 2
 
         # define layers using nn.Sequential
         self.conv_layers = nn.Sequential(
@@ -68,5 +65,4 @@ def __init__(self):
     def forward(self, x):
         x = self.conv_layers(x)
         x = self.flat_layers(x)
-        x = torch.flatten(x)
-        return x
+        return x.squeeze()

model.py

@@ -0,0 +1,37 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import argparse
+
+def plot_loss_comparison(file_relu, file_elu):
+    # Read the CSV files
+    df_relu = pd.read_csv(file_relu)
+    df_elu = pd.read_csv(file_elu)
+
+    # Create a figure and axis for the plot
+    fig, ax = plt.subplots()
+
+    # Plot the loss values from both files
+    ax.plot(df_relu.index, df_relu['loss'], label="Loss ReLU")
+    ax.plot(df_elu.index, df_elu['loss'], label="Loss ELU")
+
+    # Set labels for the x and y axes
+    ax.set_xlabel("Iteration")
+    ax.set_ylabel("Loss")
+
+    # Add a legend
+    ax.legend()
+
+    # Save the plot to a PNG file
+    fig.savefig("loss_comparison.png")
+
+    # Optionally, close the plot
+    plt.close(fig)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Compare loss values from two CSV files.")
+    parser.add_argument("file_relu", type=str, help="Path to the first (ReLU) CSV file")
+    parser.add_argument("file_elu", type=str, help="Path to the second (ELU) CSV file")
+
+    args = parser.parse_args()
+
+    plot_loss_comparison(args.file_relu, args.file_elu)

plot_loss_results.py

@@ -1,13 +1,19 @@
 import os
+import time
 import torch
 import torch.nn as nn
 import torch.optim as optim
 import torch.nn.functional as F
+import torch.optim.lr_scheduler as lr_scheduler
 import dataset_loader
 import numpy as np
 import pandas as pd
 from model import NvidiaModel
 from config import config
+import argparse
+
+parser = argparse.ArgumentParser(description="Compare loss values from two CSV files.")
+parser.add_argument("--dataset_type", type=str, help="Dataset type", choices=['sully', 'udacity'], default='sully')
 
 
 def save_model(model, log_dir="./save"):
@@ -52,8 +58,10 @@ def validation(model, val_subset_loader, loss_function):
 
 
 def main():
+    args = parser.parse_args()
+
     # train over the dataset about 30 times
-    train_subset_loader, val_subset_loader = dataset_loader.get_data_subsets_loaders()
+    train_subset_loader, val_subset_loader = dataset_loader.get_data_subsets_loaders(dataset_type=args.dataset_type)
     test_loader = iter(val_subset_loader)
     num_images = len(train_subset_loader.dataset) + len(val_subset_loader.dataset)
 
@@ -63,6 +71,9 @@ def main():
     # Optimizer
     optimizer = optim.Adam(model.parameters(), lr=config.learning_rate, weight_decay=config.weight_decay)
 
+    # Learning rate scheduler
+    scheduler = lr_scheduler.StepLR(optimizer, step_size=config.scheduler_step_size, gamma=config.scheduler_gamma)
+
     # Loss function using MSE
     loss_function = nn.MSELoss()
 
@@ -71,6 +82,8 @@ def main():
     batch_loss_mean = np.array([])
     batch_val_loss = np.array([])
 
+    start_time = time.time()  # record the start time
+
     for epoch in range(config.epochs_count):
         # change model in training mood
         model.train()
@@ -101,15 +114,23 @@ def main():
                 batch_loss_mean = np.append(batch_loss_mean, [epoch_loss])
                 print(f'Epoch: {epoch+1}/{config.epochs_count} Batch {batch_idx} \nTrain Loss: {epoch_loss:.6f}')
 
+        # Update learning rate
+        scheduler.step()
+
         val_loss_mean = validation(model, val_subset_loader, loss_function)
         batch_val_loss = np.append(batch_val_loss, [val_loss_mean.item()])
         save_model(model)
-    
+
+    end_time = time.time()  # record the end time
+    elapsed_time = end_time - start_time  # calculate the elapsed time
+    print(f"Training took {elapsed_time:.2f} seconds")
+
     if not os.path.exists('logs'):
         os.makedirs('logs')
 
-    pd.DataFrame({"loss": batch_loss_mean}).to_csv("logs/loss_acc_results.csv", index=None)
-    pd.DataFrame({"val_loss": batch_val_loss}).to_csv("logs/loss_acc_validation.csv", index=None)
+    pd.DataFrame({"loss": batch_loss_mean}).to_csv(f"logs/loss_acc_results_{config.dataset_type}.csv", index=None)
+    pd.DataFrame({"val_loss": batch_val_loss}).to_csv(f"logs/loss_acc_validation_{config.dataset_type}.csv", index=None)
+
     print("loss_acc_results.csv saved!")