UNION-pipeline__Do-evaluation-after-training__nuScenes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1b218758",
   "metadata": {},
   "source": [
    "## UNION: Unsupervised 3D Object Detection using Appearance-based Pseudo-Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e038d542",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "intermediate_results_root = 'PUT_YOUR_DIRECTORY_HERE'\n",
    "data_root                 = 'PUT_YOUR_DIRECTORY_HERE'\n",
    "\n",
    "\n",
    "assert intermediate_results_root!='PUT_YOUR_DIRECTORY_HERE', print('Folder for storing UNION results. Change to directory in your file system!')\n",
    "assert data_root!='PUT_YOUR_DIRECTORY_HERE', print('Directory to nuScenes dataset. Change to directory in your file system!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83aaf157-5257-452e-ac74-3a97c9ed5090",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "\n",
    "\n",
    "evaluation_dir = 'evaluation-results'\n",
    "if not os.path.exists(evaluation_dir):\n",
    "    os.mkdir(evaluation_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f5063d1",
   "metadata": {},
   "source": [
    "## Create nuScenes object\n",
    "\n",
    "- `sample_record = nusc.get('sample', sample_token)`\n",
    "- `sensor_data_record = nusc.get('sample_data', sample_sensor_token)`\n",
    "- `sensor_egopose_record = nusc.get('ego_pose', sensor_egopose_token)` \n",
    "- `sensor_pose_record = nusc.get('calibrated_sensor', sensor_pose_token)`\n",
    "- `annot_record = nusc.get('sample_annotation', annot)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31e97c47",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from nuscenes.nuscenes import NuScenes\n",
    "from nuscenes.utils.splits import train, val\n",
    "from utils.utils_functions import get_scene_information\n",
    "\n",
    "\n",
    "\n",
    "nuscenes_version = 'v1.0-trainval'\n",
    "nusc             = NuScenes(version=nuscenes_version, dataroot=data_root, verbose=False)\n",
    "\n",
    "\n",
    "scenes = get_scene_information(nusc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de36f6f5-de21-408a-bbf4-1452b05578e2",
   "metadata": {},
   "source": [
    "## Evaluate class-agnostic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c39fb280-623e-413c-89a4-bb801483012d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "experiment_names_ca = ['CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Class-Agnostic-Training__Labels-GT__UNION-file',   # Class-agnostic train - Ground truth.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Class-Agnostic-Training__Labels-HDBSCAN__UNION-file',   # Class-agnostic train - HDBSCAN.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Class-Agnostic-Training__Labels-Scene-Flow__UNION-file',   # Class-agnostic train - Scene flow.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Class-Agnostic-Training__Labels-UNION__UNION-file',   # Class-agnostic train - UNION.\n",
    "                      ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7825c37b-3a94-49c1-aecb-cc641e60e407",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from nuscenes.eval.detection.evaluate import DetectionEval\n",
    "from nuscenes.eval.common.config import config_factory\n",
    "\n",
    "\n",
    "\n",
    "for exp in experiment_names_ca:\n",
    "    # Create evaluate instance.\n",
    "    results_dir = os.path.join('mmdetection3d', 'work_dirs', exp, 'pred_instances_3d', 'results_nusc.json')\n",
    "    nusc_eval   = DetectionEval(nusc, config=config_factory('detection_cvpr_2019'), result_path=results_dir, eval_set='val', output_dir=os.path.join(evaluation_dir, exp), verbose=1)\n",
    "    \n",
    "    \n",
    "    # Make it class-agnostic (everything becomes car).\n",
    "    for sample_token in nusc_eval.gt_boxes.boxes:\n",
    "        for box in nusc_eval.gt_boxes.boxes[sample_token]:\n",
    "            if box.detection_name in ['bicycle', 'bus', 'car', 'construction_vehicle', 'motorcycle', 'pedestrian', 'trailer', 'truck']:\n",
    "                box.detection_name = 'car'\n",
    "                \n",
    "                \n",
    "    # Evaluate.\n",
    "    nusc_eval.main(plot_examples=10, render_curves=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d997ba6-76e8-48f6-adad-e8cce64c4000",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "\n",
    "for exp in experiment_names_ca:\n",
    "    # Load results.\n",
    "    with open(os.path.join(evaluation_dir, exp, 'metrics_summary.json'), 'r') as file:\n",
    "        data = json.load(file)\n",
    "        \n",
    "        \n",
    "    # Get results.\n",
    "    ap  = data['mean_dist_aps']['car']\n",
    "    ate = data['label_tp_errors']['car']['trans_err']\n",
    "    ase = data['label_tp_errors']['car']['scale_err']\n",
    "    aoe = data['label_tp_errors']['car']['orient_err']\n",
    "    ave = data['label_tp_errors']['car']['vel_err']\n",
    "    aae = 1.0\n",
    "    nds = (5*ap+max(1-ate,0)+max(1-ase,0)+max(1-aoe,0)+max(1-ave,0)+max(1-aae,0))/10\n",
    "    \n",
    "    \n",
    "    # Print results.\n",
    "    print(exp)\n",
    "    print(f'AP  = {np.round(100*ap,1)}')\n",
    "    print(f'NDS = {np.round(100*nds,1)}')\n",
    "    print(f'ATE = {np.round(ate,3)}')\n",
    "    print(f'ASE = {np.round(ase,3)}')\n",
    "    print(f'AOE = {np.round(aoe,3)}')\n",
    "    print(f'AVE = {np.round(ave,3)}')\n",
    "    print(f'AAE = {np.round(aae,3)}')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b81e773-2e39-4ece-9d8c-73d396919f76",
   "metadata": {},
   "source": [
    "## Evaluate multi-class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87a73758-e150-46d8-9d3d-a1b463894388",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "experiment_names_sp = ['CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Class-Agnostic-Training__Labels-HDBSCAN__UNION-file',   # Class-agnostic train - HDBSCAN.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Class-Agnostic-Training__Labels-UNION__UNION-file',   # Class-agnostic train - UNION.\n",
    "                      ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38925442-7fb8-4bca-a1d8-c54a9b74d9d9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "import pickle\n",
    "\n",
    "\n",
    "\n",
    "# Get standard train.\n",
    "train_file_dir = os.path.join(data_root, 'nuscenes_infos_train.pkl')\n",
    "with open(train_file_dir, 'rb') as f:\n",
    "    data_train = pickle.load(f)    \n",
    "data_array__train = np.array(data_train['data_list']).copy()\n",
    "\n",
    "\n",
    "# Fill shape dict (all shapes in train).\n",
    "shape_dict = {idx:[] for idx in range(8)}   # 8 detection classes.\n",
    "for sample_idx in range(len(data_array__train)):\n",
    "    instances = data_array__train[sample_idx]['instances']\n",
    "    \n",
    "    labels = [instance['bbox_label'] for instance in instances]\n",
    "    dists  = [np.linalg.norm(instance['bbox_3d'][:2], ord=2) for instance in instances]\n",
    "    bboxes = [instance['bbox_3d'] for instance in instances]\n",
    "    \n",
    "    for label, dist, bbox in zip(labels, dists, bboxes):\n",
    "        if (label>=0 and label<=4 and dist<=50) or (label>=5 and label<=7 and dist<=40):\n",
    "            l, w = max(bbox[3], bbox[4]), min(bbox[3], bbox[4])\n",
    "            shape_dict[label].append((l, w, l*w))\n",
    "            \n",
    "            \n",
    "# Get median values.\n",
    "median_shape_dict = {}\n",
    "class_names = list(data_train['metainfo']['categories'].keys())\n",
    "for label, name in zip(shape_dict.keys(), class_names):\n",
    "    dims        = np.array(shape_dict[label])\n",
    "    dims_sorted = dims[np.argsort(dims[:,2]),:]\n",
    "    median_dims = dims_sorted[len(dims)//2]\n",
    "    median_shape_dict[name] = median_dims.tolist()\n",
    "print(median_shape_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67c2f077-cb4a-481f-8b94-048d10049118",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from nuscenes.eval.detection.evaluate import DetectionEval\n",
    "from nuscenes.eval.common.config import config_factory\n",
    "\n",
    "\n",
    "\n",
    "for exp in experiment_names_sp:\n",
    "    # Create evaluate instance.\n",
    "    results_dir = os.path.join('mmdetection3d', 'work_dirs', exp, 'pred_instances_3d', 'results_nusc.json')\n",
    "    new_exp     = exp.split('Class-Agnostic')[0] + 'Multi-Class-003' + exp.split('Class-Agnostic')[1].split('__UNION-file')[0] + '-plus-SP' + '__UNION-file'\n",
    "    nusc_eval   = DetectionEval(nusc, config=config_factory('detection_cvpr_2019'), result_path=results_dir, eval_set='val', output_dir=os.path.join(evaluation_dir, new_exp), verbose=1)\n",
    "    \n",
    "    \n",
    "    # Make it multi-class with 3 classes (car, bicycle, pedestrian).\n",
    "    for sample_token in nusc_eval.gt_boxes.boxes:\n",
    "        for box in nusc_eval.gt_boxes.boxes[sample_token]:\n",
    "            if box.detection_name in ['car', 'truck', 'trailer', 'bus', 'construction_vehicle']:\n",
    "                box.detection_name = 'car'\n",
    "            elif box.detection_name in ['bicycle', 'motorcycle']:\n",
    "                box.detection_name = 'bicycle'\n",
    "            elif box.detection_name in ['pedestrian']:\n",
    "                box.detection_name = 'pedestrian'\n",
    "                \n",
    "                \n",
    "    # Assign each class-agnostic box a real class based on bounding box size.\n",
    "    prototype_dims = np.array(list(median_shape_dict.values()))[:,0:2]\n",
    "    for sample_token in nusc_eval.pred_boxes.boxes:\n",
    "        for box in nusc_eval.pred_boxes.boxes[sample_token]:\n",
    "            dims = box.size\n",
    "            l, w = max(dims[0], dims[1]), min(dims[0], dims[1])\n",
    "            \n",
    "            intersects = (l*(prototype_dims[:,0]>=l)+prototype_dims[:,0]*(prototype_dims[:,0]<l)) * (w*(prototype_dims[:,1]>=w)+prototype_dims[:,1]*(prototype_dims[:,1]<w))\n",
    "            unions     = (prototype_dims[:,0]*(prototype_dims[:,0]>=l)+l*(prototype_dims[:,0]<l)) * (prototype_dims[:,1]*(prototype_dims[:,1]>=w)+w*(prototype_dims[:,1]<w))\n",
    "            ious       = intersects/unions\n",
    "            \n",
    "            idx            = np.argmax(ious)\n",
    "            assigned_class = list(median_shape_dict.keys())[idx]\n",
    "            \n",
    "            if assigned_class in ['car', 'truck', 'trailer', 'bus', 'construction_vehicle']:\n",
    "                box.detection_name = 'car'\n",
    "            elif assigned_class in ['bicycle', 'motorcycle']:\n",
    "                box.detection_name = 'bicycle'\n",
    "            elif assigned_class in ['pedestrian']:\n",
    "                box.detection_name = 'pedestrian'\n",
    "                \n",
    "                \n",
    "    # Evaluate.\n",
    "    nusc_eval.main(plot_examples=10, render_curves=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c1ade47-91c3-49dc-8aa8-4aa53d4630d5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "\n",
    "for exp in experiment_names_sp:\n",
    "    # Load results.\n",
    "    new_exp = exp.split('Class-Agnostic')[0] + 'Multi-Class-003' + exp.split('Class-Agnostic')[1].split('__UNION-file')[0] + '-plus-SP' + '__UNION-file'\n",
    "    with open(os.path.join(evaluation_dir, new_exp, 'metrics_summary.json'), 'r') as file:\n",
    "        data = json.load(file)\n",
    "        \n",
    "        \n",
    "    # Get results.\n",
    "    ap_car  = data['mean_dist_aps']['car']\n",
    "    ate_car = data['label_tp_errors']['car']['trans_err']\n",
    "    ase_car = data['label_tp_errors']['car']['scale_err']\n",
    "    aoe_car = data['label_tp_errors']['car']['orient_err']\n",
    "    ave_car = data['label_tp_errors']['car']['vel_err']\n",
    "    aae_car = 1.0\n",
    "    \n",
    "    ap_pedestrian  = data['mean_dist_aps']['pedestrian']\n",
    "    ate_pedestrian = data['label_tp_errors']['pedestrian']['trans_err']\n",
    "    ase_pedestrian = data['label_tp_errors']['pedestrian']['scale_err']\n",
    "    aoe_pedestrian = data['label_tp_errors']['pedestrian']['orient_err']\n",
    "    ave_pedestrian = data['label_tp_errors']['pedestrian']['vel_err']\n",
    "    aae_pedestrian = 1.0\n",
    "    \n",
    "    ap_bicycle  = data['mean_dist_aps']['bicycle']\n",
    "    ate_bicycle = data['label_tp_errors']['bicycle']['trans_err']\n",
    "    ase_bicycle = data['label_tp_errors']['bicycle']['scale_err']\n",
    "    aoe_bicycle = data['label_tp_errors']['bicycle']['orient_err']\n",
    "    ave_bicycle = data['label_tp_errors']['bicycle']['vel_err']\n",
    "    aae_bicycle = 1.0\n",
    "    \n",
    "    ap  = (ap_car+ap_pedestrian+ap_bicycle)/3\n",
    "    ate = (ate_car+ate_pedestrian+ate_bicycle)/3\n",
    "    ase = (ase_car+ase_pedestrian+ase_bicycle)/3\n",
    "    aoe = (aoe_car+aoe_pedestrian+aoe_bicycle)/3\n",
    "    ave = (ave_car+ave_pedestrian+ave_bicycle)/3\n",
    "    aae = 1.0\n",
    "    \n",
    "    nds = (5*ap+max(1-ate,0)+max(1-ase,0)+max(1-aoe,0)+max(1-ave,0)+max(1-aae,0))/10\n",
    "    \n",
    "    \n",
    "    # Print results.\n",
    "    print(exp)\n",
    "    print(f'mAP    = {np.round(100*ap,1)}')\n",
    "    print(f'NDS    = {np.round(100*nds,1)}')\n",
    "    print(f'AP-car = {np.round(100*ap_car,1)}')\n",
    "    print(f'AP-ped = {np.round(100*ap_pedestrian,1)}')\n",
    "    print(f'AP-cyc = {np.round(100*ap_bicycle,1)}')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec259711-d62a-4393-b26d-4d9f8bcd6b09",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "experiment_names_mc = ['CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Multi-Class-003-Training__Labels-GT__UNION-file',   # Multi-class-003 train - Ground truth.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Multi-Class-003-Training__Labels-UNION-005pc__UNION-file',   # Multi-class-003 train - UNION-Xpc.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Multi-Class-003-Training__Labels-UNION-010pc__UNION-file',   # Multi-class-003 train - UNION-Xpc.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Multi-Class-003-Training__Labels-UNION-015pc__UNION-file',   # Multi-class-003 train - UNION-Xpc.\n",
    "                       'CenterPoint-Pillar0200__second-secfpn-8xb4-cyclic-20e-nus-3d__Multi-Class-003-Training__Labels-UNION-020pc__UNION-file',   # Multi-class-003 train - UNION-Xpc.\n",
    "                      ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01c844f9-1d08-4f8f-b678-22d784e6bcd1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from components.component_appearanceembedding import get_transform\n",
    "\n",
    "\n",
    "\n",
    "appearanceembedding_hyperparameters = {'Step0__stride': 14,}   # Unit:1.\n",
    "\n",
    "\n",
    "appearanceclustering_hyperparameters = {'Step0__feature_dim': 1024,   # Unit: 1.\n",
    "                                        'Step1__K__class_agnostic': 20,   # Unit: 1.\n",
    "                                        'Step1__max_iterations': 5000,   # Unit: 1.\n",
    "                                        'Step1__num_init': 10,   # Unit: 1.\n",
    "                                        'Step2__velocity_thres': 0.50,   # Unit: m/s.\n",
    "                                        'Step2__moving_fraction_thres': 0.050,   # Unit: 1.\n",
    "                                        'Step4__K__multi_class_list': [5, 10, 15, 20],   # Unit: 1.\n",
    "                                        'Step4__max_iterations': 5000,   # Unit: 1.\n",
    "                                        'Step4__num_init': 10,}   # Unit: 1.\n",
    "\n",
    "\n",
    "example_img_dict = {'vehicle':    {'scene_idx': 509, 'sample_idx': 10, 'sensor': 'CAM_FRONT_LEFT', 'scene_name': 'scene-0655', 'h1': 31, 'h2': 51, 'w1': 28, 'w2':  74, 'embedding': None},\n",
    "                    'pedestrian': {'scene_idx':  58, 'sample_idx': 17, 'sensor':  'CAM_BACK_LEFT', 'scene_name': 'scene-0061', 'h1': 25, 'h2': 47, 'w1': 98, 'w2': 107, 'embedding': None},\n",
    "                    'cyclist':    {'scene_idx': 437, 'sample_idx':  0, 'sensor':      'CAM_FRONT', 'scene_name': 'scene-0553', 'h1': 30, 'h2': 42, 'w1': 92, 'w2': 102, 'embedding': None},}\n",
    "\n",
    "\n",
    "for class_name in list(example_img_dict.keys()):\n",
    "    print(f'Example for {class_name}:')\n",
    "    \n",
    "    sample_token  = scenes[example_img_dict[class_name]['scene_idx']]['sample_tokens'][example_img_dict[class_name]['sample_idx']]\n",
    "    sample_record = nusc.get('sample', sample_token)\n",
    "\n",
    "    cam_record = nusc.get('sample_data', sample_record['data'][example_img_dict[class_name]['sensor']])\n",
    "    img_cam    = cv2.imread(os.path.join(nusc.dataroot, cam_record['filename']))[...,::-1]\n",
    "    \n",
    "    stride = appearanceembedding_hyperparameters['Step0__stride']\n",
    "    plt.imshow(img_cam[stride*example_img_dict[class_name]['h1']:stride*example_img_dict[class_name]['h2'],stride*example_img_dict[class_name]['w1']:stride*example_img_dict[class_name]['w2'],:])\n",
    "    plt.axis('off')\n",
    "    plt.show()\n",
    "    \n",
    "    \n",
    "# Define names, get model, and get transform.\n",
    "sensor_names = ['CAM_FRONT','CAM_FRONT_RIGHT','CAM_BACK_RIGHT','CAM_BACK','CAM_BACK_LEFT','CAM_FRONT_LEFT']\n",
    "stride       = appearanceembedding_hyperparameters['Step0__stride']\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model  = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitl14_reg').to(device)\n",
    "model.eval()\n",
    "\n",
    "transform = get_transform()\n",
    "\n",
    "\n",
    "# Compute embeddings.\n",
    "for class_name in list(example_img_dict.keys()):    \n",
    "    sample_token  = scenes[example_img_dict[class_name]['scene_idx']]['sample_tokens'][example_img_dict[class_name]['sample_idx']]\n",
    "    sample_record = nusc.get('sample', sample_token)\n",
    "\n",
    "    cam_record = nusc.get('sample_data', sample_record['data'][example_img_dict[class_name]['sensor']])\n",
    "    img_cam    = cv2.imread(os.path.join(nusc.dataroot, cam_record['filename']))[...,::-1]\n",
    "    \n",
    "    imgs_cams_tensor = transform(img_cam).unsqueeze(dim=0).to(device)\n",
    "    \n",
    "    h, w  = stride*(imgs_cams_tensor.shape[2]//stride), stride*(imgs_cams_tensor.shape[3]//stride)\n",
    "    with torch.no_grad():\n",
    "        features_cams = model.forward_features(imgs_cams_tensor[...,:h,:w])['x_norm_patchtokens']\n",
    "        features_cams = features_cams.reshape(1,h//stride,w//stride,-1).cpu()\n",
    "        \n",
    "    example_img_dict[class_name]['embedding'] = features_cams[0,example_img_dict[class_name]['h1']:example_img_dict[class_name]['h2'],example_img_dict[class_name]['w1']:example_img_dict[class_name]['w2'],:].reshape(-1,1024).mean(dim=0)\n",
    "    \n",
    "    \n",
    "# Assign pseudo-classes to real classes.\n",
    "intermediate_results_appearanceclustering_dir = os.path.join(intermediate_results_root, 'component_appearanceclustering_dinov2-vitl14-reg')\n",
    "\n",
    "moving_fraction_thres = appearanceclustering_hyperparameters['Step2__moving_fraction_thres']\n",
    "\n",
    "vehicle_prototype    = example_img_dict['vehicle']['embedding']\n",
    "pedestrian_prototype = example_img_dict['pedestrian']['embedding']\n",
    "cyclist_prototype    = example_img_dict['cyclist']['embedding']\n",
    "prototypes           = torch.stack((vehicle_prototype, pedestrian_prototype, cyclist_prototype))\n",
    "        \n",
    "assign_dict = {}\n",
    "for K__multi_class_Xpc in appearanceclustering_hyperparameters['Step4__K__multi_class_list']:\n",
    "    filename = f'K-means-centers__K-multi-class{str(K__multi_class_Xpc).zfill(3)}_moving-fraction-thres0{str(int(10000*moving_fraction_thres)).zfill(4)}__multi-class.npy'\n",
    "    centers  = torch.from_numpy(np.load(os.path.join(intermediate_results_appearanceclustering_dir, filename)))\n",
    "    \n",
    "    cos_sim     = torch.nn.functional.cosine_similarity(centers.unsqueeze(1), prototypes.unsqueeze(0), dim=2)\n",
    "    max_indices = cos_sim.argmax(dim=1)\n",
    "    \n",
    "    class_mapping  = {0: 'car', 1: 'pedestrian', 2: 'bicycle'}\n",
    "    assign_dict[K__multi_class_Xpc] = {f'pseudoclass{i:03d}': class_mapping[max_indices[i].item()] for i in range(K__multi_class_Xpc)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51d41801-73a0-4e1f-a54e-ecfb2a913dd5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from nuscenes.eval.detection.evaluate import DetectionEval\n",
    "from nuscenes.eval.common.config import config_factory\n",
    "\n",
    "\n",
    "\n",
    "for exp in experiment_names_mc:\n",
    "    # Map pseudo-classes to real classes.\n",
    "    if 'Labels-GT' not in exp:\n",
    "        pseudo_results_dir = os.path.join('mmdetection3d', 'work_dirs', exp, 'pred_instances_3d', 'results_nusc.json')\n",
    "        with open(pseudo_results_dir) as json_file:\n",
    "            json_data = json.load(json_file)\n",
    "            \n",
    "        K2 = int(exp.split('Labels-UNION-')[1].split('pc')[0])\n",
    "        for key in list(json_data['results'].keys()):\n",
    "            sample_dict = json_data['results'][key]\n",
    "            for sample in sample_dict:\n",
    "                real_class = assign_dict[K2][sample['detection_name']]\n",
    "                if real_class=='car':\n",
    "                    sample['detection_name'] = 'car'\n",
    "                    sample['attribute_name'] = 'vehicle.moving'\n",
    "                elif real_class=='bicycle':\n",
    "                    sample['detection_name'] = 'bicycle'\n",
    "                    sample['attribute_name'] = 'cycle.with_rider'\n",
    "                elif real_class=='pedestrian':\n",
    "                    sample['detection_name'] = 'pedestrian'\n",
    "                    sample['attribute_name'] = 'pedestrian.moving'\n",
    "                    \n",
    "        results_dir = os.path.join('mmdetection3d', 'work_dirs', exp, 'pred_instances_3d', 'results_nusc__mapped.json')\n",
    "        with open(results_dir, 'w') as f:\n",
    "            json.dump(json_data, f)\n",
    "    else:\n",
    "        results_dir = os.path.join('mmdetection3d', 'work_dirs', exp, 'pred_instances_3d', 'results_nusc.json')\n",
    "        \n",
    "        \n",
    "    # Create evaluate instance.\n",
    "    nusc_eval = DetectionEval(nusc, config=config_factory('detection_cvpr_2019'), result_path=results_dir, eval_set='val', output_dir=os.path.join(evaluation_dir, exp), verbose=1)\n",
    "    \n",
    "    \n",
    "    # Make it multi-class with 3 classes (car, bicycle, pedestrian).\n",
    "    for sample_token in nusc_eval.gt_boxes.boxes:\n",
    "        for box in nusc_eval.gt_boxes.boxes[sample_token]:\n",
    "            if box.detection_name in ['car', 'truck', 'trailer', 'bus', 'construction_vehicle']:\n",
    "                box.detection_name = 'car'\n",
    "            elif box.detection_name in ['bicycle', 'motorcycle']:\n",
    "                box.detection_name = 'bicycle'\n",
    "            elif box.detection_name in ['pedestrian']:\n",
    "                box.detection_name = 'pedestrian'\n",
    "                \n",
    "                \n",
    "    # Evaluate.\n",
    "    nusc_eval.main(plot_examples=10, render_curves=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad6ae4a6-0f4e-419a-acd0-8ec23a7e917e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "\n",
    "for exp in experiment_names_mc:\n",
    "    # Load results.\n",
    "    with open(os.path.join(evaluation_dir, exp, 'metrics_summary.json'), 'r') as file:\n",
    "        data = json.load(file)\n",
    "        \n",
    "        \n",
    "    # Get results.\n",
    "    ap_car  = data['mean_dist_aps']['car']\n",
    "    ate_car = data['label_tp_errors']['car']['trans_err']\n",
    "    ase_car = data['label_tp_errors']['car']['scale_err']\n",
    "    aoe_car = data['label_tp_errors']['car']['orient_err']\n",
    "    ave_car = data['label_tp_errors']['car']['vel_err']\n",
    "    aae_car = 1.0\n",
    "    \n",
    "    ap_pedestrian  = data['mean_dist_aps']['pedestrian']\n",
    "    ate_pedestrian = data['label_tp_errors']['pedestrian']['trans_err']\n",
    "    ase_pedestrian = data['label_tp_errors']['pedestrian']['scale_err']\n",
    "    aoe_pedestrian = data['label_tp_errors']['pedestrian']['orient_err']\n",
    "    ave_pedestrian = data['label_tp_errors']['pedestrian']['vel_err']\n",
    "    aae_pedestrian = 1.0\n",
    "    \n",
    "    ap_bicycle  = data['mean_dist_aps']['bicycle']\n",
    "    ate_bicycle = data['label_tp_errors']['bicycle']['trans_err']\n",
    "    ase_bicycle = data['label_tp_errors']['bicycle']['scale_err']\n",
    "    aoe_bicycle = data['label_tp_errors']['bicycle']['orient_err']\n",
    "    ave_bicycle = data['label_tp_errors']['bicycle']['vel_err']\n",
    "    aae_bicycle = 1.0\n",
    "    \n",
    "    ap  = (ap_car+ap_pedestrian+ap_bicycle)/3\n",
    "    ate = (ate_car+ate_pedestrian+ate_bicycle)/3\n",
    "    ase = (ase_car+ase_pedestrian+ase_bicycle)/3\n",
    "    aoe = (aoe_car+aoe_pedestrian+aoe_bicycle)/3\n",
    "    ave = (ave_car+ave_pedestrian+ave_bicycle)/3\n",
    "    aae = 1.0\n",
    "    \n",
    "    nds = (5*ap+max(1-ate,0)+max(1-ase,0)+max(1-aoe,0)+max(1-ave,0)+max(1-aae,0))/10\n",
    "    \n",
    "    \n",
    "    # Print results.\n",
    "    print(exp)\n",
    "    print(f'mAP    = {np.round(100*ap,1)}')\n",
    "    print(f'NDS    = {np.round(100*nds,1)}')\n",
    "    print(f'AP-car = {np.round(100*ap_car,1)}')\n",
    "    print(f'AP-ped = {np.round(100*ap_pedestrian,1)}')\n",
    "    print(f'AP-cyc = {np.round(100*ap_bicycle,1)}')\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}