diff --git a/openunreid/apis/runner.py b/openunreid/apis/runner.py index 1be08eb..3c69db6 100644 --- a/openunreid/apis/runner.py +++ b/openunreid/apis/runner.py @@ -301,7 +301,10 @@ def val(self): self._rank, print_freq=self.print_freq, ) - better_mAP = max(better_mAP, mAP) + if self.cfg.TRAIN.num_repeat != 1: + better_mAP = max(better_mAP, cmc[0]) + else: + better_mAP = max(better_mAP, mAP) return better_mAP @@ -311,7 +314,7 @@ def save(self, mAP=None): self._best_mAP = max(self._best_mAP, mAP) print( bcolors.OKGREEN - + "\n * Finished epoch {:3d} mAP: {:5.1%} best: {:5.1%}{}\n".format( + + "\n * Finished epoch {:3d} current: {:5.1%} best: {:5.1%}{}\n".format( self._epoch, mAP, self._best_mAP, " *" if is_best else "" ) + bcolors.ENDC diff --git a/openunreid/apis/test.py b/openunreid/apis/test.py index 4a139cb..80fd1de 100644 --- a/openunreid/apis/test.py +++ b/openunreid/apis/test.py @@ -8,8 +8,9 @@ import numpy as np import torch -import torchvision +from .train import set_random_seed +from openunreid.data import build_test_dataloader from ..core.metrics.rank import evaluate_rank from ..core.utils.compute_dist import build_dist from ..models.utils.dsbn_utils import switch_target_bn @@ -26,11 +27,9 @@ @torch.no_grad() def test_reid( - cfg, model, data_loader, query, gallery, dataset_name=None, rank=None, **kwargs + cfg, model, data_loader, query, gallery, dataset_name=None, num=1, rank=None, **kwargs ): - start_time = time.monotonic() - if cfg.MODEL.dsbn: assert ( dataset_name is not None @@ -47,7 +46,7 @@ def test_reid( sep = "*******************************" if dataset_name is not None: - print(f"\n{sep} Start testing {dataset_name} {sep}\n") + print(f"\n{sep} Start testing {dataset_name} {-num} {sep}\n") if rank is None: rank, _, _ = get_dist_info() @@ -78,7 +77,7 @@ def test_reid( # evaluate with original distance dist = build_dist(cfg.TEST, query_features, gallery_features) - cmc, map = evaluate_rank(dist, q_pids, g_pids, q_cids, g_cids) + cmc, map = evaluate_rank(cfg, dist, q_pids, g_pids, q_cids, g_cids) else: cmc, map = np.empty(50), 0.0 @@ -98,14 +97,10 @@ def test_reid( # dist_gg = build_dist(cfg, gallery_features, gallery_features) # final_dist = re_ranking_cpu(dist, dist_qq, dist_gg) - cmc, map = evaluate_rank(final_dist, q_pids, g_pids, q_cids, g_cids) + cmc, map = evaluate_rank(cfg, final_dist, q_pids, g_pids, q_cids, g_cids) else: cmc, map = np.empty(50), 0.0 - end_time = time.monotonic() - print("Testing time: ", timedelta(seconds=end_time - start_time)) - print(f"\n{sep} Finished testing {sep}\n") - return cmc, map @@ -142,7 +137,7 @@ def val_reid( # evaluate with original distance if rank == 0: dist = build_dist(cfg.TEST, features) - cmc, map = evaluate_rank(dist, pids, pids, cids, cids) + cmc, map = evaluate_rank(cfg, dist, pids, pids, cids, cids) else: cmc, map = np.empty(50), 0.0 @@ -207,3 +202,31 @@ def infer_gan( print(f"\n{sep} Finished translating {sep}\n") return + + +@torch.no_grad() +def final_test(cfg, model, cmc_topk=(1, 5, 10)): + sep = "*******************************" + start_time = time.monotonic() + + all_cmc = [] + all_mAP = [] + for num in range(cfg.TRAIN.num_repeat): + set_random_seed(num + 1, cfg.TRAIN.deterministic) + test_loaders, queries, galleries = build_test_dataloader(cfg) + for i, (loader, query, gallery) in enumerate(zip(test_loaders, queries, galleries)): + cmc, mAP = test_reid( + cfg, model, loader, query, gallery, dataset_name=cfg.TEST.datasets[i], num=num+1 + ) + all_cmc.append(cmc) + all_mAP.append(mAP) + + if cfg.TRAIN.num_repeat != 1: + print("\n ") + print("Average CMC Scores:") + for k in cmc_topk: + print(" top-{:<4}{:12.1%}".format(k, np.mean(all_cmc, axis=0)[k - 1])) + + end_time = time.monotonic() + print("Testing time: ", timedelta(seconds=end_time - start_time)) + print(f"\n{sep} Finished testing {sep}\n") \ No newline at end of file diff --git a/openunreid/core/metrics/rank.py b/openunreid/core/metrics/rank.py index db950dc..432259f 100644 --- a/openunreid/core/metrics/rank.py +++ b/openunreid/core/metrics/rank.py @@ -5,6 +5,8 @@ import numpy as np +from openunreid.utils.logger import display + try: from .rank_cylib.rank_cy import evaluate_cy @@ -162,6 +164,7 @@ def evaluate_py( def evaluate_rank( + cfg, distmat, q_pids, g_pids, @@ -202,10 +205,6 @@ def evaluate_rank( ) if verbose: - print("\n") - print("Mean AP: {:4.1%}".format(map)) - print("CMC Scores:") - for k in cmc_topk: - print(" top-{:<4}{:12.1%}".format(k, cmc[k - 1])) + display(cfg, map, cmc, cmc_topk) return cmc, map diff --git a/openunreid/core/metrics/rank_cylib/rank_cy.pyx b/openunreid/core/metrics/rank_cylib/rank_cy.pyx index 48dc051..865df3d 100644 --- a/openunreid/core/metrics/rank_cylib/rank_cy.pyx +++ b/openunreid/core/metrics/rank_cylib/rank_cy.pyx @@ -4,7 +4,6 @@ import cython import numpy as np cimport numpy as np from collections import defaultdict -import random """ diff --git a/openunreid/data/datasets/vehicleid.py b/openunreid/data/datasets/vehicleid.py index d1fdce7..746cc1a 100644 --- a/openunreid/data/datasets/vehicleid.py +++ b/openunreid/data/datasets/vehicleid.py @@ -2,9 +2,7 @@ import os.path as osp import random -import shutil import warnings -from collections import defaultdict from ..utils.base_dataset import ImageDataset @@ -34,6 +32,7 @@ def __init__( ): self.root = osp.abspath(osp.expanduser(root)) self.dataset_dir = osp.join(self.root, self.dataset_dir) + self.mode = mode self.del_labels = del_labels self.download_dataset(self.dataset_dir, self.dataset_url) assert (val_split > 0.0) and ( @@ -108,7 +107,6 @@ def process_split(self, list_path, data_range, relabel=False): list_data = f.readlines() for data in list_data: name, pid = data.strip().split(" ") - # pid = int(pid) if pid == -1: continue # junk images are just ignored pid_container.add(pid) @@ -123,13 +121,14 @@ def process_split(self, list_path, data_range, relabel=False): pid2label = {pid: label for label, pid in enumerate(pid_container)} data = [] + camid = 0 for ld in list_data: name, pid = ld.strip().split(" ") if (pid not in pid_container) or (pid == -1): continue img_path = osp.join(self.img_dir, name + ".jpg") - camid = 0 + camid += 1 if not self.del_labels: if relabel: pid = pid2label[pid] diff --git a/openunreid/utils/logger.py b/openunreid/utils/logger.py index b0563ae..52276bf 100644 --- a/openunreid/utils/logger.py +++ b/openunreid/utils/logger.py @@ -1,6 +1,8 @@ import os import sys +import numpy as np + from .dist_utils import get_dist_info, synchronize from .file_utils import mkdir_if_missing @@ -42,3 +44,17 @@ def close(self): self.console.close() if self.file is not None: self.file.close() + + +def display(cfg, map, cmc, cmc_topk=(1, 5, 10)): + if cfg.TRAIN.num_repeat != 1: + print("\n") + print("CMC Scores:") + for k in cmc_topk: + print(" top-{:<4}{:12.1%}".format(k, cmc[k - 1])) + else: + print("\n") + print("Mean AP: {:4.1%}".format(np.mean(map))) + print("CMC Scores:") + for k in cmc_topk: + print(" top-{:<4}{:12.1%}".format(k, cmc[k - 1])) \ No newline at end of file diff --git a/tools/CycleGAN/config.yaml b/tools/CycleGAN/config.yaml index 202a641..ab0036e 100644 --- a/tools/CycleGAN/config.yaml +++ b/tools/CycleGAN/config.yaml @@ -50,6 +50,9 @@ TRAIN: datasets: {'market1501': 'trainval', 'dukemtmcreid': 'trainval'} unsup_dataset_indexes: [1,] + # repeated number of evaluation + num_repeat: 10 # 10 only for vehicleid dataset, otherwise 1 + epochs: 50 iters: 200 diff --git a/tools/CycleGAN/main.py b/tools/CycleGAN/main.py index aef1166..d31c611 100644 --- a/tools/CycleGAN/main.py +++ b/tools/CycleGAN/main.py @@ -7,18 +7,15 @@ from pathlib import Path import torch -from torch.nn.parallel import DataParallel, DistributedDataParallel from openunreid.apis import GANBaseRunner, set_random_seed, infer_gan from openunreid.core.solvers import build_lr_scheduler, build_optimizer from openunreid.data import ( - build_test_dataloader, build_train_dataloader, build_val_dataloader, ) from openunreid.models import build_gan_model from openunreid.models.losses import build_loss -from openunreid.models.utils.extract import extract_features from openunreid.utils.config import ( cfg, cfg_from_list, diff --git a/tools/MMT/config.yaml b/tools/MMT/config.yaml index cf8c0cf..9f1deb4 100644 --- a/tools/MMT/config.yaml +++ b/tools/MMT/config.yaml @@ -61,6 +61,9 @@ TRAIN: datasets: {'market1501': 'trainval', 'dukemtmcreid': 'trainval'} unsup_dataset_indexes: [0,] + # repeated number of evaluation + num_repeat: 1 # 10 only for vehicleid dataset, otherwise 1 + epochs: 50 iters: 400 diff --git a/tools/MMT/main.py b/tools/MMT/main.py index 8c42aae..7ab93c1 100644 --- a/tools/MMT/main.py +++ b/tools/MMT/main.py @@ -8,6 +8,7 @@ import torch from openunreid.apis import BaseRunner, batch_processor, test_reid, set_random_seed +from openunreid.apis.test import final_test from openunreid.core.metrics.accuracy import accuracy from openunreid.core.solvers import build_lr_scheduler, build_optimizer from openunreid.data import build_test_dataloader, build_train_dataloader @@ -198,21 +199,11 @@ def main(): runner.resume(cfg.work_dir / "model_best.pth") # final testing - test_loaders, queries, galleries = build_test_dataloader(cfg) - for i, (loader, query, gallery) in enumerate(zip(test_loaders, queries, galleries)): - - for idx in range(len(runner.model)): - print("==> Test on the no.{} model".format(idx)) - # test_reid() on self.model[idx] will only evaluate the 'mean_net' - # for testing 'net', use self.model[idx].module.net - cmc, mAP = test_reid( - cfg, - runner.model[idx], - loader, - query, - gallery, - dataset_name=cfg.TEST.datasets[i], - ) + for idx in range(len(runner.model)): + print("==> Test on the no.{} model".format(idx)) + # test_reid() on self.model[idx] will only evaluate the 'mean_net' + # for testing 'net', use self.model[idx].module.net + final_test(cfg, runner.model[idx]) # print time end_time = time.monotonic() diff --git a/tools/SPGAN/config.yaml b/tools/SPGAN/config.yaml index abcc0a6..855712e 100644 --- a/tools/SPGAN/config.yaml +++ b/tools/SPGAN/config.yaml @@ -50,6 +50,9 @@ TRAIN: datasets: {'market1501': 'trainval', 'dukemtmcreid': 'trainval'} unsup_dataset_indexes: [1,] + # repeated number of evaluation + num_repeat: 10 # 10 only for vehicleid dataset, otherwise 1 + epochs: 50 iters: 200 diff --git a/tools/SpCL/config.yaml b/tools/SpCL/config.yaml index 5b7b3e6..7772943 100644 --- a/tools/SpCL/config.yaml +++ b/tools/SpCL/config.yaml @@ -61,6 +61,9 @@ TRAIN: # datasets: {'market1501': 'trainval', 'dukemtmcreid': 'trainval'} unsup_dataset_indexes: [0,] + # repeated number of evaluation + num_repeat: 1 # 10 only for vehicleid dataset, otherwise 1 + epochs: 50 iters: 400 @@ -71,7 +74,7 @@ TRAIN: # validate val_dataset: 'market1501' - val_freq: 5 + val_freq: 1 # sampler SAMPLER: diff --git a/tools/SpCL/main.py b/tools/SpCL/main.py index 918f997..870c14f 100644 --- a/tools/SpCL/main.py +++ b/tools/SpCL/main.py @@ -9,10 +9,10 @@ import torch from torch.nn.parallel import DataParallel, DistributedDataParallel -from openunreid.apis import BaseRunner, batch_processor, test_reid, set_random_seed +from openunreid.apis import BaseRunner, batch_processor, set_random_seed +from openunreid.apis.test import final_test from openunreid.core.solvers import build_lr_scheduler, build_optimizer from openunreid.data import ( - build_test_dataloader, build_train_dataloader, build_val_dataloader, ) @@ -265,11 +265,7 @@ def main(): runner.resume(cfg.work_dir / "model_best.pth") # final testing - test_loaders, queries, galleries = build_test_dataloader(cfg) - for i, (loader, query, gallery) in enumerate(zip(test_loaders, queries, galleries)): - cmc, mAP = test_reid( - cfg, model, loader, query, gallery, dataset_name=cfg.TEST.datasets[i] - ) + final_test(cfg, model) # print time end_time = time.monotonic() diff --git a/tools/UDA_TP/config.yaml b/tools/UDA_TP/config.yaml index 2b3eb39..bdefa4b 100644 --- a/tools/UDA_TP/config.yaml +++ b/tools/UDA_TP/config.yaml @@ -61,6 +61,9 @@ TRAIN: datasets: {'market1501': 'trainval',} unsup_dataset_indexes: [0,] + # repeated number of evaluation + num_repeat: 1 # 10 only for vehicleid dataset, otherwise 1 + epochs: 50 iters: 400 diff --git a/tools/UDA_TP/main.py b/tools/UDA_TP/main.py index e19282d..49b858b 100644 --- a/tools/UDA_TP/main.py +++ b/tools/UDA_TP/main.py @@ -8,6 +8,7 @@ import torch from openunreid.apis import BaseRunner, test_reid, set_random_seed +from openunreid.apis.test import final_test from openunreid.core.solvers import build_lr_scheduler, build_optimizer from openunreid.data import build_test_dataloader, build_train_dataloader from openunreid.models import build_model @@ -130,11 +131,7 @@ def main(): runner.resume(cfg.work_dir / "model_best.pth") # final testing - test_loaders, queries, galleries = build_test_dataloader(cfg) - for i, (loader, query, gallery) in enumerate(zip(test_loaders, queries, galleries)): - cmc, mAP = test_reid( - cfg, model, loader, query, gallery, dataset_name=cfg.TEST.datasets[i] - ) + final_test(cfg, model) # print time end_time = time.monotonic() diff --git a/tools/source_pretrain/config.yaml b/tools/source_pretrain/config.yaml index ee1b9b9..553aa66 100644 --- a/tools/source_pretrain/config.yaml +++ b/tools/source_pretrain/config.yaml @@ -59,6 +59,9 @@ TRAIN: datasets: {'market1501': 'trainval',} unsup_dataset_indexes: null + # repeated number of evaluation + num_repeat: 1 # 10 only for vehicleid dataset, otherwise 1 + epochs: 120 iters: 200 diff --git a/tools/source_pretrain/main.py b/tools/source_pretrain/main.py index c68a3ad..122ede3 100644 --- a/tools/source_pretrain/main.py +++ b/tools/source_pretrain/main.py @@ -9,6 +9,7 @@ import torch from openunreid.apis import BaseRunner, test_reid, set_random_seed +from openunreid.apis.test import final_test from openunreid.core.solvers import build_lr_scheduler, build_optimizer from openunreid.data import build_test_dataloader, build_train_dataloader from openunreid.models import build_model @@ -127,11 +128,7 @@ def main(): runner.resume(cfg.work_dir / "model_best.pth") # final testing - test_loaders, queries, galleries = build_test_dataloader(cfg) - for i, (loader, query, gallery) in enumerate(zip(test_loaders, queries, galleries)): - cmc, mAP = test_reid( - cfg, model, loader, query, gallery, dataset_name=cfg.TEST.datasets[i] - ) + final_test(cfg, model) # print time end_time = time.monotonic() diff --git a/tools/strong_baseline/config.yaml b/tools/strong_baseline/config.yaml index 58fe0b4..7bc060d 100644 --- a/tools/strong_baseline/config.yaml +++ b/tools/strong_baseline/config.yaml @@ -61,8 +61,11 @@ TRAIN: # datasets: {'market1501': 'trainval', 'dukemtmcreid': 'trainval'} unsup_dataset_indexes: [0,] + # repeated number of evaluation + num_repeat: 1 # 10 only for vehicleid dataset, otherwise 1 + epochs: 50 - iters: 400 + iters: 800 LOSS: losses: {'cross_entropy': 1., 'softmax_triplet': 1.} diff --git a/tools/strong_baseline/main.py b/tools/strong_baseline/main.py index 4840066..438e83b 100644 --- a/tools/strong_baseline/main.py +++ b/tools/strong_baseline/main.py @@ -8,6 +8,7 @@ import torch from openunreid.apis import BaseRunner, test_reid, set_random_seed +from openunreid.apis.test import final_test from openunreid.core.solvers import build_lr_scheduler, build_optimizer from openunreid.data import build_test_dataloader, build_train_dataloader from openunreid.models import build_model @@ -142,11 +143,7 @@ def main(): runner.resume(cfg.work_dir / "model_best.pth") # final testing - test_loaders, queries, galleries = build_test_dataloader(cfg) - for i, (loader, query, gallery) in enumerate(zip(test_loaders, queries, galleries)): - cmc, mAP = test_reid( - cfg, model, loader, query, gallery, dataset_name=cfg.TEST.datasets[i] - ) + final_test(cfg, model) # print time end_time = time.monotonic() diff --git a/tools/test_reid.py b/tools/test_reid.py index cf944e5..2a73575 100644 --- a/tools/test_reid.py +++ b/tools/test_reid.py @@ -18,6 +18,7 @@ from openunreid.utils.dist_utils import init_dist, synchronize from openunreid.utils.logger import Logger from openunreid.utils.torch_utils import copy_state_dict, load_checkpoint +from openunreid.apis.test import final_test def parge_config(): @@ -88,9 +89,6 @@ def main(): # load checkpoint state_dict = load_checkpoint(args.resume) - # load test data_loader - test_loaders, queries, galleries = build_test_dataloader(cfg) - for key in state_dict: if not key.startswith("state_dict"): continue @@ -99,12 +97,7 @@ def main(): copy_state_dict(state_dict[key], model) # start testing - for i, (loader, query, gallery) in enumerate( - zip(test_loaders, queries, galleries) - ): - cmc, mAP = test_reid( - cfg, model, loader, query, gallery, dataset_name=cfg.TEST.datasets[i] - ) + final_test(cfg, model) # print time end_time = time.monotonic()