Skip to content

Commit 0fccd75

Browse files
ebennequinebennequin
authored
Add new benchmark (#115)
1 parent 509dd9a commit 0fccd75

File tree

6 files changed

+67
-87
lines changed

6 files changed

+67
-87
lines changed

‎Makefile‎

Lines changed: 2 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,7 @@ extract-all-features-with-resnet12:
6767

6868
benchmark-mini-imagenet:
6969
for n_shot in 1 5; do \
70-
for method in bd_cspn prototypical_networks simple_shot ; do \
71-
python -m scripts.benchmark_methods \
72-
$${method} \
73-
data/features/mini_imagenet/test/feat_resnet12_mini_imagenet.parquet.gzip \
74-
--n-shot=$${n_shot} \
75-
--device=${DEVICE} \
76-
--num-workers=${NUM_WORKERS}; \
77-
done; \
78-
for method in tim; do \
70+
for method in bd_cspn prototypical_networks simple_shot tim finetune laplacian_shot pt_map transductive_finetuning; do \
7971
python -m scripts.benchmark_methods \
8072
$${method} \
8173
data/features/mini_imagenet/test/feat_resnet12_mini_imagenet.parquet.gzip \
@@ -84,15 +76,6 @@ benchmark-mini-imagenet:
8476
--device=${DEVICE} \
8577
--num-workers=${NUM_WORKERS}; \
8678
done; \
87-
for method in finetune laplacian_shot; do \
88-
python -m scripts.benchmark_methods \
89-
$${method} \
90-
data/features/mini_imagenet/test/feat_resnet12_mini_imagenet.parquet.gzip \
91-
--config=$${n_shot}_shot \
92-
--n-shot=$${n_shot} \
93-
--device=${DEVICE} \
94-
--num-workers=${NUM_WORKERS}; \
95-
done; \
9679
python -m scripts.benchmark_methods \
9780
feat \
9881
data/features/mini_imagenet/test/feat_resnet12_mini_imagenet.parquet.gzip \
@@ -104,15 +87,7 @@ benchmark-mini-imagenet:
10487

10588
benchmark-tiered-imagenet:
10689
for n_shot in 1 5; do \
107-
for method in bd_cspn prototypical_networks simple_shot ; do \
108-
python -m scripts.benchmark_methods \
109-
$${method} \
110-
data/features/tiered_imagenet/test/feat_resnet12_tiered_imagenet.parquet.gzip \
111-
--n-shot=$${n_shot} \
112-
--device=${DEVICE} \
113-
--num-workers=${NUM_WORKERS}; \
114-
done; \
115-
for method in tim; do \
90+
for method in bd_cspn prototypical_networks simple_shot tim finetune laplacian_shot pt_map transductive_finetuning; do \
11691
python -m scripts.benchmark_methods \
11792
$${method} \
11893
data/features/tiered_imagenet/test/feat_resnet12_tiered_imagenet.parquet.gzip \
@@ -121,15 +96,6 @@ benchmark-tiered-imagenet:
12196
--device=${DEVICE} \
12297
--num-workers=${NUM_WORKERS}; \
12398
done; \
124-
for method in finetune laplacian_shot; do \
125-
python -m scripts.benchmark_methods \
126-
$${method} \
127-
data/features/tiered_imagenet/test/feat_resnet12_tiered_imagenet.parquet.gzip \
128-
--config=$${n_shot}_shot \
129-
--n-shot=$${n_shot} \
130-
--device=${DEVICE} \
131-
--num-workers=${NUM_WORKERS}; \
132-
done; \
13399
python -m scripts.benchmark_methods \
134100
feat \
135101
data/features/tiered_imagenet/test/feat_resnet12_tiered_imagenet.parquet.gzip \

‎README.md‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -167,11 +167,14 @@ We used EasyFSL to benchmark a dozen methods.
167167
Inference times are computed over 1000 tasks using pre-extracted features. They are only indicative.
168168
Note that the inference time for fine-tuning methods highly depends on the number of fine-tuning steps.
169169

170-
All methods hyper-parameters are defined in [this JSON file](scripts/backbones_configs.json).
170+
All methods hyperparameters are defined in [this JSON file](scripts/backbones_configs.json).
171171
They were selected on miniImageNet validation set.
172-
The procedure can be reproduced from [this other project](https://github.com/ebennequin/few-shot-open-set).
172+
The procedure can be reproduced with `make hyperparameter-search`.
173173
We decided to use miniImageNet's hyperparameters for all benchmarks in order to highlight the adaptability of
174174
the different methods.
175+
Note that all methods use L2 normalization of features, except for FEAT as it harms its performance.
176+
177+
There are no results for Mathing and Relation Networks as the trained weights for their additional modules are unavailable.
175178

176179
### miniImageNet & tieredImageNet
177180

@@ -180,36 +183,26 @@ provided by the authors from [FEAT](https://github.com/Sha-Lab/FEAT)
180183
(download: [miniImageNet](https://drive.google.com/file/d/1ixqw1l9XVxl3lh1m5VXkctw6JssahGbQ/view),
181184
[tieredImageNet](https://drive.google.com/file/d/1M93jdOjAn8IihICPKJg8Mb4B-eYDSZfE/view)).
182185

186+
Best inductive and best transductive results for each column are shown in bold.
187+
183188
| Method | Ind / Trans | *mini*Imagenet<br/>1-shot | *mini*Imagenet<br/>5-shot | *tiered*Imagenet<br/>1-shot | *tiered*Imagenet<br/>5-shot | Time |
184189
|---------------------------------------------------------------------------|--------------|---------------------------|---------------------------|-----------------------------|-----------------------------|---------|
185-
| **[ProtoNet](easyfsl/methods/prototypical_networks.py)** | Inductive | 61.5 | 79.3 | 56.4 | 76.5 | 10s |
186-
| **[SimpleShot](easyfsl/methods/simple_shot.py)** | Inductive | 65.5 | 80.3 | 60.2 | 77.3 | 9s |
190+
| **[ProtoNet](easyfsl/methods/prototypical_networks.py)** | Inductive | 63.6 | 80.4 | 60.2 | 77.4 | 6s |
191+
| **[SimpleShot](easyfsl/methods/simple_shot.py)** | Inductive | 63.6 | **80.5** | 60.2 | 77.4 | 6s |
187192
| **[MatchingNet](easyfsl/methods/matching_networks.py)** | Inductive | - | - | - | - | - |
188193
| **[RelationNet](easyfsl/methods/relation_networks.py)** | Inductive | - | - | - | - | - |
189-
| **[Finetune](easyfsl/methods/finetune.py)** | Inductive | 63.4 | 80.38 | 60.1 | 77.4 | 3mn03s |
190-
| **[FEAT](easyfsl/methods/feat.py)** | Inductive | 64.7 | 80.13 | 61.3 | 76.2 | 3s |
191-
| **[BD-CSPN](easyfsl/methods/bd_cspn.py)** | Transductive | 71.7 | 82.17 | 66.1 | 79.1 | 10s |
192-
| **[LaplacianShot](easyfsl/methods/laplacian_shot.py)** | Transductive | 69.6 | 81.9 | 66.0 | 78.9 | 12s |
193-
| **[PT-MAP](easyfsl/methods/pt_map.py)** | Transductive | - | - | - | - | 22mn50s |
194-
| **[TIM](easyfsl/methods/tim.py)** | Transductive | 64.0 | 80.4 | 60.4 | 77.4 | 2mn48s |
195-
| **[Transductive Finetuning](easyfsl/methods/transductive_finetuning.py)** | Transductive | - | - | - | - | - |
194+
| **[Finetune](easyfsl/methods/finetune.py)** | Inductive | 63.3 | **80.5** | 59.8 | **77.5** | 1mn33s |
195+
| **[FEAT](easyfsl/methods/feat.py)** | Inductive | **64.7** | 80.1 | **61.3** | 76.2 | 3s |
196+
| **[BD-CSPN](easyfsl/methods/bd_cspn.py)** | Transductive | 69.8 | 82.2 | 66.3 | 79.1 | 7s |
197+
| **[LaplacianShot](easyfsl/methods/laplacian_shot.py)** | Transductive | 69.8 | 82.3 | 66.2 | 79.2 | 9s |
198+
| **[PT-MAP](easyfsl/methods/pt_map.py)** | Transductive | **76.1** | **84.2** | **71.7** | **80.7** | 39mn40s |
199+
| **[TIM](easyfsl/methods/tim.py)** | Transductive | 74.3 | **84.2** | 70.7 | **80.7** | 3mn05s |
200+
| **[Transductive Finetuning](easyfsl/methods/transductive_finetuning.py)** | Transductive | 63.0 | 80.6 | 59.1 | 77.5 | 30s |
196201

197202
To reproduce:
198203

199204
1. Download the [*mini*ImageNet](https://drive.google.com/file/d/1ixqw1l9XVxl3lh1m5VXkctw6JssahGbQ/view)
200205
and [tieredImageNet](https://drive.google.com/file/d/1M93jdOjAn8IihICPKJg8Mb4B-eYDSZfE/view) weights for ResNet12
201206
and save them under `data/models/feat_resnet12_mini_imagenet.pth` (resp. `tiered`).
202207
2. Extract all embeddings from the test sets of all datasets with `make extract-all-features-with-resnet12`.
203-
3. Run the evaluation script with `make benchmark-mini-imagenet`.
204-
205-
206-
### Next steps (01/06/2023):
207-
208-
1. Complete benchmark with Matching and Relation Nets, PT-MAP, Transductive Finetuning (est. July 23)
209-
2. Add explicit hyperparameter selection (est. July 23)
210-
3. Add feature normalization as it's been proven to have a huge impact on the results (est. July 23)
211-
4. Add cross-domain benchmarks (CUB, Fungi) and using other backbones (est. September 23)
212-
213-
214-
215-
208+
3. Run the evaluation scripts with `make benchmark-mini-imagenet` (resp. `tiered`).

‎easyfsl/methods/feat.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ def from_resnet12_checkpoint(
6666
device: str = "cpu",
6767
feature_dimension: int = 640,
6868
use_backbone: bool = True,
69+
**kwargs,
6970
):
7071
"""
7172
Load a FEAT model from a checkpoint of a resnet12 model as provided by the authors.
@@ -115,4 +116,4 @@ def from_resnet12_checkpoint(
115116
f"Missing keys for attention module: {attention_missing_keys}"
116117
)
117118

118-
return cls(backbone, attention_module=attention_module).to(device)
119+
return cls(backbone, attention_module=attention_module, **kwargs).to(device)

‎easyfsl/tests/utils_test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,4 +317,4 @@ class TestComputeAverageFeaturesFromImages:
317317
@pytest.mark.parametrize("dataloader, expected_average", cases_grid)
318318
def test_returns_expected_average(dataloader, expected_average):
319319
output_tensor = compute_average_features_from_images(dataloader, nn.Identity())
320-
torch.testing.assert_allclose(output_tensor, expected_average)
320+
torch.testing.assert_close(output_tensor, expected_average)

‎scripts/grid_search.json‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"tim": {
3+
"feature_normalization": [2],
34
"fine_tuning_steps": [50, 100, 200, 300],
45
"fine_tuning_lr": [0.001, 0.0001],
56
"cross_entropy_weight": [1.0],
@@ -8,22 +9,25 @@
89
"temperature": [1, 5, 10]
910
},
1011
"finetune": {
12+
"feature_normalization": [2],
1113
"fine_tuning_steps": [50, 100, 200, 300],
1214
"fine_tuning_lr": [0.001, 0.0001],
1315
"temperature": [1, 5, 10]
1416
},
1517
"pt_map": {
18+
"feature_normalization": [2],
1619
"fine_tuning_steps": [10, 20, 30],
1720
"fine_tuning_lr": [0.2, 0.3, 0.5],
1821
"lambda_regularization": [10, 20, 30]
1922
},
2023
"laplacian_shot": {
21-
"inference_steps": [10, 20, 30],
22-
"knn": [1, 3, 5, 7],
23-
"lambda_regularization": [0.1, 0.3, 0.5, 0.7, 0.8],
24-
"temperature": [1, 5, 10]
24+
"feature_normalization": [2],
25+
"inference_steps": [10, 20, 30],
26+
"knn": [1, 3, 5, 7],
27+
"lambda_regularization": [0.1, 0.3, 0.5, 0.7, 0.8]
2528
},
2629
"transductive_finetuning": {
30+
"feature_normalization": [2],
2731
"fine_tuning_steps": [10, 25, 40],
2832
"fine_tuning_lr": [0.001, 0.0001, 0.00001],
2933
"temperature": [1, 5, 10]

‎scripts/methods_configs.json‎

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@
22
"tim": {
33
"default": {
44
"fine_tuning_steps": 50,
5-
"fine_tuning_lr": 0.0001,
5+
"fine_tuning_lr": 0.001,
66
"cross_entropy_weight": 1.0,
77
"marginal_entropy_weight": 1.0,
8-
"conditional_entropy_weight": 1.0,
9-
"temperature": 10.0
8+
"conditional_entropy_weight": 0.5,
9+
"temperature": 10.0,
10+
"feature_normalization": 2
1011
}
1112
},
1213
"feat": {
@@ -20,36 +21,51 @@
2021
}
2122
},
2223
"finetune": {
23-
"1_shot": {
24-
"fine_tuning_steps": 200,
25-
"fine_tuning_lr": 0.0001,
26-
"temperature": 1.0
27-
},
28-
"5_shot": {
29-
"fine_tuning_steps": 100,
24+
"default": {
25+
"fine_tuning_steps": 50,
3026
"fine_tuning_lr": 0.0001,
31-
"temperature": 10.0
27+
"temperature": 10.0,
28+
"feature_normalization": 2
3229
}
3330
},
3431
"pt_map": {
3532
"default": {
36-
"fine_tuning_steps": 10,
37-
"fine_tuning_lr": 0.2,
33+
"fine_tuning_steps": 30,
34+
"fine_tuning_lr": 0.3,
3835
"lambda_regularization": 10,
3936
"power_factor": 0.5,
4037
"feature_normalization": 2
4138
}
4239
},
40+
"transductive_finetuning": {
41+
"default": {
42+
"fine_tuning_steps": 10,
43+
"fine_tuning_lr": 0.0001,
44+
"temperature": 10.0,
45+
"feature_normalization": 2
46+
}
47+
},
4348
"laplacian_shot": {
44-
"1_shot": {
45-
"inference_steps": 20,
46-
"knn": 3,
47-
"lambda_regularization": 0.7
48-
},
49-
"5_shot": {
50-
"inference_steps": 20,
49+
"default": {
50+
"inference_steps": 10,
5151
"knn": 3,
52-
"lambda_regularization": 0.1
52+
"lambda_regularization": 0.1,
53+
"feature_normalization": 2
54+
}
55+
},
56+
"bd_cspn": {
57+
"default": {
58+
"feature_normalization": 2
59+
}
60+
},
61+
"prototypical_networks": {
62+
"default": {
63+
"feature_normalization": 2
64+
}
65+
},
66+
"simple_shot": {
67+
"default": {
68+
"feature_normalization": 2
5369
}
5470
}
5571
}

0 commit comments

Comments
 (0)