Merge pull request #32 from VectorInstitute/other_metrics

fatemetkl · web-flow · commit 83063afb7560 · 2023-06-21T15:02:14.000-06:00
Other metrics
diff --git a/examples/apfl_example/server.py b/examples/apfl_example/server.py
@@ -79,7 +79,7 @@ def main(config: Dict[str, Any]) -> None:
         action="store",
         type=str,
         help="Path to configuration file.",
-        default="config.yaml",
+        default="examples/apfl_example/config.yaml",
     )
     args = parser.parse_args()
 
diff --git a/examples/basic_example/server.py b/examples/basic_example/server.py
@@ -77,7 +77,7 @@ def main(config: Dict[str, Any]) -> None:
         action="store",
         type=str,
         help="Path to configuration file.",
-        default="config.yaml",
+        default="examples/basic_example/config.yaml",
     )
     args = parser.parse_args()
 
diff --git a/examples/fedopt_example/server.py b/examples/fedopt_example/server.py
@@ -174,7 +174,7 @@ def main(config: Dict[str, Any]) -> None:
         action="store",
         type=str,
         help="Path to configuration file.",
-        default="config.yaml",
+        default="examples/fedopt_example/config.yaml",
     )
     args = parser.parse_args()
 
diff --git a/examples/fedprox_example/server.py b/examples/fedprox_example/server.py
@@ -108,7 +108,7 @@ def main(config: Dict[str, Any], server_address: str) -> None:
         action="store",
         type=str,
         help="Path to configuration file.",
-        default="config.yaml",
+        default="examples/fedprox_example/config.yaml",
     )
     parser.add_argument(
         "--server_address",
diff --git a/examples/fenda_example/server.py b/examples/fenda_example/server.py
@@ -83,7 +83,7 @@ def main(config: Dict[str, Any]) -> None:
         action="store",
         type=str,
         help="Path to configuration file.",
-        default="config.yaml",
+        default="examples/fenda_example/config.yaml",
     )
     args = parser.parse_args()
 
diff --git a/examples/fl_plus_local_ft_example/server.py b/examples/fl_plus_local_ft_example/server.py
@@ -77,7 +77,7 @@ def main(config: Dict[str, Any]) -> None:
         action="store",
         type=str,
         help="Path to configuration file.",
-        default="config.yaml",
+        default="examples/fl_plus_local_ft_example/config.yaml",
     )
     args = parser.parse_args()
 
diff --git a/examples/scaffold_example/README.md b/examples/scaffold_example/README.md
@@ -1,4 +1,4 @@
-# APFL Federated Learning Example
+# SCAFFOLD Federated Learning Example
 This is an example of [Stochastic Controlled Averaging for Federated Learning](https://arxiv.org/pdf/1910.06378.pdf)(SCAFFOLD). SCAFFOLD is a popular method for federated learning in situations where data across clients is heterogenous (non-iid). In these cases, FedAvg suffers from client drift resulting in unstable and slow
 convergence. To surmount this, SCAFFOLD uses control variates to correct for client drift during local updates. This is shown to decrease the number of communication rounds required when compared to other approaches to Federated Learning such as FedAvg.
 
diff --git a/examples/scaffold_example/client.py b/examples/scaffold_example/client.py
@@ -29,7 +29,7 @@ def setup_client(self, config: Config) -> None:
         learning_rate_local = self.narrow_config_type(config, "learning_rate_local", float)
 
         self.learning_rate_local = learning_rate_local
-        self.model: nn.Module = MnistNet()
+        self.model: nn.Module = MnistNet().to(self.device)
         self.criterion = torch.nn.CrossEntropyLoss()
         self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.learning_rate_local)
         sampler = DirichletLabelBasedSampler(list(range(10)), sample_percentage=0.75)
diff --git a/examples/scaffold_example/server.py b/examples/scaffold_example/server.py
@@ -90,7 +90,7 @@ def main(config: Dict[str, Any]) -> None:
         action="store",
         type=str,
         help="Path to configuration file.",
-        default="config.yaml",
+        default="examples/scaffold_example/config.yaml",
     )
     args = parser.parse_args()
 
diff --git a/fl4health/model_bases/apfl_base.py b/fl4health/model_bases/apfl_base.py
@@ -47,7 +47,7 @@ def update_alpha(self) -> None:
             assert local_grad is not None and global_grad is not None
             dif = local_p - global_p
             grad = torch.tensor(self.alpha) * local_grad + torch.tensor(1.0 - self.alpha) * global_grad
-            grad_alpha += torch.mul(dif, grad).sum().detach().numpy()
+            grad_alpha += torch.mul(dif, grad).sum().detach().cpu().numpy()
 
         # This update constant of 0.02 is not referenced in the paper
         # but is present in the official implementation and other ones I have seen
diff --git a/fl4health/utils/metrics.py b/fl4health/utils/metrics.py
@@ -49,6 +49,32 @@ def __call__(self, logits: torch.Tensor, target: torch.Tensor) -> Scalar:
         return metrics.balanced_accuracy_score(y_true, preds)
 
 
+class ROC_AUC(Metric):
+    def __init__(self, name: str = "ROC_AUC score"):
+        super().__init__(name)
+
+    def __call__(self, logits: torch.Tensor, target: torch.Tensor) -> Scalar:
+        assert logits.shape[0] == target.shape[0]
+        prob = torch.nn.functional.softmax(logits, dim=1)
+        prob = prob.cpu().detach()
+        target = target.cpu().detach()
+        y_true = target.reshape(-1)
+        return metrics.roc_auc_score(y_true, prob, average="weighted", multi_class="ovr")
+
+
+class F1(Metric):
+    def __init__(self, name: str = "F1 score"):
+        super().__init__(name)
+
+    def __call__(self, logits: torch.Tensor, target: torch.Tensor) -> Scalar:
+        assert logits.shape[0] == target.shape[0]
+        target = target.cpu().detach()
+        logits = logits.cpu().detach()
+        y_true = target.reshape(-1)
+        preds = np.argmax(logits, axis=1)
+        return metrics.f1_score(y_true, preds, average="weighted")
+
+
 class Meter(ABC):
     def __init__(self, metrics: Sequence[Metric], name: str = "") -> None:
         self.metrics: Sequence[Metric] = metrics
diff --git a/research/flamby/fed_isic2019/find_best_hp.py b/research/flamby/fed_isic2019/find_best_hp.py
@@ -34,7 +34,7 @@ def main(hp_sweep_dir: str) -> None:
         for run_folder in run_folders:
             run_loss = get_weighted_loss_from_server_log(run_folder)
             hp_losses.append(run_loss)
-        current_avg_loss = np.mean(hp_losses)
+        current_avg_loss = float(np.mean(hp_losses))
         if best_avg_loss is None or current_avg_loss <= best_avg_loss:
             log(INFO, f"Current Loss: {current_avg_loss} is lower than Best Loss: {best_avg_loss}")
             log(INFO, f"Best Folder: {hp_folder}, Previous Best: {best_folder}")
diff --git a/research/flamby/fed_isic2019/utils.py b/research/flamby/fed_isic2019/utils.py
@@ -39,8 +39,8 @@ def load_global_model(run_folder_dir: str) -> Baseline:
 
 
 def get_metric_avg_std(metrics: List[float]) -> Tuple[float, float]:
-    mean = np.mean(metrics)
-    std = np.std(metrics, ddof=1)
+    mean = float(np.mean(metrics))
+    std = float(np.std(metrics, ddof=1))
     return mean, std
 
 
diff --git a/tests/utils/metric_test.py b/tests/utils/metric_test.py
@@ -1,7 +1,7 @@
 import pytest
 import torch
 
-from fl4health.utils.metrics import AccumulationMeter, Accuracy, AverageMeter, BalancedAccuracy
+from fl4health.utils.metrics import F1, ROC_AUC, AccumulationMeter, Accuracy, AverageMeter, BalancedAccuracy
 
 
 def test_accuracy_metric() -> None:
@@ -124,3 +124,52 @@ def test_accumulation_meter() -> None:
     # Accumulating the batches together results in recalls of (1.0, 1/5, 5/7) for 0, 1, 2 classes, these are then
     # averaged over the number of classes giving the correct balanced accuracy for the whole
     assert pytest.approx(acc_m_balanced_accuracy, abs=0.00001) == (1.0 + 1.0 / 5.0 + 5.0 / 7.0) / 3.0
+
+
+def test_ROC_AUC_metric() -> None:
+
+    metric = ROC_AUC()
+
+    logits1 = torch.Tensor(
+        [
+            [3, 1, 2],
+            [0.88, 0.06, 0.06],
+            [0.1, 0.3, 1.2],
+            [0.9, 0.3, 0.1],
+            [3, 10, 2],
+            [1.5, 0.5, 0.5],
+        ]
+    )
+    target1 = torch.Tensor([0, 1, 2, 0, 1, 2])
+
+    assert metric(logits1, target1) == 0.75
+
+    logits2 = torch.Tensor(
+        [
+            [0.75, 0.20, 0.05],
+            [0.08, 0.86, 0.06],
+            [0.1, 0.1, 0.8],
+        ]
+    )
+    target2 = torch.Tensor([0, 0, 2])
+    # It should raise ValueError since AUC ROC score is not defined in this case.
+    with pytest.raises(ValueError):
+        metric(logits2, target2)
+
+
+def test_F1_metric() -> None:
+
+    metric = F1()
+
+    logits1 = torch.Tensor(
+        [
+            [3, 1, 2],
+            [0.88, 0.06, 0.06],
+            [0.1, 0.3, 1.2],
+            [0.9, 0.3, 1.1],
+            [0.5, 3.0, 1.5],
+        ]
+    )
+    target1 = torch.Tensor([0, 0, 2, 0, 2])
+
+    assert metric(logits1, target1) == 0.68

Original file line number	Diff line number	Diff line change
`@@ -79,7 +79,7 @@ def main(config: Dict[str, Any]) -> None:`
`79`	`79`	`action="store",`
`80`	`80`	`type=str,`
`81`	`81`	`help="Path to configuration file.",`
`82`		`- default="config.yaml",`
	`82`	`+ default="examples/apfl_example/config.yaml",`
`83`	`83`	`)`
`84`	`84`	`args = parser.parse_args()`
`85`	`85`
Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ def main(config: Dict[str, Any]) -> None:`
`77`	`77`	`action="store",`
`78`	`78`	`type=str,`
`79`	`79`	`help="Path to configuration file.",`
`80`		`- default="config.yaml",`
	`80`	`+ default="examples/basic_example/config.yaml",`
`81`	`81`	`)`
`82`	`82`	`args = parser.parse_args()`
`83`	`83`
Original file line number	Diff line number	Diff line change
`@@ -174,7 +174,7 @@ def main(config: Dict[str, Any]) -> None:`
`174`	`174`	`action="store",`
`175`	`175`	`type=str,`
`176`	`176`	`help="Path to configuration file.",`
`177`		`- default="config.yaml",`
	`177`	`+ default="examples/fedopt_example/config.yaml",`
`178`	`178`	`)`
`179`	`179`	`args = parser.parse_args()`
`180`	`180`
Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,7 @@ def main(config: Dict[str, Any], server_address: str) -> None:`
`108`	`108`	`action="store",`
`109`	`109`	`type=str,`
`110`	`110`	`help="Path to configuration file.",`
`111`		`- default="config.yaml",`
	`111`	`+ default="examples/fedprox_example/config.yaml",`
`112`	`112`	`)`
`113`	`113`	`parser.add_argument(`
`114`	`114`	`"--server_address",`
Original file line number	Diff line number	Diff line change
`@@ -83,7 +83,7 @@ def main(config: Dict[str, Any]) -> None:`
`83`	`83`	`action="store",`
`84`	`84`	`type=str,`
`85`	`85`	`help="Path to configuration file.",`
`86`		`- default="config.yaml",`
	`86`	`+ default="examples/fenda_example/config.yaml",`
`87`	`87`	`)`
`88`	`88`	`args = parser.parse_args()`
`89`	`89`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-# APFL Federated Learning Example`
	`1`	`+# SCAFFOLD Federated Learning Example`
`2`	`2`	`This is an example of [Stochastic Controlled Averaging for Federated Learning](https://arxiv.org/pdf/1910.06378.pdf)(SCAFFOLD). SCAFFOLD is a popular method for federated learning in situations where data across clients is heterogenous (non-iid). In these cases, FedAvg suffers from client drift resulting in unstable and slow`
`3`	`3`	`convergence. To surmount this, SCAFFOLD uses control variates to correct for client drift during local updates. This is shown to decrease the number of communication rounds required when compared to other approaches to Federated Learning such as FedAvg.`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,7 @@ def main(config: Dict[str, Any]) -> None:`
`90`	`90`	`action="store",`
`91`	`91`	`type=str,`
`92`	`92`	`help="Path to configuration file.",`
`93`		`- default="config.yaml",`
	`93`	`+ default="examples/scaffold_example/config.yaml",`
`94`	`94`	`)`
`95`	`95`	`args = parser.parse_args()`
`96`	`96`