working on it

Author: michielkallenberg

TODO

@@ -0,0 +1,4 @@
+# Residual models
+# run_benchmark
+# dataset.save() uses torch (nothing for sklearn)
+# LSTM model

cybench/conf/evaluation/default.yaml

@@ -1,4 +1,4 @@
 name: default
 
 # Metrics to compute; must match names registered via @metric in cybench/evaluation/eval.py
-metrics: [mse, mape, r2]
+metrics: [mse, mape, r2, normalized_rmse]

cybench/models/torch/trainer.py

@@ -8,15 +8,13 @@
 """
 import logging
 import os
-import random
 import time
 from functools import partial
 from typing import Any, Dict, Optional, Tuple
 
 import numpy as np
 import torch
 import torch.nn as nn
-from torch.optim.lr_scheduler import LRScheduler
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 
@@ -316,7 +314,7 @@ def predict(self, dataset: TorchDataset, **kwargs) -> Tuple[np.ndarray, Dict[str
         return preds, info
 
     def predict_items(self, X, **kwargs) -> Tuple[np.ndarray, Dict[str, Any]]:
-        raise NotImplementedError # TODO: evaluate whether this methode is necessary
+        raise NotImplementedError # TODO: evaluate whether this method is necessary
 
     # ------------------------------------------------------------------
     # Persistence
@@ -340,7 +338,7 @@ def save(self, path: str, seed: int):
         torch.save(checkpoint, os.path.join(path, self.name + f"_{seed}.pt"))
 
     @classmethod
-    def load(cls, model_path: str, model: nn.Module, optimizer: torch.optim.Optimizer, **kwargs):
+    def load(cls, name: str, model_path: str, model: nn.Module, optimizer: torch.optim.Optimizer, **kwargs):
         """Load a saved checkpoint into a new Trainer instance."""
         device = kwargs.get('device', 'cuda' if torch.cuda.is_available() else 'cpu')
         ckpt = torch.load(model_path, map_location=device)
@@ -350,7 +348,8 @@ def load(cls, model_path: str, model: nn.Module, optimizer: torch.optim.Optimize
 
         # Create new trainer instance
         trainer = cls(
-            model=model,
+            name=name,
+            torch_model=model,
             optimizer=optimizer,
             device=device,
             dataloader=ckpt.get("dataloader", {}),

cybench/util/optuna_hyper_opt.py

@@ -1,12 +1,9 @@
 import logging
-import os
 from pathlib import Path
-from typing import Any, Dict, Union
+from typing import Any, Union
 
-import hydra
 import numpy as np
 import optuna
-import yaml
 from hydra.utils import instantiate
 from omegaconf import DictConfig, OmegaConf
 

tests/datasets/test_dataset.py

@@ -54,14 +54,14 @@ def test_dataset_item(dataset):
     assert set(dataset[0].keys()) == set(expected_indices + expected_data)
 
 
-def test_split(dataset_cfg):
+def test_split():
     data_path_county_features = os.path.join(PATH_DATA_DIR, "features", "maize", "US")
     train_csv = os.path.join(data_path_county_features, "grain_maize_US_train.csv")
     train_df = pd.read_csv(train_csv, index_col=[KEY_LOC, KEY_YEAR])
     train_yields = train_df[[KEY_TARGET]].copy()
     feature_cols = [c for c in train_df.columns if c != KEY_TARGET]
     train_features = train_df[feature_cols].copy()
-    dataset_cv = Dataset(dataset_cfg, train_yields, {KEY_COMBINED_FEATURES: train_features})
+    dataset_cv = Dataset(None, train_yields, {KEY_COMBINED_FEATURES: train_features})
 
     even_years = {x for x in dataset_cv.years if x % 2 == 0}
     odd_years = dataset_cv.years - even_years