working on it

Author: michielkallenberg

cybench/datasets/alignment.py

@@ -383,7 +383,6 @@ def align_inputs_and_labels(df_y: pd.DataFrame, dfs_x: dict) -> tuple:
     index_y_selection = set(df_y.index.values)
 
     for df_x in dfs_x.values():
-        #print(df_x.head(), len(index_y_selection))
         if len(df_x.index.names) == 1:
             index_y_selection = {
                 (loc_id, year)
@@ -483,42 +482,42 @@ def interpolate_time_series_data_items(X: list, max_season_window_length: int):
     return df_ts
 
 
-def aggregate_time_series_data(df_ts: pd.DataFrame, aggregate_time_series_to: str):
-    """Aggregate time series data to the specified resolution.
-
-    Args:
-        df_ts (pd.DataFrame): time series data in daily resolution
-        aggregate_time_series_to (str): resolution of aggregated data
-
-    Returns:
-        pd.DataFrame with interpolated data
-    """
-    if aggregate_time_series_to not in ["week", "dekad"]:
-        raise Exception(
-            f"Unsupported time series aggregation resolution {aggregate_time_series_to}"
-        )
-
-    if "date" not in df_ts.columns:
-        assert "date" in df_ts.index.names
-        df_ts.reset_index(inplace=True)
-
-    assert "date" in df_ts.columns
-    if aggregate_time_series_to == "week":
-        df_ts["week"] = df_ts["date"].dt.isocalendar().week
-    else:
-        df_ts["dekad"] = df_ts.apply(lambda r: dekad_from_date(r["date"]), axis=1)
-
-    ts_aggrs = {k: TIME_SERIES_AGGREGATIONS[k] for k in TIME_SERIES_PREDICTORS}
-    # Primarily to avoid losing the "date" column.
-    ts_aggrs["date"] = "min"
-    df_ts = (
-        df_ts.groupby([KEY_LOC, KEY_YEAR, aggregate_time_series_to], observed=True)
-        .agg(ts_aggrs)
-        .reset_index()
-    )
-    df_ts.drop(columns=[aggregate_time_series_to], inplace=True)
-
-    return df_ts
+# def aggregate_time_series_data(df_ts: pd.DataFrame, aggregate_time_series_to: str):
+#     """Aggregate time series data to the specified resolution.
+#
+#     Args:
+#         df_ts (pd.DataFrame): time series data in daily resolution
+#         aggregate_time_series_to (str): resolution of aggregated data
+#
+#     Returns:
+#         pd.DataFrame with interpolated data
+#     """
+#     if aggregate_time_series_to not in ["week", "dekad"]:
+#         raise Exception(
+#             f"Unsupported time series aggregation resolution {aggregate_time_series_to}"
+#         )
+#
+#     if "date" not in df_ts.columns:
+#         assert "date" in df_ts.index.names
+#         df_ts.reset_index(inplace=True)
+#
+#     assert "date" in df_ts.columns
+#     if aggregate_time_series_to == "week":
+#         df_ts["week"] = df_ts["date"].dt.isocalendar().week
+#     else:
+#         df_ts["dekad"] = df_ts.apply(lambda r: dekad_from_date(r["date"]), axis=1)
+#
+#     ts_aggrs = {k: TIME_SERIES_AGGREGATIONS[k] for k in TIME_SERIES_PREDICTORS}
+#     # Primarily to avoid losing the "date" column.
+#     ts_aggrs["date"] = "min"
+#     df_ts = (
+#         df_ts.groupby([KEY_LOC, KEY_YEAR, aggregate_time_series_to], observed=True)
+#         .agg(ts_aggrs)
+#         .reset_index()
+#     )
+#     df_ts.drop(columns=[aggregate_time_series_to], inplace=True)
+#
+#     return df_ts
 
 
 def make_aligned_tensors(

cybench/datasets/data_factory.py

@@ -89,7 +89,7 @@ def build(self) -> Dataset:
             )
 
         # Caching Strategy: Save result
-        if use_cache:
+        if use_cache: #Torch?!
             torch.save(dataset, cache_path)
 
         return dataset

cybench/datasets/features.py

@@ -284,9 +284,9 @@ def design_features(
     # Add location features if available
     if "location" in input_dfs:
         location_df = input_dfs["location"]
-        # Apply spherical transformations to geographic coordinates
-        spherical_features = {"lat_lon": {"lat": [-90, 90], "lon": [-180, 180]}}
-        location_df = apply_spherical_transformations(location_df, spherical_features)
+        # # Apply spherical transformations to geographic coordinates
+        # spherical_features = {"lat_lon": {"lat": [-90, 90], "lon": [-180, 180]}}
+        # location_df = apply_spherical_transformations(location_df, spherical_features)
         soil_features = soil_features.merge(location_df, on=[KEY_LOC], how="left")
 
     # Feature design for time series

cybench/datasets/normalizer.py

@@ -99,6 +99,7 @@ def fit_normalize(self, dfs):
         return self.normalize(dfs)
 
     def normalize(self, dfs):
+        ### WARNING: this is done in-place
         """
         Normalize using already-fitted parameters.
         Returns new list of DataFrames.

cybench/datasets/torch_dataset.py

@@ -37,7 +37,7 @@ def __len__(self) -> int:
         """Return the total number of samples in the dataset."""
         return len(self.y)
 
-    def __getitem__(self, index: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    def __getitem__(self, index: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
         """
         Get a sample from the dataset.
 
@@ -51,7 +51,7 @@ def split_on_years(
     ) -> Tuple['TorchDataset', 'TorchDataset']:
         """
         Create two new datasets based on the provided split in years.
-        !!NOTE!!: This is a memory intensive operation, because its making two subsets by copying the original data.
+        !!NOTE!!: This is a memory intensive operation, because it's making two subsets by copying the original data.
         Future implementations might want to rely on more memory-efficiency, in case that becomes a bottleneck.
 
         :param years_split: Tuple of two lists, e.g., ([2012, 2014], [2015, 2017])

cybench/util/store_and_cache.py

@@ -8,10 +8,10 @@
 import os
 
 from cybench.datasets.dataset import Dataset
-from cybench.datasets.torch_dataset import TorchDataset
+from cybench.config import DatasetConfig
 
 
-def cfg_to_hash(cfg: OmegaConf, add_str: bool = True):
+def cfg_to_hash(cfg: DatasetConfig, add_str: bool = True):
     """
     Create a deterministic hash from a DatasetConfig, to use it as a keys e.g. in caching
 

pyproject.toml

@@ -35,6 +35,7 @@ fastcore = "1.7.29"
 hydra-core = "^1.3.2"
 optuna = "^4.5.0"
 codecarbon = "^3.2.0"
+omegaconf = "^2.3.0"
 
 
 [build-system]