Ensure encoding of group ids is only done if needed

Author: Jan Beitner

pytorch_forecasting/data/timeseries.py

@@ -427,7 +427,13 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
             data[group_name] = self.transform_values(name, data[name], inverse=False, group_id=True)
 
         # encode categoricals
-        for name in set(self.group_ids + self.categoricals):
+        if isinstance(
+            self.target_normalizer, GroupNormalizer
+        ):  # if we use a group normalizer, group_ids must be encoded as well
+            group_ids_to_encode = self.group_ids
+        else:
+            group_ids_to_encode = []
+        for name in set(group_ids_to_encode + self.categoricals):
             allow_nans = name in self.dropout_categoricals
             if name in self.variable_groups:  # fit groups
                 columns = self.variable_groups[name]
@@ -452,7 +458,7 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
                         self.categorical_encoders[name] = self.categorical_encoders[name].fit(data[name])
 
         # encode them
-        for name in set(self.group_ids + self.flat_categoricals):
+        for name in set(group_ids_to_encode + self.flat_categoricals):
             data[name] = self.transform_values(name, data[name], inverse=False)
 
         # save special variables
@@ -494,6 +500,10 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
                 data[self.target], scales = self.target_normalizer.transform(data[self.target], data, return_norm=True)
             elif isinstance(self.target_normalizer, NaNLabelEncoder):
                 data[self.target] = self.target_normalizer.transform(data[self.target])
+                data["__target__"] = data[
+                    self.target
+                ]  # overwrite target because it requires encoding (continuous targets should not be normalized)
+                scales = "no target scales available for categorical target"
             else:
                 data[self.target], scales = self.target_normalizer.transform(data[self.target], return_norm=True)
 
@@ -510,6 +520,8 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
 
         if self.target in self.reals:
             self.scalers[self.target] = self.target_normalizer
+        else:
+            self.categorical_encoders[self.target] = self.target_normalizer
 
         # rescale continuous variables apart from target
         for name in self.reals:
@@ -830,8 +842,8 @@ def _construct_index(self, data: pd.DataFrame, predict_mode: bool) -> pd.DataFra
         if not group_ids.isin(df_index.group_id).all():
             missing_groups = data.loc[~group_ids.isin(df_index.group_id), self._group_ids].drop_duplicates()
             # decode values
-            for name in missing_groups.columns:
-                missing_groups[name] = self.transform_values(name, missing_groups[name], inverse=True, group_id=True)
+            for name, id in self._group_ids_mapping.items():
+                missing_groups[id] = self.transform_values(name, missing_groups[id], inverse=True, group_id=True)
             warnings.warn(
                 "Min encoder length and/or min_prediction_idx and/or min prediction length is too large for "
                 f"{len(missing_groups)} series/groups which therefore are not present in the dataset index. "

tests/conftest.py

@@ -11,6 +11,18 @@
 from pytorch_forecasting.data.examples import get_stallion_data  # isort:skip
 
 
+# for vscode debugging: https://stackoverflow.com/a/62563106/14121677
+if os.getenv("_PYTEST_RAISE", "0") != "0":
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_exception_interact(call):
+        raise call.excinfo.value
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_internalerror(excinfo):
+        raise excinfo.value
+
+
 @pytest.fixture
 def test_data():
     data = get_stallion_data()

tests/test_data.py

@@ -373,10 +373,48 @@ def test_categorical_target(test_data):
         min_encoder_length=1,
     )
 
-    x, y = next(iter(dataset.to_dataloader()))
+    _, y = next(iter(dataset.to_dataloader()))
     assert y.dtype is torch.long, "target must be of type long"
 
 
 def test_pickle(test_dataset):
     pickle.dumps(test_dataset)
     pickle.dumps(test_dataset.to_dataloader())
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {},
+        dict(
+            target_normalizer=GroupNormalizer(
+                groups=["agency", "sku"], log_scale=True, scale_by_group=True, log_zero_value=1.0
+            ),
+        ),
+    ],
+)
+def test_new_group_ids(test_data, kwargs):
+    """Test for new group ids in dataset"""
+    train_agency = test_data["agency"].iloc[0]
+    train_dataset = TimeSeriesDataSet(
+        test_data[lambda x: x.agency == train_agency],
+        time_idx="time_idx",
+        target="volume",
+        group_ids=["agency", "sku"],
+        max_encoder_length=5,
+        max_prediction_length=2,
+        min_prediction_length=1,
+        min_encoder_length=1,
+        categorical_encoders=dict(agency=NaNLabelEncoder(add_nan=True), sku=NaNLabelEncoder(add_nan=True)),
+        **kwargs,
+    )
+
+    # test sampling from training dataset
+    next(iter(train_dataset.to_dataloader()))
+
+    # create test dataset with group ids that have not been observed before
+    test_dataset = TimeSeriesDataSet.from_dataset(train_dataset, test_data)
+
+    # check that we can iterate through dataset without error
+    for _ in iter(test_dataset.to_dataloader()):
+        pass