Merge pull request #115 from jdb78/feature/deepar

Adding DeepAR and GPU fixes

Author: jdb78

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.3.0
+    hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-yaml
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: ""
+    hooks:
+    - id: flake8
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.6.4
+    hooks:
+    - id: isort
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
+    hooks:
+    - id: black
+  - repo: https://github.com/godaddy/tartufo
+    rev: v1.1.2
+    hooks:
+    - id: tartufo
+      language: python
+      args: [--pre-commit, --cleanup]

README.md

@@ -53,6 +53,8 @@ documentation with detailed tutorials.
 - [N-BEATS: Neural basis expansion analysis for interpretable time series forecasting](http://arxiv.org/abs/1905.10437)
   which has (if used as ensemble) outperformed all other methods including ensembles of traditional statical
   methods in the M4 competition. The M4 competition is arguably the most important benchmark for univariate time series forecasting.
+- [DeepAR: Probabilistic forecasting with autoregressive recurrent networks](https://www.sciencedirect.com/science/article/pii/S0169207019301888)
+  which is the one of the most popular forecasting algorithms and is often used as a baseline
 
 # Usage
 

pytorch_forecasting/__init__.py

@@ -2,7 +2,7 @@
 PyTorch Forecasting package for timeseries forecasting with PyTorch.
 """
 from pytorch_forecasting.data import EncoderNormalizer, GroupNormalizer, TimeSeriesDataSet
-from pytorch_forecasting.models import Baseline, NBeats, TemporalFusionTransformer
+from pytorch_forecasting.models import Baseline, DeepAR, NBeats, TemporalFusionTransformer
 
 __all__ = [
     "TimeSeriesDataSet",
@@ -11,6 +11,7 @@
     "TemporalFusionTransformer",
     "NBeats",
     "Baseline",
+    "DeepAR",
 ]
 
 __version__ = "0.0.0"

pytorch_forecasting/data/encoders.py

@@ -155,7 +155,7 @@ def __init__(
         method: str = "standard",
         center: bool = True,
         log_scale: Union[bool, float] = False,
-        log_zero_value: float = 0.0,
+        log_zero_value: float = -np.inf,
         coerce_positive: Union[float, bool] = None,
         eps: float = 1e-8,
     ):
@@ -167,13 +167,14 @@ def __init__(
                 (scale using quantiles 0.25-0.75). Defaults to "standard".
             center (bool, optional): If to center the output to zero. Defaults to True.
             log_scale (bool, optional): If to take log of values. Defaults to False. Defaults to False.
-            log_zero_value (float, optional): Value to map 0 to for ``log_scale=True`` or in softplus. Defaults to 0.0
+            log_zero_value (float, optional): Value to map 0 to for ``log_scale=True`` or in softplus. Defaults to -inf.
             coerce_positive (Union[bool, float, str], optional): If to coerce output to positive. Valid values:
                 * None, i.e. is automatically determined and might change to True if all values are >= 0 (Default).
                 * True, i.e. output is clamped at 0.
                 * False, i.e. values are not coerced
                 * float, i.e. softmax is applied with beta = coerce_positive.
-            eps (float, optional): Number for numerical stability of calcualtions. Defaults to 1e-8.
+            eps (float, optional): Number for numerical stability of calcualtions.
+                Defaults to 1e-8. For count data, 1.0 is recommended.
         """
         self.method = method
         assert method in ["standard", "robust"], f"method has invalid value {method}"
@@ -202,7 +203,7 @@ def get_parameters(self, *args, **kwargs) -> torch.Tensor:
         Returns:
             torch.Tensor: First element is center of data and second is scale
         """
-        return torch.tensor([self.center_, self.scale_])
+        return torch.stack([torch.as_tensor(self.center_), torch.as_tensor(self.scale_)], dim=-1)
 
     def _preprocess_y(self, y: Union[pd.Series, np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]:
         """
@@ -213,14 +214,11 @@ def _preprocess_y(self, y: Union[pd.Series, np.ndarray, torch.Tensor]) -> Union[
         Returns:
             Union[np.ndarray, torch.Tensor]: return rescaled series with type depending on input type
         """
-        if self.coerce_positive is None and not self.log_scale:
-            self.coerce_positive = (y >= 0).all()
-
         if self.log_scale:
             if isinstance(y, torch.Tensor):
-                y = torch.log(y + self.log_zero_value)
+                y = torch.log(y + self.log_zero_value + self.eps)
             else:
-                y = np.log(y + self.log_zero_value)
+                y = np.log(y + self.log_zero_value + self.eps)
         return y
 
     def fit(self, y: Union[pd.Series, np.ndarray, torch.Tensor]):
@@ -233,53 +231,77 @@ def fit(self, y: Union[pd.Series, np.ndarray, torch.Tensor]):
         Returns:
             TorchNormalizer: self
         """
+        if self.coerce_positive is None and not self.log_scale:
+            self.coerce_positive = (y >= 0).all()
         y = self._preprocess_y(y)
 
         if self.method == "standard":
             if isinstance(y, torch.Tensor):
-                self.center_ = torch.mean(y)
-                self.scale_ = torch.std(y) / (self.center_ + self.eps)
+                self.center_ = torch.mean(y, dim=-1) + self.eps
+                self.scale_ = torch.std(y, dim=-1) + self.eps
+            elif isinstance(y, np.ndarray):
+                self.center_ = np.mean(y, axis=-1) + self.eps
+                self.scale_ = np.std(y, axis=-1) + self.eps
             else:
-                self.center_ = np.mean(y)
-                self.scale_ = np.std(y) / (self.center_ + self.eps)
+                self.center_ = np.mean(y) + self.eps
+                self.scale_ = np.std(y) + self.eps
 
         elif self.method == "robust":
             if isinstance(y, torch.Tensor):
-                self.center_ = torch.median(y)
-                q_75 = y.kthvalue(int(len(y) * 0.75)).values
-                q_25 = y.kthvalue(int(len(y) * 0.25)).values
+                self.center_ = torch.median(y, dim=-1).values + self.eps
+                q_75 = y.kthvalue(int(len(y) * 0.75), dim=-1).values
+                q_25 = y.kthvalue(int(len(y) * 0.25), dim=-1).values
+            elif isinstance(y, np.ndarray):
+                self.center_ = np.median(y, axis=-1) + self.eps
+                q_75 = np.percentiley(y, 75, axis=-1)
+                q_25 = np.percentiley(y, 25, axis=-1)
             else:
-                self.center_ = np.median(y)
+                self.center_ = np.median(y) + self.eps
                 q_75 = np.percentiley(y, 75)
                 q_25 = np.percentiley(y, 25)
-            self.scale_ = (q_75 - q_25) / (self.center_ + self.eps) / 2.0
+            self.scale_ = (q_75 - q_25) / 2.0 + self.eps
+        if not self.center:
+            self.scale_ = self.center_
+            if isinstance(y, torch.Tensor):
+                self.center_ = torch.zeros_like(self.center_)
+            else:
+                self.center_ = np.zeros_like(self.center_)
         return self
 
     def transform(
-        self, y: Union[pd.Series, np.ndarray, torch.Tensor], return_norm: bool = False
+        self,
+        y: Union[pd.Series, np.ndarray, torch.Tensor],
+        return_norm: bool = False,
+        target_scale: torch.Tensor = None,
     ) -> Union[Tuple[Union[np.ndarray, torch.Tensor], np.ndarray], Union[np.ndarray, torch.Tensor]]:
         """
         Rescale data.
 
         Args:
             y (Union[pd.Series, np.ndarray, torch.Tensor]): input data
             return_norm (bool, optional): [description]. Defaults to False.
+            target_scale (torch.Tensor): target scale to use instead of fitted center and scale
 
         Returns:
             Union[Tuple[Union[np.ndarray, torch.Tensor], np.ndarray], Union[np.ndarray, torch.Tensor]]: rescaled
                 data with type depending on input type. returns second element if ``return_norm=True``
         """
-        if self.log_scale:
-            if isinstance(y, torch.Tensor):
-                y = (y + self.log_zero_value + self.eps).log()
-            else:
-                y = np.log(y + self.log_zero_value + self.eps)
-        if self.center:
-            y = (y / (self.center_ + self.eps) - 1) / (self.scale_ + self.eps)
-        else:
-            y = y / (self.center_ + self.eps)
+        y = self._preprocess_y(y)
+        # get center and scale
+        if target_scale is None:
+            target_scale = self.get_parameters().numpy()[None, :]
+        center = target_scale[..., 0]
+        scale = target_scale[..., 1]
+        if y.ndim > center.ndim:  # multiple batches -> expand size
+            center = center.view(*center.size(), *(1,) * (y.ndim - center.ndim))
+            scale = scale.view(*scale.size(), *(1,) * (y.ndim - scale.ndim))
+
+        # transform
+        y = (y - center) / scale
+
+        # return with center and scale or without
         if return_norm:
-            return y, self.get_parameters().numpy()[None, :]
+            return y, target_scale
         else:
             return y
 
@@ -303,6 +325,8 @@ def __call__(self, data: Dict[str, torch.Tensor]) -> torch.Tensor:
             data (Dict[str, torch.Tensor]): Dictionary with entries
                 * prediction: data to de-scale
                 * target_scale: center and scale of data
+            scale_only (bool): if to only scale prediction and not center it (even if `self.center is True`).
+                Defaults to False.
 
         Returns:
             torch.Tensor: de-scaled data
@@ -315,10 +339,8 @@ def __call__(self, data: Dict[str, torch.Tensor]) -> torch.Tensor:
             norm = norm.unsqueeze(-1)
 
         # transform
-        if self.center:
-            y_normed = (data["prediction"] * norm[:, 1, None] + 1) * norm[:, 0, None]
-        else:
-            y_normed = data["prediction"] * norm[:, 0, None]
+        y_normed = data["prediction"] * norm[:, 1, None] + norm[:, 0, None]
+
         if self.log_scale:
             y_normed = (y_normed.exp() - self.log_zero_value).clamp_min(0.0)
         elif isinstance(self.coerce_positive, bool) and self.coerce_positive:
@@ -379,7 +401,8 @@ def __init__(
                 * True, i.e. output is clamped at 0.
                 * False, i.e. values are not coerced
                 * float, i.e. softmax is applied with beta = coerce_positive.
-            eps (float, optional): Number for numerical stability of calcualtions. Defaults to 1e-8.
+            eps (float, optional): Number for numerical stability of calcualtions.
+                Defaults to 1e-8. For count data, 1.0 is recommended.
         """
         self.groups = groups
         self.scale_by_group = scale_by_group
@@ -403,24 +426,31 @@ def fit(self, y: pd.Series, X: pd.DataFrame):
         Returns:
             self
         """
+        if self.coerce_positive is None and not self.log_scale:
+            self.coerce_positive = (y >= 0).all()
         y = self._preprocess_y(y)
         if len(self.groups) == 0:
             assert not self.scale_by_group, "No groups are defined, i.e. `scale_by_group=[]`"
             if self.method == "standard":
-                mean = np.mean(y)
-                self.norm_ = mean, np.std(y) / (mean + self.eps)
+                self.norm_ = [np.mean(y) + self.eps, np.std(y) + self.eps]  # center and scale
             else:
                 quantiles = np.quantile(y, [0.25, 0.5, 0.75])
-                self.norm_ = quantiles[1], (quantiles[2] - quantiles[0]) / (quantiles[1] + self.eps)
+                self.norm_ = [
+                    quantiles[1] + self.eps,
+                    (quantiles[2] - quantiles[0]) / 2.0 + self.eps,
+                ]  # center and scale
+            if not self.center:
+                self.norm_[1] = self.norm_[0]
+                self.norm_[0] = 0.0
 
         elif self.scale_by_group:
             if self.method == "standard":
                 self.norm_ = {
                     g: X[[g]]
                     .assign(y=y)
                     .groupby(g, observed=True)
-                    .agg(mean=("y", "mean"), scale=("y", "std"))
-                    .assign(scale=lambda x: x.scale / (x["mean"] + self.eps))
+                    .agg(center=("y", "mean"), scale=("y", "std"))
+                    .assign(center=lambda x: x["center"] + self.eps, scale=lambda x: x.scale + self.eps)
                     for g in self.groups
                 }
             else:
@@ -431,12 +461,20 @@ def fit(self, y: pd.Series, X: pd.DataFrame):
                     .y.quantile([0.25, 0.5, 0.75])
                     .unstack(-1)
                     .assign(
-                        median=lambda x: x[0.5] + self.eps,
-                        scale=lambda x: (x[0.75] - x[0.25] + self.eps) / (x[0.5] + self.eps),
-                    )[["median", "scale"]]
+                        center=lambda x: x[0.5] + self.eps,
+                        scale=lambda x: (x[0.75] - x[0.25]) / 2.0 + self.eps,
+                    )[["center", "scale"]]
                     for g in self.groups
                 }
             # calculate missings
+            if not self.center:  # swap center and scale
+
+                def swap_parameters(norm):
+                    norm["scale"] = norm["center"]
+                    norm["center"] = 0.0
+                    return norm
+
+                self.norm = {g: swap_parameters(norm) for g, norm in self.norm_.items()}
             self.missing_ = {group: scales.median().to_dict() for group, scales in self.norm_.items()}
 
         else:
@@ -445,8 +483,8 @@ def fit(self, y: pd.Series, X: pd.DataFrame):
                     X[self.groups]
                     .assign(y=y)
                     .groupby(self.groups, observed=True)
-                    .agg(mean=("y", "mean"), scale=("y", "std"))
-                    .assign(scale=lambda x: x.scale / (x["mean"] + self.eps))
+                    .agg(center=("y", "mean"), scale=("y", "std"))
+                    .assign(center=lambda x: x["center"] + self.eps, scale=lambda x: x.scale + self.eps)
                 )
             else:
                 self.norm_ = (
@@ -456,10 +494,13 @@ def fit(self, y: pd.Series, X: pd.DataFrame):
                     .y.quantile([0.25, 0.5, 0.75])
                     .unstack(-1)
                     .assign(
-                        median=lambda x: x[0.5] + self.eps,
-                        scale=lambda x: (x[0.75] - x[0.25] + self.eps) / (x[0.5] + self.eps) / 2.0,
-                    )[["median", "scale"]]
+                        center=lambda x: x[0.5] + self.eps,
+                        scale=lambda x: (x[0.75] - x[0.25]) / 2.0 + self.eps,
+                    )[["center", "scale"]]
                 )
+            if not self.center:  # swap center and scale
+                self.norm_["scale"] = self.norm_["center"]
+                self.norm_["center"] = 0.0
             self.missing_ = self.norm_.median().to_dict()
         return self
 
@@ -471,10 +512,7 @@ def names(self) -> List[str]:
         Returns:
             List[str]: list of names
         """
-        if self.method == "standard":
-            return ["mean", "scale"]
-        else:
-            return ["median", "scale"]
+        return ["center", "scale"]
 
     def fit_transform(
         self, y: pd.Series, X: pd.DataFrame, return_norm: bool = False
@@ -495,12 +533,12 @@ def fit_transform(
 
     def inverse_transform(self, y: pd.Series, X: pd.DataFrame):
         """
-        Rescaling data to original scale - not implemented.
+        Rescaling data to original scale - not implemented - call class with target scale instead.
         """
         raise NotImplementedError()
 
     def transform(
-        self, y: pd.Series, X: pd.DataFrame, return_norm: bool = False
+        self, y: pd.Series, X: pd.DataFrame = None, return_norm: bool = False, target_scale: torch.Tensor = None
     ) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
         """
         Scale input data.
@@ -509,21 +547,16 @@ def transform(
             y (pd.Series): data to scale
             X (pd.DataFrame): dataframe with ``groups`` columns
             return_norm (bool, optional): If to return . Defaults to False.
+            target_scale (torch.Tensor): target scale to use instead of fitted center and scale
 
         Returns:
             Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]: Scaled data, if ``return_norm=True``, returns also scales
                 as second element
         """
-        norm = self.get_norm(X)
-        y = self._preprocess_y(y)
-        if self.center:
-            y_normed = (y / (norm[:, 0] + self.eps) - 1) / (norm[:, 1] + self.eps)
-        else:
-            y_normed = y / (norm[:, 0] + self.eps)
-        if return_norm:
-            return y_normed, norm
-        else:
-            return y_normed
+        if target_scale is None:
+            assert X is not None, "either target_scale or X has to be passed"
+            target_scale = self.get_norm(X)
+        return super().transform(y=y, return_norm=return_norm, target_scale=target_scale)
 
     def get_parameters(self, groups: Union[torch.Tensor, list, tuple], group_names: List[str] = None) -> np.ndarray:
         """