adding missing values for time series, fixing outliers (2d and more),… (#21)

… refactoring notebooks

Author: Patricia-IESE

badgers/generators/time_series/missingness.py

@@ -0,0 +1,60 @@
+import abc
+from typing import Tuple
+
+import numpy as np
+from numpy.random import default_rng
+
+from badgers.core.base import GeneratorMixin
+from badgers.core.decorators.time_series import preprocess_inputs
+
+
+class MissingValuesGenerator(GeneratorMixin):
+    """
+    Base class for transformers that generate point outliers in time-series data
+    """
+
+    def __init__(self, random_generator=default_rng(seed=0)):
+        """
+        :param random_generator: a random number generator
+        :param n_outliers: the number of outliers to generate
+        """
+        self.random_generator = random_generator
+        self.missing_indices_ = []
+
+    @abc.abstractmethod
+    def generate(self, X, y, **params) -> Tuple:
+        pass
+
+
+class MissingAtRandomGenerator(MissingValuesGenerator):
+    """
+    Randomly set data points to nan (missing)
+    """
+
+    def __init__(self, random_generator=default_rng(seed=0)):
+        """
+
+        :param random_generator: a random number generator
+
+        """
+        super().__init__(random_generator=random_generator)
+
+    @preprocess_inputs
+    def generate(self, X, y, n_missing: int = 10) -> Tuple:
+        """
+        Randomly set values to np.nan (missing)
+        :param X:
+        :param y:
+        :param n_missing: the number of outliers to generate
+        :return:
+        """
+        # generate missing values indices and values
+        rows = self.random_generator.choice(X.shape[0], size=n_missing, replace=False, p=None)
+        cols = self.random_generator.integers(low=0, high=X.shape[1], size=n_missing)
+
+        self.missing_indices_ = list(zip(rows, cols))
+
+        for r, c in self.missing_indices_:
+            X.iloc[r, c] = np.nan
+
+        return X, y

badgers/generators/time_series/outliers.py

@@ -1,11 +1,13 @@
 import abc
 from typing import Tuple
 
+import numpy as np
 from numpy.random import default_rng
 
 from badgers.core.base import GeneratorMixin
-from badgers.core.utils import random_sign
 from badgers.core.decorators.time_series import preprocess_inputs
+from badgers.core.utils import random_sign
+
 
 class OutliersGenerator(GeneratorMixin):
     """
@@ -48,10 +50,13 @@ def generate(self, X, y, n_outliers: int = 10) -> Tuple:
         :return:
         """
         # generate extreme values indices and values
-        self.outliers_indices_ = self.random_generator.choice(X.shape[0], size=n_outliers, replace=False, p=None)
+        rows = self.random_generator.choice(X.shape[0], size=n_outliers, replace=False, p=None)
+        cols = self.random_generator.integers(low=0, high=X.shape[1], size=n_outliers)
 
-        for idx in self.outliers_indices_:
-            X.iloc[idx, :] = 0
+        self.outliers_indices_ = list(zip(rows, cols))
+
+        for r, c in self.outliers_indices_:
+            X.iloc[r, c] = 0
 
         return X, y
 
@@ -84,15 +89,21 @@ def generate(self, X, y, n_outliers: int = 10,
         :return: the transformed array
         """
         # generate extreme values indices and values
-        self.outliers_indices_ = self.random_generator.choice(X.shape[0], size=n_outliers, replace=False, p=None)
+        delta = int(local_window_size / 2)
+
+        rows = self.random_generator.choice(
+            np.arange(delta, X.shape[0] - delta, dtype=int), size=n_outliers, replace=False, p=None)
+        cols = self.random_generator.integers(low=0, high=X.shape[1], size=n_outliers)
+
+        self.outliers_indices_ = list(zip(rows, cols))
 
-        for idx in self.outliers_indices_:
-            local_window = X.iloc[idx - int(local_window_size / 2):idx + int(local_window_size / 2), :]
+        for r, c in self.outliers_indices_:
+            local_window = X.iloc[r - delta:r + delta, c]
             local_mean = local_window.mean(axis=0)
             local_std = local_window.std(axis=0)
-            value = local_mean + random_sign(self.random_generator, size=X.shape[1]) * (
-                3. * local_std + self.random_generator.exponential(size=X.shape[1]))
+            value = local_mean + random_sign(self.random_generator) * (
+                3. * local_std + self.random_generator.exponential())
             # updating with new outliers
-            X.iloc[idx, :] = value
+            X.iloc[r, c] = value
 
         return X, y

docs/dev/Create-New-Tabular-Generators.ipynb

@@ -228,7 +228,7 @@
     "\n",
     "First, we generate some data using sklearn make_blob utility.\n",
     "\n",
-    "Second, we instanciate our generator and apply the generate function on our data.\n",
+    "Second, we instantiate our generator and apply the generate function on our data.\n",
     "\n",
     "Finally, we plot the data using seaborn (or matplotlib or plotly, or bokeh...)"
    ]
@@ -319,7 +319,7 @@
    "id": "5b40fee6-c51a-4948-9cd5-4e4ea4c2a1ce",
    "metadata": {},
    "source": [
-    "## Instanciate our generator"
+    "## Instantiate our generator"
    ]
   },
   {
@@ -470,7 +470,7 @@
    "id": "5ce36322-a02e-4c95-a181-deefc3473a49",
    "metadata": {},
    "source": [
-    "## Instanciate the generator, generate data with different `lam` values and plot the results"
+    "## Instantiate the generator, generate data with different `lam` values and plot the results"
    ]
   },
   {