Timeseries data generation - TimeGAN

Author: Diyago

README.md

@@ -10,7 +10,7 @@ We well know GANs for success in the realistic image generation. However, they c
 * Arxiv article: ["Tabular GANs for uneven distribution"](https://arxiv.org/abs/2010.00638)
 * Medium post: [GANs for tabular data](https://towardsdatascience.com/review-of-gans-for-tabular-data-a30a2199342)
 
-### How to use library
+## How to use library
 
 * Installation: `pip install tabgan`
 * To generate new data to train by sampling and then filtering by adversarial training
@@ -88,7 +88,38 @@ print("OriginalGenerator metric", fit_predict(clf, new_train1, new_target1, X_te
 new_train1, new_target1 = GANGenerator().generate_data_pipe(X_train, y_train, X_test, )
 print("GANGenerator metric", fit_predict(clf, new_train1, new_target1, X_test, y_test))
 ```
+## Timeseries GAN generation TimeGAN
 
+You can easily adjust code to generate multidimensional timeseries data.
+Basically it extracts days, months and year from _date_. Demo how to use in the example below:
+```python
+import pandas as pd
+import numpy as np
+from tabgan.utils import get_year_mnth_dt_from_date,make_two_digit,collect_dates
+from tabgan.sampler import OriginalGenerator, GANGenerator
+
+
+train_size = 100
+train = pd.DataFrame(
+        np.random.randint(-10, 150, size=(train_size, 4)), columns=list("ABCD")
+    )
+min_date = pd.to_datetime('2019-01-01')
+max_date = pd.to_datetime('2021-12-31')
+d = (max_date - min_date).days + 1
+
+train['Date'] = min_date + pd.to_timedelta(pd.np.random.randint(d, size=train_size), unit='d')
+train = get_year_mnth_dt_from_date(train, 'Date')
+
+new_train, new_target = GANGenerator(gen_x_times=1.1, cat_cols=['year'], bot_filter_quantile=0.001,
+                                     top_filter_quantile=0.999,
+                                     is_post_process=True, pregeneration_frac=2, only_generated_data=False).\
+                                     generate_data_pipe(train.drop('Date', axis=1), None,
+                                                        train.drop('Date', axis=1)
+                                                                    )
+new_train = collect_dates(new_train)
+```
+
+## Experiments
 ### Datasets and experiment design
 
 **Running experiment**

pip_desc.md

@@ -97,6 +97,38 @@ if __name__ == "__main__":
     print("GANGenerator metric", fit_predict(clf, new_train1, new_target1, X_test, y_test))
 ```
 
+## Timeseries GAN generation TimeGAN
+
+You can easily adjust code to generate multidimensional timeseries data.
+Basically it extracts days, months and year from _date_. Demo how to use in the example below:
+```python
+import pandas as pd
+import numpy as np
+from tabgan.utils import get_year_mnth_dt_from_date,make_two_digit,collect_dates
+from tabgan.sampler import OriginalGenerator, GANGenerator
+
+
+train_size = 100
+train = pd.DataFrame(
+        np.random.randint(-10, 150, size=(train_size, 4)), columns=list("ABCD")
+    )
+min_date = pd.to_datetime('2019-01-01')
+max_date = pd.to_datetime('2021-12-31')
+d = (max_date - min_date).days + 1
+
+train['Date'] = min_date + pd.to_timedelta(pd.np.random.randint(d, size=train_size), unit='d')
+train = get_year_mnth_dt_from_date(train, 'Date')
+
+new_train, new_target = GANGenerator(gen_x_times=1.1, cat_cols=['year'], bot_filter_quantile=0.001,
+                                     top_filter_quantile=0.999,
+                                     is_post_process=True, pregeneration_frac=2, only_generated_data=False).\
+                                     generate_data_pipe(train.drop('Date', axis=1), None,
+                                                        train.drop('Date', axis=1)
+                                                                    )
+new_train = collect_dates(new_train)
+```
+
+## Experiments
 ### Datasets and experiment design
 
 **Running experiment**

src/tabgan/sampler.py

@@ -14,7 +14,7 @@
 from _ctgan.synthesizer import _CTGANSynthesizer as CTGAN
 from tabgan.abc_sampler import Sampler, SampleData
 from tabgan.adversarial_model import AdversarialModel
-from tabgan.utils import setup_logging
+from tabgan.utils import setup_logging, get_year_mnth_dt_from_date, collect_dates
 
 warnings.filterwarnings("ignore", category=FutureWarning)
 
@@ -317,12 +317,13 @@ def get_columns_if_exists(df, col) -> pd.DataFrame:
 
 if __name__ == "__main__":
     setup_logging(logging.DEBUG)
+    train_size = 100
     train = pd.DataFrame(
-        np.random.randint(-10, 150, size=(100, 4)), columns=list("ABCD")
+        np.random.randint(-10, 150, size=(train_size, 4)), columns=list("ABCD")
     )
     logging.info(train)
-    target = pd.DataFrame(np.random.randint(0, 2, size=(100, 1)), columns=list("Y"))
-    test = pd.DataFrame(np.random.randint(0, 100, size=(100, 4)), columns=list("ABCD"))
+    target = pd.DataFrame(np.random.randint(0, 2, size=(train_size, 1)), columns=list("Y"))
+    test = pd.DataFrame(np.random.randint(0, 100, size=(train_size, 4)), columns=list("ABCD"))
     _sampler(OriginalGenerator(gen_x_times=15), train, target, test)
     _sampler(
         GANGenerator(gen_x_times=10, only_generated_data=False,
@@ -336,3 +337,18 @@ def get_columns_if_exists(df, col) -> pd.DataFrame:
         None,
         train,
     )
+    min_date = pd.to_datetime('2019-01-01')
+    max_date = pd.to_datetime('2021-12-31')
+
+    d = (max_date - min_date).days + 1
+
+    train['Date'] = min_date + pd.to_timedelta(pd.np.random.randint(d, size=train_size), unit='d')
+    train = get_year_mnth_dt_from_date(train, 'Date')
+
+    new_train, new_target = GANGenerator(gen_x_times=1.1, cat_cols=['year'], bot_filter_quantile=0.001,
+                                         top_filter_quantile=0.999,
+                                         is_post_process=True, pregeneration_frac=2, only_generated_data=False).\
+                                         generate_data_pipe(train.drop('Date', axis=1), None,
+                                                            train.drop('Date', axis=1)
+                                                                        )
+    new_train = collect_dates(new_train)

src/tabgan/utils.py

@@ -1,6 +1,8 @@
 import logging
 import sys
 
+import pandas as pd
+
 
 def setup_logging(loglevel):
     """Setup basic logging
@@ -15,4 +17,27 @@ def setup_logging(loglevel):
     )
 
 
+def make_two_digit(num_as_str: str) -> pd.DataFrame:
+    if len(num_as_str) == 2:
+        return num_as_str
+    else:
+        return '0' + num_as_str
+
+
+def get_year_mnth_dt_from_date(df: pd.DataFrame, date_col='Date') -> pd.DataFrame:
+    df[date_col] = pd.to_datetime(df[date_col])
+    df['year'] = df[date_col].dt.year
+    df['month'] = df[date_col].dt.month
+    df['day'] = df[date_col].dt.day
+    return df
+
+
+def collect_dates(df: pd.DataFrame)-> pd.DataFrame:
+    df["Date"] = df['year'].astype(str) + '-' \
+                        + df['month'].astype(str).apply(make_two_digit) + '-' \
+                        + df['day'].astype(str).apply(make_two_digit)
+    df.drop(['year','month','day'], axis=1,inplace=True)
+    return df
+
+
 TEMP_TARGET = "_temp_target"