Merge pull request #219 from winedarksea/dev

0.6.6

Author: winedarksea

TODO.md

@@ -12,9 +12,8 @@
 * The most recent data will generally be the most important
 * Forecasts are desired for the future immediately following the most recent data.
 
-# 0.6.5 🏮🏮🏮
-* horizontal and mosaic upgrades
-* bug fixes and template updates
+# 0.6.6 🐌🐌🐌
+* bug fixes, particularly compatability for the archaic pandas 1.0.3 still used at a certain big tech company
 
 ### Unstable Upstream Pacakges (those that are frequently broken by maintainers)
 * Pytorch-Forecasting

autots/__init__.py

@@ -26,7 +26,7 @@
 from autots.models.cassandra import Cassandra
 
 
-__version__ = '0.6.5'
+__version__ = '0.6.6'
 
 TransformTS = GeneralTransformer
 

autots/evaluator/auto_ts.py

@@ -134,7 +134,7 @@ class AutoTS(object):
             if True, recommend use in conjunction with `verbose` > 0 and `result_file` in the event of accidental complete termination.
             if "end_generation", as True and also ends entire generation of run. Note skipped models will not be tried again.
         generation_timeout (int): if not None, this is the number of minutes from start at which the generational search ends, then proceeding to validation
-            This is only checked after the end of each generation, so only offers an 'approximate' timeout for searching
+            This is only checked after the end of each generation, so only offers an 'approximate' timeout for searching. It is an overall cap for total generation search time, not per generation.
         current_model_file (str): file path to write to disk of current model params (for debugging if computer crashes). .json is appended
         force_gc (bool): if True, run gc.collect() after each model run. Probably won't make much difference.
         verbose (int): setting to 0 or lower should reduce most output. Higher numbers give more output.
@@ -1198,7 +1198,7 @@ def fit(
 
         # unpack ensemble models so sub models appear at highest level
         self.initial_template = unpack_ensemble_models(
-            self.initial_template,
+            self.initial_template.copy(),
             self.template_cols,
             keep_ensemble=True,
             recursive=True,
@@ -3225,6 +3225,7 @@ def plot_validations(
         if df_wide is None:
             df_wide = self.df_wide_numeric
         # choose which series to plot
+        agg_flag = False
         if series is None:
             if subset is None:
                 series = random.choice(df_wide.columns)
@@ -3241,6 +3242,9 @@ def plot_validations(
                     series = mapes[0]
                 elif str(subset).lower() == "worst score":
                     series = scores[0]
+                elif str(subset).lower() == "agg":
+                    agg_flag = True
+                    series = "Aggregate Forecasts"
                 else:
                     raise ValueError(
                         "plot_validations arg subset must be None, 'best' or 'worst'"
@@ -3260,23 +3264,40 @@ def plot_validations(
             mname = x.split("_")[1]
             if mname == "chosen" or mname in needed_mods:
                 new_df = pd.DataFrame(index=df_wide.index)
-                new_df[mname] = self.validation_forecasts[x].forecast[series]
-                new_df[mname + "_" + "upper"] = self.validation_forecasts[
-                    x
-                ].upper_forecast[series]
-                new_df[mname + "_" + "lower"] = self.validation_forecasts[
-                    x
-                ].lower_forecast[series]
+                if agg_flag:
+                    new_df[mname] = self.validation_forecasts[x].forecast.sum(axis=1)
+                    new_df[mname + "_" + "upper"] = self.validation_forecasts[
+                        x
+                    ].upper_forecast.sum(axis=1)
+                    new_df[mname + "_" + "lower"] = self.validation_forecasts[
+                        x
+                    ].lower_forecast.sum(axis=1)
+                else:
+                    new_df[mname] = self.validation_forecasts[x].forecast[series]
+                    new_df[mname + "_" + "upper"] = self.validation_forecasts[
+                        x
+                    ].upper_forecast[series]
+                    new_df[mname + "_" + "lower"] = self.validation_forecasts[
+                        x
+                    ].lower_forecast[series]
                 df_list.append(new_df)
         plot_df = pd.concat(df_list, sort=True, axis=0)
         # self.val_plot_df = plot_df.copy()
         plot_df = plot_df.groupby(level=0).last()
-        plot_df = (
-            df_wide[series]
-            .rename("actuals")
-            .to_frame()
-            .merge(plot_df, left_index=True, right_index=True, how="left")
-        )
+        if agg_flag:
+            plot_df = (
+                df_wide.sum(axis=1)
+                .rename("actuals")
+                .to_frame()
+                .merge(plot_df, left_index=True, right_index=True, how="left")
+            )
+        else:
+            plot_df = (
+                df_wide[series]
+                .rename("actuals")
+                .to_frame()
+                .merge(plot_df, left_index=True, right_index=True, how="left")
+            )
         if not include_bounds:
             colb = [
                 x for x in plot_df.columns if "_lower" not in x and "_upper" not in x
@@ -3366,11 +3387,12 @@ def list_failed_model_types(self):
         return temp[temp <= 0].index.to_list()
 
     def best_model_per_series_mape(self):
+        """This isn't quite classic mape but is a percentage mean error intended for quick visuals not final statistics (see model.results())."""
         best_model_per_series_mae = self.initial_results.per_series_mae[
             self.initial_results.per_series_mae.index == self.best_model_id
         ].mean(axis=0)
         # obsess over avoiding division by zero
-        scaler = self.df_wide_numeric.mean(axis=0)
+        scaler = self.df_wide_numeric.abs().mean(axis=0)
         scaler[scaler == 0] == np.nan
         scaler = scaler.fillna(self.df_wide_numeric.max(axis=0))
         scaler[scaler == 0] == 1

autots/models/base.py

@@ -510,14 +510,14 @@ def long_form_results(
             pd.DataFrame
         """
         upload = pd.melt(
-            self.forecast.reset_index(names='datetime'),
+            self.forecast.rename_axis(index='datetime').reset_index(),
             var_name="SeriesID",
             value_name="Value",
             id_vars="datetime",
         ).set_index("datetime")
         upload[interval_name] = "50%"
         upload_upper = pd.melt(
-            self.upper_forecast.reset_index(names='datetime'),
+            self.upper_forecast.rename_axis(index='datetime').reset_index(),
             var_name="SeriesID",
             value_name="Value",
             id_vars="datetime",
@@ -526,7 +526,7 @@ def long_form_results(
             interval_name
         ] = f"{round(100 - ((1- self.prediction_interval)/2) * 100, 0)}%"
         upload_lower = pd.melt(
-            self.lower_forecast.reset_index(names='datetime'),
+            self.lower_forecast.rename_axis(index='datetime').reset_index(),
             var_name="SeriesID",
             value_name="Value",
             id_vars="datetime",

autots/models/neural_forecast.py

@@ -207,7 +207,7 @@ def fit(self, df, future_regressor=None):
             raise ValueError(f"models not recognized: {models}")
 
         # model params
-
+        # requires pandas >= 1.5
         silly_format = df.reset_index(names='ds').melt(
             id_vars='ds', value_name='y', var_name='unique_id'
         )

autots/models/statsmodels.py

@@ -2005,12 +2005,16 @@ def theta_forecast_by_column(current_series, args):
 
     def get_new_params(self, method: str = 'random'):
         """Return dict of new parameters for parameter tuning."""
+        if method in ["deep"]:
+            period = random.choices([None, 7, 28, 288], [0.8, 0.1, 0.1, 0.01])[0]
+        else:
+            period = None
         return {
             'deseasonalize': random.choices([True, False], [0.8, 0.2])[0],
             'difference': random.choice([True, False]),
             'use_test': random.choices([True, False], [0.4, 0.2])[0],
             'method': "auto",
-            'period': None,
+            'period': period,
             'theta': random.choice([1.2, 1.4, 1.6, 2, 2.5, 3, 4]),
             'use_mle': random.choices([True, False], [0.0001, 0.99])[0],
         }

autots/tools/seasonal.py

@@ -33,6 +33,7 @@ def seasonal_int(include_one: bool = False, small=False, very_small=False):
         60: 0.05,
         96: 0.04,  # quarter in days
         168: 0.01,
+        288: 0.001,  # daily at 5 minute intervals
         364: 0.1,  # year to weekday
         1440: 0.01,
         420: 0.01,