feat: drop the refit through validation for forecasting

Author: Irozuku

DashAI/back/evaluation/forecasting_holdout.py

@@ -5,107 +5,44 @@
 
 
 class ForecastingHoldoutEvaluationStrategy(SinglePartitionEvaluationStrategy):
-    """Holdout evaluation that treats validation as history rather than a sample.
+    """Holdout evaluation that records no in-sample metrics.
 
-    Two things the ordinary holdout strategy assumes are wrong for a
-    forecaster, and both of them are decisions about evaluation rather than
-    about any model.
+    One thing the ordinary holdout strategy assumes is wrong for a forecaster,
+    and it is a decision about evaluation rather than about any model.
 
     **The training partition is not scored.** Scoring it would mean asking the
     model about dates it was fitted on. That is an in-sample fit statistic,
     which is a real diagnostic but is not comparable with a forecast made
     several steps out; showing the two side by side in one results table
     invites exactly that comparison. Only validation and test are recorded.
 
-    **The kept model is fitted through validation.** For most tasks the
-    validation partition is a held out sample that has to stay out of the fit.
-    For a forecaster it is simply the most recent stretch of the series, and
-    the stretch nearest to whatever comes next. Leaving it out makes the model
-    reach across the whole validation window before arriving at the first test
-    row, so the test metrics describe a longer horizon than the one being
-    asked about.
+    **The kept model is fitted on the training partition alone**, like every
+    other holdout run, and nothing is fed to it afterwards. Two approaches that
+    would have changed that were tried and dropped, both because they hand the
+    model data from a partition it was meant to be held out from:
 
-    The validation metrics are still measured on a model fitted on training
-    data alone, which is what makes them honest: they are recorded before the
-    refit. So the two columns in the results table answer different questions,
-    and both answer them fairly.
+        refitting through validation before scoring test, which overwrote the
+        fit the validation metrics came from, so the saved model could not
+        reproduce its own results table;
 
-        validation metrics  <- model fitted on train
-        test metrics        <- model fitted on train + validation
+        advancing the model through the observed validation rows at predict
+        time, which re-estimates nothing but still lets a held out partition
+        reach the model, which no other task in DashAI does.
 
-    Hyperparameter search is untouched. Its trials are scored on validation,
-    so they must not be fitted on it.
-    """
-
-    COMPATIBLE_COMPONENTS = ["ForecastingTask"]
-    SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST)
-
-    def execute(self, x, y, run, db):
-        """Score validation on a trial fit, then refit and score test.
-
-        Parameters
-        ----------
-        x : DatasetDict
-            Input partitions, keyed by split name.
-        y : DatasetDict
-            Target partitions, keyed by split name.
-        run : Run
-            Database model representing the current run.
-        db : Session
-            SQLAlchemy session used to persist metrics.
-
-        Returns
-        -------
-        tuple
-            The trained model and the paths of any HPO plots.
-        """
-        plot_paths = []
-        model = self.model
-
-        model.x_data = x
-        model.y_data = y
-
-        if self.optimizer and self.run_optimizable_parameters:
-            self._report_progress(0.2, "Hyperparameter optimization")
-            model = self._do_hpo(model, x, y, run, db)
-            plot_paths = self._generate_hpo_plots(run)
+    So the two columns describe different horizons, and deliberately:
 
-        # Fitted on training data only, so the validation score below measures
-        # a model that has not seen the rows it is being scored on.
-        self._report_progress(0.5, "Training")
-        model.train(x["train"], y["train"])
+        validation metrics  <- forecasting 1..len(val) past the fit
+        test metrics        <- forecasting len(val)+1..len(val)+len(test),
+                               its own forecasts standing in for validation
 
-        self._report_progress(0.8, "Computing validation metrics")
-        self._calculate_metrics_if_missing(model, run, db, SplitEnum.VALIDATION)
+    The test column is therefore the harder question, not the same one further
+    along. Comparing like with like over a chosen horizon is what
+    ``RollingOriginSplitter`` is for, since its ``horizon`` says outright how
+    many steps ahead each refit is scored on.
 
-        # Now the model that gets kept: the same configuration, refitted with
-        # the validation rows included, since for a series they are history.
-        self._report_progress(0.9, "Refitting on train and validation")
-        self._fit_final_model(model, x, y)
-
-        self._report_progress(0.95, "Computing test metrics")
-        self._calculate_metrics_if_missing(model, run, db, SplitEnum.TEST)
-
-        return model, plot_paths
-
-    def _fit_final_model(self, model, x, y):
-        """Fit the kept model on the training and validation rows together.
-
-        Parameters
-        ----------
-        model : BaseModel
-            The model to fit.
-        x : DatasetDict
-            Input partitions.
-        y : DatasetDict
-            Target partitions.
-        """
-        validation_x = x.get("validation")
-        validation_y = y.get("validation")
-
-        if validation_x is None or validation_y is None or len(validation_x) == 0:
-            model.train(x["train"], y["train"])
-            return
+    Hyperparameter search is untouched. Its trials are scored on validation, so
+    they must not be fitted on it.
+    """
 
-        extend = type(model)._extend
-        model.train(extend(x["train"], validation_x), extend(y["train"], validation_y))
+    COMPATIBLE_COMPONENTS = ["ForecastingTask"]
+    SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST)

DashAI/back/evaluation/holdout.py

@@ -156,10 +156,7 @@ def evaluate(self, model, input_dataset, output_dataset, metric):
             output_dataset["validation"], is_fit=False
         )
 
-        # Calculate metric for train and validation data each trial. The
-        # training partition is skipped for a strategy that does not score it,
-        # which for a forecaster is not a preference: predicting on dates it
-        # was fitted on is refused, so asking would fail every trial.
+        # Calculate metric for train and validation data each trial.
         if SplitEnum.TRAIN in self.SCORED_SPLITS:
             model.calculate_metrics(split=SplitEnum.TRAIN, level=LevelEnum.TRIAL)
         model.calculate_metrics(split=SplitEnum.VALIDATION, level=LevelEnum.TRIAL)

DashAI/back/job/predict_job.py

@@ -30,7 +30,6 @@ def _run_prediction_pipeline(
     train_dataset: "DashAIDataset",
     loaded_dataset: "DashAIDataset",
     model_session: ModelSession,
-    history_dataset: "DashAIDataset" = None,
 ) -> Tuple["DashAIDataset", Any]:
     """Run shared prediction steps from prepared input data to final predictions.
 
@@ -46,13 +45,6 @@ def _run_prediction_pipeline(
         The rows to predict.
     model_session : ModelSession
         The session declaring the input and output columns.
-    history_dataset : DashAIDataset, optional
-        Every row of the dataset the requested ones came from, target column
-        included. A forecasting model is advanced through the rows that were
-        observed between the end of its fit and the window being asked about,
-        so it forecasts from the last thing that actually happened rather than
-        across a stretch it has to invent. Models that do not read a history
-        never see it.
 
     Returns
     -------
@@ -62,20 +54,7 @@ def _run_prediction_pipeline(
     import numpy as np
 
     prepared_dataset = loaded_dataset.select_columns(model_session.input_columns)
-
-    predict_kwargs = {}
-    output_column = model_session.output_columns[0]
-    if (
-        history_dataset is not None
-        and getattr(trained_model, "ACCEPTS_HISTORY", False)
-        and output_column in history_dataset.column_names
-    ):
-        predict_kwargs["history"] = (
-            history_dataset.select_columns(model_session.input_columns),
-            history_dataset.select_columns([output_column]),
-        )
-
-    y_pred_proba = np.array(trained_model.predict(prepared_dataset, **predict_kwargs))
+    y_pred_proba = np.array(trained_model.predict(prepared_dataset))
     y_pred = task.process_predictions(
         train_dataset, y_pred_proba, model_session.output_columns[0]
     )
@@ -471,12 +450,10 @@ def run(
 
             try:
                 # Load or create prediction dataset
-                history_dataset = None
                 if dataset_id:
                     loaded_dataset: "DashAIDataset" = load_dataset(
                         str(Path(f"{dataset.file_path}/dataset/"))
                     )
-                    history_dataset = loaded_dataset
                     if row_indexes is not None:
                         loaded_dataset = loaded_dataset.select(row_indexes)
                 else:
@@ -494,7 +471,6 @@ def run(
                     train_dataset=train_dataset,
                     loaded_dataset=loaded_dataset,
                     model_session=model_session,
-                    history_dataset=history_dataset,
                 )
 
             except ValueError as ve:

DashAI/back/models/forecasting/arima.py

@@ -250,9 +250,6 @@ def train(
             )
 
         with warnings.catch_warnings():
-            # statsmodels warns that it is assuming evenly spaced observations
-            # because no date index was supplied. That is the assumption this
-            # model makes on purpose, so the warning says nothing new.
             warnings.simplefilter("ignore")
             self._result = _ARIMA(series, order=(self.p, self.d, self.q)).fit()
 
@@ -261,18 +258,17 @@ def train(
         self._fitted = True
         return self
 
-    def predict(self, x: "DashAIDataset") -> "np.ndarray":
-        """Forecast forward from the end of the training series.
+    def _forecast(self, steps: int) -> "np.ndarray":
+        """Forecast the next ``steps`` periods after the end of the history.
 
         Parameters
         ----------
-        x : DashAIDataset
-            The rows to forecast, whose dates say how far ahead each one is.
+        steps : int
+            How many periods to forecast.
 
         Returns
         -------
         np.ndarray
-            One forecast value per requested row.
+            One value per period, in order.
         """
-        self._require_fitted()
-        return self._forecast_at(x, lambda steps: self._result.forecast(steps=steps))
+        return self._result.forecast(steps=steps)