Fixed develop conflicts again

Author: cristian-tamblay

DashAI/back/api/api_v1/endpoints/datasets.py

@@ -2157,13 +2157,16 @@ async def validate_type_changes(
                 filepath_or_buffer=tmp_file_path, params=parsed_params, n_rows=1000
             )
 
-            all_valid, errors = validate_multiple_type_changes(
+            all_valid, errors, resolved_dtypes = validate_multiple_type_changes(
                 sample_df, parsed_type_changes
             )
 
             return {
                 "valid": all_valid,
                 "errors": errors,
+                # A Date column's strptime format is detected from the data, so
+                # the frontend learns it here rather than choosing it.
+                "resolved_dtypes": resolved_dtypes,
             }
 
         finally:

DashAI/back/converters/simple_converters/time_series_window.py

@@ -817,8 +817,23 @@ def transform_dataset_with_schema(
                     # we are saving them as strings to preserve the original format.
                     # Can modify classes in value_types.py
                     # if want to use PyArrow date, time or timestamp types.
+                    #
+                    # Two dict shapes reach this function. The one built by
+                    # type inference and by get_columns_spec carries the
+                    # strptime format in "dtype"; the one a column emits
+                    # through to_string() carries it in "format" and leaves
+                    # "dtype" as the arrow type. Reading only "dtype" turned
+                    # the second shape's format into the literal "string".
+                    _format = info.get("format") or dtype
+                    if not _format:
+                        raise ValueError(
+                            f"Column '{column_name}' is typed as {_type} but "
+                            "carries no format. A date, time or timestamp "
+                            "column is stored as text plus a strptime format, "
+                            "so the format has to be resolved before saving."
+                        )
                     dashai_types[column_name] = arrow_to_dashai_types(
-                        arrow_type=_type, format=dtype
+                        arrow_type=_type, format=_format
                     )
                     pa_type = to_arrow_types("string")
                     dai_table[column_name] = table.column(column_name)

DashAI/back/dataloaders/classes/dashai_dataset.py

@@ -6,6 +6,7 @@
 from kink import di
 
 from DashAI.back.core.artifacts import normalize_artifacts
+from DashAI.back.core.enums.metrics import SplitEnum
 from DashAI.back.dependencies.database.models import Run
 from DashAI.back.models.base_model import BaseModel
 from DashAI.back.models.model_factory import ModelFactory
@@ -22,6 +23,28 @@ class BaseEvaluationStrategy(metaclass=ABCMeta):
 
     TYPE: Final[str] = "EvaluationStrategy"
 
+    # How this strategy divides the dataset. The frontend renders holdout
+    # controls or fold controls from this rather than comparing class names,
+    # which is what previously made a new strategy unreachable from the UI.
+    KIND: str = "holdout"
+
+    # Which partitions this strategy records metrics for. Scoring the training
+    # partition means predicting on rows the model was fitted on, which is a
+    # fit statistic; a forecaster has no such thing to report.
+    SCORED_SPLITS: tuple = (SplitEnum.TRAIN, SplitEnum.VALIDATION, SplitEnum.TEST)
+
+    @classmethod
+    def get_metadata(cls) -> dict:
+        """Describe the strategy for the frontend.
+
+        Returns
+        -------
+        dict
+            Mapping with ``kind``, which says whether this strategy splits the
+            dataset once or into folds.
+        """
+        return {"kind": cls.KIND}
+
     def __init__(
         self,
         factory: ModelFactory,

DashAI/back/evaluation/base_evaluation_strategy.py

@@ -9,7 +9,7 @@
 from DashAI.back.splitters.base_splitter import BaseSplitter
 
 
-class CrossValidationEvaluationStrategy(BaseEvaluationStrategy):
+class FoldEvaluationStrategy(BaseEvaluationStrategy):
     """Evaluation strategy implementing k-fold cross-validation with optional
     nested CV and HPO.
 
@@ -25,6 +25,8 @@ class CrossValidationEvaluationStrategy(BaseEvaluationStrategy):
     - LAST/LAST_OUTER: Aggregated metrics (mean and std) for simple/nested CV
     """
 
+    KIND: str = "cv"
+
     def execute(self, x, y, run: Run, db):
         """Execute k-fold cross-validation with optional nested CV and HPO.
 
@@ -107,9 +109,10 @@ def execute(self, x, y, run: Run, db):
             model.train(x_fold["train"], y_fold["train"])
 
             # Compute and store metrics for this fold
-            model.calculate_metrics(
-                split=SplitEnum.TRAIN, level=LevelEnum.FOLD, fold_index=i
-            )
+            if SplitEnum.TRAIN in self.SCORED_SPLITS:
+                model.calculate_metrics(
+                    split=SplitEnum.TRAIN, level=LevelEnum.FOLD, fold_index=i
+                )
             model.calculate_metrics(
                 split=SplitEnum.VALIDATION, level=LevelEnum.FOLD, fold_index=i
             )
@@ -200,7 +203,11 @@ def evaluate(self, model, input_dataset, output_dataset, metric, **kwargs):
             model.train(x_fold["train"], y_fold["train"])
 
             # Compute metrics on both training and validation sets
-            train_scores = model.compute_metrics(split=SplitEnum.TRAIN)
+            train_scores = (
+                model.compute_metrics(split=SplitEnum.TRAIN)
+                if SplitEnum.TRAIN in self.SCORED_SPLITS
+                else {}
+            )
             validation_scores = model.compute_metrics(split=SplitEnum.VALIDATION)
 
             # Collect the goal metric value from this fold
@@ -228,11 +235,12 @@ def evaluate(self, model, input_dataset, output_dataset, metric, **kwargs):
             }
 
             # Persist averaged metrics as TRIAL level (intermediate HPO result)
-            model._save_metrics(
-                results=averaged_train_results,
-                split=SplitEnum.TRAIN,
-                level=LevelEnum.TRIAL,
-            )
+            if SplitEnum.TRAIN in self.SCORED_SPLITS:
+                model._save_metrics(
+                    results=averaged_train_results,
+                    split=SplitEnum.TRAIN,
+                    level=LevelEnum.TRIAL,
+                )
             model._save_metrics(
                 results=averaged_validation_results,
                 split=SplitEnum.VALIDATION,
@@ -313,9 +321,10 @@ def _nested_cv(self, run_id, model, input_dataset, output_dataset, db):
             outer_model.calculate_metrics(
                 split=SplitEnum.VALIDATION, level=LevelEnum.OUTER_FOLD, fold_index=i
             )
-            outer_model.calculate_metrics(
-                split=SplitEnum.TRAIN, level=LevelEnum.OUTER_FOLD, fold_index=i
-            )
+            if SplitEnum.TRAIN in self.SCORED_SPLITS:
+                outer_model.calculate_metrics(
+                    split=SplitEnum.TRAIN, level=LevelEnum.OUTER_FOLD, fold_index=i
+                )
 
         # Aggregate outer fold metrics
         # Compute mean and std of OUTER_FOLD metrics and store as LAST_OUTER level
@@ -406,3 +415,20 @@ def _aggregate_fold_metrics(
 
         # Persist aggregated metrics to database
         db.commit()
+
+
+class CrossValidationEvaluationStrategy(FoldEvaluationStrategy):
+    """Score a model across folds, recording train and validation for each.
+
+    The ordinary cross-validation evaluation. Not offered for
+    ``ForecastingTask``, whose folds have no in-sample score to report;
+    ``ForecastingCrossValidationEvaluationStrategy`` handles that.
+    """
+
+    COMPATIBLE_COMPONENTS = [
+        "TabularClassificationTask",
+        "TextClassificationTask",
+        "ImageClassificationTask",
+        "TranslationTask",
+        "RegressionTask",
+    ]

DashAI/back/evaluation/cv.py

@@ -0,0 +1,25 @@
+"""Cross-validation for models that forecast a series from its own history."""
+
+from DashAI.back.core.enums.metrics import SplitEnum
+from DashAI.back.evaluation.cv import FoldEvaluationStrategy
+
+
+class ForecastingCrossValidationEvaluationStrategy(FoldEvaluationStrategy):
+    """Rolling origin cross-validation that records no in-sample metrics.
+
+    Only one thing separates this from the ordinary cross-validation
+    strategy: the training partition of a fold is not scored. Scoring it would
+    mean asking the model about dates it was fitted on, which is a fit
+    statistic rather than a forecast and is not comparable with the validation
+    score of the same fold.
+
+    Nothing else needs to change, and that is worth stating because it was not
+    obvious. Each fold already trains on everything before its own validation
+    window, and the final refit already uses the whole pool of rows outside
+    the reserved tail, so this strategy never had the horizon problem that
+    holdout did. Pair it with ``RollingOriginSplitter``, whose folds walk the
+    origin forward through time.
+    """
+
+    COMPATIBLE_COMPONENTS = ["ForecastingTask"]
+    SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST)

DashAI/back/evaluation/forecasting_cv.py

@@ -0,0 +1,111 @@
+"""Holdout evaluation for models that forecast a series from its own history."""
+
+from DashAI.back.core.enums.metrics import SplitEnum
+from DashAI.back.evaluation.holdout import SinglePartitionEvaluationStrategy
+
+
+class ForecastingHoldoutEvaluationStrategy(SinglePartitionEvaluationStrategy):
+    """Holdout evaluation that treats validation as history rather than a sample.
+
+    Two things the ordinary holdout strategy assumes are wrong for a
+    forecaster, and both of them are decisions about evaluation rather than
+    about any model.
+
+    **The training partition is not scored.** Scoring it would mean asking the
+    model about dates it was fitted on. That is an in-sample fit statistic,
+    which is a real diagnostic but is not comparable with a forecast made
+    several steps out; showing the two side by side in one results table
+    invites exactly that comparison. Only validation and test are recorded.
+
+    **The kept model is fitted through validation.** For most tasks the
+    validation partition is a held out sample that has to stay out of the fit.
+    For a forecaster it is simply the most recent stretch of the series, and
+    the stretch nearest to whatever comes next. Leaving it out makes the model
+    reach across the whole validation window before arriving at the first test
+    row, so the test metrics describe a longer horizon than the one being
+    asked about.
+
+    The validation metrics are still measured on a model fitted on training
+    data alone, which is what makes them honest: they are recorded before the
+    refit. So the two columns in the results table answer different questions,
+    and both answer them fairly.
+
+        validation metrics  <- model fitted on train
+        test metrics        <- model fitted on train + validation
+
+    Hyperparameter search is untouched. Its trials are scored on validation,
+    so they must not be fitted on it.
+    """
+
+    COMPATIBLE_COMPONENTS = ["ForecastingTask"]
+    SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST)
+
+    def execute(self, x, y, run, db):
+        """Score validation on a trial fit, then refit and score test.
+
+        Parameters
+        ----------
+        x : DatasetDict
+            Input partitions, keyed by split name.
+        y : DatasetDict
+            Target partitions, keyed by split name.
+        run : Run
+            Database model representing the current run.
+        db : Session
+            SQLAlchemy session used to persist metrics.
+
+        Returns
+        -------
+        tuple
+            The trained model and the paths of any HPO plots.
+        """
+        plot_paths = []
+        model = self.model
+
+        model.x_data = x
+        model.y_data = y
+
+        if self.optimizer and self.run_optimizable_parameters:
+            self._report_progress(0.2, "Hyperparameter optimization")
+            model = self._do_hpo(model, x, y, run, db)
+            plot_paths = self._generate_hpo_plots(run)
+
+        # Fitted on training data only, so the validation score below measures
+        # a model that has not seen the rows it is being scored on.
+        self._report_progress(0.5, "Training")
+        model.train(x["train"], y["train"])
+
+        self._report_progress(0.8, "Computing validation metrics")
+        self._calculate_metrics_if_missing(model, run, db, SplitEnum.VALIDATION)
+
+        # Now the model that gets kept: the same configuration, refitted with
+        # the validation rows included, since for a series they are history.
+        self._report_progress(0.9, "Refitting on train and validation")
+        self._fit_final_model(model, x, y)
+
+        self._report_progress(0.95, "Computing test metrics")
+        self._calculate_metrics_if_missing(model, run, db, SplitEnum.TEST)
+
+        return model, plot_paths
+
+    def _fit_final_model(self, model, x, y):
+        """Fit the kept model on the training and validation rows together.
+
+        Parameters
+        ----------
+        model : BaseModel
+            The model to fit.
+        x : DatasetDict
+            Input partitions.
+        y : DatasetDict
+            Target partitions.
+        """
+        validation_x = x.get("validation")
+        validation_y = y.get("validation")
+
+        if validation_x is None or validation_y is None or len(validation_x) == 0:
+            model.train(x["train"], y["train"])
+            return
+
+        extend = type(model)._extend
+        model.train(extend(x["train"], validation_x), extend(y["train"], validation_y))