update bench

Author: LongxingTan

benchmark/formatter.py

@@ -3,7 +3,6 @@
 Supports console tables, CSV, JSON, and LaTeX output for papers."""
 
 import csv
-from dataclasses import asdict
 import json
 import logging
 import os
@@ -99,7 +98,7 @@ def to_csv(self, path: str) -> None:
             logger.info("Results saved to %s", path)
         except ImportError:
             # Fallback with csv module
-            _dicts = [asdict(r) for r in self.results]
+            _dicts = [dict(r) for r in self.results]
             if not _dicts:
                 return
             keys = _dicts[0].keys()

benchmark/registry.py

@@ -66,14 +66,13 @@ def get(self, name: str) -> Type[Dataset]:
             class _LazyDataset(Dataset):
                 """Lazy-loaded dataset wrapper."""
 
-                name = name
-
                 def prepare_data(self, **kwargs):
                     return factory().prepare_data(**kwargs)
 
                 def get_train_valid_split(self, **kwargs):
                     return factory().get_train_valid_split(**kwargs)
 
+            _LazyDataset.name = name
             return _LazyDataset
         return super().get(name)
 

examples/run_anomaly.py

@@ -8,8 +8,9 @@
 import numpy as np
 import pandas as pd
 from sklearn.preprocessing import StandardScaler
+import tensorflow as tf
 
-from tfts import AutoConfig, AutoModel, AutoModelForAnomaly, KerasTrainer
+from tfts import AutoConfig, AutoModelForAnomaly, KerasTrainer, set_seed
 
 
 def parse_args():
@@ -28,7 +29,7 @@ def parse_args():
 def create_subsequences(time_series, train_length, pred_length):
     """Create subsequences for training and prediction."""
     subsequences, next_values = [], []
-    for i in range(len(time_series) - train_length - pred_length):
+    for i in range(len(time_series) - train_length - pred_length + 1):
         subsequences.append(time_series[i : i + train_length])
         next_values.append(time_series[i + train_length : i + train_length + pred_length].T[0])
     return subsequences, next_values
@@ -53,14 +54,21 @@ def load_and_preprocess_data(args):
 
 def train_model(args):
     """Train the model using the specified arguments."""
+    set_seed(args.seed)
     x_train, y_train, _ = load_and_preprocess_data(args)
 
     config = AutoConfig.for_model(args.use_model)
     config.train_sequence_length = args.train_length
     model = AutoModelForAnomaly.from_config(config)
 
     trainer = KerasTrainer(model)
-    trainer.train((x_train, y_train), (x_train, y_train), epochs=args.epochs)
+    trainer.train(
+        (x_train, y_train),
+        (x_train, y_train),
+        optimizer=tf.keras.optimizers.Adam(args.learning_rate),
+        epochs=args.epochs,
+        batch_size=args.batch_size,
+    )
     trainer.save_model(args.output_dir)
     print(f"Model trained and saved to {args.output_dir}")
 

examples/run_classification.py

@@ -8,7 +8,7 @@
 from sklearn.model_selection import train_test_split
 import tensorflow as tf
 
-from tfts import AutoConfig, AutoModelForClassification, KerasTrainer
+from tfts import AutoConfig, AutoModelForClassification, KerasTrainer, set_seed
 
 logging.getLogger("tensorflow").setLevel(logging.ERROR)
 
@@ -49,6 +49,7 @@ def readucr(filename):
 
 
 def run_train(args):
+    set_seed(args.seed)
     x_train, y_train, x_test, y_test = prepare_data()
 
     x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=42)

examples/run_tuner.py

@@ -1,6 +1,7 @@
 """Demo to tune the model parameters by Autotune"""
 
 import numpy as np
+import tensorflow as tf
 
 from tfts import AutoConfig, AutoModel, KerasTrainer, get_data
 
@@ -30,9 +31,15 @@ def objective(self, trial):
         config.num_stacked_layers = num_layers
 
         model = AutoModel.from_config(config, predict_sequence_length=self.predict_sequence_length)
-        trainer = KerasTrainer(model, optimizer_config={"learning_rate": learning_rate})
-
-        trainer.train(self.train_data, self.valid_data, epochs=epochs, verbose=0)
+        trainer = KerasTrainer(model)
+
+        trainer.train(
+            self.train_data,
+            self.valid_data,
+            optimizer=tf.keras.optimizers.Adam(learning_rate),
+            epochs=epochs,
+            verbose=0,
+        )
 
         x_valid, y_valid = self.valid_data
         predictions = trainer.predict(x_valid)

tests/test_benchmark_formatter.py

@@ -0,0 +1,62 @@
+import builtins
+import csv
+import tempfile
+import unittest
+
+from benchmark.formatter import BenchmarkResults
+from benchmark.registry import DatasetRegistry
+
+
+class BenchmarkResultsTest(unittest.TestCase):
+    def test_to_csv_fallback_handles_dict_results_without_pandas(self):
+        results = BenchmarkResults(
+            [
+                {"dataset": "synthetic", "model": "rnn", "metrics": {"mae": 0.1}},
+                {"dataset": "synthetic", "model": "tcn", "metrics": {"mae": 0.2}},
+            ]
+        )
+
+        original_import = builtins.__import__
+
+        def import_without_pandas(name, *args, **kwargs):
+            if name == "pandas":
+                raise ImportError("pandas disabled for fallback test")
+            return original_import(name, *args, **kwargs)
+
+        with tempfile.NamedTemporaryFile(suffix=".csv") as tmp:
+            try:
+                builtins.__import__ = import_without_pandas
+                results.to_csv(tmp.name)
+            finally:
+                builtins.__import__ = original_import
+
+            tmp.seek(0)
+            rows = list(csv.DictReader(line.decode("utf-8") for line in tmp.readlines()))
+
+        self.assertEqual(rows[0]["dataset"], "synthetic")
+        self.assertEqual(rows[0]["model"], "rnn")
+        self.assertEqual(rows[0]["metrics"], "{'mae': 0.1}")
+
+
+class DatasetRegistryTest(unittest.TestCase):
+    def test_lazy_dataset_registration_returns_instantiable_wrapper(self):
+        class ToyDataset:
+            def prepare_data(self, **kwargs):
+                return "prepared"
+
+            def get_train_valid_split(self, **kwargs):
+                return "split"
+
+        registry = DatasetRegistry()
+        registry.register_lazy("toy", lambda: ToyDataset())
+
+        dataset_cls = registry.get("toy")
+        dataset = dataset_cls()
+
+        self.assertEqual(dataset_cls.name, "toy")
+        self.assertEqual(dataset.prepare_data(), "prepared")
+        self.assertEqual(dataset.get_train_valid_split(), "split")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_data/test_processor.py

@@ -0,0 +1,67 @@
+import unittest
+
+import numpy as np
+import pandas as pd
+
+from tfts.data.auto_preprocessor import AutoPreprocessor
+from tfts.data.processor import DataProcessor
+
+
+class DataProcessorTest(unittest.TestCase):
+    def setUp(self):
+        self.df = pd.DataFrame(
+            {
+                "date": pd.date_range("2024-01-01", periods=20, freq="D"),
+                "value": np.arange(20, dtype=float),
+            }
+        )
+
+    def test_standard_inverse_transform(self):
+        processor = DataProcessor(lookback=4, horizon=2, normalize="standard", validation_split=0)
+        processor.prepare(self.df, target_col="value", time_col="date")
+        normalized = (self.df["value"].to_numpy() - self.df["value"].mean()) / self.df["value"].std()
+        np.testing.assert_allclose(processor.inverse_transform(normalized), self.df["value"], rtol=1e-6)
+
+    def test_inference_reuses_training_scaler_and_latest_window(self):
+        processor = DataProcessor(lookback=4, horizon=2, normalize="minmax", validation_split=0)
+        processor.prepare(self.df, target_col="value", time_col="date")
+        scaler = dict(processor._scaler_params)
+
+        inference_df = self.df.tail(4).copy()
+        inference_ds = processor.prepare_for_inference(inference_df, target_col="value", time_col="date")
+        batches = list(inference_ds.as_numpy_iterator())
+
+        self.assertEqual(processor._scaler_params, scaler)
+        self.assertEqual(len(batches), 1)
+        self.assertEqual(batches[0][0].shape, (1, 4, 1))
+
+    def test_validation_split_is_chronological(self):
+        processor = DataProcessor(
+            lookback=3,
+            horizon=1,
+            normalize=None,
+            validation_split=0.25,
+            shuffle=False,
+            batch_size=64,
+        )
+        train_ds, valid_ds = processor.prepare(self.df, target_col="value", time_col="date")
+        train_x = next(iter(train_ds))[0].numpy()
+        valid_x = next(iter(valid_ds))[0].numpy()
+        self.assertLess(train_x[-1, -1, 0], valid_x[0, -1, 0])
+
+    def test_scaler_is_fitted_without_validation_leakage(self):
+        processor = DataProcessor(lookback=3, horizon=1, normalize="minmax", validation_split=0.25)
+        processor.prepare(self.df, target_col="value", time_col="date")
+        self.assertEqual(processor._scaler_params["max"], 14.0)
+
+
+class AutoPreprocessorTest(unittest.TestCase):
+    def test_forward_fill_preserves_unprocessed_columns(self):
+        df = pd.DataFrame({"time": [1, 2, 3], "value": [1.0, np.nan, 3.0]})
+        result = AutoPreprocessor(handle_missing="ffill", columns=["value"]).fit_transform(df)
+        self.assertEqual(list(result.columns), ["time", "value"])
+        self.assertEqual(result.loc[1, "value"], 1.0)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_data/test_timeseries.py

@@ -551,6 +551,66 @@ def test_multiple_targets(self):
         self.assertEqual(len(seq.target), 2)
         self.assertIn("value", seq.target)
         self.assertIn("value2", seq.target)
+        x, y = seq[0]
+        self.assertEqual(x.shape[-1], 2)
+        self.assertEqual(y.shape[-1], 2)
+
+    def test_feature_columns_are_included_in_encoder_inputs(self):
+        seq = TimeSeriesSequence(
+            data=self.data,
+            time_idx="date",
+            target_column="value",
+            feature_columns=["feature1", "feature2"],
+            train_sequence_length=10,
+            predict_sequence_length=1,
+        )
+        x, y = seq[0]
+        self.assertEqual(x.shape[-1], 3)
+        self.assertEqual(y.shape[-1], 1)
+        np.testing.assert_allclose(x[0, :, 1], self.data["feature1"].iloc[:10])
+
+    def test_generated_feature_columns_are_included_in_encoder_inputs(self):
+        config = {"date_features": {"type": "datetime", "features": ["dayofweek"], "time_col": "date"}}
+        seq = TimeSeriesSequence(
+            data=self.data,
+            time_idx="date",
+            target_column="value",
+            feature_columns=["date_dayofweek"],
+            train_sequence_length=10,
+            predict_sequence_length=1,
+            feature_config=config,
+        )
+        x, y = seq[0]
+        self.assertEqual(x.shape[-1], 2)
+        self.assertEqual(y.shape[-1], 1)
+        np.testing.assert_allclose(x[0, :, 1], self.data["date"].dt.dayofweek.iloc[:10])
+
+    def test_one_step_horizon_and_boundary_continuity(self):
+        data = pd.DataFrame({"time": [0, 1, 2, 4, 5], "value": np.arange(5)})
+        seq = TimeSeriesSequence(
+            data=data,
+            time_idx="time",
+            target_column="value",
+            train_sequence_length=2,
+            predict_sequence_length=1,
+        )
+        self.assertEqual(len(seq.sequences), 1)
+        np.testing.assert_array_equal(seq.sequences[0][0][:, 0], [0, 1])
+        np.testing.assert_array_equal(seq.sequences[0][1][:, 0], [2])
+
+    def test_inference_mode_uses_latest_complete_window(self):
+        data = pd.DataFrame({"time": range(5), "value": np.arange(5)})
+        seq = TimeSeriesSequence(
+            data=data,
+            time_idx="time",
+            target_column="value",
+            train_sequence_length=3,
+            predict_sequence_length=2,
+            mode="inference",
+        )
+        self.assertEqual(len(seq.sequences), 3)
+        np.testing.assert_array_equal(seq.sequences[-1][0][:, 0], [2, 3, 4])
+        self.assertEqual(seq.sequences[-1][1].shape, (2, 1))
 
     def test_multiple_targets_as_list(self):
         """Test target column provided as list."""

tests/test_models/test_auto_model.py

@@ -1,3 +1,4 @@
+import tempfile
 import unittest
 
 import numpy as np
@@ -70,3 +71,16 @@ def test_auto_model_for_uncertainty(self):
         x = tf.random.normal([2, 14, 4])
         output = model(x)
         print(output.shape)
+
+    def test_save_and_load_preserves_prediction_length(self):
+        config = AutoConfig.for_model("rnn")
+        model = AutoModel.from_config(config, predict_sequence_length=3)
+        model.build_model(tf.keras.Input(shape=(8, 2)))
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            model.save_pretrained(tmpdir)
+            loaded = AutoModel.from_pretrained(tmpdir)
+            output = loaded(tf.random.normal([2, 8, 2]))
+
+        self.assertEqual(loaded.predict_sequence_length, 3)
+        self.assertEqual(output.shape, (2, 3, 1))

tfts/cli/forecasting.py

@@ -171,7 +171,7 @@ def predict(self, steps: Optional[int] = None, df: Optional[pd.DataFrame] = None
         Returns:
             Numpy array of shape ``(n_series, steps, n_targets)``.
         """
-        if not self._fitted and self.trainer is None:
+        if not self._fitted:
             raise RuntimeError("Pipeline must be fitted before prediction. Call .fit() first.")
 
         steps = steps or self.horizon