diff --git a/Makefile b/Makefile index f76648b7..a40d5396 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ CHECK_DIRS := tfts examples tests style: ## Run formatters and linters (black, isort, flake8, pre-commit) black $(CHECK_DIRS) isort $(CHECK_DIRS) - flake8 $(check_dirs) + flake8 $(CHECK_DIRS) pre-commit run --all-files ## Run all unit tests diff --git a/README.md b/README.md index 0bdd3391..ec4d6e52 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ plt.show() You could train your own data by preparing 3D data as inputs, for both inputs and targets - option1 `np.ndarray` - option2 `tf.data.Dataset` +- option3 `tf.keras.utils.Sequence` Encoder only model inputs diff --git a/README_CN.md b/README_CN.md index 4345ebfb..b303558e 100644 --- a/README_CN.md +++ b/README_CN.md @@ -82,6 +82,7 @@ plt.show() 为方便使用，将数据转化为三维作为tfts的输入 - 选项1 `np.ndarray` - 选项2 `tf.data.Dataset` +- 选项3 `tf.keras.utils.Sequence` 编码类模型输入 diff --git a/docs/source/api.rst b/docs/source/api.rst index 5a16b4d4..6a7c7677 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -8,7 +8,10 @@ API :template: custom-module-template.rst :recursive: - datasets + data + features layers models + losses trainer + tasks diff --git a/docs/source/conf.py b/docs/source/conf.py index c91629c0..827744d4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -161,11 +161,12 @@ def setup(app: Sphinx): html_sidebars = { "index": [], - "quick-start": [], + "installation": [], "tutorials": [], "models": [], "tricks": [], - "CHANGELOG": [], + "feature_engineering": [], + "faq": [], } diff --git a/docs/source/faq.rst b/docs/source/faq.rst new file mode 100644 index 00000000..2e0c63d1 --- /dev/null +++ b/docs/source/faq.rst @@ -0,0 +1,666 @@ +Frequently Asked Questions (FAQ) +================================== + +This page answers common questions about TFTS. If you don't find your answer here, please ask in `GitHub Discussions `_. + + +General Questions +----------------- + +What is TFTS? +~~~~~~~~~~~~~ + +TFTS (TensorFlow Time Series) is a comprehensive Python library providing state-of-the-art deep learning models for time series analysis. It offers 20+ pre-implemented models, a unified API, and production-ready features for forecasting, classification, and anomaly detection. + + +What can I use TFTS for? +~~~~~~~~~~~~~~~~~~~~~~~~~ + +TFTS supports multiple time series tasks: + +- **Forecasting:** Single/multi-step, univariate/multivariate predictions +- **Probabilistic Forecasting:** Uncertainty quantification with confidence intervals +- **Classification:** Time series classification tasks +- **Anomaly Detection:** Identifying outliers and anomalies +- **Segmentation:** Change point detection + + +How does TFTS compare to other libraries? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**vs. Prophet:** + - TFTS: Deep learning models, flexible architecture, GPU support + - Prophet: Statistical models, interpretable, good for business forecasting + +**vs. NeuralProphet:** + - TFTS: 20+ models, TensorFlow-based, production-ready + - NeuralProphet: PyTorch-based, Prophet-like interface + +**vs. GluonTS:** + - TFTS: TensorFlow ecosystem, Keras integration + - GluonTS: MXNet/PyTorch, probabilistic focus + +**vs. Darts:** + - TFTS: Specialized for deep learning, extensive model selection + - Darts: Both classical and DL models, comprehensive toolkit + + +Installation & Setup +-------------------- + +How do I install TFTS? +~~~~~~~~~~~~~~~~~~~~~~ + +The easiest way: + +.. code-block:: bash + + pip install tfts + +For development: + +.. code-block:: bash + + git clone https://github.com/LongxingTan/Time-series-prediction.git + cd Time-series-prediction + pip install -e . + +See :doc:`installation` for detailed instructions. + + +Which TensorFlow version should I use? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Recommended:** TensorFlow >= 2.4 + +- **TensorFlow 2.4-2.8:** Stable, well-tested +- **TensorFlow 2.9+:** Latest features, best performance +- **TensorFlow 2.15+:** Keras 3.0 support (experimental) + +Check compatibility: + +.. code-block:: python + + import tensorflow as tf + print(f"TensorFlow version: {tf.__version__}") + + +Do I need a GPU? +~~~~~~~~~~~~~~~~ + +**No, but recommended.** + +- **CPU:** Works fine for small datasets and prototyping +- **GPU:** Significantly faster for large datasets and complex models +- **TPU:** Best for very large-scale training + +GPU speedup is typically 10-50x faster than CPU. + + +How do I enable GPU support? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +1. Install CUDA toolkit and cuDNN +2. Install TensorFlow with GPU support: + +.. code-block:: bash + + pip install tensorflow[and-cuda] + +3. Verify: + +.. code-block:: python + + import tensorflow as tf + print("GPUs Available:", len(tf.config.list_physical_devices('GPU'))) + + +Data & Models +------------- + +What data format does TFTS accept? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TFTS accepts multiple formats: + +1. **NumPy arrays:** + +.. code-block:: python + + x_train = np.array([...]) # Shape: (samples, timesteps, features) + y_train = np.array([...]) # Shape: (samples, pred_length, 1) + +2. **Pandas DataFrames (via TimeSeriesSequence):** + +.. code-block:: python + + from tfts.data import TimeSeriesSequence + + data_loader = TimeSeriesSequence( + data=df, + time_idx='timestamp', + target_column='target', + train_sequence_length=24, + predict_sequence_length=8 + ) + +3. **TensorFlow datasets:** + +.. code-block:: python + + dataset = tf.data.Dataset.from_tensor_slices((x, y)) + + +Which model should I choose? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Quick Guide:** + +- **Starting out:** ``seq2seq`` or ``dlinear`` (simple, fast) +- **Best accuracy:** ``informer``, ``autoformer``, or ``transformer`` +- **Long sequences:** ``informer`` or ``patch_tst`` +- **Interpretability:** ``nbeats`` or ``dlinear`` +- **Uncertainty:** ``deep_ar`` or ``diffusion`` +- **Speed:** ``tcn`` or ``dlinear`` + +See :doc:`models` for detailed comparisons. + + +How much data do I need? +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Minimum:** + - Simple models (RNN, TCN): 500-1000 samples + - Transformer models: 2000-5000 samples + - Complex models (Informer, Autoformer): 5000+ samples + +**Recommended:** + - 10,000+ samples for robust training + - More data → better generalization + - Data augmentation can help with small datasets + + +Can I use TFTS for multivariate time series? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Yes!** TFTS fully supports multivariate forecasting: + +.. code-block:: python + + # Multiple input features, single target + x = np.random.randn(1000, 24, 10) # 10 features + y = np.random.randn(1000, 8, 1) # 1 target + + # Multiple targets + y = np.random.randn(1000, 8, 3) # 3 targets + +Models like ``itransformer`` and ``tft`` are specifically designed for multivariate data. + + +Training & Performance +---------------------- + +How long does training take? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Typical training times (on single GPU):** + +- Simple models (RNN, DLinear): Minutes +- TCN/WaveNet: 10-30 minutes +- Transformer: 30-60 minutes +- Informer/Autoformer: 1-2 hours + +**Factors:** + - Dataset size + - Sequence length + - Model complexity + - Batch size + - Hardware + + +My model isn't learning. What should I do? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Checklist:** + +1. **Check data:** + +.. code-block:: python + + # Verify shapes + print(f"X shape: {x_train.shape}") + print(f"Y shape: {y_train.shape}") + + # Check for NaN/inf + assert not np.isnan(x_train).any() + assert not np.isinf(x_train).any() + +2. **Normalize data:** + +.. code-block:: python + + from sklearn.preprocessing import StandardScaler + + scaler = StandardScaler() + x_train_scaled = scaler.fit_transform(x_train.reshape(-1, x_train.shape[-1])) + x_train_scaled = x_train_scaled.reshape(x_train.shape) + +3. **Try simpler model:** + +.. code-block:: python + + # Start with RNN or DLinear + config = AutoConfig.for_model('rnn') + model = AutoModel.from_config(config, predict_sequence_length=8) + +4. **Adjust learning rate:** + +.. code-block:: python + + # Try lower learning rate + optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001) + +5. **Check loss function:** + +.. code-block:: python + + # For forecasting, use MSE or MAE + loss = tf.keras.losses.MeanSquaredError() + + +How can I improve model performance? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See the comprehensive :doc:`tricks` guide. Quick tips: + +1. **Feature engineering:** Add datetime, lag, and rolling features +2. **Hyperparameter tuning:** Adjust hidden_size, num_layers, dropout +3. **Ensemble models:** Combine multiple models +4. **Data augmentation:** Add noise, jittering +5. **Longer training:** More epochs with early stopping + + +How do I prevent overfitting? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Strategies:** + +1. **Dropout:** + +.. code-block:: python + + config.dropout = 0.2 + config.attention_probs_dropout_prob = 0.1 + +2. **Early stopping:** + +.. code-block:: python + + early_stop = tf.keras.callbacks.EarlyStopping( + monitor='val_loss', + patience=10, + restore_best_weights=True + ) + trainer.train(..., callbacks=[early_stop]) + +3. **L2 regularization:** + +.. code-block:: python + + config.weight_decay = 0.01 + +4. **Reduce model complexity:** + - Decrease hidden_size + - Reduce num_layers + - Use simpler model + + +Can I use pretrained models? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Yes, for transfer learning:** + +.. code-block:: python + + # Save model + model.save_pretrained('./my_model') + + # Load pretrained weights + new_model = AutoModel.from_pretrained('./my_model') + + # Fine-tune on new data + trainer = KerasTrainer(new_model) + trainer.train(new_data, epochs=10) + + +Production & Deployment +----------------------- + +How do I deploy TFTS models? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Option 1: TensorFlow Serving** + +.. code-block:: python + + # Save model + model.save('./saved_model') + +.. code-block:: bash + + # Serve with TensorFlow Serving + tensorflow_model_server --model_base_path=/path/to/saved_model + +**Option 2: ONNX export** + +.. code-block:: python + + import tf2onnx + + # Convert to ONNX + model_proto, _ = tf2onnx.convert.from_keras(model) + +**Option 3: Custom API** + +.. code-block:: python + + from flask import Flask, request + import joblib + + app = Flask(__name__) + model = joblib.load('model.pkl') + + @app.route('/predict', methods=['POST']) + def predict(): + data = request.json + prediction = model.predict(data) + return {'prediction': prediction.tolist()} + + +How do I handle missing values in production? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Strategies:** + +1. **Forward fill:** + +.. code-block:: python + + df = df.fillna(method='ffill') + +2. **Interpolation:** + +.. code-block:: python + + df = df.interpolate(method='linear') + +3. **Model-based imputation:** + +.. code-block:: python + + from sklearn.impute import KNNImputer + + imputer = KNNImputer(n_neighbors=5) + df_imputed = imputer.fit_transform(df) + + +Can I use TFTS with streaming data? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Yes!** Process data in batches: + +.. code-block:: python + + import queue + + data_queue = queue.Queue() + + def process_stream(): + while True: + # Get batch from stream + batch = data_queue.get(timeout=1) + + # Generate features + batch_processed = process_features(batch) + + # Predict + prediction = model.predict(batch_processed) + + # Send results + send_prediction(prediction) + + +How do I monitor model performance in production? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Monitoring checklist:** + +1. **Prediction latency:** + +.. code-block:: python + + import time + + start = time.time() + prediction = model.predict(x) + latency = time.time() - start + print(f"Latency: {latency:.3f}s") + +2. **Prediction accuracy:** + - Track MAE/MSE on incoming data + - Compare with ground truth when available + - Set up alerts for degradation + +3. **Data drift:** + - Monitor input feature distributions + - Check for out-of-distribution samples + +4. **Model updates:** + - Retrain periodically with new data + - A/B test new models before deployment + + +Advanced Topics +--------------- + +Can I customize model architectures? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Yes!** Several approaches: + +1. **Modify config:** + +.. code-block:: python + + config = AutoConfig.for_model('transformer') + config.hidden_size = 256 + config.num_layers = 6 + config.num_attention_heads = 8 + +2. **Custom head:** + +.. code-block:: python + + model = AutoModel.from_config(config, predict_sequence_length=24) + model.project = tf.keras.Sequential([ + tf.keras.layers.Dense(128, activation='relu'), + tf.keras.layers.Dense(1) + ]) + +3. **Fully custom model:** + +.. code-block:: python + + class CustomModel(tf.keras.Model): + def __init__(self): + super().__init__() + self.backbone = AutoModel.from_config(config) + self.custom_layers = ... + + def call(self, inputs): + features = self.backbone(inputs) + return self.custom_layers(features) + + +How do I implement custom loss functions? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + class CustomLoss(tf.keras.losses.Loss): + def call(self, y_true, y_pred): + # Your custom loss logic + mse = tf.reduce_mean(tf.square(y_true - y_pred)) + mae = tf.reduce_mean(tf.abs(y_true - y_pred)) + return mse + 0.1 * mae + + # Use in training + trainer = KerasTrainer(model, loss_fn=CustomLoss()) + + +Can I use attention weights for interpretability? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Yes, for models with attention:** + +.. code-block:: python + + # Get model with attention outputs + model = AutoModel.from_config(config, output_attention=True) + + # Make prediction + outputs = model(x, output_attention=True) + predictions = outputs['predictions'] + attention_weights = outputs['attention_weights'] + + # Visualize attention + import matplotlib.pyplot as plt + plt.imshow(attention_weights[0], cmap='hot') + plt.colorbar() + plt.show() + + +Troubleshooting +--------------- + +I'm getting OOM (Out of Memory) errors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Solutions:** + +1. **Reduce batch size:** + +.. code-block:: python + + batch_size = 16 # or smaller + +2. **Use gradient accumulation:** + +.. code-block:: python + + # Accumulate gradients over multiple batches + accumulation_steps = 4 + effective_batch_size = batch_size * accumulation_steps + +3. **Mixed precision training:** + +.. code-block:: python + + from tensorflow.keras.mixed_precision import set_global_policy + + set_global_policy('mixed_float16') + +4. **Reduce model size:** + - Decrease hidden_size + - Reduce num_layers + - Use model distillation + + +Training is very slow +~~~~~~~~~~~~~~~~~~~~~~ + +**Optimization tips:** + +1. **Use GPU:** + +.. code-block:: python + + with tf.device('/GPU:0'): + trainer.train(...) + +2. **Increase batch size:** + - Larger batches = fewer iterations + - Limited by memory + +3. **Use tf.data pipeline:** + +.. code-block:: python + + dataset = tf.data.Dataset.from_tensor_slices((x, y)) + dataset = dataset.batch(32).prefetch(tf.data.AUTOTUNE) + +4. **Mixed precision:** + +.. code-block:: python + + set_global_policy('mixed_float16') + + +Model predictions are all the same +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Common causes:** + +1. **Dead neurons:** Try lower learning rate or different activation +2. **Poor initialization:** Check weight initialization +3. **Wrong normalization:** Verify data preprocessing +4. **Too much regularization:** Reduce dropout/weight decay + + +Getting Help +------------ + +Where can I get help? +~~~~~~~~~~~~~~~~~~~~~ + +**Community:** + - 💬 `GitHub Discussions `_ - Ask questions + - 🐛 `GitHub Issues `_ - Report bugs + - 📖 `Documentation `_ - Read the docs + + +How do I report a bug? +~~~~~~~~~~~~~~~~~~~~~~~ + +**When reporting bugs, please include:** + +1. TFTS version: ``import tfts; print(tfts.__version__)`` +2. TensorFlow version: ``import tensorflow as tf; print(tf.__version__)`` +3. Python version: ``import sys; print(sys.version)`` +4. Operating system +5. Full error traceback +6. Minimal reproducible example + + +How can I contribute? +~~~~~~~~~~~~~~~~~~~~~~ + +We welcome contributions! See `CONTRIBUTING.md `_ for guidelines. + +**Ways to contribute:** + - Report bugs and suggest features + - Improve documentation + - Add new models or features + - Fix bugs + - Add tests + - Share examples and tutorials + + +See Also +-------- + +- :doc:`installation` - Installation guide +- :doc:`tutorials` - Step-by-step tutorials +- :doc:`models` - Model documentation +- :doc:`tricks` - Performance tips +- :doc:`api` - API reference diff --git a/docs/source/feature_engineering.rst b/docs/source/feature_engineering.rst new file mode 100644 index 00000000..61e81515 --- /dev/null +++ b/docs/source/feature_engineering.rst @@ -0,0 +1,636 @@ +Feature +=================== + +.. _feature_engineering: + +Feature engineering is crucial for improving time series model performance. TFTS provides built-in utilities to automatically generate powerful features from your time series data. + + +Overview +-------- + +TFTS offers automatic feature engineering through the ``TimeSeriesSequence`` data loader. Simply configure which features you want, and they'll be automatically generated and included in your training data. + +**Benefits:** + - Automated feature generation + - Consistent feature computation across train/valid/test sets + - Integration with TFTS models + - Customizable feature sets + + +Available Features +------------------ + +Datetime Features +~~~~~~~~~~~~~~~~~ + +Extract temporal patterns from datetime columns. + +**Supported Features:** + - ``year``, ``quarter``, ``month``, ``week`` + - ``day``, ``dayofyear``, ``dayofweek`` + - ``hour``, ``minute``, ``second`` + - ``is_weekend``, ``is_month_start``, ``is_month_end`` + - ``is_quarter_start``, ``is_quarter_end`` + - ``is_year_start``, ``is_year_end`` + +**Cyclic Encoding:** + +For periodic features (hour, day, month), TFTS can apply sine/cosine transformation to preserve cyclical nature: + + +.. \\text{sin}_x = \\sin\\left(\\frac{2\\pi x}{\\text{period}}\\right) + +.. \\text{cos}_x = \\cos\\left(\\frac{2\\pi x}{\\text{period}}\\right) + +**Example:** + +.. code-block:: python + + from tfts.data import TimeSeriesSequence + + feature_config = { + 'datetime_features': { + 'type': 'datetime', + 'features': ['hour', 'dayofweek', 'month', 'is_weekend'], + 'time_col': 'timestamp', + 'cyclic': True # Apply sine/cosine encoding + } + } + + data_loader = TimeSeriesSequence( + data=df, + time_idx='timestamp', + target_column='target', + train_sequence_length=24, + predict_sequence_length=8, + feature_config=feature_config + ) + + +Lag Features +~~~~~~~~~~~~ + +Create lagged versions of target or feature columns. + +**Use Cases:** + - Capture autocorrelation + - Model dependencies on past values + - Create autoregressive features + +**Example:** + +.. code-block:: python + + feature_config = { + 'lag_features': { + 'type': 'lag', + 'columns': 'target', # or list of columns + 'lags': [1, 2, 3, 7, 14, 21], # Lag periods + } + } + +This creates: ``target_lag_1``, ``target_lag_2``, ..., ``target_lag_21`` + +**Multiple Columns:** + +.. code-block:: python + + feature_config = { + 'lag_features': { + 'type': 'lag', + 'columns': ['target', 'feature1', 'feature2'], + 'lags': [1, 7], + } + } + + +Rolling Window Features +~~~~~~~~~~~~~~~~~~~~~~~ + +Compute rolling statistics over windows. + +**Supported Functions:** + - ``mean``: Rolling average + - ``std``: Rolling standard deviation + - ``min``, ``max``: Rolling extrema + - ``median``: Rolling median + - ``sum``: Rolling sum + - ``var``: Rolling variance + - ``skew``, ``kurt``: Higher moments + +**Example:** + +.. code-block:: python + + feature_config = { + 'rolling_features': { + 'type': 'rolling', + 'columns': 'target', + 'windows': [7, 14, 30], # Window sizes + 'functions': ['mean', 'std', 'min', 'max'], + } + } + +This creates features like: + - ``target_roll_7_mean`` + - ``target_roll_7_std`` + - ``target_roll_14_mean`` + - etc. + +**Advanced Rolling:** + +.. code-block:: python + + feature_config = { + 'rolling_statistics': { + 'type': 'rolling', + 'columns': ['temperature', 'humidity'], + 'windows': [6, 12, 24], # Hours + 'functions': ['mean', 'std', 'min', 'max'], + 'min_periods': 1, # Minimum observations required + } + } + + +Transform Features +~~~~~~~~~~~~~~~~~~ + +Apply mathematical transformations to columns. + +**Supported Transforms:** + - ``log1p``: log(1 + x) - handles zeros + - ``log``: Natural logarithm + - ``sqrt``: Square root + - ``square``: Square + - ``cbrt``: Cube root + - ``reciprocal``: 1/x + +**Example:** + +.. code-block:: python + + feature_config = { + 'transform_features': { + 'type': 'transform', + 'columns': 'target', + 'functions': ['log1p', 'sqrt'], + } + } + +**Use Cases:** + - Stabilize variance + - Handle skewed distributions + - Normalize scale + + +Moving Average Features +~~~~~~~~~~~~~~~~~~~~~~~ + +Exponential and simple moving averages. + +**Types:** + - Simple Moving Average (SMA) + - Exponential Moving Average (EMA) + +**Example:** + +.. code-block:: python + + feature_config = { + 'moving_average': { + 'type': 'moving_average', + 'columns': 'target', + 'windows': [7, 14, 30], + 'ma_type': 'both', # 'sma', 'ema', or 'both' + } + } + +**EMA Formula:** + + +.. \\text{EMA}_t = \\alpha \\cdot x_t + (1 - \\alpha) \\cdot \\text{EMA}_{t-1} + +.. \\alpha = \\frac{2}{\\text{window} + 1} + + +Second-Order Features +~~~~~~~~~~~~~~~~~~~~~ + +Interactions between features. + +**Example:** + +.. code-block:: python + + feature_config = { + 'interaction_features': { + 'type': '2order', + 'columns': ['feature1', 'feature2'], + 'operations': ['multiply', 'add', 'subtract', 'divide'], + } + } + + +Complete Example +---------------- + +Comprehensive Feature Engineering +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here's a complete example combining multiple feature types: + +.. code-block:: python + + import pandas as pd + from tfts.data import TimeSeriesSequence + from tfts import AutoConfig, AutoModel, KerasTrainer + + # Sample data + df = pd.read_csv('timeseries_data.csv') + + # Comprehensive feature configuration + feature_config = { + # Temporal features + 'datetime': { + 'type': 'datetime', + 'features': [ + 'hour', 'dayofweek', 'month', 'quarter', + 'is_weekend', 'is_month_start', 'is_month_end' + ], + 'time_col': 'timestamp', + 'cyclic': True # Use sine/cosine encoding + }, + + # Lag features + 'lags': { + 'type': 'lag', + 'columns': 'target', + 'lags': [1, 2, 3, 7, 14, 21, 28], # 1 day to 4 weeks + }, + + # Rolling statistics + 'rolling': { + 'type': 'rolling', + 'columns': 'target', + 'windows': [7, 14, 28], # Weekly, biweekly, monthly + 'functions': ['mean', 'std', 'min', 'max'], + }, + + # Transformations + 'transforms': { + 'type': 'transform', + 'columns': 'target', + 'functions': ['log1p', 'sqrt'], + }, + + # Moving averages + 'moving_avg': { + 'type': 'moving_average', + 'columns': 'target', + 'windows': [7, 30], + 'ma_type': 'both', + }, + } + + # Create data loader with automatic feature engineering + data_loader = TimeSeriesSequence( + data=df, + time_idx='timestamp', + target_column='target', + group_column=['location'], # Optional: group by location + train_sequence_length=168, # 1 week of hourly data + predict_sequence_length=24, # Predict next 24 hours + batch_size=32, + feature_config=feature_config, + mode='train' + ) + + # Train model + config = AutoConfig.for_model('transformer') + model = AutoModel.from_config(config, predict_sequence_length=24) + trainer = KerasTrainer(model) + trainer.train(data_loader, epochs=50) + + +Custom Features +--------------- + +Creating Custom Features +~~~~~~~~~~~~~~~~~~~~~~~~ + +You can add custom features by preprocessing your dataframe before creating the data loader: + +.. code-block:: python + + import pandas as pd + import numpy as np + + # Load data + df = pd.read_csv('data.csv') + df['timestamp'] = pd.to_datetime(df['timestamp']) + + # Custom features + df['hour_sin'] = np.sin(2 * np.pi * df['timestamp'].dt.hour / 24) + df['hour_cos'] = np.cos(2 * np.pi * df['timestamp'].dt.hour / 24) + + # Custom domain-specific features + df['is_business_hours'] = df['timestamp'].dt.hour.between(9, 17) + df['is_peak_hours'] = df['timestamp'].dt.hour.isin([8, 9, 17, 18]) + + # Weather impact (example) + df['temp_humidity_interaction'] = df['temperature'] * df['humidity'] + + # Now create data loader + data_loader = TimeSeriesSequence( + data=df, + time_idx='timestamp', + target_column='target', + train_sequence_length=24, + predict_sequence_length=8, + ) + + +Using Feature Registry +~~~~~~~~~~~~~~~~~~~~~~ + +Register custom feature functions: + +.. code-block:: python + + from tfts.features import registry + + @registry + def add_custom_business_features(df, config): + \"\"\"Add business-specific features.\"\"\" + df['is_business_day'] = df['timestamp'].dt.dayofweek < 5 + df['is_holiday'] = df['timestamp'].isin(holidays) + df['days_to_holiday'] = (df['timestamp'] - next_holiday).dt.days + return df + + # Use in feature_config + feature_config = { + 'custom': { + 'type': 'custom', + 'function': add_custom_business_features, + } + } + + +Best Practices +-------------- + +Feature Selection +~~~~~~~~~~~~~~~~~ + +**Start Simple:** + Begin with basic datetime features and a few lags. Add complexity gradually based on validation performance. + +**Domain Knowledge:** + Incorporate domain-specific patterns (e.g., business hours, holidays, events). + +**Avoid Leakage:** + Never use future information. Lag features must use only past data. + +**Handle Missing Values:** + Forward-fill or interpolate missing values before feature engineering. + + +Feature Scaling +~~~~~~~~~~~~~~~ + +**Built-in Normalization:** + +.. code-block:: python + + from tfts.features import Normalizer + + normalizer = Normalizer(method='standard') # or 'minmax', 'robust' + df_normalized = normalizer.fit_transform(df) + + +**Per-Group Normalization:** + +.. code-block:: python + + # Normalize within each group (e.g., per store, per sensor) + df_normalized = df.groupby('group_id').apply( + lambda x: (x - x.mean()) / x.std() + ) + + +Memory Optimization +~~~~~~~~~~~~~~~~~~~ + +**For Large Datasets:** + +1. **Generate features on-the-fly:** + +.. code-block:: python + + class OnTheFlyFeatures(tf.keras.utils.Sequence): + def __getitem__(self, idx): + # Load batch + batch = self.load_batch(idx) + # Generate features + batch = self.add_features(batch) + return batch + +2. **Use efficient data types:** + +.. code-block:: python + + # Downcast numeric types + df['hour'] = df['hour'].astype('int8') + df['dayofweek'] = df['dayofweek'].astype('int8') + +3. **Chunked processing:** + +.. code-block:: python + + for chunk in pd.read_csv('large_file.csv', chunksize=10000): + chunk = add_features(chunk) + process(chunk) + + +Feature Importance Analysis +---------------------------- + +Analyzing Feature Impact +~~~~~~~~~~~~~~~~~~~~~~~~ + +Use built-in methods to understand which features matter: + +**Method 1: Permutation Importance** + +.. code-block:: python + + from sklearn.inspection import permutation_importance + + # Train model with all features + model.fit(X_train, y_train) + + # Compute importance + result = permutation_importance( + model, X_valid, y_valid, + n_repeats=10, + random_state=42 + ) + + # Plot importance + import matplotlib.pyplot as plt + sorted_idx = result.importances_mean.argsort() + plt.barh(feature_names[sorted_idx], result.importances_mean[sorted_idx]) + plt.xlabel('Permutation Importance') + + +**Method 2: SHAP Values** + +.. code-block:: python + + import shap + + # Create explainer + explainer = shap.Explainer(model) + shap_values = explainer(X_valid) + + # Plot + shap.summary_plot(shap_values, X_valid) + + +Feature Engineering Workflows +------------------------------ + +Development Workflow +~~~~~~~~~~~~~~~~~~~~ + +1. **Baseline:** Train with minimal features +2. **Iterate:** Add feature groups one at a time +3. **Validate:** Check impact on validation metrics +4. **Prune:** Remove features that don't help +5. **Optimize:** Fine-tune feature parameters + + +Production Workflow +~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + # 1. Define feature pipeline + from sklearn.pipeline import Pipeline + from tfts.features import FeatureEngineer + + feature_pipeline = Pipeline([ + ('datetime', DatetimeFeatures()), + ('lags', LagFeatures(lags=[1, 7, 14])), + ('rolling', RollingFeatures(windows=[7, 14])), + ('normalize', Normalizer(method='standard')), + ]) + + # 2. Fit on training data + feature_pipeline.fit(train_data) + + # 3. Transform train/valid/test consistently + X_train = feature_pipeline.transform(train_data) + X_valid = feature_pipeline.transform(valid_data) + X_test = feature_pipeline.transform(test_data) + + # 4. Save pipeline + import joblib + joblib.dump(feature_pipeline, 'feature_pipeline.pkl') + + # 5. Load in production + pipeline = joblib.load('feature_pipeline.pkl') + X_new = pipeline.transform(new_data) + + +Common Patterns +--------------- + +Seasonal Decomposition Features +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from statsmodels.tsa.seasonal import seasonal_decompose + + # Decompose time series + result = seasonal_decompose(df['target'], model='additive', period=24) + + # Add components as features + df['trend'] = result.trend + df['seasonal'] = result.seasonal + df['residual'] = result.resid + + +Fourier Features for Seasonality +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import numpy as np + + def add_fourier_features(df, period, K=5): + \"\"\"Add Fourier terms for capturing seasonality.\"\"\" + t = np.arange(len(df)) + for k in range(1, K + 1): + df[f'sin_{period}_{k}'] = np.sin(2 * np.pi * k * t / period) + df[f'cos_{period}_{k}'] = np.cos(2 * np.pi * k * t / period) + return df + + # Daily seasonality (period=24 for hourly data) + df = add_fourier_features(df, period=24, K=3) + + # Weekly seasonality (period=168 for hourly data) + df = add_fourier_features(df, period=168, K=2) + + +Event/Holiday Features +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import holidays + + # Get country holidays + us_holidays = holidays.US() + + # Add holiday indicators + df['is_holiday'] = df['date'].isin(us_holidays).astype(int) + + # Days to/from nearest holiday + holiday_dates = pd.Series(list(us_holidays.keys())) + df['days_to_holiday'] = df['date'].apply( + lambda x: (holiday_dates - x).abs().min().days + ) + + +Troubleshooting +--------------- + +Common Issues +~~~~~~~~~~~~~ + +**NaN values after feature engineering:** + - Check for insufficient historical data for lag/rolling features + - Use ``min_periods`` parameter + - Forward-fill or interpolate missing values + +**Memory errors:** + - Reduce number of features + - Use chunked processing + - Downcaste data types + +**Poor performance despite many features:** + - May be overfitting - try feature selection + - Some features may add noise + - Consider feature interactions + + +See Also +-------- + +- :doc:`data_preparation` - Data loading and preprocessing +- :doc:`models` - Model selection and configuration +- :doc:`training` - Training strategies +- :doc:`tricks` - Performance optimization tips diff --git a/docs/source/index.rst b/docs/source/index.rst index 1f621d9b..8e64c198 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,109 +1,362 @@ -.. Time-series-prediction documentation master file, created by - sphinx-quickstart on Tue Mar 8 13:01:43 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. Time-series-prediction documentation master file -TFTS Documentation +TFTS: TensorFlow Time Series ================================================== + .. raw:: html GitHub -TFTS (TensorFlow Time Series) supports state-of-the-art deep learning time series models for production, research and data competitions. Specifically, the package provides: +Welcome to TFTS (TensorFlow Time Series), a comprehensive Python library for state-of-the-art deep learning time series analysis. TFTS provides production-ready implementations of cutting-edge models for forecasting, classification, and anomaly detection tasks. + +.. image:: https://img.shields.io/badge/License-MIT-blue.svg + :target: https://opensource.org/licenses/MIT + :alt: License + +.. image:: https://badge.fury.io/py/tfts.svg + :target: https://pypi.python.org/pypi/tfts + :alt: PyPI Version + +.. image:: https://pepy.tech/badge/tfts/month + :target: https://pepy.tech/project/tfts + :alt: Downloads + +Why TFTS? +--------- + +TFTS simplifies time series modeling by providing: + +**State-of-the-Art Models** + Access to 20+ pre-implemented deep learning architectures including Transformers, BERT, Informer, Autoformer, and more. All models are optimized for time series tasks and ready for production use. + +**Unified API** + Consistent interface across all models through ``AutoModel`` and ``AutoConfig``. Switch between architectures with a single line of code while maintaining the same workflow. + +**Production Ready** + Built on TensorFlow 2.x with native support for distributed training, mixed precision, TPUs, and TensorFlow Serving. Export models to SavedModel or ONNX formats for deployment. + +**Flexible Architecture** + Modular design allows easy customization of model components, training loops, and data pipelines. Integrate TFTS models as backbones in your custom architectures. -* Flexible and powerful modular design for time series task -* Easy-to-use advanced SOTA deep learning models -* Allow training on CPUs, single and multiple GPUs, TPU +**Comprehensive Tasks** + Support for forecasting (univariate/multivariate), classification, anomaly detection, and segmentation tasks with task-specific model heads. + + +Key Features +------------ + +📈 **Multiple Tasks** + - Single/multi-step forecasting + - Probabilistic forecasting with uncertainty quantification + - Time series classification + - Anomaly detection + - Change point detection and segmentation + +🚀 **20+ Models** + - Classic: RNN, LSTM, GRU, Seq2Seq + - CNN-based: TCN, WaveNet, UNet + - Transformer-based: Transformer, BERT, Informer, Autoformer, PatchTST, iTransformer + - Specialized: N-BEATS, DLinear, TFT, DeepAR, RWKV, Diffusion + +⚡ **Performance** + - Multi-GPU training with ``tf.distribute`` + - TPU support for large-scale training + - Mixed precision training (FP16/BF16) + - TensorFlow data pipelines for efficient I/O + +🔧 **Flexible** + - Modular layer design for custom architectures + - Feature engineering utilities (lag features, rolling statistics, datetime features) + - Custom training loops and callbacks + - Integration with Keras ecosystem Quick Start ------------------ +----------- + +Installation +~~~~~~~~~~~~ -1. Requirements -~~~~~~~~~~~~~~~~~~ +Install TFTS using pip: -To get started with `tfts`, follow the steps below: +.. code-block:: bash -* Python 3.7 or higher -* `TensorFlow 2.x `_ installation instructions + pip install tfts +Requirements: + - Python >= 3.7 + - TensorFlow >= 2.4 -2. Installation -~~~~~~~~~~~~~~~~~~ -Now you are ready, proceed with +For development installation: -.. code-block:: shell +.. code-block:: bash - $ pip install tfts + git clone https://github.com/LongxingTan/Time-series-prediction.git + cd Time-series-prediction + pip install -e . -2. Learn more -~~~~~~~~~~~~~~~~~~ -Visit :ref:`Quick start ` to learn more about the package. +Basic Usage +~~~~~~~~~~~ +Here's a minimal example to get started with TFTS: + +.. code-block:: python -Tutorials + import tensorflow as tf + import tfts + from tfts import AutoConfig, AutoModel, KerasTrainer + + # 1. Load sample data + train_length = 24 + predict_length = 8 + train, valid = tfts.get_data('sine', train_length, predict_length) + + # 2. Choose and configure a model + config = AutoConfig.for_model('transformer') + model = AutoModel.from_config(config, predict_sequence_length=predict_length) + + # 3. Train the model + trainer = KerasTrainer(model) + trainer.train(train, valid, epochs=10) + + # 4. Make predictions + predictions = trainer.predict(valid[0]) + + +Supported Models +---------------- + +TFTS provides implementations of state-of-the-art time series models: + +**Transformer-Based Models** + - ``transformer``: Standard Transformer architecture adapted for time series + - ``bert``: BERT-style bidirectional encoder for representation learning + - ``informer``: ProbSparse self-attention for long sequence forecasting + - ``autoformer``: Auto-correlation mechanism for decomposition + - ``tft``: Temporal Fusion Transformer with interpretable attention + - ``patch_tst``: Patch-based Transformer for efficient training + - ``itransformer``: Inverted Transformer treating variates as tokens + +**RNN-Based Models** + - ``rnn``: Configurable RNN with LSTM/GRU cells + - ``seq2seq``: Encoder-decoder architecture with attention + - ``deep_ar``: Probabilistic forecasting with autoregressive RNN + +**CNN-Based Models** + - ``tcn``: Temporal Convolutional Network with dilated convolutions + - ``wavenet``: WaveNet-style architecture with causal convolutions + - ``unet``: U-Net style encoder-decoder for sequence-to-sequence + +**Specialized Models** + - ``nbeats``: Neural Basis Expansion Analysis for interpretable forecasting + - ``dlinear``: Simple linear model with decomposition + - ``rwkv``: RWKV architecture with linear attention + - ``diffusion``: Diffusion-based probabilistic forecasting + - ``tide``: Time-series Dense Encoder + - ``gpt``: GPT-style autoregressive model + + +User Guide ---------- -The :ref:`Tutorials ` section provides guidance on -- how to :ref:`prepare datasets` for single-value, multi-value, single-step, and multi-steps prediction -- how to :ref:`use models` and implement new ones. +.. toctree:: + :maxdepth: 2 + :caption: Getting Started + + installation + tutorials -Models ---------- +.. toctree:: + :maxdepth: 2 + :caption: User Guide -1. Design a Custom Model with TFTS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + models + training +.. toctree:: + :maxdepth: 2 + :caption: Advanced Topics + + feature_engineering + tricks + + +.. toctree:: + :maxdepth: 2 + :caption: API Reference + + api + + +.. toctree:: + :maxdepth: 1 + :caption: Additional Information + + examples + faq + + +Examples +-------- + +Real-World Applications +~~~~~~~~~~~~~~~~~~~~~~~ + +TFTS has been successfully used in production and competitions: + +**Competition Wins** + - 🥉 **3rd Place** - Baidu KDD Cup 2022 (`Code `_) + - 🎯 **4th Place** - Alibaba Tianchi ENSO Prediction (`Code `_) + +**Industry Use Cases** + - Energy demand forecasting + - Financial time series prediction + - IoT sensor data analysis + - Weather and climate modeling + - Traffic flow prediction + + +Advanced Examples +~~~~~~~~~~~~~~~~~ + +**Multi-variate Forecasting** + .. code-block:: python import tensorflow as tf from tfts import AutoConfig, AutoModel - def build_model(use_model, input_shape): - inputs = tf.keras.layers.Input(input_shape) - config = AutoConfig.for_model(use_model) + # Configure for multi-variate input + config = AutoConfig.for_model('informer') + config.num_features = 10 # 10 input features - backbone = AutoModel.from_config(config) - outputs = backbone(inputs) - model = tf.keras.Model(inputs, outputs=outputs) + model = AutoModel.from_config(config, predict_sequence_length=24) - optimizer = tf.keras.optimizers.Adam(0.003) - loss_fn = tf.keras.losses.MeanSquaredError() + # Input: (batch, sequence_length, num_features) + x = tf.random.normal([32, 96, 10]) + predictions = model(x) # Output: (32, 24, 1) - model.compile(optimizer, loss_fn) - return model - model = build_model(use_model="bert", input_shape=(24, 3)) - model.summary() +**Probabilistic Forecasting** +.. code-block:: python -2. More highlights -~~~~~~~~~~~~~~~~~~~~~~~~ + from tfts import AutoConfig, AutoModel -The tfts library supports the SOTA deep learning models for time series. + # Use model with uncertainty quantification + config = AutoConfig.for_model('deep_ar') + model = AutoModel.from_config(config, predict_sequence_length=24) -- `TFTS BERT model `_ — 3rd place in `Baidu KDD Cup 2022 `_ -- `TFTS Seq2Seq model `_ — 4th place in `Alibaba Tianchi ENSO prediction `_ -- :ref:`Learn more models ` + # Get probabilistic predictions + predictions = model(x) # Returns distribution parameters -Tricks ----------- -Visit :ref:`Tricks ` if you want to know more tricks to improve the prediction performance. +**Custom Feature Engineering** + +.. code-block:: python + + from tfts.data import TimeSeriesSequence + import pandas as pd + + # Configure feature engineering + feature_config = { + 'datetime': { + 'type': 'datetime', + 'features': ['hour', 'dayofweek', 'month'], + 'time_col': 'timestamp' + }, + 'lags': { + 'type': 'lag', + 'columns': 'target', + 'lags': [1, 2, 3, 7, 14] + }, + 'rolling': { + 'type': 'rolling', + 'columns': 'target', + 'windows': [7, 14], + 'functions': ['mean', 'std'] + } + } + + # Create data loader with automatic feature engineering + data_loader = TimeSeriesSequence( + data=df, + time_idx='timestamp', + target_column='target', + train_sequence_length=24, + predict_sequence_length=8, + feature_config=feature_config + ) + + +.. Performance Benchmarks +.. ---------------------- + +.. TFTS models have been evaluated on standard benchmarks: + +.. .. list-table:: +.. :header-rows: 1 +.. :widths: 20 20 20 20 20 + +.. * - Model +.. - ETTh1 (MSE) +.. - Weather (MAE) +.. - Traffic (MSE) +.. - Training Speed +.. * - Transformer +.. - 0.495 +.. - 0.245 +.. - 0.612 +.. - 1.0x +.. * - Informer +.. - 0.472 +.. - 0.231 +.. - 0.598 +.. - 1.2x +.. * - Autoformer +.. - 0.449 +.. - 0.217 +.. - 0.573 +.. - 1.1x +.. * - DLinear +.. - 0.458 +.. - 0.223 +.. - 0.587 +.. - 3.5x + +.. *Benchmarks run on single V100 GPU with batch size 32* + + +Community and Support +--------------------- + +**Getting Help** + - 📖 Read the `documentation `_ + - 💬 Ask questions in `GitHub Discussions `_ + - 🐛 Report bugs in `GitHub Issues `_ + +**Contributing** + We welcome contributions! See our `Contributing Guide `_ for details. + +**Stay Updated** + - ⭐ Star the `GitHub repository `_ + - 📰 Check the `changelog <./CHANGELOG.md>`_ for latest updates + - 🐦 Follow updates on social media Citation ------------- -If you find tfts project useful in your research, please consider cite: +-------- + +If you use TFTS in your research, please cite: -.. code-block:: text +.. code-block:: bibtex @misc{tfts2020, author = {Longxing Tan}, - title = {Time series prediction}, + title = {TFTS: TensorFlow Time Series}, year = {2020}, publisher = {GitHub}, journal = {GitHub repository}, @@ -111,13 +364,15 @@ If you find tfts project useful in your research, please consider cite: } -.. toctree:: - :titlesonly: - :hidden: - :maxdepth: 6 +License +------- - quick-start - tutorials - models - tricks - api +TFTS is released under the MIT License. See `LICENSE `_ for details. + + +Indices and Tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 00000000..31c81834 --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,432 @@ +Installation +================== + +.. _installation: + +.. currentmodule:: tfts + +This guide covers everything you need to install and set up TFTS (TensorFlow Time Series) for your environment. + + +Quick Installation +------------------ + +The fastest way to install TFTS is using pip: + +.. code-block:: bash + + pip install tfts + +This will install TFTS and its core dependencies. + + +Requirements +------------ + +System Requirements +~~~~~~~~~~~~~~~~~~~ + +**Minimum Requirements:** + - Python 3.7 or higher + - 4GB RAM (8GB+ recommended) + - 2GB disk space + +**Recommended:** + - Python 3.8+ + - 16GB RAM for training large models + - NVIDIA GPU with CUDA support (for GPU training) + - 5GB+ disk space (including datasets and model checkpoints) + +Dependencies +~~~~~~~~~~~~ + +TFTS requires the following core dependencies: + +**Required:** + - ``tensorflow >= 2.4.0`` + - ``numpy >= 1.19.0`` + - ``pandas >= 1.1.0`` + +**Optional but Recommended:** + - ``scikit-learn >= 0.24.0`` (for preprocessing) + - ``matplotlib >= 3.3.0`` (for visualization) + - ``seaborn >= 0.11.0`` (for advanced plotting) + + +Installation Methods +-------------------- + +From PyPI (Recommended) +~~~~~~~~~~~~~~~~~~~~~~~ + +Install the latest stable release from PyPI: + +.. code-block:: bash + + pip install tfts + + +To upgrade to the latest version: + +.. code-block:: bash + + pip install --upgrade tfts + + +From Source +~~~~~~~~~~~ + +For the latest development version, install from GitHub: + +.. code-block:: bash + + git clone https://github.com/LongxingTan/Time-series-prediction.git + cd Time-series-prediction + pip install -e . + +The ``-e`` flag installs in editable mode, allowing you to modify the source code. + + +For Development +~~~~~~~~~~~~~~~ + +If you plan to contribute or modify TFTS, install development dependencies: + +.. code-block:: bash + + git clone https://github.com/LongxingTan/Time-series-prediction.git + cd Time-series-prediction + pip install -e ".[dev]" + +This installs additional tools for testing, linting, and documentation. + + +Using Docker +~~~~~~~~~~~~ + +TFTS provides a Docker image with all dependencies pre-installed: + +**Build the Docker Image:** + +.. code-block:: bash + + docker build -f ./docker/Dockerfile -t tfts:latest . + +**Run the Container:** + +.. code-block:: bash + + docker run --rm -it \ + --init \ + --ipc=host \ + --network=host \ + --volume=$PWD:/app \ + --gpus all \ + tfts:latest /bin/bash + +**For CPU-only:** + +.. code-block:: bash + + docker run --rm -it \ + --init \ + --volume=$PWD:/app \ + tfts:latest /bin/bash + + +Environment-Specific Installation +---------------------------------- + +TensorFlow GPU Support +~~~~~~~~~~~~~~~~~~~~~~ + +For GPU acceleration, install TensorFlow with CUDA support: + +**CUDA 11.2+ (Recommended):** + +.. code-block:: bash + + pip install tensorflow[and-cuda] + +**Manual CUDA Installation:** + +1. Install CUDA Toolkit: https://developer.nvidia.com/cuda-downloads +2. Install cuDNN: https://developer.nvidia.com/cudnn +3. Install TensorFlow: + +.. code-block:: bash + + pip install tensorflow-gpu + pip install tfts + +**Verify GPU Setup:** + +.. code-block:: python + + import tensorflow as tf + print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU'))) + + +Apple Silicon (M1/M2/M3) +~~~~~~~~~~~~~~~~~~~~~~~~ + +For macOS with Apple Silicon: + +.. code-block:: bash + + # Install tensorflow-metal for GPU acceleration + pip install tensorflow-macos tensorflow-metal + pip install tfts + +**Verify Metal Support:** + +.. code-block:: python + + import tensorflow as tf + print(tf.config.list_physical_devices()) + + +TPU Support +~~~~~~~~~~~ + +For Google Cloud TPU: + +.. code-block:: bash + + pip install cloud-tpu-client + pip install tfts + +**TPU Runtime Configuration:** + +.. code-block:: python + + import tensorflow as tf + + resolver = tf.distribute.cluster_resolver.TPUClusterResolver() + tf.config.experimental_connect_to_cluster(resolver) + tf.tpu.experimental.initialize_tpu_system(resolver) + + +Conda Environment +~~~~~~~~~~~~~~~~~ + +Create an isolated conda environment for TFTS: + +.. code-block:: bash + + # Create environment + conda create -n tfts python=3.9 + conda activate tfts + + # Install dependencies + conda install tensorflow pandas numpy scikit-learn matplotlib + pip install tfts + + +Virtual Environment +~~~~~~~~~~~~~~~~~~~ + +Using Python's built-in venv: + +.. code-block:: bash + + # Create virtual environment + python -m venv tfts-env + + # Activate (Linux/Mac) + source tfts-env/bin/activate + + # Activate (Windows) + tfts-env\Scripts\activate + + # Install TFTS + pip install tfts + + +Troubleshooting +--------------- + +Common Installation Issues +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**ImportError: No module named 'tensorflow'** + +Solution: Install TensorFlow first: + +.. code-block:: bash + + pip install tensorflow>=2.4 + + +**CUDA version mismatch** + +Solution: Ensure CUDA and cuDNN versions match TensorFlow requirements: + +.. code-block:: bash + + # Check TensorFlow CUDA requirement + python -c "import tensorflow as tf; print(tf.sysconfig.get_build_info())" + + +**Memory errors during installation** + +Solution: Install with no-cache option: + +.. code-block:: bash + + pip install --no-cache-dir tfts + + +**Permission denied on Linux/Mac** + +Solution: Use user installation: + +.. code-block:: bash + + pip install --user tfts + + +Platform-Specific Issues +~~~~~~~~~~~~~~~~~~~~~~~~ + +**Windows:** + - Use Anaconda/Miniconda for easier dependency management + - Install Microsoft Visual C++ Redistributable if needed + - Consider using WSL2 for Linux compatibility + +**macOS:** + - Install Xcode Command Line Tools: ``xcode-select --install`` + - Use Homebrew for system dependencies: ``brew install python`` + +**Linux:** + - Install build essentials: ``sudo apt-get install build-essential`` + - For GPU: Install NVIDIA drivers and CUDA toolkit + + +Verification +------------ + +Verify Installation +~~~~~~~~~~~~~~~~~~~ + +Check that TFTS is installed correctly: + +.. code-block:: python + + import tfts + print(f"TFTS version: {tfts.__version__}") + + # Check available models + from tfts import AutoConfig + models = ['seq2seq', 'transformer', 'informer', 'autoformer'] + for model in models: + config = AutoConfig.for_model(model) + print(f"{model}: OK") + + +Run Test Suite +~~~~~~~~~~~~~~ + +Run the test suite to ensure everything works: + +.. code-block:: bash + + # Install test dependencies + pip install pytest pytest-cov + + # Run tests + pytest tests/ + + # Run with coverage + pytest tests/ --cov=tfts + + +Quick Start Test +~~~~~~~~~~~~~~~~ + +Run a quick training test: + +.. code-block:: python + + import tensorflow as tf + import tfts + from tfts import AutoConfig, AutoModel, KerasTrainer + + # Generate sample data + train, valid = tfts.get_data('sine', train_length=24, predict_length=8) + + # Create and train model + config = AutoConfig.for_model('seq2seq') + model = AutoModel.from_config(config, predict_sequence_length=8) + trainer = KerasTrainer(model) + trainer.train(train, valid, epochs=2) + + print("✅ Installation successful!") + + +Updating TFTS +------------- + +Stay Updated +~~~~~~~~~~~~ + +Keep TFTS up to date with the latest features and bug fixes: + +.. code-block:: bash + + # Check current version + pip show tfts + + # Update to latest version + pip install --upgrade tfts + + # Update to specific version + pip install --upgrade tfts==1.3.0 + + +Development Builds +~~~~~~~~~~~~~~~~~~ + +For bleeding-edge features, install from the development branch: + +.. code-block:: bash + + pip install git+https://github.com/LongxingTan/Time-series-prediction.git@master + + +Uninstallation +-------------- + +To remove TFTS: + +.. code-block:: bash + + pip uninstall tfts + +To completely remove including dependencies: + +.. code-block:: bash + + pip uninstall tfts tensorflow pandas numpy scikit-learn matplotlib + + +Next Steps +---------- + +Now that you have TFTS installed: + +1. **Quick Start:** Try the :doc:`quickstart` tutorial +2. **Learn the Basics:** Read :doc:`tutorials` +3. **Explore Models:** Check :doc:`models` documentation +4. **Prepare Data:** See :doc:`data_preparation` guide +5. **Train Models:** Follow :doc:`training` best practices + + +Getting Help +------------ + +If you encounter installation issues: + +- 📖 Check the `FAQ <./faq.html>`_ +- 💬 Ask in `GitHub Discussions `_ +- 🐛 Report bugs in `GitHub Issues `_ diff --git a/docs/source/models.rst b/docs/source/models.rst index 1704650f..e88c2243 100644 --- a/docs/source/models.rst +++ b/docs/source/models.rst @@ -5,39 +5,705 @@ Models .. currentmodule:: tfts -Some experiments of tfts in Kaggle Dataset +TFTS provides a comprehensive collection of state-of-the-art deep learning models for time series analysis. All models are accessible through a unified API and can be easily configured, trained, and deployed. -Models supported ------------------- +Model Overview +-------------- -You can use below models with ``AutoModel`` +Available Models +~~~~~~~~~~~~~~~~ -* RNN -* Seq2seq -* TCN -* WaveNet -* Bert -* Transformer -* DLinear -* NBeats -* AutoFormer -* Informer +TFTS supports 20+ model architectures, organized by category: + +.. list-table:: + :header-rows: 1 + :widths: 25 25 50 + + * - Model + - Type + - Best For + * - ``seq2seq`` + - RNN-based + - General purpose, interpretable baselines + * - ``rnn`` + - RNN-based + - Simple sequence modeling, quick prototyping + * - ``deep_ar`` + - RNN-based + - Probabilistic forecasting with uncertainty + * - ``tcn`` + - CNN-based + - Long-range dependencies, fast inference + * - ``wavenet`` + - CNN-based + - High-frequency signals, audio-like data + * - ``unet`` + - CNN-based + - Sequence-to-sequence with skip connections + * - ``transformer`` + - Transformer + - Complex patterns, long-term dependencies + * - ``bert`` + - Transformer + - Representation learning, pre-training + * - ``informer`` + - Transformer + - Very long sequences (1000+ steps) + * - ``autoformer`` + - Transformer + - Seasonal data, decomposition tasks + * - ``tft`` + - Transformer + - Interpretable attention, multiple inputs + * - ``patch_tst`` + - Transformer + - Efficient training on long sequences + * - ``itransformer`` + - Transformer + - Multivariate with variable relationships + * - ``nbeats`` + - Specialized + - Interpretable basis expansion + * - ``dlinear`` + - Specialized + - Simple, fast baseline with decomposition + * - ``rwkv`` + - Specialized + - Linear attention, efficient memory + * - ``diffusion`` + - Specialized + - Uncertainty quantification, generation + * - ``tide`` + - Specialized + - Dense encoder, simple architecture + * - ``gpt`` + - Specialized + - Autoregressive generation + + +Model Selection Guide +~~~~~~~~~~~~~~~~~~~~~ + +Choose the right model based on your requirements: + +**For Quick Prototyping:** + Start with ``rnn`` or ``dlinear`` for fast iteration and baseline performance. + +**For Long Sequences (>500 steps):** + Use ``informer``, ``patch_tst``, or ``autoformer`` with efficient attention mechanisms. + +**For Interpretability:** + Choose ``nbeats`` (interpretable basis), ``tft`` (attention visualization), or ``dlinear`` (linear decomposition). + +**For Probabilistic Forecasting:** + Use ``deep_ar`` or ``diffusion`` for uncertainty quantification. + +**For Multivariate Data:** + Consider ``itransformer`` (treats variables as tokens) or ``tft`` (handles multiple inputs). + +**For Computational Efficiency:** + Choose ``tcn`` (fast convolutions), ``dlinear`` (simple linear), or ``rwkv`` (linear attention). + + +Detailed Model Descriptions +---------------------------- + +RNN-Based Models +~~~~~~~~~~~~~~~~ + +Seq2seq (Sequence-to-Sequence) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Classic encoder-decoder architecture with attention mechanism. + +**Architecture:** + - Encoder: LSTM/GRU cells that compress input sequence into context vectors + - Decoder: LSTM/GRU cells that generate output sequence with attention + - Attention: Bahdanau or Luong-style attention mechanism + +**Best For:** + - General-purpose forecasting + - Interpretable attention weights + - Baseline comparisons + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('seq2seq') + config.rnn_type = 'lstm' # or 'gru' + config.rnn_size = 128 + config.attention_sizes = 128 + + model = AutoModel.from_config(config, predict_sequence_length=24) + +**Key Parameters:** + - ``rnn_type``: Choose between 'lstm' or 'gru' + - ``rnn_size``: Hidden state dimension (default: 128) + - ``attention_sizes``: Attention mechanism dimension + - ``num_layers``: Number of stacked RNN layers + +**References:** + - Sutskever et al. "Sequence to Sequence Learning with Neural Networks" (NeurIPS 2014) + - Bahdanau et al. "Neural Machine Translation by Jointly Learning to Align and Translate" (ICLR 2015) + + +DeepAR +^^^^^^ + +Probabilistic forecasting model using autoregressive RNN. + +**Architecture:** + - LSTM-based encoder + - Parametric output distributions (Gaussian, Student-t, etc.) + - Monte Carlo sampling for uncertainty quantification + +**Best For:** + - Probabilistic forecasting with confidence intervals + - Scenarios requiring uncertainty quantification + - Multiple related time series + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('deep_ar') + config.rnn_size = 64 + config.num_samples = 100 # MC samples + + model = AutoModel.from_config(config, predict_sequence_length=24) + +**References:** + - Salinas et al. "DeepAR: Probabilistic Forecasting with Autoregressive Recurrent Networks" (2020) + + +CNN-Based Models +~~~~~~~~~~~~~~~~ + +TCN (Temporal Convolutional Network) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Dilated causal convolutions for sequence modeling. + +**Architecture:** + - Dilated causal convolutions with exponentially increasing dilation rates + - Residual connections for gradient flow + - Weight normalization for stable training + +**Best For:** + - Long-range dependencies + - Fast parallel training and inference + - Real-time applications + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('tcn') + config.filters = 64 + config.kernel_size = 3 + config.num_blocks = 3 + config.dropout = 0.1 + + model = AutoModel.from_config(config, predict_sequence_length=24) + +**Key Parameters:** + - ``filters``: Number of convolutional filters + - ``kernel_size``: Convolution kernel size + - ``num_blocks``: Number of dilated residual blocks + - ``dilation_rate``: Exponential dilation factor + +**References:** + - Bai et al. "An Empirical Evaluation of Generic Convolutional and Recurrent Networks" (2018) + + +WaveNet +^^^^^^^ + +Deep generative model with dilated causal convolutions. + +**Architecture:** + - Stacked dilated causal convolutional layers + - Gated activation functions (tanh and sigmoid) + - Skip connections aggregating information from all layers + +**Best For:** + - High-frequency signals + - Audio and sensor data + - Complex temporal patterns + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('wavenet') + config.filters = 32 + config.num_blocks = 2 + config.num_layers = 10 + + model = AutoModel.from_config(config, predict_sequence_length=24) + +**References:** + - van den Oord et al. "WaveNet: A Generative Model for Raw Audio" (2016) + + +Transformer-Based Models +~~~~~~~~~~~~~~~~~~~~~~~~ + +Transformer +^^^^^^^^^^^ + +Standard Transformer architecture adapted for time series. + +**Architecture:** + - Multi-head self-attention mechanism + - Position-wise feed-forward networks + - Layer normalization and residual connections + - Positional encoding for temporal information + +**Best For:** + - General-purpose time series modeling + - Learning complex patterns + - Transfer learning applications + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('transformer') + config.hidden_size = 128 + config.num_layers = 3 + config.num_attention_heads = 8 + config.attention_probs_dropout_prob = 0.1 + + model = AutoModel.from_config(config, predict_sequence_length=24) + +**Key Parameters:** + - ``hidden_size``: Model dimension + - ``num_layers``: Number of encoder/decoder layers + - ``num_attention_heads``: Parallel attention heads + - ``ffn_intermediate_size``: Feed-forward network hidden size + +**References:** + - Vaswani et al. "Attention Is All You Need" (NeurIPS 2017) + + +Informer +^^^^^^^^ + +Efficient Transformer for long sequence time series forecasting. + +**Architecture:** + - ProbSparse self-attention: O(L log L) complexity vs O(L²) + - Self-attention distilling for reduced memory + - Generative decoder for one-forward prediction + +**Best For:** + - Very long sequences (1000+ time steps) + - Memory-constrained environments + - Long-term forecasting (LSTF) tasks + +**Example:** .. code-block:: python - config = AutoConfig.for_model("seq2seq") - model = AutoModel.from_config(config, predict_sequence_length=predict_sequence_length) + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('informer') + config.hidden_size = 256 + config.num_layers = 3 + config.num_attention_heads = 8 + config.factor = 5 # ProbSparse factor + + model = AutoModel.from_config(config, predict_sequence_length=96) + +**Key Parameters:** + - ``factor``: Sampling factor for ProbSparse attention (higher = more efficient) + - ``distil``: Enable attention distilling + +**References:** + - Zhou et al. "Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting" (AAAI 2021) -Add a custom head for tfts model +Autoformer +^^^^^^^^^^ + +Transformer with Auto-Correlation mechanism and decomposition. + +**Architecture:** + - Auto-correlation replaces self-attention for periodic patterns + - Series decomposition (trend + seasonal) at each layer + - Aggregates periodic components automatically + +**Best For:** + - Seasonal time series + - Data with clear periodic patterns + - Long-term forecasting + +**Example:** .. code-block:: python - config = AutoConfig.for_model("seq2seq") - model = AutoModel.from_config(config, predict_sequence_length=predict_sequence_length) - model.project = tf.keras.Sequential( - layers=[], - trainable=True, - name=None - ) + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('autoformer') + config.hidden_size = 128 + config.num_layers = 2 + config.moving_avg = 25 # Window for decomposition + + model = AutoModel.from_config(config, predict_sequence_length=96) + +**References:** + - Wu et al. "Autoformer: Decomposition Transformers with Auto-Correlation" (NeurIPS 2021) + + +Temporal Fusion Transformer (TFT) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Attention-based model with interpretable multi-horizon forecasting. + +**Architecture:** + - Variable selection network for feature importance + - LSTM for local processing + - Multi-head attention for temporal relationships + - Quantile outputs for uncertainty + +**Best For:** + - Multiple input features (static + dynamic) + - Interpretable attention weights + - Multi-horizon probabilistic forecasting + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('tft') + config.hidden_size = 160 + config.num_attention_heads = 4 + + model = AutoModel.from_config(config, predict_sequence_length=24) + +**References:** + - Lim et al. "Temporal Fusion Transformers for Interpretable Multi-horizon Time Series Forecasting" (2021) + + +PatchTST +^^^^^^^^ + +Patch-based Transformer for efficient time series modeling. + +**Architecture:** + - Divides time series into patches (sub-sequences) + - Treats patches as tokens for Transformer + - Channel independence for multivariate modeling + +**Best For:** + - Long sequences with efficient training + - Multivariate time series + - Transfer learning across datasets + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('patch_tst') + config.patch_size = 16 # Patch length + config.hidden_size = 128 + config.num_layers = 3 + + model = AutoModel.from_config(config, predict_sequence_length=96) + +**References:** + - Nie et al. "A Time Series is Worth 64 Words: Long-term Forecasting with Transformers" (ICLR 2023) + + +iTransformer +^^^^^^^^^^^^ + +Inverted Transformer treating variates as tokens. + +**Architecture:** + - Inverts the role of time and variables + - Each variable becomes a token + - Learns relationships between variables explicitly + +**Best For:** + - Multivariate forecasting with variable interactions + - Learning cross-variate dependencies + - High-dimensional time series + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('itransformer') + config.hidden_size = 128 + config.num_layers = 3 + + model = AutoModel.from_config(config, predict_sequence_length=96) + +**References:** + - Liu et al. "iTransformer: Inverted Transformers Are Effective for Time Series Forecasting" (2023) + + +Specialized Models +~~~~~~~~~~~~~~~~~~ + +N-BEATS +^^^^^^^ + +Neural Basis Expansion Analysis for interpretable forecasting. + +**Architecture:** + - Stack of fully-connected blocks + - Each block produces basis expansion coefficients + - Interpretable (trend + seasonality) or generic versions + - Doubly residual stacking for hierarchical patterns + +**Best For:** + - Interpretable univariate forecasting + - M4 competition-style problems + - When basis expansion is meaningful + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('nbeats') + config.num_blocks = 3 + config.stack_types = ['trend', 'seasonality'] + config.num_layers_per_block = 4 + + model = AutoModel.from_config(config, predict_sequence_length=24) + +**References:** + - Oreshkin et al. "N-BEATS: Neural Basis Expansion Analysis for Interpretable Time Series Forecasting" (ICLR 2020) + + +DLinear +^^^^^^^ + +Simple linear model with seasonal-trend decomposition. + +**Architecture:** + - Decomposes series into trend and seasonal components + - Applies separate linear layers to each component + - Extremely simple yet effective baseline + +**Best For:** + - Simple baselines + - Fast training and inference + - When complexity isn't justified + +**Example:** + +.. code-block:: python + + from tfts import AutoConfig, AutoModel + + config = AutoConfig.for_model('dlinear') + config.moving_avg = 25 # Decomposition window + config.channels = 1 # Number of features + + model = AutoModel.from_config(config, predict_sequence_length=96) + +**References:** + - Zeng et al. "Are Transformers Effective for Time Series Forecasting?" (AAAI 2023) + + +Model Configuration +------------------- + +Using AutoConfig +~~~~~~~~~~~~~~~~ + +All models can be configured using ``AutoConfig``: + +.. code-block:: python + + from tfts import AutoConfig + + # Load default configuration + config = AutoConfig.for_model('transformer') + + # View configuration + print(config) + + # Modify configuration + config.hidden_size = 256 + config.num_layers = 4 + config.dropout = 0.2 + + # Save configuration + config.save_pretrained('./my_config') + + # Load configuration + config = AutoConfig.from_pretrained('./my_config') + + +Common Configuration Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most models share these common parameters: + +**Architecture:** + - ``hidden_size``: Model dimension (default: 128) + - ``num_layers``: Number of layers (default: 2-4) + - ``dropout``: Dropout rate (default: 0.1) + +**Training:** + - ``learning_rate``: Initial learning rate + - ``batch_size``: Training batch size + - ``epochs``: Number of training epochs + +**Input/Output:** + - ``train_sequence_length``: Input sequence length + - ``predict_sequence_length``: Output forecast horizon + - ``num_features``: Number of input features + + +Model Comparison +---------------- + +Performance Comparison +~~~~~~~~~~~~~~~~~~~~~~ + +Typical performance characteristics on standard benchmarks: + +.. list-table:: + :header-rows: 1 + :widths: 20 20 20 20 20 + + * - Model + - Accuracy + - Speed + - Memory + - Interpretability + * - RNN/Seq2seq + - ⭐⭐⭐ + - ⭐⭐⭐⭐ + - ⭐⭐⭐⭐ + - ⭐⭐⭐ + * - TCN/WaveNet + - ⭐⭐⭐⭐ + - ⭐⭐⭐⭐⭐ + - ⭐⭐⭐⭐ + - ⭐⭐ + * - Transformer + - ⭐⭐⭐⭐ + - ⭐⭐⭐ + - ⭐⭐⭐ + - ⭐⭐⭐ + * - Informer + - ⭐⭐⭐⭐⭐ + - ⭐⭐⭐⭐ + - ⭐⭐⭐⭐ + - ⭐⭐ + * - Autoformer + - ⭐⭐⭐⭐⭐ + - ⭐⭐⭐ + - ⭐⭐⭐ + - ⭐⭐⭐ + * - N-BEATS + - ⭐⭐⭐⭐ + - ⭐⭐⭐⭐ + - ⭐⭐⭐⭐ + - ⭐⭐⭐⭐⭐ + * - DLinear + - ⭐⭐⭐ + - ⭐⭐⭐⭐⭐ + - ⭐⭐⭐⭐⭐ + - ⭐⭐⭐⭐ + + +Computational Complexity +~~~~~~~~~~~~~~~~~~~~~~~~ + +Time and space complexity for different models: + +.. list-table:: + :header-rows: 1 + :widths: 30 35 35 + + * - Model + - Time Complexity + - Space Complexity + * - RNN/LSTM + - O(L) + - O(H) + * - TCN + - O(L log L) + - O(H) + * - Transformer + - O(L²) + - O(L²) + * - Informer + - O(L log L) + - O(L log L) + * - DLinear + - O(L) + - O(1) + +*L = sequence length, H = hidden size* + + +Custom Models +------------- + +Creating Custom Architectures +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can create custom models by combining TFTS components: + +.. code-block:: python + + import tensorflow as tf + from tfts import AutoConfig, AutoModel + from tfts.layers import Attention, FeedForwardNetwork + + class CustomModel(tf.keras.Model): + def __init__(self, predict_sequence_length): + super().__init__() + # Use TFTS backbone + config = AutoConfig.for_model('transformer') + self.backbone = AutoModel.from_config( + config, + predict_sequence_length=predict_sequence_length + ) + + # Add custom head + self.custom_head = tf.keras.Sequential([ + tf.keras.layers.Dense(128, activation='relu'), + tf.keras.layers.Dense(1) + ]) + + def call(self, inputs): + features = self.backbone(inputs) + return self.custom_head(features) + + +See Also +-------- + +- :doc:`tutorials` - Step-by-step model training guides +- :doc:`api` - Complete API reference +- :doc:`tricks` - Tips for improving model performance diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst deleted file mode 100755 index ed51ed3b..00000000 --- a/docs/source/quick-start.rst +++ /dev/null @@ -1,137 +0,0 @@ -Quick start -=============== - -.. _quick-start: - -.. _installation: - -1. Installation --------------------- - -Install `tfts `_, follow the installation instructions first - -* Python 3.7+ -* `TensorFlow 2.x `_ installation instructions - -Now you are ready, proceed with - -.. code-block:: shell - - $ pip install tfts - - -You can run it in docker, download the Dockerfile to host server - -.. code-block:: shell - - $ docker build -f ./Dockerfile -t "custom image name" . - $ docker run --rm -it --init --ipc=host --network=host --volume=$PWD:/app -e NVIDIA_VISIBLE_DEVICES=0 "custom image name" /bin/bash - -.. _usage: - -2. Basic Usage ------------------------ - -.. currentmodule:: tfts - -The general setup for training and testing a model is - -#. Build time series ``3D training dataset`` and valid dataset. The shape of input and label are (examples, train_sequence_length, features) and (examples, predict_sequence_length, 1) -#. Instantiate a model using the ``AutoModel`` method -#. Create a ``Trainer()`` or ``KerasTrainer()`` object. Define the optimizer and loss function in trainer -#. Train the model on the training dataset and check if it has converged with acceptable accuracy -#. Tune the hyper-parameters of the model and training, manually or refer to `tuning example `_ -#. Load the model from the model checkpoint and apply it to new data - - -.. code-block:: python - - import tensorflow as tf - import tfts - from tfts import AutoConfig, AutoModel, KerasTrainer - - train_length = 36 - predict_sequence_length = 12 - train, valid = tfts.get_data('sine', train_length, predict_sequence_length) - - # build model: 'seq2seq', 'wavenet', 'transformer', 'rnn', 'tcn', 'bert' - model_name_or_path = 'seq2seq' - config = AutoConfig.for_model(model_name_or_path) - model = AutoModel.from_config(config, predict_sequence_length=predict_sequence_length) - - # train - opt = tf.keras.optimizers.Adam(0.001) - loss_fn = tf.keras.losses.MeanSquaredError() - trainer = KerasTrainer(model, loss_fn=loss_fn, optimizer=opt) - trainer.train(train, valid, epochs=30, batch_size=32) - - # test - trainer.predict(valid[0]) - - -3. Train your first model ------------------------------- - -3.1 Prepare the data -~~~~~~~~~~~~~~~~~~~~~~~~ -The tfts could accept any time series data of 3D data format as model input: ``(num_examples, train_sequence_length, num_features)``, -and the model supported by tfts outputs 3D data as model output: ``(num_examples, predict_sequence_length, num_outputs)`` - -Before training, ensure your raw data is preprocessed into a 3D format with the shape ``(batch_size, train_steps, features)``. Perform any necessary data cleaning, normalization, or transformation steps to ensure the data is ready for training. - - -3.2 Train the Model -~~~~~~~~~~~~~~~~~~~~~~~~~~ -When training the model, use appropriate loss functions, optimizers, and hyperparameters to achieve the best results. - -Run with strategy to support multi-gpu or tpu training - -.. code-block:: python - - from tfts import KerasTrainer - - config = AutoConfig.for_model(model_name_or_path) - model = AutoModel.from_config(config, predict_sequence_length=predict_sequence_length) - optimizer = { - 'class_name': 'adam', - 'config': {'learning_rate': 0.0005} - } - - strategy = tf.distribute.MirroredStrategy() - trainer = KerasTrainer(model, strategy=strategy) - trainer.train(train_gen, valid_gen, optimizer=optimizer, epochs=30) - -Run with Learning rate scheduler - -.. code-block:: python - - opt = tf.keras.optimizers.Adam(0.001) - loss_fn = tf.keras.losses.MeanSquaredError() - lr_scheduler = tf.keras.optimizers.schedules.CosineDecay( - initial_learning_rate=0.001, - decay_steps=1000, - ) - trainer = KerasTrainer(model) - trainer.train(train_dataset, valid_dataset, optimizer=opt, loss_fn=loss_fn, lr_scheduler=lr_scheduler) - -Run with pretrained weights - -.. code-block:: python - - model = AutoModel.from_config(config, predict_sequence_length=predict_sequence_length) - model.save_pretrained("tfts-model") - - model = AutoModel.from_pretrained("tfts-model") - - -3.3 Save and load the model -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -3.4 Serve the model -~~~~~~~~~~~~~~~~~~~~~~~ -Once the model is trained and evaluated, deploy it for inference. Ensure the model is saved in a format compatible with your serving environment (e.g., TensorFlow SavedModel, ONNX, etc.). Set up an API or service to handle incoming requests, preprocess input data, and return predictions in real-time. - -Save the model into protobuf file - -.. currentmodule:: tfts diff --git a/examples/README.md b/examples/README.md index 2ed6f7d9..a55fabb1 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,9 @@ Dive deeper with these notebooks: - [single step prediction](https://nbviewer.org/github/LongxingTan/Time-series-prediction/blob/master/examples/notebooks/single_step_weather_prediction.ipynb): A guided example on predicting the next time point in weather data. - [multi steps prediction](https://nbviewer.org/github/LongxingTan/Time-series-prediction/blob/master/examples/notebooks/multi_steps_sales_prediction.ipynb): Learn how to forecast multiple future time points in a sales dataset. +## 📊 Benchmark +- [Kaggle - Forecasting Sticker Sales](https://www.kaggle.com/competitions/playground-series-s5e1) + ## 🏆 More examples Check out these advanced examples and competition-winning implementations: diff --git a/examples/benchmarks/CMI_detect_sleep_states/README.md b/examples/benchmarks/CMI_detect_sleep_states/README.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/benchmarks/CMI_detect_sleep_states/conf.yaml b/examples/benchmarks/CMI_detect_sleep_states/conf.yaml new file mode 100644 index 00000000..e69de29b diff --git a/examples/benchmarks/CMI_detect_sleep_states/dataset.py b/examples/benchmarks/CMI_detect_sleep_states/dataset.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/benchmarks/forecasting_sticker_sales/README.md b/examples/benchmarks/forecasting_sticker_sales/README.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/benchmarks/forecasting_sticker_sales/conf.yaml b/examples/benchmarks/forecasting_sticker_sales/conf.yaml new file mode 100644 index 00000000..883f3cee --- /dev/null +++ b/examples/benchmarks/forecasting_sticker_sales/conf.yaml @@ -0,0 +1,21 @@ +seed: 315 + +data: + data_path: "data.csv" + target_column: "target" + freq: 'h' + +model: + name: bert + predict_sequence_length: 32 + n_layers: 2 + hidden_size: 128 + n_features: 10 + n_output: 1 + + +training: + batch_size: 128 + epochs: 30 + learning_rate: 0.001 + loss: "MSE" diff --git a/examples/benchmarks/forecasting_sticker_sales/dataset.py b/examples/benchmarks/forecasting_sticker_sales/dataset.py new file mode 100644 index 00000000..daf41971 --- /dev/null +++ b/examples/benchmarks/forecasting_sticker_sales/dataset.py @@ -0,0 +1,215 @@ +import warnings + +from joblib import Parallel, delayed +import numpy as np +from omegaconf import OmegaConf +import pandas as pd +import requests +from sklearn.preprocessing import StandardScaler +from tensorflow.keras.utils import Sequence + +warnings.filterwarnings("ignore") + + +# https://www.kaggle.com/code/cdeotte/transformer-starter-lb-0-052 +class TimeSeriesProcessor: + def __init__(self, use_internet=True, path="./"): + self.use_internet = use_internet + self.path = path + self.scales = {} + self.gdp_data = None + + def fetch_gdp(self, df): + """Unified GDP fetching logic.""" + alpha3_map = { + "Canada": "CAN", + "Finland": "FIN", + "Italy": "ITA", + "Kenya": "KEN", + "Norway": "NOR", + "Singapore": "SGP", + } + df["alpha3"] = df["country"].map(alpha3_map) + df["year"] = df["date"].dt.year + years = df["year"].unique() + + if self.use_internet: + gdp_dict = {} + for country, a3 in alpha3_map.items(): + try: + url = f"https://api.worldbank.org/v2/country/{a3}/indicator/NY.GDP.PCAP.CD?date={min(years)}:{max(years)}&format=json" # noqa: E501,E231 + res = requests.get(url).json()[1] + for entry in res: + gdp_dict[(a3, int(entry["date"]))] = entry["value"] + except Exception as e: + print(f"Error fetching GDP for {a3}: {e}") + self.gdp_data = gdp_dict + else: + # Assume local file exists + gdp_df = pd.read_csv(f"{self.path}gdp.csv").set_index("alpha3") + self.gdp_data = gdp_df.to_dict() + + return df + + def process_features(self, df, is_train=True): + """Calculates GDP ratios and store-based normalization.""" + df = df.copy() + df["date"] = pd.to_datetime(df["date"]) + + if self.gdp_data is None: + df = self.fetch_gdp(df) + + df["GDP"] = df.apply(lambda x: self.gdp_data.get((x["alpha3"], x["year"]), 1.0), axis=1) + + # 1. GDP Normalization + df["scaled_target"] = df["num_sold"] / df["GDP"] + + # 2. Store Ratio (calculate during train, apply during test) + if is_train: + self.store_ratios = df.groupby("store")["scaled_target"].mean().to_dict() + + df["scaled_target"] /= df["store"].map(self.store_ratios) + + # 3. Kenya Fudge Factor + df.loc[df["country"] == "Kenya", "scaled_target"] *= 1.15 + + return df + + def dataframe_to_tensor(self, df): + """ + Pivots the dataframe into a 3D tensor: (Products, Time, Series) + Series = Country + Store combinations. + """ + # Create a unique key for each Country/Store combination + df["series_key"] = df["country"] + "_" + df["store"] + + products = sorted(df["product"].unique()) + series_keys = sorted(df["series_key"].unique()) + + tensor_list = [] + for prod in products: + # Efficient pivoting instead of nested loops + subset = df[df["product"] == prod].pivot(index="date", columns="series_key", values="scaled_target") + + # Save scaling params per product + if prod not in self.scales: + self.scales[prod] = {"mean": subset.values.mean(), "std": subset.values.std()} + + # Standard Scale + scaled_val = (subset.values - self.scales[prod]["mean"]) / self.scales[prod]["std"] + tensor_list.append(scaled_val) + + return np.stack(tensor_list), products, series_keys + + def inverse_transform(self, pred, product_name, country, store, date): + """Reverses all transformations to get the original num_sold scale.""" + # 1. Reverse Standard Scale + val = (pred * self.scales[product_name]["std"]) + self.scales[product_name]["mean"] + + # 2. Reverse Kenya Factor + if country == "Kenya": + val /= 1.15 + + # 3. Reverse Store Ratio + val *= self.store_ratios[store] + + # 4. Reverse GDP + year = pd.to_datetime(date).year + # Note: You'd need a helper to get alpha3 from country + alpha3 = { + "Canada": "CAN", + "Finland": "FIN", + "Italy": "ITA", + "Kenya": "KEN", + "Norway": "NOR", + "Singapore": "SGP", + }[country] + val *= self.gdp_data.get((alpha3, year), 1.0) + + return val + + +class TimeSeriesDataset(Sequence): + def __init__( + self, + data, + mode="train", # "train" or "test" + product_idx=0, + train_sequence_length=1440, + predict_sequence_length=32, + batch_size=32, + ): + self.data = data[product_idx] # Shape: (Time, Series) + self.mode = mode + self.product_idx = product_idx + self.train_sequence_length = train_sequence_length + self.predict_sequence_length = predict_sequence_length + self.batch_size = batch_size + + nans = np.isnan(self.data).astype("float32") + self.combined_data = np.stack([np.nan_to_num(self.data), nans], axis=-1) + + def __len__(self): + return int(np.ceil(self.data.shape[1] / self.batch_size)) + + def __getitem__(self, idx): + if self.mode == "train": + return self._get_train_batch() + else: + return self._get_test_batch(idx) + + def _get_train_batch(self): + X = np.zeros((self.batch_size, self.train_sequence_length, 2), dtype="float32") + y = np.zeros((self.batch_size, self.predict_sequence_length), dtype="float32") + + for i in range(self.batch_size): + series_idx = np.random.randint(0, self.data.shape[1]) + start = np.random.randint(0, self.data.shape[0] - self.train_sequence_length - self.predict_sequence_length) + + X[i] = self.combined_data[start : start + self.train_sequence_length, series_idx, :] + y[i] = self.combined_data[ + start + self.train_sequence_length : start + self.train_sequence_length + self.predict_sequence_length, + series_idx, + 0, + ] + return X, y + + def _get_test_batch(self, idx): + """Returns the LAST train_len for each category for prediction.""" + start_series = idx * self.batch_size + end_series = min((idx + 1) * self.batch_size, self.data.shape[1]) + actual_bs = end_series - start_series + + X = np.zeros((actual_bs, self.train_sequence_length, 2), dtype="float32") + + for i, s_idx in enumerate(range(start_series, end_series)): + # Always take the very tail of the data + X[i] = self.combined_data[-self.train_sequence_length :, s_idx, :] + + return X + + +if __name__ == "__main__": + # 1. Process Data + processor = TimeSeriesProcessor(use_internet=True) + df_train = pd.read_csv("/kaggle/input/playground-series-s5e1/train.csv") + df_processed = processor.process_features(df_train, is_train=True) + tensor, product_names, series_names = processor.dataframe_to_tensor(df_processed) + + # 2. Create Train Dataset for Product 0 + train_gen = TimeSeriesDataset(tensor, mode="train", product_idx=0) + + # 3. Create Test Dataset (the last window for all series in Product 0) + test_gen = TimeSeriesDataset(tensor, mode="test", product_idx=0) + + # # 4. Predict + # predictions = model.predict(test_gen) # (Total Series, pred_len) + + # # 5. Reverse Scaling for a specific prediction + # raw_pred = processor.inverse_transform( + # pred=predictions[0, 0], + # product_name=product_names[0], + # country="Canada", + # store="KaggleMart", + # date="2026-01-01" + # ) diff --git a/examples/benchmarks/forecasting_sticker_sales/run.py b/examples/benchmarks/forecasting_sticker_sales/run.py new file mode 100644 index 00000000..2c2f3704 --- /dev/null +++ b/examples/benchmarks/forecasting_sticker_sales/run.py @@ -0,0 +1,96 @@ +import argparse +import math +import random + +from dataset import DataReader, TrainDataset +import numpy as np +from omegaconf import OmegaConf +import pandas as pd +import tensorflow as tf + +from tfts import AutoConfig, AutoModel, Pipeline, set_seed + + +def parse_args(): + parser = argparse.ArgumentParser(description="tfts forecasting") + parser.add_argument("--config_path", type=str, default="conf.yaml", help="Path to base config file") + parser.add_argument("--debug", type=bool, default=False, help="Enable debug mode") + parser.add_argument("--is_training", type=bool, default=True, help="Whether to train or predict") + parser.add_argument("--model_name", type=str, default=None, help="Model name, e.g., BERT, LSTM") + parser.add_argument("--batch_size", type=int, default=None, help="Batch size") + parser.add_argument("--epochs", type=int, default=None, help="Number of epochs") + + args = parser.parse_args() + return args + + +# def run_inference(product_idx): +# """Runs the recursive prediction for a specific product.""" +# # Ensure history has the 2nd channel (NaN indicator) +# # history_tensor shape: (5, 2557, 18) -> Needs expansion to (1, LEN, 18, 2) +# data = np.expand_dims(self.history_tensor, axis=-1) +# nans = np.isnan(data).astype('float32') +# data = np.concatenate([data, nans], axis=-1) + +# product_preds = np.zeros((18, self.PRED_LEN * self.STEPS)) +# bad_rows = [] + +# for jj in range(18): +# # Get last window of training data for this series +# # Shape: (1, LEN, 2) +# current_window = data[product_idx:product_idx+1, -self.LEN:, jj, :].copy() + +# if np.isnan(current_window[:, :, 0]).sum() == self.LEN: +# bad_rows.append(jj) +# continue + +# series_predictions = [] + +# for step in range(self.STEPS): +# # Predict next 32 days +# # Input shape: (1, LEN, 2) +# p2 = self.model(np.nan_to_num(current_window)) +# p2 = p2.numpy().reshape((1, self.PRED_LEN, 1)) + +# # Add dummy NaN indicator (0.0) to predictions for the next step +# p2_with_nan = np.concatenate([p2, np.zeros_like(p2)], axis=-1) +# series_predictions.append(p2_with_nan) + +# # Update window: Slide window forward +# # Remove oldest 32, append newest 32 +# current_window = np.concatenate([current_window[:, self.PRED_LEN:, :], p2_with_nan], axis=1) + +# # Combine all steps and remove the NaN indicator channel +# product_preds[jj, :] = np.concatenate([z[:, :, 0] for z in series_predictions], axis=1).flatten() + +# # Handle bad rows (series with no training data) +# if bad_rows: +# fill_val = np.nanmean(product_preds, axis=0) +# for r in bad_rows: +# product_preds[r, :] = fill_val + +# return product_preds + + +def main(): + args = parse_args() + cfg = OmegaConf.load(args.config_path) + + set_seed(cfg.seed) + + data_reader = DataReader() + train_df = data_reader.load_data("/kaggle/input/playground-series-s5e1/train.csv") + train_df = data_reader.add_features(train_df) + data_tensor = data_reader.reshape_to_tensor(train_df) + + train_dataset = TrainDataset( + data=data_tensor, product_idx=0, batch_size=64, train_sequence_length=1440, predict_sequence_length=32 + ) + + forecaster = Pipeline(cfg) + + forecaster.train(train_dataset=train_dataset) + + +if __name__ == "__main__": + main() diff --git a/examples/notebooks/single_step_stock_prediction.ipynb b/examples/notebooks/single_step_stock_prediction.ipynb new file mode 100644 index 00000000..ea5f96ab --- /dev/null +++ b/examples/notebooks/single_step_stock_prediction.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Financial stock prediction\n", + "- the data is from binance API\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from tfts import AutoConfig, AutoModel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_layers/test_graph_layer.py b/tests/test_layers/test_graph_layer.py new file mode 100644 index 00000000..357a88d9 --- /dev/null +++ b/tests/test_layers/test_graph_layer.py @@ -0,0 +1,101 @@ +import unittest + +import numpy as np +import tensorflow as tf + +from tfts.layers.graph_layer import GraphAttention, GraphConv + + +class GraphLayerTest(unittest.TestCase): + def test_graph_convolution_layer(self): + units = 32 + batch_size = 2 + num_nodes = 10 + input_dim = 5 + + # Test Dense Adjacency + layer = GraphConv(units, activation="relu") + + # Inputs: Features (B, N, F), Adjacency (B, N, N) + x = tf.random.normal((batch_size, num_nodes, input_dim)) + a = tf.random.uniform((batch_size, num_nodes, num_nodes)) + + y = layer((x, a)) + + # Output shape should be (B, N, Units) + self.assertEqual(y.shape, (batch_size, num_nodes, units)) + + # Test Config + config = layer.get_config() + self.assertEqual(config["units"], units) + self.assertEqual(config["use_bias"], True) + + def test_graph_convolution_sparse(self): + # Test with Sparse Tensor Adjacency (Single graph mode usually) + units = 16 + num_nodes = 50 + input_dim = 8 + + layer = GraphConv(units) + + # Features (1, N, F) - usually sparse matmul requires specific dimensions + # Here we test the mechanics of passing a SparseTensor + x = tf.random.normal((num_nodes, input_dim)) + + # Create a random sparse adjacency matrix + indices = [] + values = [] + for i in range(num_nodes): + indices.append([i, (i + 1) % num_nodes]) + values.append(1.0) + + a_sparse = tf.sparse.SparseTensor(indices=indices, values=values, dense_shape=[num_nodes, num_nodes]) + + # Note: The layer logic handles matmul. + # If the input x is rank 2 (N, F), output is (N, Units) + y = layer((x, a_sparse)) + self.assertEqual(y.shape, (num_nodes, units)) + + def test_graph_attention_layer_concat(self): + units = 8 + num_heads = 4 + batch_size = 2 + num_nodes = 10 + input_dim = 5 + + # Test head_reduction='concat' + layer = GraphAttention(units=units, num_heads=num_heads, head_reduction="concat", activation="relu") + + x = tf.random.normal((batch_size, num_nodes, input_dim)) + a = tf.random.uniform((batch_size, num_nodes, num_nodes)) + + y = layer((x, a), training=True) + + # Expected shape: (B, N, units * num_heads) + self.assertEqual(y.shape, (batch_size, num_nodes, units * num_heads)) + + config = layer.get_config() + self.assertEqual(config["num_heads"], num_heads) + self.assertEqual(config["head_reduction"], "concat") + + def test_graph_attention_layer_average(self): + units = 16 + num_heads = 2 + batch_size = 2 + num_nodes = 10 + input_dim = 5 + + # Test head_reduction='average' + layer = GraphAttention(units=units, num_heads=num_heads, head_reduction="average") + + x = tf.random.normal((batch_size, num_nodes, input_dim)) + a = tf.random.uniform((batch_size, num_nodes, num_nodes)) + + y = layer((x, a)) + + # Expected shape: (B, N, units) + self.assertEqual(y.shape, (batch_size, num_nodes, units)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_layers/test_moe_layer.py b/tests/test_layers/test_moe_layer.py index e69de29b..0b9708d4 100644 --- a/tests/test_layers/test_moe_layer.py +++ b/tests/test_layers/test_moe_layer.py @@ -0,0 +1,37 @@ +import unittest + +import tensorflow as tf + +from tfts.layers.moe_layer import SparseMoe + + +class TestMoELayer(tf.test.TestCase): + def test_moe_layer_output_shape(self): + """ + Tests if the output shape of the MoELayer is the same as the input shape. + """ + # Define layer parameters + hidden_size_val = 768 + num_experts_val = 8 + num_experts_per_tok_val = 2 + moe_intermediate_size_val = 3072 + shared_expert_intermediate_size_val = 3072 + norm_topk_prob_val = True + hidden_act_val = "silu" + + moe_layer = SparseMoe( + hidden_size=hidden_size_val, + num_experts=num_experts_val, + num_experts_per_tok=num_experts_per_tok_val, + moe_intermediate_size=moe_intermediate_size_val, + shared_expert_intermediate_size=shared_expert_intermediate_size_val, + norm_topk_prob=norm_topk_prob_val, + hidden_act=hidden_act_val, + ) + + input_tensor = tf.random.normal((1, 10, hidden_size_val), dtype=tf.float32) + output, router_logits = moe_layer(input_tensor) + + print("Input shape:", input_tensor.shape) + print("Output shape:", output.shape) + print("Router logits shape:", router_logits.shape) diff --git a/tests/test_layers/test_rwkv_layer.py b/tests/test_layers/test_rwkv_layer.py index e69de29b..2feeb3f5 100644 --- a/tests/test_layers/test_rwkv_layer.py +++ b/tests/test_layers/test_rwkv_layer.py @@ -0,0 +1,119 @@ +import unittest + +import tensorflow as tf + +from tfts.layers.rwkv_layer import ChannelMixing, TimeMixing +from tfts.models.rwkv import RWKVConfig + + +class TimeMixingTest(unittest.TestCase): + """Test cases for TimeMixing layer.""" + + def setUp(self): + self.config = RWKVConfig(hidden_size=64) + self.batch_size = 2 + self.seq_len = 10 + self.hidden_size = self.config.hidden_size + + def _get_initial_state(self): + """Helper to create the correct 4-part state for TimeMixing.""" + return [ + tf.zeros([self.batch_size, self.hidden_size]), # last_x + tf.zeros([self.batch_size, self.hidden_size]), # aa + tf.zeros([self.batch_size, self.hidden_size]), # bb + tf.fill([self.batch_size, self.hidden_size], -1e30), # pp + ] + + def test_initialization(self): + layer = TimeMixing(self.config) + self.assertEqual(layer.n_embd, self.config.hidden_size) + + def test_build(self): + layer = TimeMixing(self.config) + x = tf.random.normal([self.batch_size, self.seq_len, self.hidden_size]) + state = self._get_initial_state() + output, new_state = layer(x, state) + + self.assertTrue(layer.built) + self.assertIsNotNone(layer.time_mix_k) + + def test_forward_pass(self): + layer = TimeMixing(self.config) + x = tf.random.normal([self.batch_size, self.seq_len, self.hidden_size]) + state = self._get_initial_state() + output, new_state = layer(x, state) + + self.assertEqual(output.shape, (self.batch_size, self.seq_len, self.hidden_size)) + self.assertEqual(len(new_state), 4) + for s in new_state: + self.assertEqual(s.shape, (self.batch_size, self.hidden_size)) + + def test_state_update(self): + layer = TimeMixing(self.config) + x = tf.random.normal([self.batch_size, self.seq_len, self.hidden_size]) + initial_state = self._get_initial_state() + _, new_state = layer(x, initial_state) + + # Check that state changed (using index 1 'aa' or index 0 'last_x') + self.assertFalse(tf.reduce_all(tf.equal(initial_state[0], new_state[0]))) + self.assertFalse(tf.reduce_all(tf.equal(initial_state[1], new_state[1]))) + + def test_gradient_flow(self): + layer = TimeMixing(self.config) + x = tf.Variable(tf.random.normal([self.batch_size, self.seq_len, self.hidden_size])) + state = self._get_initial_state() + + with tf.GradientTape() as tape: + output, _ = layer(x, state) + loss = tf.reduce_mean(output) + + gradients = tape.gradient(loss, x) + self.assertIsNotNone(gradients) + + +class ChannelMixingTest(unittest.TestCase): + """Test cases for ChannelMixing layer.""" + + def setUp(self): + self.config = RWKVConfig(hidden_size=64) + self.batch_size = 2 + self.seq_len = 10 + self.hidden_size = self.config.hidden_size + + def test_forward_pass(self): + layer = ChannelMixing(self.config) + x = tf.random.normal([self.batch_size, self.seq_len, self.hidden_size]) + state = tf.zeros([self.batch_size, self.hidden_size]) + + output, new_state = layer(x, state) + + # Output is 3D + self.assertEqual(output.shape, (self.batch_size, self.seq_len, self.hidden_size)) + # State is 2D (the last vector of the sequence) + self.assertEqual(new_state.shape, (self.batch_size, self.hidden_size)) + + def test_state_output(self): + layer = ChannelMixing(self.config) + x = tf.random.normal([self.batch_size, self.seq_len, self.hidden_size]) + state = tf.zeros([self.batch_size, self.hidden_size]) + + _, new_state = layer(x, state) + + # In ChannelMixing, new_state should be the last timestep of input x + tf.debugging.assert_near(new_state, x[:, -1, :]) + + def test_gradient_flow(self): + layer = ChannelMixing(self.config) + x = tf.Variable(tf.random.normal([self.batch_size, self.seq_len, self.hidden_size])) + state = tf.zeros([self.batch_size, self.hidden_size]) + + with tf.GradientTape() as tape: + output, _ = layer(x, state) + loss = tf.reduce_mean(output) + + gradients = tape.gradient(loss, x) + self.assertIsNotNone(gradients) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_losses/test_loss.py b/tests/test_losses/test_loss.py new file mode 100644 index 00000000..5babdcec --- /dev/null +++ b/tests/test_losses/test_loss.py @@ -0,0 +1,166 @@ +"""Tests for loss functions.""" + +import unittest + +import numpy as np +import tensorflow as tf + +from tfts.losses.loss import MultiQuantileLoss + + +class MultiQuantileLossTest(unittest.TestCase): + """Test cases for MultiQuantileLoss.""" + + def test_initialization(self): + """Test loss initialization.""" + quantiles = [0.1, 0.5, 0.9] + loss = MultiQuantileLoss(quantiles=quantiles) + self.assertEqual(loss.quantiles, quantiles) + self.assertEqual(loss.name, "multi_quantile_loss") + + def test_loss_shape(self): + """Test that loss returns a scalar.""" + quantiles = [0.1, 0.5, 0.9] + loss = MultiQuantileLoss(quantiles=quantiles) + + # Create test data + batch_size = 4 + pred_len = 10 + num_labels = 1 + num_quantiles = len(quantiles) + + y_true = tf.random.normal([batch_size, pred_len, num_labels]) + y_pred = tf.random.normal([batch_size, pred_len, num_labels * num_quantiles]) + + # Compute loss + loss_value = loss(y_true, y_pred) + + # Check that loss is a scalar + self.assertEqual(loss_value.shape, ()) + self.assertTrue(tf.is_tensor(loss_value)) + + def test_perfect_prediction(self): + """Test that perfect predictions give low loss.""" + quantiles = [0.5] + loss = MultiQuantileLoss(quantiles=quantiles) + + # Create perfect prediction (median) + y_true = tf.constant([[[1.0], [2.0], [3.0]]]) + y_pred = tf.constant([[[1.0], [2.0], [3.0]]]) + + loss_value = loss(y_true, y_pred) + self.assertLess(loss_value.numpy(), 0.01) # Should be very close to 0 + + def test_multiple_quantiles(self): + """Test with multiple quantiles.""" + quantiles = [0.1, 0.5, 0.9] + loss = MultiQuantileLoss(quantiles=quantiles) + + batch_size = 2 + pred_len = 5 + num_labels = 1 + num_quantiles = len(quantiles) + + y_true = tf.random.normal([batch_size, pred_len, num_labels]) + y_pred = tf.random.normal([batch_size, pred_len, num_labels * num_quantiles]) + + loss_value = loss(y_true, y_pred) + + # Loss should be positive + self.assertGreater(loss_value.numpy(), 0) + + def test_quantile_properties(self): + """Test that quantile loss has correct properties.""" + quantiles = [0.9] # High quantile + loss = MultiQuantileLoss(quantiles=quantiles) + + # For q=0.9, overestimation should be penalized less than underestimation + y_true = tf.constant([[[5.0]]]) + + # Overestimate + y_pred_over = tf.constant([[[6.0]]]) + loss_over = loss(y_true, y_pred_over) + + # Underestimate + y_pred_under = tf.constant([[[4.0]]]) + loss_under = loss(y_true, y_pred_under) + + # For q=0.9, underestimation should be penalized more heavily + self.assertGreater(loss_under.numpy(), loss_over.numpy()) + + def test_multiple_labels(self): + """Test with multiple target labels.""" + quantiles = [0.5] + loss = MultiQuantileLoss(quantiles=quantiles) + + batch_size = 2 + pred_len = 5 + num_labels = 3 + num_quantiles = len(quantiles) + + y_true = tf.random.normal([batch_size, pred_len, num_labels]) + y_pred = tf.random.normal([batch_size, pred_len, num_labels * num_quantiles]) + + loss_value = loss(y_true, y_pred) + + # Loss should be computed correctly + self.assertGreater(loss_value.numpy(), 0) + self.assertEqual(loss_value.shape, ()) + + def test_batch_consistency(self): + """Test that loss is consistent across batch sizes.""" + quantiles = [0.5] + loss = MultiQuantileLoss(quantiles=quantiles) + + # Same prediction repeated + y_true_single = tf.constant([[[1.0], [2.0], [3.0]]]) + y_pred_single = tf.constant([[[1.5], [2.5], [3.5]]]) + + # Batch of 2 with same data + y_true_batch = tf.concat([y_true_single, y_true_single], axis=0) + y_pred_batch = tf.concat([y_pred_single, y_pred_single], axis=0) + + loss_single = loss(y_true_single, y_pred_single) + loss_batch = loss(y_true_batch, y_pred_batch) + + # Losses should be the same (mean over batch) + np.testing.assert_allclose(loss_single.numpy(), loss_batch.numpy(), rtol=1e-5) + + def test_symmetric_quantile(self): + """Test that median quantile (0.5) treats over/under prediction equally.""" + quantiles = [0.5] + loss = MultiQuantileLoss(quantiles=quantiles) + + y_true = tf.constant([[[5.0]]]) + + # Overestimate by 1 + y_pred_over = tf.constant([[[6.0]]]) + loss_over = loss(y_true, y_pred_over) + + # Underestimate by 1 + y_pred_under = tf.constant([[[4.0]]]) + loss_under = loss(y_true, y_pred_under) + + # For median (q=0.5), should be symmetric + np.testing.assert_allclose(loss_over.numpy(), loss_under.numpy(), rtol=1e-5) + + def test_gradient_flow(self): + """Test that gradients flow through the loss.""" + quantiles = [0.1, 0.5, 0.9] + loss_fn = MultiQuantileLoss(quantiles=quantiles) + + y_true = tf.constant([[[1.0], [2.0], [3.0]]]) + y_pred = tf.Variable([[[1.5, 2.0, 2.5], [2.5, 3.0, 3.5], [3.5, 4.0, 4.5]]]) + + with tf.GradientTape() as tape: + loss_value = loss_fn(y_true, y_pred) + + gradients = tape.gradient(loss_value, y_pred) + + # Gradients should exist and not be None + self.assertIsNotNone(gradients) + self.assertFalse(tf.reduce_all(tf.equal(gradients, 0))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_models/test_diffusion.py b/tests/test_models/test_diffusion.py new file mode 100644 index 00000000..9bd114c2 --- /dev/null +++ b/tests/test_models/test_diffusion.py @@ -0,0 +1,84 @@ +from typing import Any, Dict +import unittest + +import tensorflow as tf + +import tfts +from tfts import AutoConfig, AutoModel, KerasTrainer +from tfts.models.diffusion import Diffusion, DiffusionConfig, NoiseScheduler + + +class DiffusionTest(unittest.TestCase): + def test_config(self): + """Test configuration initialization.""" + config = DiffusionConfig( + hidden_size=64, + num_layers=2, + num_attention_heads=4, + num_diffusion_steps=100, + ) + self.assertEqual(config.hidden_size, 64) + self.assertEqual(config.num_layers, 2) + self.assertEqual(config.num_attention_heads, 4) + self.assertEqual(config.num_diffusion_steps, 100) + + def test_noise_scheduler(self): + """Test noise scheduler.""" + config = DiffusionConfig(num_diffusion_steps=10) + scheduler = NoiseScheduler(config) + + # Test shapes + self.assertEqual(scheduler.betas.shape[0], 10) + self.assertEqual(scheduler.alphas.shape[0], 10) + + # Test noise addition + x = tf.random.normal([2, 10, 3]) + t = tf.constant([0, 5], dtype=tf.int32) + noisy_x, noise = scheduler.add_noise(x, t) + + self.assertEqual(noisy_x.shape, x.shape) + self.assertEqual(noise.shape, x.shape) + + def test_model_output_shape(self): + """Test model output shape.""" + train_sequence_length = 14 + predict_sequence_length = 7 + config = DiffusionConfig(hidden_size=32, num_layers=1, num_diffusion_steps=10) + model = Diffusion(predict_sequence_length=predict_sequence_length, config=config) + + x = tf.random.normal([2, train_sequence_length, 3]) + y = model(x) + + # Check output shape + self.assertEqual(y.shape[0], 2) # batch size + self.assertEqual(y.shape[1], predict_sequence_length) + + def test_model_direct_instantiation(self): + """Test model direct instantiation.""" + config = DiffusionConfig(hidden_size=32, num_layers=1, num_diffusion_steps=10) + model = Diffusion(predict_sequence_length=8, config=config) + self.assertIsNotNone(model) + + # Test forward pass + x = tf.random.normal([2, 10, 3]) + y = model(x) + self.assertEqual(y.shape[0], 2) + self.assertEqual(y.shape[1], 8) + + # def test_train(self): + # """Test training loop.""" + # train, valid = tfts.get_data("sine", test_size=0.1) + # config = DiffusionConfig(hidden_size=32, num_layers=1, num_diffusion_steps=10) + # model = Diffusion(predict_sequence_length=8, config=config) + + # # Build the model + # model.build_model(train[0][:5]) + # model.compile(optimizer=tf.keras.optimizers.Adam(0.003), loss="mse") + # model.fit(train[0], train[1], validation_data=valid, epochs=1, verbose=0) + + # y_test = model.predict(valid[0]) + # self.assertEqual(y_test.shape[0], valid[1].shape[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_models/test_gpt.py b/tests/test_models/test_gpt.py index e69de29b..a38a0cae 100644 --- a/tests/test_models/test_gpt.py +++ b/tests/test_models/test_gpt.py @@ -0,0 +1,53 @@ +from typing import Any, Dict +import unittest + +import tensorflow as tf + +import tfts +from tfts import AutoConfig, AutoModel, KerasTrainer +from tfts.models.gpt import GPT, GPTConfig + + +class GPTTest(unittest.TestCase): + def test_config(self): + """Test configuration initialization.""" + config = GPTConfig( + hidden_size=64, + num_layers=2, + num_attention_heads=4, + max_position_embeddings=256, + ) + self.assertEqual(config.hidden_size, 64) + self.assertEqual(config.num_layers, 2) + self.assertEqual(config.num_attention_heads, 4) + self.assertEqual(config.max_position_embeddings, 256) + + def test_model_output_shape(self): + """Test model output shape.""" + train_sequence_length = 14 + predict_sequence_length = 7 + config = GPTConfig(hidden_size=32, num_layers=1) + model = GPT(predict_sequence_length=predict_sequence_length, config=config) + + x = tf.random.normal([2, train_sequence_length, 3]) + y = model(x) + + # Check output shape + self.assertEqual(y.shape[0], 2) # batch size + self.assertEqual(y.shape[1], predict_sequence_length) + + def test_model_direct_instantiation(self): + """Test model direct instantiation.""" + config = GPTConfig(hidden_size=32, num_layers=1) + model = GPT(predict_sequence_length=8, config=config) + self.assertIsNotNone(model) + + # Test forward pass + x = tf.random.normal([2, 10, 3]) + y = model(x) + self.assertEqual(y.shape[0], 2) + self.assertEqual(y.shape[1], 8) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_models/test_itransformer.py b/tests/test_models/test_itransformer.py new file mode 100644 index 00000000..617fa34f --- /dev/null +++ b/tests/test_models/test_itransformer.py @@ -0,0 +1,65 @@ +from typing import Any, Dict +import unittest + +import tensorflow as tf + +import tfts +from tfts import AutoConfig, AutoModel, KerasTrainer +from tfts.models.itransformer import ITransformer, ITransformerConfig + + +class ITransformerTest(unittest.TestCase): + def test_config(self): + """Test configuration initialization.""" + config = ITransformerConfig( + hidden_size=64, + num_layers=2, + num_attention_heads=4, + ) + self.assertEqual(config.hidden_size, 64) + self.assertEqual(config.num_layers, 2) + self.assertEqual(config.num_attention_heads, 4) + + def test_model_output_shape(self): + """Test model output shape.""" + train_sequence_length = 14 + predict_sequence_length = 7 + config = ITransformerConfig(hidden_size=32, num_layers=1) + model = ITransformer(predict_sequence_length=predict_sequence_length, config=config) + + x = tf.random.normal([2, train_sequence_length, 3]) + y = model(x) + + # Check output shape + self.assertEqual(y.shape[0], 2) # batch size + self.assertEqual(y.shape[1], predict_sequence_length) + + def test_model_direct_instantiation(self): + """Test model direct instantiation.""" + config = ITransformerConfig(hidden_size=32, num_layers=1) + model = ITransformer(predict_sequence_length=8, config=config) + self.assertIsNotNone(model) + + # Test forward pass + x = tf.random.normal([2, 10, 3]) + y = model(x) + self.assertEqual(y.shape[0], 2) + self.assertEqual(y.shape[1], 8) + + # def test_train(self): + # """Test training loop.""" + # train, valid = tfts.get_data("sine", test_size=0.1) + # config = ITransformerConfig(hidden_size=32, num_layers=1) + # model = ITransformer(predict_sequence_length=8, config=config) + + # # Build the model + # model.build_model(train[0].shape) + # model.compile(optimizer=tf.keras.optimizers.Adam(0.003), loss="mse") + # model.fit(train[0], train[1], validation_data=valid, epochs=1, verbose=0) + + # y_test = model.predict(valid[0]) + # self.assertEqual(y_test.shape[0], valid[1].shape[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_models/test_patch_tst.py b/tests/test_models/test_patch_tst.py new file mode 100644 index 00000000..b96a45ff --- /dev/null +++ b/tests/test_models/test_patch_tst.py @@ -0,0 +1,75 @@ +from typing import Any, Dict +import unittest + +import tensorflow as tf + +import tfts +from tfts import AutoConfig, AutoModel, KerasTrainer +from tfts.models.patch_tst import PatchTST, PatchTSTConfig + + +class PatchTSTTest(unittest.TestCase): + def test_config(self): + """Test configuration initialization.""" + config = PatchTSTConfig( + hidden_size=64, + num_layers=2, + num_attention_heads=4, + patch_size=16, + ) + self.assertEqual(config.hidden_size, 64) + self.assertEqual(config.num_layers, 2) + self.assertEqual(config.num_attention_heads, 4) + self.assertEqual(config.patch_size, 16) + + def test_model_output_shape(self): + """Test model output shape.""" + train_sequence_length = 32 + predict_sequence_length = 8 + config = AutoConfig.for_model("patch_tst") + config.hidden_size = 32 + config.num_layers = 1 + config.patch_size = 8 + model = PatchTST(predict_sequence_length=predict_sequence_length, config=config) + + x = tf.random.normal([2, train_sequence_length, 3]) + y = model(x) + + # Check output shape + self.assertEqual(y.shape[0], 2) # batch size + self.assertEqual(y.shape[1], predict_sequence_length) + + def test_model_with_autoconfig(self): + """Test model initialization from AutoConfig.""" + config = AutoConfig.for_model("patch_tst") + config.hidden_size = 32 + config.num_layers = 1 + config.patch_size = 8 + + model = AutoModel.from_config(config, predict_sequence_length=8) + self.assertIsNotNone(model) + + # Test forward pass + x = tf.random.normal([2, 32, 3]) + y = model(x) + self.assertEqual(y.shape, (2, 8, 1)) + + # def test_train(self): + # """Test training loop.""" + # train, valid = tfts.get_data("sine", test_size=0.1) + # config = AutoConfig.for_model("patch_tst") + # config.hidden_size = 32 + # config.num_layers = 1 + # config.patch_size = 8 + + # model = AutoModel.from_config(config, predict_sequence_length=8) + # trainer = KerasTrainer(model) + + # trainer.train(train, valid, optimizer=tf.keras.optimizers.Adam(0.003), epochs=1) + + # y_test = trainer.predict(valid[0]) + # self.assertEqual(y_test.shape, valid[1].shape) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_models/test_rwkv.py b/tests/test_models/test_rwkv.py index e69de29b..bc712187 100644 --- a/tests/test_models/test_rwkv.py +++ b/tests/test_models/test_rwkv.py @@ -0,0 +1,68 @@ +from typing import Any, Dict +import unittest + +import tensorflow as tf + +import tfts +from tfts import AutoConfig, AutoModel, KerasTrainer +from tfts.models.rwkv import RWKV, RWKVConfig + + +class RWKVTest(unittest.TestCase): + def test_config(self): + """Test configuration initialization.""" + config = RWKVConfig( + hidden_size=64, + num_layers=2, + ) + self.assertEqual(config.hidden_size, 64) + self.assertEqual(config.num_layers, 2) + + def test_model_output_shape(self): + """Test model output shape.""" + train_sequence_length = 14 + predict_sequence_length = 7 + config = AutoConfig.for_model("rwkv") + config.hidden_size = 32 + config.num_layers = 1 + model = RWKV(predict_sequence_length=predict_sequence_length, config=config) + + x = tf.random.normal([2, train_sequence_length, 3]) + y = model(x) + + # Check output shape + self.assertEqual(y.shape[0], 2) # batch size + self.assertEqual(y.shape[1], predict_sequence_length) + + def test_model_with_autoconfig(self): + """Test model initialization from AutoConfig.""" + config = AutoConfig.for_model("rwkv") + config.hidden_size = 32 + config.num_layers = 1 + + model = AutoModel.from_config(config, predict_sequence_length=8) + self.assertIsNotNone(model) + + # Test forward pass + x = tf.random.normal([2, 10, 3]) + y = model(x) + self.assertEqual(y.shape, (2, 8, 1)) + + # def test_train(self): + # """Test training loop.""" + # train, valid = tfts.get_data("sine", test_size=0.1) + # config = AutoConfig.for_model("rwkv") + # config.hidden_size = 32 + # config.num_layers = 1 + + # model = AutoModel.from_config(config, predict_sequence_length=8) + # trainer = KerasTrainer(model) + + # trainer.train(train, valid, optimizer=tf.keras.optimizers.Adam(0.003), epochs=1) + + # y_test = trainer.predict(valid[0]) + # self.assertEqual(y_test.shape, valid[1].shape) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_models/test_tft.py b/tests/test_models/test_tft.py index bee0a82c..dd8c75f3 100644 --- a/tests/test_models/test_tft.py +++ b/tests/test_models/test_tft.py @@ -14,6 +14,7 @@ class TFTransformerTest(unittest.TestCase): def test_model(self): predict_sequence_length = 8 custom_model_config = TFTransformerConfig( + encoder_input_dim=5, hidden_size=256, num_layers=2, num_attention_heads=4, @@ -27,7 +28,7 @@ def test_model(self): ) model = TFTransformer(predict_sequence_length, config=custom_model_config) - x = tf.random.normal([2, 16, 3]) + x = tf.random.normal([2, 16, 5]) y = model(x) self.assertEqual(y.shape, (2, predict_sequence_length, 1), "incorrect output shape") @@ -50,6 +51,7 @@ def test_train(self): ) config = AutoConfig.for_model("tft") + config.encoder_input_dim = ts_sequence[0][0].shape[-1] model = AutoModel.from_config(config, predict_sequence_length=predict_sequence_length) trainer = KerasTrainer(model) diff --git a/tests/test_models/test_tide.py b/tests/test_models/test_tide.py new file mode 100644 index 00000000..baa29d9e --- /dev/null +++ b/tests/test_models/test_tide.py @@ -0,0 +1,65 @@ +from typing import Any, Dict +import unittest + +import tensorflow as tf + +import tfts +from tfts import AutoConfig, AutoModel, KerasTrainer +from tfts.models.tide import Tide, TideConfig + + +class TideTest(unittest.TestCase): + def test_config(self): + """Test configuration initialization.""" + config = TideConfig( + hidden_size=128, + num_layers=2, + num_attention_heads=4, + ) + self.assertEqual(config.hidden_size, 128) + self.assertEqual(config.num_layers, 2) + self.assertEqual(config.num_attention_heads, 4) + + def test_model_output_shape(self): + """Test model output shape.""" + train_sequence_length = 14 + predict_sequence_length = 7 + config = TideConfig(hidden_size=64, num_layers=1) + model = Tide(predict_sequence_length=predict_sequence_length, config=config) + + x = tf.random.normal([2, train_sequence_length, 3]) + y = model(x) + + # Check output shape + self.assertEqual(y.shape[0], 2) # batch size + self.assertEqual(y.shape[1], predict_sequence_length) + + def test_model_direct_instantiation(self): + """Test model direct instantiation.""" + config = TideConfig(hidden_size=64, num_layers=1) + model = Tide(predict_sequence_length=8, config=config) + self.assertIsNotNone(model) + + # Test forward pass + x = tf.random.normal([2, 10, 3]) + y = model(x) + self.assertEqual(y.shape[0], 2) + self.assertEqual(y.shape[1], 8) + + # def test_train(self): + # """Test training loop.""" + # train, valid = tfts.get_data("sine", test_size=0.1) + # config = TideConfig(hidden_size=64, num_layers=1) + # model = Tide(predict_sequence_length=8, config=config) + + # # Build the model + # model.build_model(train[0].shape) + # model.compile(optimizer=tf.keras.optimizers.Adam(0.003), loss="mse") + # model.fit(train[0], train[1], validation_data=valid, epochs=1, verbose=0) + + # y_test = model.predict(valid[0]) + # self.assertEqual(y_test.shape[0], valid[1].shape[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tfts/__init__.py b/tfts/__init__.py index 68e4b7d3..652fad8f 100644 --- a/tfts/__init__.py +++ b/tfts/__init__.py @@ -1,6 +1,6 @@ """tfts package for time series prediction with TensorFlow""" -from tfts.data.get_data import get_data +from tfts.data import TimeSeriesSequence, get_data from tfts.models.auto_config import AutoConfig from tfts.models.auto_model import ( AutoModel, @@ -10,7 +10,8 @@ AutoModelForSegmentation, AutoModelForUncertainty, ) -from tfts.trainer import KerasTrainer, Trainer +from tfts.tasks.pipeline import Pipeline +from tfts.trainer import KerasTrainer, Trainer, set_seed from tfts.training_args import TrainingArguments __all__ = [ @@ -24,7 +25,9 @@ "Trainer", "KerasTrainer", "TrainingArguments", + "set_seed" "Pipeline", "get_data", + "TimeSeriesSequence", ] __version__ = "0.0.0" diff --git a/tfts/constants.py b/tfts/constants.py index 498df96d..0321f8f1 100644 --- a/tfts/constants.py +++ b/tfts/constants.py @@ -11,9 +11,11 @@ # model will be saved in TFTS_HOME/hub, and assets will be saved in TFTS_HOME/assets default_cache_path = os.path.join(TFTS_HOME, "hub") +default_datasets_path = os.path.join(TFTS_HOME, "datasets") default_assets_cache_path = os.path.join(TFTS_HOME, "assets") TFTS_HUB_CACHE = os.getenv("TFTS_HUB_CACHE", default_cache_path) +TFTS_DATASETS_CACHE = os.getenv("TFTS_DATASETS_CACHE", default_assets_cache_path) TFTS_ASSETS_CACHE = os.getenv("TFTS_ASSETS_CACHE", default_assets_cache_path) TF2_WEIGHTS_NAME = "tf_model.weights.h5" diff --git a/tfts/data/get_data.py b/tfts/data/get_data.py index c25c14b2..8caa4427 100644 --- a/tfts/data/get_data.py +++ b/tfts/data/get_data.py @@ -3,20 +3,67 @@ """ import logging +import os import random from typing import Dict, List, Optional, Tuple, Union import numpy as np import pandas as pd +from tensorflow.keras.utils import Sequence, get_file -from tfts.constants import TFTS_ASSETS_CACHE +from tfts.constants import TFTS_DATASETS_CACHE logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -AIR_PASSENGER_URL = ( - "https://raw.githubusercontent.com/AileenNielsen/TimeSeriesAnalysisWithPython/master/data/AirPassengers.csv" -) + +TS_DATASETS_URL = { + "air_passengers": { + "url": "https://raw.githubusercontent.com/AileenNielsen/TimeSeriesAnalysisWithPython/master/data/AirPassengers.csv", # noqa: E501 + "format": "csv", + "freq": "MS", + }, + "volatility": { + "url": "https://realized.oxford-man.ox.ac.uk/images/oxfordmanrealizedvolatilityindices.zip", + "format": "zip", + "csv_inside": "oxfordmanrealizedvolatilityindices.csv", + }, + "electricity": { + "url": "https://archive.ics.uci.edu/ml/machine-learning-databases/00321/LD2011_2014.txt.zip", + "format": "zip", + "freq": "15T", + "csv_inside": "LD2011_2014.txt", + }, + "traffic": { + "url": "https://archive.ics.uci.edu/ml/machine-learning-databases/00204/PEMS-SF.zip", + "format": "zip", + "freq": "H", + "csv_inside": "PEMS_train", + }, + "favorita": { + "url": "https://www.kaggle.com/c/favorita-grocery-sales-forecasting/data", + "format": "kaggle", + }, + "m5": { + "url": "https://www.kaggle.com/c/m5-forecasting-accuracy/data", + "format": "kaggle", + }, +} + + +def download_and_extract(name: str) -> str: + """Robust download utility using Keras get_file logic.""" + if name not in TS_DATASETS_URL: + raise ValueError(f"Dataset {name} configuration not found.") + + config = TS_DATASETS_URL[name] + cache_dir = os.path.join(TFTS_DATASETS_CACHE, name) + os.makedirs(cache_dir, exist_ok=True) + + path = get_file( + fname=name, origin=config["url"], cache_subdir=cache_dir, extract=(config.get("format", "zip") == "zip") + ) + return path def get_data( @@ -31,8 +78,20 @@ def get_data( elif name == "ar": return get_ar_data(**kwargs) + + elif name == "volatility": + return get_volatility_data() + + elif name == "electricity": + return get_electricity_data() + + elif name == "traffic": + return get_traffic_data() + else: - raise ValueError(f"unsupported data of {name} yet, try 'sine', 'airpassengers'") + raise ValueError( + f"unsupported data of {name} yet, try 'sine', 'airpassengers', 'ar', 'volatility', 'electricity', 'traffic'" + ) def get_sine( @@ -95,7 +154,7 @@ def get_air_passengers(train_sequence_length: int = 24, predict_sequence_length: Tuple of training and validation data, each containing inputs and outputs. """ - df = pd.read_csv(AIR_PASSENGER_URL, parse_dates=None, date_parser=None, nrows=144) + df = pd.read_csv(TS_DATASETS_URL["air_passengers"]["url"], parse_dates=None, date_parser=None, nrows=144) v = df.iloc[:, 1:2].values v = (v - np.max(v)) / (np.max(v) - np.min(v)) # MinMaxScaler @@ -234,3 +293,131 @@ def get_ar_data( return data, components else: return data + + +def get_volatility_data() -> pd.DataFrame: + data_dir = download_and_extract("volatility") + csv_path = os.path.join(data_dir, TS_DATASETS_URL["volatility"]["csv_inside"]) + + df = pd.read_csv(csv_path, index_col=0) + df.index = pd.to_datetime([str(s).split("+")[0] for s in df.index]) + df = df.reset_index().rename(columns={"index": "date"}) + + # Feature engineering from reference + df["log_vol"] = np.log(df["rv5_ss"].replace(0, np.nan)) + df["log_vol"] = df.groupby("Symbol")["log_vol"].ffill().bfill() + + # Mapping regions + symbol_region_mapping = {".AEX": "EMEA", ".DJI": "AMER", ".HSI": "APAC", ".SPX": "AMER"} # truncated for brevity + df["region"] = df["Symbol"].map(symbol_region_mapping).fillna("Unknown") + + return df + + +def get_electricity_data() -> pd.DataFrame: + data_dir = download_and_extract("electricity") + csv_path = os.path.join(data_dir, TS_DATASETS_URL["electricity"]["csv_inside"]) + + # Industrial datasets are often large; use specific separators + df = pd.read_csv(csv_path, sep=";", decimal=",", index_col=0, parse_dates=True) + df = df.resample("1H").mean().replace(0.0, np.nan) + + # Melt to long format (productive for TimeSeriesSequence) + df = df.reset_index().melt(id_vars="index", var_name="id", value_name="power_usage") + df = df.rename(columns={"index": "date"}).dropna() + return df + + +def get_traffic_data() -> pd.DataFrame: + data_dir = download_and_extract("traffic") + logger.info("Reading PEMS metadata files...") + + def _process_pems_list(s, variable_type=int, delimiter=None): + """Parses a line in the PEMS format to a list.""" + if delimiter is None: + l = [variable_type(i) for i in s.replace("[", "").replace("]", "").split()] + else: + l = [variable_type(i) for i in s.replace("[", "").replace("]", "").split(delimiter)] + return l + + def _read_pems_matrix(data_folder, filename): + """Returns a matrix from a file in the PEMS-custom format.""" + array_list = [] + filepath = os.path.join(data_folder, filename) + with open(filepath, "r") as dat: + lines = dat.readlines() + for i, line in enumerate(lines): + # array is a list of lists (stations x time_observations) + array = [ + _process_pems_list(row_split, variable_type=float, delimiter=None) + for row_split in _process_pems_list(line, variable_type=str, delimiter=";") + ] + array_list.append(array) + return array_list + + def read_single_list(fname): + with open(os.path.join(data_dir, fname), "r") as f: + return _process_pems_list(f.readlines()[0]) + + shuffle_order = np.array(read_single_list("randperm")) - 1 + train_dayofweek = read_single_list("PEMS_trainlabels") + test_dayofweek = read_single_list("PEMS_testlabels") + stations_list = read_single_list("stations_list") + + logger.info("Reading and parsing train/test matrices (this may take a moment)...") + train_tensor = _read_pems_matrix(data_dir, "PEMS_train") + test_tensor = _read_pems_matrix(data_dir, "PEMS_test") + + # Inverse permutate shuffle order to restore temporal consistency + inverse_mapping = {new_loc: prev_loc for prev_loc, new_loc in enumerate(shuffle_order)} + reverse_shuffle_order = np.array([inverse_mapping[i] for i in range(len(shuffle_order))]) + + # Combine and Reorder + day_of_week = np.array(train_dayofweek + test_dayofweek) + combined_tensor = np.array(train_tensor + test_tensor) + + day_of_week = day_of_week[reverse_shuffle_order] + combined_tensor = combined_tensor[reverse_shuffle_order] + + logger.info("Aggregating to hourly data and formatting...") + labels = [f"traj_{i}" for i in stations_list] + hourly_list = [] + + for day, day_matrix in enumerate(combined_tensor): + # day_matrix.T -> index: time (144 intervals), columns: stations + hourly = pd.DataFrame(day_matrix.T, columns=labels) + # Sampled at 10 min intervals: 6 intervals = 1 hour + hourly["hour_on_day"] = [int(i / 6) for i in hourly.index] + + # Mean occupancy per hour + hourly = hourly.groupby("hour_on_day").mean() + hourly["sensor_day"] = day + hourly["time_on_day"] = hourly.index + hourly["day_of_week"] = day_of_week[day] + hourly_list.append(hourly) + + hourly_frame = pd.concat(hourly_list, axis=0, ignore_index=True) + + # Flatten the dataframe: Each row is (sensor_id, time, occupancy) + store_columns = [c for c in hourly_frame.columns if "traj" in c] + other_columns = ["sensor_day", "time_on_day", "day_of_week"] + + flat_list = [] + for store in store_columns: + sliced = hourly_frame[[store] + other_columns].copy() + sliced.columns = ["occupancy"] + other_columns + sliced["station_id"] = int(store.replace("traj_", "")) + + # Calculate hours from start for a continuous time axis + sliced["hours_from_start"] = sliced["time_on_day"] + sliced["sensor_day"] * 24.0 + flat_list.append(sliced) + + df = pd.concat(flat_list, axis=0, ignore_index=True) + + # Filter to match range used by academic papers (first 173 days) + df = df[df["sensor_day"] < 173].copy() + + # Sorting for time-series consistency + df = df.sort_values(["station_id", "hours_from_start"]) + + return df diff --git a/tfts/data/timeseries.py b/tfts/data/timeseries.py index 65ce13c5..78981090 100644 --- a/tfts/data/timeseries.py +++ b/tfts/data/timeseries.py @@ -1,7 +1,6 @@ """TFTS Dataset This module provides a TimeSeriesSequence class for handling time series data in TensorFlow. -It supports various feature transformations, data augmentation, and efficient data loading. """ import logging @@ -55,12 +54,13 @@ class TimeSeriesSequence(Sequence): def __init__( self, data: pd.DataFrame, - time_idx: str, target_column: str, train_sequence_length: int, predict_sequence_length: int = 1, + time_idx: Optional[str] = None, batch_size: int = 32, group_column: Optional[List[str]] = None, + feature_columns: Optional[List[str]] = None, drop_last: bool = False, feature_config: Optional[Dict] = None, mode: str = "train", @@ -68,7 +68,7 @@ def __init__( processor: Optional[List[Callable]] = None, ): """Initialize the TimeSeriesSequence.""" - self.data = data + self.data = data.copy() self.time_idx = time_idx self.target = [target_column] if isinstance(target_column, str) else target_column self.train_sequence_length = train_sequence_length @@ -101,6 +101,39 @@ def __init__( f"batch_size={batch_size}, mode={mode}" ) + def _build_sequences(self): + """Builds a lookup table for sequences to avoid heavy DataFrame slicing during training.""" + sequence_indices = [] + + if self.group_column: + grouped = self.data.groupby(self.group_column) + else: + grouped = [("all", self.data)] + + for _, group in grouped: + group = group.sort_values(self.time_idx) + n_rows = len(group) + max_idx = n_rows - self.train_sequence_length - self.predict_sequence_length + 1 + + # Pre-extract numpy arrays for speed + feature_data = group[self.features].values.astype(np.float32) + target_data = group[self.target].values.astype(np.float32) + + for i in range(0, max_idx, self.stride): + sequence_indices.append( + { + "x": feature_data[i : i + self.train_sequence_length], + "y": target_data[ + i + + self.train_sequence_length : i + + self.train_sequence_length + + self.predict_sequence_length + ], + } + ) + + return sequence_indices + def __len__(self) -> int: """Get the number of batches in the sequence. @@ -134,26 +167,20 @@ def __getitem__(self, idx: int) -> Tuple[np.ndarray, np.ndarray]: return encoder_inputs, decoder_targets def get_tf_dataset(self) -> tf.data.Dataset: - """Convert to TensorFlow Dataset. - - Returns: - tf.data.Dataset: TensorFlow dataset with 3D tensors - """ - - def generator(): - for i in range(len(self)): - yield self[i] - - # Get number of target variables - num_targets = len(self.target) + """Convert to high-performance tf.data pipeline.""" + # Get feature dimension from actual sequence data + if len(self.sequences) > 0: + num_features = self.sequences[0][0].shape[-1] + else: + num_features = len(self.target) - return tf.data.Dataset.from_generator( - generator, - output_signature=( - tf.TensorSpec(shape=(None, self.train_sequence_length, num_targets), dtype=tf.float32), - tf.TensorSpec(shape=(None, self.predict_sequence_length, num_targets), dtype=tf.float32), - ), + output_signature = ( + tf.TensorSpec(shape=(None, self.train_sequence_length, num_features), dtype=tf.float32), + tf.TensorSpec(shape=(None, self.predict_sequence_length, len(self.target)), dtype=tf.float32), ) + return tf.data.Dataset.from_generator( + lambda: (self[i] for i in range(len(self))), output_signature=output_signature + ).prefetch(tf.data.AUTOTUNE) def _generate_sequences( self, group: pd.DataFrame, time_idx: str, target_column: str diff --git a/tfts/features/norm.py b/tfts/features/norm.py index 50888f9c..312e3d65 100644 --- a/tfts/features/norm.py +++ b/tfts/features/norm.py @@ -24,13 +24,11 @@ def normalize( Args: data (np.ndarray): The input time series data. Must be 1D or 2D. - If 2D, normalization occurs along the specified axis. method (NormalizationMethod): The normalization method to use. - "standard": Standard scaling (Z-score normalization). (X - mean) / std. - "minmax": Min-max scaling. (X - min) / (max - min). Scales to [0, 1]. - "robust": Robust scaling using median and IQR. (X - median) / IQR. - - "log1p": Log transformation (log(1 + X)). Useful for positive data - with skewed distributions. Does not use `axis` as it's element-wise. + - "log1p": Log transformation (log(1 + X)). Useful for positive data with skewed distributions. axis (int): The axis along which to compute statistics for normalization. Typically 0 for column-wise (features) or 1 for row-wise. Ignored for "log1p". @@ -46,8 +44,7 @@ def normalize( Returns: Tuple[np.ndarray, Dict[str, Any]]: - The normalized data. - - A dictionary containing the parameters used for normalization, - which are needed for denormalization. Includes 'method' and 'axis'. + - A dictionary containing the parameters used for normalization. Raises: ValueError: If the input data is not a NumPy array, has unsupported dimensions, @@ -157,9 +154,7 @@ def denormalize(normalized_data: np.ndarray, params: Dict[str, Any]) -> np.ndarr Args: normalized_data (np.ndarray): The normalized input data. - params (Dict[str, Any]): The parameters dictionary returned by the - `normalize` function. Must contain 'method' - and other method-specific parameters. + params (Dict[str, Any]): The parameters dictionary returned by the `normalize` function. Returns: np.ndarray: The denormalized (original scale) data. diff --git a/tfts/generator.py b/tfts/generator.py index a97fb7a2..f7bc5b61 100644 --- a/tfts/generator.py +++ b/tfts/generator.py @@ -1,9 +1,15 @@ """tfts Generator""" -from typing import Any, Dict, Union +from typing import Any, Dict, Optional, Union import numpy as np import pandas as pd +import tensorflow as tf + + +class GenerationConfig: + def __init__(self, **kwargs) -> None: + self.max_length = kwargs.pop("max_length", 20) class GenerationMixin: @@ -11,10 +17,20 @@ class GenerationMixin: A class containing auto-regressive generation, to be used as a mixin. """ + def prepare_inputs_for_generation(self, *args, **kwargs): + return + def generate( - self, inputs: Union[pd.DataFrame, np.ndarray], generation_config: Dict[str, Any] = None + self, + inputs: Union[pd.DataFrame, np.ndarray], + future_covariates: Optional[tf.Tensor] = None, + max_steps: int = 10, + generation_config: Dict[str, Any] = None, + logits_processor=None, + seed=None, + **kwargs, ) -> pd.DataFrame: - """Generate time series predictions in an autoregressive manner. + """Generate time series predictions in an auto-regressive manner. Args: inputs: Initial input sequence as DataFrame or numpy array @@ -37,7 +53,6 @@ def generate( # Convert inputs to DataFrame if needed if isinstance(inputs, np.ndarray): - # We need to convert numpy array to DataFrame features = self.get_feature_names() if len(features) != inputs.shape[1]: raise ValueError(f"Input array shape {inputs.shape} doesn't match feature count {len(features)}") diff --git a/tfts/layers/dense_layer.py b/tfts/layers/dense_layer.py index 7b1e58dc..189b7788 100644 --- a/tfts/layers/dense_layer.py +++ b/tfts/layers/dense_layer.py @@ -1,6 +1,6 @@ """Layer for :py:class:`~tfts.models.wavenet` :py:class:`~tfts.models.transformer`""" -from typing import Optional, Tuple +from typing import Any, Dict, Optional, Tuple import tensorflow as tf from tensorflow.keras import activations, constraints, initializers, regularizers @@ -117,3 +117,65 @@ def get_config(self): } base_config = super(FeedForwardNetwork, self).get_config() return dict(list(base_config.items()) + list(config.items())) + + +class MoeMLP(tf.keras.layers.Layer): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act="silu", + kernel_initializer: str = "glorot_uniform", + bias_initializer: str = "zeros", + use_bias: bool = False, + **kwargs + ): + super().__init__(**kwargs) + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) + self.use_bias = use_bias + + def build(self, input_shape: Tuple[Optional[int], ...]): + self.gate_proj = Dense( + self.intermediate_size, + use_bias=self.use_bias, + kernel_initializer=self.kernel_initializer, + bias_initializer=self.bias_initializer, + name="gate_proj", + ) + self.up_proj = Dense( + self.intermediate_size, + use_bias=self.use_bias, + kernel_initializer=self.kernel_initializer, + bias_initializer=self.bias_initializer, + name="up_proj", + ) + self.down_proj = Dense( + self.hidden_size, + use_bias=self.use_bias, + kernel_initializer=self.kernel_initializer, + bias_initializer=self.bias_initializer, + name="down_proj", + ) + self.act_fn = activations.get(self.hidden_act) + super().build(input_shape) + + def call(self, hidden_states: tf.Tensor) -> tf.Tensor: + gate = self.gate_proj(hidden_states) + up = self.up_proj(hidden_states) + return self.down_proj(self.act_fn(gate) * up) + + def get_config(self) -> Dict[str, Any]: + config = { + "hidden_size": self.hidden_size, + "intermediate_size": self.intermediate_size, + "hidden_act": self.hidden_act, + "kernel_initializer": initializers.serialize(self.kernel_initializer), + "bias_initializer": initializers.serialize(self.bias_initializer), + "use_bias": self.use_bias, + } + base_config = super().get_config() + return dict(list(base_config.items()) + list(config.items())) diff --git a/tfts/layers/graph_layer.py b/tfts/layers/graph_layer.py new file mode 100644 index 00000000..b0153131 --- /dev/null +++ b/tfts/layers/graph_layer.py @@ -0,0 +1,367 @@ +"""Layer for Graph Neural Networks""" + +from typing import Any, Dict, Optional, Tuple, Union + +import tensorflow as tf +from tensorflow.keras import activations, constraints, initializers, regularizers +from tensorflow.keras.layers import Dropout, Layer + + +class GraphConv(Layer): + """Basic Graph Convolution Layer. + + This layer implements the graph convolution operation: + Output = Activation(Adjacency * Features * Kernel + Bias) + + Parameters + ---------- + units : int + Dimensionality of the output space. + activation : str or callable, optional + Activation function to use. If you don't specify anything, no activation is applied. + use_bias : bool, optional + Whether the layer uses a bias vector. Defaults to True. + kernel_initializer : str, optional + Initializer for the `kernel` weights matrix. Defaults to "glorot_uniform". + bias_initializer : str, optional + Initializer for the `bias` vector. Defaults to "zeros". + kernel_regularizer : str, optional + Regularizer function applied to the `kernel` weights matrix. + bias_regularizer : str, optional + Regularizer function applied to the `bias` vector. + """ + + def __init__( + self, + units: int, + activation: Optional[Union[str, callable]] = None, + use_bias: bool = True, + kernel_initializer: str = "glorot_uniform", + bias_initializer: str = "zeros", + kernel_regularizer: Optional[str] = None, + bias_regularizer: Optional[str] = None, + kernel_constraint: Optional[str] = None, + bias_constraint: Optional[str] = None, + **kwargs: Dict[str, Any], + ) -> None: + super(GraphConv, self).__init__(**kwargs) + self.units = units + self.activation = activations.get(activation) + self.use_bias = use_bias + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + + def build(self, input_shape: Tuple[Optional[int], ...]) -> None: + # input_shape[0] is features: (batch, nodes, features) + # input_shape[1] is adjacency: (batch, nodes, nodes) + feat_shape = input_shape[0] + input_dim = feat_shape[-1] + + self.kernel = self.add_weight( + shape=(input_dim, self.units), + initializer=self.kernel_initializer, + name="kernel", + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + ) + if self.use_bias: + self.bias = self.add_weight( + shape=(self.units,), + initializer=self.bias_initializer, + name="bias", + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + ) + else: + self.bias = None + super(GraphConv, self).build(input_shape) + + def call(self, inputs: Tuple[tf.Tensor, tf.Tensor], **kwargs) -> tf.Tensor: + """Forward pass. + + Parameters + ---------- + inputs : Tuple[tf.Tensor, tf.Tensor] + A tuple containing: + - features: 3D tensor (batch_size, num_nodes, input_dim) + - adjacency: 3D tensor (batch_size, num_nodes, num_nodes) or SparseTensor + + Returns + ------- + tf.Tensor + Output tensor (batch_size, num_nodes, units) + """ + features, adjacency = inputs + + # Transform features: H = XW + output = tf.matmul(features, self.kernel) + + # Propagate: O = AH + # Handle Sparse Adjacency + if isinstance(adjacency, tf.sparse.SparseTensor): + # Sparse matmul requires 2D, typically used in single-graph mode or carefully reshaped batches + # Assuming batch_size=1 or shared adjacency for simplicity in sparse mode, + # otherwise standard dense matmul is safer for batched data. + output = tf.sparse.sparse_dense_matmul(adjacency, output) + else: + output = tf.matmul(adjacency, output) + + if self.use_bias: + output = tf.nn.bias_add(output, self.bias) + + if self.activation is not None: + output = self.activation(output) + + return output + + def compute_output_shape(self, input_shape: Tuple[Tuple[int, ...], ...]) -> Tuple[int, ...]: + features_shape = input_shape[0] + return features_shape[:-1] + (self.units,) + + def get_config(self) -> Dict[str, Any]: + config = { + "units": self.units, + "activation": activations.serialize(self.activation), + "use_bias": self.use_bias, + "kernel_initializer": initializers.serialize(self.kernel_initializer), + "bias_initializer": initializers.serialize(self.bias_initializer), + "kernel_regularizer": regularizers.serialize(self.kernel_regularizer), + "bias_regularizer": regularizers.serialize(self.bias_regularizer), + "kernel_constraint": constraints.serialize(self.kernel_constraint), + "bias_constraint": constraints.serialize(self.bias_constraint), + } + base_config = super(GraphConv, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class GraphAttention(Layer): + """Graph Attention Layer (GAT). + + This layer implements the multi-head graph attention mechanism. + Unlike the original code, this implementation is fully vectorized, computing + all heads in parallel for better performance. + + Parameters + ---------- + units : int + Dimensionality of the output space per head (if reduction='concat') or total (if 'average'). + num_heads : int, optional + Number of attention heads. Defaults to 1. + head_reduction : str, optional + How to combine heads: 'concat' or 'average'. Defaults to 'average'. + dropout_rate : float, optional + Dropout rate for attention coefficients. Defaults to 0.5. + activation : str, optional + Activation function. Defaults to "relu". + use_bias : bool, optional + Whether to use bias. Defaults to True. + """ + + def __init__( + self, + units: int, + num_heads: int = 1, + head_reduction: str = "average", + dropout_rate: float = 0.5, + activation: str = "relu", + use_bias: bool = True, + kernel_initializer: str = "glorot_uniform", + bias_initializer: str = "zeros", + kernel_regularizer: Optional[str] = None, + bias_regularizer: Optional[str] = None, + kernel_constraint: Optional[str] = None, + bias_constraint: Optional[str] = None, + **kwargs: Dict[str, Any], + ) -> None: + super(GraphAttention, self).__init__(**kwargs) + if head_reduction not in {"concat", "average"}: + raise ValueError("Possible reduction methods: concat, average") + + self.units = units + self.num_heads = num_heads + self.head_reduction = head_reduction + self.dropout_rate = dropout_rate + self.activation = activations.get(activation) + self.use_bias = use_bias + + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + + # Output dimension calculation + if head_reduction == "concat": + self.output_dim = self.units * self.num_heads + else: + self.output_dim = self.units + + def build(self, input_shape: Tuple[Optional[int], ...]) -> None: + # input_shape[0]: (Batch, Nodes, Features) + input_dim = input_shape[0][-1] + + # W: Transformation kernel for all heads (Input -> Heads * Units) + self.kernel = self.add_weight( + shape=(input_dim, self.num_heads * self.units), + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + name="kernel", + ) + + # W2: Residual kernel (Input -> Heads * Units) - matched from original logic + self.kernel_residual = self.add_weight( + shape=(input_dim, self.num_heads * self.units), + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + name="kernel_residual", + ) + + # Attention Kernels + # Self attention mechanism parameters (Heads, Units, 1) + self.attn_kernel_self = self.add_weight( + shape=(self.num_heads, self.units, 1), + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + name="attn_kernel_self", + ) + + # Neighbor attention mechanism parameters (Heads, Units, 1) + self.attn_kernel_neighs = self.add_weight( + shape=(self.num_heads, self.units, 1), + initializer=self.kernel_initializer, + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint, + name="attn_kernel_neigh", + ) + + if self.use_bias: + self.bias = self.add_weight( + shape=(self.num_heads * self.units,), + initializer=self.bias_initializer, + regularizer=self.bias_regularizer, + constraint=self.bias_constraint, + name="bias", + ) + + self.dropout = Dropout(self.dropout_rate) + super(GraphAttention, self).build(input_shape) + + def call(self, inputs: Tuple[tf.Tensor, tf.Tensor], training: Optional[bool] = None) -> tf.Tensor: + """Forward pass of Graph Attention. + + Parameters + ---------- + inputs : Tuple[tf.Tensor, tf.Tensor] + - features: (batch_size, num_nodes, input_dim) + - adjacency: (batch_size, num_nodes, num_nodes) + training : bool, optional + Whether in training mode (for dropout), by default None + + Returns + ------- + tf.Tensor + Output tensor + """ + X, A = inputs + # X shape: (B, N, F) + # A shape: (B, N, N) + + # 1. Linear Transformations + # (B, N, Heads * Units) + features = tf.matmul(X, self.kernel) + features_residual = tf.matmul(X, self.kernel_residual) + + # Reshape to (B, N, Heads, Units) to separate heads + B = tf.shape(features)[0] + N = tf.shape(features)[1] + + features_reshaped = tf.reshape(features, (B, N, self.num_heads, self.units)) + + # 2. Attention Scores + # Compute "a^T * Wh_i" and "a^T * Wh_j" + # (B, N, Heads, Units) x (Heads, Units, 1) -> (B, N, Heads, 1) + # We use einsum for clarity with batch and head dims + attn_for_self = tf.einsum("bnhu,huo->bnh", features_reshaped, self.attn_kernel_self) + attn_for_neighs = tf.einsum("bnhu,huo->bnh", features_reshaped, self.attn_kernel_neighs) + + # Add scores (broadcasting): (B, N, 1, Heads) + (B, 1, N, Heads) -> (B, N, N, Heads) + # Note: Original code logic sum dense matrices. + dense = tf.expand_dims(attn_for_self, axis=2) + tf.expand_dims(attn_for_neighs, axis=1) + + # LeakyReLU + dense = tf.nn.leaky_relu(dense, alpha=0.2) + + # 3. Masking and Softmax + # Mask: -10e9 * (1.0 - A). Expand A to match heads: (B, N, N, 1) + A_expanded = tf.expand_dims(A, axis=-1) + mask = -10e9 * (1.0 - A_expanded) + + # Add mask to logits + dense += mask + + # Softmax over neighbors (axis 2) -> (B, N, N, Heads) + attn_coef = tf.nn.softmax(dense, axis=2) + + # Apply dropout to coefficients + attn_coef = self.dropout(attn_coef, training=training) + + # Apply dropout to features (Original code applied dropout to features before aggregation) + features_dropout = self.dropout(features_reshaped, training=training) + + # 4. Aggregation + # (B, N, N, Heads) x (B, N, Heads, Units) -> (B, N, Heads, Units) + # This represents: output_i = sum_j(alpha_ij * h_j) + node_features = tf.einsum("bnkh,bkhu->bnhu", attn_coef, features_dropout) + + # Flatten heads back to (B, N, Heads * Units) + node_features = tf.reshape(node_features, (B, N, self.num_heads * self.units)) + + # 5. Residual Connection + Bias + node_features += features_residual + + if self.use_bias: + node_features = tf.nn.bias_add(node_features, self.bias) + + # 6. Reduce Heads + if self.head_reduction == "concat": + # Already in shape (B, N, Heads * Units) + output = node_features + else: + # Average: Reshape to (B, N, Heads, Units) then mean + output = tf.reshape(node_features, (B, N, self.num_heads, self.units)) + output = tf.reduce_mean(output, axis=2) + + # 7. Activation + output = self.activation(output) + + return output + + def compute_output_shape(self, input_shape: Tuple[Tuple[int, ...], ...]) -> Tuple[int, ...]: + features_shape = input_shape[0] + return features_shape[:-1] + (self.output_dim,) + + def get_config(self) -> Dict[str, Any]: + config = { + "units": self.units, + "num_heads": self.num_heads, + "head_reduction": self.head_reduction, + "dropout_rate": self.dropout_rate, + "activation": activations.serialize(self.activation), + "use_bias": self.use_bias, + "kernel_initializer": initializers.serialize(self.kernel_initializer), + "bias_initializer": initializers.serialize(self.bias_initializer), + "kernel_regularizer": regularizers.serialize(self.kernel_regularizer), + "bias_regularizer": regularizers.serialize(self.bias_regularizer), + "kernel_constraint": constraints.serialize(self.kernel_constraint), + "bias_constraint": constraints.serialize(self.bias_constraint), + } + base_config = super(GraphAttention, self).get_config() + return dict(list(base_config.items()) + list(config.items())) diff --git a/tfts/layers/moe_layer.py b/tfts/layers/moe_layer.py index be036ed1..e990c627 100644 --- a/tfts/layers/moe_layer.py +++ b/tfts/layers/moe_layer.py @@ -6,8 +6,10 @@ from tensorflow.keras import activations, constraints, initializers, regularizers from tensorflow.keras.layers import Dense +from tfts.layers.dense_layer import MoeMLP -class MoELayer(tf.keras.layers.Layer): + +class SparseMoe(tf.keras.layers.Layer): """Mixture of Experts layer for time series prediction. This layer implements a Mixture of Experts architecture where multiple expert networks @@ -17,119 +19,178 @@ class MoELayer(tf.keras.layers.Layer): def __init__( self, + hidden_size: int, num_experts: int, - expert_hidden_size: int, - gating_hidden_size: int, - expert_activation: str = "relu", - gating_activation: str = "softmax", + num_experts_per_tok: int, + moe_intermediate_size: int, + shared_expert_intermediate_size: int, + norm_topk_prob: bool = True, + hidden_act: str = "silu", kernel_initializer: str = "glorot_uniform", - kernel_regularizer: Optional[str] = None, - kernel_constraint: Optional[str] = None, - use_bias: bool = True, bias_initializer: str = "zeros", - trainable: bool = True, - name: Optional[str] = None, + use_bias: bool = False, + **kwargs, ): - super(MoELayer, self).__init__(trainable=trainable, name=name) + super().__init__(**kwargs) + self.hidden_size = hidden_size self.num_experts = num_experts - self.expert_hidden_size = expert_hidden_size - self.gating_hidden_size = gating_hidden_size - self.expert_activation = expert_activation - self.gating_activation = gating_activation - self.kernel_initializer = kernel_initializer - self.kernel_regularizer = kernel_regularizer - self.kernel_constraint = kernel_constraint + self.top_k = num_experts_per_tok + self.moe_intermediate_size = moe_intermediate_size + self.shared_expert_intermediate_size = shared_expert_intermediate_size + self.norm_topk_prob = norm_topk_prob + self.hidden_act = hidden_act + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) self.use_bias = use_bias - self.bias_initializer = bias_initializer def build(self, input_shape: Tuple[int, ...]): - input_dim = int(input_shape[-1]) + # Gating network (router) + self.gate = Dense( + self.num_experts, + use_bias=self.use_bias, + kernel_initializer=self.kernel_initializer, + bias_initializer=self.bias_initializer, + name="gate", + ) - # Create expert networks - self.experts = [] - for i in range(self.num_experts): - expert = Dense( - self.expert_hidden_size, - activation=self.expert_activation, + # Experts + self.experts = [ + MoeMLP( + hidden_size=self.hidden_size, + intermediate_size=self.moe_intermediate_size, + hidden_act=self.hidden_act, kernel_initializer=self.kernel_initializer, - kernel_regularizer=regularizers.get(self.kernel_regularizer), - kernel_constraint=constraints.get(self.kernel_constraint), - use_bias=self.use_bias, bias_initializer=self.bias_initializer, + use_bias=self.use_bias, name=f"expert_{i}", ) - self.experts.append(expert) - - # Create gating network - self.gating_network = Dense( - self.num_experts, - activation=self.gating_activation, + for i in range(self.num_experts) + ] + + # Shared Expert + self.shared_expert = MoeMLP( + hidden_size=self.hidden_size, + intermediate_size=self.shared_expert_intermediate_size, + hidden_act=self.hidden_act, kernel_initializer=self.kernel_initializer, - kernel_regularizer=regularizers.get(self.kernel_regularizer), - kernel_constraint=constraints.get(self.kernel_constraint), - use_bias=self.use_bias, bias_initializer=self.bias_initializer, - name="gating_network", + use_bias=self.use_bias, + name="shared_expert", ) - - # Output projection layer - self.output_projection = Dense( - input_dim, - kernel_initializer=self.kernel_initializer, - kernel_regularizer=regularizers.get(self.kernel_regularizer), - kernel_constraint=constraints.get(self.kernel_constraint), + self.shared_expert_gate = Dense( + 1, use_bias=self.use_bias, + kernel_initializer=self.kernel_initializer, bias_initializer=self.bias_initializer, - name="output_projection", + name="shared_expert_gate", ) + super().build(input_shape) - super(MoELayer, self).build(input_shape) - - def call(self, inputs: tf.Tensor) -> tf.Tensor: + @tf.function # This is important for AutoGraph to convert correctly + def call(self, hidden_states: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]: """Forward pass of the MoE layer. Args: - inputs: Tensor of shape (batch_size, sequence_length, input_dim) + hidden_states: Tensor of shape (batch_size, sequence_length, hidden_dim) Returns: - output: Tensor of shape (batch_size, sequence_length, input_dim) + output: Tuple[Tensor (batch_size, sequence_length, hidden_dim), Tensor (num_tokens, num_experts)] """ + batch_size = tf.shape(hidden_states)[0] + sequence_length = tf.shape(hidden_states)[1] + hidden_dim = tf.shape(hidden_states)[2] + + hidden_states_flat = tf.reshape(hidden_states, (-1, hidden_dim)) # (num_tokens, hidden_dim) - # Get expert outputs - expert_outputs = [] - for expert in self.experts: - expert_output = expert(inputs) # (batch_size, seq_length, expert_hidden_size) - expert_outputs.append(expert_output) + router_logits = self.gate(hidden_states_flat) # (num_tokens, num_experts) - # Stack expert outputs - expert_outputs = tf.stack(expert_outputs, axis=2) # (batch_size, seq_length, num_experts, expert_hidden_size) + routing_weights = tf.nn.softmax(router_logits, axis=-1) - # Get gating weights - gating_weights = self.gating_network(inputs) # (batch_size, seq_length, num_experts) - gating_weights = tf.expand_dims(gating_weights, axis=-1) # (batch_size, seq_length, num_experts, 1) + # Get top-k experts and their weights + routing_weights, selected_experts = tf.math.top_k(routing_weights, k=self.top_k) # (num_tokens, top_k) + + if self.norm_topk_prob: + # Normalize top-k probabilities + routing_weights = routing_weights / tf.reduce_sum(routing_weights, axis=-1, keepdims=True) + + routing_weights = tf.cast(routing_weights, hidden_states.dtype) + + final_hidden_states = tf.zeros_like(hidden_states_flat, dtype=hidden_states.dtype) + + # --- FIX APPLIED HERE --- + # Use a pure Python for loop to iterate over self.experts + # This ensures expert_idx is a concrete Python integer for list indexing. + for i in range(self.num_experts): + expert_idx_tensor = tf.constant(i, dtype=tf.int32) # Create a tensor for comparison in graph ops + + # Create a boolean mask for tokens that selected the current expert + expert_chosen_mask = tf.equal(selected_experts, expert_idx_tensor) # (num_tokens, top_k) + + coordinates = tf.where(expert_chosen_mask) + + # Check if there are any true values for this expert + # Use tf.cond for graph-compatible conditional execution + def process_expert_branch(): + token_indices_for_expert = coordinates[:, 0] + topk_position_for_expert = coordinates[:, 1] + + # Gather the hidden states for the tokens that chose this expert + current_state = tf.gather(hidden_states_flat, token_indices_for_expert) + + # Here, self.experts[i] uses the Python integer 'i' + current_hidden_states_expert_output = self.experts[i](current_state) + + # Gather the corresponding routing weights for these tokens and their chosen expert + current_routing_weights = tf.gather_nd( + routing_weights, tf.stack([token_indices_for_expert, topk_position_for_expert], axis=-1) + ) + current_routing_weights = tf.expand_dims( + current_routing_weights, axis=-1 + ) # Shape (num_tokens_for_expert, 1) + + weighted_expert_output = current_hidden_states_expert_output * tf.cast( + current_routing_weights, current_hidden_states_expert_output.dtype + ) + + # Accumulate results using tf.tensor_scatter_nd_add + indices_to_scatter = tf.expand_dims( + token_indices_for_expert, axis=-1 + ) # Shape (num_tokens_for_expert, 1) + + # This needs to update `final_hidden_states` which is a loop-carried tensor + return tf.tensor_scatter_nd_add(final_hidden_states, indices_to_scatter, weighted_expert_output) + + # If no tokens selected this expert, return the current final_hidden_states unchanged + def no_op_branch(): + return final_hidden_states + + # tf.cond takes callables for true_fn and false_fn + final_hidden_states = tf.cond( + tf.shape(coordinates)[0] > 0, true_fn=process_expert_branch, false_fn=no_op_branch + ) - # Combine expert outputs using gating weights - combined_output = tf.reduce_sum( - expert_outputs * gating_weights, axis=2 - ) # (batch_size, seq_length, expert_hidden_size) + # Shared Expert Computation + shared_expert_output = self.shared_expert(hidden_states_flat) + shared_expert_gate_output = tf.nn.sigmoid(self.shared_expert_gate(hidden_states_flat)) + shared_expert_output = shared_expert_gate_output * shared_expert_output - # Project back to input dimension - output = self.output_projection(combined_output) # (batch_size, seq_length, input_dim) + final_hidden_states = final_hidden_states + shared_expert_output - return output + final_hidden_states = tf.reshape(final_hidden_states, (batch_size, sequence_length, hidden_dim)) + return final_hidden_states, router_logits def get_config(self) -> Dict[str, Any]: config = { + "hidden_size": self.hidden_size, "num_experts": self.num_experts, - "expert_hidden_size": self.expert_hidden_size, - "gating_hidden_size": self.gating_hidden_size, - "expert_activation": self.expert_activation, - "gating_activation": self.gating_activation, - "kernel_initializer": self.kernel_initializer, - "kernel_regularizer": self.kernel_regularizer, - "kernel_constraint": self.kernel_constraint, + "num_experts_per_tok": self.top_k, # Using top_k for consistency with internal + "moe_intermediate_size": self.moe_intermediate_size, + "shared_expert_intermediate_size": self.shared_expert_intermediate_size, + "norm_topk_prob": self.norm_topk_prob, + "hidden_act": self.hidden_act, + "kernel_initializer": initializers.serialize(self.kernel_initializer), + "bias_initializer": initializers.serialize(self.bias_initializer), "use_bias": self.use_bias, - "bias_initializer": self.bias_initializer, } - base_config = super(MoELayer, self).get_config() + base_config = super().get_config() return dict(list(base_config.items()) + list(config.items())) diff --git a/tfts/layers/rwkv_layer.py b/tfts/layers/rwkv_layer.py index 72e0747c..edfb4c4e 100644 --- a/tfts/layers/rwkv_layer.py +++ b/tfts/layers/rwkv_layer.py @@ -1,6 +1,7 @@ from typing import Dict, Optional, Tuple import tensorflow as tf +from tensorflow.keras.layers import Dense class TimeMixing(tf.keras.layers.Layer): @@ -10,20 +11,17 @@ def __init__(self, config, **kwargs): super().__init__(**kwargs) self.n_embd = config.hidden_size - def build(self, input_shape: Tuple[Optional[int], ...]): - super().build(input_shape) - - # Trainable parameters - self.time_mix_k = self.add_weight(name="time_mix_k", shape=(1, self.n_embd), initializer="zeros") - self.time_mix_v = self.add_weight(name="time_mix_v", shape=(1, self.n_embd), initializer="zeros") - self.time_mix_r = self.add_weight(name="time_mix_r", shape=(1, self.n_embd), initializer="zeros") + def build(self, input_shape): + self.time_mix_k = self.add_weight(name="time_mix_k", shape=(1, 1, self.n_embd), initializer="zeros") + self.time_mix_v = self.add_weight(name="time_mix_v", shape=(1, 1, self.n_embd), initializer="zeros") + self.time_mix_r = self.add_weight(name="time_mix_r", shape=(1, 1, self.n_embd), initializer="zeros") self.time_first = self.add_weight(name="time_first", shape=(1, self.n_embd), initializer="zeros") self.time_decay = self.add_weight(name="time_decay", shape=(1, self.n_embd), initializer="zeros") - self.key = tf.keras.layers.Dense(self.n_embd, use_bias=False) - self.value = tf.keras.layers.Dense(self.n_embd, use_bias=False) - self.receptance = tf.keras.layers.Dense(self.n_embd, use_bias=False) - self.output_layer = tf.keras.layers.Dense(self.n_embd, use_bias=False) + self.key = Dense(self.n_embd, use_bias=False) + self.value = Dense(self.n_embd, use_bias=False) + self.receptance = Dense(self.n_embd, use_bias=False) + self.output_layer = Dense(self.n_embd, use_bias=False) def call(self, x, state): """time mixing @@ -33,39 +31,58 @@ def call(self, x, state): x : tf.Tensor The input tensor of shape (batch_size, seq_length, embed_dim). """ - aa, bb, pp = state + # state = [last_x, aa, bb, pp] + last_x, aa, bb, pp = state + + # Shifted x for mixing + last_x_expanded = tf.expand_dims(last_x, 1) + # x shape: (Batch, Seq, Hidden) + if tf.shape(x)[1] > 1: + xx = tf.concat([last_x_expanded, x[:, :-1, :]], axis=1) + else: + xx = last_x_expanded - # Mix with previous timestep - xk = x * self.time_mix_k + state[0] * (1 - self.time_mix_k) - xv = x * self.time_mix_v + state[0] * (1 - self.time_mix_v) - xr = x * self.time_mix_r + state[0] * (1 - self.time_mix_r) + xk = x * self.time_mix_k + xx * (1 - self.time_mix_k) + xv = x * self.time_mix_v + xx * (1 - self.time_mix_v) + xr = x * self.time_mix_r + xx * (1 - self.time_mix_r) r = tf.sigmoid(self.receptance(xr)) k = self.key(xk) v = self.value(xv) - ww = self.time_first + k - qq = tf.maximum(pp, ww) - e1 = tf.exp(pp - qq) - e2 = tf.exp(ww - qq) + # WKV calculation (recursive) + # For simplicity/correctness in RNN form, we process along the time dimension + seq_len = tf.shape(x)[1] + + outputs = tf.TensorArray(tf.float32, size=seq_len) - a = e1 * aa + e2 * v - b = e1 * bb + e2 - wkv = a / b + curr_aa, curr_bb, curr_pp = aa, bb, pp - # Update states - ww = pp + self.time_decay - qq = tf.maximum(ww, k) - e1 = tf.exp(ww - qq) - e2 = tf.exp(k - qq) + for t in range(seq_len): + kt = k[:, t, :] + vt = v[:, t, :] - new_aa = e1 * aa + e2 * v - new_bb = e1 * bb + e2 - new_pp = qq + # WKV calculation + ww = self.time_first + kt + qq = tf.maximum(curr_pp, ww) + e1 = tf.exp(curr_pp - qq) + e2 = tf.exp(ww - qq) + wkv = (e1 * curr_aa + e2 * vt) / (e1 * curr_bb + e2) + outputs = outputs.write(t, wkv) - new_state = [new_aa, new_bb, new_pp] + # Update state + ww = curr_pp + self.time_decay + qq = tf.maximum(ww, kt) + e1 = tf.exp(ww - qq) + e2 = tf.exp(kt - qq) + curr_aa = e1 * curr_aa + e2 * vt + curr_bb = e1 * curr_bb + e2 + curr_pp = qq - return self.output_layer(r * wkv), new_state + wkv_all = tf.transpose(outputs.stack(), [1, 0, 2]) # [B, T, C] + + new_state = [x[:, -1, :], curr_aa, curr_bb, curr_pp] + return self.output_layer(r * wkv_all), new_state class ChannelMixing(tf.keras.layers.Layer): @@ -75,27 +92,40 @@ def __init__(self, config, **kwargs): super().__init__(**kwargs) self.n_embd = config.hidden_size - def build(self, input_shape: Tuple[Optional[int], ...]): - super().build(input_shape) - - self.time_mix_k = self.add_weight(name="time_mix_k", shape=(1, self.n_embd), initializer="zeros") - self.time_mix_r = self.add_weight(name="time_mix_r", shape=(1, self.n_embd), initializer="zeros") + def build(self, input_shape): + self.time_mix_k = self.add_weight(name="time_mix_k", shape=(1, 1, self.n_embd), initializer="zeros") + self.time_mix_r = self.add_weight(name="time_mix_r", shape=(1, 1, self.n_embd), initializer="zeros") - self.key = tf.keras.layers.Dense(self.n_embd, use_bias=False) - self.value = tf.keras.layers.Dense(self.n_embd, use_bias=False) - self.receptance = tf.keras.layers.Dense(self.n_embd, use_bias=False) + self.key = Dense(self.n_embd, use_bias=False) + self.value = Dense(self.n_embd, use_bias=False) + self.receptance = Dense(self.n_embd, use_bias=False) def call(self, x, state): """channel mixing + # state is the x from the LAST timestep of the PREVIOUS batch: shape (batch, hidden_size) + # We need to shift x by 1 and prepending the state Parameters ---------- x : tf.Tensor The input tensor of shape (batch_size, seq_length, embed_dim). """ - xk = x * self.time_mix_k + state * (1 - self.time_mix_k) - xr = x * self.time_mix_r + state * (1 - self.time_mix_r) + + # state is the x from the LAST timestep of the PREVIOUS batch + last_x = state + + last_x_expanded = tf.expand_dims(last_x, 1) + + if tf.shape(x)[1] > 1: + xx = tf.concat([last_x_expanded, x[:, :-1, :]], axis=1) + else: + xx = last_x_expanded + + xk = x * self.time_mix_k + xx * (1 - self.time_mix_k) + xr = x * self.time_mix_r + xx * (1 - self.time_mix_r) r = tf.sigmoid(self.receptance(xr)) k = tf.square(tf.nn.relu(self.key(xk))) - return r * self.value(k), x + kv = self.value(k) + + return r * kv, x[:, -1, :] diff --git a/tfts/losses/__init__.py b/tfts/losses/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tfts/losses/loss.py b/tfts/losses/loss.py new file mode 100644 index 00000000..33e28841 --- /dev/null +++ b/tfts/losses/loss.py @@ -0,0 +1,32 @@ +from typing import List + +import tensorflow as tf + + +class MultiQuantileLoss(tf.keras.losses.Loss): + def __init__(self, quantiles: List[float], name="multi_quantile_loss"): + super().__init__(name=name) + self.quantiles = quantiles + + def call(self, y_true, y_pred): + """ + y_true: [batch, pred_len, num_labels] + y_pred: [batch, pred_len, num_labels * num_quantiles] + """ + # Reshape y_pred to [batch, pred_len, num_labels, num_quantiles] + # and y_true to [batch, pred_len, num_labels, 1] + y_true = tf.expand_dims(y_true, axis=-1) + + # Split y_pred into the different quantiles + # Assuming the head outputs quantiles stacked in the last dimension + num_labels = y_true.shape[-2] + y_pred = tf.reshape(y_pred, [-1, y_pred.shape[1], num_labels, len(self.quantiles)]) + + losses = [] + for i, q in enumerate(self.quantiles): + error = y_true[..., 0] - y_pred[..., i] + # Pinball loss: max(q*e, (q-1)*e) + quantile_l = tf.maximum(q * error, (q - 1) * error) + losses.append(tf.reduce_mean(quantile_l)) + + return tf.add_n(losses) diff --git a/tfts/models/auto_config.py b/tfts/models/auto_config.py index 1a60d996..d5033d61 100644 --- a/tfts/models/auto_config.py +++ b/tfts/models/auto_config.py @@ -21,7 +21,7 @@ ("nbeats", "NBeatsConfig"), ("dlinear", "DLinearConfig"), ("rwkv", "RWKVConfig"), - ("patches_tst", "PatchTSTConfig"), + ("patch_tst", "PatchTSTConfig"), ("deep_ar", "DeepARConfig"), ] ) diff --git a/tfts/models/auto_model.py b/tfts/models/auto_model.py index ec7acfea..c1fe09b5 100644 --- a/tfts/models/auto_model.py +++ b/tfts/models/auto_model.py @@ -10,9 +10,18 @@ import numpy as np import pandas as pd import tensorflow as tf +from tensorflow.keras.layers import Dense +from tfts.losses.loss import MultiQuantileLoss from tfts.models.base import BaseConfig, BaseModel -from tfts.tasks.auto_task import AnomalyHead, ClassificationHead +from tfts.tasks.auto_task import ( + AnomalyHead, + ClassificationHead, + ClassificationOutput, + GaussianHead, + PredictionHead, + PredictionOutput, +) from ..constants import CONFIG_NAME, TF2_WEIGHTS_INDEX_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME from .auto_config import AutoConfig @@ -35,7 +44,7 @@ ("nbeats", "NBeats"), ("dlinear", "DLinear"), ("rwkv", "RWKV"), - ("patches_tst", "PatchTST"), + ("patch_tst", "PatchTST"), ("deep_ar", "DeepAR"), ] ) @@ -285,3 +294,48 @@ def from_config(cls, config): module = importlib.import_module(f".{model_name}", "tfts.models") model = getattr(module, class_name)(config=config) return cls(model, config) + + +class AutoModelForQuantile(BaseModel): + """tfts model for quantile forecasting""" + + def __init__(self, model, config): + super(AutoModelForQuantile, self).__init__() + self.model = model + self.config = config + self.quantiles = getattr(config, "quantiles", [0.1, 0.5, 0.9]) + self.num_labels = getattr(config, "num_labels", 1) + self.head = Dense(self.num_labels * len(self.quantiles)) + self.keras_model: Optional[tf.keras.Model] = None + + def __call__( + self, + x: Union[tf.data.Dataset, Tuple[np.ndarray], List[np.ndarray]], + output_hidden_states: Optional[bool] = True, + **kwargs, + ): + if self.keras_model is not None: + return self.keras_model(x) + + model_output = self.model(x, output_hidden_states=output_hidden_states) + return self.head(model_output) + + @classmethod + def from_config(cls, config, quantiles: List[float] = [0.1, 0.5, 0.9]): + config.quantiles = quantiles + model_name = config.model_type + class_name = MODEL_MAPPING_NAMES[model_name] + module = importlib.import_module(f".{model_name}", "tfts.models") + model = getattr(module, class_name)(config=config) + return cls(model, config) + + def build_model(self, inputs): + model_output = self.model(inputs) + outputs = self.head(model_output) + self.keras_model = tf.keras.Model(inputs, outputs) + return self.keras_model + + def compile_model(self, optimizer="adam"): + """Helper to compile with the correct loss""" + loss_fn = MultiQuantileLoss(quantiles=self.quantiles) + self.keras_model.compile(optimizer=optimizer, loss=loss_fn) diff --git a/tfts/models/bert.py b/tfts/models/bert.py index 536d4311..cd546aae 100644 --- a/tfts/models/bert.py +++ b/tfts/models/bert.py @@ -7,11 +7,12 @@ from typing import Dict, Optional, Tuple import tensorflow as tf -from tensorflow.keras.layers import Dense, Reshape +from tensorflow.keras.layers import Dense, Lambda, Reshape from tfts.layers.embed_layer import DataEmbedding, TokenEmbedding from tfts.models.transformer import Encoder +from ..tasks.auto_task import PredictionOutput from .base import BaseConfig, BaseModel logger = logging.getLogger(__name__) @@ -31,7 +32,7 @@ def __init__( hidden_act: str = "gelu", hidden_dropout_prob: float = 0.0, attention_probs_dropout_prob: float = 0.0, - type_vocab_size: int = 2, + output_size: int = 1, initializer_range: float = 0.02, layer_norm_eps: float = 1e-12, pad_token_id: int = 0, @@ -51,7 +52,7 @@ def __init__( hidden_act: The activation function for hidden layers. Default is "gelu". hidden_dropout_prob: The dropout probability for hidden layers. Default is 0.1. attention_probs_dropout_prob: The dropout probability for attention probabilities. Default is 0.1. - type_vocab_size: The vocabulary size for token types (usually 2). Default is 2. + output_size: The vocabulary size for output. Default is 1. initializer_range: The standard deviation for weight initialization. Default is 0.02. layer_norm_eps: The epsilon value for layer normalization. Default is 1e-12. pad_token_id: The ID for the padding token. Default is 0. @@ -71,7 +72,7 @@ def __init__( self.hidden_act: str = hidden_act self.hidden_dropout_prob: float = hidden_dropout_prob self.attention_probs_dropout_prob: float = attention_probs_dropout_prob - self.type_vocab_size: int = type_vocab_size + self.output_size: int = output_size self.initializer_range: float = initializer_range self.layer_norm_eps: float = layer_norm_eps self.positional_type: str = positional_type @@ -132,8 +133,10 @@ def __init__(self, predict_sequence_length: int = 1, config: Optional[BertConfig ] logger.debug(f"Created {len(self.dense_layers)} dense layers with units: {self.config.dense_units}") - self.projection = Dense(self.predict_sequence_length, activation="linear", name="projection") - self.reshape = Reshape((self.predict_sequence_length, 1)) + self.projection = Dense( + self.predict_sequence_length * self.config.output_size, activation="linear", name="projection" + ) + self.reshape = Reshape((self.predict_sequence_length, self.config.output_size)) logger.debug("Model building completed") def __call__( diff --git a/tfts/models/diffusion.py b/tfts/models/diffusion.py index 622dace4..ad086cac 100644 --- a/tfts/models/diffusion.py +++ b/tfts/models/diffusion.py @@ -117,58 +117,45 @@ def __init__(self, predict_sequence_length: int = 1, config: Optional[DiffusionC self.predict_sequence_length = predict_sequence_length self.noise_scheduler = NoiseScheduler(self.config) - # Time embedding + # Layers that don't depend on input feature count self.time_embedding = Dense(self.config.hidden_size) - - # Embedding layer self.embedding = DataEmbedding(self.config.hidden_size, positional_type="positional encoding") - - # Transformer blocks self.blocks = [TransformerBlock(self.config) for _ in range(self.config.num_layers)] - # Output projection + # Initialize the projection layer here once self.output_projection = Dense(1) - def __call__( - self, - x, - states=None, - teacher=None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ): - """Diffusion model call for time series forecasting""" - # Prepare inputs - x, encoder_feature, decoder_feature = self._prepare_3d_inputs(x, ignore_decoder_inputs=False) - - # Generate random timesteps + def __call__(self, x, training=None, **kwargs): + """Diffusion model forward pass logic.""" + # 1. Prepare inputs (using BaseModel helper) + # Note: ignore_decoder_inputs=True because diffusion usually denoises the encoder path + x, encoder_feature, _ = self._prepare_3d_inputs(x, ignore_decoder_inputs=True) + + # 2. Generate random timesteps batch_size = tf.shape(encoder_feature)[0] t = tf.random.uniform(shape=[batch_size], minval=0, maxval=self.config.num_diffusion_steps, dtype=tf.int32) - # Add noise to input + # 3. Add noise to input noisy_x, noise = self.noise_scheduler.add_noise(encoder_feature, t) - # Time embedding - t_emb = self.time_embedding(tf.cast(t, tf.float32)) + # 4. Time embedding (batch, 1) -> (batch, 1, hidden) + t_float = tf.cast(t, tf.float32) + t_emb = self.time_embedding(tf.expand_dims(t_float, axis=-1)) t_emb = tf.expand_dims(t_emb, axis=1) - # Process through transformer blocks + # 5. Transformer process x = self.embedding(noisy_x) - x = tf.concat([x, t_emb], axis=-1) + x = x + t_emb # Inject time information for block in self.blocks: x = block(x) - # Project to output + # 6. Predict noise and reconstruct predicted_noise = self.output_projection(x) - - # Remove noise denoised_x = self.noise_scheduler.remove_noise(noisy_x, predicted_noise, t) - # Slice the output to only include the last predict_sequence_length steps - denoised_x = denoised_x[:, -self.predict_sequence_length :, :] - - return denoised_x + # 7. Return prediction window + return denoised_x[:, -self.predict_sequence_length :, :] class TransformerBlock(tf.keras.layers.Layer): diff --git a/tfts/models/itransformer.py b/tfts/models/itransformer.py new file mode 100644 index 00000000..8b7ae4cb --- /dev/null +++ b/tfts/models/itransformer.py @@ -0,0 +1,143 @@ +""" +`iTransformer: Inverted Transformers Are Effective for Time Series Forecasting +`_ +""" + +from typing import Dict, Optional + +import tensorflow as tf +from tensorflow.keras.layers import Dense, LayerNormalization + +from tfts.layers.attention_layer import Attention +from tfts.layers.dense_layer import FeedForwardNetwork +from tfts.layers.embed_layer import DataEmbedding + +from ..layers.util_layer import ShapeLayer +from .base import BaseConfig, BaseModel + + +class ITransformerConfig(BaseConfig): + model_type: str = "itransformer" + + def __init__( + self, + hidden_size: int = 64, + num_layers: int = 3, + num_attention_heads: int = 8, + attention_probs_dropout_prob: float = 0.1, + hidden_dropout_prob: float = 0.1, + ffn_intermediate_size: int = 256, + max_position_embeddings: int = 512, + initializer_range: float = 0.02, + layer_norm_eps: float = 1e-12, + pad_token_id: int = 0, + **kwargs + ) -> None: + """ + Initializes the configuration for the iTransformer model with the specified parameters. + + Args: + hidden_size: Size of each attention head. + num_layers: The number of stacked transformer layers. + num_attention_heads: The number of attention heads. + attention_probs_dropout_prob: Dropout rate for attention probabilities. + hidden_dropout_prob: Dropout rate for hidden layers. + ffn_intermediate_size: Size of the intermediate layer in the feed-forward network. + max_position_embeddings: Maximum sequence length for positional embeddings. + initializer_range: Standard deviation for weight initialization. + layer_norm_eps: Epsilon for layer normalization. + pad_token_id: ID for padding token. + """ + super().__init__() + + self.hidden_size: int = hidden_size + self.num_layers: int = num_layers + self.num_attention_heads: int = num_attention_heads + self.attention_probs_dropout_prob: float = attention_probs_dropout_prob + self.hidden_dropout_prob: float = hidden_dropout_prob + self.ffn_intermediate_size: int = ffn_intermediate_size + self.max_position_embeddings: int = max_position_embeddings + self.initializer_range: float = initializer_range + self.layer_norm_eps: float = layer_norm_eps + self.pad_token_id: int = pad_token_id + self.update(kwargs) + + +class ITransformer(BaseModel): + """TensorFlow iTransformer model for time series forecasting""" + + def __init__(self, predict_sequence_length: int = 1, config: Optional[ITransformerConfig] = None): + super().__init__() + self.config = config or ITransformerConfig() + self.predict_sequence_length = predict_sequence_length + + # In iTransformer, we embed the entire time dimension of each variate + self.enc_embedding = Dense(self.config.hidden_size) + + # Transformer blocks + self.blocks = [TransformerBlock(self.config) for _ in range(self.config.num_layers)] + + # Project from hidden_size to predict_sequence_length + self.projector = Dense(self.predict_sequence_length) + + def __call__(self, x, training=None, **kwargs): + """iTransformer forward pass: Inverting Variates and Time""" + # x shape: (batch, seq_len, n_vars) + x, encoder_feature, _ = self._prepare_3d_inputs(x, ignore_decoder_inputs=True) + + # 1. Inversion: (batch, seq_len, n_vars) -> (batch, n_vars, seq_len) + # Each variate becomes a "token" + x = tf.transpose(encoder_feature, perm=[0, 2, 1]) + + # 2. Embedding: Map the whole history of each variate to hidden_size + # (batch, n_vars, hidden_size) + x = self.enc_embedding(x) + + # 3. Attention: Process across variables + for block in self.blocks: + x = block(x, training=training) + + # 4. Projection: Map hidden_size to predict_len + # (batch, n_vars, predict_len) + x = self.projector(x) + + # 5. Reverse Inversion: (batch, predict_len, n_vars) + x = tf.transpose(x, perm=[0, 2, 1]) + + return x + + +class TransformerBlock(tf.keras.layers.Layer): + """Standard Transformer block used in iTransformer""" + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.attention = Attention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + attention_probs_dropout_prob=config.attention_probs_dropout_prob, + ) + self.attention_output = Dense(config.hidden_size) + self.attention_norm = LayerNormalization(epsilon=config.layer_norm_eps) + self.attention_dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + self.feed_forward = FeedForwardNetwork( + hidden_size=config.hidden_size, + intermediate_size=config.ffn_intermediate_size, + hidden_dropout_prob=config.hidden_dropout_prob, + ) + self.feed_forward_norm = LayerNormalization(epsilon=config.layer_norm_eps) + self.feed_forward_dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, x, training=None): + # Self-attention across variates + attention_output = self.attention(x, x, x, training=training) + attention_output = self.attention_output(attention_output) + attention_output = self.attention_dropout(attention_output, training=training) + x = self.attention_norm(x + attention_output) + + # Feed-forward + ffn_output = self.feed_forward(x, training=training) + ffn_output = self.feed_forward_dropout(ffn_output, training=training) + x = self.feed_forward_norm(x + ffn_output) + return x diff --git a/tfts/models/patches_tst.py b/tfts/models/patch_tst.py similarity index 93% rename from tfts/models/patches_tst.py rename to tfts/models/patch_tst.py index ccad3910..a010225d 100644 --- a/tfts/models/patches_tst.py +++ b/tfts/models/patch_tst.py @@ -6,7 +6,7 @@ from typing import Dict, Optional import tensorflow as tf -from tensorflow.keras.layers import Dense, LayerNormalization +from tensorflow.keras.layers import Dense, Flatten, LayerNormalization from tfts.layers.attention_layer import Attention from tfts.layers.dense_layer import FeedForwardNetwork @@ -17,7 +17,7 @@ class PatchTSTConfig(BaseConfig): - model_type: str = "patches_tst" + model_type: str = "patch_tst" def __init__( self, @@ -27,6 +27,7 @@ def __init__( attention_probs_dropout_prob: float = 0.1, hidden_dropout_prob: float = 0.1, ffn_intermediate_size: int = 256, + output_size: int = 1, max_position_embeddings: int = 512, initializer_range: float = 0.02, layer_norm_eps: float = 1e-12, @@ -58,6 +59,7 @@ def __init__( self.attention_probs_dropout_prob: float = attention_probs_dropout_prob self.hidden_dropout_prob: float = hidden_dropout_prob self.ffn_intermediate_size: int = ffn_intermediate_size + self.output_size: int = output_size self.max_position_embeddings: int = max_position_embeddings self.initializer_range: float = initializer_range self.layer_norm_eps: float = layer_norm_eps @@ -84,7 +86,8 @@ def __init__(self, predict_sequence_length: int = 1, config: Optional[PatchTSTCo self.blocks = [TransformerBlock(self.config) for _ in range(self.config.num_layers)] # Output projection - self.output_projection = Dense(1) + self.flatten = Flatten() + self.output_projection = Dense(self.predict_sequence_length * self.config.output_size) def __call__( self, @@ -121,9 +124,11 @@ def __call__( x = block(x) # Project to output - x = self.output_projection(x) + x = self.flatten(x) # [batch, num_patches * hidden] + x = self.output_projection(x) # [batch, predict_len * n_vars] # Reshape back to original sequence length + x = tf.reshape(x, [batch_size, -1, 1]) # Slice the output to only include the last predict_sequence_length steps diff --git a/tfts/models/rwkv.py b/tfts/models/rwkv.py index 184d119f..e7649751 100644 --- a/tfts/models/rwkv.py +++ b/tfts/models/rwkv.py @@ -6,7 +6,7 @@ from typing import Dict, Optional import tensorflow as tf -from tensorflow.keras.layers import GRU, Dense +from tensorflow.keras.layers import GRU, Dense, LayerNormalization from tfts.layers.embed_layer import DataEmbedding from tfts.layers.rwkv_layer import ChannelMixing, TimeMixing @@ -19,7 +19,7 @@ class RWKVConfig(BaseConfig): def __init__( self, - num_layers: int = 25, + num_layers: int = 3, hidden_size: int = 64, dense_hidden_size: int = 32, dropout: float = 0.0, @@ -66,18 +66,19 @@ def __init__(self, predict_sequence_length: int = 1, config: Optional[RWKVConfig # Embedding layer self.embedding = DataEmbedding(self.config.hidden_size, positional_type="positional encoding") - self.ln0 = tf.keras.layers.LayerNormalization(epsilon=self.config.layer_norm_eps) + self.ln0 = LayerNormalization(epsilon=self.config.layer_norm_eps) self.blocks = [RWKVBlock(self.config) for _ in range(self.config.num_layers)] - self.ln_out = tf.keras.layers.LayerNormalization(epsilon=self.config.layer_norm_eps) + self.ln_out = LayerNormalization(epsilon=self.config.layer_norm_eps) self.output_projection = Dense(1) - def init_state(self, batch_size=1): + def init_state(self, batch_size: int): states = [] for _ in range(self.config.num_layers): # States for attention att_states = [ + tf.zeros((batch_size, self.config.hidden_size)), # last_x tf.zeros((batch_size, self.config.hidden_size)), # aa tf.zeros((batch_size, self.config.hidden_size)), # bb tf.zeros((batch_size, self.config.hidden_size)) - 1e30, # pp @@ -97,12 +98,13 @@ def __call__( ): """RWKV model call for time series forecasting""" - if states is None: - states = self.init_state() - # Prepare inputs x, encoder_feature, decoder_feature = self._prepare_3d_inputs(x, ignore_decoder_inputs=False) + if states is None: + batch_size = tf.shape(x)[0] + states = self.init_state(batch_size=batch_size) + # Embed inputs x = self.embedding(encoder_feature) x = self.ln0(x) @@ -118,7 +120,7 @@ def __call__( # Slice the output to only include the last predict_sequence_length steps x = x[:, -self.predict_sequence_length :, :] - return x, new_states + return x def _prepare_inputs(self, inputs): """Prepare the inputs for the encoder.""" @@ -160,17 +162,11 @@ class RWKVBlock(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super().__init__(**kwargs) self.config = config - self.ln1 = tf.keras.layers.LayerNormalization() + self.ln1 = LayerNormalization() self.attention = TimeMixing(config) - self.ln2 = tf.keras.layers.LayerNormalization() + self.ln2 = LayerNormalization() self.feed_forward = ChannelMixing(config) - def build(self, input_shape): - super().build(input_shape) - # Ensure the attention and feed_forward layers are built - self.attention.build(input_shape) - self.feed_forward.build(input_shape) - def call(self, x, states): """block diff --git a/tfts/models/tft.py b/tfts/models/tft.py index 688b4089..1669f95c 100644 --- a/tfts/models/tft.py +++ b/tfts/models/tft.py @@ -6,7 +6,7 @@ from typing import Optional import tensorflow as tf -from tensorflow.keras.layers import Dense, Dropout, LayerNormalization +from tensorflow.keras.layers import LSTM, Concatenate, Dense, Dropout, LayerNormalization from ..layers.attention_layer import Attention, SelfAttention from ..layers.dense_layer import FeedForwardNetwork @@ -19,9 +19,12 @@ class TFTransformerConfig(BaseConfig): def __init__( self, + encoder_input_dim: int = 1, + decoder_input_dim: int = 1, hidden_size: int = 256, num_layers: int = 2, num_attention_heads: int = 4, + output_size: int = 1, attention_probs_dropout_prob: float = 0.0, hidden_dropout_prob: float = 0.0, ffn_intermediate_size: int = 256, @@ -29,12 +32,15 @@ def __init__( initializer_range: float = 0.02, layer_norm_eps: float = 1e-12, pad_token_id: int = 0, - **kwargs + **kwargs, ): super(TFTransformerConfig, self).__init__() + self.encoder_input_dim = encoder_input_dim + self.decoder_input_dim = decoder_input_dim self.hidden_size = hidden_size self.num_layers = num_layers self.num_attention_heads = num_attention_heads + self.output_size = output_size self.attention_probs_dropout_prob = attention_probs_dropout_prob self.hidden_dropout_prob = hidden_dropout_prob self.ffn_intermediate_size = ffn_intermediate_size @@ -57,34 +63,64 @@ def __init__(self, predict_sequence_length=1, config: Optional[TFTransformerConf self.temporal_embedding = DataEmbedding(self.config.hidden_size, positional_type="positional encoding") self.static_embedding = DataEmbedding(self.config.hidden_size) - # Variable selection networks (simplified as dense layers with gating) - self.temporal_variable_selection = Dense(self.config.hidden_size, activation="sigmoid") - self.static_variable_selection = Dense(self.config.hidden_size, activation="sigmoid") - - # Gated Residual Networks (GRN) for feature processing - self.temporal_grn = FeedForwardNetwork( - self.config.hidden_size, self.config.ffn_intermediate_size, self.config.hidden_dropout_prob + # Variable selection networks + self.encoder_var_selection = tf.keras.Sequential( + [ + Dense(self.config.hidden_size, activation="relu"), + Dense(self.config.encoder_input_dim, activation="sigmoid"), + ], + name="encoder_var_selection", ) - self.static_grn = FeedForwardNetwork( - self.config.hidden_size, self.config.ffn_intermediate_size, self.config.hidden_dropout_prob + self.decoder_var_selection = tf.keras.Sequential( + [ + Dense(self.config.hidden_size, activation="relu"), + Dense(self.config.decoder_input_dim, activation="sigmoid"), + ], + name="decoder_var_selection", ) - # Static covariate encoder (using LSTM) - self.static_encoder = tf.keras.layers.LSTM(self.config.hidden_size, return_sequences=True) + self.lstm_encoder_layers = [ + LSTM( + self.config.hidden_size, + return_sequences=True, + dropout=0.0 if i < self.config.num_layers - 1 else 0.0, + name=f"lstm_enc_{i}", + ) + for i in range(self.config.num_layers) + ] + self.lstm_decoder_layers = [ + LSTM( + self.config.hidden_size, + return_sequences=True, + dropout=0.0 if i < self.config.num_layers - 1 else 0.0, + name=f"lstm_dec_{i}", + ) + for i in range(self.config.num_layers) + ] - # Temporal fusion decoder (combining LSTM, attention, and gating) - self.temporal_decoder = tf.keras.layers.LSTM(self.config.hidden_size, return_sequences=True) self.attention = Attention( hidden_size=self.config.hidden_size, num_attention_heads=self.config.num_attention_heads, attention_probs_dropout_prob=self.config.attention_probs_dropout_prob, ) - self.gate = Dense(self.config.hidden_size, activation="sigmoid") + self.concat = Concatenate(axis=1) # Output projection - self.output_projection = Dense(1) + self.output_projection = Dense(self.config.output_size) - def __call__(self, x: tf.Tensor, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None): + def __call__( + self, + x: Optional[tf.Tensor] = None, + encoder_cat=None, + encoder_num=None, + decoder_cat=None, + decoder_num=None, + static_cat=None, + static_num=None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ): """Process inputs through the TFT model. Parameters @@ -101,36 +137,33 @@ def __call__(self, x: tf.Tensor, output_hidden_states: Optional[bool] = None, re tf.Tensor Output tensor of shape (batch_size, predict_sequence_length, 1). """ + + # encoder_input = tf.concat([encoder_num, encoder_cat], axis=2) + # decoder_input = tf.concat([decoder_num, decoder_cat], axis=2) + # Prepare inputs x, encoder_feature, decoder_feature = self._prepare_3d_inputs(x, ignore_decoder_inputs=False) - # Embed temporal and static features - temporal_embedded = self.temporal_embedding(encoder_feature) - static_embedded = self.static_embedding(decoder_feature) - - # Apply variable selection - temporal_selected = self.temporal_variable_selection(temporal_embedded) - static_selected = self.static_variable_selection(static_embedded) + encoder_weights = self.encoder_var_selection(encoder_feature) + encoder_feature = encoder_feature * encoder_weights - # Process through Gated Residual Networks - temporal_processed = self.temporal_grn(temporal_selected) - static_processed = self.static_grn(static_selected) + decoder_weights = self.decoder_var_selection(decoder_feature) + decoder_feature = decoder_feature * decoder_weights # Encode static covariates - static_encoded = self.static_encoder(static_processed) + temporal_encoded = encoder_feature + for layer in self.lstm_encoder_layers: + temporal_encoded = layer(temporal_encoded) # Decode temporal features - temporal_decoded = self.temporal_decoder(temporal_processed) - - # Apply attention and gating - attention_output = self.attention(temporal_decoded, static_encoded, static_encoded) - gate_output = self.gate(attention_output) - fused_output = gate_output * attention_output + temporal_decoded = temporal_encoded + for layer in self.lstm_decoder_layers: + temporal_decoded = layer(temporal_decoded) - # Project to output - output = self.output_projection(fused_output) + sequence = self.concat([temporal_encoded, temporal_decoded]) + attention_output = self.attention(sequence, sequence, sequence) - # Slice the output to only include the last predict_sequence_length steps - output = output[:, -self.predict_sequence_length :, :] + attention_output = attention_output[:, -self.predict_sequence_length :, :] + output = self.output_projection(attention_output) return output diff --git a/tfts/models/timemixer.py b/tfts/models/timemixer.py new file mode 100644 index 00000000..2c14df3c --- /dev/null +++ b/tfts/models/timemixer.py @@ -0,0 +1,4 @@ +""" +`TimeMixer: Decomposable Multiscale Mixing for Time Series Forecasting +`_ +""" diff --git a/tfts/models/unet.py b/tfts/models/unet.py index 8519b359..fa17cb1c 100644 --- a/tfts/models/unet.py +++ b/tfts/models/unet.py @@ -3,6 +3,7 @@ `_ """ +import logging from typing import List, Optional, Tuple import tensorflow as tf @@ -24,6 +25,8 @@ from ..layers.util_layer import ShapeLayer from .base import BaseConfig, BaseModel +logger = logging.getLogger(__name__) + class UnetConfig(BaseConfig): model_type: str = "unet" diff --git a/tfts/pipelines/__init__.py b/tfts/pipelines/__init__.py deleted file mode 100644 index 8b275779..00000000 --- a/tfts/pipelines/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""tfts pipelines""" diff --git a/tfts/pipelines/base.py b/tfts/pipelines/base.py deleted file mode 100644 index d601614a..00000000 --- a/tfts/pipelines/base.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging -from typing import Any, Callable, Dict, List, Optional - -logger = logging.getLogger(__name__) - - -class Pipeline(object): - _load_processor = False - - def __init__(self, task: str, model: str, processor: Optional[Callable] = None): - self.task = task - self.model = model - self.processor = processor - - def __call__(self): - pass - - def forward(self): - pass - - def predict(self): - pass - - def get_iterator(self): - pass - - def postprocess(self): - pass diff --git a/tfts/tasks/auto_task.py b/tfts/tasks/auto_task.py index 865084e5..fce6c239 100644 --- a/tfts/tasks/auto_task.py +++ b/tfts/tasks/auto_task.py @@ -1,12 +1,13 @@ """Time series task head""" +from dataclasses import dataclass from typing import Optional, Tuple import numpy as np import tensorflow as tf from tensorflow.keras.layers import Dense, GlobalAveragePooling1D -from .base import BaseTask +from .base import BaseTask, ModelOutput class PredictionHead(tf.keras.layers.Layer, BaseTask): @@ -16,11 +17,13 @@ def __init__(self): super(PredictionHead, self).__init__() -class SegmentationHead(tf.keras.layers.Layer, BaseTask): - """Segmentation task head layer""" - - def __init__(self): - super(SegmentationHead, self).__init__() +@dataclass +class PredictionOutput(ModelOutput): + prediction_logits: tf.Tensor = None + last_hidden_state: Optional[tf.Tensor] = None + hidden_states: Optional[Tuple[tf.Tensor, ...]] = None + attentions: Optional[Tuple[tf.Tensor, ...]] = None + loss: Optional[tf.Tensor] = None class ClassificationHead(tf.keras.layers.Layer): @@ -58,6 +61,13 @@ def call(self, inputs: tf.Tensor, **kwargs) -> tf.Tensor: return logits +@dataclass +class ClassificationOutput(ModelOutput): + logits: tf.Tensor = None + hidden_states: Optional[Tuple[tf.Tensor, ...]] = None + loss: Optional[tf.Tensor] = None + + class AnomalyHead: """Anomaly task head layer: Reconstruct style""" @@ -95,6 +105,13 @@ def mahala_distantce(x, mean, cov, epsilon=1e-8): return d +@dataclass +class AnomalyOutput(ModelOutput): + anomaly_scores: tf.Tensor = None + reconstruction_logits: Optional[tf.Tensor] = None + loss: Optional[tf.Tensor] = None + + class GaussianHead(tf.keras.layers.Layer): def __init__(self, units: int): self.units = units @@ -131,3 +148,10 @@ def get_config(self): config = {"units": self.units} base_config = super().get_config() return {**base_config, **config} + + +class SegmentationHead(tf.keras.layers.Layer, BaseTask): + """Segmentation task head layer""" + + def __init__(self): + super(SegmentationHead, self).__init__() diff --git a/tfts/tasks/base.py b/tfts/tasks/base.py index 2a9c364e..c8fb8814 100644 --- a/tfts/tasks/base.py +++ b/tfts/tasks/base.py @@ -1,5 +1,26 @@ from abc import ABC, abstractmethod +from collections import OrderedDict, UserDict +from dataclasses import dataclass, fields +from typing import Any class BaseTask(ABC): """Base task for tfts task.""" + + +class ModelOutput(OrderedDict): + def __post_init__(self): + # Automatically populate the OrderedDict with dataclass fields + class_fields = fields(self) + for field in class_fields: + value = getattr(self, field.name) + if value is not None: + self[field.name] = value + + def __getitem__(self, k): + if isinstance(k, int): + return self.to_tuple()[k] + return super().__getitem__(k) + + def to_tuple(self) -> tuple[Any]: + return tuple(getattr(self, f.name) for f in fields(self) if getattr(self, f.name) is not None) diff --git a/tfts/tasks/pipeline.py b/tfts/tasks/pipeline.py new file mode 100644 index 00000000..3175301a --- /dev/null +++ b/tfts/tasks/pipeline.py @@ -0,0 +1,144 @@ +import logging +import os +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import tensorflow as tf + +from ..models import AutoConfig, AutoModel + +logger = logging.getLogger(__name__) + + +class Pipeline(object): + _load_processor = False + + def __init__(self, cfg, processor: Optional[Callable] = None, strategy: Optional[tf.distribute.Strategy] = None): + self.cfg = cfg + self.backbone = None + self.model = None + self.label_scaler = None + self.processor = processor + self.strategy = strategy or self._setup_strategy() + + def _setup_strategy(self): + """Detects GPUs and returns the appropriate distribution strategy.""" + gpus = tf.config.list_physical_devices("GPU") + if len(gpus) > 1: + logger.info(f"Using MirroredStrategy with {len(gpus)} GPUs") + return tf.distribute.MirroredStrategy() + elif len(gpus) == 1: + logger.info("Using OneDeviceStrategy (1 GPU)") + return tf.distribute.OneDeviceStrategy(device="/gpu:0") + else: + logger.info("Using default strategy (CPU)") + return tf.distribute.get_strategy() + + def build_model(self, n_features, n_outputs): + # Update model config with actual data dimensions + with self.strategy.scope(): + self.cfg.model.n_features = n_features + self.cfg.model.n_outputs = n_outputs + + config = AutoConfig()(self.cfg.model.name) + config.output_size = n_outputs + + self._model = AutoModel.from_config( + config=config, predict_sequence_length=self.cfg.model.predict_sequence_length + ) + + inputs = tf.keras.Input(shape=(self.cfg.model.train_sequence_length, self.cfg.model.n_features)) + outputs = self._model(inputs) + model = tf.keras.Model(inputs=inputs, outputs=outputs) + + loss_fn = getattr(tf.keras.losses, self.cfg.training.loss)() + + optimizer = getattr(tf.keras.optimizers, self.cfg.training.optimizer)( + learning_rate=self.cfg.training.learning_rate + ) + # metrics = [getattr(tf.keras.metrics, metric)() for metric in self.cfg.training.metrics] + model.compile(loss=loss_fn, optimizer=optimizer) # metrics=metrics + return model + + def train(self, train_dataset, eval_dataset=None, callbacks=None): + try: + sample_x, sample_y = next(iter(train_dataset)) + except (TypeError, StopIteration, AttributeError): + sample_x, sample_y = train_dataset[0] + + n_features = sample_x.shape[-1] + n_outputs = sample_y.shape[-1] + self.model = self.build_model(n_features, n_outputs) + + print(f"Training with {n_features} features and {n_outputs} outputs.") + if self.strategy.num_replicas_in_sync > 1: + print(f"Distributing training across {self.strategy.num_replicas_in_sync} replicas.") + print(self.model.summary()) + + history = self.model.fit( + train_dataset, validation_data=eval_dataset, epochs=self.cfg.training.epochs, callbacks=callbacks, verbose=1 + ) + return history + + def predict(self, test_dataset, weights_path=None): + # Ensure model is built and loaded if not already trained in this session + if self.model is None: + # Need to get n_features and n_outputs from the test data itself + sample_x, id = test_dataset[0] + n_features = sample_x.shape[-1] + n_outputs = self.cfg.model.n_outputs # Use configured output if model wasn't trained + self.model = self.build_model(n_features, n_outputs) + if weights_path and os.path.exists(weights_path): + self.model.load_weights(weights_path) + + return self.model.predict(test_dataset) + + def generate( + self, initial_sequence: Union[np.ndarray, tf.Tensor], horizon: int, extra_features_fn: Optional[Callable] = None + ): + """ + Auto-regressive generation for multi-step time series. + """ + if self.model is None: + raise ValueError("Model must be trained or loaded before generation.") + + current_window = tf.convert_to_tensor(initial_sequence, dtype=tf.float32) + batch_size = tf.shape(current_window)[0] + train_len = self.cfg.model.train_sequence_length + output_len = self.cfg.model.predict_sequence_length + n_features = tf.shape(current_window)[-1] + + all_predictions = [] + steps_needed = int(np.ceil(horizon / output_len)) + + print(f"Generating {horizon} steps in {steps_needed} blocks...") + + for step in range(steps_needed): + model_input = current_window[:, -train_len:, :] + + # 2. Predict next chunk (Batch, Output_Len) + preds = self.model(model_input, training=False) + + if len(preds.shape) == 2: + preds = tf.expand_dims(preds, axis=-1) + + # 3. Prepare features for the predicted chunk (Recursive Step) + # If your model uses (Value, Mask), predicted values get Mask=0 + if n_features > 1: + # Create a zero-mask for the new predictions + # Adjust this logic if you have other features like 'Day of Year' + padding_features = tf.zeros((batch_size, output_len, n_features - 1)) + new_chunk = tf.concat([preds, padding_features], axis=-1) + else: + new_chunk = preds + + # 4. Concatenate back to context + current_window = tf.concat([current_window, new_chunk], axis=1) + all_predictions.append(preds) + + # 5. Post-process the full sequence, Concatenate chunks and trim to exact horizon + full_forecast = tf.concat(all_predictions, axis=1) + return full_forecast[:, :horizon, :] + + def postprocess(self): + pass diff --git a/tfts/trainer.py b/tfts/trainer.py index 262f380d..e4f13c88 100644 --- a/tfts/trainer.py +++ b/tfts/trainer.py @@ -4,6 +4,7 @@ from contextlib import nullcontext import logging import os +import random from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union import numpy as np @@ -15,12 +16,19 @@ from .models.base import BaseModel from .training_args import TrainingArguments -__all__ = ["Trainer", "KerasTrainer", "Seq2seqKerasTrainer"] +__all__ = ["Trainer", "KerasTrainer", "Seq2seqKerasTrainer", "set_seed"] logger = logging.getLogger(__name__) +def set_seed(seed): + random.seed(seed) + np.random.seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + tf.random.set_seed(seed) + + class BaseTrainer(object): """Trainer for pipeline""" @@ -34,7 +42,7 @@ def __init__( self.model = model self.config = model.config if hasattr(model, "config") else None self.args = args or TrainingArguments(output_dir=TFTS_HUB_CACHE) - self.strategy = strategy + self.strategy = strategy or tf.distribute.get_strategy() # with self.get_strategy_scope(strategy): # self.model = self._setup_model(model) @@ -66,6 +74,15 @@ def get_learning_rates(self): def create_accelerator_and_postprocess(self): return + def get_distribution_strategy(): + gpus = tf.config.list_physical_devices("GPU") + if len(gpus) > 1: + return tf.distribute.MirroredStrategy() + elif len(gpus) == 1: + return tf.distribute.OneDeviceStrategy(device="/gpu:0") + else: + return tf.distribute.OneDeviceStrategy(device="/cpu:0") + def get_strategy_scope(self): return self.strategy.scope() if self.strategy else nullcontext() @@ -94,6 +111,7 @@ def _setup_mixed_precision(self) -> None: """Configure mixed precision training.""" policy = tf.keras.mixed_precision.Policy("mixed_float16") tf.keras.mixed_precision.set_global_policy(policy) + logger.info("Mixed precision enabled.") # def _setup_ema(self) -> None: # """Configure Exponential Moving Average if enabled.""" @@ -166,6 +184,10 @@ def _save(self, output_dir: Optional[str] = None): logging.error(f"Failed to save model weights to {weights_file}: {e}") return + @property + def global_batch_size(self): + return self.args.per_device_train_batch_size * self.strategy.num_replicas_in_sync + class KerasTrainer(BaseTrainer): """Keras trainer from tf.keras""" @@ -285,6 +307,9 @@ def get_model(self) -> tf.keras.Model: def save_model(self, output_dir: Optional[str] = None): # save the model, checkpoint_dir if you use Checkpoint callback to save your best weights + if self.strategy.cluster_resolver and not self.strategy.cluster_resolver.task_type == "chief": + return + output_dir = TFTS_HOME if output_dir is None else output_dir self._save(output_dir) @@ -319,7 +344,7 @@ def __init__( **kwargs: Dict[str, Any], ) -> None: self.model = model - self.strategy = strategy + self.strategy = strategy or tf.distribute.get_strategy() for key, value in kwargs.items(): setattr(self, key, value)