diff --git a/pydentification/data/__init__.py b/pydentification/data/__init__.py index e546a5b..547109c 100644 --- a/pydentification/data/__init__.py +++ b/pydentification/data/__init__.py @@ -1,11 +1,7 @@ -from .process import decay, unbatch from .sequences import generate_time_series_windows from .splits import compute_n_validation_samples, draw_validation_indices, time_series_train_test_split __all__ = [ - # torch utils - "decay", - "unbatch", # windowing utils "generate_time_series_windows", # train-test-validation splitting utils diff --git a/pydentification/data/process.py b/pydentification/data/process.py deleted file mode 100644 index c7bd504..0000000 --- a/pydentification/data/process.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import Iterable, Sequence - -import torch -from torch import Tensor - - -def decay(x: Tensor, gamma: float) -> Tensor: - """Apply exponential decay to batched time-series data.""" - n_time_steps = x.size(-1) # get number of time steps from input tensor - factors = torch.exp(-gamma * torch.arange(n_time_steps, dtype=x.dtype, device=x.device)) - return x * factors - - -def unbatch(batched: Iterable[tuple[Tensor, ...]]) -> tuple[Tensor, ...]: - """ - Converts batched dataset given as iterable (usually lazy iterable) to tuple of tensors - - :example: - >>> loader: DataLoader = get_loader(batch_size=32) # assume get_loader is implemented - >>> x, y = unbatch(loader) - >>> x.shape - ... (320, 10, 1) # (BATCH_SIZE * N_BATCHES, *DATA_SHAPE) - """ - for batch_idx, batch in enumerate(batched): - if batch_idx == 0: # initialize unbatched list of first batch - n_tensors = len(batch) if isinstance(batch, Sequence) else 1 - unbatched = [Tensor() for _ in range(n_tensors)] - - for i, tensor in enumerate(batch): - unbatched[i] = torch.cat([unbatched[i], tensor]) - - return tuple(unbatched) diff --git a/pydentification/models/README.md b/pydentification/models/README.md index a27f139..1855eaa 100644 --- a/pydentification/models/README.md +++ b/pydentification/models/README.md @@ -7,4 +7,3 @@ stacking. This package contains following submodules: * `modules` - building blocks of the models, extending PyTorch modules with custom layers, activation functions, etc. * `networks` - models composed of building blocks, such as transformer etc. -* `nonparametric` - non-parametric models and supporting utils diff --git a/pydentification/models/modules/README.md b/pydentification/models/modules/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/pydentification/models/modules/activations/__init__.py b/pydentification/models/modules/activations/__init__.py index b927060..dac7f21 100644 --- a/pydentification/models/modules/activations/__init__.py +++ b/pydentification/models/modules/activations/__init__.py @@ -1,9 +1,7 @@ -from .functional import bounded_linear_unit, universal_activation -from .modules import BoundedLinearUnit, UniversalActivation +from .functional import universal_activation +from .modules import UniversalActivation __all__ = [ - "bounded_linear_unit", - "BoundedLinearUnit", "universal_activation", "UniversalActivation", ] diff --git a/pydentification/models/modules/activations/functional.py b/pydentification/models/modules/activations/functional.py index 7fbf810..c240ed0 100644 --- a/pydentification/models/modules/activations/functional.py +++ b/pydentification/models/modules/activations/functional.py @@ -2,15 +2,6 @@ from torch import Tensor -def bounded_linear_unit(inputs: Tensor, lower: float | Tensor, upper: float | Tensor, inplace: bool = False) -> Tensor: - """ - Bounded linear activation function. It means that the output is linear in range [lower, upper] and clamped - outside of it to the values of the bounds. Bounds can be scalar of tensor of the same shape as inputs. - """ - out = inputs if inplace else None - return torch.clamp(inputs, min=lower, max=upper, out=out) - - def universal_activation(inputs: Tensor, inplace: bool = False) -> Tensor: """ Universal activation function, for which finite number of neurons can approximate any continuous function on diff --git a/pydentification/models/modules/activations/modules.py b/pydentification/models/modules/activations/modules.py index 0622f60..ceee3e6 100644 --- a/pydentification/models/modules/activations/modules.py +++ b/pydentification/models/modules/activations/modules.py @@ -4,33 +4,6 @@ from . import functional as func -class BoundedLinearUnit(Module): - """ - Bounded linear activation function. It means that the output is linear in range [-bounds, bounds] and clamped - outside of it to the values of the bounds. Bounds can be scalar of tensor of the same shape as inputs. - """ - - def __init__( - self, - lower: float | Tensor | None = None, - upper: float | Tensor | None = None, - ): - """ - Bounds given in __init__ are static, applied irrespective of the input bounds - they can be scalar or tensor of the same shape as inputs - """ - super(BoundedLinearUnit, self).__init__() - - self.static_lower_bound = lower - self.static_upper_bound = upper - - def forward(self, inputs: Tensor, bounds: float | Tensor | None = None) -> Tensor: - lower = bounds if self.static_lower_bound is None else self.static_lower_bound - upper = bounds if self.static_upper_bound is None else self.static_upper_bound - - return func.bounded_linear_unit(inputs, lower=lower, upper=upper) - - class UniversalActivation(Module): """ Universal activation function, for which finite number of neurons can approximate any continuous function on diff --git a/pydentification/models/modules/losses/__init__.py b/pydentification/models/modules/losses/__init__.py deleted file mode 100644 index b628c1a..0000000 --- a/pydentification/models/modules/losses/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .modules import BoundedMSELoss # noqa: F401 diff --git a/pydentification/models/modules/losses/modules.py b/pydentification/models/modules/losses/modules.py deleted file mode 100644 index 5e4e6e0..0000000 --- a/pydentification/models/modules/losses/modules.py +++ /dev/null @@ -1,35 +0,0 @@ -import torch -from torch import Tensor -from torch.nn import Module -from torch.nn import functional as F - - -class BoundedMSELoss(Module): - """MSE loss with penalty for crossing bounds of the nonparametric estimator""" - - def __init__(self, gamma: float): - """ - :param gamma: penalty factor for crossing bounds, defaults to 0.0 - """ - self.gamma = gamma - - super(BoundedMSELoss, self).__init__() - - def forward( - self, y_true: Tensor, y_pred: Tensor, lower: Tensor | None = None, upper: Tensor | None = None - ) -> float: - loss = F.mse_loss(y_pred, y_true, reduction="mean") - - # if gamma is given as 0 or if bounds are not given, - # no penalty is applied and BoundedMSELoss is equivalent to MSE, for example when used as validation loss - if self.gamma == 0 or lower is None or upper is None: - return loss - - penalty = torch.where( - (y_pred < lower) | (y_pred > upper), # find predictions outside of bounds - torch.min(torch.abs(y_pred - upper), torch.abs(y_pred - lower)), # calculate distance to closer bound - torch.tensor(0.0, device=y_pred.device), # zero-fill for predictions inside bounds - ) - - # returns loss as sum of MSE and cumulated penalty for crossing bounds with gamma factor - return loss + self.gamma * torch.sum(penalty) diff --git a/pydentification/models/networks/transformer/feedforward.py b/pydentification/models/networks/transformer/feedforward.py index 9c94173..7917c7e 100644 --- a/pydentification/models/networks/transformer/feedforward.py +++ b/pydentification/models/networks/transformer/feedforward.py @@ -49,7 +49,7 @@ def forward(self, inputs: Tensor) -> Tensor: class CausalDelayLineFeedforward(nn.Module): """ Linear transformation of the input signal using delay line with casual mask, allowing only connections from future - to past time steps, which means single row of weight matrix is multiplied with each time step of the input signal. + to past-time steps, which means single row of weight matrix is multiplied with each time step of the input signal. For MIMO systems, different weight matrix is applied for each dimension, where each of the matrices is stored in MaskedLinear module in torch.nn.ModuleList. diff --git a/pyproject.toml b/pyproject.toml index e0ea0c0..b1e659c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ include = ["pydentification*"] [project] name = "pydentification" -version = "0.4.1" +version = "0.5.0" description = "Toolbox for dynamical system identification with neural networks" license = {text = "MIT"} readme = {file = "README.md", content-type = "text/markdown"} diff --git a/tests/test_data/test_process.py b/tests/test_data/test_process.py deleted file mode 100644 index 08f8e1a..0000000 --- a/tests/test_data/test_process.py +++ /dev/null @@ -1,57 +0,0 @@ -import math - -import pytest -import torch -from torch import Tensor - -from pydentification.data import decay, unbatch -from tests.test_data.utils import tensor_batch_iterable - - -@pytest.mark.parametrize( - ["x", "gamma", "expected"], - [ - # decay with gamma=0 should return original tensor - (torch.ones(10), float(0), torch.ones(10)), - # decay with gamma=0 applied to N-D tensor should return original tensor - (torch.ones((10, 1)), float(0), torch.ones((10, 1))), - # decay with gamma=1 should return elements of tensor multiplied by 1/e in each step - (torch.ones(5), float(1), torch.Tensor([1, 1 / math.e, 1 / math.e**2, 1 / math.e**3, 1 / math.e**4])), - # decay with gamma=1 applied to N-D tensor should return elements of tensor multiplied by 1/e in each step - ( - torch.ones((1, 5)), - float(1), - torch.Tensor( - [ - [1, 1 / math.e, 1 / math.e**2, 1 / math.e**3, 1 / math.e**4], - ] - ), - ), - ], -) -def test_decay(x: Tensor, gamma: float, expected: Tensor): - assert torch.allclose(decay(x, gamma=gamma), expected) - - -@pytest.mark.parametrize( - ["n_batches", "batch_size", "batch_shape", "n_tensors", "expected_shape"], - [ - # single batch with 32 items with shape (10, 1) - (1, 32, (10, 1), 1, (32, 10, 1)), - # single batch with 32 items with shape (10,) for features and targets - (1, 32, (10,), 2, (32, 10)), - # 10 batches with 32 items with shape (10, 1) - (10, 32, (10, 1), 1, (320, 10, 1)), - # 10 batches with 32 items with shape (10,) for features and targets - (10, 32, (10,), 2, (320, 10)), - # single item batches - (10, 1, (10, 1), 1, (10, 10, 1)), - # 4-tuple dataloader - (10, 32, (10, 1), 4, (320, 10, 1)), - ], -) -def test_unbatch(n_batches: int, batch_size: int, batch_shape: tuple, n_tensors: int, expected_shape: tuple): - iterable = tensor_batch_iterable(n_batches, batch_size, batch_shape, n_tensors) - - for tensor in unbatch(iterable): - assert tensor.shape == expected_shape # in the test all input tensors have the same shape diff --git a/tests/test_models/test_losses.py b/tests/test_models/test_losses.py deleted file mode 100644 index 29e2693..0000000 --- a/tests/test_models/test_losses.py +++ /dev/null @@ -1,34 +0,0 @@ -import pytest -import torch -from torch import Tensor - -from pydentification.models.modules.losses import BoundedMSELoss - - -@pytest.mark.parametrize( - ["y_true", "y_pred", "lower", "upper", "gamma", "expected"], - ( - # perfect prediction - (torch.ones(5), torch.ones(5), torch.zeros(5), 5 * torch.ones(5), 1.0, 0.0), - # loss falls back to MSE if gamma is 0, in this case error is 1.0 - (torch.ones(5), torch.Tensor([2, 2, 2, 2, 2]), torch.zeros(5), 5 * torch.ones(5), 1.0, 1.0), - # loss is 0 if prediction is inside bounds - (torch.ones(5), torch.Tensor([2, 2, 2, 2, 2]), torch.zeros(5), 5 * torch.ones(5), 1.0, 1.0), - # loss is 1 + 0.5 for crossing bounds in the last element - (torch.ones(5), torch.Tensor([2, 2, 2, 2, 2]), torch.zeros(5), torch.Tensor([5, 5, 5, 5, 1.5]), 1.0, 1.5), - # loss is 1 + 5 for crossing bounds in each element (for test purpose true value and bound are equal) - (torch.ones(5), torch.Tensor([2, 2, 2, 2, 2]), torch.zeros(5), torch.ones(5), 1.0, 6.0), - # loss is 1 + 2 * 0.5 (gamma = 2) for crossing bounds in the last element - (torch.ones(5), torch.Tensor([2, 2, 2, 2, 2]), torch.zeros(5), torch.Tensor([5, 5, 5, 5, 1.5]), 2.0, 2.0), - # loss is 1 + 0.5 * 0.5 (gamma = 1/2) for crossing bounds in the last element - (torch.ones(5), torch.Tensor([2, 2, 2, 2, 2]), torch.zeros(5), torch.Tensor([5, 5, 5, 5, 1.5]), 0.5, 1.25), - # gamma = 0 so loss falls back to regular MSE - (torch.ones(5), torch.Tensor([2, 2, 2, 2, 2]), torch.zeros(5), torch.Tensor([5, 5, 5, 5, 1.5]), 0.0, 1.0), - ), -) -def test_bounded_mse_loss(y_true: Tensor, y_pred: Tensor, lower: Tensor, upper: Tensor, gamma: float, expected: float): - loss = BoundedMSELoss(gamma) - - with torch.no_grad(): - result = loss(y_true, y_pred, lower, upper) # type: ignore - assert result.item() == expected diff --git a/tests/test_models/test_modules/test_activation.py b/tests/test_models/test_modules/test_activation.py deleted file mode 100644 index 5c871b8..0000000 --- a/tests/test_models/test_modules/test_activation.py +++ /dev/null @@ -1,58 +0,0 @@ -import pytest -import torch -from torch import Tensor - -from pydentification.models.modules.activations import bounded_linear_unit - - -@pytest.mark.parametrize( - ["inputs", "lower", "upper", "expected"], - [ - # all elements in scalar bounds - output is the same as input - (torch.ones(5), -1, 2, torch.ones(5)), - # all elements outside upper scalar bound - output is upper bound - (torch.ones(5), -1, 0, torch.zeros(5)), - # all elements outside lower scalar bound - output is lower bound - (torch.ones(5), 2, 3, 2 * torch.ones(5)), - # all elements in varying bounds - output is the same as input - (torch.ones(5), torch.Tensor([0, -1, 0, -1, 0]), torch.Tensor([2, 3, 5, 6, 7]), torch.ones(5)), - # all elements outside upper varying bounds - output is upper bound with every element different - ( - 10 * torch.ones(5), - torch.Tensor([0, -1, 0, -1, 0]), - torch.Tensor([0, 2, 5, 6, 7]), - torch.Tensor([0, 2, 5, 6, 7]), - ), - # all elements outside lower varying bounds - output is lower bound with every element different - ( - -10 * torch.ones(5), - torch.Tensor([0, -1, 0, -1, 0]), - torch.Tensor([2, 3, 5, 6, 1]), - torch.Tensor([0, -1, 0, -1, 0]), - ), - # some elements outside upper varying bounds - output contains elements from input and upper bound - ( - torch.Tensor([1, 2, 3, 4, 5]), - torch.Tensor([0, 0, 0, 0, 0]), - torch.Tensor([2, 1, 2, 1, 2]), - torch.Tensor([1, 1, 2, 1, 2]), - ), - # some elements outside lower varying bounds - output contains elements from input and lower bound - ( - torch.Tensor([1, 2, 3, 4, 5]), - torch.Tensor([0, 5, 0, 5, 0]), - torch.Tensor([10, 10, 10, 10, 10]), - torch.Tensor([1, 5, 3, 5, 5]), - ), - # some elements outside upper and lower varying bounds - output contains elements from input and both bounds - ( - torch.Tensor([1, 2, 3, 4, 5]), - torch.Tensor([5, 0, 5, 0, 0]), - torch.Tensor([10, 10, 10, 3, 3]), - torch.Tensor([5, 2, 5, 3, 3]), - ), - ], -) -def test_bounded_linear_unit(inputs: Tensor, lower: float | Tensor, upper: float | Tensor, expected: Tensor): - outputs = bounded_linear_unit(inputs, lower, upper) - assert torch.allclose(outputs, expected)