diff --git a/docs/api/cells.rst b/docs/api/cells.rst index 6402afd..a664297 100755 --- a/docs/api/cells.rst +++ b/docs/api/cells.rst @@ -23,6 +23,7 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.LightRUCell torchrecurrent.LiGRUCell torchrecurrent.MGUCell + torchrecurrent.MinimalRNNCell torchrecurrent.MiRU1Cell torchrecurrent.MiRU2Cell torchrecurrent.MultiplicativeLSTMCell diff --git a/docs/api/layers.rst b/docs/api/layers.rst index 2e6487c..1cf9623 100644 --- a/docs/api/layers.rst +++ b/docs/api/layers.rst @@ -23,6 +23,7 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.LightRU torchrecurrent.LiGRU torchrecurrent.MGU + torchrecurrent.MinimalRNN torchrecurrent.MiRU1 torchrecurrent.MiRU2 torchrecurrent.MultiplicativeLSTM diff --git a/docs/generated/torchrecurrent.MinimalRNN.rst b/docs/generated/torchrecurrent.MinimalRNN.rst new file mode 100644 index 0000000..aefcfb8 --- /dev/null +++ b/docs/generated/torchrecurrent.MinimalRNN.rst @@ -0,0 +1,86 @@ +torchrecurrent.MinimalRNN +========================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: MinimalRNN + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MinimalRNN.__init__ + ~MinimalRNN.add_module + ~MinimalRNN.apply + ~MinimalRNN.bfloat16 + ~MinimalRNN.buffers + ~MinimalRNN.children + ~MinimalRNN.compile + ~MinimalRNN.cpu + ~MinimalRNN.cuda + ~MinimalRNN.double + ~MinimalRNN.eval + ~MinimalRNN.extra_repr + ~MinimalRNN.float + ~MinimalRNN.forward + ~MinimalRNN.get_buffer + ~MinimalRNN.get_extra_state + ~MinimalRNN.get_parameter + ~MinimalRNN.get_submodule + ~MinimalRNN.half + ~MinimalRNN.initialize_cells + ~MinimalRNN.ipu + ~MinimalRNN.load_state_dict + ~MinimalRNN.modules + ~MinimalRNN.mtia + ~MinimalRNN.named_buffers + ~MinimalRNN.named_children + ~MinimalRNN.named_modules + ~MinimalRNN.named_parameters + ~MinimalRNN.parameters + ~MinimalRNN.register_backward_hook + ~MinimalRNN.register_buffer + ~MinimalRNN.register_forward_hook + ~MinimalRNN.register_forward_pre_hook + ~MinimalRNN.register_full_backward_hook + ~MinimalRNN.register_full_backward_pre_hook + ~MinimalRNN.register_load_state_dict_post_hook + ~MinimalRNN.register_load_state_dict_pre_hook + ~MinimalRNN.register_module + ~MinimalRNN.register_parameter + ~MinimalRNN.register_state_dict_post_hook + ~MinimalRNN.register_state_dict_pre_hook + ~MinimalRNN.requires_grad_ + ~MinimalRNN.set_extra_state + ~MinimalRNN.set_submodule + ~MinimalRNN.share_memory + ~MinimalRNN.state_dict + ~MinimalRNN.to + ~MinimalRNN.to_empty + ~MinimalRNN.train + ~MinimalRNN.type + ~MinimalRNN.xpu + ~MinimalRNN.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MinimalRNN.T_destination + ~MinimalRNN.call_super_init + ~MinimalRNN.dump_patches + ~MinimalRNN.input_size + ~MinimalRNN.hidden_size + ~MinimalRNN.bias + ~MinimalRNN.dropout + ~MinimalRNN.batch_first + ~MinimalRNN.training + diff --git a/docs/generated/torchrecurrent.MinimalRNNCell.rst b/docs/generated/torchrecurrent.MinimalRNNCell.rst new file mode 100644 index 0000000..fce9ad2 --- /dev/null +++ b/docs/generated/torchrecurrent.MinimalRNNCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.MinimalRNNCell +============================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: MinimalRNNCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MinimalRNNCell.__init__ + ~MinimalRNNCell.add_module + ~MinimalRNNCell.apply + ~MinimalRNNCell.bfloat16 + ~MinimalRNNCell.buffers + ~MinimalRNNCell.children + ~MinimalRNNCell.compile + ~MinimalRNNCell.cpu + ~MinimalRNNCell.cuda + ~MinimalRNNCell.double + ~MinimalRNNCell.eval + ~MinimalRNNCell.extra_repr + ~MinimalRNNCell.float + ~MinimalRNNCell.forward + ~MinimalRNNCell.get_buffer + ~MinimalRNNCell.get_extra_state + ~MinimalRNNCell.get_parameter + ~MinimalRNNCell.get_submodule + ~MinimalRNNCell.half + ~MinimalRNNCell.ipu + ~MinimalRNNCell.load_state_dict + ~MinimalRNNCell.modules + ~MinimalRNNCell.mtia + ~MinimalRNNCell.named_buffers + ~MinimalRNNCell.named_children + ~MinimalRNNCell.named_modules + ~MinimalRNNCell.named_parameters + ~MinimalRNNCell.parameters + ~MinimalRNNCell.register_backward_hook + ~MinimalRNNCell.register_buffer + ~MinimalRNNCell.register_forward_hook + ~MinimalRNNCell.register_forward_pre_hook + ~MinimalRNNCell.register_full_backward_hook + ~MinimalRNNCell.register_full_backward_pre_hook + ~MinimalRNNCell.register_load_state_dict_post_hook + ~MinimalRNNCell.register_load_state_dict_pre_hook + ~MinimalRNNCell.register_module + ~MinimalRNNCell.register_parameter + ~MinimalRNNCell.register_state_dict_post_hook + ~MinimalRNNCell.register_state_dict_pre_hook + ~MinimalRNNCell.requires_grad_ + ~MinimalRNNCell.reset_parameters + ~MinimalRNNCell.set_extra_state + ~MinimalRNNCell.set_submodule + ~MinimalRNNCell.share_memory + ~MinimalRNNCell.state_dict + ~MinimalRNNCell.to + ~MinimalRNNCell.to_empty + ~MinimalRNNCell.train + ~MinimalRNNCell.type + ~MinimalRNNCell.uses_double_state + ~MinimalRNNCell.xpu + ~MinimalRNNCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MinimalRNNCell.T_destination + ~MinimalRNNCell.call_super_init + ~MinimalRNNCell.dump_patches + ~MinimalRNNCell.weight_ih + ~MinimalRNNCell.weight_hh + ~MinimalRNNCell.weight_mm + ~MinimalRNNCell.bias_ih + ~MinimalRNNCell.bias_hh + ~MinimalRNNCell.input_size + ~MinimalRNNCell.hidden_size + ~MinimalRNNCell.bias + ~MinimalRNNCell.recurrent_bias + ~MinimalRNNCell.training + diff --git a/docs/models.rst b/docs/models.rst index 0a3daaa..1e00a46 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -62,6 +62,9 @@ references and official implementations where available. * - :doc:`MGU ` - `IJAC 2016 `__ - – + * - :doc:`MinimalRNN ` + - `NeurIPS 2017 Workshop `__ + - – * - :doc:`MiRU1 ` - `Neurocomputing 2026 `__ - – diff --git a/pyproject.toml b/pyproject.toml index 2a5687f..77f0b3c 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "torchrecurrent" -version = "0.2.3" +version = "0.2.4" description = "A package for recurrent neural networks in PyTorch" readme = "README.md" authors = [ diff --git a/tests/test_cells.py b/tests/test_cells.py index 2c8ab9c..5425f50 100755 --- a/tests/test_cells.py +++ b/tests/test_cells.py @@ -4,10 +4,6 @@ import torch from torch import Tensor -skip_windows = pytest.mark.skipif( - sys.platform == "win32", - reason="torch.compile requires Triton, which is not supported on Windows", -) from torchrecurrent import ( AntisymmetricRNNCell, ATRCell, @@ -22,6 +18,7 @@ LEMCell, GatedAntisymmetricRNNCell, MGUCell, + MinimalRNNCell, IndRNNCell, LiGRUCell, LightRUCell, @@ -44,6 +41,11 @@ WMCLSTMCell, ) +skip_windows = pytest.mark.skipif( + sys.platform == "win32", + reason="torch.compile requires Triton, which is not supported on Windows", +) + CELL_CASES = [ # (CellClass, input_size, hidden_size, uses_double_state) (AntisymmetricRNNCell, 3, 5, False), @@ -59,6 +61,7 @@ (JANETCell, 3, 5, True), (LEMCell, 3, 5, True), (MGUCell, 4, 8, False), + (MinimalRNNCell, 3, 5, False), (IndRNNCell, 3, 5, False), (LiGRUCell, 6, 12, False), (LightRUCell, 3, 5, False), @@ -124,6 +127,13 @@ def test_cell_output_and_state_shapes(Cell, in_size, hid_size, double): assert h3.shape == (B, hid_size) +@pytest.mark.parametrize("Cell, in_size, hid_size, double", CELL_CASES) +def test_cell_reset_parameters_after_construction(Cell, in_size, hid_size, double): + """reset_parameters() must remain callable after __init__, not just during it.""" + cell = Cell(in_size, hid_size) + cell.reset_parameters() + + def test_reslstm_cell_parameter_shapes(): cell = ResLSTMCell(4, 9) diff --git a/tests/test_layers.py b/tests/test_layers.py index 7935257..04645ab 100755 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -17,6 +17,7 @@ LiGRU, LightRU, MGU, + MinimalRNN, MultiplicativeLSTM, MUT1, MUT2, @@ -51,6 +52,7 @@ LiGRU, LightRU, MGU, + MinimalRNN, MultiplicativeLSTM, MUT1, MUT2, @@ -86,6 +88,7 @@ (LiGRU, False), (LightRU, False), (MGU, False), + (MinimalRNN, False), (MultiplicativeLSTM, True), (MUT1, False), (MUT2, False), diff --git a/torchrecurrent/__init__.py b/torchrecurrent/__init__.py index eecd4e5..7d2925f 100755 --- a/torchrecurrent/__init__.py +++ b/torchrecurrent/__init__.py @@ -17,6 +17,7 @@ LiGRUCell, LightRUCell, MGUCell, + MinimalRNNCell, MiRU1Cell, MiRU2Cell, MultiplicativeLSTMCell, @@ -56,6 +57,7 @@ LiGRU, LightRU, MGU, + MinimalRNN, MiRU1, MiRU2, MultiplicativeLSTM, @@ -95,6 +97,8 @@ "DSGUCell", "MGU", "MGUCell", + "MinimalRNN", + "MinimalRNNCell", "MiRU1", "MiRU1Cell", "MiRU2", diff --git a/torchrecurrent/base.py b/torchrecurrent/base.py index d27b322..824856a 100755 --- a/torchrecurrent/base.py +++ b/torchrecurrent/base.py @@ -2,7 +2,6 @@ from __future__ import annotations -import warnings from typing import Dict, Optional, Tuple, List, Any import torch @@ -236,9 +235,10 @@ def reset_parameters(self) -> None: apply_init_(p, self.init_cfg["recurrent_bias"]) def _cleanup_non_scriptable(self) -> None: + # init_cfg is kept: reset_parameters() must remain callable after + # construction, and a Dict[str, str] is TorchScript-compatible. del self._init_device del self._init_dtype - del self.init_cfg class SingleStateCellBase(RecurrentCellBase): diff --git a/torchrecurrent/cells/__init__.py b/torchrecurrent/cells/__init__.py index 7d0aa18..ad1a6bb 100755 --- a/torchrecurrent/cells/__init__.py +++ b/torchrecurrent/cells/__init__.py @@ -10,6 +10,7 @@ from .br_cell import BR, BRCell, NBR, NBRCell from .cfn_cell import CFN, CFNCell from .mgu_cell import MGU, MGUCell +from .minimalrnn_cell import MinimalRNN, MinimalRNNCell from .miru_cell import MiRU1, MiRU1Cell, MiRU2, MiRU2Cell from .cornn_cell import coRNN, coRNNCell from .fastrnn_cell import FastRNN, FastRNNCell, FastGRNN, FastGRNNCell @@ -52,6 +53,8 @@ "CFNCell", "MGU", "MGUCell", + "MinimalRNN", + "MinimalRNNCell", "MiRU1", "MiRU1Cell", "MiRU2", diff --git a/torchrecurrent/cells/antisymmetricrnn_cell.py b/torchrecurrent/cells/antisymmetricrnn_cell.py index 05e3fd9..3fb0d54 100755 --- a/torchrecurrent/cells/antisymmetricrnn_cell.py +++ b/torchrecurrent/cells/antisymmetricrnn_cell.py @@ -2,7 +2,13 @@ import torch import torch.nn as nn from torch import Tensor -from ..base import SingleStateCellBase, SingleStateRecurrentLayerBase, resolve_activation, resolve_init_name, apply_init_ +from ..base import ( + SingleStateCellBase, + SingleStateRecurrentLayerBase, + resolve_activation, + resolve_init_name, + apply_init_, +) class AntisymmetricRNN(SingleStateRecurrentLayerBase): @@ -62,14 +68,12 @@ class AntisymmetricRNN(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -84,14 +88,13 @@ class AntisymmetricRNN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the AntisymmetricRNN, for each - `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the - input, the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -109,9 +112,6 @@ class AntisymmetricRNN(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`AntisymmetricRNNCell` @@ -213,7 +213,14 @@ class AntisymmetricRNNCell(SingleStateCellBase): >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) """ - __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias", "epsilon", "gamma"] + __constants__ = [ + "input_size", + "hidden_size", + "bias", + "recurrent_bias", + "epsilon", + "gamma", + ] weight_ih: Tensor weight_hh: Tensor @@ -279,7 +286,12 @@ def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state recurrent_matrix = _compute_asym(self.weight_hh, self.gamma) - pre_act = b_inp @ self.weight_ih.t() + self.bias_ih + b_state @ recurrent_matrix.t() + self.bias_hh + pre_act = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + b_state @ recurrent_matrix.t() + + self.bias_hh + ) h_new = b_state + self.epsilon * self.act(pre_act) if not is_batched: @@ -348,14 +360,12 @@ class GatedAntisymmetricRNN(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -370,14 +380,13 @@ class GatedAntisymmetricRNN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the GatedAntisymmetricRNN, for each - `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the - input, the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -395,9 +404,6 @@ class GatedAntisymmetricRNN(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`GatedAntisymmetricRNNCell` @@ -506,7 +512,14 @@ class GatedAntisymmetricRNNCell(SingleStateCellBase): >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) """ - __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias", "epsilon", "gamma"] + __constants__ = [ + "input_size", + "hidden_size", + "bias", + "recurrent_bias", + "epsilon", + "gamma", + ] weight_ih: Tensor weight_hh: Tensor @@ -584,5 +597,5 @@ def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: def _compute_asym(W_hh: Tensor, gamma: float) -> Tensor: n = W_hh.size(0) - I = torch.eye(n, device=W_hh.device, dtype=W_hh.dtype) - return W_hh - W_hh.transpose(0, 1) - gamma * I + identity = torch.eye(n, device=W_hh.device, dtype=W_hh.dtype) + return W_hh - W_hh.transpose(0, 1) - gamma * identity diff --git a/torchrecurrent/cells/atr_cell.py b/torchrecurrent/cells/atr_cell.py index d2314d3..d26cf44 100755 --- a/torchrecurrent/cells/atr_cell.py +++ b/torchrecurrent/cells/atr_cell.py @@ -1,8 +1,12 @@ import torch from torch import Tensor import torch.nn as nn -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, +) class ATR(SingleStateRecurrentLayerBase): @@ -58,14 +62,12 @@ class ATR(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -80,14 +82,12 @@ class ATR(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the ATR, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the ATR, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -105,9 +105,6 @@ class ATR(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`ATRCell` diff --git a/torchrecurrent/cells/br_cell.py b/torchrecurrent/cells/br_cell.py index 39a4b83..eb995f8 100755 --- a/torchrecurrent/cells/br_cell.py +++ b/torchrecurrent/cells/br_cell.py @@ -1,8 +1,13 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, + apply_init_, +) class BR(SingleStateRecurrentLayerBase): @@ -60,14 +65,12 @@ class BR(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -82,14 +85,12 @@ class BR(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the BR, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the BR, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -107,9 +108,6 @@ class BR(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`BRCell` @@ -347,14 +345,12 @@ class NBR(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -369,14 +365,12 @@ class NBR(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the NBR, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the NBR, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -394,9 +388,6 @@ class NBR(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`NBRCell` diff --git a/torchrecurrent/cells/cfn_cell.py b/torchrecurrent/cells/cfn_cell.py index e1076a6..50e80a7 100755 --- a/torchrecurrent/cells/cfn_cell.py +++ b/torchrecurrent/cells/cfn_cell.py @@ -1,8 +1,12 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, +) class CFN(SingleStateRecurrentLayerBase): @@ -63,14 +67,12 @@ class CFN(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -85,14 +87,12 @@ class CFN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the CFN, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the CFN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -110,9 +110,6 @@ class CFN(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`CFNCell` diff --git a/torchrecurrent/cells/cornn_cell.py b/torchrecurrent/cells/cornn_cell.py index 162de18..eb047cc 100755 --- a/torchrecurrent/cells/cornn_cell.py +++ b/torchrecurrent/cells/cornn_cell.py @@ -75,18 +75,16 @@ class coRNN(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state for each element in the input sequence. Defaults to zeros if not provided. @@ -101,17 +99,15 @@ class coRNN(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the coRNN, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the coRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final cell + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -133,9 +129,6 @@ class coRNN(DoubleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`coRNNCell` diff --git a/torchrecurrent/cells/fastrnn_cell.py b/torchrecurrent/cells/fastrnn_cell.py index 06cd736..0f0b53d 100755 --- a/torchrecurrent/cells/fastrnn_cell.py +++ b/torchrecurrent/cells/fastrnn_cell.py @@ -1,7 +1,7 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple +from typing import Optional from ..base import ( SingleStateRecurrentLayerBase, SingleStateCellBase, @@ -67,14 +67,12 @@ class FastRNN(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -89,14 +87,12 @@ class FastRNN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the FastRNN, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the FastRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -120,9 +116,6 @@ class FastRNN(SingleStateRecurrentLayerBase): :math:`\alpha` and :math:`\beta` are initialized from `alpha_init` and `beta_init`. - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`FastRNNCell` @@ -378,11 +371,9 @@ class FastGRNN(SingleStateRecurrentLayerBase): dtype: Desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape `(L, H_in)` for unbatched input, - `(L, N, H_in)` when ``batch_first=False``, or `(N, L, H_in)` when - ``batch_first=True``. - - **h_0**: tensor of shape `(num_layers, H_out)` for unbatched input, - or `(num_layers, N, H_out)` when batched. Defaults to zeros. + - **input**: tensor of shape `(L, N, H_in)` when ``batch_first=False``, + or `(N, L, H_in)` when ``batch_first=True``. + - **h_0**: tensor of shape `(num_layers, N, H_out)`. Defaults to zeros. Where: @@ -395,11 +386,11 @@ class FastGRNN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape `(L, H_out)` for unbatched input, - `(L, N, H_out)` when ``batch_first=False``, or `(N, L, H_out)` when - ``batch_first=True``, containing hidden states from the last layer. - - **h_n**: tensor of shape `(num_layers, H_out)` (unbatched) or - `(num_layers, N, H_out)` with the final hidden state. + - **output**: tensor of shape `(L, N, H_out)` when ``batch_first=False``, + or `(N, L, H_out)` when ``batch_first=True``, containing hidden states + from the last layer. + - **h_n**: tensor of shape `(num_layers, N, H_out)` with the final + hidden state. Attributes: cells.{k}.weight_ih : input–hidden weights of the :math:`k`-th layer, diff --git a/torchrecurrent/cells/indrnn_cell.py b/torchrecurrent/cells/indrnn_cell.py index 99fae00..0aaf9b9 100755 --- a/torchrecurrent/cells/indrnn_cell.py +++ b/torchrecurrent/cells/indrnn_cell.py @@ -1,7 +1,6 @@ -import torch from torch import Tensor import torch.nn as nn -from typing import Optional, Tuple +from typing import Optional from ..base import ( SingleStateRecurrentLayerBase, SingleStateCellBase, @@ -58,14 +57,12 @@ class IndRNN(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -80,14 +77,12 @@ class IndRNN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the IndRNN, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the IndRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -103,9 +98,6 @@ class IndRNN(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`IndRNNCell` @@ -269,7 +261,10 @@ def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state h_new = ( - b_inp @ self.weight_ih.t() + self.bias_ih + self.vector_u * b_state + self.bias_hh + b_inp @ self.weight_ih.t() + + self.bias_ih + + self.vector_u * b_state + + self.bias_hh ) h_new = self.act(h_new) diff --git a/torchrecurrent/cells/janet_cell.py b/torchrecurrent/cells/janet_cell.py index 483539c..c6374ab 100755 --- a/torchrecurrent/cells/janet_cell.py +++ b/torchrecurrent/cells/janet_cell.py @@ -67,18 +67,16 @@ class JANET(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state for each element in the input sequence. Defaults to zeros if not provided. @@ -93,17 +91,15 @@ class JANET(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the JANET, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the JANET, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final cell + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -124,9 +120,6 @@ class JANET(DoubleStateRecurrentLayerBase): initializers (`kernel_init`, `recurrent_kernel_init`, etc.). The threshold shift :math:`\beta` is initialized from the given value of `beta`. - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`JANETCell` @@ -300,8 +293,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) gates = ( b_inp @ self.weight_ih.t() diff --git a/torchrecurrent/cells/lem_cell.py b/torchrecurrent/cells/lem_cell.py index 10588cb..c1faf77 100755 --- a/torchrecurrent/cells/lem_cell.py +++ b/torchrecurrent/cells/lem_cell.py @@ -79,18 +79,16 @@ class LEM(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state for each element in the input sequence. Defaults to zeros if not provided. @@ -105,17 +103,15 @@ class LEM(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the LEM, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the LEM, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final cell + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -138,9 +134,6 @@ class LEM(DoubleStateRecurrentLayerBase): initializers (`kernel_init`, `recurrent_kernel_init`, etc.). The integration step :math:`\Delta t` is scaled by the given parameter `dt`. - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`LEMCell` @@ -307,9 +300,7 @@ def __init__( self.init_cfg["recurrent_bias"] = resolve_init_name( recurrent_bias_init, self.init_cfg["recurrent_bias"] ) - self.init_cfg["cell_kernel"] = resolve_init_name( - cell_kernel_init, "xavier_uniform" - ) + self.init_cfg["cell_kernel"] = resolve_init_name(cell_kernel_init, "xavier_uniform") self.init_cfg["cell_bias"] = resolve_init_name(cell_bias_init, "zeros") self._register_tensors( @@ -347,8 +338,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) inp_expanded = b_inp @ self.weight_ih.t() + self.bias_ih state_expanded = b_h @ self.weight_hh.t() + self.bias_hh diff --git a/torchrecurrent/cells/lightru_cell.py b/torchrecurrent/cells/lightru_cell.py index 6f117b2..ca713f3 100755 --- a/torchrecurrent/cells/lightru_cell.py +++ b/torchrecurrent/cells/lightru_cell.py @@ -1,8 +1,12 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, +) class LightRU(SingleStateRecurrentLayerBase): @@ -57,14 +61,12 @@ class LightRU(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -79,14 +81,12 @@ class LightRU(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the LightRU, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the LightRU, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -104,9 +104,6 @@ class LightRU(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`LightRUCell` diff --git a/torchrecurrent/cells/ligru_cell.py b/torchrecurrent/cells/ligru_cell.py index dbd1599..155e75c 100755 --- a/torchrecurrent/cells/ligru_cell.py +++ b/torchrecurrent/cells/ligru_cell.py @@ -1,13 +1,12 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple +from typing import Optional from ..base import ( SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_activation, resolve_init_name, - apply_init_, ) @@ -69,14 +68,12 @@ class LiGRU(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -91,14 +88,12 @@ class LiGRU(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the LiGRU, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the LiGRU, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -116,9 +111,6 @@ class LiGRU(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`LiGRUCell` diff --git a/torchrecurrent/cells/mgu_cell.py b/torchrecurrent/cells/mgu_cell.py index f204b3d..58a7113 100755 --- a/torchrecurrent/cells/mgu_cell.py +++ b/torchrecurrent/cells/mgu_cell.py @@ -1,13 +1,12 @@ import torch import torch.nn as nn from torch import Tensor -from typing import Optional, Tuple +from typing import Optional from ..base import ( SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_activation, resolve_init_name, - apply_init_, ) @@ -70,14 +69,12 @@ class MGU(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -92,14 +89,12 @@ class MGU(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the MGU, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the MGU, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -117,9 +112,6 @@ class MGU(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`MGUCell` diff --git a/torchrecurrent/cells/minimalrnn_cell.py b/torchrecurrent/cells/minimalrnn_cell.py new file mode 100644 index 0000000..b0fe2dc --- /dev/null +++ b/torchrecurrent/cells/minimalrnn_cell.py @@ -0,0 +1,308 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_activation, + resolve_init_name, + apply_init_, +) + + +class MinimalRNN(SingleStateRecurrentLayerBase): + r"""Multi-layer minimal recurrent neural network. + + [`arXiv `_] + + Each layer consists of an :class:`MinimalRNNCell`, which updates the hidden + state according to: + + .. math:: + \begin{aligned} + z_t &= \phi(W_{ih} x_t + b_{ih}), \\ + u_t &= \sigma(W_{hh} h_{t-1} + W_{mm} z_t + b_{hh}), \\ + h_t &= u_t \circ h_{t-1} + (1 - u_t) \circ z_t + \end{aligned} + + where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the + input at time `t`, :math:`z_t` is the latent representation of the input, + :math:`\sigma` is the sigmoid function, :math:`\phi` is a pointwise + nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + multiplication. + + In a multilayer MinimalRNN, the input :math:`x^{(l)}_t` of the + :math:`l`-th layer (:math:`l \ge 2`) is the hidden state + :math:`h^{(l-1)}_t` of the previous layer multiplied by dropout + :math:`\delta^{(l-1)}_t`, where each :math:`\delta^{(l-1)}_t` is a + Bernoulli random variable which is 0 with probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two MinimalRNN layers, with the second receiving the outputs + of the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + nonlinearity: Nonlinearity :math:`\phi` for the input encoder. Default: + :func:`torch.tanh` + gate_nonlinearity: Activation for the update gate. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + memory_kernel_init: Initializer for `W_{mm}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for input-side biases. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for recurrent biases. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape + :math:`(L, N, H_{in})` when ``batch_first=False`` or + :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial + hidden state for each element in the input sequence. Defaults to zeros if + not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape + :math:`(L, N, H_{out})` when ``batch_first=False`` or + :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output + features `(h_t)` from the last layer of the MinimalRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final + hidden state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable hidden-hidden weights of the :math:`k`-th + layer, of shape `(hidden_size, hidden_size)`. + cells.{k}.weight_mm : the learnable latent-hidden weights of the :math:`k`-th + layer, of shape `(hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable hidden-hidden biases of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`MinimalRNNCell` + + Examples:: + + >>> rnn = MinimalRNN(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(MinimalRNN, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(MinimalRNNCell, **kwargs) + + +class MinimalRNNCell(SingleStateCellBase): + r"""A Minimal recurrent neural network (MinimalRNN) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{z}(t) &= \phi\bigl( + \mathbf{W}_{ih}\,\mathbf{x}(t) + \mathbf{b}_{ih} + \bigr), \\[6pt] + \mathbf{u}(t) &= \sigma\bigl( + \mathbf{W}_{hh}\,\mathbf{h}(t-1) + + \mathbf{W}_{mm}\,\mathbf{z}(t) + + \mathbf{b}_{hh} + \bigr), \\[6pt] + \mathbf{h}(t) &= \mathbf{u}(t)\circ\mathbf{h}(t-1) + \;+\;\bigl(1 - \mathbf{u}(t)\bigr)\circ\mathbf{z}(t), + + where :math:`\circ` is element-wise product, :math:`\phi` is a pointwise + nonlinearity (e.g., tanh) mapping the input into the latent space of the + hidden state, and :math:`\mathbf{z}(t)` is recomputed from the current + input at every step (it is not part of the recurrent state). + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use recurrent biases. + Default: ``True``. + nonlinearity: Nonlinearity :math:`\phi` for the input encoder. + Default: :func:`torch.tanh`. + gate_nonlinearity: Activation for the update gate. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + memory_kernel_init: Initializer for ``W_{mm}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for input-side biases when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for recurrent biases when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: h_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(hidden_size, input_size)``. + weight_hh: The learnable hidden-hidden weights, + of shape ``(hidden_size, hidden_size)``. + weight_mm: The learnable latent-hidden weights, + of shape ``(hidden_size, hidden_size)``. + bias_ih: The learnable input-hidden biases, + of shape ``(hidden_size)`` if ``bias=True``. + bias_hh: The learnable hidden-hidden biases, + of shape ``(hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = MinimalRNNCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h = cell(x[t], h) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + weight_mm: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + nonlinearity="tanh", + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + memory_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.act = resolve_activation(nonlinearity) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["memory_kernel"] = resolve_init_name( + memory_kernel_init, "xavier_uniform" + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._default_register_tensors(ih_mult=1, hh_mult=1) + self.weight_mm = nn.Parameter( + torch.empty( + self.hidden_size, + self.hidden_size, + device=self._init_device, + dtype=self._init_dtype, + ) + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def reset_parameters(self) -> None: + super().reset_parameters() + apply_init_(self.weight_mm, self.init_cfg["memory_kernel"]) + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + latent = self.act(b_inp @ self.weight_ih.t() + self.bias_ih) + update_gate = self.gate_act( + b_state @ self.weight_hh.t() + latent @ self.weight_mm.t() + self.bias_hh + ) + new_state = update_gate * b_state + (1.0 - update_gate) * latent + + if not is_batched: + new_state = new_state.squeeze(0) + + return new_state diff --git a/torchrecurrent/cells/miru_cell.py b/torchrecurrent/cells/miru_cell.py index 328c7d9..81c93e0 100644 --- a/torchrecurrent/cells/miru_cell.py +++ b/torchrecurrent/cells/miru_cell.py @@ -1,13 +1,12 @@ import torch import torch.nn as nn from torch import Tensor -from typing import Optional, Tuple +from typing import Optional from ..base import ( SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_activation, resolve_init_name, - apply_init_, ) @@ -60,19 +59,16 @@ class MiRU1(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True``. - - **h_0**: tensor of shape :math:`(\text{num\_layers}, H_{out})` for - unbatched input or :math:`(\text{num\_layers}, N, H_{out})`. + - **h_0**: tensor of shape :math:`(\text{num\_layers}, N, H_{out})`. Defaults to zeros if not provided. Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched - input, :math:`(L, N, H_{out})` when ``batch_first=False`` or + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True``. - - **h_n**: tensor of shape :math:`(\text{num\_layers}, H_{out})` for - unbatched input or :math:`(\text{num\_layers}, N, H_{out})`. + - **h_n**: tensor of shape :math:`(\text{num\_layers}, N, H_{out})`. Examples:: @@ -249,7 +245,9 @@ def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: ) candidate = self.act(ch) - new_state = self.update_coefficient * b_state + (1.0 - self.update_coefficient) * candidate + new_state = ( + self.update_coefficient * b_state + (1.0 - self.update_coefficient) * candidate + ) if not is_batched: new_state = new_state.squeeze(0) @@ -303,19 +301,16 @@ class MiRU2(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True``. - - **h_0**: tensor of shape :math:`(\text{num\_layers}, H_{out})` for - unbatched input or :math:`(\text{num\_layers}, N, H_{out})`. + - **h_0**: tensor of shape :math:`(\text{num\_layers}, N, H_{out})`. Defaults to zeros if not provided. Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched - input, :math:`(L, N, H_{out})` when ``batch_first=False`` or + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True``. - - **h_n**: tensor of shape :math:`(\text{num\_layers}, H_{out})` for - unbatched input or :math:`(\text{num\_layers}, N, H_{out})`. + - **h_n**: tensor of shape :math:`(\text{num\_layers}, N, H_{out})`. Examples:: @@ -480,7 +475,9 @@ def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: ) candidate = self.act(ch) - new_state = self.update_coefficient * b_state + (1.0 - self.update_coefficient) * candidate + new_state = ( + self.update_coefficient * b_state + (1.0 - self.update_coefficient) * candidate + ) if not is_batched: new_state = new_state.squeeze(0) diff --git a/torchrecurrent/cells/multiplicativelstm_cell.py b/torchrecurrent/cells/multiplicativelstm_cell.py index 7ed3a5e..492f2eb 100755 --- a/torchrecurrent/cells/multiplicativelstm_cell.py +++ b/torchrecurrent/cells/multiplicativelstm_cell.py @@ -71,18 +71,16 @@ class MultiplicativeLSTM(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state for each element in the input sequence. Defaults to zeros if not provided. @@ -97,17 +95,16 @@ class MultiplicativeLSTM(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the multiplicative LSTM, for each - `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the - input, the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final cell + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -130,9 +127,6 @@ class MultiplicativeLSTM(DoubleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`MultiplicativeLSTMCell` @@ -343,8 +337,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) inp_expanded = b_inp @ self.weight_ih.t() + self.bias_ih gxs1, gxs2, gxs3, gxs4, gxs5 = inp_expanded.chunk(5, 1) diff --git a/torchrecurrent/cells/mut_cell.py b/torchrecurrent/cells/mut_cell.py index 3b6169d..d496fd2 100755 --- a/torchrecurrent/cells/mut_cell.py +++ b/torchrecurrent/cells/mut_cell.py @@ -1,8 +1,12 @@ import torch import torch.nn as nn from torch import Tensor -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, +) class MUT1(SingleStateRecurrentLayerBase): @@ -60,14 +64,12 @@ class MUT1(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -82,14 +84,12 @@ class MUT1(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the MUT1, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the MUT1, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -107,9 +107,6 @@ class MUT1(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`MUT1Cell` @@ -339,14 +336,12 @@ class MUT2(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -361,14 +356,12 @@ class MUT2(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the MUT2, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the MUT2, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -386,9 +379,6 @@ class MUT2(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`MUT2Cell` @@ -617,14 +607,12 @@ class MUT3(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of - the input sequence. The input can also be a packed variable length - sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the initial + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. @@ -639,14 +627,12 @@ class MUT3(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output - features `(h_t)` from the last layer of the MUT3, for each `t`. If a - :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, - the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for unbatched - input or :math:`(\text{num_layers}, N, H_{out})` containing the final + features `(h_t)` from the last layer of the MUT3, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: @@ -664,9 +650,6 @@ class MUT3(SingleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`MUT3Cell` diff --git a/torchrecurrent/cells/nas_cell.py b/torchrecurrent/cells/nas_cell.py index 177efe6..aab9785 100755 --- a/torchrecurrent/cells/nas_cell.py +++ b/torchrecurrent/cells/nas_cell.py @@ -27,7 +27,11 @@ import torch.nn as nn from typing import Optional, Tuple from torch import Tensor -from ..base import DoubleStateRecurrentLayerBase, DoubleStateCellBase, resolve_init_name, apply_init_ +from ..base import ( + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + resolve_init_name, +) class NAS(DoubleStateRecurrentLayerBase): @@ -96,18 +100,15 @@ class NAS(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the - features of the input sequence. The input can also be a packed - variable length sequence. See - :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + features of the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not provided. where: @@ -121,17 +122,15 @@ class NAS(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the NAS, for each `t`. - If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as - the input, the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -150,9 +149,6 @@ class NAS(DoubleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`NASCell` @@ -308,8 +304,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) gates = ( b_inp @ self.weight_ih.t() diff --git a/torchrecurrent/cells/originallstm_cell.py b/torchrecurrent/cells/originallstm_cell.py index c7a34aa..38012fe 100755 --- a/torchrecurrent/cells/originallstm_cell.py +++ b/torchrecurrent/cells/originallstm_cell.py @@ -2,7 +2,11 @@ from torch import Tensor import torch.nn as nn from typing import Optional, Tuple -from ..base import DoubleStateRecurrentLayerBase, DoubleStateCellBase, resolve_init_name, apply_init_ +from ..base import ( + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + resolve_init_name, +) class OriginalLSTM(DoubleStateRecurrentLayerBase): @@ -54,18 +58,15 @@ class OriginalLSTM(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the - features of the input sequence. The input can also be a packed - variable length sequence. See - :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + features of the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not provided. where: @@ -79,17 +80,15 @@ class OriginalLSTM(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the LSTM, for each `t`. - If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as - the input, the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -109,9 +108,6 @@ class OriginalLSTM(DoubleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`OriginalLSTMCell` @@ -263,8 +259,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) gates = ( b_inp @ self.weight_ih.t() diff --git a/torchrecurrent/cells/peepholelstm_cell.py b/torchrecurrent/cells/peepholelstm_cell.py index f852bc0..b590086 100755 --- a/torchrecurrent/cells/peepholelstm_cell.py +++ b/torchrecurrent/cells/peepholelstm_cell.py @@ -2,7 +2,13 @@ from torch import Tensor import torch.nn as nn from typing import Optional, Tuple -from ..base import DoubleStateRecurrentLayerBase, DoubleStateCellBase, resolve_activation, resolve_init_name, apply_init_ +from ..base import ( + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + resolve_activation, + resolve_init_name, + apply_init_, +) class PeepholeLSTM(DoubleStateRecurrentLayerBase): @@ -68,15 +74,15 @@ class PeepholeLSTM(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not provided. where: @@ -90,15 +96,15 @@ class PeepholeLSTM(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features from the last layer, for each timestep. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -287,9 +293,7 @@ def __init__( self.init_cfg["recurrent_bias"] = resolve_init_name( recurrent_bias_init, self.init_cfg["recurrent_bias"] ) - self.init_cfg["peephole_kernel"] = resolve_init_name( - peephole_kernel_init, "normal" - ) + self.init_cfg["peephole_kernel"] = resolve_init_name(peephole_kernel_init, "normal") self._register_tensors( { @@ -323,8 +327,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) weight_ih_i, weight_ih_f, weight_ih_c, weight_ih_o = self.weight_ih.chunk(4, 0) weight_hh_i, weight_hh_f, weight_hh_c, weight_hh_o = self.weight_hh.chunk(4, 0) diff --git a/torchrecurrent/cells/ran_cell.py b/torchrecurrent/cells/ran_cell.py index aead0af..49034cf 100755 --- a/torchrecurrent/cells/ran_cell.py +++ b/torchrecurrent/cells/ran_cell.py @@ -2,7 +2,11 @@ import torch.nn as nn from torch import Tensor from typing import Optional, Tuple -from ..base import DoubleStateRecurrentLayerBase, DoubleStateCellBase, resolve_init_name, apply_init_ +from ..base import ( + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + resolve_init_name, +) class RAN(DoubleStateRecurrentLayerBase): @@ -53,15 +57,15 @@ class RAN(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not provided. where: @@ -75,15 +79,15 @@ class RAN(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features from the last layer, for each timestep. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -261,8 +265,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) weight_ih_c, weight_ih_i, weight_ih_f = self.weight_ih.chunk(3, 0) weight_hh_i, weight_hh_f = self.weight_hh.chunk(2, 0) diff --git a/torchrecurrent/cells/reslstm_cell.py b/torchrecurrent/cells/reslstm_cell.py index 1c03421..a7cef19 100644 --- a/torchrecurrent/cells/reslstm_cell.py +++ b/torchrecurrent/cells/reslstm_cell.py @@ -83,18 +83,15 @@ class ResLSTM(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the - features of the input sequence. The input can also be a packed - variable length sequence. See - :func:`torch.nn.utils.rnn.pack_padded_sequence` or - :func:`torch.nn.utils.rnn.pack_sequence` for details. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + features of the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. - - **c_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not provided. where: @@ -108,17 +105,16 @@ class ResLSTM(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the ResLSTM, for each - `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as - the input, the output will also be a packed sequence. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **c_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. Attributes: @@ -145,9 +141,6 @@ class ResLSTM(DoubleStateRecurrentLayerBase): All the weights and biases are initialized according to the provided initializers (`kernel_init`, `recurrent_kernel_init`, etc.). - .. note:: - ``batch_first`` argument is ignored for unbatched inputs. - .. seealso:: :class:`ResLSTMCell` @@ -323,9 +316,7 @@ def __init__( self.init_cfg["recurrent_kernel"] = resolve_init_name( recurrent_kernel_init, self.init_cfg["recurrent_kernel"] ) - self.init_cfg["peephole_kernel"] = resolve_init_name( - peephole_kernel_init, "normal" - ) + self.init_cfg["peephole_kernel"] = resolve_init_name(peephole_kernel_init, "normal") self.init_cfg["proj"] = resolve_init_name(proj_init, "xavier_uniform") self.init_cfg["residual"] = resolve_init_name(residual_init, "xavier_uniform") self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) diff --git a/torchrecurrent/cells/scrn_cell.py b/torchrecurrent/cells/scrn_cell.py index c476134..0846bc2 100755 --- a/torchrecurrent/cells/scrn_cell.py +++ b/torchrecurrent/cells/scrn_cell.py @@ -2,7 +2,12 @@ from torch import Tensor import torch.nn as nn from typing import Optional, Tuple -from ..base import DoubleStateRecurrentLayerBase, DoubleStateCellBase, resolve_init_name, apply_init_ +from ..base import ( + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + resolve_init_name, + apply_init_, +) class SCRN(DoubleStateRecurrentLayerBase): @@ -63,15 +68,15 @@ class SCRN(DoubleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, (h_0, s_0) - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. - - **s_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` + - **s_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial context state. Defaults to zeros if not provided. where: @@ -85,15 +90,15 @@ class SCRN(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, s_n) - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features from the last layer, for each timestep. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. - - **s_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **s_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final context state for each element in the sequence. Attributes: @@ -319,8 +324,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) inp_expanded = b_inp @ self.weight_ih.t() + self.bias_ih gxs1, gxs2 = inp_expanded.chunk(2, 1) diff --git a/torchrecurrent/cells/sgrn_cell.py b/torchrecurrent/cells/sgrn_cell.py index 59067cc..28f2a6e 100755 --- a/torchrecurrent/cells/sgrn_cell.py +++ b/torchrecurrent/cells/sgrn_cell.py @@ -1,8 +1,12 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, +) class SGRN(SingleStateRecurrentLayerBase): @@ -53,12 +57,12 @@ class SGRN(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. where: @@ -72,12 +76,12 @@ class SGRN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features from the last layer, for each timestep. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: diff --git a/torchrecurrent/cells/sgu_cell.py b/torchrecurrent/cells/sgu_cell.py index c7b71b6..44d4945 100644 --- a/torchrecurrent/cells/sgu_cell.py +++ b/torchrecurrent/cells/sgu_cell.py @@ -65,22 +65,22 @@ class DSGU(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing the + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the DSGU, for each `t`. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing the + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. .. seealso:: @@ -163,22 +163,22 @@ class SGU(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing the + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input sequence. Defaults to zeros if not provided. Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the SGU, for each `t`. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing the + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. .. seealso:: diff --git a/torchrecurrent/cells/star_cell.py b/torchrecurrent/cells/star_cell.py index 3a2c079..121fd2d 100755 --- a/torchrecurrent/cells/star_cell.py +++ b/torchrecurrent/cells/star_cell.py @@ -1,8 +1,12 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, +) class STAR(SingleStateRecurrentLayerBase): @@ -51,12 +55,12 @@ class STAR(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. where: @@ -70,12 +74,12 @@ class STAR(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features from the last layer, for each timestep. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: diff --git a/torchrecurrent/cells/taugru_cell.py b/torchrecurrent/cells/taugru_cell.py index cd3e2c7..6aa382b 100644 --- a/torchrecurrent/cells/taugru_cell.py +++ b/torchrecurrent/cells/taugru_cell.py @@ -62,11 +62,11 @@ class tauGRU(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters. Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True``. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})`. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + . Defaults to zeros if not provided. Outputs: output, h_n diff --git a/torchrecurrent/cells/ugrnn_cell.py b/torchrecurrent/cells/ugrnn_cell.py index 383d493..a6f84c7 100755 --- a/torchrecurrent/cells/ugrnn_cell.py +++ b/torchrecurrent/cells/ugrnn_cell.py @@ -1,8 +1,12 @@ import torch from torch import nn from torch import Tensor -from typing import Optional, Tuple -from ..base import SingleStateRecurrentLayerBase, SingleStateCellBase, resolve_init_name, apply_init_ +from typing import Optional +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + resolve_init_name, +) class UGRNN(SingleStateRecurrentLayerBase): @@ -53,12 +57,12 @@ class UGRNN(SingleStateRecurrentLayerBase): dtype: The desired floating point type of parameters Inputs: input, h_0 - - **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, + - **input**: tensor of shape :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. - - **h_0**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not provided. where: @@ -72,12 +76,12 @@ class UGRNN(SingleStateRecurrentLayerBase): \end{aligned} Outputs: output, h_n - - **output**: tensor of shape :math:`(L, H_{out})` for unbatched input, + - **output**: tensor of shape :math:`(L, N, H_{out})` when ``batch_first=False`` or :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output features from the last layer, for each timestep. - - **h_n**: tensor of shape :math:`(\text{num_layers}, H_{out})` for - unbatched input or :math:`(\text{num_layers}, N, H_{out})` containing + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. Attributes: diff --git a/torchrecurrent/cells/unicornn_cell.py b/torchrecurrent/cells/unicornn_cell.py index d42bcaf..59b8034 100755 --- a/torchrecurrent/cells/unicornn_cell.py +++ b/torchrecurrent/cells/unicornn_cell.py @@ -59,12 +59,11 @@ class UnICORNN(DoubleStateRecurrentLayerBase): dtype: Desired floating point type of parameters. Inputs: input, (h_0, z_0) - - **input**: tensor of shape `(L, H_in)` for unbatched input, - `(L, N, H_in)` when ``batch_first=False``, or `(N, L, H_in)` when - ``batch_first=True`` containing input sequence features. - - **h_0**: tensor of shape `(num_layers, H_out)` for unbatched input or - `(num_layers, N, H_out)` containing the initial hidden state. Defaults - to zeros if not provided. + - **input**: tensor of shape `(L, N, H_in)` when ``batch_first=False``, + or `(N, L, H_in)` when ``batch_first=True`` containing input sequence + features. + - **h_0**: tensor of shape `(num_layers, N, H_out)` containing the + initial hidden state. Defaults to zeros if not provided. - **z_0**: tensor of the same shape as `h_0`, containing the initial control state. Defaults to zeros if not provided. @@ -79,12 +78,10 @@ class UnICORNN(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, z_n) - - **output**: tensor of shape `(L, H_out)` for unbatched input, - `(L, N, H_out)` when ``batch_first=False``, or `(N, L, H_out)` when - ``batch_first=True`` containing the hidden states from the last layer - at each timestep. - - **h_n**: final hidden state for each layer, - shape `(num_layers, H_out)` (unbatched) or `(num_layers, N, H_out)`. + - **output**: tensor of shape `(L, N, H_out)` when ``batch_first=False``, + or `(N, L, H_out)` when ``batch_first=True`` containing the hidden + states from the last layer at each timestep. + - **h_n**: final hidden state for each layer, shape `(num_layers, N, H_out)`. - **z_n**: final control state for each layer, same shape as `h_n`. Attributes: diff --git a/torchrecurrent/cells/wmclstm_cell.py b/torchrecurrent/cells/wmclstm_cell.py index f2bb2b3..708b567 100755 --- a/torchrecurrent/cells/wmclstm_cell.py +++ b/torchrecurrent/cells/wmclstm_cell.py @@ -2,7 +2,12 @@ from torch import Tensor import torch.nn as nn from typing import Optional, Tuple -from ..base import DoubleStateRecurrentLayerBase, DoubleStateCellBase, resolve_init_name, apply_init_ +from ..base import ( + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + resolve_init_name, + apply_init_, +) class WMCLSTM(DoubleStateRecurrentLayerBase): @@ -62,12 +67,10 @@ class WMCLSTM(DoubleStateRecurrentLayerBase): dtype: Desired floating point type of parameters. Inputs: input, (h_0, c_0) - - **input**: tensor of shape `(L, H_in)` for unbatched input, - `(L, N, H_in)` when ``batch_first=False``, or `(N, L, H_in)` when - ``batch_first=True``. - - **h_0**: tensor of shape `(num_layers, H_out)` (unbatched) or - `(num_layers, N, H_out)` containing initial hidden state. Defaults to - zeros if not provided. + - **input**: tensor of shape `(L, N, H_in)` when ``batch_first=False``, + or `(N, L, H_in)` when ``batch_first=True``. + - **h_0**: tensor of shape `(num_layers, N, H_out)` containing initial + hidden state. Defaults to zeros if not provided. - **c_0**: tensor of same shape as `h_0`, containing initial cell state. Defaults to zeros if not provided. @@ -82,12 +85,10 @@ class WMCLSTM(DoubleStateRecurrentLayerBase): \end{aligned} Outputs: output, (h_n, c_n) - - **output**: tensor of shape `(L, H_out)` for unbatched input, - `(L, N, H_out)` when ``batch_first=False``, or `(N, L, H_out)` when - ``batch_first=True`` containing hidden states from the last layer at - each timestep. - - **h_n**: final hidden state for each layer, - shape `(num_layers, H_out)` (unbatched) or `(num_layers, N, H_out)`. + - **output**: tensor of shape `(L, N, H_out)` when ``batch_first=False``, + or `(N, L, H_out)` when ``batch_first=True`` containing hidden states + from the last layer at each timestep. + - **h_n**: final hidden state for each layer, shape `(num_layers, N, H_out)`. - **c_n**: final cell state for each layer, same shape as `h_n`. Attributes: @@ -316,8 +317,16 @@ def forward( b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) else: h, c = state - b_h = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if h is None else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) - b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) if c is None else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + b_h = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h is None + else (h.unsqueeze(0) if (not is_batched and h.dim() == 1) else h) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c is None + else (c.unsqueeze(0) if (not is_batched and c.dim() == 1) else c) + ) gates = ( b_inp @ self.weight_ih.t()