From 77cc27113d6dd7df20e6dcd2f0110ce8e65bb414 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 20 Sep 2026 11:00:00 +0200 Subject: [PATCH 1/5] feat: add strongly-typed RNN family (TRNN, TGRU) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports T-RNN and T-GRU from Balduzzi & Ghifary, "Strongly-Typed Recurrent Neural Networks" (ICML 2016, arXiv:1602.02218), found missing by diffing against the sister Julia libraries RecurrentLayers.jl and LuxRecurrentLayers.jl. Verified against the actual paper (fetched and read directly, not just the Julia ports, which cite a couple of unrelated arXiv IDs in code comments for TGRU/TLSTM — the underlying equations are correct in both libraries, matching the real paper). Both cells have no weight matrix acting on the hidden state at all, by design ("strongly typed" architectures forbid a learned linear map on h(t-1)): T-RNN mixes h(t-1) in only via an elementwise gate, and T-GRU's recurrent weights act on the previous *input* x(t-1) rather than h(t-1). This means TGRU's second recurrent state is input_size-shaped for the first layer but hidden_size-shaped for every subsequent stacked layer (since layer k>=1's input is the previous layer's hidden_size output), which breaks this library's shared multi-layer stacking loop and shared test harness (both assume every double-state component is uniformly hidden_size-shaped). Rather than change the shared base class, TGRU gets its own forward() override that tracks per-layer state as Python lists and returns the final previous-input state as a tuple of per-layer tensors instead of one stacked tensor. TGRU is excluded from the generic CELL_CASES/LAYER_CASES harnesses for this reason and gets bespoke tests instead, including state-continuity, per-layer value correctness, and batch_first checks (added after a Codex audit flagged the initial bespoke tests as shape-only). TLSTM, MCLSTM, and IntersectionRNN are not included here: they share a deeper issue where the paper-defined "output" is a different tensor from what needs to recur in time, which neither of this library's two base classes can cleanly express. Follow-up needed before porting those. Co-Authored-By: Claude Sonnet 5 --- docs/api/cells.rst | 2 + docs/api/layers.rst | 2 + docs/generated/torchrecurrent.TGRU.rst | 88 +++ docs/generated/torchrecurrent.TGRUCell.rst | 91 +++ docs/generated/torchrecurrent.TRNN.rst | 88 +++ docs/generated/torchrecurrent.TRNNCell.rst | 89 +++ docs/models.rst | 6 + tests/test_cells.py | 81 ++- tests/test_layers.py | 98 ++++ torchrecurrent/__init__.py | 8 + torchrecurrent/cells/__init__.py | 5 + torchrecurrent/cells/trnn_cell.py | 619 +++++++++++++++++++++ 12 files changed, 1173 insertions(+), 4 deletions(-) create mode 100644 docs/generated/torchrecurrent.TGRU.rst create mode 100644 docs/generated/torchrecurrent.TGRUCell.rst create mode 100644 docs/generated/torchrecurrent.TRNN.rst create mode 100644 docs/generated/torchrecurrent.TRNNCell.rst create mode 100644 torchrecurrent/cells/trnn_cell.py diff --git a/docs/api/cells.rst b/docs/api/cells.rst index 6402afd..ab6d9f3 100755 --- a/docs/api/cells.rst +++ b/docs/api/cells.rst @@ -39,6 +39,8 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.SGRNCell torchrecurrent.STARCell torchrecurrent.tauGRUCell + torchrecurrent.TRNNCell + torchrecurrent.TGRUCell torchrecurrent.UGRNNCell torchrecurrent.UnICORNNCell torchrecurrent.WMCLSTMCell diff --git a/docs/api/layers.rst b/docs/api/layers.rst index 2e6487c..0c749df 100644 --- a/docs/api/layers.rst +++ b/docs/api/layers.rst @@ -39,6 +39,8 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.SGRN torchrecurrent.STAR torchrecurrent.tauGRU + torchrecurrent.TRNN + torchrecurrent.TGRU torchrecurrent.UGRNN torchrecurrent.UnICORNN torchrecurrent.WMCLSTM diff --git a/docs/generated/torchrecurrent.TGRU.rst b/docs/generated/torchrecurrent.TGRU.rst new file mode 100644 index 0000000..19329c2 --- /dev/null +++ b/docs/generated/torchrecurrent.TGRU.rst @@ -0,0 +1,88 @@ +torchrecurrent.TGRU +=================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TGRU + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TGRU.__init__ + ~TGRU.add_module + ~TGRU.apply + ~TGRU.bfloat16 + ~TGRU.buffers + ~TGRU.children + ~TGRU.compile + ~TGRU.cpu + ~TGRU.cuda + ~TGRU.double + ~TGRU.eval + ~TGRU.extra_repr + ~TGRU.float + ~TGRU.forward + ~TGRU.get_buffer + ~TGRU.get_extra_state + ~TGRU.get_parameter + ~TGRU.get_submodule + ~TGRU.half + ~TGRU.initialize_cells + ~TGRU.ipu + ~TGRU.load_state_dict + ~TGRU.modules + ~TGRU.mtia + ~TGRU.named_buffers + ~TGRU.named_children + ~TGRU.named_modules + ~TGRU.named_parameters + ~TGRU.parameters + ~TGRU.register_backward_hook + ~TGRU.register_buffer + ~TGRU.register_forward_hook + ~TGRU.register_forward_pre_hook + ~TGRU.register_full_backward_hook + ~TGRU.register_full_backward_pre_hook + ~TGRU.register_load_state_dict_post_hook + ~TGRU.register_load_state_dict_pre_hook + ~TGRU.register_module + ~TGRU.register_parameter + ~TGRU.register_state_dict_post_hook + ~TGRU.register_state_dict_pre_hook + ~TGRU.requires_grad_ + ~TGRU.set_extra_state + ~TGRU.set_submodule + ~TGRU.share_memory + ~TGRU.state_dict + ~TGRU.to + ~TGRU.to_empty + ~TGRU.train + ~TGRU.type + ~TGRU.xpu + ~TGRU.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TGRU.T_destination + ~TGRU.call_super_init + ~TGRU.dump_patches + ~TGRU.input_size + ~TGRU.hidden_size + ~TGRU.num_layers + ~TGRU.batch_first + ~TGRU.dropout + ~TGRU.cells + ~TGRU.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TGRUCell.rst b/docs/generated/torchrecurrent.TGRUCell.rst new file mode 100644 index 0000000..b85ee50 --- /dev/null +++ b/docs/generated/torchrecurrent.TGRUCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.TGRUCell +======================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: TGRUCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TGRUCell.__init__ + ~TGRUCell.add_module + ~TGRUCell.apply + ~TGRUCell.bfloat16 + ~TGRUCell.buffers + ~TGRUCell.children + ~TGRUCell.compile + ~TGRUCell.cpu + ~TGRUCell.cuda + ~TGRUCell.double + ~TGRUCell.eval + ~TGRUCell.extra_repr + ~TGRUCell.float + ~TGRUCell.forward + ~TGRUCell.get_buffer + ~TGRUCell.get_extra_state + ~TGRUCell.get_parameter + ~TGRUCell.get_submodule + ~TGRUCell.half + ~TGRUCell.ipu + ~TGRUCell.load_state_dict + ~TGRUCell.modules + ~TGRUCell.mtia + ~TGRUCell.named_buffers + ~TGRUCell.named_children + ~TGRUCell.named_modules + ~TGRUCell.named_parameters + ~TGRUCell.parameters + ~TGRUCell.register_backward_hook + ~TGRUCell.register_buffer + ~TGRUCell.register_forward_hook + ~TGRUCell.register_forward_pre_hook + ~TGRUCell.register_full_backward_hook + ~TGRUCell.register_full_backward_pre_hook + ~TGRUCell.register_load_state_dict_post_hook + ~TGRUCell.register_load_state_dict_pre_hook + ~TGRUCell.register_module + ~TGRUCell.register_parameter + ~TGRUCell.register_state_dict_post_hook + ~TGRUCell.register_state_dict_pre_hook + ~TGRUCell.requires_grad_ + ~TGRUCell.reset_parameters + ~TGRUCell.set_extra_state + ~TGRUCell.set_submodule + ~TGRUCell.share_memory + ~TGRUCell.state_dict + ~TGRUCell.to + ~TGRUCell.to_empty + ~TGRUCell.train + ~TGRUCell.type + ~TGRUCell.uses_double_state + ~TGRUCell.xpu + ~TGRUCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TGRUCell.T_destination + ~TGRUCell.call_super_init + ~TGRUCell.dump_patches + ~TGRUCell.weight_ih + ~TGRUCell.weight_hh + ~TGRUCell.bias_ih + ~TGRUCell.bias_hh + ~TGRUCell.input_size + ~TGRUCell.hidden_size + ~TGRUCell.bias + ~TGRUCell.recurrent_bias + ~TGRUCell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TRNN.rst b/docs/generated/torchrecurrent.TRNN.rst new file mode 100644 index 0000000..3943393 --- /dev/null +++ b/docs/generated/torchrecurrent.TRNN.rst @@ -0,0 +1,88 @@ +torchrecurrent.TRNN +=================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TRNN + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TRNN.__init__ + ~TRNN.add_module + ~TRNN.apply + ~TRNN.bfloat16 + ~TRNN.buffers + ~TRNN.children + ~TRNN.compile + ~TRNN.cpu + ~TRNN.cuda + ~TRNN.double + ~TRNN.eval + ~TRNN.extra_repr + ~TRNN.float + ~TRNN.forward + ~TRNN.get_buffer + ~TRNN.get_extra_state + ~TRNN.get_parameter + ~TRNN.get_submodule + ~TRNN.half + ~TRNN.initialize_cells + ~TRNN.ipu + ~TRNN.load_state_dict + ~TRNN.modules + ~TRNN.mtia + ~TRNN.named_buffers + ~TRNN.named_children + ~TRNN.named_modules + ~TRNN.named_parameters + ~TRNN.parameters + ~TRNN.register_backward_hook + ~TRNN.register_buffer + ~TRNN.register_forward_hook + ~TRNN.register_forward_pre_hook + ~TRNN.register_full_backward_hook + ~TRNN.register_full_backward_pre_hook + ~TRNN.register_load_state_dict_post_hook + ~TRNN.register_load_state_dict_pre_hook + ~TRNN.register_module + ~TRNN.register_parameter + ~TRNN.register_state_dict_post_hook + ~TRNN.register_state_dict_pre_hook + ~TRNN.requires_grad_ + ~TRNN.set_extra_state + ~TRNN.set_submodule + ~TRNN.share_memory + ~TRNN.state_dict + ~TRNN.to + ~TRNN.to_empty + ~TRNN.train + ~TRNN.type + ~TRNN.xpu + ~TRNN.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TRNN.T_destination + ~TRNN.call_super_init + ~TRNN.dump_patches + ~TRNN.input_size + ~TRNN.hidden_size + ~TRNN.num_layers + ~TRNN.batch_first + ~TRNN.dropout + ~TRNN.cells + ~TRNN.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TRNNCell.rst b/docs/generated/torchrecurrent.TRNNCell.rst new file mode 100644 index 0000000..9b81f87 --- /dev/null +++ b/docs/generated/torchrecurrent.TRNNCell.rst @@ -0,0 +1,89 @@ +torchrecurrent.TRNNCell +======================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: TRNNCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TRNNCell.__init__ + ~TRNNCell.add_module + ~TRNNCell.apply + ~TRNNCell.bfloat16 + ~TRNNCell.buffers + ~TRNNCell.children + ~TRNNCell.compile + ~TRNNCell.cpu + ~TRNNCell.cuda + ~TRNNCell.double + ~TRNNCell.eval + ~TRNNCell.extra_repr + ~TRNNCell.float + ~TRNNCell.forward + ~TRNNCell.get_buffer + ~TRNNCell.get_extra_state + ~TRNNCell.get_parameter + ~TRNNCell.get_submodule + ~TRNNCell.half + ~TRNNCell.ipu + ~TRNNCell.load_state_dict + ~TRNNCell.modules + ~TRNNCell.mtia + ~TRNNCell.named_buffers + ~TRNNCell.named_children + ~TRNNCell.named_modules + ~TRNNCell.named_parameters + ~TRNNCell.parameters + ~TRNNCell.register_backward_hook + ~TRNNCell.register_buffer + ~TRNNCell.register_forward_hook + ~TRNNCell.register_forward_pre_hook + ~TRNNCell.register_full_backward_hook + ~TRNNCell.register_full_backward_pre_hook + ~TRNNCell.register_load_state_dict_post_hook + ~TRNNCell.register_load_state_dict_pre_hook + ~TRNNCell.register_module + ~TRNNCell.register_parameter + ~TRNNCell.register_state_dict_post_hook + ~TRNNCell.register_state_dict_pre_hook + ~TRNNCell.requires_grad_ + ~TRNNCell.reset_parameters + ~TRNNCell.set_extra_state + ~TRNNCell.set_submodule + ~TRNNCell.share_memory + ~TRNNCell.state_dict + ~TRNNCell.to + ~TRNNCell.to_empty + ~TRNNCell.train + ~TRNNCell.type + ~TRNNCell.uses_double_state + ~TRNNCell.xpu + ~TRNNCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TRNNCell.T_destination + ~TRNNCell.call_super_init + ~TRNNCell.dump_patches + ~TRNNCell.weight_ih + ~TRNNCell.bias_ih + ~TRNNCell.input_size + ~TRNNCell.hidden_size + ~TRNNCell.bias + ~TRNNCell.recurrent_bias + ~TRNNCell.training + + \ No newline at end of file diff --git a/docs/models.rst b/docs/models.rst index 0a3daaa..9080363 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -113,6 +113,12 @@ references and official implementations where available. * - :doc:`tauGRU ` - `AISTATS 2025 `__ - – + * - :doc:`TRNN ` + - `ICML 2016 `__ + - – + * - :doc:`TGRU ` + - `ICML 2016 `__ + - – * - :doc:`UGRNN ` - `ICLR 2017 `__ - – diff --git a/tests/test_cells.py b/tests/test_cells.py index 2c8ab9c..61d87d1 100755 --- a/tests/test_cells.py +++ b/tests/test_cells.py @@ -4,10 +4,6 @@ import torch from torch import Tensor -skip_windows = pytest.mark.skipif( - sys.platform == "win32", - reason="torch.compile requires Triton, which is not supported on Windows", -) from torchrecurrent import ( AntisymmetricRNNCell, ATRCell, @@ -39,11 +35,18 @@ SGRNCell, STARCell, tauGRUCell, + TRNNCell, + TGRUCell, UGRNNCell, UnICORNNCell, WMCLSTMCell, ) +skip_windows = pytest.mark.skipif( + sys.platform == "win32", + reason="torch.compile requires Triton, which is not supported on Windows", +) + CELL_CASES = [ # (CellClass, input_size, hidden_size, uses_double_state) (AntisymmetricRNNCell, 3, 5, False), @@ -76,6 +79,7 @@ (SGRNCell, 3, 5, False), (STARCell, 3, 5, False), (tauGRUCell, 3, 5, False), + (TRNNCell, 3, 5, False), (UGRNNCell, 3, 5, False), (UnICORNNCell, 3, 5, True), (WMCLSTMCell, 3, 5, True), @@ -231,6 +235,75 @@ def test_taugru_cell_uses_delayed_state(): assert torch.allclose(out, torch.tanh(delayed), atol=1e-4) +def test_trnn_cell_has_no_recurrent_weight(): + """T-RNN's forget gate depends only on x(t): there is no weight_hh at all.""" + cell = TRNNCell(4, 9) + + assert cell.weight_ih.shape == (18, 4) + assert cell.bias_ih.shape == (18,) + assert not hasattr(cell, "weight_hh") + assert not hasattr(cell, "bias_hh") + + +def test_trnn_cell_matches_paper_update(): + cell = TRNNCell(2, 2, bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6], [0.7, 0.8]])) + + x = torch.tensor([[0.2, -0.3]]) + h = torch.tensor([[0.4, -0.5]]) + + new_h = cell(x, h) + latent, forget_pre = (x @ cell.weight_ih.t()).chunk(2, 1) + forget_gate = torch.sigmoid(forget_pre) + expected = forget_gate * h + (1 - forget_gate) * latent + + assert torch.allclose(new_h, expected) + + +def test_tgru_cell_parameter_shapes(): + """weight_hh acts on the previous *input*, so it shares weight_ih's shape.""" + cell = TGRUCell(4, 9) + + assert cell.weight_ih.shape == (27, 4) + assert cell.weight_hh.shape == (27, 4) + assert cell.bias_ih.shape == (27,) + assert cell.bias_hh.shape == (27,) + + +def test_tgru_cell_default_state_uses_input_size_not_hidden_size(): + """The second state component is x(t-1): shaped like the input, not like h.""" + cell = TGRUCell(4, 9) + x = torch.randn(3, 4) + + h, x_prev = cell(x) + + assert h.shape == (3, 9) + assert x_prev.shape == (3, 4) + assert torch.equal(x_prev, x) + + +def test_tgru_cell_matches_paper_update(): + cell = TGRUCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + h = torch.tensor([[0.4, -0.5]]) + x_prev = torch.tensor([[-0.1, 0.6]]) + + new_h, new_x_prev = cell(x, (h, x_prev)) + gates = x @ cell.weight_ih.t() + x_prev @ cell.weight_hh.t() + latent, forget_pre, out_pre = gates.chunk(3, 1) + forget_gate = torch.sigmoid(forget_pre) + candidate = torch.tanh(out_pre) + expected_h = forget_gate * h + latent * candidate + + assert torch.allclose(new_h, expected_h) + assert torch.equal(new_x_prev, x) + + @pytest.mark.parametrize("Cell, in_size, hid_size, _", CELL_CASES) def test_cell_gradients(Cell, in_size, hid_size, _): """A quick smoke test: outputs should be differentiable wrt parameters.""" diff --git a/tests/test_layers.py b/tests/test_layers.py index 7935257..3c40509 100755 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -31,6 +31,8 @@ SGRN, STAR, tauGRU, + TRNN, + TGRU, UGRNN, UnICORNN, WMCLSTM, @@ -65,6 +67,8 @@ SGRN, STAR, tauGRU, + TRNN, + TGRU, UGRNN, UnICORNN, WMCLSTM, @@ -100,6 +104,7 @@ (SGRN, False), (STAR, False), (tauGRU, False), + (TRNN, False), (UGRNN, False), (UnICORNN, True), (WMCLSTM, True), @@ -200,6 +205,99 @@ def test_layer_runs_on_device(Layer, is_double, device): assert p.grad.device.type == device.type +def test_tgru_single_layer_supports_differing_sizes(): + """TGRU's second state (x(t-1)) is input_size-shaped, not hidden_size-shaped: + a single layer works fine even when input_size != hidden_size.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size = 4, 3 + + layer = TGRU(input_size, hidden_size, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert hn.shape == (1, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == 1 + assert xn[0].shape == (batch_size, input_size) + + +def test_tgru_stacked_layers_have_per_layer_previous_input_shapes(): + """Layer 0's previous-input state is input_size-shaped; every later layer's + is hidden_size-shaped, since it receives the previous layer's output.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert hn.shape == (num_layers, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == num_layers + assert xn[0].shape == (batch_size, input_size) + assert xn[1].shape == (batch_size, hidden_size) + + +def test_tgru_previous_input_state_holds_last_seen_input(): + """x_n[k] should be the actual last input layer k saw, not just correctly shaped.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + + assert torch.equal(xn[0], x[-1]) + + # layer 1's own input at each timestep is layer 0's output at that timestep; + # recompute layer 0's per-timestep outputs manually to check xn[1]. + h0 = torch.zeros(batch_size, hidden_size) + x0_prev = torch.zeros(batch_size, input_size) + for t in range(seq_len): + h0, x0_prev = layer.cells[0](x[t], (h0, x0_prev)) + assert torch.allclose(xn[1], h0) + + +def test_tgru_state_continuity_matches_single_call(): + """Splitting a sequence in two and carrying state across calls must match + running the whole sequence in one call.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out_full, _ = layer(x) + + out1, state1 = layer(x[:split]) + out2, _ = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + + +def test_tgru_batch_first_matches_seq_first(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + layer_bf = TGRU( + input_size, hidden_size, num_layers=num_layers, bias=False, batch_first=True + ) + layer_bf.load_state_dict(layer.state_dict()) + + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + out_bf, (hn_bf, xn_bf) = layer_bf(x.transpose(0, 1)) + + assert torch.allclose(out_bf, out.transpose(0, 1)) + assert torch.allclose(hn_bf, hn) + for a, b in zip(xn_bf, xn): + assert torch.allclose(a, b) + assert xn[1].shape == (batch_size, hidden_size) + + @pytest.mark.parametrize("Layer", LAYER_CLASSES) def test_default_repr_shows_input_hidden(Layer): # Default repr should exactly match "Class(input_size, hidden_size)" diff --git a/torchrecurrent/__init__.py b/torchrecurrent/__init__.py index eecd4e5..58159c9 100755 --- a/torchrecurrent/__init__.py +++ b/torchrecurrent/__init__.py @@ -34,6 +34,8 @@ SGRNCell, STARCell, tauGRUCell, + TRNNCell, + TGRUCell, UGRNNCell, UnICORNNCell, WMCLSTMCell, @@ -73,6 +75,8 @@ SGRN, STAR, tauGRU, + TRNN, + TGRU, UGRNN, UnICORNN, WMCLSTM, @@ -143,6 +147,10 @@ "STARCell", "tauGRU", "tauGRUCell", + "TRNN", + "TRNNCell", + "TGRU", + "TGRUCell", "UGRNN", "UGRNNCell", "UnICORNN", diff --git a/torchrecurrent/cells/__init__.py b/torchrecurrent/cells/__init__.py index 7d0aa18..9a5639d 100755 --- a/torchrecurrent/cells/__init__.py +++ b/torchrecurrent/cells/__init__.py @@ -30,6 +30,7 @@ from .sgrn_cell import SGRN, SGRNCell from .star_cell import STAR, STARCell from .taugru_cell import tauGRU, tauGRUCell +from .trnn_cell import TRNN, TRNNCell, TGRU, TGRUCell from .ugrnn_cell import UGRNN, UGRNNCell from .unicornn_cell import UnICORNN, UnICORNNCell from .wmclstm_cell import WMCLSTM, WMCLSTMCell @@ -102,6 +103,10 @@ "STARCell", "tauGRU", "tauGRUCell", + "TRNN", + "TRNNCell", + "TGRU", + "TGRUCell", "UGRNN", "UGRNNCell", "UnICORNN", diff --git a/torchrecurrent/cells/trnn_cell.py b/torchrecurrent/cells/trnn_cell.py new file mode 100644 index 0000000..17186b5 --- /dev/null +++ b/torchrecurrent/cells/trnn_cell.py @@ -0,0 +1,619 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional, Tuple +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + resolve_activation, + resolve_init_name, +) + + +class TRNN(SingleStateRecurrentLayerBase): + r"""Multi-layer strongly typed recurrent neural network (T-RNN). + + [`arXiv `_] + + Each layer consists of a :class:`TRNNCell`, which updates the hidden state + according to: + + .. math:: + \begin{aligned} + z_t &= W_{ih}^z x_t + b_{ih}^z, \\ + f_t &= \sigma(W_{ih}^f x_t + b_{ih}^f), \\ + h_t &= f_t \circ h_{t-1} + (1 - f_t) \circ z_t + \end{aligned} + + where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the + input at time `t`, :math:`\sigma` is the sigmoid function, and + :math:`\circ` denotes elementwise (Hadamard) product. Unlike most + recurrent cells, T-RNN has no hidden-to-hidden weight matrix: the type + constraint of "strongly typed" architectures forbids a learned linear + map acting on :math:`h_{t-1}`, so it only ever enters the update through + the elementwise gate above. + + In a multilayer TRNN, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two TRNN layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input + sequence. Defaults to zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the TRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(2*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(2*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(2*hidden_size)`. Only present when ``bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `bias_init`). + + .. seealso:: + :class:`TRNNCell` + + Examples:: + + >>> rnn = TRNN(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(TRNN, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(TRNNCell, **kwargs) + + +class TRNNCell(SingleStateCellBase): + r"""A strongly typed recurrent neural network (T-RNN) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{z}(t) &= \mathbf{W}_{ih}^{z}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{z}, \\[4pt] + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{f}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{f} + \bigr), \\[4pt] + \mathbf{h}(t) &= \mathbf{f}(t)\circ\mathbf{h}(t-1) + + \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{z}(t), + + where :math:`\circ` is element-wise product. Following the "strongly + typed" design, there is no weight matrix acting on :math:`\mathbf{h}(t-1)`: + it enters the update only through the elementwise mix with the gate + :math:`\mathbf{f}(t)`, which itself depends only on the current input. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use biases. + Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: h_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(2*hidden_size, input_size)`` (``z, f`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(2*hidden_size)`` if ``bias=True``. + + Examples:: + + >>> cell = TRNNCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h = cell(x[t], h) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias"] + + weight_ih: Tensor + bias_ih: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=False, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + + self._register_tensors( + { + "weight_ih": ((2 * hidden_size, input_size), True), + "bias_ih": ((2 * hidden_size,), self.bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + latent, forget_pre = (b_inp @ self.weight_ih.t() + self.bias_ih).chunk(2, 1) + forget_gate = self.gate_act(forget_pre) + new_state = forget_gate * b_state + (1.0 - forget_gate) * latent + + if not is_batched: + new_state = new_state.squeeze(0) + + return new_state + + +class TGRU(DoubleStateRecurrentLayerBase): + r"""Multi-layer strongly typed gated recurrent unit (T-GRU). + + [`arXiv `_] + + Each layer consists of a :class:`TGRUCell`, which updates the hidden + state according to: + + .. math:: + \begin{aligned} + z_t &= W_{ih}^z x_t + b_{ih}^z + W_{hh}^z x_{t-1} + b_{hh}^z, \\ + f_t &= \sigma(W_{ih}^f x_t + b_{ih}^f + W_{hh}^f x_{t-1} + b_{hh}^f), \\ + o_t &= \tau(W_{ih}^o x_t + b_{ih}^o + W_{hh}^o x_{t-1} + b_{hh}^o), \\ + h_t &= f_t \circ h_{t-1} + z_t \circ o_t + \end{aligned} + + where :math:`\sigma` is the sigmoid function, :math:`\tau` is a pointwise + nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + (Hadamard) product. As in T-RNN, no weight matrix acts on the hidden + state: the recurrent weights :math:`W_{hh}^{*}` act on the *previous + input* :math:`x_{t-1}` rather than on :math:`h_{t-1}`, keeping the type + of every linear map consistent with the "strongly typed" design. + + In a multilayer TGRU, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two TGRU layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. Default: + :func:`torch.sigmoid` + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (h_0, x_{-1}) + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not + provided. + - **x_{-1}**: tuple of :attr:`num_layers` tensors containing the + "previous input" fed to each layer before the sequence starts. The + :math:`k`-th tensor has shape :math:`(N, \text{input\_size})` for + `k = 0` and :math:`(N, H_{out})` for `k \ge 1`, since layers beyond + the first receive the previous layer's hidden state as their + input. A single stacked tensor of shape + :math:`(\text{num_layers}, N, H_{in})` is also accepted whenever + `input_size == hidden_size`. Defaults to zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, (h_n, x_n) + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the TGRU, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. + - **x_n**: tuple of :attr:`num_layers` tensors, with the same + per-layer shapes as **x_{-1}**, containing the last input seen by + each layer. Returned as a tuple (not stacked into one tensor) + because layer 0's input dimension may differ from every other + layer's. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(3*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(3*hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable weights applied to the previous input of + the :math:`k`-th layer, of the same shape as `weight_ih`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(3*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable biases paired with `weight_hh` of the + :math:`k`-th layer, of shape `(3*hidden_size)`. Only present when + ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. note:: + Unlike every other multi-layer recurrent layer in this package, + TGRU does not stack its per-layer state into a single tensor for the + "previous input" component, since layer 0's input dimension can + differ from every subsequent layer's hidden dimension. This layer + therefore overrides the generic double-state forward loop. + + .. seealso:: + :class:`TGRUCell` + + Examples:: + + >>> rnn = TGRU(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> output, (hn, xn) = rnn(input) # default zero state + >>> xn[0].shape # layer 0: input_size-shaped + torch.Size([3, 10]) + >>> xn[1].shape # layer 1: hidden_size-shaped + torch.Size([3, 20]) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(TGRU, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(TGRUCell, **kwargs) + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tuple[Tensor, ...]]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tuple[Tensor, ...]]]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + h = [ + torch.zeros( + batch_size, self.hidden_size, dtype=inp.dtype, device=inp.device + ) + for _ in range(self.num_layers) + ] + x_prev = [ + torch.zeros(batch_size, cell.input_size, dtype=inp.dtype, device=inp.device) + for cell in self.cells + ] + else: + h0, x_prev0 = state + h = [h0[layer_idx] for layer_idx in range(self.num_layers)] + x_prev = list(x_prev0) + + outputs = [] + for t in range(seq_len): + x = inp[t] + new_h = [] + new_x_prev = [] + + for layer_idx, cell in enumerate(self.cells): + h_i, x_prev_i = cell(x, (h[layer_idx], x_prev[layer_idx])) + new_h.append(h_i) + new_x_prev.append(x_prev_i) + x = h_i + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + h = new_h + x_prev = new_x_prev + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + + h_n = torch.stack(h, dim=0) + x_n = tuple(x_prev) + return out, (h_n, x_n) + + +class TGRUCell(DoubleStateCellBase): + r"""A strongly typed gated recurrent unit (T-GRU) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{z}(t) &= \mathbf{W}_{ih}^{z}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{z} + + \mathbf{W}_{hh}^{z}\,\mathbf{x}(t-1) + \mathbf{b}_{hh}^{z}, \\[4pt] + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{f}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{f} + + \mathbf{W}_{hh}^{f}\,\mathbf{x}(t-1) + \mathbf{b}_{hh}^{f} + \bigr), \\[4pt] + \mathbf{o}(t) &= \tau\bigl( + \mathbf{W}_{ih}^{o}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{o} + + \mathbf{W}_{hh}^{o}\,\mathbf{x}(t-1) + \mathbf{b}_{hh}^{o} + \bigr), \\[4pt] + \mathbf{h}(t) &= \mathbf{f}(t)\circ\mathbf{h}(t-1) + + \mathbf{z}(t)\circ\mathbf{o}(t), + + where :math:`\circ` is element-wise product and :math:`\tau` is a + pointwise nonlinearity (e.g., tanh). The weights :math:`\mathbf{W}_{hh}^{*}` + act on the previous *input* :math:`\mathbf{x}(t-1)`, not on + :math:`\mathbf{h}(t-1)`, per the "strongly typed" design. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use the biases + paired with ``W_{hh}``. Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. + Default: :func:`torch.sigmoid`. + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (h_0, x_{-1}) + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + - **x_{-1}** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing the "previous input". + + If not provided, both default to zero. + + Outputs: h_1, x_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + - **x_1**: the current input ``x``, carried forward unchanged to + serve as :math:`\mathbf{x}(t-1)` on the following call. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + weight_hh: The learnable weights applied to the previous input, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(3*hidden_size)`` if ``bias=True``. + bias_hh: The learnable biases paired with ``weight_hh``, + of shape ``(3*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = TGRUCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> x_prev = torch.zeros(3, 10) # (batch, input_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h, x_prev = cell(x[t], (h, x_prev)) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + gate_nonlinearity="sigmoid", + output_nonlinearity="tanh", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.out_act = resolve_activation(output_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((3 * hidden_size, input_size), True), + "weight_hh": ((3 * hidden_size, input_size), True), + "bias_ih": ((3 * hidden_size,), self.bias), + "bias_hh": ((3 * hidden_size,), self.recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tensor]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + b_prev_inp = torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + else: + h0, prev_inp0 = state + b_state = h0.unsqueeze(0) if (not is_batched and h0.dim() == 1) else h0 + b_prev_inp = ( + prev_inp0.unsqueeze(0) + if (not is_batched and prev_inp0.dim() == 1) + else prev_inp0 + ) + + gates_ih = b_inp @ self.weight_ih.t() + self.bias_ih + gates_hh = b_prev_inp @ self.weight_hh.t() + self.bias_hh + latent, forget_pre, out_pre = (gates_ih + gates_hh).chunk(3, 1) + + forget_gate = self.gate_act(forget_pre) + candidate = self.out_act(out_pre) + new_state = forget_gate * b_state + latent * candidate + new_prev_inp = b_inp + + if not is_batched: + new_state = new_state.squeeze(0) + new_prev_inp = new_prev_inp.squeeze(0) + + return new_state, new_prev_inp From d2866c12774b9b1d74120c22748e5084b9f72ca7 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 20 Sep 2026 11:23:41 +0200 Subject: [PATCH 2/5] feat: add decoupled-output base classes and IntersectionRNN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends base.py with two new interface pairs (DecoupledSingleStateCellBase /DecoupledSingleStateRecurrentLayerBase and their double-state counterparts) for cells whose per-timestep output is not the same tensor as what recurs in time or feeds the next layer in a stack. This is purely additive: existing SingleState/DoubleState classes and every existing cell are untouched. This was needed because TLSTM, MCLSTM, and IntersectionRNN all define an "output" that genuinely differs from their recurring state, which neither of this library's original two base classes could express (both conflate "returned output" with "next-timestep state" with "next-layer input" into a single tensor). Implements IntersectionRNN/IntersectionRNNCell (Collins, Sohl-Dickstein, Sussillo, "Capacity and Trainability in Recurrent Neural Networks", ICLR 2017, arXiv:1611.09913) on top of the new single-state pair — the first real user of the extension. Verified against the actual paper PDF (page 4, equations 15-20). Requires input_size == hidden_size, per the paper: y(t) is a gated mix of x(t) and a hidden_size-dimensional candidate. The paper's own experiments exclude single-layer use as "not appropriate" (documented, not enforced, since the per-step math stays well-defined). Bespoke tests throughout (not added to the shared CELL_CASES/ LAYER_CASES/LAYER_CLASSES harnesses, since even the generic repr test uses differing input/hidden sizes) added after a Codex audit found the initial set didn't independently verify that multi-layer stacking forwards y (output), not h (state), to the next layer, and didn't numerically check dropout is applied only between layers. Both fixed with an explicit cell-by-cell unroll oracle and a dropout test. Co-Authored-By: Claude Sonnet 5 --- docs/api/cells.rst | 1 + docs/api/layers.rst | 1 + .../torchrecurrent.IntersectionRNN.rst | 88 +++++ .../torchrecurrent.IntersectionRNNCell.rst | 91 +++++ docs/models.rst | 3 + tests/test_cells.py | 67 ++++ tests/test_layers.py | 127 +++++++ torchrecurrent/__init__.py | 4 + torchrecurrent/base.py | 130 ++++++- torchrecurrent/cells/__init__.py | 3 + torchrecurrent/cells/intersectionrnn_cell.py | 336 ++++++++++++++++++ 11 files changed, 850 insertions(+), 1 deletion(-) create mode 100644 docs/generated/torchrecurrent.IntersectionRNN.rst create mode 100644 docs/generated/torchrecurrent.IntersectionRNNCell.rst create mode 100644 torchrecurrent/cells/intersectionrnn_cell.py diff --git a/docs/api/cells.rst b/docs/api/cells.rst index ab6d9f3..aca6972 100755 --- a/docs/api/cells.rst +++ b/docs/api/cells.rst @@ -18,6 +18,7 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.FastGRNNCell torchrecurrent.ATRCell torchrecurrent.IndRNNCell + torchrecurrent.IntersectionRNNCell torchrecurrent.JANETCell torchrecurrent.LEMCell torchrecurrent.LightRUCell diff --git a/docs/api/layers.rst b/docs/api/layers.rst index 0c749df..dcdb4c1 100644 --- a/docs/api/layers.rst +++ b/docs/api/layers.rst @@ -18,6 +18,7 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.FastGRNN torchrecurrent.ATR torchrecurrent.IndRNN + torchrecurrent.IntersectionRNN torchrecurrent.JANET torchrecurrent.LEM torchrecurrent.LightRU diff --git a/docs/generated/torchrecurrent.IntersectionRNN.rst b/docs/generated/torchrecurrent.IntersectionRNN.rst new file mode 100644 index 0000000..5dc16d0 --- /dev/null +++ b/docs/generated/torchrecurrent.IntersectionRNN.rst @@ -0,0 +1,88 @@ +torchrecurrent.IntersectionRNN +============================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: IntersectionRNN + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~IntersectionRNN.__init__ + ~IntersectionRNN.add_module + ~IntersectionRNN.apply + ~IntersectionRNN.bfloat16 + ~IntersectionRNN.buffers + ~IntersectionRNN.children + ~IntersectionRNN.compile + ~IntersectionRNN.cpu + ~IntersectionRNN.cuda + ~IntersectionRNN.double + ~IntersectionRNN.eval + ~IntersectionRNN.extra_repr + ~IntersectionRNN.float + ~IntersectionRNN.forward + ~IntersectionRNN.get_buffer + ~IntersectionRNN.get_extra_state + ~IntersectionRNN.get_parameter + ~IntersectionRNN.get_submodule + ~IntersectionRNN.half + ~IntersectionRNN.initialize_cells + ~IntersectionRNN.ipu + ~IntersectionRNN.load_state_dict + ~IntersectionRNN.modules + ~IntersectionRNN.mtia + ~IntersectionRNN.named_buffers + ~IntersectionRNN.named_children + ~IntersectionRNN.named_modules + ~IntersectionRNN.named_parameters + ~IntersectionRNN.parameters + ~IntersectionRNN.register_backward_hook + ~IntersectionRNN.register_buffer + ~IntersectionRNN.register_forward_hook + ~IntersectionRNN.register_forward_pre_hook + ~IntersectionRNN.register_full_backward_hook + ~IntersectionRNN.register_full_backward_pre_hook + ~IntersectionRNN.register_load_state_dict_post_hook + ~IntersectionRNN.register_load_state_dict_pre_hook + ~IntersectionRNN.register_module + ~IntersectionRNN.register_parameter + ~IntersectionRNN.register_state_dict_post_hook + ~IntersectionRNN.register_state_dict_pre_hook + ~IntersectionRNN.requires_grad_ + ~IntersectionRNN.set_extra_state + ~IntersectionRNN.set_submodule + ~IntersectionRNN.share_memory + ~IntersectionRNN.state_dict + ~IntersectionRNN.to + ~IntersectionRNN.to_empty + ~IntersectionRNN.train + ~IntersectionRNN.type + ~IntersectionRNN.xpu + ~IntersectionRNN.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~IntersectionRNN.T_destination + ~IntersectionRNN.call_super_init + ~IntersectionRNN.dump_patches + ~IntersectionRNN.input_size + ~IntersectionRNN.hidden_size + ~IntersectionRNN.num_layers + ~IntersectionRNN.batch_first + ~IntersectionRNN.dropout + ~IntersectionRNN.cells + ~IntersectionRNN.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.IntersectionRNNCell.rst b/docs/generated/torchrecurrent.IntersectionRNNCell.rst new file mode 100644 index 0000000..b331b40 --- /dev/null +++ b/docs/generated/torchrecurrent.IntersectionRNNCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.IntersectionRNNCell +================================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: IntersectionRNNCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~IntersectionRNNCell.__init__ + ~IntersectionRNNCell.add_module + ~IntersectionRNNCell.apply + ~IntersectionRNNCell.bfloat16 + ~IntersectionRNNCell.buffers + ~IntersectionRNNCell.children + ~IntersectionRNNCell.compile + ~IntersectionRNNCell.cpu + ~IntersectionRNNCell.cuda + ~IntersectionRNNCell.double + ~IntersectionRNNCell.eval + ~IntersectionRNNCell.extra_repr + ~IntersectionRNNCell.float + ~IntersectionRNNCell.forward + ~IntersectionRNNCell.get_buffer + ~IntersectionRNNCell.get_extra_state + ~IntersectionRNNCell.get_parameter + ~IntersectionRNNCell.get_submodule + ~IntersectionRNNCell.half + ~IntersectionRNNCell.ipu + ~IntersectionRNNCell.load_state_dict + ~IntersectionRNNCell.modules + ~IntersectionRNNCell.mtia + ~IntersectionRNNCell.named_buffers + ~IntersectionRNNCell.named_children + ~IntersectionRNNCell.named_modules + ~IntersectionRNNCell.named_parameters + ~IntersectionRNNCell.parameters + ~IntersectionRNNCell.register_backward_hook + ~IntersectionRNNCell.register_buffer + ~IntersectionRNNCell.register_forward_hook + ~IntersectionRNNCell.register_forward_pre_hook + ~IntersectionRNNCell.register_full_backward_hook + ~IntersectionRNNCell.register_full_backward_pre_hook + ~IntersectionRNNCell.register_load_state_dict_post_hook + ~IntersectionRNNCell.register_load_state_dict_pre_hook + ~IntersectionRNNCell.register_module + ~IntersectionRNNCell.register_parameter + ~IntersectionRNNCell.register_state_dict_post_hook + ~IntersectionRNNCell.register_state_dict_pre_hook + ~IntersectionRNNCell.requires_grad_ + ~IntersectionRNNCell.reset_parameters + ~IntersectionRNNCell.set_extra_state + ~IntersectionRNNCell.set_submodule + ~IntersectionRNNCell.share_memory + ~IntersectionRNNCell.state_dict + ~IntersectionRNNCell.to + ~IntersectionRNNCell.to_empty + ~IntersectionRNNCell.train + ~IntersectionRNNCell.type + ~IntersectionRNNCell.uses_double_state + ~IntersectionRNNCell.xpu + ~IntersectionRNNCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~IntersectionRNNCell.T_destination + ~IntersectionRNNCell.call_super_init + ~IntersectionRNNCell.dump_patches + ~IntersectionRNNCell.weight_ih + ~IntersectionRNNCell.weight_hh + ~IntersectionRNNCell.bias_ih + ~IntersectionRNNCell.bias_hh + ~IntersectionRNNCell.input_size + ~IntersectionRNNCell.hidden_size + ~IntersectionRNNCell.bias + ~IntersectionRNNCell.recurrent_bias + ~IntersectionRNNCell.training + + \ No newline at end of file diff --git a/docs/models.rst b/docs/models.rst index 9080363..5c5534b 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -47,6 +47,9 @@ references and official implementations where available. * - :doc:`IndRNN ` - `CVPR 2018 `__ - `Sunnydreamrain/IndRNN_Theano_Lasagne `__ + * - :doc:`IntersectionRNN ` + - `ICLR 2017 `__ + - – * - :doc:`JANET ` - `arXiv 2018 `__ - `JosvanderWesthuizen/janet `__ diff --git a/tests/test_cells.py b/tests/test_cells.py index 61d87d1..d06bf3d 100755 --- a/tests/test_cells.py +++ b/tests/test_cells.py @@ -19,6 +19,7 @@ GatedAntisymmetricRNNCell, MGUCell, IndRNNCell, + IntersectionRNNCell, LiGRUCell, LightRUCell, MultiplicativeLSTMCell, @@ -304,6 +305,72 @@ def test_tgru_cell_matches_paper_update(): assert torch.equal(new_x_prev, x) +def test_intersectionrnn_cell_requires_equal_sizes(): + with pytest.raises(ValueError, match="input_size == hidden_size"): + IntersectionRNNCell(3, 5) + + +def test_intersectionrnn_cell_shapes(): + cell = IntersectionRNNCell(4, 4) + + assert cell.weight_ih.shape == (16, 4) + assert cell.weight_hh.shape == (16, 4) + assert cell.bias_ih.shape == (16,) + assert cell.bias_hh.shape == (16,) + + +def test_intersectionrnn_cell_output_differs_from_state(): + """y(t) and h(t) are genuinely different tensors, not aliases.""" + cell = IntersectionRNNCell(4, 4) + + x_unbatched = torch.randn(4) + y_u, h_u = cell(x_unbatched) + assert y_u.shape == h_u.shape == (4,) + assert not torch.allclose(y_u, h_u) + + x = torch.randn(3, 4) + y, h = cell(x) + + assert y.shape == h.shape == (3, 4) + assert not torch.allclose(y, h) + + +def test_intersectionrnn_cell_gradients(): + cell = IntersectionRNNCell(4, 4, bias=False) + x = torch.randn(2, 4, requires_grad=True) + + y, h = cell(x) + (y.sum() + h.sum()).backward() + + assert x.grad is not None + for p in cell.parameters(): + if p.requires_grad: + assert p.grad is not None + + +def test_intersectionrnn_cell_matches_paper_update(): + cell = IntersectionRNNCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(16, dtype=torch.float32).reshape(8, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(16, dtype=torch.float32).reshape(8, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + h = torch.tensor([[0.4, -0.5]]) + + y, new_h = cell(x, h) + gates = x @ cell.weight_ih.t() + h @ cell.weight_hh.t() + depth_pre, recurrent_pre, gy_pre, gh_pre = gates.chunk(4, 1) + depth_candidate = torch.relu(depth_pre) + recurrent_candidate = torch.tanh(recurrent_pre) + gate_y = torch.sigmoid(gy_pre) + gate_h = torch.sigmoid(gh_pre) + expected_y = gate_y * x + (1 - gate_y) * depth_candidate + expected_h = gate_h * h + (1 - gate_h) * recurrent_candidate + + assert torch.allclose(y, expected_y) + assert torch.allclose(new_h, expected_h) + + @pytest.mark.parametrize("Cell, in_size, hid_size, _", CELL_CASES) def test_cell_gradients(Cell, in_size, hid_size, _): """A quick smoke test: outputs should be differentiable wrt parameters.""" diff --git a/tests/test_layers.py b/tests/test_layers.py index 3c40509..17da956 100755 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -14,6 +14,7 @@ FastGRNN, GatedAntisymmetricRNN, IndRNN, + IntersectionRNN, LiGRU, LightRU, MGU, @@ -298,6 +299,132 @@ def test_tgru_batch_first_matches_seq_first(): assert xn[1].shape == (batch_size, hidden_size) +def test_intersectionrnn_requires_equal_sizes(): + with pytest.raises(ValueError, match="input_size == hidden_size"): + IntersectionRNN(3, 5) + + +def test_intersectionrnn_shapes_and_state(): + size = 6 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, size) + out, hn = layer(x) + + assert out.shape == (seq_len, batch_size, size) + assert isinstance(hn, Tensor) + assert hn.shape == (num_layers, batch_size, size) + # output and recurrent state are genuinely different tensors + assert not torch.allclose(out[-1], hn[-1]) + + +def test_intersectionrnn_batch_first_matches_seq_first(): + size = 6 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + layer_bf = IntersectionRNN( + size, size, num_layers=num_layers, bias=False, batch_first=True + ) + layer_bf.load_state_dict(layer.state_dict()) + + x = torch.randn(seq_len, batch_size, size) + out, hn = layer(x) + out_bf, hn_bf = layer_bf(x.transpose(0, 1)) + + assert torch.allclose(out_bf, out.transpose(0, 1)) + assert torch.allclose(hn_bf, hn) + + +def test_intersectionrnn_state_continuity_matches_single_call(): + size = 6 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, size) + + out_full, _ = layer(x) + + out1, state1 = layer(x[:split]) + out2, _ = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + + +def test_intersectionrnn_stacking_forwards_output_not_state(): + """Layer k+1 must receive layer k's y(t) (output), not its h(t) (state).""" + size = 5 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, size) + + out, hn = layer(x) + + h_prev = [torch.zeros(batch_size, size) for _ in range(num_layers)] + expected_out = None + for t in range(seq_len): + layer_inp = x[t] + for layer_idx, cell in enumerate(layer.cells): + y, h_new = cell(layer_inp, h_prev[layer_idx]) + h_prev[layer_idx] = h_new + layer_inp = y + expected_out = layer_inp + + assert torch.allclose(out[-1], expected_out, atol=1e-6) + assert torch.allclose(hn[0], h_prev[0], atol=1e-6) + assert torch.allclose(hn[1], h_prev[1], atol=1e-6) + + +def test_intersectionrnn_dropout_only_between_layers(): + size = 5 + seq_len, batch_size = 4, 3 + + # num_layers=1: dropout is never applied (no "next layer" to drop before), + # so output must be bit-identical regardless of the dropout probability. + torch.manual_seed(0) + layer_no_drop = IntersectionRNN(size, size, dropout=0.0, bias=False) + torch.manual_seed(0) + layer_high_drop = IntersectionRNN(size, size, dropout=0.9, bias=False) + + x = torch.randn(seq_len, batch_size, size) + out_no_drop, _ = layer_no_drop(x) + out_high_drop, _ = layer_high_drop(x) + + assert torch.equal(out_no_drop, out_high_drop) + + # num_layers=2: dropout between layers must actually change the output. + torch.manual_seed(0) + stacked_no_drop = IntersectionRNN(size, size, num_layers=2, dropout=0.0, bias=False) + torch.manual_seed(0) + stacked_drop = IntersectionRNN(size, size, num_layers=2, dropout=0.9, bias=False) + stacked_drop.load_state_dict(stacked_no_drop.state_dict()) + + torch.manual_seed(1) + out_stacked_no_drop, _ = stacked_no_drop(x) + torch.manual_seed(1) + out_stacked_drop, _ = stacked_drop(x) + + assert not torch.allclose(out_stacked_no_drop, out_stacked_drop) + + +def test_intersectionrnn_repr(): + r = repr(IntersectionRNN(5, 5)) + assert r == "IntersectionRNN(5, 5)" + + r = repr(IntersectionRNN(5, 5, num_layers=2)) + assert "num_layers=2" in r + + r = repr(IntersectionRNN(5, 5, dropout=0.5)) + assert "dropout=0.5" in r + + r = repr(IntersectionRNN(5, 5, batch_first=True)) + assert "batch_first=True" in r + + @pytest.mark.parametrize("Layer", LAYER_CLASSES) def test_default_repr_shows_input_hidden(Layer): # Default repr should exactly match "Class(input_size, hidden_size)" diff --git a/torchrecurrent/__init__.py b/torchrecurrent/__init__.py index 58159c9..9cf1899 100755 --- a/torchrecurrent/__init__.py +++ b/torchrecurrent/__init__.py @@ -12,6 +12,7 @@ FastGRNNCell, ATRCell, IndRNNCell, + IntersectionRNNCell, JANETCell, LEMCell, LiGRUCell, @@ -53,6 +54,7 @@ FastRNN, FastGRNN, IndRNN, + IntersectionRNN, JANET, LEM, LiGRU, @@ -111,6 +113,8 @@ "FastGRNNCell", "IndRNN", "IndRNNCell", + "IntersectionRNN", + "IntersectionRNNCell", "JANET", "JANETCell", "LEM", diff --git a/torchrecurrent/base.py b/torchrecurrent/base.py index d27b322..65b3bb9 100755 --- a/torchrecurrent/base.py +++ b/torchrecurrent/base.py @@ -2,7 +2,6 @@ from __future__ import annotations -import warnings from typing import Dict, Optional, Tuple, List, Any import torch @@ -269,6 +268,38 @@ def forward( raise NotImplementedError +class DecoupledSingleStateCellBase(RecurrentCellBase): + """ + Interface for cells whose per-step output is not the tensor that recurs + in time (e.g. a model with a separate "prediction" and "control" signal): + forward(inp, state) -> (output, new_state) + Unlike SingleStateCellBase, `output` is returned separately from + `new_state` instead of being the same tensor. + """ + + def uses_double_state(self) -> bool: + return False + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: + raise NotImplementedError + + +class DecoupledDoubleStateCellBase(RecurrentCellBase): + """ + Interface for double-state cells whose per-step output is not either + state component: + forward(inp, (s1, s2)) -> (output, (new_s1, new_s2)) + """ + + def uses_double_state(self) -> bool: + return True + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + raise NotImplementedError + + # Recurrent layers class RecurrentLayerBase(nn.Module): __constants__ = ["input_size", "hidden_size", "num_layers", "batch_first", "dropout"] @@ -402,3 +433,100 @@ def forward( if self.batch_first: out = out.transpose(0, 1) return out, state + + +class DecoupledSingleStateRecurrentLayerBase(RecurrentLayerBase): + """ + Like SingleStateRecurrentLayerBase, but each cell returns (output, new_state) + with output distinct from new_state: output feeds the next layer, new_state + feeds the same layer's next timestep. + """ + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + state = torch.zeros( + self.num_layers, + batch_size, + self.hidden_size, + dtype=inp.dtype, + device=inp.device, + ) + + outputs = torch.jit.annotate(List[Tensor], []) + + for t in range(seq_len): + x = inp[t] + new_states = torch.jit.annotate(List[Tensor], []) + + for layer_idx, cell in enumerate(self.cells): + s_prev = state[layer_idx] + out, s_new = cell(x, s_prev) + new_states.append(s_new) + x = out + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + state = torch.stack(new_states, dim=0) + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + return out, state + + +class DecoupledDoubleStateRecurrentLayerBase(RecurrentLayerBase): + """ + Like DoubleStateRecurrentLayerBase, but each cell returns + (output, (new_s1, new_s2)) with output distinct from either state + component: output feeds the next layer, (new_s1, new_s2) feed the same + layer's next timestep. + """ + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + s1 = torch.zeros( + self.num_layers, + batch_size, + self.hidden_size, + dtype=inp.dtype, + device=inp.device, + ) + s2 = torch.zeros_like(s1) + state = (s1, s2) + + outputs = torch.jit.annotate(List[Tensor], []) + + for t in range(seq_len): + x = inp[t] + new_s1 = torch.jit.annotate(List[Tensor], []) + new_s2 = torch.jit.annotate(List[Tensor], []) + s1_prev, s2_prev = state + + for layer_idx, cell in enumerate(self.cells): + out, (s1_i, s2_i) = cell(x, (s1_prev[layer_idx], s2_prev[layer_idx])) + new_s1.append(s1_i) + new_s2.append(s2_i) + x = out + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + state = (torch.stack(new_s1, dim=0), torch.stack(new_s2, dim=0)) + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + return out, state diff --git a/torchrecurrent/cells/__init__.py b/torchrecurrent/cells/__init__.py index 9a5639d..7bd5eeb 100755 --- a/torchrecurrent/cells/__init__.py +++ b/torchrecurrent/cells/__init__.py @@ -14,6 +14,7 @@ from .cornn_cell import coRNN, coRNNCell from .fastrnn_cell import FastRNN, FastRNNCell, FastGRNN, FastGRNNCell from .indrnn_cell import IndRNN, IndRNNCell +from .intersectionrnn_cell import IntersectionRNN, IntersectionRNNCell from .janet_cell import JANET, JANETCell from .lem_cell import LEM, LEMCell from .ligru_cell import LiGRU, LiGRUCell @@ -65,6 +66,8 @@ "FastGRNNCell", "IndRNN", "IndRNNCell", + "IntersectionRNN", + "IntersectionRNNCell", "JANET", "JANETCell", "LEM", diff --git a/torchrecurrent/cells/intersectionrnn_cell.py b/torchrecurrent/cells/intersectionrnn_cell.py new file mode 100644 index 0000000..581e7e0 --- /dev/null +++ b/torchrecurrent/cells/intersectionrnn_cell.py @@ -0,0 +1,336 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional, Tuple +from ..base import ( + DecoupledSingleStateRecurrentLayerBase, + DecoupledSingleStateCellBase, + resolve_activation, + resolve_init_name, +) + + +class IntersectionRNN(DecoupledSingleStateRecurrentLayerBase): + r"""Multi-layer Intersection RNN (+RNN). + + [`arXiv `_] + + Each layer consists of an :class:`IntersectionRNNCell`, which couples a + "depth" gate (between layers) and a "recurrent" gate (through time): + + .. math:: + \begin{aligned} + y^{in}_t &= \phi(W_{ih}^{y} x_t + b_{ih}^{y} + W_{hh}^{y} h_{t-1} + b_{hh}^{y}), \\ + h^{in}_t &= \tau(W_{ih}^{h} x_t + b_{ih}^{h} + W_{hh}^{h} h_{t-1} + b_{hh}^{h}), \\ + g^{y}_t &= \sigma(W_{ih}^{g^y} x_t + b_{ih}^{g^y} + W_{hh}^{g^y} h_{t-1} + b_{hh}^{g^y}), \\ + g^{h}_t &= \sigma(W_{ih}^{g^h} x_t + b_{ih}^{g^h} + W_{hh}^{g^h} h_{t-1} + b_{hh}^{g^h}), \\ + y_t &= g^{y}_t \circ x_t + (1 - g^{y}_t) \circ y^{in}_t, \\ + h_t &= g^{h}_t \circ h_{t-1} + (1 - g^{h}_t) \circ h^{in}_t + \end{aligned} + + where :math:`\sigma` is the sigmoid function, :math:`\phi` is ReLU, + :math:`\tau` is a pointwise nonlinearity (e.g., tanh), and :math:`\circ` + denotes elementwise (Hadamard) product. Unlike every other layer in this + package, the per-timestep output :math:`y_t` fed to the next layer is + *not* the same tensor as the recurrent state :math:`h_t` carried to the + next timestep: :math:`y_t` is a "depth" gate mixing the raw input + :math:`x_t` with a candidate, while :math:`h_t` is an ordinary recurrent + gate mix. This requires ``input_size == hidden_size``, since :math:`y_t` + mixes :math:`x_t` and :math:`y^{in}_t` elementwise. + + The original paper studies this architecture for ``num_layers >= 2`` and + explicitly excludes single-layer (``num_layers=1``) configurations from + its experiments, since the depth gate has no second layer to feed. The + per-timestep computation is still well-defined for ``num_layers=1`` + (:math:`y_t` is simply returned as the layer's output), but this + configuration was not studied by the authors. + + In a multilayer IntersectionRNN, the input :math:`x^{(l)}_t` of the + :math:`l`-th layer (:math:`l \ge 2`) is the *output* :math:`y^{(l-1)}_t` + of the previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, + where each :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which + is 0 with probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. Must + equal `hidden_size`. + hidden_size: The number of features in the hidden state `h`. Must + equal `input_size`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two IntersectionRNN layers, with the second receiving the + outputs of the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + depth_nonlinearity: Nonlinearity :math:`\phi` for the depth candidate + :math:`y^{in}`. Default: :func:`torch.relu` + recurrent_nonlinearity: Nonlinearity :math:`\tau` for the recurrent + candidate :math:`h^{in}`. Default: :func:`torch.tanh` + gate_nonlinearity: Activation :math:`\sigma` for both gates. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape :math:`(L, N, H)` when ``batch_first=False`` + or :math:`(N, L, H)` when ``batch_first=True`` containing the features of + the input sequence, where :math:`H` = `input_size` = `hidden_size`. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H)` containing the + initial hidden state for each element in the input sequence. Defaults to + zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H ={} & \text{input\_size} = \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape :math:`(L, N, H)` when ``batch_first=False`` + or :math:`(N, L, H)` when ``batch_first=True`` containing the output + features `(y_t)` from the last layer of the IntersectionRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H)` containing the + final recurrent state for each element in the sequence. Note this is + `h_n`, not `y_n`: the recurrent state and the returned output are + different tensors for every layer. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(4*hidden_size, input_size)`. + cells.{k}.weight_hh : the learnable hidden-hidden weights of the :math:`k`-th + layer, of shape `(4*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(4*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable hidden-hidden biases of the :math:`k`-th + layer, of shape `(4*hidden_size)`. Only present when ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`IntersectionRNNCell` + + Examples:: + + >>> rnn = IntersectionRNN(10, 10, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 10) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(IntersectionRNN, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(IntersectionRNNCell, **kwargs) + + +class IntersectionRNNCell(DecoupledSingleStateCellBase): + r"""An Intersection RNN (+RNN) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{y}^{in}(t) &= \phi\bigl( + \mathbf{W}_{ih}^{y}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{y} + + \mathbf{W}_{hh}^{y}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{y} + \bigr), \\[4pt] + \mathbf{h}^{in}(t) &= \tau\bigl( + \mathbf{W}_{ih}^{h}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{h} + + \mathbf{W}_{hh}^{h}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{h} + \bigr), \\[4pt] + \mathbf{g}^{y}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{g^y}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{g^y} + + \mathbf{W}_{hh}^{g^y}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{g^y} + \bigr), \\[4pt] + \mathbf{g}^{h}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{g^h}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{g^h} + + \mathbf{W}_{hh}^{g^h}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{g^h} + \bigr), \\[4pt] + \mathbf{y}(t) &= \mathbf{g}^{y}(t)\circ\mathbf{x}(t) + + \bigl(1 - \mathbf{g}^{y}(t)\bigr)\circ\mathbf{y}^{in}(t), \\[4pt] + \mathbf{h}(t) &= \mathbf{g}^{h}(t)\circ\mathbf{h}(t-1) + + \bigl(1 - \mathbf{g}^{h}(t)\bigr)\circ\mathbf{h}^{in}(t), + + where :math:`\circ` is element-wise product, :math:`\phi` is ReLU, and + :math:`\tau` is a pointwise nonlinearity (e.g., tanh). The returned + output :math:`\mathbf{y}(t)` and the recurrent state + :math:`\mathbf{h}(t)` are different tensors: :math:`\mathbf{y}(t)` gates + a mix of the raw input and a candidate, while :math:`\mathbf{h}(t)` + gates a mix of the previous state and a (different) candidate. Requires + ``input_size == hidden_size``, since :math:`\mathbf{y}(t)` mixes + :math:`\mathbf{x}(t)` and :math:`\mathbf{y}^{in}(t)` elementwise. + + Args: + input_size: The number of expected features in the input ``x``. Must + equal ``hidden_size``. + hidden_size: The number of features in the hidden state ``h``. Must + equal ``input_size``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use recurrent biases. + Default: ``True``. + depth_nonlinearity: Nonlinearity :math:`\phi` for the depth candidate. + Default: :func:`torch.relu`. + recurrent_nonlinearity: Nonlinearity :math:`\tau` for the recurrent + candidate. Default: :func:`torch.tanh`. + gate_nonlinearity: Activation :math:`\sigma` for both gates. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: y_1, h_1 + - **y_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the depth output, to be passed to the next layer. + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next recurrent state. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(4*hidden_size, input_size)`` (``y, h, g^y, g^h`` parts). + weight_hh: The learnable hidden-hidden weights, + of shape ``(4*hidden_size, hidden_size)`` (``y, h, g^y, g^h`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(4*hidden_size)`` if ``bias=True``. + bias_hh: The learnable hidden-hidden biases, + of shape ``(4*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = IntersectionRNNCell(10, 10) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 10) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... y, h = cell(x[t], h) + ... out.append(y) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + depth_nonlinearity="relu", + recurrent_nonlinearity="tanh", + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + if input_size != hidden_size: + raise ValueError( + "IntersectionRNNCell requires input_size == hidden_size (got " + f"input_size={input_size}, hidden_size={hidden_size}), since the " + "output is a gated mix of x(t) and a candidate of dimension " + "hidden_size." + ) + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.depth_act = resolve_activation(depth_nonlinearity) + self.recurrent_act = resolve_activation(recurrent_nonlinearity) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._default_register_tensors(ih_mult=4, hh_mult=4) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + gates = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + b_state @ self.weight_hh.t() + + self.bias_hh + ) + depth_pre, recurrent_pre, gate_y_pre, gate_h_pre = gates.chunk(4, 1) + + depth_candidate = self.depth_act(depth_pre) + recurrent_candidate = self.recurrent_act(recurrent_pre) + gate_y = self.gate_act(gate_y_pre) + gate_h = self.gate_act(gate_h_pre) + + new_output = gate_y * b_inp + (1.0 - gate_y) * depth_candidate + new_state = gate_h * b_state + (1.0 - gate_h) * recurrent_candidate + + if not is_batched: + new_output = new_output.squeeze(0) + new_state = new_state.squeeze(0) + + return new_output, new_state From ba5e7161544e462ba46c02a3f4f9cee95ccb49dc Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 20 Sep 2026 16:37:44 +0200 Subject: [PATCH 3/5] feat: add MCLSTM (memory-controller LSTM) Ports MC-LSTM from Ben-Ari & Shwartz-Ziv, "Sequence modeling using a memory controller extension for LSTM" (NIPS 2017 time series workshop), found missing by diffing against the sister Julia libraries. Verified against the actual paper PDF: every gate (f, i, o, m, n) is driven by the control vector v(t-1), never by h(t-1) -- the paper's central idea is decoupling the LSTM's "prediction" output h(t) from the memory-management role a standard LSTM's hidden state also has to play. Built on the DecoupledDoubleStateCellBase/RecurrentLayerBase pair added in the previous commit -- MCLSTM is the first real user of the double-state half of that extension (IntersectionRNN used the single-state half). Unlike TGRU/TLSTM, MCLSTM's recurring state (v, c) is uniformly hidden_size-shaped across every layer (no ragged-shape issue), so the layer class relies entirely on the inherited generic stacking loop with no custom forward() override, and fits cleanly into the shared LAYER_CASES/LAYER_CLASSES test harness (unlike TGRU and IntersectionRNN, which needed fully bespoke layer tests). The cell itself still needs bespoke tests, since its forward() returns Tuple[Tensor, Tuple[Tensor, Tensor]], structurally incompatible with the shared CELL_CASES harness's Tuple[Tensor, Tensor] assumption. While implementing this, found and fixed a real gap in both MCLSTMCell and the already-committed TGRUCell: neither handled the per-component-None state convention (e.g. cell(x, (None, some_c))) that every other double-state cell in this library supports (see janet_cell.py's forward for the reference pattern) -- both only checked "state is None" at the top level, not per-component. Fixed in both. Codex audit found no correctness bugs (23 focused tests + 28 independent audit cases + a dedicated dropout-placement probe all passed) but flagged real test-thoroughness gaps: the None-state tests only covered unbatched inputs, the state-continuity test discarded final states instead of comparing them, and the stacking-oracle test only checked the last timestep's output and one state component per layer instead of the full sequence and every layer. All four strengthened before commit. Co-Authored-By: Claude Sonnet 5 --- docs/api/cells.rst | 1 + docs/api/layers.rst | 1 + docs/generated/torchrecurrent.MCLSTM.rst | 88 +++++ docs/generated/torchrecurrent.MCLSTMCell.rst | 91 +++++ docs/models.rst | 3 + tests/test_cells.py | 110 ++++++ tests/test_layers.py | 82 +++++ torchrecurrent/__init__.py | 4 + torchrecurrent/cells/__init__.py | 3 + torchrecurrent/cells/mclstm_cell.py | 347 +++++++++++++++++++ torchrecurrent/cells/trnn_cell.py | 18 +- 11 files changed, 744 insertions(+), 4 deletions(-) create mode 100644 docs/generated/torchrecurrent.MCLSTM.rst create mode 100644 docs/generated/torchrecurrent.MCLSTMCell.rst create mode 100644 torchrecurrent/cells/mclstm_cell.py diff --git a/docs/api/cells.rst b/docs/api/cells.rst index aca6972..5b91712 100755 --- a/docs/api/cells.rst +++ b/docs/api/cells.rst @@ -23,6 +23,7 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.LEMCell torchrecurrent.LightRUCell torchrecurrent.LiGRUCell + torchrecurrent.MCLSTMCell torchrecurrent.MGUCell torchrecurrent.MiRU1Cell torchrecurrent.MiRU2Cell diff --git a/docs/api/layers.rst b/docs/api/layers.rst index dcdb4c1..715e436 100644 --- a/docs/api/layers.rst +++ b/docs/api/layers.rst @@ -23,6 +23,7 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.LEM torchrecurrent.LightRU torchrecurrent.LiGRU + torchrecurrent.MCLSTM torchrecurrent.MGU torchrecurrent.MiRU1 torchrecurrent.MiRU2 diff --git a/docs/generated/torchrecurrent.MCLSTM.rst b/docs/generated/torchrecurrent.MCLSTM.rst new file mode 100644 index 0000000..32bea58 --- /dev/null +++ b/docs/generated/torchrecurrent.MCLSTM.rst @@ -0,0 +1,88 @@ +torchrecurrent.MCLSTM +===================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: MCLSTM + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MCLSTM.__init__ + ~MCLSTM.add_module + ~MCLSTM.apply + ~MCLSTM.bfloat16 + ~MCLSTM.buffers + ~MCLSTM.children + ~MCLSTM.compile + ~MCLSTM.cpu + ~MCLSTM.cuda + ~MCLSTM.double + ~MCLSTM.eval + ~MCLSTM.extra_repr + ~MCLSTM.float + ~MCLSTM.forward + ~MCLSTM.get_buffer + ~MCLSTM.get_extra_state + ~MCLSTM.get_parameter + ~MCLSTM.get_submodule + ~MCLSTM.half + ~MCLSTM.initialize_cells + ~MCLSTM.ipu + ~MCLSTM.load_state_dict + ~MCLSTM.modules + ~MCLSTM.mtia + ~MCLSTM.named_buffers + ~MCLSTM.named_children + ~MCLSTM.named_modules + ~MCLSTM.named_parameters + ~MCLSTM.parameters + ~MCLSTM.register_backward_hook + ~MCLSTM.register_buffer + ~MCLSTM.register_forward_hook + ~MCLSTM.register_forward_pre_hook + ~MCLSTM.register_full_backward_hook + ~MCLSTM.register_full_backward_pre_hook + ~MCLSTM.register_load_state_dict_post_hook + ~MCLSTM.register_load_state_dict_pre_hook + ~MCLSTM.register_module + ~MCLSTM.register_parameter + ~MCLSTM.register_state_dict_post_hook + ~MCLSTM.register_state_dict_pre_hook + ~MCLSTM.requires_grad_ + ~MCLSTM.set_extra_state + ~MCLSTM.set_submodule + ~MCLSTM.share_memory + ~MCLSTM.state_dict + ~MCLSTM.to + ~MCLSTM.to_empty + ~MCLSTM.train + ~MCLSTM.type + ~MCLSTM.xpu + ~MCLSTM.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MCLSTM.T_destination + ~MCLSTM.call_super_init + ~MCLSTM.dump_patches + ~MCLSTM.input_size + ~MCLSTM.hidden_size + ~MCLSTM.num_layers + ~MCLSTM.batch_first + ~MCLSTM.dropout + ~MCLSTM.cells + ~MCLSTM.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MCLSTMCell.rst b/docs/generated/torchrecurrent.MCLSTMCell.rst new file mode 100644 index 0000000..a6d4005 --- /dev/null +++ b/docs/generated/torchrecurrent.MCLSTMCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.MCLSTMCell +========================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: MCLSTMCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MCLSTMCell.__init__ + ~MCLSTMCell.add_module + ~MCLSTMCell.apply + ~MCLSTMCell.bfloat16 + ~MCLSTMCell.buffers + ~MCLSTMCell.children + ~MCLSTMCell.compile + ~MCLSTMCell.cpu + ~MCLSTMCell.cuda + ~MCLSTMCell.double + ~MCLSTMCell.eval + ~MCLSTMCell.extra_repr + ~MCLSTMCell.float + ~MCLSTMCell.forward + ~MCLSTMCell.get_buffer + ~MCLSTMCell.get_extra_state + ~MCLSTMCell.get_parameter + ~MCLSTMCell.get_submodule + ~MCLSTMCell.half + ~MCLSTMCell.ipu + ~MCLSTMCell.load_state_dict + ~MCLSTMCell.modules + ~MCLSTMCell.mtia + ~MCLSTMCell.named_buffers + ~MCLSTMCell.named_children + ~MCLSTMCell.named_modules + ~MCLSTMCell.named_parameters + ~MCLSTMCell.parameters + ~MCLSTMCell.register_backward_hook + ~MCLSTMCell.register_buffer + ~MCLSTMCell.register_forward_hook + ~MCLSTMCell.register_forward_pre_hook + ~MCLSTMCell.register_full_backward_hook + ~MCLSTMCell.register_full_backward_pre_hook + ~MCLSTMCell.register_load_state_dict_post_hook + ~MCLSTMCell.register_load_state_dict_pre_hook + ~MCLSTMCell.register_module + ~MCLSTMCell.register_parameter + ~MCLSTMCell.register_state_dict_post_hook + ~MCLSTMCell.register_state_dict_pre_hook + ~MCLSTMCell.requires_grad_ + ~MCLSTMCell.reset_parameters + ~MCLSTMCell.set_extra_state + ~MCLSTMCell.set_submodule + ~MCLSTMCell.share_memory + ~MCLSTMCell.state_dict + ~MCLSTMCell.to + ~MCLSTMCell.to_empty + ~MCLSTMCell.train + ~MCLSTMCell.type + ~MCLSTMCell.uses_double_state + ~MCLSTMCell.xpu + ~MCLSTMCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MCLSTMCell.T_destination + ~MCLSTMCell.call_super_init + ~MCLSTMCell.dump_patches + ~MCLSTMCell.weight_ih + ~MCLSTMCell.weight_hh + ~MCLSTMCell.bias_ih + ~MCLSTMCell.bias_hh + ~MCLSTMCell.input_size + ~MCLSTMCell.hidden_size + ~MCLSTMCell.bias + ~MCLSTMCell.recurrent_bias + ~MCLSTMCell.training + + \ No newline at end of file diff --git a/docs/models.rst b/docs/models.rst index 5c5534b..043b6b0 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -62,6 +62,9 @@ references and official implementations where available. * - :doc:`LightRU ` - `MDPI Electronics 2023 `__ - – + * - :doc:`MCLSTM ` + - `NIPS 2017 workshop `__ + - – * - :doc:`MGU ` - `IJAC 2016 `__ - – diff --git a/tests/test_cells.py b/tests/test_cells.py index d06bf3d..66bbd4f 100755 --- a/tests/test_cells.py +++ b/tests/test_cells.py @@ -22,6 +22,7 @@ IntersectionRNNCell, LiGRUCell, LightRUCell, + MCLSTMCell, MultiplicativeLSTMCell, MUT1Cell, MUT2Cell, @@ -305,6 +306,31 @@ def test_tgru_cell_matches_paper_update(): assert torch.equal(new_x_prev, x) +def test_tgru_cell_accepts_per_component_none_state(): + """Matches the (None, None) sentinel convention every other double-state + cell in this library supports (see tests/test_cells.py's CELL_CASES + harness), even though TGRU isn't itself in that harness.""" + cell = TGRUCell(3, 5) + x = torch.randn(3) + + h, x_prev = cell(x, (None, None)) + assert h.shape == (5,) + assert x_prev.shape == (3,) + + h2, x_prev2 = cell(x, (h, None)) + assert h2.shape == (5,) + assert x_prev2.shape == (3,) + + x_batched = torch.randn(2, 3) + h3, x_prev3 = cell(x_batched, (None, None)) + assert h3.shape == (2, 5) + assert x_prev3.shape == (2, 3) + + h4, x_prev4 = cell(x_batched, (None, x_prev3)) + assert h4.shape == (2, 5) + assert x_prev4.shape == (2, 3) + + def test_intersectionrnn_cell_requires_equal_sizes(): with pytest.raises(ValueError, match="input_size == hidden_size"): IntersectionRNNCell(3, 5) @@ -371,6 +397,90 @@ def test_intersectionrnn_cell_matches_paper_update(): assert torch.allclose(new_h, expected_h) +def test_mclstm_cell_shapes(): + cell = MCLSTMCell(4, 9) + + assert cell.weight_ih.shape == (45, 4) + assert cell.weight_hh.shape == (45, 9) + assert cell.bias_ih.shape == (45,) + assert cell.bias_hh.shape == (45,) + + +def test_mclstm_cell_output_differs_from_state(): + """h(t) is a pure output: it must differ from both v(t) and c(t).""" + cell = MCLSTMCell(4, 4) + + x_unbatched = torch.randn(4) + h_u, (v_u, c_u) = cell(x_unbatched) + assert h_u.shape == v_u.shape == c_u.shape == (4,) + assert not torch.allclose(h_u, v_u) + assert not torch.allclose(h_u, c_u) + + x = torch.randn(3, 4) + h, (v, c) = cell(x) + assert h.shape == v.shape == c.shape == (3, 4) + assert not torch.allclose(h, v) + + +def test_mclstm_cell_accepts_per_component_none_state(): + cell = MCLSTMCell(3, 5) + x = torch.randn(3) + + h, (v, c) = cell(x, (None, None)) + assert h.shape == v.shape == c.shape == (5,) + + h2, (v2, c2) = cell(x, (v, None)) + assert h2.shape == v2.shape == c2.shape == (5,) + + x_batched = torch.randn(2, 3) + h3, (v3, c3) = cell(x_batched, (None, None)) + assert h3.shape == v3.shape == c3.shape == (2, 5) + + h4, (v4, c4) = cell(x_batched, (None, c3)) + assert h4.shape == v4.shape == c4.shape == (2, 5) + + +def test_mclstm_cell_gradients(): + cell = MCLSTMCell(4, 5, bias=False) + x = torch.randn(2, 4, requires_grad=True) + + h, (v, c) = cell(x) + (h.sum() + v.sum() + c.sum()).backward() + + assert x.grad is not None + for p in cell.parameters(): + if p.requires_grad: + assert p.grad is not None + + +def test_mclstm_cell_matches_paper_update(): + cell = MCLSTMCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(20, dtype=torch.float32).reshape(10, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(20, dtype=torch.float32).reshape(10, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + v = torch.tensor([[0.1, -0.2]]) + c = torch.tensor([[0.4, -0.5]]) + + h, (new_v, new_c) = cell(x, (v, c)) + gates = x @ cell.weight_ih.t() + v @ cell.weight_hh.t() + f_pre, i_pre, o_pre, m_pre, n_pre = gates.chunk(5, 1) + forget_gate = torch.sigmoid(f_pre) + input_gate = torch.sigmoid(i_pre) + output_gate = torch.sigmoid(o_pre) + control_gate = torch.sigmoid(m_pre) + candidate = torch.tanh(n_pre) + + expected_c = forget_gate * c + input_gate * candidate + expected_h = output_gate * torch.tanh(expected_c) + expected_v = control_gate * torch.tanh(expected_c) + + assert torch.allclose(new_c, expected_c) + assert torch.allclose(h, expected_h) + assert torch.allclose(new_v, expected_v) + + @pytest.mark.parametrize("Cell, in_size, hid_size, _", CELL_CASES) def test_cell_gradients(Cell, in_size, hid_size, _): """A quick smoke test: outputs should be differentiable wrt parameters.""" diff --git a/tests/test_layers.py b/tests/test_layers.py index 17da956..ebf4dd2 100755 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -17,6 +17,7 @@ IntersectionRNN, LiGRU, LightRU, + MCLSTM, MGU, MultiplicativeLSTM, MUT1, @@ -53,6 +54,7 @@ IndRNN, LiGRU, LightRU, + MCLSTM, MGU, MultiplicativeLSTM, MUT1, @@ -90,6 +92,7 @@ (IndRNN, False), (LiGRU, False), (LightRU, False), + (MCLSTM, True), (MGU, False), (MultiplicativeLSTM, True), (MUT1, False), @@ -425,6 +428,85 @@ def test_intersectionrnn_repr(): assert "batch_first=True" in r +def test_mclstm_state_continuity_matches_single_call(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = MCLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out_full, (v_full, c_full) = layer(x) + + out1, state1 = layer(x[:split]) + out2, (v2, c2) = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + assert torch.allclose(v2, v_full, atol=1e-6) + assert torch.allclose(c2, c_full, atol=1e-6) + + +def test_mclstm_stacking_forwards_output_not_control_state(): + """Layer k+1 must receive layer k's h(t) (output), not its v(t)/c(t).""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = MCLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out, (vn, cn) = layer(x) + + v_prev = [torch.zeros(batch_size, hidden_size) for _ in range(num_layers)] + c_prev = [torch.zeros(batch_size, hidden_size) for _ in range(num_layers)] + expected_outputs = [] + for t in range(seq_len): + layer_inp = x[t] + for layer_idx, cell in enumerate(layer.cells): + h, (v_new, c_new) = cell(layer_inp, (v_prev[layer_idx], c_prev[layer_idx])) + v_prev[layer_idx] = v_new + c_prev[layer_idx] = c_new + layer_inp = h + expected_outputs.append(layer_inp) + + assert torch.allclose(out, torch.stack(expected_outputs, dim=0), atol=1e-6) + for layer_idx in range(num_layers): + assert torch.allclose(vn[layer_idx], v_prev[layer_idx], atol=1e-6) + assert torch.allclose(cn[layer_idx], c_prev[layer_idx], atol=1e-6) + + +def test_mclstm_dropout_only_between_layers(): + input_size, hidden_size = 5, 5 + seq_len, batch_size = 4, 3 + + # num_layers=1: dropout is never applied, so the output must be + # bit-identical regardless of the dropout probability. + torch.manual_seed(0) + layer_no_drop = MCLSTM(input_size, hidden_size, dropout=0.0, bias=False) + torch.manual_seed(0) + layer_high_drop = MCLSTM(input_size, hidden_size, dropout=0.9, bias=False) + + x = torch.randn(seq_len, batch_size, input_size) + out_no_drop, _ = layer_no_drop(x) + out_high_drop, _ = layer_high_drop(x) + + assert torch.equal(out_no_drop, out_high_drop) + + # num_layers=2: dropout between layers must actually change the output. + torch.manual_seed(0) + stacked_no_drop = MCLSTM(input_size, hidden_size, num_layers=2, dropout=0.0, bias=False) + torch.manual_seed(0) + stacked_drop = MCLSTM(input_size, hidden_size, num_layers=2, dropout=0.9, bias=False) + stacked_drop.load_state_dict(stacked_no_drop.state_dict()) + + torch.manual_seed(1) + out_stacked_no_drop, _ = stacked_no_drop(x) + torch.manual_seed(1) + out_stacked_drop, _ = stacked_drop(x) + + assert not torch.allclose(out_stacked_no_drop, out_stacked_drop) + + @pytest.mark.parametrize("Layer", LAYER_CLASSES) def test_default_repr_shows_input_hidden(Layer): # Default repr should exactly match "Class(input_size, hidden_size)" diff --git a/torchrecurrent/__init__.py b/torchrecurrent/__init__.py index 9cf1899..9214294 100755 --- a/torchrecurrent/__init__.py +++ b/torchrecurrent/__init__.py @@ -17,6 +17,7 @@ LEMCell, LiGRUCell, LightRUCell, + MCLSTMCell, MGUCell, MiRU1Cell, MiRU2Cell, @@ -59,6 +60,7 @@ LEM, LiGRU, LightRU, + MCLSTM, MGU, MiRU1, MiRU2, @@ -99,6 +101,8 @@ "CFNCell", "DSGU", "DSGUCell", + "MCLSTM", + "MCLSTMCell", "MGU", "MGUCell", "MiRU1", diff --git a/torchrecurrent/cells/__init__.py b/torchrecurrent/cells/__init__.py index 7bd5eeb..7d1fb02 100755 --- a/torchrecurrent/cells/__init__.py +++ b/torchrecurrent/cells/__init__.py @@ -9,6 +9,7 @@ from .atr_cell import ATRCell, ATR from .br_cell import BR, BRCell, NBR, NBRCell from .cfn_cell import CFN, CFNCell +from .mclstm_cell import MCLSTM, MCLSTMCell from .mgu_cell import MGU, MGUCell from .miru_cell import MiRU1, MiRU1Cell, MiRU2, MiRU2Cell from .cornn_cell import coRNN, coRNNCell @@ -52,6 +53,8 @@ "NBRCell", "CFN", "CFNCell", + "MCLSTM", + "MCLSTMCell", "MGU", "MGUCell", "MiRU1", diff --git a/torchrecurrent/cells/mclstm_cell.py b/torchrecurrent/cells/mclstm_cell.py new file mode 100644 index 0000000..e6fc94c --- /dev/null +++ b/torchrecurrent/cells/mclstm_cell.py @@ -0,0 +1,347 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional, Tuple +from ..base import ( + DecoupledDoubleStateRecurrentLayerBase, + DecoupledDoubleStateCellBase, + resolve_activation, + resolve_init_name, +) + + +class MCLSTM(DecoupledDoubleStateRecurrentLayerBase): + r"""Multi-layer memory-controller LSTM (MC-LSTM). + + [`NIPS 2017 workshop `_] + + Each layer consists of an :class:`MCLSTMCell`, which decouples the LSTM's + "prediction" output from the recurrent signal that drives its own gates: + + .. math:: + \begin{aligned} + f_t &= \sigma(W_{ih}^f x_t + b_{ih}^f + W_{hh}^f v_{t-1} + b_{hh}^f), \\ + i_t &= \sigma(W_{ih}^i x_t + b_{ih}^i + W_{hh}^i v_{t-1} + b_{hh}^i), \\ + o_t &= \sigma(W_{ih}^o x_t + b_{ih}^o + W_{hh}^o v_{t-1} + b_{hh}^o), \\ + m_t &= \sigma(W_{ih}^m x_t + b_{ih}^m + W_{hh}^m v_{t-1} + b_{hh}^m), \\ + n_t &= \tanh(W_{ih}^n x_t + b_{ih}^n + W_{hh}^n v_{t-1} + b_{hh}^n), \\ + c_t &= f_t \circ c_{t-1} + i_t \circ n_t, \\ + h_t &= o_t \circ \tanh(c_t), \\ + v_t &= m_t \circ \tanh(c_t) + \end{aligned} + + where :math:`\sigma` is the sigmoid function and :math:`\circ` denotes + elementwise (Hadamard) product. Every gate is driven by the *control + vector* :math:`v_{t-1}`, never by :math:`h_{t-1}`: this is the paper's + central idea, decoupling the "prediction" output :math:`h_t` from the + memory-management role a standard LSTM's hidden state also has to play. + :math:`h_t` is returned as the layer's output but does not itself recur; + the recurring state is :math:`(v_t, c_t)`. + + In a multilayer MCLSTM, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the *output* :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden and control states. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two MCLSTM layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the four gates. Default: + :func:`torch.sigmoid` + cell_nonlinearity: Activation for the candidate content `n_t`. Default: + :func:`torch.tanh` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (v_0, c_0) + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **v_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial control state. Defaults to zeros if not + provided. + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not + provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, (v_n, c_n) + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the MCLSTM, for each `t`. + - **v_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final control state for each element in the + sequence. Note this is `v_n`, not `h_n`: the control state and the + returned output are different tensors for every layer. + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(5*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(5*hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable weights applied to the control state of + the :math:`k`-th layer, of shape `(5*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(5*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable biases paired with `weight_hh` of the + :math:`k`-th layer, of shape `(5*hidden_size)`. Only present when + ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`MCLSTMCell` + + Examples:: + + >>> rnn = MCLSTM(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> v0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> c0 = torch.zeros(2, 3, 20) + >>> output, (vn, cn) = rnn(input, (v0, c0)) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(MCLSTM, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(MCLSTMCell, **kwargs) + + +class MCLSTMCell(DecoupledDoubleStateCellBase): + r"""A memory-controller LSTM (MC-LSTM) cell. + + [`NIPS 2017 workshop `_] + + .. math:: + + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{f}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{f} + + \mathbf{W}_{hh}^{f}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{f} + \bigr), \\[4pt] + \mathbf{i}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{i}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{i} + + \mathbf{W}_{hh}^{i}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{i} + \bigr), \\[4pt] + \mathbf{o}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{o}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{o} + + \mathbf{W}_{hh}^{o}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{o} + \bigr), \\[4pt] + \mathbf{m}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{m}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{m} + + \mathbf{W}_{hh}^{m}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{m} + \bigr), \\[4pt] + \mathbf{n}(t) &= \tanh\bigl( + \mathbf{W}_{ih}^{n}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{n} + + \mathbf{W}_{hh}^{n}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{n} + \bigr), \\[4pt] + \mathbf{c}(t) &= \mathbf{f}(t)\circ\mathbf{c}(t-1) + + \mathbf{i}(t)\circ\mathbf{n}(t), \\[4pt] + \mathbf{h}(t) &= \mathbf{o}(t)\circ\tanh\bigl(\mathbf{c}(t)\bigr), \\[4pt] + \mathbf{v}(t) &= \mathbf{m}(t)\circ\tanh\bigl(\mathbf{c}(t)\bigr), + + where :math:`\sigma` is the sigmoid function and :math:`\circ` is + element-wise product. Every gate depends on the *control vector* + :math:`\mathbf{v}(t-1)`, never on :math:`\mathbf{h}(t-1)`. The returned + output :math:`\mathbf{h}(t)` and the recurring state + :math:`(\mathbf{v}(t), \mathbf{c}(t))` are different tensors: + :math:`\mathbf{h}(t)` is free to specialize as a pure prediction, since + it plays no role in the next step's gating. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden and control states. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use the biases + paired with ``W_{hh}``. Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the four gates. + Default: :func:`torch.sigmoid`. + cell_nonlinearity: Activation for the candidate content ``n``. + Default: :func:`torch.tanh`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (v_0, c_0) + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **v_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial control state. + - **c_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial cell state. + + If not provided, both default to zero. + + Outputs: h_1, (v_1, c_1) + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the output (prediction). + - **v_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next control state. + - **c_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next cell state. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(5*hidden_size, input_size)`` (``f, i, o, m, n`` parts). + weight_hh: The learnable weights applied to the control state, + of shape ``(5*hidden_size, hidden_size)`` (``f, i, o, m, n`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(5*hidden_size)`` if ``bias=True``. + bias_hh: The learnable biases paired with ``weight_hh``, + of shape ``(5*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = MCLSTMCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> v = torch.zeros(3, 20) # (batch, hidden_size) + >>> c = torch.zeros(3, 20) + >>> out = [] + >>> for t in range(x.size(0)): + ... h, (v, c) = cell(x[t], (v, c)) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + gate_nonlinearity="sigmoid", + cell_nonlinearity="tanh", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.cell_act = resolve_activation(cell_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._default_register_tensors(ih_mult=5, hh_mult=5) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_v = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + v0, c0 = state + b_v = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if v0 is None + else (v0.unsqueeze(0) if (not is_batched and v0.dim() == 1) else v0) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c0 is None + else (c0.unsqueeze(0) if (not is_batched and c0.dim() == 1) else c0) + ) + + gates = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + b_v @ self.weight_hh.t() + + self.bias_hh + ) + forget_pre, input_pre, output_pre, control_pre, cell_pre = gates.chunk(5, 1) + + forget_gate = self.gate_act(forget_pre) + input_gate = self.gate_act(input_pre) + output_gate = self.gate_act(output_pre) + control_gate = self.gate_act(control_pre) + candidate = self.cell_act(cell_pre) + + new_c = forget_gate * b_c + input_gate * candidate + squashed_c = torch.tanh(new_c) + new_h = output_gate * squashed_c + new_v = control_gate * squashed_c + + if not is_batched: + new_h = new_h.squeeze(0) + new_v = new_v.squeeze(0) + new_c = new_c.squeeze(0) + + return new_h, (new_v, new_c) diff --git a/torchrecurrent/cells/trnn_cell.py b/torchrecurrent/cells/trnn_cell.py index 17186b5..d509c81 100644 --- a/torchrecurrent/cells/trnn_cell.py +++ b/torchrecurrent/cells/trnn_cell.py @@ -596,11 +596,21 @@ def forward( ) else: h0, prev_inp0 = state - b_state = h0.unsqueeze(0) if (not is_batched and h0.dim() == 1) else h0 + b_state = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h0 is None + else (h0.unsqueeze(0) if (not is_batched and h0.dim() == 1) else h0) + ) b_prev_inp = ( - prev_inp0.unsqueeze(0) - if (not is_batched and prev_inp0.dim() == 1) - else prev_inp0 + torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + if prev_inp0 is None + else ( + prev_inp0.unsqueeze(0) + if (not is_batched and prev_inp0.dim() == 1) + else prev_inp0 + ) ) gates_ih = b_inp @ self.weight_ih.t() + self.bias_ih From 646aa7987ccdfb907394da01bef3b80a8508272a Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 20 Sep 2026 16:48:57 +0200 Subject: [PATCH 4/5] feat: add TLSTM, completing the strongly-typed RNN family Ports T-LSTM from Balduzzi & Ghifary, "Strongly-Typed Recurrent Neural Networks" (ICML 2016, arXiv:1602.02218) -- the same paper TRNN and TGRU (already committed) came from. Verified against the actual paper HTML (equations 25-29). This is the trickiest of the five cells this branch set out to add, since it combines two issues each independently solved for other cells on this branch: - Like TGRU, the recurrent weights act on the previous *input* x(t-1), not a hidden state, so the second recurring state component is ragged across stacked layers (input_size-shaped for layer 0, hidden_size-shaped for later layers). - Like MCLSTM and IntersectionRNN, h(t) is a pure output that plays no role in any subsequent computation -- only (c(t), x(t)) recur. TLSTM therefore needed a custom forward() override (like TGRU, for the ragged state) built on DecoupledDoubleStateCellBase (like MCLSTM, for the decoupled output) -- the first cell needing both fixes at once. The critical correctness risk was the layer-stacking loop routing the wrong tensor to the next layer (e.g. threading c or x_prev instead of h, which would silently produce a different, incorrect architecture while still passing shape-only checks). This is independently verified by test_tlstm_stacking_forwards_output_not_state, which calls the cells directly rather than reusing any of the layer's own routing logic. Codex audit found no implementation bugs -- it specifically confirmed the stacking oracle would catch a routing bug by mutation-testing it (deliberately routing the wrong tensor and confirming the test fails) -- but found two docstring inaccuracies in the TLSTM layer docstring: it claimed to be "unlike every other cell" in decoupling output from state (false; MCLSTM and IntersectionRNN already do this), and it described the recurring state using x(t-1) notation when it's actually x(t) that carries forward (becoming x(t-1) on the next call). Both fixed. This completes all 5 models found missing from the sister Julia libraries (RecurrentLayers.jl, LuxRecurrentLayers.jl) on this branch: TRNN, TGRU, TLSTM, MCLSTM, IntersectionRNN. Co-Authored-By: Claude Sonnet 5 --- docs/api/cells.rst | 1 + docs/api/layers.rst | 1 + docs/generated/torchrecurrent.TLSTM.rst | 88 +++++ docs/generated/torchrecurrent.TLSTMCell.rst | 91 +++++ docs/models.rst | 3 + tests/test_cells.py | 85 +++++ tests/test_layers.py | 152 ++++++++ torchrecurrent/__init__.py | 4 + torchrecurrent/cells/__init__.py | 4 +- torchrecurrent/cells/trnn_cell.py | 396 ++++++++++++++++++++ 10 files changed, 824 insertions(+), 1 deletion(-) create mode 100644 docs/generated/torchrecurrent.TLSTM.rst create mode 100644 docs/generated/torchrecurrent.TLSTMCell.rst diff --git a/docs/api/cells.rst b/docs/api/cells.rst index 5b91712..187be37 100755 --- a/docs/api/cells.rst +++ b/docs/api/cells.rst @@ -43,6 +43,7 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.tauGRUCell torchrecurrent.TRNNCell torchrecurrent.TGRUCell + torchrecurrent.TLSTMCell torchrecurrent.UGRNNCell torchrecurrent.UnICORNNCell torchrecurrent.WMCLSTMCell diff --git a/docs/api/layers.rst b/docs/api/layers.rst index 715e436..1e07e17 100644 --- a/docs/api/layers.rst +++ b/docs/api/layers.rst @@ -43,6 +43,7 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.tauGRU torchrecurrent.TRNN torchrecurrent.TGRU + torchrecurrent.TLSTM torchrecurrent.UGRNN torchrecurrent.UnICORNN torchrecurrent.WMCLSTM diff --git a/docs/generated/torchrecurrent.TLSTM.rst b/docs/generated/torchrecurrent.TLSTM.rst new file mode 100644 index 0000000..7dceb0a --- /dev/null +++ b/docs/generated/torchrecurrent.TLSTM.rst @@ -0,0 +1,88 @@ +torchrecurrent.TLSTM +==================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TLSTM + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TLSTM.__init__ + ~TLSTM.add_module + ~TLSTM.apply + ~TLSTM.bfloat16 + ~TLSTM.buffers + ~TLSTM.children + ~TLSTM.compile + ~TLSTM.cpu + ~TLSTM.cuda + ~TLSTM.double + ~TLSTM.eval + ~TLSTM.extra_repr + ~TLSTM.float + ~TLSTM.forward + ~TLSTM.get_buffer + ~TLSTM.get_extra_state + ~TLSTM.get_parameter + ~TLSTM.get_submodule + ~TLSTM.half + ~TLSTM.initialize_cells + ~TLSTM.ipu + ~TLSTM.load_state_dict + ~TLSTM.modules + ~TLSTM.mtia + ~TLSTM.named_buffers + ~TLSTM.named_children + ~TLSTM.named_modules + ~TLSTM.named_parameters + ~TLSTM.parameters + ~TLSTM.register_backward_hook + ~TLSTM.register_buffer + ~TLSTM.register_forward_hook + ~TLSTM.register_forward_pre_hook + ~TLSTM.register_full_backward_hook + ~TLSTM.register_full_backward_pre_hook + ~TLSTM.register_load_state_dict_post_hook + ~TLSTM.register_load_state_dict_pre_hook + ~TLSTM.register_module + ~TLSTM.register_parameter + ~TLSTM.register_state_dict_post_hook + ~TLSTM.register_state_dict_pre_hook + ~TLSTM.requires_grad_ + ~TLSTM.set_extra_state + ~TLSTM.set_submodule + ~TLSTM.share_memory + ~TLSTM.state_dict + ~TLSTM.to + ~TLSTM.to_empty + ~TLSTM.train + ~TLSTM.type + ~TLSTM.xpu + ~TLSTM.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TLSTM.T_destination + ~TLSTM.call_super_init + ~TLSTM.dump_patches + ~TLSTM.input_size + ~TLSTM.hidden_size + ~TLSTM.num_layers + ~TLSTM.batch_first + ~TLSTM.dropout + ~TLSTM.cells + ~TLSTM.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TLSTMCell.rst b/docs/generated/torchrecurrent.TLSTMCell.rst new file mode 100644 index 0000000..dc7d782 --- /dev/null +++ b/docs/generated/torchrecurrent.TLSTMCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.TLSTMCell +======================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TLSTMCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TLSTMCell.__init__ + ~TLSTMCell.add_module + ~TLSTMCell.apply + ~TLSTMCell.bfloat16 + ~TLSTMCell.buffers + ~TLSTMCell.children + ~TLSTMCell.compile + ~TLSTMCell.cpu + ~TLSTMCell.cuda + ~TLSTMCell.double + ~TLSTMCell.eval + ~TLSTMCell.extra_repr + ~TLSTMCell.float + ~TLSTMCell.forward + ~TLSTMCell.get_buffer + ~TLSTMCell.get_extra_state + ~TLSTMCell.get_parameter + ~TLSTMCell.get_submodule + ~TLSTMCell.half + ~TLSTMCell.ipu + ~TLSTMCell.load_state_dict + ~TLSTMCell.modules + ~TLSTMCell.mtia + ~TLSTMCell.named_buffers + ~TLSTMCell.named_children + ~TLSTMCell.named_modules + ~TLSTMCell.named_parameters + ~TLSTMCell.parameters + ~TLSTMCell.register_backward_hook + ~TLSTMCell.register_buffer + ~TLSTMCell.register_forward_hook + ~TLSTMCell.register_forward_pre_hook + ~TLSTMCell.register_full_backward_hook + ~TLSTMCell.register_full_backward_pre_hook + ~TLSTMCell.register_load_state_dict_post_hook + ~TLSTMCell.register_load_state_dict_pre_hook + ~TLSTMCell.register_module + ~TLSTMCell.register_parameter + ~TLSTMCell.register_state_dict_post_hook + ~TLSTMCell.register_state_dict_pre_hook + ~TLSTMCell.requires_grad_ + ~TLSTMCell.reset_parameters + ~TLSTMCell.set_extra_state + ~TLSTMCell.set_submodule + ~TLSTMCell.share_memory + ~TLSTMCell.state_dict + ~TLSTMCell.to + ~TLSTMCell.to_empty + ~TLSTMCell.train + ~TLSTMCell.type + ~TLSTMCell.uses_double_state + ~TLSTMCell.xpu + ~TLSTMCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TLSTMCell.T_destination + ~TLSTMCell.call_super_init + ~TLSTMCell.dump_patches + ~TLSTMCell.weight_ih + ~TLSTMCell.weight_hh + ~TLSTMCell.bias_ih + ~TLSTMCell.bias_hh + ~TLSTMCell.input_size + ~TLSTMCell.hidden_size + ~TLSTMCell.bias + ~TLSTMCell.recurrent_bias + ~TLSTMCell.training + + \ No newline at end of file diff --git a/docs/models.rst b/docs/models.rst index 043b6b0..8fd2aba 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -125,6 +125,9 @@ references and official implementations where available. * - :doc:`TGRU ` - `ICML 2016 `__ - – + * - :doc:`TLSTM ` + - `ICML 2016 `__ + - – * - :doc:`UGRNN ` - `ICLR 2017 `__ - – diff --git a/tests/test_cells.py b/tests/test_cells.py index 66bbd4f..b1c3dd9 100755 --- a/tests/test_cells.py +++ b/tests/test_cells.py @@ -39,6 +39,7 @@ tauGRUCell, TRNNCell, TGRUCell, + TLSTMCell, UGRNNCell, UnICORNNCell, WMCLSTMCell, @@ -331,6 +332,90 @@ def test_tgru_cell_accepts_per_component_none_state(): assert x_prev4.shape == (2, 3) +def test_tlstm_cell_parameter_shapes(): + """weight_hh acts on the previous *input*, so it shares weight_ih's shape.""" + cell = TLSTMCell(4, 9) + + assert cell.weight_ih.shape == (27, 4) + assert cell.weight_hh.shape == (27, 4) + assert cell.bias_ih.shape == (27,) + assert cell.bias_hh.shape == (27,) + + +def test_tlstm_cell_output_differs_from_state(): + """h(t) is a pure output: it plays no role in any later computation.""" + cell = TLSTMCell(4, 4) + + x_unbatched = torch.randn(4) + h_u, (c_u, xprev_u) = cell(x_unbatched) + assert h_u.shape == c_u.shape == (4,) + assert xprev_u.shape == (4,) + assert not torch.allclose(h_u, c_u) + + x = torch.randn(3, 4) + h, (c, xprev) = cell(x) + assert h.shape == c.shape == (3, 4) + assert not torch.allclose(h, c) + + +def test_tlstm_cell_accepts_per_component_none_state(): + cell = TLSTMCell(3, 5) + x = torch.randn(3) + + h, (c, xprev) = cell(x, (None, None)) + assert h.shape == c.shape == (5,) + assert xprev.shape == (3,) + + h2, (c2, xprev2) = cell(x, (c, None)) + assert h2.shape == c2.shape == (5,) + assert xprev2.shape == (3,) + + x_batched = torch.randn(2, 3) + h3, (c3, xprev3) = cell(x_batched, (None, None)) + assert h3.shape == c3.shape == (2, 5) + assert xprev3.shape == (2, 3) + + h4, (c4, xprev4) = cell(x_batched, (None, xprev3)) + assert h4.shape == c4.shape == (2, 5) + assert xprev4.shape == (2, 3) + + +def test_tlstm_cell_gradients(): + cell = TLSTMCell(4, 5, bias=False) + x = torch.randn(2, 4, requires_grad=True) + + h, (c, xprev) = cell(x) + (h.sum() + c.sum()).backward() + + assert x.grad is not None + for p in cell.parameters(): + if p.requires_grad: + assert p.grad is not None + + +def test_tlstm_cell_matches_paper_update(): + cell = TLSTMCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + c = torch.tensor([[0.4, -0.5]]) + x_prev = torch.tensor([[-0.1, 0.6]]) + + h, (new_c, new_x_prev) = cell(x, (c, x_prev)) + gates = x @ cell.weight_ih.t() + x_prev @ cell.weight_hh.t() + latent, forget_pre, out_pre = gates.chunk(3, 1) + forget_gate = torch.sigmoid(forget_pre) + output_candidate = torch.tanh(out_pre) + expected_c = forget_gate * c + (1 - forget_gate) * latent + expected_h = expected_c * output_candidate + + assert torch.allclose(new_c, expected_c) + assert torch.allclose(h, expected_h) + assert torch.equal(new_x_prev, x) + + def test_intersectionrnn_cell_requires_equal_sizes(): with pytest.raises(ValueError, match="input_size == hidden_size"): IntersectionRNNCell(3, 5) diff --git a/tests/test_layers.py b/tests/test_layers.py index ebf4dd2..566ac5a 100755 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -35,6 +35,7 @@ tauGRU, TRNN, TGRU, + TLSTM, UGRNN, UnICORNN, WMCLSTM, @@ -72,6 +73,7 @@ tauGRU, TRNN, TGRU, + TLSTM, UGRNN, UnICORNN, WMCLSTM, @@ -302,6 +304,156 @@ def test_tgru_batch_first_matches_seq_first(): assert xn[1].shape == (batch_size, hidden_size) +def test_tlstm_single_layer_supports_differing_sizes(): + input_size, hidden_size = 5, 7 + seq_len, batch_size = 4, 3 + + layer = TLSTM(input_size, hidden_size, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert cn.shape == (1, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == 1 + assert xn[0].shape == (batch_size, input_size) + # output and cell state are genuinely different tensors + assert not torch.allclose(out[-1], cn[-1]) + + +def test_tlstm_stacked_layers_have_per_layer_previous_input_shapes(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert cn.shape == (num_layers, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == num_layers + assert xn[0].shape == (batch_size, input_size) + assert xn[1].shape == (batch_size, hidden_size) + + +def test_tlstm_previous_input_state_holds_last_seen_input(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + + assert torch.equal(xn[0], x[-1]) + + # layer 1's last seen input is layer 0's last *output* h(t), not layer 0's + # own (c, x_prev) state -- output and state are different tensors here. + c0 = torch.zeros(batch_size, hidden_size) + x0_prev = torch.zeros(batch_size, input_size) + h0 = None + for t in range(seq_len): + h0, (c0, x0_prev) = layer.cells[0](x[t], (c0, x0_prev)) + assert torch.allclose(xn[1], h0) + + +def test_tlstm_stacking_forwards_output_not_state(): + """Layer k+1 must receive layer k's h(t) (output), not its c(t)/x(t-1).""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out, (cn, xn) = layer(x) + + c_prev = [torch.zeros(batch_size, hidden_size) for _ in range(num_layers)] + x_prev = [ + torch.zeros(batch_size, layer.cells[idx].input_size) for idx in range(num_layers) + ] + expected_outputs = [] + for t in range(seq_len): + layer_inp = x[t] + for layer_idx, cell in enumerate(layer.cells): + h, (c_new, x_new) = cell(layer_inp, (c_prev[layer_idx], x_prev[layer_idx])) + c_prev[layer_idx] = c_new + x_prev[layer_idx] = x_new + layer_inp = h + expected_outputs.append(layer_inp) + + assert torch.allclose(out, torch.stack(expected_outputs, dim=0), atol=1e-6) + for layer_idx in range(num_layers): + assert torch.allclose(cn[layer_idx], c_prev[layer_idx], atol=1e-6) + assert torch.allclose(xn[layer_idx], x_prev[layer_idx], atol=1e-6) + + +def test_tlstm_state_continuity_matches_single_call(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out_full, (c_full, x_full) = layer(x) + + out1, state1 = layer(x[:split]) + out2, (c2, x2) = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + assert torch.allclose(c2, c_full, atol=1e-6) + for a, b in zip(x2, x_full): + assert torch.allclose(a, b, atol=1e-6) + + +def test_tlstm_dropout_only_between_layers(): + input_size, hidden_size = 5, 5 + seq_len, batch_size = 4, 3 + + torch.manual_seed(0) + layer_no_drop = TLSTM(input_size, hidden_size, dropout=0.0, bias=False) + torch.manual_seed(0) + layer_high_drop = TLSTM(input_size, hidden_size, dropout=0.9, bias=False) + + x = torch.randn(seq_len, batch_size, input_size) + out_no_drop, _ = layer_no_drop(x) + out_high_drop, _ = layer_high_drop(x) + + assert torch.equal(out_no_drop, out_high_drop) + + torch.manual_seed(0) + stacked_no_drop = TLSTM(input_size, hidden_size, num_layers=2, dropout=0.0, bias=False) + torch.manual_seed(0) + stacked_drop = TLSTM(input_size, hidden_size, num_layers=2, dropout=0.9, bias=False) + stacked_drop.load_state_dict(stacked_no_drop.state_dict()) + + torch.manual_seed(1) + out_stacked_no_drop, _ = stacked_no_drop(x) + torch.manual_seed(1) + out_stacked_drop, _ = stacked_drop(x) + + assert not torch.allclose(out_stacked_no_drop, out_stacked_drop) + + +def test_tlstm_batch_first_matches_seq_first(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + layer_bf = TLSTM( + input_size, hidden_size, num_layers=num_layers, bias=False, batch_first=True + ) + layer_bf.load_state_dict(layer.state_dict()) + + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + out_bf, (cn_bf, xn_bf) = layer_bf(x.transpose(0, 1)) + + assert torch.allclose(out_bf, out.transpose(0, 1)) + assert torch.allclose(cn_bf, cn) + for a, b in zip(xn_bf, xn): + assert torch.allclose(a, b) + + def test_intersectionrnn_requires_equal_sizes(): with pytest.raises(ValueError, match="input_size == hidden_size"): IntersectionRNN(3, 5) diff --git a/torchrecurrent/__init__.py b/torchrecurrent/__init__.py index 9214294..0cc8266 100755 --- a/torchrecurrent/__init__.py +++ b/torchrecurrent/__init__.py @@ -38,6 +38,7 @@ tauGRUCell, TRNNCell, TGRUCell, + TLSTMCell, UGRNNCell, UnICORNNCell, WMCLSTMCell, @@ -81,6 +82,7 @@ tauGRU, TRNN, TGRU, + TLSTM, UGRNN, UnICORNN, WMCLSTM, @@ -159,6 +161,8 @@ "TRNNCell", "TGRU", "TGRUCell", + "TLSTM", + "TLSTMCell", "UGRNN", "UGRNNCell", "UnICORNN", diff --git a/torchrecurrent/cells/__init__.py b/torchrecurrent/cells/__init__.py index 7d1fb02..211dcfd 100755 --- a/torchrecurrent/cells/__init__.py +++ b/torchrecurrent/cells/__init__.py @@ -32,7 +32,7 @@ from .sgrn_cell import SGRN, SGRNCell from .star_cell import STAR, STARCell from .taugru_cell import tauGRU, tauGRUCell -from .trnn_cell import TRNN, TRNNCell, TGRU, TGRUCell +from .trnn_cell import TRNN, TRNNCell, TGRU, TGRUCell, TLSTM, TLSTMCell from .ugrnn_cell import UGRNN, UGRNNCell from .unicornn_cell import UnICORNN, UnICORNNCell from .wmclstm_cell import WMCLSTM, WMCLSTMCell @@ -113,6 +113,8 @@ "TRNNCell", "TGRU", "TGRUCell", + "TLSTM", + "TLSTMCell", "UGRNN", "UGRNNCell", "UnICORNN", diff --git a/torchrecurrent/cells/trnn_cell.py b/torchrecurrent/cells/trnn_cell.py index d509c81..160c407 100644 --- a/torchrecurrent/cells/trnn_cell.py +++ b/torchrecurrent/cells/trnn_cell.py @@ -7,6 +7,8 @@ SingleStateCellBase, DoubleStateRecurrentLayerBase, DoubleStateCellBase, + DecoupledDoubleStateRecurrentLayerBase, + DecoupledDoubleStateCellBase, resolve_activation, resolve_init_name, ) @@ -627,3 +629,397 @@ def forward( new_prev_inp = new_prev_inp.squeeze(0) return new_state, new_prev_inp + + +class TLSTM(DecoupledDoubleStateRecurrentLayerBase): + r"""Multi-layer strongly typed long short-term memory (T-LSTM). + + [`arXiv `_] + + Each layer consists of a :class:`TLSTMCell`, which updates the cell + state according to: + + .. math:: + \begin{aligned} + z_t &= V^z x_{t-1} + W^z x_t + b^z, \\ + f_t &= \sigma(V^f x_{t-1} + W^f x_t + b^f), \\ + o_t &= \tau(V^o x_{t-1} + W^o x_t + b^o), \\ + c_t &= f_t \circ c_{t-1} + (1 - f_t) \circ z_t, \\ + h_t &= c_t \circ o_t + \end{aligned} + + where :math:`\sigma` is the sigmoid function, :math:`\tau` is a pointwise + nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + (Hadamard) product. As in T-GRU, the recurrent weights act on the + *previous input* :math:`x_{t-1}` rather than on a hidden state. As with + :class:`MCLSTM` and :class:`IntersectionRNN`, the returned output + :math:`h_t` is *not* part of the recurring state at all: only + :math:`(c_t, x_t)` carry forward to the next timestep (:math:`x_t` + becoming :math:`x_{t-1}` there). :math:`h_t` is a pure function of + quantities that are otherwise discarded after being used, so it is free + to specialize as an output without affecting the recurrence. + + In a multilayer TLSTM, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the *output* :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden and cell states. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two TLSTM layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. Default: + :func:`torch.sigmoid` + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh` + kernel_init: Initializer for `W`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `V`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (c_0, x_{-1}) + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not + provided. + - **x_{-1}**: tuple of :attr:`num_layers` tensors containing the + "previous input" fed to each layer before the sequence starts. The + :math:`k`-th tensor has shape :math:`(N, \text{input\_size})` for + `k = 0` and :math:`(N, H_{out})` for `k \ge 1`, since layers beyond + the first receive the previous layer's output as their input. A + single stacked tensor of shape :math:`(\text{num_layers}, N, + H_{in})` is also accepted whenever `input_size == hidden_size`. + Defaults to zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, (c_n, x_n) + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the TLSTM, for each `t`. + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. + - **x_n**: tuple of :attr:`num_layers` tensors, with the same + per-layer shapes as **x_{-1}**, containing the last input seen by + each layer. Returned as a tuple (not stacked into one tensor) + because layer 0's input dimension may differ from every other + layer's. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(3*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(3*hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable weights applied to the previous input of + the :math:`k`-th layer, of the same shape as `weight_ih`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(3*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable biases paired with `weight_hh` of the + :math:`k`-th layer, of shape `(3*hidden_size)`. Only present when + ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. note:: + Like TGRU, this layer overrides the generic decoupled double-state + forward loop: the "previous input" component of the state cannot be + stacked into a single tensor across layers when `input_size != + hidden_size`, and (unlike TGRU) the returned output is not part of + the recurring state at all. + + .. seealso:: + :class:`TLSTMCell` + + Examples:: + + >>> rnn = TLSTM(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> output, (cn, xn) = rnn(input) # default zero state + >>> xn[0].shape # layer 0: input_size-shaped + torch.Size([3, 10]) + >>> xn[1].shape # layer 1: hidden_size-shaped + torch.Size([3, 20]) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(TLSTM, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(TLSTMCell, **kwargs) + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tuple[Tensor, ...]]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tuple[Tensor, ...]]]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + c = [ + torch.zeros( + batch_size, self.hidden_size, dtype=inp.dtype, device=inp.device + ) + for _ in range(self.num_layers) + ] + x_prev = [ + torch.zeros(batch_size, cell.input_size, dtype=inp.dtype, device=inp.device) + for cell in self.cells + ] + else: + c0, x_prev0 = state + c = [c0[layer_idx] for layer_idx in range(self.num_layers)] + x_prev = list(x_prev0) + + outputs = [] + for t in range(seq_len): + x = inp[t] + new_c = [] + new_x_prev = [] + + for layer_idx, cell in enumerate(self.cells): + h, (c_i, x_prev_i) = cell(x, (c[layer_idx], x_prev[layer_idx])) + new_c.append(c_i) + new_x_prev.append(x_prev_i) + x = h + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + c = new_c + x_prev = new_x_prev + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + + c_n = torch.stack(c, dim=0) + x_n = tuple(x_prev) + return out, (c_n, x_n) + + +class TLSTMCell(DecoupledDoubleStateCellBase): + r"""A strongly typed long short-term memory (T-LSTM) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{z}(t) &= \mathbf{V}^{z}\,\mathbf{x}(t-1) + + \mathbf{W}^{z}\,\mathbf{x}(t) + \mathbf{b}^{z}, \\[4pt] + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{V}^{f}\,\mathbf{x}(t-1) + \mathbf{W}^{f}\,\mathbf{x}(t) + \mathbf{b}^{f} + \bigr), \\[4pt] + \mathbf{o}(t) &= \tau\bigl( + \mathbf{V}^{o}\,\mathbf{x}(t-1) + \mathbf{W}^{o}\,\mathbf{x}(t) + \mathbf{b}^{o} + \bigr), \\[4pt] + \mathbf{c}(t) &= \mathbf{f}(t)\circ\mathbf{c}(t-1) + + \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{z}(t), \\[4pt] + \mathbf{h}(t) &= \mathbf{c}(t)\circ\mathbf{o}(t), + + where :math:`\circ` is element-wise product and :math:`\tau` is a + pointwise nonlinearity (e.g., tanh). The recurrent weights act on the + previous input :math:`\mathbf{x}(t-1)`, not on a hidden state, per the + "strongly typed" design. The recurring state is + :math:`(\mathbf{c}(t), \mathbf{x}(t))`; the returned output + :math:`\mathbf{h}(t)` plays no role in any subsequent computation. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden and cell states. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use the biases + paired with ``V``. Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. + Default: :func:`torch.sigmoid`. + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh`. + kernel_init: Initializer for ``W``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``V``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (c_0, x_{-1}) + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **c_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial cell state. + - **x_{-1}** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing the "previous input". + + If not provided, both default to zero. + + Outputs: h_1, (c_1, x_1) + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the output. + - **c_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next cell state. + - **x_1**: the current input ``x``, carried forward unchanged to + serve as :math:`\mathbf{x}(t-1)` on the following call. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + weight_hh: The learnable weights applied to the previous input, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(3*hidden_size)`` if ``bias=True``. + bias_hh: The learnable biases paired with ``weight_hh``, + of shape ``(3*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = TLSTMCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> c = torch.zeros(3, 20) # (batch, hidden_size) + >>> x_prev = torch.zeros(3, 10) # (batch, input_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h, (c, x_prev) = cell(x[t], (c, x_prev)) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + gate_nonlinearity="sigmoid", + output_nonlinearity="tanh", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.out_act = resolve_activation(output_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((3 * hidden_size, input_size), True), + "weight_hh": ((3 * hidden_size, input_size), True), + "bias_ih": ((3 * hidden_size,), self.bias), + "bias_hh": ((3 * hidden_size,), self.recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + b_prev_inp = torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + else: + c0, prev_inp0 = state + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c0 is None + else (c0.unsqueeze(0) if (not is_batched and c0.dim() == 1) else c0) + ) + b_prev_inp = ( + torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + if prev_inp0 is None + else ( + prev_inp0.unsqueeze(0) + if (not is_batched and prev_inp0.dim() == 1) + else prev_inp0 + ) + ) + + gates_ih = b_inp @ self.weight_ih.t() + self.bias_ih + gates_hh = b_prev_inp @ self.weight_hh.t() + self.bias_hh + latent, forget_pre, out_pre = (gates_ih + gates_hh).chunk(3, 1) + + forget_gate = self.gate_act(forget_pre) + output_candidate = self.out_act(out_pre) + new_c = forget_gate * b_c + (1.0 - forget_gate) * latent + new_h = new_c * output_candidate + new_prev_inp = b_inp + + if not is_batched: + new_h = new_h.squeeze(0) + new_c = new_c.squeeze(0) + new_prev_inp = new_prev_inp.squeeze(0) + + return new_h, (new_c, new_prev_inp) From aee7c244166ef7454dd10445a2dbacc5a0234104 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Sun, 20 Sep 2026 17:12:54 +0200 Subject: [PATCH 5/5] chore: up version Co-Authored-By: Claude Sonnet 5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 77f0b3c..cf04671 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "torchrecurrent" -version = "0.2.4" +version = "0.2.5" description = "A package for recurrent neural networks in PyTorch" readme = "README.md" authors = [