diff --git a/docs/api/cells.rst b/docs/api/cells.rst index a923158..ef4c6e6 100755 --- a/docs/api/cells.rst +++ b/docs/api/cells.rst @@ -13,6 +13,7 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.BRCell torchrecurrent.CFNCell torchrecurrent.DSGUCell + torchrecurrent.eLSTMCell torchrecurrent.coRNNCell torchrecurrent.FastRNNCell torchrecurrent.FastGRNNCell @@ -25,6 +26,9 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.LiGRUCell torchrecurrent.MCLSTMCell torchrecurrent.MGUCell + torchrecurrent.MGU1Cell + torchrecurrent.MGU2Cell + torchrecurrent.MGU3Cell torchrecurrent.MinimalRNNCell torchrecurrent.MiRU1Cell torchrecurrent.MiRU2Cell diff --git a/docs/api/layers.rst b/docs/api/layers.rst index b254891..43c664e 100644 --- a/docs/api/layers.rst +++ b/docs/api/layers.rst @@ -13,6 +13,7 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.BR torchrecurrent.CFN torchrecurrent.DSGU + torchrecurrent.eLSTM torchrecurrent.coRNN torchrecurrent.FastRNN torchrecurrent.FastGRNN @@ -25,6 +26,9 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.LiGRU torchrecurrent.MCLSTM torchrecurrent.MGU + torchrecurrent.MGU1 + torchrecurrent.MGU2 + torchrecurrent.MGU3 torchrecurrent.MinimalRNN torchrecurrent.MiRU1 torchrecurrent.MiRU2 diff --git a/docs/generated/torchrecurrent.MGU1.rst b/docs/generated/torchrecurrent.MGU1.rst new file mode 100644 index 0000000..b43a418 --- /dev/null +++ b/docs/generated/torchrecurrent.MGU1.rst @@ -0,0 +1,87 @@ +torchrecurrent.MGU1 +=================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: MGU1 + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MGU1.__init__ + ~MGU1.add_module + ~MGU1.apply + ~MGU1.bfloat16 + ~MGU1.buffers + ~MGU1.children + ~MGU1.compile + ~MGU1.cpu + ~MGU1.cuda + ~MGU1.double + ~MGU1.eval + ~MGU1.extra_repr + ~MGU1.float + ~MGU1.forward + ~MGU1.get_buffer + ~MGU1.get_extra_state + ~MGU1.get_parameter + ~MGU1.get_submodule + ~MGU1.half + ~MGU1.initialize_cells + ~MGU1.ipu + ~MGU1.load_state_dict + ~MGU1.modules + ~MGU1.mtia + ~MGU1.named_buffers + ~MGU1.named_children + ~MGU1.named_modules + ~MGU1.named_parameters + ~MGU1.parameters + ~MGU1.register_backward_hook + ~MGU1.register_buffer + ~MGU1.register_forward_hook + ~MGU1.register_forward_pre_hook + ~MGU1.register_full_backward_hook + ~MGU1.register_full_backward_pre_hook + ~MGU1.register_load_state_dict_post_hook + ~MGU1.register_load_state_dict_pre_hook + ~MGU1.register_module + ~MGU1.register_parameter + ~MGU1.register_state_dict_post_hook + ~MGU1.register_state_dict_pre_hook + ~MGU1.requires_grad_ + ~MGU1.set_extra_state + ~MGU1.set_submodule + ~MGU1.share_memory + ~MGU1.state_dict + ~MGU1.to + ~MGU1.to_empty + ~MGU1.train + ~MGU1.type + ~MGU1.xpu + ~MGU1.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MGU1.T_destination + ~MGU1.call_super_init + ~MGU1.dump_patches + ~MGU1.input_size + ~MGU1.hidden_size + ~MGU1.bias + ~MGU1.dropout + ~MGU1.batch_first + ~MGU1.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MGU1Cell.rst b/docs/generated/torchrecurrent.MGU1Cell.rst new file mode 100644 index 0000000..23a6c7c --- /dev/null +++ b/docs/generated/torchrecurrent.MGU1Cell.rst @@ -0,0 +1,91 @@ +torchrecurrent.MGU1Cell +======================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: MGU1Cell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MGU1Cell.__init__ + ~MGU1Cell.add_module + ~MGU1Cell.apply + ~MGU1Cell.bfloat16 + ~MGU1Cell.buffers + ~MGU1Cell.children + ~MGU1Cell.compile + ~MGU1Cell.cpu + ~MGU1Cell.cuda + ~MGU1Cell.double + ~MGU1Cell.eval + ~MGU1Cell.extra_repr + ~MGU1Cell.float + ~MGU1Cell.forward + ~MGU1Cell.get_buffer + ~MGU1Cell.get_extra_state + ~MGU1Cell.get_parameter + ~MGU1Cell.get_submodule + ~MGU1Cell.half + ~MGU1Cell.init_weights + ~MGU1Cell.ipu + ~MGU1Cell.load_state_dict + ~MGU1Cell.modules + ~MGU1Cell.mtia + ~MGU1Cell.named_buffers + ~MGU1Cell.named_children + ~MGU1Cell.named_modules + ~MGU1Cell.named_parameters + ~MGU1Cell.parameters + ~MGU1Cell.register_backward_hook + ~MGU1Cell.register_buffer + ~MGU1Cell.register_forward_hook + ~MGU1Cell.register_forward_pre_hook + ~MGU1Cell.register_full_backward_hook + ~MGU1Cell.register_full_backward_pre_hook + ~MGU1Cell.register_load_state_dict_post_hook + ~MGU1Cell.register_load_state_dict_pre_hook + ~MGU1Cell.register_module + ~MGU1Cell.register_parameter + ~MGU1Cell.register_state_dict_post_hook + ~MGU1Cell.register_state_dict_pre_hook + ~MGU1Cell.requires_grad_ + ~MGU1Cell.set_extra_state + ~MGU1Cell.set_submodule + ~MGU1Cell.share_memory + ~MGU1Cell.state_dict + ~MGU1Cell.to + ~MGU1Cell.to_empty + ~MGU1Cell.train + ~MGU1Cell.type + ~MGU1Cell.uses_double_state + ~MGU1Cell.xpu + ~MGU1Cell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MGU1Cell.T_destination + ~MGU1Cell.call_super_init + ~MGU1Cell.dump_patches + ~MGU1Cell.weight_ih + ~MGU1Cell.weight_hh + ~MGU1Cell.bias_ih + ~MGU1Cell.bias_hh + ~MGU1Cell.input_size + ~MGU1Cell.hidden_size + ~MGU1Cell.bias + ~MGU1Cell.recurrent_bias + ~MGU1Cell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MGU2.rst b/docs/generated/torchrecurrent.MGU2.rst new file mode 100644 index 0000000..917aec1 --- /dev/null +++ b/docs/generated/torchrecurrent.MGU2.rst @@ -0,0 +1,87 @@ +torchrecurrent.MGU2 +=================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: MGU2 + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MGU2.__init__ + ~MGU2.add_module + ~MGU2.apply + ~MGU2.bfloat16 + ~MGU2.buffers + ~MGU2.children + ~MGU2.compile + ~MGU2.cpu + ~MGU2.cuda + ~MGU2.double + ~MGU2.eval + ~MGU2.extra_repr + ~MGU2.float + ~MGU2.forward + ~MGU2.get_buffer + ~MGU2.get_extra_state + ~MGU2.get_parameter + ~MGU2.get_submodule + ~MGU2.half + ~MGU2.initialize_cells + ~MGU2.ipu + ~MGU2.load_state_dict + ~MGU2.modules + ~MGU2.mtia + ~MGU2.named_buffers + ~MGU2.named_children + ~MGU2.named_modules + ~MGU2.named_parameters + ~MGU2.parameters + ~MGU2.register_backward_hook + ~MGU2.register_buffer + ~MGU2.register_forward_hook + ~MGU2.register_forward_pre_hook + ~MGU2.register_full_backward_hook + ~MGU2.register_full_backward_pre_hook + ~MGU2.register_load_state_dict_post_hook + ~MGU2.register_load_state_dict_pre_hook + ~MGU2.register_module + ~MGU2.register_parameter + ~MGU2.register_state_dict_post_hook + ~MGU2.register_state_dict_pre_hook + ~MGU2.requires_grad_ + ~MGU2.set_extra_state + ~MGU2.set_submodule + ~MGU2.share_memory + ~MGU2.state_dict + ~MGU2.to + ~MGU2.to_empty + ~MGU2.train + ~MGU2.type + ~MGU2.xpu + ~MGU2.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MGU2.T_destination + ~MGU2.call_super_init + ~MGU2.dump_patches + ~MGU2.input_size + ~MGU2.hidden_size + ~MGU2.bias + ~MGU2.dropout + ~MGU2.batch_first + ~MGU2.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MGU2Cell.rst b/docs/generated/torchrecurrent.MGU2Cell.rst new file mode 100644 index 0000000..029ea8a --- /dev/null +++ b/docs/generated/torchrecurrent.MGU2Cell.rst @@ -0,0 +1,91 @@ +torchrecurrent.MGU2Cell +======================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: MGU2Cell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MGU2Cell.__init__ + ~MGU2Cell.add_module + ~MGU2Cell.apply + ~MGU2Cell.bfloat16 + ~MGU2Cell.buffers + ~MGU2Cell.children + ~MGU2Cell.compile + ~MGU2Cell.cpu + ~MGU2Cell.cuda + ~MGU2Cell.double + ~MGU2Cell.eval + ~MGU2Cell.extra_repr + ~MGU2Cell.float + ~MGU2Cell.forward + ~MGU2Cell.get_buffer + ~MGU2Cell.get_extra_state + ~MGU2Cell.get_parameter + ~MGU2Cell.get_submodule + ~MGU2Cell.half + ~MGU2Cell.init_weights + ~MGU2Cell.ipu + ~MGU2Cell.load_state_dict + ~MGU2Cell.modules + ~MGU2Cell.mtia + ~MGU2Cell.named_buffers + ~MGU2Cell.named_children + ~MGU2Cell.named_modules + ~MGU2Cell.named_parameters + ~MGU2Cell.parameters + ~MGU2Cell.register_backward_hook + ~MGU2Cell.register_buffer + ~MGU2Cell.register_forward_hook + ~MGU2Cell.register_forward_pre_hook + ~MGU2Cell.register_full_backward_hook + ~MGU2Cell.register_full_backward_pre_hook + ~MGU2Cell.register_load_state_dict_post_hook + ~MGU2Cell.register_load_state_dict_pre_hook + ~MGU2Cell.register_module + ~MGU2Cell.register_parameter + ~MGU2Cell.register_state_dict_post_hook + ~MGU2Cell.register_state_dict_pre_hook + ~MGU2Cell.requires_grad_ + ~MGU2Cell.set_extra_state + ~MGU2Cell.set_submodule + ~MGU2Cell.share_memory + ~MGU2Cell.state_dict + ~MGU2Cell.to + ~MGU2Cell.to_empty + ~MGU2Cell.train + ~MGU2Cell.type + ~MGU2Cell.uses_double_state + ~MGU2Cell.xpu + ~MGU2Cell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MGU2Cell.T_destination + ~MGU2Cell.call_super_init + ~MGU2Cell.dump_patches + ~MGU2Cell.weight_ih + ~MGU2Cell.weight_hh + ~MGU2Cell.bias_ih + ~MGU2Cell.bias_hh + ~MGU2Cell.input_size + ~MGU2Cell.hidden_size + ~MGU2Cell.bias + ~MGU2Cell.recurrent_bias + ~MGU2Cell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MGU3.rst b/docs/generated/torchrecurrent.MGU3.rst new file mode 100644 index 0000000..4704543 --- /dev/null +++ b/docs/generated/torchrecurrent.MGU3.rst @@ -0,0 +1,87 @@ +torchrecurrent.MGU3 +=================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: MGU3 + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MGU3.__init__ + ~MGU3.add_module + ~MGU3.apply + ~MGU3.bfloat16 + ~MGU3.buffers + ~MGU3.children + ~MGU3.compile + ~MGU3.cpu + ~MGU3.cuda + ~MGU3.double + ~MGU3.eval + ~MGU3.extra_repr + ~MGU3.float + ~MGU3.forward + ~MGU3.get_buffer + ~MGU3.get_extra_state + ~MGU3.get_parameter + ~MGU3.get_submodule + ~MGU3.half + ~MGU3.initialize_cells + ~MGU3.ipu + ~MGU3.load_state_dict + ~MGU3.modules + ~MGU3.mtia + ~MGU3.named_buffers + ~MGU3.named_children + ~MGU3.named_modules + ~MGU3.named_parameters + ~MGU3.parameters + ~MGU3.register_backward_hook + ~MGU3.register_buffer + ~MGU3.register_forward_hook + ~MGU3.register_forward_pre_hook + ~MGU3.register_full_backward_hook + ~MGU3.register_full_backward_pre_hook + ~MGU3.register_load_state_dict_post_hook + ~MGU3.register_load_state_dict_pre_hook + ~MGU3.register_module + ~MGU3.register_parameter + ~MGU3.register_state_dict_post_hook + ~MGU3.register_state_dict_pre_hook + ~MGU3.requires_grad_ + ~MGU3.set_extra_state + ~MGU3.set_submodule + ~MGU3.share_memory + ~MGU3.state_dict + ~MGU3.to + ~MGU3.to_empty + ~MGU3.train + ~MGU3.type + ~MGU3.xpu + ~MGU3.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MGU3.T_destination + ~MGU3.call_super_init + ~MGU3.dump_patches + ~MGU3.input_size + ~MGU3.hidden_size + ~MGU3.bias + ~MGU3.dropout + ~MGU3.batch_first + ~MGU3.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MGU3Cell.rst b/docs/generated/torchrecurrent.MGU3Cell.rst new file mode 100644 index 0000000..07fbdb3 --- /dev/null +++ b/docs/generated/torchrecurrent.MGU3Cell.rst @@ -0,0 +1,92 @@ +torchrecurrent.MGU3Cell +======================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: MGU3Cell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MGU3Cell.__init__ + ~MGU3Cell.add_module + ~MGU3Cell.apply + ~MGU3Cell.bfloat16 + ~MGU3Cell.buffers + ~MGU3Cell.children + ~MGU3Cell.compile + ~MGU3Cell.cpu + ~MGU3Cell.cuda + ~MGU3Cell.double + ~MGU3Cell.eval + ~MGU3Cell.extra_repr + ~MGU3Cell.float + ~MGU3Cell.forward + ~MGU3Cell.get_buffer + ~MGU3Cell.get_extra_state + ~MGU3Cell.get_parameter + ~MGU3Cell.get_submodule + ~MGU3Cell.half + ~MGU3Cell.init_weights + ~MGU3Cell.ipu + ~MGU3Cell.load_state_dict + ~MGU3Cell.modules + ~MGU3Cell.mtia + ~MGU3Cell.named_buffers + ~MGU3Cell.named_children + ~MGU3Cell.named_modules + ~MGU3Cell.named_parameters + ~MGU3Cell.parameters + ~MGU3Cell.register_backward_hook + ~MGU3Cell.register_buffer + ~MGU3Cell.register_forward_hook + ~MGU3Cell.register_forward_pre_hook + ~MGU3Cell.register_full_backward_hook + ~MGU3Cell.register_full_backward_pre_hook + ~MGU3Cell.register_load_state_dict_post_hook + ~MGU3Cell.register_load_state_dict_pre_hook + ~MGU3Cell.register_module + ~MGU3Cell.register_parameter + ~MGU3Cell.register_state_dict_post_hook + ~MGU3Cell.register_state_dict_pre_hook + ~MGU3Cell.requires_grad_ + ~MGU3Cell.set_extra_state + ~MGU3Cell.set_submodule + ~MGU3Cell.share_memory + ~MGU3Cell.state_dict + ~MGU3Cell.to + ~MGU3Cell.to_empty + ~MGU3Cell.train + ~MGU3Cell.type + ~MGU3Cell.uses_double_state + ~MGU3Cell.xpu + ~MGU3Cell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MGU3Cell.T_destination + ~MGU3Cell.call_super_init + ~MGU3Cell.dump_patches + ~MGU3Cell.weight_ih + ~MGU3Cell.weight_hh + ~MGU3Cell.bias_f + ~MGU3Cell.bias_ih + ~MGU3Cell.bias_hh + ~MGU3Cell.input_size + ~MGU3Cell.hidden_size + ~MGU3Cell.bias + ~MGU3Cell.recurrent_bias + ~MGU3Cell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.eLSTM.rst b/docs/generated/torchrecurrent.eLSTM.rst new file mode 100644 index 0000000..3baae1e --- /dev/null +++ b/docs/generated/torchrecurrent.eLSTM.rst @@ -0,0 +1,87 @@ +torchrecurrent.eLSTM +==================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: eLSTM + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~eLSTM.__init__ + ~eLSTM.add_module + ~eLSTM.apply + ~eLSTM.bfloat16 + ~eLSTM.buffers + ~eLSTM.children + ~eLSTM.compile + ~eLSTM.cpu + ~eLSTM.cuda + ~eLSTM.double + ~eLSTM.eval + ~eLSTM.extra_repr + ~eLSTM.float + ~eLSTM.forward + ~eLSTM.get_buffer + ~eLSTM.get_extra_state + ~eLSTM.get_parameter + ~eLSTM.get_submodule + ~eLSTM.half + ~eLSTM.initialize_cells + ~eLSTM.ipu + ~eLSTM.load_state_dict + ~eLSTM.modules + ~eLSTM.mtia + ~eLSTM.named_buffers + ~eLSTM.named_children + ~eLSTM.named_modules + ~eLSTM.named_parameters + ~eLSTM.parameters + ~eLSTM.register_backward_hook + ~eLSTM.register_buffer + ~eLSTM.register_forward_hook + ~eLSTM.register_forward_pre_hook + ~eLSTM.register_full_backward_hook + ~eLSTM.register_full_backward_pre_hook + ~eLSTM.register_load_state_dict_post_hook + ~eLSTM.register_load_state_dict_pre_hook + ~eLSTM.register_module + ~eLSTM.register_parameter + ~eLSTM.register_state_dict_post_hook + ~eLSTM.register_state_dict_pre_hook + ~eLSTM.requires_grad_ + ~eLSTM.set_extra_state + ~eLSTM.set_submodule + ~eLSTM.share_memory + ~eLSTM.state_dict + ~eLSTM.to + ~eLSTM.to_empty + ~eLSTM.train + ~eLSTM.type + ~eLSTM.xpu + ~eLSTM.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~eLSTM.T_destination + ~eLSTM.call_super_init + ~eLSTM.dump_patches + ~eLSTM.input_size + ~eLSTM.hidden_size + ~eLSTM.bias + ~eLSTM.dropout + ~eLSTM.batch_first + ~eLSTM.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.eLSTMCell.rst b/docs/generated/torchrecurrent.eLSTMCell.rst new file mode 100644 index 0000000..c535eb1 --- /dev/null +++ b/docs/generated/torchrecurrent.eLSTMCell.rst @@ -0,0 +1,92 @@ +torchrecurrent.eLSTMCell +======================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: eLSTMCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~eLSTMCell.__init__ + ~eLSTMCell.add_module + ~eLSTMCell.apply + ~eLSTMCell.bfloat16 + ~eLSTMCell.buffers + ~eLSTMCell.children + ~eLSTMCell.compile + ~eLSTMCell.cpu + ~eLSTMCell.cuda + ~eLSTMCell.double + ~eLSTMCell.eval + ~eLSTMCell.extra_repr + ~eLSTMCell.float + ~eLSTMCell.forward + ~eLSTMCell.get_buffer + ~eLSTMCell.get_extra_state + ~eLSTMCell.get_parameter + ~eLSTMCell.get_submodule + ~eLSTMCell.half + ~eLSTMCell.init_weights + ~eLSTMCell.ipu + ~eLSTMCell.load_state_dict + ~eLSTMCell.modules + ~eLSTMCell.mtia + ~eLSTMCell.named_buffers + ~eLSTMCell.named_children + ~eLSTMCell.named_modules + ~eLSTMCell.named_parameters + ~eLSTMCell.parameters + ~eLSTMCell.register_backward_hook + ~eLSTMCell.register_buffer + ~eLSTMCell.register_forward_hook + ~eLSTMCell.register_forward_pre_hook + ~eLSTMCell.register_full_backward_hook + ~eLSTMCell.register_full_backward_pre_hook + ~eLSTMCell.register_load_state_dict_post_hook + ~eLSTMCell.register_load_state_dict_pre_hook + ~eLSTMCell.register_module + ~eLSTMCell.register_parameter + ~eLSTMCell.register_state_dict_post_hook + ~eLSTMCell.register_state_dict_pre_hook + ~eLSTMCell.requires_grad_ + ~eLSTMCell.set_extra_state + ~eLSTMCell.set_submodule + ~eLSTMCell.share_memory + ~eLSTMCell.state_dict + ~eLSTMCell.to + ~eLSTMCell.to_empty + ~eLSTMCell.train + ~eLSTMCell.type + ~eLSTMCell.uses_double_state + ~eLSTMCell.xpu + ~eLSTMCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~eLSTMCell.T_destination + ~eLSTMCell.call_super_init + ~eLSTMCell.dump_patches + ~eLSTMCell.weight_ih + ~eLSTMCell.weight_hh + ~eLSTMCell.weight_c + ~eLSTMCell.bias_ih + ~eLSTMCell.bias_hh + ~eLSTMCell.input_size + ~eLSTMCell.hidden_size + ~eLSTMCell.bias + ~eLSTMCell.recurrent_bias + ~eLSTMCell.training + + \ No newline at end of file diff --git a/docs/models.rst b/docs/models.rst index 7eedea5..811ae8a 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -35,6 +35,9 @@ references and official implementations where available. * - :doc:`DSGU ` - `ACML 2016 `__ - – + * - :doc:`eLSTM ` + - `ICLR 2024 `__ + - `IDSIA/rtrl-elstm `__ * - :doc:`FastGRNN ` - `NeurIPS 2018 `__ - `Microsoft/EdgeML `__ @@ -68,6 +71,15 @@ references and official implementations where available. * - :doc:`MGU ` - `IJAC 2016 `__ - – + * - :doc:`MGU1 ` + - `IEEE MWSCAS 2017 `__ + - – + * - :doc:`MGU2 ` + - `IEEE MWSCAS 2017 `__ + - – + * - :doc:`MGU3 ` + - `IEEE MWSCAS 2017 `__ + - – * - :doc:`MinimalRNN ` - `NeurIPS 2017 Workshop `__ - – diff --git a/tests/test_cells.py b/tests/test_cells.py index b6a543a..325976e 100755 --- a/tests/test_cells.py +++ b/tests/test_cells.py @@ -1,3 +1,4 @@ +import inspect import sys import pytest @@ -11,6 +12,7 @@ BRCell, CFNCell, DSGUCell, + eLSTMCell, coRNNCell, FastRNNCell, FastGRNNCell, @@ -18,6 +20,9 @@ LEMCell, GatedAntisymmetricRNNCell, MGUCell, + MGU1Cell, + MGU2Cell, + MGU3Cell, MinimalRNNCell, IndRNNCell, IntersectionRNNCell, @@ -66,6 +71,9 @@ (JANETCell, 3, 5, True), (LEMCell, 3, 5, True), (MGUCell, 4, 8, False), + (MGU1Cell, 4, 8, False), + (MGU2Cell, 4, 8, False), + (MGU3Cell, 4, 8, False), (MinimalRNNCell, 3, 5, False), (IndRNNCell, 3, 5, False), (LiGRUCell, 6, 12, False), @@ -140,6 +148,21 @@ def test_cell_reset_parameters_after_construction(Cell, in_size, hid_size, doubl cell.reset_parameters() +@pytest.mark.parametrize( + "Cell, in_size, hid_size", [c[:3] for c in CELL_CASES] + [(eLSTMCell, 3, 5)] +) +def test_cell_disabled_biases_stay_zero(Cell, in_size, hid_size): + """Non-zero bias initializers must not write into disabled (buffer) biases.""" + kwargs = dict( + bias=False, recurrent_bias=False, bias_init="ones", recurrent_bias_init="ones" + ) + accepted = inspect.signature(Cell.__init__).parameters + cell = Cell(in_size, hid_size, **{k: v for k, v in kwargs.items() if k in accepted}) + cell.reset_parameters() + for name, buf in cell.named_buffers(): + assert torch.count_nonzero(buf) == 0, name + + def test_reslstm_cell_parameter_shapes(): cell = ResLSTMCell(4, 9) @@ -668,3 +691,101 @@ def test_cell_compile_with_state(Cell, in_size, hid_size, double): h0 = torch.randn(B, hid_size) h = compiled(x, h0) assert h.shape == (B, hid_size) + + +def _mgu_candidate_update(cell, x, h, forget_gate, weight_hh_h, bias_hh_h): + candidate = torch.tanh( + x @ cell.weight_ih.t() + + cell.bias_ih + + (forget_gate * h) @ weight_hh_h.t() + + bias_hh_h + ) + return forget_gate * candidate + (1 - forget_gate) * h + + +@pytest.mark.parametrize("Cell", [MGU1Cell, MGU2Cell, MGU3Cell]) +def test_mgu_variant_cells_match_paper_update(Cell): + torch.manual_seed(0) + cell = Cell( + 3, 4, bias_init=torch.nn.init.normal_, recurrent_bias_init=torch.nn.init.normal_ + ) + x = torch.randn(2, 3) + h = torch.randn(2, 4) + + if Cell is MGU3Cell: + forget_gate = torch.sigmoid(cell.bias_f).expand_as(h) + weight_hh_h, bias_hh_h = cell.weight_hh, cell.bias_hh + else: + weight_hh_f, weight_hh_h = cell.weight_hh.chunk(2, 0) + if Cell is MGU1Cell: + bias_hh_f, bias_hh_h = cell.bias_hh.chunk(2, 0) + forget_gate = torch.sigmoid(h @ weight_hh_f.t() + bias_hh_f) + else: + bias_hh_h = cell.bias_hh + forget_gate = torch.sigmoid(h @ weight_hh_f.t()) + + expected = _mgu_candidate_update(cell, x, h, forget_gate, weight_hh_h, bias_hh_h) + assert torch.allclose(cell(x, h), expected, atol=1e-6) + + +def test_mgu_variant_cell_parameter_shapes(): + assert MGU1Cell(3, 4).weight_hh.shape == (8, 4) + assert MGU1Cell(3, 4).bias_hh.shape == (8,) + assert MGU2Cell(3, 4).weight_hh.shape == (8, 4) + assert MGU2Cell(3, 4).bias_hh.shape == (4,) + cell = MGU3Cell(3, 4, bias=False, recurrent_bias=False) + assert cell.weight_ih.shape == (4, 3) + assert cell.weight_hh.shape == (4, 4) + assert isinstance(cell.bias_f, torch.nn.Parameter) + + +def test_mgu_variant_gates_ignore_input(): + """MGU1-3 gates must not see x: with h = 0, MGU2's gate is exactly 0.5.""" + cell = MGU2Cell(3, 4, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_hh.zero_() + x = torch.randn(5, 3) + expected = 0.5 * torch.tanh(x @ cell.weight_ih.t()) + assert torch.allclose(cell(x, torch.zeros(5, 4)), expected) + + +def test_elstm_cell_shapes(): + cell = eLSTMCell(3, 5) + + assert cell.weight_ih.shape == (15, 3) + assert cell.weight_hh.shape == (5, 5) + assert cell.weight_c.shape == (10,) + assert cell.bias_ih.shape == (15,) + assert cell.bias_hh.shape == (5,) + + h_u, c_u = cell(torch.randn(3)) + assert h_u.shape == c_u.shape == (5,) + h, c = cell(torch.randn(4, 3), torch.randn(4, 5)) + assert h.shape == c.shape == (4, 5) + assert not torch.allclose(h, c) + + +def test_elstm_cell_reset_parameters_after_construction(): + eLSTMCell(3, 5).reset_parameters() + + +def test_elstm_cell_matches_paper_update(): + torch.manual_seed(0) + cell = eLSTMCell( + 3, 4, bias_init=torch.nn.init.normal_, recurrent_bias_init=torch.nn.init.normal_ + ) + x = torch.randn(2, 3) + c = torch.randn(2, 4) + + w_f, w_z, w_o = cell.weight_ih.chunk(3, 0) + b_f, b_z, b_o = cell.bias_ih.chunk(3, 0) + v_f, v_z = cell.weight_c.chunk(2, 0) + f = torch.sigmoid(x @ w_f.t() + b_f + v_f * c) + z = torch.tanh(x @ w_z.t() + b_z + v_z * c) + expected_c = f * c + (1 - f) * z + o = torch.sigmoid(x @ w_o.t() + b_o + expected_c @ cell.weight_hh.t() + cell.bias_hh) + expected_h = o * expected_c + + h, new_c = cell(x, c) + assert torch.allclose(new_c, expected_c, atol=1e-6) + assert torch.allclose(h, expected_h, atol=1e-6) diff --git a/tests/test_layers.py b/tests/test_layers.py index 162a13c..3167db9 100755 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -9,6 +9,7 @@ BR, CFN, DSGU, + eLSTM, coRNN, FastRNN, FastGRNN, @@ -19,6 +20,9 @@ LightRU, MCLSTM, MGU, + MGU1, + MGU2, + MGU3, MinimalRNN, MultiplicativeLSTM, MUT1, @@ -49,6 +53,7 @@ BR, CFN, DSGU, + eLSTM, coRNN, FastRNN, FastGRNN, @@ -58,6 +63,9 @@ LightRU, MCLSTM, MGU, + MGU1, + MGU2, + MGU3, MinimalRNN, MultiplicativeLSTM, MUT1, @@ -89,6 +97,7 @@ (BR, False), (CFN, False), (DSGU, False), + (eLSTM, False), (coRNN, True), (FastRNN, False), (FastGRNN, False), @@ -98,6 +107,9 @@ (LightRU, False), (MCLSTM, True), (MGU, False), + (MGU1, False), + (MGU2, False), + (MGU3, False), (MinimalRNN, False), (MultiplicativeLSTM, True), (MUT1, False), @@ -682,3 +694,25 @@ def test_repr_includes_nondefault_kwargs(Layer): # batch_first=True r = repr(Layer(3, 5, batch_first=True)) assert "batch_first=True" in r + + +def test_elstm_stacking_forwards_output_not_state(): + """Layer k+1 must receive layer k's h(t) (output), not its c(t) (state).""" + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = eLSTM(3, 5, num_layers=num_layers) + x = torch.randn(seq_len, batch_size, 3) + + out, cn = layer(x) + + c_prev = [torch.zeros(batch_size, 5) for _ in range(num_layers)] + expected = [] + for t in range(seq_len): + layer_inp = x[t] + for layer_idx, cell in enumerate(layer.cells): + h, c_prev[layer_idx] = cell(layer_inp, c_prev[layer_idx]) + layer_inp = h + expected.append(layer_inp) + + assert torch.allclose(out, torch.stack(expected), atol=1e-6) + assert torch.allclose(cn, torch.stack(c_prev), atol=1e-6) diff --git a/torchrecurrent/__init__.py b/torchrecurrent/__init__.py index d374558..75b9f27 100755 --- a/torchrecurrent/__init__.py +++ b/torchrecurrent/__init__.py @@ -8,6 +8,7 @@ BRCell, CFNCell, DSGUCell, + eLSTMCell, FastRNNCell, FastGRNNCell, ATRCell, @@ -19,6 +20,9 @@ LightRUCell, MCLSTMCell, MGUCell, + MGU1Cell, + MGU2Cell, + MGU3Cell, MinimalRNNCell, MiRU1Cell, MiRU2Cell, @@ -54,6 +58,7 @@ BR, CFN, DSGU, + eLSTM, FastRNN, FastGRNN, IndRNN, @@ -64,6 +69,9 @@ LightRU, MCLSTM, MGU, + MGU1, + MGU2, + MGU3, MinimalRNN, MiRU1, MiRU2, @@ -105,10 +113,18 @@ "CFNCell", "DSGU", "DSGUCell", + "eLSTM", + "eLSTMCell", "MCLSTM", "MCLSTMCell", "MGU", "MGUCell", + "MGU1", + "MGU1Cell", + "MGU2", + "MGU2Cell", + "MGU3", + "MGU3Cell", "MinimalRNN", "MinimalRNNCell", "MiRU1", diff --git a/torchrecurrent/cells/__init__.py b/torchrecurrent/cells/__init__.py index 645561d..ab73199 100755 --- a/torchrecurrent/cells/__init__.py +++ b/torchrecurrent/cells/__init__.py @@ -10,9 +10,10 @@ from .br_cell import BR, BRCell, NBR, NBRCell from .cfn_cell import CFN, CFNCell from .mclstm_cell import MCLSTM, MCLSTMCell -from .mgu_cell import MGU, MGUCell +from .mgu_cell import MGU, MGUCell, MGU1, MGU1Cell, MGU2, MGU2Cell, MGU3, MGU3Cell from .minimalrnn_cell import MinimalRNN, MinimalRNNCell from .miru_cell import MiRU1, MiRU1Cell, MiRU2, MiRU2Cell +from .elstm_cell import eLSTM, eLSTMCell from .cornn_cell import coRNN, coRNNCell from .fastrnn_cell import FastRNN, FastRNNCell, FastGRNN, FastGRNNCell from .indrnn_cell import IndRNN, IndRNNCell @@ -58,12 +59,20 @@ "MCLSTMCell", "MGU", "MGUCell", + "MGU1", + "MGU1Cell", + "MGU2", + "MGU2Cell", + "MGU3", + "MGU3Cell", "MinimalRNN", "MinimalRNNCell", "MiRU1", "MiRU1Cell", "MiRU2", "MiRU2Cell", + "eLSTM", + "eLSTMCell", "coRNN", "coRNNCell", "FastRNN", diff --git a/torchrecurrent/cells/antisymmetricrnn_cell.py b/torchrecurrent/cells/antisymmetricrnn_cell.py index 3fb0d54..09da23d 100755 --- a/torchrecurrent/cells/antisymmetricrnn_cell.py +++ b/torchrecurrent/cells/antisymmetricrnn_cell.py @@ -271,9 +271,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: @@ -568,9 +568,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: diff --git a/torchrecurrent/cells/br_cell.py b/torchrecurrent/cells/br_cell.py index eb995f8..6d4054d 100755 --- a/torchrecurrent/cells/br_cell.py +++ b/torchrecurrent/cells/br_cell.py @@ -256,9 +256,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: @@ -536,9 +536,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: diff --git a/torchrecurrent/cells/cornn_cell.py b/torchrecurrent/cells/cornn_cell.py index eb047cc..8791046 100755 --- a/torchrecurrent/cells/cornn_cell.py +++ b/torchrecurrent/cells/cornn_cell.py @@ -330,11 +330,11 @@ def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) apply_init_(self.weight_ch, self.init_cfg["cell_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) - if hasattr(self, "bias_ch"): + if isinstance(self.bias_ch, nn.Parameter): apply_init_(self.bias_ch, self.init_cfg["cell_bias"]) def forward( diff --git a/torchrecurrent/cells/elstm_cell.py b/torchrecurrent/cells/elstm_cell.py new file mode 100644 index 0000000..ef92ff6 --- /dev/null +++ b/torchrecurrent/cells/elstm_cell.py @@ -0,0 +1,328 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional, Tuple +from ..base import ( + DecoupledSingleStateRecurrentLayerBase, + DecoupledSingleStateCellBase, + apply_init_, + resolve_activation, + resolve_init_name, +) + + +class eLSTM(DecoupledSingleStateRecurrentLayerBase): + r"""Multi-layer LSTM with element-wise recurrence (eLSTM). + + [`arXiv `_] + + Each layer consists of an :class:`eLSTMCell`, which updates the cell + state according to: + + .. math:: + \begin{aligned} + f_t &= \sigma(W_{ih}^f x_t + b_{ih}^f + w^f \circ c_{t-1}), \\ + z_t &= \phi(W_{ih}^z x_t + b_{ih}^z + w^z \circ c_{t-1}), \\ + c_t &= f_t \circ c_{t-1} + (1 - f_t) \circ z_t, \\ + o_t &= \sigma(W_{ih}^o x_t + b_{ih}^o + W_{hh} c_t + b_{hh}), \\ + h_t &= o_t \circ c_t + \end{aligned} + + where :math:`c_t` is the recurrent cell state at time `t`, :math:`h_t` is + the output at time `t`, :math:`x_t` is the input at time `t`, + :math:`\sigma` is the sigmoid function, :math:`\phi` is a pointwise + nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + multiplication. Only :math:`c_t` recurs, through the element-wise weights + :math:`w^f` and :math:`w^z`; :math:`h_t` is passed to the next layer. + + In a multilayer eLSTM, the input :math:`x^{(l)}_t` of the :math:`l`-th layer + (:math:`l \ge 2`) is the output :math:`h^{(l-1)}_t` of the previous + layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the cell state `c`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two eLSTM layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use the bias + :math:`b_{hh}` of the output gate. Default: True + nonlinearity: Nonlinearity :math:`\phi` for the candidate. Default: + :func:`torch.tanh` + gate_nonlinearity: Activation for the forget and output gates. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}^*`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + cell_kernel_init: Initializer for the element-wise weights `w^f, w^z`. + Default: uniform in :math:`[-0.1, 0.1]` + bias_init: Initializer for input-side biases. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for the output-gate bias `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, c_0 + - **input**: tensor of shape + :math:`(L, N, H_{in})` when ``batch_first=False`` or + :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of + the input sequence. + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state for each element in the input + sequence. Defaults to zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, c_n + - **output**: tensor of shape + :math:`(L, N, H_{out})` when ``batch_first=False`` or + :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output + features `(h_t)` from the last layer of the eLSTM, for each `t`. + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. + Note this is `c_n`, not `h_n`: the recurrent state and the returned + output are different tensors for every layer. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(3*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(3*hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable cell-to-output-gate weights of the + :math:`k`-th layer, of shape `(hidden_size, hidden_size)`. + cells.{k}.weight_c : the learnable element-wise recurrent weights of the + :math:`k`-th layer, of shape `(2*hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(3*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable output-gate bias of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`eLSTMCell` + + Examples:: + + >>> rnn = eLSTM(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> c0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> output, cn = rnn(input, c0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(eLSTM, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(eLSTMCell, **kwargs) + + +class eLSTMCell(DecoupledSingleStateCellBase): + r"""An LSTM cell with element-wise recurrence (eLSTM). + + [`arXiv `_] + + .. math:: + + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{f}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{f} + + \mathbf{w}^{f}\circ\mathbf{c}(t-1) + \bigr), \\[6pt] + \mathbf{z}(t) &= \phi\bigl( + \mathbf{W}_{ih}^{z}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{z} + + \mathbf{w}^{z}\circ\mathbf{c}(t-1) + \bigr), \\[6pt] + \mathbf{c}(t) &= \mathbf{f}(t)\circ\mathbf{c}(t-1) + + \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{z}(t), \\[6pt] + \mathbf{o}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{o}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{o} + + \mathbf{W}_{hh}\,\mathbf{c}(t) + \mathbf{b}_{hh} + \bigr), \\[6pt] + \mathbf{h}(t) &= \mathbf{o}(t)\circ\mathbf{c}(t), + + where :math:`\circ` is element‐wise product and :math:`\phi` is a + pointwise nonlinearity (e.g., tanh). Only :math:`\mathbf{c}` recurs; + :math:`\mathbf{h}` is the output. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the cell state ``c``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use the output-gate + bias ``b_{hh}``. Default: ``True``. + nonlinearity: Nonlinearity :math:`\phi` for the candidate. + Default: :func:`torch.tanh`. + gate_nonlinearity: Activation for the forget and output gates. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}^*``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + cell_kernel_init: Initializer for the element-wise weights ``w^f, w^z``. + Default: uniform in :math:`[-0.1, 0.1]`. + bias_init: Initializer for input-side biases when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, c_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **c_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial cell state. + + If **c_0** is not provided, it defaults to zero. + + Outputs: h_1, c_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the output, to be passed to the next layer. + - **c_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next cell state. + + Variables: + weight_ih: The learnable input–hidden weights, + of shape ``(3*hidden_size, input_size)`` (``f, z, o`` parts). + weight_hh: The learnable cell-to-output-gate weights, + of shape ``(hidden_size, hidden_size)``. + weight_c: The learnable element-wise recurrent weights, + of shape ``(2*hidden_size)`` (``w^f, w^z`` parts). + bias_ih: The learnable input–hidden biases, + of shape ``(3*hidden_size)`` if ``bias=True``. + bias_hh: The learnable output-gate bias, + of shape ``(hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = eLSTMCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> c = torch.zeros(3, 20) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h, c = cell(x[t], c) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + weight_c: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + nonlinearity="tanh", + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + cell_kernel_init="uniform_centered", + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.act = resolve_activation(nonlinearity) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["cell_kernel"] = resolve_init_name( + cell_kernel_init, "uniform_centered" + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((3 * hidden_size, input_size), True), + "weight_hh": ((hidden_size, hidden_size), True), + "weight_c": ((2 * hidden_size,), True), + "bias_ih": ((3 * hidden_size,), bias), + "bias_hh": ((hidden_size,), recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def reset_parameters(self) -> None: + apply_init_(self.weight_ih, self.init_cfg["kernel"]) + apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) + apply_init_(self.weight_c, self.init_cfg["cell_kernel"]) + if isinstance(self.bias_ih, nn.Parameter): + apply_init_(self.bias_ih, self.init_cfg["bias"]) + if isinstance(self.bias_hh, nn.Parameter): + apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + forget_pre, cand_pre, out_pre = (b_inp @ self.weight_ih.t() + self.bias_ih).chunk( + 3, 1 + ) + weight_c_f, weight_c_z = self.weight_c.chunk(2, 0) + + forget_gate = self.gate_act(forget_pre + weight_c_f * b_state) + candidate = self.act(cand_pre + weight_c_z * b_state) + new_state = forget_gate * b_state + (1.0 - forget_gate) * candidate + out_gate = self.gate_act(out_pre + new_state @ self.weight_hh.t() + self.bias_hh) + new_output = out_gate * new_state + + if not is_batched: + new_output = new_output.squeeze(0) + new_state = new_state.squeeze(0) + + return new_output, new_state diff --git a/torchrecurrent/cells/fastrnn_cell.py b/torchrecurrent/cells/fastrnn_cell.py index 0f0b53d..286e479 100755 --- a/torchrecurrent/cells/fastrnn_cell.py +++ b/torchrecurrent/cells/fastrnn_cell.py @@ -291,9 +291,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) nn.init.constant_(self.alpha, self.alpha_init) nn.init.constant_(self.beta, self.beta_init) @@ -575,9 +575,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) nn.init.constant_(self.zeta, self.zeta_init) nn.init.constant_(self.nu, self.nu_init) diff --git a/torchrecurrent/cells/indrnn_cell.py b/torchrecurrent/cells/indrnn_cell.py index 0aaf9b9..907a09d 100755 --- a/torchrecurrent/cells/indrnn_cell.py +++ b/torchrecurrent/cells/indrnn_cell.py @@ -246,9 +246,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.vector_u, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: diff --git a/torchrecurrent/cells/janet_cell.py b/torchrecurrent/cells/janet_cell.py index c6374ab..d21d85f 100755 --- a/torchrecurrent/cells/janet_cell.py +++ b/torchrecurrent/cells/janet_cell.py @@ -277,9 +277,9 @@ def __init__( def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward( diff --git a/torchrecurrent/cells/lem_cell.py b/torchrecurrent/cells/lem_cell.py index c1faf77..7062f7b 100755 --- a/torchrecurrent/cells/lem_cell.py +++ b/torchrecurrent/cells/lem_cell.py @@ -320,11 +320,11 @@ def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) apply_init_(self.weight_ch, self.init_cfg["cell_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) - if hasattr(self, "bias_ch"): + if isinstance(self.bias_ch, nn.Parameter): apply_init_(self.bias_ch, self.init_cfg["cell_bias"]) def forward( diff --git a/torchrecurrent/cells/mgu_cell.py b/torchrecurrent/cells/mgu_cell.py index 58a7113..e708e80 100755 --- a/torchrecurrent/cells/mgu_cell.py +++ b/torchrecurrent/cells/mgu_cell.py @@ -5,6 +5,7 @@ from ..base import ( SingleStateRecurrentLayerBase, SingleStateCellBase, + apply_init_, resolve_activation, resolve_init_name, ) @@ -292,3 +293,878 @@ def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: new_state = new_state.squeeze(0) return new_state + + +class MGU1(SingleStateRecurrentLayerBase): + r"""Multi-layer minimal gated unit variant 1 (MGU1) network. + + [`IEEE MWSCAS 2017 `_] + + Each layer consists of an :class:`MGU1Cell`, which updates the hidden + state according to: + + .. math:: + \begin{aligned} + f_t &= \sigma(W_{hh}^f h_{t-1} + b_{hh}^f), \\ + \tilde{h}_t &= \phi(W_{ih} x_t + b_{ih} + + W_{hh}^h (f_t \circ h_{t-1}) + b_{hh}^h), \\ + h_t &= (1 - f_t) \circ h_{t-1} + f_t \circ \tilde{h}_t + \end{aligned} + + where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the + input at time `t`, :math:`\sigma` is the sigmoid function, :math:`\phi` is + a pointwise nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + multiplication. This is :class:`MGU` with a reduced forget gate. The gate + has no input term. + + In a multilayer MGU1, the input :math:`x^{(l)}_t` of the :math:`l`-th layer + (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous + layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two MGU1 layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + nonlinearity: Nonlinearity :math:`\phi` for the candidate. Default: + :func:`torch.tanh` + gate_nonlinearity: Activation for the forget gate. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}^*`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for input-side biases. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for recurrent biases. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape + :math:`(L, N, H_{in})` when ``batch_first=False`` or + :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial + hidden state for each element in the input sequence. Defaults to zeros if + not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape + :math:`(L, N, H_{out})` when ``batch_first=False`` or + :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output + features `(h_t)` from the last layer of the MGU1, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final + hidden state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable hidden-hidden weights of the :math:`k`-th + layer, of shape `(2*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable hidden-hidden biases of the :math:`k`-th + layer, of shape `(2*hidden_size)`. Only present when ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`MGU1Cell`, :class:`MGU` + + Examples:: + + >>> rnn = MGU1(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(MGU1, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(MGU1Cell, **kwargs) + + +class MGU1Cell(SingleStateCellBase): + r"""A minimal gated unit variant 1 (MGU1) cell. + + [`IEEE MWSCAS 2017 `_] + + .. math:: + + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{hh}^{f}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{f} + \bigr), \\[6pt] + \tilde{\mathbf{h}}(t) &= \phi\bigl( + \mathbf{W}_{ih}\,\mathbf{x}(t) + + \mathbf{b}_{ih} + + \mathbf{W}_{hh}^{h}\bigl(\mathbf{f}(t)\circ\mathbf{h}(t-1)\bigr) + + \mathbf{b}_{hh}^{h} + \bigr), \\[6pt] + \mathbf{h}(t) &= \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{h}(t-1) + \;+\;\mathbf{f}(t)\circ\tilde{\mathbf{h}}(t), + + where :math:`\circ` is element‐wise product and :math:`\phi` is a + pointwise nonlinearity (e.g., tanh). The gate has no input term. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use recurrent biases. + Default: ``True``. + nonlinearity: Nonlinearity :math:`\phi` for the candidate. + Default: :func:`torch.tanh`. + gate_nonlinearity: Activation for the forget gate. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}^*``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for input-side biases when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for recurrent biases when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: h_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + + Variables: + weight_ih: The learnable input–hidden weights, + of shape ``(hidden_size, input_size)`` (candidate part only). + weight_hh: The learnable hidden–hidden weights, + of shape ``(2*hidden_size, hidden_size)`` (forget & candidate parts). + bias_ih: The learnable input–hidden biases, + of shape ``(hidden_size)`` if ``bias=True``. + bias_hh: The learnable hidden–hidden biases, + of shape ``(2*hidden_size)`` if ``recurrent_bias=True`` + (forget & candidate parts). + + Examples:: + + >>> cell = MGU1Cell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h = cell(x[t], h) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + nonlinearity="tanh", + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.act = resolve_activation(nonlinearity) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((hidden_size, input_size), True), + "weight_hh": ((2 * hidden_size, hidden_size), True), + "bias_ih": ((hidden_size,), bias), + "bias_hh": ((2 * hidden_size,), recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + weight_hh_f, weight_hh_h = self.weight_hh.chunk(2, 0) + bias_hh_f, bias_hh_h = self.bias_hh.chunk(2, 0) + forget_gate = self.gate_act(b_state @ weight_hh_f.t() + bias_hh_f) + ch = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + (forget_gate * b_state) @ weight_hh_h.t() + + bias_hh_h + ) + candidate_hidden = self.act(ch) + new_state = forget_gate * candidate_hidden + (1.0 - forget_gate) * b_state + + if not is_batched: + new_state = new_state.squeeze(0) + + return new_state + + +class MGU2(SingleStateRecurrentLayerBase): + r"""Multi-layer minimal gated unit variant 2 (MGU2) network. + + [`IEEE MWSCAS 2017 `_] + + Each layer consists of an :class:`MGU2Cell`, which updates the hidden + state according to: + + .. math:: + \begin{aligned} + f_t &= \sigma(W_{hh}^f h_{t-1}), \\ + \tilde{h}_t &= \phi(W_{ih} x_t + b_{ih} + + W_{hh}^h (f_t \circ h_{t-1}) + b_{hh}^h), \\ + h_t &= (1 - f_t) \circ h_{t-1} + f_t \circ \tilde{h}_t + \end{aligned} + + where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the + input at time `t`, :math:`\sigma` is the sigmoid function, :math:`\phi` is + a pointwise nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + multiplication. This is :class:`MGU` with a reduced forget gate. The gate + has no input term. + + In a multilayer MGU2, the input :math:`x^{(l)}_t` of the :math:`l`-th layer + (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous + layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two MGU2 layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + nonlinearity: Nonlinearity :math:`\phi` for the candidate. Default: + :func:`torch.tanh` + gate_nonlinearity: Activation for the forget gate. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}^*`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for input-side biases. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for recurrent biases. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape + :math:`(L, N, H_{in})` when ``batch_first=False`` or + :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial + hidden state for each element in the input sequence. Defaults to zeros if + not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape + :math:`(L, N, H_{out})` when ``batch_first=False`` or + :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output + features `(h_t)` from the last layer of the MGU2, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final + hidden state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable hidden-hidden weights of the :math:`k`-th + layer, of shape `(2*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable hidden-hidden biases of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`MGU2Cell`, :class:`MGU` + + Examples:: + + >>> rnn = MGU2(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(MGU2, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(MGU2Cell, **kwargs) + + +class MGU2Cell(SingleStateCellBase): + r"""A minimal gated unit variant 2 (MGU2) cell. + + [`IEEE MWSCAS 2017 `_] + + .. math:: + + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{hh}^{f}\,\mathbf{h}(t-1) + \bigr), \\[6pt] + \tilde{\mathbf{h}}(t) &= \phi\bigl( + \mathbf{W}_{ih}\,\mathbf{x}(t) + + \mathbf{b}_{ih} + + \mathbf{W}_{hh}^{h}\bigl(\mathbf{f}(t)\circ\mathbf{h}(t-1)\bigr) + + \mathbf{b}_{hh}^{h} + \bigr), \\[6pt] + \mathbf{h}(t) &= \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{h}(t-1) + \;+\;\mathbf{f}(t)\circ\tilde{\mathbf{h}}(t), + + where :math:`\circ` is element‐wise product and :math:`\phi` is a + pointwise nonlinearity (e.g., tanh). The gate has no input term. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use recurrent biases. + Default: ``True``. + nonlinearity: Nonlinearity :math:`\phi` for the candidate. + Default: :func:`torch.tanh`. + gate_nonlinearity: Activation for the forget gate. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}^*``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for input-side biases when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for recurrent biases when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: h_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + + Variables: + weight_ih: The learnable input–hidden weights, + of shape ``(hidden_size, input_size)`` (candidate part only). + weight_hh: The learnable hidden–hidden weights, + of shape ``(2*hidden_size, hidden_size)`` (forget & candidate parts). + bias_ih: The learnable input–hidden biases, + of shape ``(hidden_size)`` if ``bias=True``. + bias_hh: The learnable hidden–hidden biases, + of shape ``(hidden_size)`` if ``recurrent_bias=True`` + (candidate part only; the gate has no bias). + + Examples:: + + >>> cell = MGU2Cell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h = cell(x[t], h) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + nonlinearity="tanh", + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.act = resolve_activation(nonlinearity) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((hidden_size, input_size), True), + "weight_hh": ((2 * hidden_size, hidden_size), True), + "bias_ih": ((hidden_size,), bias), + "bias_hh": ((hidden_size,), recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + weight_hh_f, weight_hh_h = self.weight_hh.chunk(2, 0) + bias_hh_h = self.bias_hh + forget_gate = self.gate_act(b_state @ weight_hh_f.t()) + ch = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + (forget_gate * b_state) @ weight_hh_h.t() + + bias_hh_h + ) + candidate_hidden = self.act(ch) + new_state = forget_gate * candidate_hidden + (1.0 - forget_gate) * b_state + + if not is_batched: + new_state = new_state.squeeze(0) + + return new_state + + +class MGU3(SingleStateRecurrentLayerBase): + r"""Multi-layer minimal gated unit variant 3 (MGU3) network. + + [`IEEE MWSCAS 2017 `_] + + Each layer consists of an :class:`MGU3Cell`, which updates the hidden + state according to: + + .. math:: + \begin{aligned} + f_t &= \sigma(b^f), \\ + \tilde{h}_t &= \phi(W_{ih} x_t + b_{ih} + + W_{hh}^h (f_t \circ h_{t-1}) + b_{hh}^h), \\ + h_t &= (1 - f_t) \circ h_{t-1} + f_t \circ \tilde{h}_t + \end{aligned} + + where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the + input at time `t`, :math:`\sigma` is the sigmoid function, :math:`\phi` is + a pointwise nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + multiplication. This is :class:`MGU` with a reduced forget gate. The gate + is a learned constant, independent of both input and state. + + In a multilayer MGU3, the input :math:`x^{(l)}_t` of the :math:`l`-th layer + (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous + layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two MGU3 layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + nonlinearity: Nonlinearity :math:`\phi` for the candidate. Default: + :func:`torch.tanh` + gate_nonlinearity: Activation for the forget gate. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}^*`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for input-side biases and `b^f`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for recurrent biases. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape + :math:`(L, N, H_{in})` when ``batch_first=False`` or + :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of + the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial + hidden state for each element in the input sequence. Defaults to zeros if + not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape + :math:`(L, N, H_{out})` when ``batch_first=False`` or + :math:`(N, L, H_{out})` when ``batch_first=True`` containing the output + features `(h_t)` from the last layer of the MGU3, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final + hidden state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable hidden-hidden weights of the :math:`k`-th + layer, of shape `(hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable hidden-hidden biases of the :math:`k`-th + layer, of shape `(hidden_size)`. Only present when ``recurrent_bias=True``. + cells.{k}.bias_f : the learnable forget-gate bias of the :math:`k`-th + layer, of shape `(hidden_size)`. Always present. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + Unlike :class:`MGU`, the forget gate uses a learnable bias ``bias_f`` + that is present regardless of ``bias`` and ``recurrent_bias``. + + .. seealso:: + :class:`MGU3Cell`, :class:`MGU` + + Examples:: + + >>> rnn = MGU3(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(MGU3, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(MGU3Cell, **kwargs) + + +class MGU3Cell(SingleStateCellBase): + r"""A minimal gated unit variant 3 (MGU3) cell. + + [`IEEE MWSCAS 2017 `_] + + .. math:: + + \mathbf{f}(t) &= \sigma\bigl(\mathbf{b}^{f}\bigr), \\[6pt] + \tilde{\mathbf{h}}(t) &= \phi\bigl( + \mathbf{W}_{ih}\,\mathbf{x}(t) + + \mathbf{b}_{ih} + + \mathbf{W}_{hh}^{h}\bigl(\mathbf{f}(t)\circ\mathbf{h}(t-1)\bigr) + + \mathbf{b}_{hh}^{h} + \bigr), \\[6pt] + \mathbf{h}(t) &= \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{h}(t-1) + \;+\;\mathbf{f}(t)\circ\tilde{\mathbf{h}}(t), + + where :math:`\circ` is element‐wise product and :math:`\phi` is a + pointwise nonlinearity (e.g., tanh). The gate is a learned constant, + independent of both input and state. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use recurrent biases. + Default: ``True``. + nonlinearity: Nonlinearity :math:`\phi` for the candidate. + Default: :func:`torch.tanh`. + gate_nonlinearity: Activation for the forget gate. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}^*``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for input-side biases and ``b^f``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for recurrent biases when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: h_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + + Variables: + weight_ih: The learnable input–hidden weights, + of shape ``(hidden_size, input_size)`` (candidate part only). + weight_hh: The learnable hidden–hidden weights, + of shape ``(hidden_size, hidden_size)`` (candidate part only). + bias_ih: The learnable input–hidden biases, + of shape ``(hidden_size)`` if ``bias=True``. + bias_hh: The learnable hidden–hidden biases, + of shape ``(hidden_size)`` if ``recurrent_bias=True`` + (candidate part only). + bias_f: The learnable forget-gate bias, of shape ``(hidden_size)``. + Always present, since it is the only term of the gate. + + Examples:: + + >>> cell = MGU3Cell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h = cell(x[t], h) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + bias_f: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + nonlinearity="tanh", + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.act = resolve_activation(nonlinearity) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((hidden_size, input_size), True), + "weight_hh": ((hidden_size, hidden_size), True), + "bias_f": ((hidden_size,), True), + "bias_ih": ((hidden_size,), bias), + "bias_hh": ((hidden_size,), recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def reset_parameters(self) -> None: + apply_init_(self.weight_ih, self.init_cfg["kernel"]) + apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) + apply_init_(self.bias_f, self.init_cfg["bias"]) + if isinstance(self.bias_ih, nn.Parameter): + apply_init_(self.bias_ih, self.init_cfg["bias"]) + if isinstance(self.bias_hh, nn.Parameter): + apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + weight_hh_h = self.weight_hh + bias_hh_h = self.bias_hh + forget_gate = self.gate_act(self.bias_f).expand_as(b_state) + ch = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + (forget_gate * b_state) @ weight_hh_h.t() + + bias_hh_h + ) + candidate_hidden = self.act(ch) + new_state = forget_gate * candidate_hidden + (1.0 - forget_gate) * b_state + + if not is_batched: + new_state = new_state.squeeze(0) + + return new_state diff --git a/torchrecurrent/cells/multiplicativelstm_cell.py b/torchrecurrent/cells/multiplicativelstm_cell.py index 492f2eb..a3a9c4e 100755 --- a/torchrecurrent/cells/multiplicativelstm_cell.py +++ b/torchrecurrent/cells/multiplicativelstm_cell.py @@ -319,11 +319,11 @@ def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) apply_init_(self.weight_mh, self.init_cfg["multiplicative_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) - if hasattr(self, "bias_mh"): + if isinstance(self.bias_mh, nn.Parameter): apply_init_(self.bias_mh, self.init_cfg["multiplicative_bias"]) def forward( diff --git a/torchrecurrent/cells/peepholelstm_cell.py b/torchrecurrent/cells/peepholelstm_cell.py index b590086..83274ce 100755 --- a/torchrecurrent/cells/peepholelstm_cell.py +++ b/torchrecurrent/cells/peepholelstm_cell.py @@ -311,9 +311,9 @@ def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) apply_init_(self.weight_ph, self.init_cfg["peephole_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward( diff --git a/torchrecurrent/cells/reslstm_cell.py b/torchrecurrent/cells/reslstm_cell.py index a7cef19..829e3a6 100644 --- a/torchrecurrent/cells/reslstm_cell.py +++ b/torchrecurrent/cells/reslstm_cell.py @@ -344,9 +344,9 @@ def reset_parameters(self) -> None: apply_init_(self.weight_proj, self.init_cfg["proj"]) apply_init_(self.weight_res, self.init_cfg["residual"]) apply_init_(self.weight_ph, self.init_cfg["peephole_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward( diff --git a/torchrecurrent/cells/scrn_cell.py b/torchrecurrent/cells/scrn_cell.py index 0846bc2..69ba4b8 100755 --- a/torchrecurrent/cells/scrn_cell.py +++ b/torchrecurrent/cells/scrn_cell.py @@ -306,11 +306,11 @@ def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) apply_init_(self.weight_ch, self.init_cfg["context_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) - if hasattr(self, "bias_ch"): + if isinstance(self.bias_ch, nn.Parameter): apply_init_(self.bias_ch, self.init_cfg["context_bias"]) def forward( diff --git a/torchrecurrent/cells/unicornn_cell.py b/torchrecurrent/cells/unicornn_cell.py index 59b8034..246682e 100755 --- a/torchrecurrent/cells/unicornn_cell.py +++ b/torchrecurrent/cells/unicornn_cell.py @@ -277,9 +277,9 @@ def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) apply_init_(self.weight_ch, self.init_cfg["control_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) def forward( diff --git a/torchrecurrent/cells/wmclstm_cell.py b/torchrecurrent/cells/wmclstm_cell.py index 708b567..101dde6 100755 --- a/torchrecurrent/cells/wmclstm_cell.py +++ b/torchrecurrent/cells/wmclstm_cell.py @@ -299,11 +299,11 @@ def reset_parameters(self) -> None: apply_init_(self.weight_ih, self.init_cfg["kernel"]) apply_init_(self.weight_hh, self.init_cfg["recurrent_kernel"]) apply_init_(self.weight_mh, self.init_cfg["memory_kernel"]) - if hasattr(self, "bias_ih"): + if isinstance(self.bias_ih, nn.Parameter): apply_init_(self.bias_ih, self.init_cfg["bias"]) - if hasattr(self, "bias_hh"): + if isinstance(self.bias_hh, nn.Parameter): apply_init_(self.bias_hh, self.init_cfg["recurrent_bias"]) - if hasattr(self, "bias_mh"): + if isinstance(self.bias_mh, nn.Parameter): apply_init_(self.bias_mh, self.init_cfg["memory_bias"]) def forward(