diff --git a/docs/api/cells.rst b/docs/api/cells.rst index a664297..a923158 100755 --- a/docs/api/cells.rst +++ b/docs/api/cells.rst @@ -18,10 +18,12 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.FastGRNNCell torchrecurrent.ATRCell torchrecurrent.IndRNNCell + torchrecurrent.IntersectionRNNCell torchrecurrent.JANETCell torchrecurrent.LEMCell torchrecurrent.LightRUCell torchrecurrent.LiGRUCell + torchrecurrent.MCLSTMCell torchrecurrent.MGUCell torchrecurrent.MinimalRNNCell torchrecurrent.MiRU1Cell @@ -40,6 +42,9 @@ This page documents all custom recurrent cells provided in the `torchrecurrent.c torchrecurrent.SGRNCell torchrecurrent.STARCell torchrecurrent.tauGRUCell + torchrecurrent.TRNNCell + torchrecurrent.TGRUCell + torchrecurrent.TLSTMCell torchrecurrent.UGRNNCell torchrecurrent.UnICORNNCell torchrecurrent.WMCLSTMCell diff --git a/docs/api/layers.rst b/docs/api/layers.rst index 1cf9623..b254891 100644 --- a/docs/api/layers.rst +++ b/docs/api/layers.rst @@ -18,10 +18,12 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.FastGRNN torchrecurrent.ATR torchrecurrent.IndRNN + torchrecurrent.IntersectionRNN torchrecurrent.JANET torchrecurrent.LEM torchrecurrent.LightRU torchrecurrent.LiGRU + torchrecurrent.MCLSTM torchrecurrent.MGU torchrecurrent.MinimalRNN torchrecurrent.MiRU1 @@ -40,6 +42,9 @@ This page documents all custom recurrent layers provided in the `torchrecurrent` torchrecurrent.SGRN torchrecurrent.STAR torchrecurrent.tauGRU + torchrecurrent.TRNN + torchrecurrent.TGRU + torchrecurrent.TLSTM torchrecurrent.UGRNN torchrecurrent.UnICORNN torchrecurrent.WMCLSTM diff --git a/docs/generated/torchrecurrent.IntersectionRNN.rst b/docs/generated/torchrecurrent.IntersectionRNN.rst new file mode 100644 index 0000000..5dc16d0 --- /dev/null +++ b/docs/generated/torchrecurrent.IntersectionRNN.rst @@ -0,0 +1,88 @@ +torchrecurrent.IntersectionRNN +============================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: IntersectionRNN + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~IntersectionRNN.__init__ + ~IntersectionRNN.add_module + ~IntersectionRNN.apply + ~IntersectionRNN.bfloat16 + ~IntersectionRNN.buffers + ~IntersectionRNN.children + ~IntersectionRNN.compile + ~IntersectionRNN.cpu + ~IntersectionRNN.cuda + ~IntersectionRNN.double + ~IntersectionRNN.eval + ~IntersectionRNN.extra_repr + ~IntersectionRNN.float + ~IntersectionRNN.forward + ~IntersectionRNN.get_buffer + ~IntersectionRNN.get_extra_state + ~IntersectionRNN.get_parameter + ~IntersectionRNN.get_submodule + ~IntersectionRNN.half + ~IntersectionRNN.initialize_cells + ~IntersectionRNN.ipu + ~IntersectionRNN.load_state_dict + ~IntersectionRNN.modules + ~IntersectionRNN.mtia + ~IntersectionRNN.named_buffers + ~IntersectionRNN.named_children + ~IntersectionRNN.named_modules + ~IntersectionRNN.named_parameters + ~IntersectionRNN.parameters + ~IntersectionRNN.register_backward_hook + ~IntersectionRNN.register_buffer + ~IntersectionRNN.register_forward_hook + ~IntersectionRNN.register_forward_pre_hook + ~IntersectionRNN.register_full_backward_hook + ~IntersectionRNN.register_full_backward_pre_hook + ~IntersectionRNN.register_load_state_dict_post_hook + ~IntersectionRNN.register_load_state_dict_pre_hook + ~IntersectionRNN.register_module + ~IntersectionRNN.register_parameter + ~IntersectionRNN.register_state_dict_post_hook + ~IntersectionRNN.register_state_dict_pre_hook + ~IntersectionRNN.requires_grad_ + ~IntersectionRNN.set_extra_state + ~IntersectionRNN.set_submodule + ~IntersectionRNN.share_memory + ~IntersectionRNN.state_dict + ~IntersectionRNN.to + ~IntersectionRNN.to_empty + ~IntersectionRNN.train + ~IntersectionRNN.type + ~IntersectionRNN.xpu + ~IntersectionRNN.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~IntersectionRNN.T_destination + ~IntersectionRNN.call_super_init + ~IntersectionRNN.dump_patches + ~IntersectionRNN.input_size + ~IntersectionRNN.hidden_size + ~IntersectionRNN.num_layers + ~IntersectionRNN.batch_first + ~IntersectionRNN.dropout + ~IntersectionRNN.cells + ~IntersectionRNN.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.IntersectionRNNCell.rst b/docs/generated/torchrecurrent.IntersectionRNNCell.rst new file mode 100644 index 0000000..b331b40 --- /dev/null +++ b/docs/generated/torchrecurrent.IntersectionRNNCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.IntersectionRNNCell +================================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: IntersectionRNNCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~IntersectionRNNCell.__init__ + ~IntersectionRNNCell.add_module + ~IntersectionRNNCell.apply + ~IntersectionRNNCell.bfloat16 + ~IntersectionRNNCell.buffers + ~IntersectionRNNCell.children + ~IntersectionRNNCell.compile + ~IntersectionRNNCell.cpu + ~IntersectionRNNCell.cuda + ~IntersectionRNNCell.double + ~IntersectionRNNCell.eval + ~IntersectionRNNCell.extra_repr + ~IntersectionRNNCell.float + ~IntersectionRNNCell.forward + ~IntersectionRNNCell.get_buffer + ~IntersectionRNNCell.get_extra_state + ~IntersectionRNNCell.get_parameter + ~IntersectionRNNCell.get_submodule + ~IntersectionRNNCell.half + ~IntersectionRNNCell.ipu + ~IntersectionRNNCell.load_state_dict + ~IntersectionRNNCell.modules + ~IntersectionRNNCell.mtia + ~IntersectionRNNCell.named_buffers + ~IntersectionRNNCell.named_children + ~IntersectionRNNCell.named_modules + ~IntersectionRNNCell.named_parameters + ~IntersectionRNNCell.parameters + ~IntersectionRNNCell.register_backward_hook + ~IntersectionRNNCell.register_buffer + ~IntersectionRNNCell.register_forward_hook + ~IntersectionRNNCell.register_forward_pre_hook + ~IntersectionRNNCell.register_full_backward_hook + ~IntersectionRNNCell.register_full_backward_pre_hook + ~IntersectionRNNCell.register_load_state_dict_post_hook + ~IntersectionRNNCell.register_load_state_dict_pre_hook + ~IntersectionRNNCell.register_module + ~IntersectionRNNCell.register_parameter + ~IntersectionRNNCell.register_state_dict_post_hook + ~IntersectionRNNCell.register_state_dict_pre_hook + ~IntersectionRNNCell.requires_grad_ + ~IntersectionRNNCell.reset_parameters + ~IntersectionRNNCell.set_extra_state + ~IntersectionRNNCell.set_submodule + ~IntersectionRNNCell.share_memory + ~IntersectionRNNCell.state_dict + ~IntersectionRNNCell.to + ~IntersectionRNNCell.to_empty + ~IntersectionRNNCell.train + ~IntersectionRNNCell.type + ~IntersectionRNNCell.uses_double_state + ~IntersectionRNNCell.xpu + ~IntersectionRNNCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~IntersectionRNNCell.T_destination + ~IntersectionRNNCell.call_super_init + ~IntersectionRNNCell.dump_patches + ~IntersectionRNNCell.weight_ih + ~IntersectionRNNCell.weight_hh + ~IntersectionRNNCell.bias_ih + ~IntersectionRNNCell.bias_hh + ~IntersectionRNNCell.input_size + ~IntersectionRNNCell.hidden_size + ~IntersectionRNNCell.bias + ~IntersectionRNNCell.recurrent_bias + ~IntersectionRNNCell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MCLSTM.rst b/docs/generated/torchrecurrent.MCLSTM.rst new file mode 100644 index 0000000..32bea58 --- /dev/null +++ b/docs/generated/torchrecurrent.MCLSTM.rst @@ -0,0 +1,88 @@ +torchrecurrent.MCLSTM +===================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: MCLSTM + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MCLSTM.__init__ + ~MCLSTM.add_module + ~MCLSTM.apply + ~MCLSTM.bfloat16 + ~MCLSTM.buffers + ~MCLSTM.children + ~MCLSTM.compile + ~MCLSTM.cpu + ~MCLSTM.cuda + ~MCLSTM.double + ~MCLSTM.eval + ~MCLSTM.extra_repr + ~MCLSTM.float + ~MCLSTM.forward + ~MCLSTM.get_buffer + ~MCLSTM.get_extra_state + ~MCLSTM.get_parameter + ~MCLSTM.get_submodule + ~MCLSTM.half + ~MCLSTM.initialize_cells + ~MCLSTM.ipu + ~MCLSTM.load_state_dict + ~MCLSTM.modules + ~MCLSTM.mtia + ~MCLSTM.named_buffers + ~MCLSTM.named_children + ~MCLSTM.named_modules + ~MCLSTM.named_parameters + ~MCLSTM.parameters + ~MCLSTM.register_backward_hook + ~MCLSTM.register_buffer + ~MCLSTM.register_forward_hook + ~MCLSTM.register_forward_pre_hook + ~MCLSTM.register_full_backward_hook + ~MCLSTM.register_full_backward_pre_hook + ~MCLSTM.register_load_state_dict_post_hook + ~MCLSTM.register_load_state_dict_pre_hook + ~MCLSTM.register_module + ~MCLSTM.register_parameter + ~MCLSTM.register_state_dict_post_hook + ~MCLSTM.register_state_dict_pre_hook + ~MCLSTM.requires_grad_ + ~MCLSTM.set_extra_state + ~MCLSTM.set_submodule + ~MCLSTM.share_memory + ~MCLSTM.state_dict + ~MCLSTM.to + ~MCLSTM.to_empty + ~MCLSTM.train + ~MCLSTM.type + ~MCLSTM.xpu + ~MCLSTM.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MCLSTM.T_destination + ~MCLSTM.call_super_init + ~MCLSTM.dump_patches + ~MCLSTM.input_size + ~MCLSTM.hidden_size + ~MCLSTM.num_layers + ~MCLSTM.batch_first + ~MCLSTM.dropout + ~MCLSTM.cells + ~MCLSTM.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.MCLSTMCell.rst b/docs/generated/torchrecurrent.MCLSTMCell.rst new file mode 100644 index 0000000..a6d4005 --- /dev/null +++ b/docs/generated/torchrecurrent.MCLSTMCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.MCLSTMCell +========================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: MCLSTMCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~MCLSTMCell.__init__ + ~MCLSTMCell.add_module + ~MCLSTMCell.apply + ~MCLSTMCell.bfloat16 + ~MCLSTMCell.buffers + ~MCLSTMCell.children + ~MCLSTMCell.compile + ~MCLSTMCell.cpu + ~MCLSTMCell.cuda + ~MCLSTMCell.double + ~MCLSTMCell.eval + ~MCLSTMCell.extra_repr + ~MCLSTMCell.float + ~MCLSTMCell.forward + ~MCLSTMCell.get_buffer + ~MCLSTMCell.get_extra_state + ~MCLSTMCell.get_parameter + ~MCLSTMCell.get_submodule + ~MCLSTMCell.half + ~MCLSTMCell.ipu + ~MCLSTMCell.load_state_dict + ~MCLSTMCell.modules + ~MCLSTMCell.mtia + ~MCLSTMCell.named_buffers + ~MCLSTMCell.named_children + ~MCLSTMCell.named_modules + ~MCLSTMCell.named_parameters + ~MCLSTMCell.parameters + ~MCLSTMCell.register_backward_hook + ~MCLSTMCell.register_buffer + ~MCLSTMCell.register_forward_hook + ~MCLSTMCell.register_forward_pre_hook + ~MCLSTMCell.register_full_backward_hook + ~MCLSTMCell.register_full_backward_pre_hook + ~MCLSTMCell.register_load_state_dict_post_hook + ~MCLSTMCell.register_load_state_dict_pre_hook + ~MCLSTMCell.register_module + ~MCLSTMCell.register_parameter + ~MCLSTMCell.register_state_dict_post_hook + ~MCLSTMCell.register_state_dict_pre_hook + ~MCLSTMCell.requires_grad_ + ~MCLSTMCell.reset_parameters + ~MCLSTMCell.set_extra_state + ~MCLSTMCell.set_submodule + ~MCLSTMCell.share_memory + ~MCLSTMCell.state_dict + ~MCLSTMCell.to + ~MCLSTMCell.to_empty + ~MCLSTMCell.train + ~MCLSTMCell.type + ~MCLSTMCell.uses_double_state + ~MCLSTMCell.xpu + ~MCLSTMCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~MCLSTMCell.T_destination + ~MCLSTMCell.call_super_init + ~MCLSTMCell.dump_patches + ~MCLSTMCell.weight_ih + ~MCLSTMCell.weight_hh + ~MCLSTMCell.bias_ih + ~MCLSTMCell.bias_hh + ~MCLSTMCell.input_size + ~MCLSTMCell.hidden_size + ~MCLSTMCell.bias + ~MCLSTMCell.recurrent_bias + ~MCLSTMCell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TGRU.rst b/docs/generated/torchrecurrent.TGRU.rst new file mode 100644 index 0000000..19329c2 --- /dev/null +++ b/docs/generated/torchrecurrent.TGRU.rst @@ -0,0 +1,88 @@ +torchrecurrent.TGRU +=================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TGRU + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TGRU.__init__ + ~TGRU.add_module + ~TGRU.apply + ~TGRU.bfloat16 + ~TGRU.buffers + ~TGRU.children + ~TGRU.compile + ~TGRU.cpu + ~TGRU.cuda + ~TGRU.double + ~TGRU.eval + ~TGRU.extra_repr + ~TGRU.float + ~TGRU.forward + ~TGRU.get_buffer + ~TGRU.get_extra_state + ~TGRU.get_parameter + ~TGRU.get_submodule + ~TGRU.half + ~TGRU.initialize_cells + ~TGRU.ipu + ~TGRU.load_state_dict + ~TGRU.modules + ~TGRU.mtia + ~TGRU.named_buffers + ~TGRU.named_children + ~TGRU.named_modules + ~TGRU.named_parameters + ~TGRU.parameters + ~TGRU.register_backward_hook + ~TGRU.register_buffer + ~TGRU.register_forward_hook + ~TGRU.register_forward_pre_hook + ~TGRU.register_full_backward_hook + ~TGRU.register_full_backward_pre_hook + ~TGRU.register_load_state_dict_post_hook + ~TGRU.register_load_state_dict_pre_hook + ~TGRU.register_module + ~TGRU.register_parameter + ~TGRU.register_state_dict_post_hook + ~TGRU.register_state_dict_pre_hook + ~TGRU.requires_grad_ + ~TGRU.set_extra_state + ~TGRU.set_submodule + ~TGRU.share_memory + ~TGRU.state_dict + ~TGRU.to + ~TGRU.to_empty + ~TGRU.train + ~TGRU.type + ~TGRU.xpu + ~TGRU.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TGRU.T_destination + ~TGRU.call_super_init + ~TGRU.dump_patches + ~TGRU.input_size + ~TGRU.hidden_size + ~TGRU.num_layers + ~TGRU.batch_first + ~TGRU.dropout + ~TGRU.cells + ~TGRU.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TGRUCell.rst b/docs/generated/torchrecurrent.TGRUCell.rst new file mode 100644 index 0000000..b85ee50 --- /dev/null +++ b/docs/generated/torchrecurrent.TGRUCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.TGRUCell +======================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: TGRUCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TGRUCell.__init__ + ~TGRUCell.add_module + ~TGRUCell.apply + ~TGRUCell.bfloat16 + ~TGRUCell.buffers + ~TGRUCell.children + ~TGRUCell.compile + ~TGRUCell.cpu + ~TGRUCell.cuda + ~TGRUCell.double + ~TGRUCell.eval + ~TGRUCell.extra_repr + ~TGRUCell.float + ~TGRUCell.forward + ~TGRUCell.get_buffer + ~TGRUCell.get_extra_state + ~TGRUCell.get_parameter + ~TGRUCell.get_submodule + ~TGRUCell.half + ~TGRUCell.ipu + ~TGRUCell.load_state_dict + ~TGRUCell.modules + ~TGRUCell.mtia + ~TGRUCell.named_buffers + ~TGRUCell.named_children + ~TGRUCell.named_modules + ~TGRUCell.named_parameters + ~TGRUCell.parameters + ~TGRUCell.register_backward_hook + ~TGRUCell.register_buffer + ~TGRUCell.register_forward_hook + ~TGRUCell.register_forward_pre_hook + ~TGRUCell.register_full_backward_hook + ~TGRUCell.register_full_backward_pre_hook + ~TGRUCell.register_load_state_dict_post_hook + ~TGRUCell.register_load_state_dict_pre_hook + ~TGRUCell.register_module + ~TGRUCell.register_parameter + ~TGRUCell.register_state_dict_post_hook + ~TGRUCell.register_state_dict_pre_hook + ~TGRUCell.requires_grad_ + ~TGRUCell.reset_parameters + ~TGRUCell.set_extra_state + ~TGRUCell.set_submodule + ~TGRUCell.share_memory + ~TGRUCell.state_dict + ~TGRUCell.to + ~TGRUCell.to_empty + ~TGRUCell.train + ~TGRUCell.type + ~TGRUCell.uses_double_state + ~TGRUCell.xpu + ~TGRUCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TGRUCell.T_destination + ~TGRUCell.call_super_init + ~TGRUCell.dump_patches + ~TGRUCell.weight_ih + ~TGRUCell.weight_hh + ~TGRUCell.bias_ih + ~TGRUCell.bias_hh + ~TGRUCell.input_size + ~TGRUCell.hidden_size + ~TGRUCell.bias + ~TGRUCell.recurrent_bias + ~TGRUCell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TLSTM.rst b/docs/generated/torchrecurrent.TLSTM.rst new file mode 100644 index 0000000..7dceb0a --- /dev/null +++ b/docs/generated/torchrecurrent.TLSTM.rst @@ -0,0 +1,88 @@ +torchrecurrent.TLSTM +==================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TLSTM + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TLSTM.__init__ + ~TLSTM.add_module + ~TLSTM.apply + ~TLSTM.bfloat16 + ~TLSTM.buffers + ~TLSTM.children + ~TLSTM.compile + ~TLSTM.cpu + ~TLSTM.cuda + ~TLSTM.double + ~TLSTM.eval + ~TLSTM.extra_repr + ~TLSTM.float + ~TLSTM.forward + ~TLSTM.get_buffer + ~TLSTM.get_extra_state + ~TLSTM.get_parameter + ~TLSTM.get_submodule + ~TLSTM.half + ~TLSTM.initialize_cells + ~TLSTM.ipu + ~TLSTM.load_state_dict + ~TLSTM.modules + ~TLSTM.mtia + ~TLSTM.named_buffers + ~TLSTM.named_children + ~TLSTM.named_modules + ~TLSTM.named_parameters + ~TLSTM.parameters + ~TLSTM.register_backward_hook + ~TLSTM.register_buffer + ~TLSTM.register_forward_hook + ~TLSTM.register_forward_pre_hook + ~TLSTM.register_full_backward_hook + ~TLSTM.register_full_backward_pre_hook + ~TLSTM.register_load_state_dict_post_hook + ~TLSTM.register_load_state_dict_pre_hook + ~TLSTM.register_module + ~TLSTM.register_parameter + ~TLSTM.register_state_dict_post_hook + ~TLSTM.register_state_dict_pre_hook + ~TLSTM.requires_grad_ + ~TLSTM.set_extra_state + ~TLSTM.set_submodule + ~TLSTM.share_memory + ~TLSTM.state_dict + ~TLSTM.to + ~TLSTM.to_empty + ~TLSTM.train + ~TLSTM.type + ~TLSTM.xpu + ~TLSTM.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TLSTM.T_destination + ~TLSTM.call_super_init + ~TLSTM.dump_patches + ~TLSTM.input_size + ~TLSTM.hidden_size + ~TLSTM.num_layers + ~TLSTM.batch_first + ~TLSTM.dropout + ~TLSTM.cells + ~TLSTM.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TLSTMCell.rst b/docs/generated/torchrecurrent.TLSTMCell.rst new file mode 100644 index 0000000..dc7d782 --- /dev/null +++ b/docs/generated/torchrecurrent.TLSTMCell.rst @@ -0,0 +1,91 @@ +torchrecurrent.TLSTMCell +======================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TLSTMCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TLSTMCell.__init__ + ~TLSTMCell.add_module + ~TLSTMCell.apply + ~TLSTMCell.bfloat16 + ~TLSTMCell.buffers + ~TLSTMCell.children + ~TLSTMCell.compile + ~TLSTMCell.cpu + ~TLSTMCell.cuda + ~TLSTMCell.double + ~TLSTMCell.eval + ~TLSTMCell.extra_repr + ~TLSTMCell.float + ~TLSTMCell.forward + ~TLSTMCell.get_buffer + ~TLSTMCell.get_extra_state + ~TLSTMCell.get_parameter + ~TLSTMCell.get_submodule + ~TLSTMCell.half + ~TLSTMCell.ipu + ~TLSTMCell.load_state_dict + ~TLSTMCell.modules + ~TLSTMCell.mtia + ~TLSTMCell.named_buffers + ~TLSTMCell.named_children + ~TLSTMCell.named_modules + ~TLSTMCell.named_parameters + ~TLSTMCell.parameters + ~TLSTMCell.register_backward_hook + ~TLSTMCell.register_buffer + ~TLSTMCell.register_forward_hook + ~TLSTMCell.register_forward_pre_hook + ~TLSTMCell.register_full_backward_hook + ~TLSTMCell.register_full_backward_pre_hook + ~TLSTMCell.register_load_state_dict_post_hook + ~TLSTMCell.register_load_state_dict_pre_hook + ~TLSTMCell.register_module + ~TLSTMCell.register_parameter + ~TLSTMCell.register_state_dict_post_hook + ~TLSTMCell.register_state_dict_pre_hook + ~TLSTMCell.requires_grad_ + ~TLSTMCell.reset_parameters + ~TLSTMCell.set_extra_state + ~TLSTMCell.set_submodule + ~TLSTMCell.share_memory + ~TLSTMCell.state_dict + ~TLSTMCell.to + ~TLSTMCell.to_empty + ~TLSTMCell.train + ~TLSTMCell.type + ~TLSTMCell.uses_double_state + ~TLSTMCell.xpu + ~TLSTMCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TLSTMCell.T_destination + ~TLSTMCell.call_super_init + ~TLSTMCell.dump_patches + ~TLSTMCell.weight_ih + ~TLSTMCell.weight_hh + ~TLSTMCell.bias_ih + ~TLSTMCell.bias_hh + ~TLSTMCell.input_size + ~TLSTMCell.hidden_size + ~TLSTMCell.bias + ~TLSTMCell.recurrent_bias + ~TLSTMCell.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TRNN.rst b/docs/generated/torchrecurrent.TRNN.rst new file mode 100644 index 0000000..3943393 --- /dev/null +++ b/docs/generated/torchrecurrent.TRNN.rst @@ -0,0 +1,88 @@ +torchrecurrent.TRNN +=================== + +.. currentmodule:: torchrecurrent + +.. autoclass:: TRNN + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TRNN.__init__ + ~TRNN.add_module + ~TRNN.apply + ~TRNN.bfloat16 + ~TRNN.buffers + ~TRNN.children + ~TRNN.compile + ~TRNN.cpu + ~TRNN.cuda + ~TRNN.double + ~TRNN.eval + ~TRNN.extra_repr + ~TRNN.float + ~TRNN.forward + ~TRNN.get_buffer + ~TRNN.get_extra_state + ~TRNN.get_parameter + ~TRNN.get_submodule + ~TRNN.half + ~TRNN.initialize_cells + ~TRNN.ipu + ~TRNN.load_state_dict + ~TRNN.modules + ~TRNN.mtia + ~TRNN.named_buffers + ~TRNN.named_children + ~TRNN.named_modules + ~TRNN.named_parameters + ~TRNN.parameters + ~TRNN.register_backward_hook + ~TRNN.register_buffer + ~TRNN.register_forward_hook + ~TRNN.register_forward_pre_hook + ~TRNN.register_full_backward_hook + ~TRNN.register_full_backward_pre_hook + ~TRNN.register_load_state_dict_post_hook + ~TRNN.register_load_state_dict_pre_hook + ~TRNN.register_module + ~TRNN.register_parameter + ~TRNN.register_state_dict_post_hook + ~TRNN.register_state_dict_pre_hook + ~TRNN.requires_grad_ + ~TRNN.set_extra_state + ~TRNN.set_submodule + ~TRNN.share_memory + ~TRNN.state_dict + ~TRNN.to + ~TRNN.to_empty + ~TRNN.train + ~TRNN.type + ~TRNN.xpu + ~TRNN.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TRNN.T_destination + ~TRNN.call_super_init + ~TRNN.dump_patches + ~TRNN.input_size + ~TRNN.hidden_size + ~TRNN.num_layers + ~TRNN.batch_first + ~TRNN.dropout + ~TRNN.cells + ~TRNN.training + + \ No newline at end of file diff --git a/docs/generated/torchrecurrent.TRNNCell.rst b/docs/generated/torchrecurrent.TRNNCell.rst new file mode 100644 index 0000000..9b81f87 --- /dev/null +++ b/docs/generated/torchrecurrent.TRNNCell.rst @@ -0,0 +1,89 @@ +torchrecurrent.TRNNCell +======================= + +.. currentmodule:: torchrecurrent + +.. autoclass:: TRNNCell + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~TRNNCell.__init__ + ~TRNNCell.add_module + ~TRNNCell.apply + ~TRNNCell.bfloat16 + ~TRNNCell.buffers + ~TRNNCell.children + ~TRNNCell.compile + ~TRNNCell.cpu + ~TRNNCell.cuda + ~TRNNCell.double + ~TRNNCell.eval + ~TRNNCell.extra_repr + ~TRNNCell.float + ~TRNNCell.forward + ~TRNNCell.get_buffer + ~TRNNCell.get_extra_state + ~TRNNCell.get_parameter + ~TRNNCell.get_submodule + ~TRNNCell.half + ~TRNNCell.ipu + ~TRNNCell.load_state_dict + ~TRNNCell.modules + ~TRNNCell.mtia + ~TRNNCell.named_buffers + ~TRNNCell.named_children + ~TRNNCell.named_modules + ~TRNNCell.named_parameters + ~TRNNCell.parameters + ~TRNNCell.register_backward_hook + ~TRNNCell.register_buffer + ~TRNNCell.register_forward_hook + ~TRNNCell.register_forward_pre_hook + ~TRNNCell.register_full_backward_hook + ~TRNNCell.register_full_backward_pre_hook + ~TRNNCell.register_load_state_dict_post_hook + ~TRNNCell.register_load_state_dict_pre_hook + ~TRNNCell.register_module + ~TRNNCell.register_parameter + ~TRNNCell.register_state_dict_post_hook + ~TRNNCell.register_state_dict_pre_hook + ~TRNNCell.requires_grad_ + ~TRNNCell.reset_parameters + ~TRNNCell.set_extra_state + ~TRNNCell.set_submodule + ~TRNNCell.share_memory + ~TRNNCell.state_dict + ~TRNNCell.to + ~TRNNCell.to_empty + ~TRNNCell.train + ~TRNNCell.type + ~TRNNCell.uses_double_state + ~TRNNCell.xpu + ~TRNNCell.zero_grad + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~TRNNCell.T_destination + ~TRNNCell.call_super_init + ~TRNNCell.dump_patches + ~TRNNCell.weight_ih + ~TRNNCell.bias_ih + ~TRNNCell.input_size + ~TRNNCell.hidden_size + ~TRNNCell.bias + ~TRNNCell.recurrent_bias + ~TRNNCell.training + + \ No newline at end of file diff --git a/docs/models.rst b/docs/models.rst index 1e00a46..7eedea5 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -47,6 +47,9 @@ references and official implementations where available. * - :doc:`IndRNN ` - `CVPR 2018 `__ - `Sunnydreamrain/IndRNN_Theano_Lasagne `__ + * - :doc:`IntersectionRNN ` + - `ICLR 2017 `__ + - – * - :doc:`JANET ` - `arXiv 2018 `__ - `JosvanderWesthuizen/janet `__ @@ -59,6 +62,9 @@ references and official implementations where available. * - :doc:`LightRU ` - `MDPI Electronics 2023 `__ - – + * - :doc:`MCLSTM ` + - `NIPS 2017 workshop `__ + - – * - :doc:`MGU ` - `IJAC 2016 `__ - – @@ -116,6 +122,15 @@ references and official implementations where available. * - :doc:`tauGRU ` - `AISTATS 2025 `__ - – + * - :doc:`TRNN ` + - `ICML 2016 `__ + - – + * - :doc:`TGRU ` + - `ICML 2016 `__ + - – + * - :doc:`TLSTM ` + - `ICML 2016 `__ + - – * - :doc:`UGRNN ` - `ICLR 2017 `__ - – diff --git a/pyproject.toml b/pyproject.toml index 77f0b3c..cf04671 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "torchrecurrent" -version = "0.2.4" +version = "0.2.5" description = "A package for recurrent neural networks in PyTorch" readme = "README.md" authors = [ diff --git a/tests/test_cells.py b/tests/test_cells.py index 5425f50..b6a543a 100755 --- a/tests/test_cells.py +++ b/tests/test_cells.py @@ -20,8 +20,10 @@ MGUCell, MinimalRNNCell, IndRNNCell, + IntersectionRNNCell, LiGRUCell, LightRUCell, + MCLSTMCell, MultiplicativeLSTMCell, MUT1Cell, MUT2Cell, @@ -36,6 +38,9 @@ SGRNCell, STARCell, tauGRUCell, + TRNNCell, + TGRUCell, + TLSTMCell, UGRNNCell, UnICORNNCell, WMCLSTMCell, @@ -79,6 +84,7 @@ (SGRNCell, 3, 5, False), (STARCell, 3, 5, False), (tauGRUCell, 3, 5, False), + (TRNNCell, 3, 5, False), (UGRNNCell, 3, 5, False), (UnICORNNCell, 3, 5, True), (WMCLSTMCell, 3, 5, True), @@ -241,6 +247,334 @@ def test_taugru_cell_uses_delayed_state(): assert torch.allclose(out, torch.tanh(delayed), atol=1e-4) +def test_trnn_cell_has_no_recurrent_weight(): + """T-RNN's forget gate depends only on x(t): there is no weight_hh at all.""" + cell = TRNNCell(4, 9) + + assert cell.weight_ih.shape == (18, 4) + assert cell.bias_ih.shape == (18,) + assert not hasattr(cell, "weight_hh") + assert not hasattr(cell, "bias_hh") + + +def test_trnn_cell_matches_paper_update(): + cell = TRNNCell(2, 2, bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6], [0.7, 0.8]])) + + x = torch.tensor([[0.2, -0.3]]) + h = torch.tensor([[0.4, -0.5]]) + + new_h = cell(x, h) + latent, forget_pre = (x @ cell.weight_ih.t()).chunk(2, 1) + forget_gate = torch.sigmoid(forget_pre) + expected = forget_gate * h + (1 - forget_gate) * latent + + assert torch.allclose(new_h, expected) + + +def test_tgru_cell_parameter_shapes(): + """weight_hh acts on the previous *input*, so it shares weight_ih's shape.""" + cell = TGRUCell(4, 9) + + assert cell.weight_ih.shape == (27, 4) + assert cell.weight_hh.shape == (27, 4) + assert cell.bias_ih.shape == (27,) + assert cell.bias_hh.shape == (27,) + + +def test_tgru_cell_default_state_uses_input_size_not_hidden_size(): + """The second state component is x(t-1): shaped like the input, not like h.""" + cell = TGRUCell(4, 9) + x = torch.randn(3, 4) + + h, x_prev = cell(x) + + assert h.shape == (3, 9) + assert x_prev.shape == (3, 4) + assert torch.equal(x_prev, x) + + +def test_tgru_cell_matches_paper_update(): + cell = TGRUCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + h = torch.tensor([[0.4, -0.5]]) + x_prev = torch.tensor([[-0.1, 0.6]]) + + new_h, new_x_prev = cell(x, (h, x_prev)) + gates = x @ cell.weight_ih.t() + x_prev @ cell.weight_hh.t() + latent, forget_pre, out_pre = gates.chunk(3, 1) + forget_gate = torch.sigmoid(forget_pre) + candidate = torch.tanh(out_pre) + expected_h = forget_gate * h + latent * candidate + + assert torch.allclose(new_h, expected_h) + assert torch.equal(new_x_prev, x) + + +def test_tgru_cell_accepts_per_component_none_state(): + """Matches the (None, None) sentinel convention every other double-state + cell in this library supports (see tests/test_cells.py's CELL_CASES + harness), even though TGRU isn't itself in that harness.""" + cell = TGRUCell(3, 5) + x = torch.randn(3) + + h, x_prev = cell(x, (None, None)) + assert h.shape == (5,) + assert x_prev.shape == (3,) + + h2, x_prev2 = cell(x, (h, None)) + assert h2.shape == (5,) + assert x_prev2.shape == (3,) + + x_batched = torch.randn(2, 3) + h3, x_prev3 = cell(x_batched, (None, None)) + assert h3.shape == (2, 5) + assert x_prev3.shape == (2, 3) + + h4, x_prev4 = cell(x_batched, (None, x_prev3)) + assert h4.shape == (2, 5) + assert x_prev4.shape == (2, 3) + + +def test_tlstm_cell_parameter_shapes(): + """weight_hh acts on the previous *input*, so it shares weight_ih's shape.""" + cell = TLSTMCell(4, 9) + + assert cell.weight_ih.shape == (27, 4) + assert cell.weight_hh.shape == (27, 4) + assert cell.bias_ih.shape == (27,) + assert cell.bias_hh.shape == (27,) + + +def test_tlstm_cell_output_differs_from_state(): + """h(t) is a pure output: it plays no role in any later computation.""" + cell = TLSTMCell(4, 4) + + x_unbatched = torch.randn(4) + h_u, (c_u, xprev_u) = cell(x_unbatched) + assert h_u.shape == c_u.shape == (4,) + assert xprev_u.shape == (4,) + assert not torch.allclose(h_u, c_u) + + x = torch.randn(3, 4) + h, (c, xprev) = cell(x) + assert h.shape == c.shape == (3, 4) + assert not torch.allclose(h, c) + + +def test_tlstm_cell_accepts_per_component_none_state(): + cell = TLSTMCell(3, 5) + x = torch.randn(3) + + h, (c, xprev) = cell(x, (None, None)) + assert h.shape == c.shape == (5,) + assert xprev.shape == (3,) + + h2, (c2, xprev2) = cell(x, (c, None)) + assert h2.shape == c2.shape == (5,) + assert xprev2.shape == (3,) + + x_batched = torch.randn(2, 3) + h3, (c3, xprev3) = cell(x_batched, (None, None)) + assert h3.shape == c3.shape == (2, 5) + assert xprev3.shape == (2, 3) + + h4, (c4, xprev4) = cell(x_batched, (None, xprev3)) + assert h4.shape == c4.shape == (2, 5) + assert xprev4.shape == (2, 3) + + +def test_tlstm_cell_gradients(): + cell = TLSTMCell(4, 5, bias=False) + x = torch.randn(2, 4, requires_grad=True) + + h, (c, xprev) = cell(x) + (h.sum() + c.sum()).backward() + + assert x.grad is not None + for p in cell.parameters(): + if p.requires_grad: + assert p.grad is not None + + +def test_tlstm_cell_matches_paper_update(): + cell = TLSTMCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(12, dtype=torch.float32).reshape(6, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + c = torch.tensor([[0.4, -0.5]]) + x_prev = torch.tensor([[-0.1, 0.6]]) + + h, (new_c, new_x_prev) = cell(x, (c, x_prev)) + gates = x @ cell.weight_ih.t() + x_prev @ cell.weight_hh.t() + latent, forget_pre, out_pre = gates.chunk(3, 1) + forget_gate = torch.sigmoid(forget_pre) + output_candidate = torch.tanh(out_pre) + expected_c = forget_gate * c + (1 - forget_gate) * latent + expected_h = expected_c * output_candidate + + assert torch.allclose(new_c, expected_c) + assert torch.allclose(h, expected_h) + assert torch.equal(new_x_prev, x) + + +def test_intersectionrnn_cell_requires_equal_sizes(): + with pytest.raises(ValueError, match="input_size == hidden_size"): + IntersectionRNNCell(3, 5) + + +def test_intersectionrnn_cell_shapes(): + cell = IntersectionRNNCell(4, 4) + + assert cell.weight_ih.shape == (16, 4) + assert cell.weight_hh.shape == (16, 4) + assert cell.bias_ih.shape == (16,) + assert cell.bias_hh.shape == (16,) + + +def test_intersectionrnn_cell_output_differs_from_state(): + """y(t) and h(t) are genuinely different tensors, not aliases.""" + cell = IntersectionRNNCell(4, 4) + + x_unbatched = torch.randn(4) + y_u, h_u = cell(x_unbatched) + assert y_u.shape == h_u.shape == (4,) + assert not torch.allclose(y_u, h_u) + + x = torch.randn(3, 4) + y, h = cell(x) + + assert y.shape == h.shape == (3, 4) + assert not torch.allclose(y, h) + + +def test_intersectionrnn_cell_gradients(): + cell = IntersectionRNNCell(4, 4, bias=False) + x = torch.randn(2, 4, requires_grad=True) + + y, h = cell(x) + (y.sum() + h.sum()).backward() + + assert x.grad is not None + for p in cell.parameters(): + if p.requires_grad: + assert p.grad is not None + + +def test_intersectionrnn_cell_matches_paper_update(): + cell = IntersectionRNNCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(16, dtype=torch.float32).reshape(8, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(16, dtype=torch.float32).reshape(8, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + h = torch.tensor([[0.4, -0.5]]) + + y, new_h = cell(x, h) + gates = x @ cell.weight_ih.t() + h @ cell.weight_hh.t() + depth_pre, recurrent_pre, gy_pre, gh_pre = gates.chunk(4, 1) + depth_candidate = torch.relu(depth_pre) + recurrent_candidate = torch.tanh(recurrent_pre) + gate_y = torch.sigmoid(gy_pre) + gate_h = torch.sigmoid(gh_pre) + expected_y = gate_y * x + (1 - gate_y) * depth_candidate + expected_h = gate_h * h + (1 - gate_h) * recurrent_candidate + + assert torch.allclose(y, expected_y) + assert torch.allclose(new_h, expected_h) + + +def test_mclstm_cell_shapes(): + cell = MCLSTMCell(4, 9) + + assert cell.weight_ih.shape == (45, 4) + assert cell.weight_hh.shape == (45, 9) + assert cell.bias_ih.shape == (45,) + assert cell.bias_hh.shape == (45,) + + +def test_mclstm_cell_output_differs_from_state(): + """h(t) is a pure output: it must differ from both v(t) and c(t).""" + cell = MCLSTMCell(4, 4) + + x_unbatched = torch.randn(4) + h_u, (v_u, c_u) = cell(x_unbatched) + assert h_u.shape == v_u.shape == c_u.shape == (4,) + assert not torch.allclose(h_u, v_u) + assert not torch.allclose(h_u, c_u) + + x = torch.randn(3, 4) + h, (v, c) = cell(x) + assert h.shape == v.shape == c.shape == (3, 4) + assert not torch.allclose(h, v) + + +def test_mclstm_cell_accepts_per_component_none_state(): + cell = MCLSTMCell(3, 5) + x = torch.randn(3) + + h, (v, c) = cell(x, (None, None)) + assert h.shape == v.shape == c.shape == (5,) + + h2, (v2, c2) = cell(x, (v, None)) + assert h2.shape == v2.shape == c2.shape == (5,) + + x_batched = torch.randn(2, 3) + h3, (v3, c3) = cell(x_batched, (None, None)) + assert h3.shape == v3.shape == c3.shape == (2, 5) + + h4, (v4, c4) = cell(x_batched, (None, c3)) + assert h4.shape == v4.shape == c4.shape == (2, 5) + + +def test_mclstm_cell_gradients(): + cell = MCLSTMCell(4, 5, bias=False) + x = torch.randn(2, 4, requires_grad=True) + + h, (v, c) = cell(x) + (h.sum() + v.sum() + c.sum()).backward() + + assert x.grad is not None + for p in cell.parameters(): + if p.requires_grad: + assert p.grad is not None + + +def test_mclstm_cell_matches_paper_update(): + cell = MCLSTMCell(2, 2, bias=False, recurrent_bias=False) + with torch.no_grad(): + cell.weight_ih.copy_(torch.arange(20, dtype=torch.float32).reshape(10, 2) * 0.1) + cell.weight_hh.copy_(torch.arange(20, dtype=torch.float32).reshape(10, 2) * -0.1) + + x = torch.tensor([[0.2, -0.3]]) + v = torch.tensor([[0.1, -0.2]]) + c = torch.tensor([[0.4, -0.5]]) + + h, (new_v, new_c) = cell(x, (v, c)) + gates = x @ cell.weight_ih.t() + v @ cell.weight_hh.t() + f_pre, i_pre, o_pre, m_pre, n_pre = gates.chunk(5, 1) + forget_gate = torch.sigmoid(f_pre) + input_gate = torch.sigmoid(i_pre) + output_gate = torch.sigmoid(o_pre) + control_gate = torch.sigmoid(m_pre) + candidate = torch.tanh(n_pre) + + expected_c = forget_gate * c + input_gate * candidate + expected_h = output_gate * torch.tanh(expected_c) + expected_v = control_gate * torch.tanh(expected_c) + + assert torch.allclose(new_c, expected_c) + assert torch.allclose(h, expected_h) + assert torch.allclose(new_v, expected_v) + + @pytest.mark.parametrize("Cell, in_size, hid_size, _", CELL_CASES) def test_cell_gradients(Cell, in_size, hid_size, _): """A quick smoke test: outputs should be differentiable wrt parameters.""" diff --git a/tests/test_layers.py b/tests/test_layers.py index 04645ab..162a13c 100755 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -14,8 +14,10 @@ FastGRNN, GatedAntisymmetricRNN, IndRNN, + IntersectionRNN, LiGRU, LightRU, + MCLSTM, MGU, MinimalRNN, MultiplicativeLSTM, @@ -32,6 +34,9 @@ SGRN, STAR, tauGRU, + TRNN, + TGRU, + TLSTM, UGRNN, UnICORNN, WMCLSTM, @@ -51,6 +56,7 @@ IndRNN, LiGRU, LightRU, + MCLSTM, MGU, MinimalRNN, MultiplicativeLSTM, @@ -67,6 +73,9 @@ SGRN, STAR, tauGRU, + TRNN, + TGRU, + TLSTM, UGRNN, UnICORNN, WMCLSTM, @@ -87,6 +96,7 @@ (IndRNN, False), (LiGRU, False), (LightRU, False), + (MCLSTM, True), (MGU, False), (MinimalRNN, False), (MultiplicativeLSTM, True), @@ -103,6 +113,7 @@ (SGRN, False), (STAR, False), (tauGRU, False), + (TRNN, False), (UGRNN, False), (UnICORNN, True), (WMCLSTM, True), @@ -203,6 +214,454 @@ def test_layer_runs_on_device(Layer, is_double, device): assert p.grad.device.type == device.type +def test_tgru_single_layer_supports_differing_sizes(): + """TGRU's second state (x(t-1)) is input_size-shaped, not hidden_size-shaped: + a single layer works fine even when input_size != hidden_size.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size = 4, 3 + + layer = TGRU(input_size, hidden_size, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert hn.shape == (1, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == 1 + assert xn[0].shape == (batch_size, input_size) + + +def test_tgru_stacked_layers_have_per_layer_previous_input_shapes(): + """Layer 0's previous-input state is input_size-shaped; every later layer's + is hidden_size-shaped, since it receives the previous layer's output.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert hn.shape == (num_layers, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == num_layers + assert xn[0].shape == (batch_size, input_size) + assert xn[1].shape == (batch_size, hidden_size) + + +def test_tgru_previous_input_state_holds_last_seen_input(): + """x_n[k] should be the actual last input layer k saw, not just correctly shaped.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + + assert torch.equal(xn[0], x[-1]) + + # layer 1's own input at each timestep is layer 0's output at that timestep; + # recompute layer 0's per-timestep outputs manually to check xn[1]. + h0 = torch.zeros(batch_size, hidden_size) + x0_prev = torch.zeros(batch_size, input_size) + for t in range(seq_len): + h0, x0_prev = layer.cells[0](x[t], (h0, x0_prev)) + assert torch.allclose(xn[1], h0) + + +def test_tgru_state_continuity_matches_single_call(): + """Splitting a sequence in two and carrying state across calls must match + running the whole sequence in one call.""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out_full, _ = layer(x) + + out1, state1 = layer(x[:split]) + out2, _ = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + + +def test_tgru_batch_first_matches_seq_first(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TGRU(input_size, hidden_size, num_layers=num_layers, bias=False) + layer_bf = TGRU( + input_size, hidden_size, num_layers=num_layers, bias=False, batch_first=True + ) + layer_bf.load_state_dict(layer.state_dict()) + + x = torch.randn(seq_len, batch_size, input_size) + out, (hn, xn) = layer(x) + out_bf, (hn_bf, xn_bf) = layer_bf(x.transpose(0, 1)) + + assert torch.allclose(out_bf, out.transpose(0, 1)) + assert torch.allclose(hn_bf, hn) + for a, b in zip(xn_bf, xn): + assert torch.allclose(a, b) + assert xn[1].shape == (batch_size, hidden_size) + + +def test_tlstm_single_layer_supports_differing_sizes(): + input_size, hidden_size = 5, 7 + seq_len, batch_size = 4, 3 + + layer = TLSTM(input_size, hidden_size, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert cn.shape == (1, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == 1 + assert xn[0].shape == (batch_size, input_size) + # output and cell state are genuinely different tensors + assert not torch.allclose(out[-1], cn[-1]) + + +def test_tlstm_stacked_layers_have_per_layer_previous_input_shapes(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + + assert out.shape == (seq_len, batch_size, hidden_size) + assert cn.shape == (num_layers, batch_size, hidden_size) + assert isinstance(xn, tuple) and len(xn) == num_layers + assert xn[0].shape == (batch_size, input_size) + assert xn[1].shape == (batch_size, hidden_size) + + +def test_tlstm_previous_input_state_holds_last_seen_input(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + + assert torch.equal(xn[0], x[-1]) + + # layer 1's last seen input is layer 0's last *output* h(t), not layer 0's + # own (c, x_prev) state -- output and state are different tensors here. + c0 = torch.zeros(batch_size, hidden_size) + x0_prev = torch.zeros(batch_size, input_size) + h0 = None + for t in range(seq_len): + h0, (c0, x0_prev) = layer.cells[0](x[t], (c0, x0_prev)) + assert torch.allclose(xn[1], h0) + + +def test_tlstm_stacking_forwards_output_not_state(): + """Layer k+1 must receive layer k's h(t) (output), not its c(t)/x(t-1).""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out, (cn, xn) = layer(x) + + c_prev = [torch.zeros(batch_size, hidden_size) for _ in range(num_layers)] + x_prev = [ + torch.zeros(batch_size, layer.cells[idx].input_size) for idx in range(num_layers) + ] + expected_outputs = [] + for t in range(seq_len): + layer_inp = x[t] + for layer_idx, cell in enumerate(layer.cells): + h, (c_new, x_new) = cell(layer_inp, (c_prev[layer_idx], x_prev[layer_idx])) + c_prev[layer_idx] = c_new + x_prev[layer_idx] = x_new + layer_inp = h + expected_outputs.append(layer_inp) + + assert torch.allclose(out, torch.stack(expected_outputs, dim=0), atol=1e-6) + for layer_idx in range(num_layers): + assert torch.allclose(cn[layer_idx], c_prev[layer_idx], atol=1e-6) + assert torch.allclose(xn[layer_idx], x_prev[layer_idx], atol=1e-6) + + +def test_tlstm_state_continuity_matches_single_call(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out_full, (c_full, x_full) = layer(x) + + out1, state1 = layer(x[:split]) + out2, (c2, x2) = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + assert torch.allclose(c2, c_full, atol=1e-6) + for a, b in zip(x2, x_full): + assert torch.allclose(a, b, atol=1e-6) + + +def test_tlstm_dropout_only_between_layers(): + input_size, hidden_size = 5, 5 + seq_len, batch_size = 4, 3 + + torch.manual_seed(0) + layer_no_drop = TLSTM(input_size, hidden_size, dropout=0.0, bias=False) + torch.manual_seed(0) + layer_high_drop = TLSTM(input_size, hidden_size, dropout=0.9, bias=False) + + x = torch.randn(seq_len, batch_size, input_size) + out_no_drop, _ = layer_no_drop(x) + out_high_drop, _ = layer_high_drop(x) + + assert torch.equal(out_no_drop, out_high_drop) + + torch.manual_seed(0) + stacked_no_drop = TLSTM(input_size, hidden_size, num_layers=2, dropout=0.0, bias=False) + torch.manual_seed(0) + stacked_drop = TLSTM(input_size, hidden_size, num_layers=2, dropout=0.9, bias=False) + stacked_drop.load_state_dict(stacked_no_drop.state_dict()) + + torch.manual_seed(1) + out_stacked_no_drop, _ = stacked_no_drop(x) + torch.manual_seed(1) + out_stacked_drop, _ = stacked_drop(x) + + assert not torch.allclose(out_stacked_no_drop, out_stacked_drop) + + +def test_tlstm_batch_first_matches_seq_first(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = TLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + layer_bf = TLSTM( + input_size, hidden_size, num_layers=num_layers, bias=False, batch_first=True + ) + layer_bf.load_state_dict(layer.state_dict()) + + x = torch.randn(seq_len, batch_size, input_size) + out, (cn, xn) = layer(x) + out_bf, (cn_bf, xn_bf) = layer_bf(x.transpose(0, 1)) + + assert torch.allclose(out_bf, out.transpose(0, 1)) + assert torch.allclose(cn_bf, cn) + for a, b in zip(xn_bf, xn): + assert torch.allclose(a, b) + + +def test_intersectionrnn_requires_equal_sizes(): + with pytest.raises(ValueError, match="input_size == hidden_size"): + IntersectionRNN(3, 5) + + +def test_intersectionrnn_shapes_and_state(): + size = 6 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, size) + out, hn = layer(x) + + assert out.shape == (seq_len, batch_size, size) + assert isinstance(hn, Tensor) + assert hn.shape == (num_layers, batch_size, size) + # output and recurrent state are genuinely different tensors + assert not torch.allclose(out[-1], hn[-1]) + + +def test_intersectionrnn_batch_first_matches_seq_first(): + size = 6 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + layer_bf = IntersectionRNN( + size, size, num_layers=num_layers, bias=False, batch_first=True + ) + layer_bf.load_state_dict(layer.state_dict()) + + x = torch.randn(seq_len, batch_size, size) + out, hn = layer(x) + out_bf, hn_bf = layer_bf(x.transpose(0, 1)) + + assert torch.allclose(out_bf, out.transpose(0, 1)) + assert torch.allclose(hn_bf, hn) + + +def test_intersectionrnn_state_continuity_matches_single_call(): + size = 6 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, size) + + out_full, _ = layer(x) + + out1, state1 = layer(x[:split]) + out2, _ = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + + +def test_intersectionrnn_stacking_forwards_output_not_state(): + """Layer k+1 must receive layer k's y(t) (output), not its h(t) (state).""" + size = 5 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = IntersectionRNN(size, size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, size) + + out, hn = layer(x) + + h_prev = [torch.zeros(batch_size, size) for _ in range(num_layers)] + expected_out = None + for t in range(seq_len): + layer_inp = x[t] + for layer_idx, cell in enumerate(layer.cells): + y, h_new = cell(layer_inp, h_prev[layer_idx]) + h_prev[layer_idx] = h_new + layer_inp = y + expected_out = layer_inp + + assert torch.allclose(out[-1], expected_out, atol=1e-6) + assert torch.allclose(hn[0], h_prev[0], atol=1e-6) + assert torch.allclose(hn[1], h_prev[1], atol=1e-6) + + +def test_intersectionrnn_dropout_only_between_layers(): + size = 5 + seq_len, batch_size = 4, 3 + + # num_layers=1: dropout is never applied (no "next layer" to drop before), + # so output must be bit-identical regardless of the dropout probability. + torch.manual_seed(0) + layer_no_drop = IntersectionRNN(size, size, dropout=0.0, bias=False) + torch.manual_seed(0) + layer_high_drop = IntersectionRNN(size, size, dropout=0.9, bias=False) + + x = torch.randn(seq_len, batch_size, size) + out_no_drop, _ = layer_no_drop(x) + out_high_drop, _ = layer_high_drop(x) + + assert torch.equal(out_no_drop, out_high_drop) + + # num_layers=2: dropout between layers must actually change the output. + torch.manual_seed(0) + stacked_no_drop = IntersectionRNN(size, size, num_layers=2, dropout=0.0, bias=False) + torch.manual_seed(0) + stacked_drop = IntersectionRNN(size, size, num_layers=2, dropout=0.9, bias=False) + stacked_drop.load_state_dict(stacked_no_drop.state_dict()) + + torch.manual_seed(1) + out_stacked_no_drop, _ = stacked_no_drop(x) + torch.manual_seed(1) + out_stacked_drop, _ = stacked_drop(x) + + assert not torch.allclose(out_stacked_no_drop, out_stacked_drop) + + +def test_intersectionrnn_repr(): + r = repr(IntersectionRNN(5, 5)) + assert r == "IntersectionRNN(5, 5)" + + r = repr(IntersectionRNN(5, 5, num_layers=2)) + assert "num_layers=2" in r + + r = repr(IntersectionRNN(5, 5, dropout=0.5)) + assert "dropout=0.5" in r + + r = repr(IntersectionRNN(5, 5, batch_first=True)) + assert "batch_first=True" in r + + +def test_mclstm_state_continuity_matches_single_call(): + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 6, 3, 2 + split = 2 + + layer = MCLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out_full, (v_full, c_full) = layer(x) + + out1, state1 = layer(x[:split]) + out2, (v2, c2) = layer(x[split:], state1) + out_chunked = torch.cat([out1, out2], dim=0) + + assert torch.allclose(out_chunked, out_full, atol=1e-6) + assert torch.allclose(v2, v_full, atol=1e-6) + assert torch.allclose(c2, c_full, atol=1e-6) + + +def test_mclstm_stacking_forwards_output_not_control_state(): + """Layer k+1 must receive layer k's h(t) (output), not its v(t)/c(t).""" + input_size, hidden_size = 5, 7 + seq_len, batch_size, num_layers = 4, 3, 2 + + layer = MCLSTM(input_size, hidden_size, num_layers=num_layers, bias=False) + x = torch.randn(seq_len, batch_size, input_size) + + out, (vn, cn) = layer(x) + + v_prev = [torch.zeros(batch_size, hidden_size) for _ in range(num_layers)] + c_prev = [torch.zeros(batch_size, hidden_size) for _ in range(num_layers)] + expected_outputs = [] + for t in range(seq_len): + layer_inp = x[t] + for layer_idx, cell in enumerate(layer.cells): + h, (v_new, c_new) = cell(layer_inp, (v_prev[layer_idx], c_prev[layer_idx])) + v_prev[layer_idx] = v_new + c_prev[layer_idx] = c_new + layer_inp = h + expected_outputs.append(layer_inp) + + assert torch.allclose(out, torch.stack(expected_outputs, dim=0), atol=1e-6) + for layer_idx in range(num_layers): + assert torch.allclose(vn[layer_idx], v_prev[layer_idx], atol=1e-6) + assert torch.allclose(cn[layer_idx], c_prev[layer_idx], atol=1e-6) + + +def test_mclstm_dropout_only_between_layers(): + input_size, hidden_size = 5, 5 + seq_len, batch_size = 4, 3 + + # num_layers=1: dropout is never applied, so the output must be + # bit-identical regardless of the dropout probability. + torch.manual_seed(0) + layer_no_drop = MCLSTM(input_size, hidden_size, dropout=0.0, bias=False) + torch.manual_seed(0) + layer_high_drop = MCLSTM(input_size, hidden_size, dropout=0.9, bias=False) + + x = torch.randn(seq_len, batch_size, input_size) + out_no_drop, _ = layer_no_drop(x) + out_high_drop, _ = layer_high_drop(x) + + assert torch.equal(out_no_drop, out_high_drop) + + # num_layers=2: dropout between layers must actually change the output. + torch.manual_seed(0) + stacked_no_drop = MCLSTM(input_size, hidden_size, num_layers=2, dropout=0.0, bias=False) + torch.manual_seed(0) + stacked_drop = MCLSTM(input_size, hidden_size, num_layers=2, dropout=0.9, bias=False) + stacked_drop.load_state_dict(stacked_no_drop.state_dict()) + + torch.manual_seed(1) + out_stacked_no_drop, _ = stacked_no_drop(x) + torch.manual_seed(1) + out_stacked_drop, _ = stacked_drop(x) + + assert not torch.allclose(out_stacked_no_drop, out_stacked_drop) + + @pytest.mark.parametrize("Layer", LAYER_CLASSES) def test_default_repr_shows_input_hidden(Layer): # Default repr should exactly match "Class(input_size, hidden_size)" diff --git a/torchrecurrent/__init__.py b/torchrecurrent/__init__.py index 7d2925f..d374558 100755 --- a/torchrecurrent/__init__.py +++ b/torchrecurrent/__init__.py @@ -12,10 +12,12 @@ FastGRNNCell, ATRCell, IndRNNCell, + IntersectionRNNCell, JANETCell, LEMCell, LiGRUCell, LightRUCell, + MCLSTMCell, MGUCell, MinimalRNNCell, MiRU1Cell, @@ -35,6 +37,9 @@ SGRNCell, STARCell, tauGRUCell, + TRNNCell, + TGRUCell, + TLSTMCell, UGRNNCell, UnICORNNCell, WMCLSTMCell, @@ -52,10 +57,12 @@ FastRNN, FastGRNN, IndRNN, + IntersectionRNN, JANET, LEM, LiGRU, LightRU, + MCLSTM, MGU, MinimalRNN, MiRU1, @@ -75,6 +82,9 @@ SGRN, STAR, tauGRU, + TRNN, + TGRU, + TLSTM, UGRNN, UnICORNN, WMCLSTM, @@ -95,6 +105,8 @@ "CFNCell", "DSGU", "DSGUCell", + "MCLSTM", + "MCLSTMCell", "MGU", "MGUCell", "MinimalRNN", @@ -111,6 +123,8 @@ "FastGRNNCell", "IndRNN", "IndRNNCell", + "IntersectionRNN", + "IntersectionRNNCell", "JANET", "JANETCell", "LEM", @@ -147,6 +161,12 @@ "STARCell", "tauGRU", "tauGRUCell", + "TRNN", + "TRNNCell", + "TGRU", + "TGRUCell", + "TLSTM", + "TLSTMCell", "UGRNN", "UGRNNCell", "UnICORNN", diff --git a/torchrecurrent/base.py b/torchrecurrent/base.py index 824856a..23fc59b 100755 --- a/torchrecurrent/base.py +++ b/torchrecurrent/base.py @@ -269,6 +269,38 @@ def forward( raise NotImplementedError +class DecoupledSingleStateCellBase(RecurrentCellBase): + """ + Interface for cells whose per-step output is not the tensor that recurs + in time (e.g. a model with a separate "prediction" and "control" signal): + forward(inp, state) -> (output, new_state) + Unlike SingleStateCellBase, `output` is returned separately from + `new_state` instead of being the same tensor. + """ + + def uses_double_state(self) -> bool: + return False + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: + raise NotImplementedError + + +class DecoupledDoubleStateCellBase(RecurrentCellBase): + """ + Interface for double-state cells whose per-step output is not either + state component: + forward(inp, (s1, s2)) -> (output, (new_s1, new_s2)) + """ + + def uses_double_state(self) -> bool: + return True + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + raise NotImplementedError + + # Recurrent layers class RecurrentLayerBase(nn.Module): __constants__ = ["input_size", "hidden_size", "num_layers", "batch_first", "dropout"] @@ -402,3 +434,100 @@ def forward( if self.batch_first: out = out.transpose(0, 1) return out, state + + +class DecoupledSingleStateRecurrentLayerBase(RecurrentLayerBase): + """ + Like SingleStateRecurrentLayerBase, but each cell returns (output, new_state) + with output distinct from new_state: output feeds the next layer, new_state + feeds the same layer's next timestep. + """ + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + state = torch.zeros( + self.num_layers, + batch_size, + self.hidden_size, + dtype=inp.dtype, + device=inp.device, + ) + + outputs = torch.jit.annotate(List[Tensor], []) + + for t in range(seq_len): + x = inp[t] + new_states = torch.jit.annotate(List[Tensor], []) + + for layer_idx, cell in enumerate(self.cells): + s_prev = state[layer_idx] + out, s_new = cell(x, s_prev) + new_states.append(s_new) + x = out + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + state = torch.stack(new_states, dim=0) + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + return out, state + + +class DecoupledDoubleStateRecurrentLayerBase(RecurrentLayerBase): + """ + Like DoubleStateRecurrentLayerBase, but each cell returns + (output, (new_s1, new_s2)) with output distinct from either state + component: output feeds the next layer, (new_s1, new_s2) feed the same + layer's next timestep. + """ + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + s1 = torch.zeros( + self.num_layers, + batch_size, + self.hidden_size, + dtype=inp.dtype, + device=inp.device, + ) + s2 = torch.zeros_like(s1) + state = (s1, s2) + + outputs = torch.jit.annotate(List[Tensor], []) + + for t in range(seq_len): + x = inp[t] + new_s1 = torch.jit.annotate(List[Tensor], []) + new_s2 = torch.jit.annotate(List[Tensor], []) + s1_prev, s2_prev = state + + for layer_idx, cell in enumerate(self.cells): + out, (s1_i, s2_i) = cell(x, (s1_prev[layer_idx], s2_prev[layer_idx])) + new_s1.append(s1_i) + new_s2.append(s2_i) + x = out + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + state = (torch.stack(new_s1, dim=0), torch.stack(new_s2, dim=0)) + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + return out, state diff --git a/torchrecurrent/cells/__init__.py b/torchrecurrent/cells/__init__.py index ad1a6bb..645561d 100755 --- a/torchrecurrent/cells/__init__.py +++ b/torchrecurrent/cells/__init__.py @@ -9,12 +9,14 @@ from .atr_cell import ATRCell, ATR from .br_cell import BR, BRCell, NBR, NBRCell from .cfn_cell import CFN, CFNCell +from .mclstm_cell import MCLSTM, MCLSTMCell from .mgu_cell import MGU, MGUCell from .minimalrnn_cell import MinimalRNN, MinimalRNNCell from .miru_cell import MiRU1, MiRU1Cell, MiRU2, MiRU2Cell from .cornn_cell import coRNN, coRNNCell from .fastrnn_cell import FastRNN, FastRNNCell, FastGRNN, FastGRNNCell from .indrnn_cell import IndRNN, IndRNNCell +from .intersectionrnn_cell import IntersectionRNN, IntersectionRNNCell from .janet_cell import JANET, JANETCell from .lem_cell import LEM, LEMCell from .ligru_cell import LiGRU, LiGRUCell @@ -31,6 +33,7 @@ from .sgrn_cell import SGRN, SGRNCell from .star_cell import STAR, STARCell from .taugru_cell import tauGRU, tauGRUCell +from .trnn_cell import TRNN, TRNNCell, TGRU, TGRUCell, TLSTM, TLSTMCell from .ugrnn_cell import UGRNN, UGRNNCell from .unicornn_cell import UnICORNN, UnICORNNCell from .wmclstm_cell import WMCLSTM, WMCLSTMCell @@ -51,6 +54,8 @@ "NBRCell", "CFN", "CFNCell", + "MCLSTM", + "MCLSTMCell", "MGU", "MGUCell", "MinimalRNN", @@ -67,6 +72,8 @@ "FastGRNNCell", "IndRNN", "IndRNNCell", + "IntersectionRNN", + "IntersectionRNNCell", "JANET", "JANETCell", "LEM", @@ -105,6 +112,12 @@ "STARCell", "tauGRU", "tauGRUCell", + "TRNN", + "TRNNCell", + "TGRU", + "TGRUCell", + "TLSTM", + "TLSTMCell", "UGRNN", "UGRNNCell", "UnICORNN", diff --git a/torchrecurrent/cells/intersectionrnn_cell.py b/torchrecurrent/cells/intersectionrnn_cell.py new file mode 100644 index 0000000..581e7e0 --- /dev/null +++ b/torchrecurrent/cells/intersectionrnn_cell.py @@ -0,0 +1,336 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional, Tuple +from ..base import ( + DecoupledSingleStateRecurrentLayerBase, + DecoupledSingleStateCellBase, + resolve_activation, + resolve_init_name, +) + + +class IntersectionRNN(DecoupledSingleStateRecurrentLayerBase): + r"""Multi-layer Intersection RNN (+RNN). + + [`arXiv `_] + + Each layer consists of an :class:`IntersectionRNNCell`, which couples a + "depth" gate (between layers) and a "recurrent" gate (through time): + + .. math:: + \begin{aligned} + y^{in}_t &= \phi(W_{ih}^{y} x_t + b_{ih}^{y} + W_{hh}^{y} h_{t-1} + b_{hh}^{y}), \\ + h^{in}_t &= \tau(W_{ih}^{h} x_t + b_{ih}^{h} + W_{hh}^{h} h_{t-1} + b_{hh}^{h}), \\ + g^{y}_t &= \sigma(W_{ih}^{g^y} x_t + b_{ih}^{g^y} + W_{hh}^{g^y} h_{t-1} + b_{hh}^{g^y}), \\ + g^{h}_t &= \sigma(W_{ih}^{g^h} x_t + b_{ih}^{g^h} + W_{hh}^{g^h} h_{t-1} + b_{hh}^{g^h}), \\ + y_t &= g^{y}_t \circ x_t + (1 - g^{y}_t) \circ y^{in}_t, \\ + h_t &= g^{h}_t \circ h_{t-1} + (1 - g^{h}_t) \circ h^{in}_t + \end{aligned} + + where :math:`\sigma` is the sigmoid function, :math:`\phi` is ReLU, + :math:`\tau` is a pointwise nonlinearity (e.g., tanh), and :math:`\circ` + denotes elementwise (Hadamard) product. Unlike every other layer in this + package, the per-timestep output :math:`y_t` fed to the next layer is + *not* the same tensor as the recurrent state :math:`h_t` carried to the + next timestep: :math:`y_t` is a "depth" gate mixing the raw input + :math:`x_t` with a candidate, while :math:`h_t` is an ordinary recurrent + gate mix. This requires ``input_size == hidden_size``, since :math:`y_t` + mixes :math:`x_t` and :math:`y^{in}_t` elementwise. + + The original paper studies this architecture for ``num_layers >= 2`` and + explicitly excludes single-layer (``num_layers=1``) configurations from + its experiments, since the depth gate has no second layer to feed. The + per-timestep computation is still well-defined for ``num_layers=1`` + (:math:`y_t` is simply returned as the layer's output), but this + configuration was not studied by the authors. + + In a multilayer IntersectionRNN, the input :math:`x^{(l)}_t` of the + :math:`l`-th layer (:math:`l \ge 2`) is the *output* :math:`y^{(l-1)}_t` + of the previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, + where each :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which + is 0 with probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. Must + equal `hidden_size`. + hidden_size: The number of features in the hidden state `h`. Must + equal `input_size`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two IntersectionRNN layers, with the second receiving the + outputs of the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + depth_nonlinearity: Nonlinearity :math:`\phi` for the depth candidate + :math:`y^{in}`. Default: :func:`torch.relu` + recurrent_nonlinearity: Nonlinearity :math:`\tau` for the recurrent + candidate :math:`h^{in}`. Default: :func:`torch.tanh` + gate_nonlinearity: Activation :math:`\sigma` for both gates. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape :math:`(L, N, H)` when ``batch_first=False`` + or :math:`(N, L, H)` when ``batch_first=True`` containing the features of + the input sequence, where :math:`H` = `input_size` = `hidden_size`. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H)` containing the + initial hidden state for each element in the input sequence. Defaults to + zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H ={} & \text{input\_size} = \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape :math:`(L, N, H)` when ``batch_first=False`` + or :math:`(N, L, H)` when ``batch_first=True`` containing the output + features `(y_t)` from the last layer of the IntersectionRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H)` containing the + final recurrent state for each element in the sequence. Note this is + `h_n`, not `y_n`: the recurrent state and the returned output are + different tensors for every layer. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(4*hidden_size, input_size)`. + cells.{k}.weight_hh : the learnable hidden-hidden weights of the :math:`k`-th + layer, of shape `(4*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(4*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable hidden-hidden biases of the :math:`k`-th + layer, of shape `(4*hidden_size)`. Only present when ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`IntersectionRNNCell` + + Examples:: + + >>> rnn = IntersectionRNN(10, 10, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 10) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(IntersectionRNN, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(IntersectionRNNCell, **kwargs) + + +class IntersectionRNNCell(DecoupledSingleStateCellBase): + r"""An Intersection RNN (+RNN) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{y}^{in}(t) &= \phi\bigl( + \mathbf{W}_{ih}^{y}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{y} + + \mathbf{W}_{hh}^{y}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{y} + \bigr), \\[4pt] + \mathbf{h}^{in}(t) &= \tau\bigl( + \mathbf{W}_{ih}^{h}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{h} + + \mathbf{W}_{hh}^{h}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{h} + \bigr), \\[4pt] + \mathbf{g}^{y}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{g^y}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{g^y} + + \mathbf{W}_{hh}^{g^y}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{g^y} + \bigr), \\[4pt] + \mathbf{g}^{h}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{g^h}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{g^h} + + \mathbf{W}_{hh}^{g^h}\,\mathbf{h}(t-1) + \mathbf{b}_{hh}^{g^h} + \bigr), \\[4pt] + \mathbf{y}(t) &= \mathbf{g}^{y}(t)\circ\mathbf{x}(t) + + \bigl(1 - \mathbf{g}^{y}(t)\bigr)\circ\mathbf{y}^{in}(t), \\[4pt] + \mathbf{h}(t) &= \mathbf{g}^{h}(t)\circ\mathbf{h}(t-1) + + \bigl(1 - \mathbf{g}^{h}(t)\bigr)\circ\mathbf{h}^{in}(t), + + where :math:`\circ` is element-wise product, :math:`\phi` is ReLU, and + :math:`\tau` is a pointwise nonlinearity (e.g., tanh). The returned + output :math:`\mathbf{y}(t)` and the recurrent state + :math:`\mathbf{h}(t)` are different tensors: :math:`\mathbf{y}(t)` gates + a mix of the raw input and a candidate, while :math:`\mathbf{h}(t)` + gates a mix of the previous state and a (different) candidate. Requires + ``input_size == hidden_size``, since :math:`\mathbf{y}(t)` mixes + :math:`\mathbf{x}(t)` and :math:`\mathbf{y}^{in}(t)` elementwise. + + Args: + input_size: The number of expected features in the input ``x``. Must + equal ``hidden_size``. + hidden_size: The number of features in the hidden state ``h``. Must + equal ``input_size``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use recurrent biases. + Default: ``True``. + depth_nonlinearity: Nonlinearity :math:`\phi` for the depth candidate. + Default: :func:`torch.relu`. + recurrent_nonlinearity: Nonlinearity :math:`\tau` for the recurrent + candidate. Default: :func:`torch.tanh`. + gate_nonlinearity: Activation :math:`\sigma` for both gates. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: y_1, h_1 + - **y_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the depth output, to be passed to the next layer. + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next recurrent state. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(4*hidden_size, input_size)`` (``y, h, g^y, g^h`` parts). + weight_hh: The learnable hidden-hidden weights, + of shape ``(4*hidden_size, hidden_size)`` (``y, h, g^y, g^h`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(4*hidden_size)`` if ``bias=True``. + bias_hh: The learnable hidden-hidden biases, + of shape ``(4*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = IntersectionRNNCell(10, 10) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 10) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... y, h = cell(x[t], h) + ... out.append(y) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + depth_nonlinearity="relu", + recurrent_nonlinearity="tanh", + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + if input_size != hidden_size: + raise ValueError( + "IntersectionRNNCell requires input_size == hidden_size (got " + f"input_size={input_size}, hidden_size={hidden_size}), since the " + "output is a gated mix of x(t) and a candidate of dimension " + "hidden_size." + ) + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.depth_act = resolve_activation(depth_nonlinearity) + self.recurrent_act = resolve_activation(recurrent_nonlinearity) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._default_register_tensors(ih_mult=4, hh_mult=4) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + gates = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + b_state @ self.weight_hh.t() + + self.bias_hh + ) + depth_pre, recurrent_pre, gate_y_pre, gate_h_pre = gates.chunk(4, 1) + + depth_candidate = self.depth_act(depth_pre) + recurrent_candidate = self.recurrent_act(recurrent_pre) + gate_y = self.gate_act(gate_y_pre) + gate_h = self.gate_act(gate_h_pre) + + new_output = gate_y * b_inp + (1.0 - gate_y) * depth_candidate + new_state = gate_h * b_state + (1.0 - gate_h) * recurrent_candidate + + if not is_batched: + new_output = new_output.squeeze(0) + new_state = new_state.squeeze(0) + + return new_output, new_state diff --git a/torchrecurrent/cells/mclstm_cell.py b/torchrecurrent/cells/mclstm_cell.py new file mode 100644 index 0000000..e6fc94c --- /dev/null +++ b/torchrecurrent/cells/mclstm_cell.py @@ -0,0 +1,347 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional, Tuple +from ..base import ( + DecoupledDoubleStateRecurrentLayerBase, + DecoupledDoubleStateCellBase, + resolve_activation, + resolve_init_name, +) + + +class MCLSTM(DecoupledDoubleStateRecurrentLayerBase): + r"""Multi-layer memory-controller LSTM (MC-LSTM). + + [`NIPS 2017 workshop `_] + + Each layer consists of an :class:`MCLSTMCell`, which decouples the LSTM's + "prediction" output from the recurrent signal that drives its own gates: + + .. math:: + \begin{aligned} + f_t &= \sigma(W_{ih}^f x_t + b_{ih}^f + W_{hh}^f v_{t-1} + b_{hh}^f), \\ + i_t &= \sigma(W_{ih}^i x_t + b_{ih}^i + W_{hh}^i v_{t-1} + b_{hh}^i), \\ + o_t &= \sigma(W_{ih}^o x_t + b_{ih}^o + W_{hh}^o v_{t-1} + b_{hh}^o), \\ + m_t &= \sigma(W_{ih}^m x_t + b_{ih}^m + W_{hh}^m v_{t-1} + b_{hh}^m), \\ + n_t &= \tanh(W_{ih}^n x_t + b_{ih}^n + W_{hh}^n v_{t-1} + b_{hh}^n), \\ + c_t &= f_t \circ c_{t-1} + i_t \circ n_t, \\ + h_t &= o_t \circ \tanh(c_t), \\ + v_t &= m_t \circ \tanh(c_t) + \end{aligned} + + where :math:`\sigma` is the sigmoid function and :math:`\circ` denotes + elementwise (Hadamard) product. Every gate is driven by the *control + vector* :math:`v_{t-1}`, never by :math:`h_{t-1}`: this is the paper's + central idea, decoupling the "prediction" output :math:`h_t` from the + memory-management role a standard LSTM's hidden state also has to play. + :math:`h_t` is returned as the layer's output but does not itself recur; + the recurring state is :math:`(v_t, c_t)`. + + In a multilayer MCLSTM, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the *output* :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden and control states. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two MCLSTM layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the four gates. Default: + :func:`torch.sigmoid` + cell_nonlinearity: Activation for the candidate content `n_t`. Default: + :func:`torch.tanh` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (v_0, c_0) + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **v_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial control state. Defaults to zeros if not + provided. + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not + provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, (v_n, c_n) + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the MCLSTM, for each `t`. + - **v_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final control state for each element in the + sequence. Note this is `v_n`, not `h_n`: the control state and the + returned output are different tensors for every layer. + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(5*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(5*hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable weights applied to the control state of + the :math:`k`-th layer, of shape `(5*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(5*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable biases paired with `weight_hh` of the + :math:`k`-th layer, of shape `(5*hidden_size)`. Only present when + ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. seealso:: + :class:`MCLSTMCell` + + Examples:: + + >>> rnn = MCLSTM(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> v0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> c0 = torch.zeros(2, 3, 20) + >>> output, (vn, cn) = rnn(input, (v0, c0)) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(MCLSTM, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(MCLSTMCell, **kwargs) + + +class MCLSTMCell(DecoupledDoubleStateCellBase): + r"""A memory-controller LSTM (MC-LSTM) cell. + + [`NIPS 2017 workshop `_] + + .. math:: + + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{f}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{f} + + \mathbf{W}_{hh}^{f}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{f} + \bigr), \\[4pt] + \mathbf{i}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{i}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{i} + + \mathbf{W}_{hh}^{i}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{i} + \bigr), \\[4pt] + \mathbf{o}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{o}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{o} + + \mathbf{W}_{hh}^{o}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{o} + \bigr), \\[4pt] + \mathbf{m}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{m}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{m} + + \mathbf{W}_{hh}^{m}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{m} + \bigr), \\[4pt] + \mathbf{n}(t) &= \tanh\bigl( + \mathbf{W}_{ih}^{n}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{n} + + \mathbf{W}_{hh}^{n}\,\mathbf{v}(t-1) + \mathbf{b}_{hh}^{n} + \bigr), \\[4pt] + \mathbf{c}(t) &= \mathbf{f}(t)\circ\mathbf{c}(t-1) + + \mathbf{i}(t)\circ\mathbf{n}(t), \\[4pt] + \mathbf{h}(t) &= \mathbf{o}(t)\circ\tanh\bigl(\mathbf{c}(t)\bigr), \\[4pt] + \mathbf{v}(t) &= \mathbf{m}(t)\circ\tanh\bigl(\mathbf{c}(t)\bigr), + + where :math:`\sigma` is the sigmoid function and :math:`\circ` is + element-wise product. Every gate depends on the *control vector* + :math:`\mathbf{v}(t-1)`, never on :math:`\mathbf{h}(t-1)`. The returned + output :math:`\mathbf{h}(t)` and the recurring state + :math:`(\mathbf{v}(t), \mathbf{c}(t))` are different tensors: + :math:`\mathbf{h}(t)` is free to specialize as a pure prediction, since + it plays no role in the next step's gating. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden and control states. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use the biases + paired with ``W_{hh}``. Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the four gates. + Default: :func:`torch.sigmoid`. + cell_nonlinearity: Activation for the candidate content ``n``. + Default: :func:`torch.tanh`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (v_0, c_0) + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **v_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial control state. + - **c_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial cell state. + + If not provided, both default to zero. + + Outputs: h_1, (v_1, c_1) + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the output (prediction). + - **v_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next control state. + - **c_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next cell state. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(5*hidden_size, input_size)`` (``f, i, o, m, n`` parts). + weight_hh: The learnable weights applied to the control state, + of shape ``(5*hidden_size, hidden_size)`` (``f, i, o, m, n`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(5*hidden_size)`` if ``bias=True``. + bias_hh: The learnable biases paired with ``weight_hh``, + of shape ``(5*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = MCLSTMCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> v = torch.zeros(3, 20) # (batch, hidden_size) + >>> c = torch.zeros(3, 20) + >>> out = [] + >>> for t in range(x.size(0)): + ... h, (v, c) = cell(x[t], (v, c)) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + gate_nonlinearity="sigmoid", + cell_nonlinearity="tanh", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.cell_act = resolve_activation(cell_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._default_register_tensors(ih_mult=5, hh_mult=5) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_v = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + v0, c0 = state + b_v = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if v0 is None + else (v0.unsqueeze(0) if (not is_batched and v0.dim() == 1) else v0) + ) + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c0 is None + else (c0.unsqueeze(0) if (not is_batched and c0.dim() == 1) else c0) + ) + + gates = ( + b_inp @ self.weight_ih.t() + + self.bias_ih + + b_v @ self.weight_hh.t() + + self.bias_hh + ) + forget_pre, input_pre, output_pre, control_pre, cell_pre = gates.chunk(5, 1) + + forget_gate = self.gate_act(forget_pre) + input_gate = self.gate_act(input_pre) + output_gate = self.gate_act(output_pre) + control_gate = self.gate_act(control_pre) + candidate = self.cell_act(cell_pre) + + new_c = forget_gate * b_c + input_gate * candidate + squashed_c = torch.tanh(new_c) + new_h = output_gate * squashed_c + new_v = control_gate * squashed_c + + if not is_batched: + new_h = new_h.squeeze(0) + new_v = new_v.squeeze(0) + new_c = new_c.squeeze(0) + + return new_h, (new_v, new_c) diff --git a/torchrecurrent/cells/trnn_cell.py b/torchrecurrent/cells/trnn_cell.py new file mode 100644 index 0000000..160c407 --- /dev/null +++ b/torchrecurrent/cells/trnn_cell.py @@ -0,0 +1,1025 @@ +import torch +import torch.nn as nn +from torch import Tensor +from typing import Optional, Tuple +from ..base import ( + SingleStateRecurrentLayerBase, + SingleStateCellBase, + DoubleStateRecurrentLayerBase, + DoubleStateCellBase, + DecoupledDoubleStateRecurrentLayerBase, + DecoupledDoubleStateCellBase, + resolve_activation, + resolve_init_name, +) + + +class TRNN(SingleStateRecurrentLayerBase): + r"""Multi-layer strongly typed recurrent neural network (T-RNN). + + [`arXiv `_] + + Each layer consists of a :class:`TRNNCell`, which updates the hidden state + according to: + + .. math:: + \begin{aligned} + z_t &= W_{ih}^z x_t + b_{ih}^z, \\ + f_t &= \sigma(W_{ih}^f x_t + b_{ih}^f), \\ + h_t &= f_t \circ h_{t-1} + (1 - f_t) \circ z_t + \end{aligned} + + where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the + input at time `t`, :math:`\sigma` is the sigmoid function, and + :math:`\circ` denotes elementwise (Hadamard) product. Unlike most + recurrent cells, T-RNN has no hidden-to-hidden weight matrix: the type + constraint of "strongly typed" architectures forbids a learned linear + map acting on :math:`h_{t-1}`, so it only ever enters the update through + the elementwise gate above. + + In a multilayer TRNN, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two TRNN layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. Default: + :func:`torch.sigmoid` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state for each element in the input + sequence. Defaults to zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, h_n + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the TRNN, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(2*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(2*hidden_size, hidden_size)`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(2*hidden_size)`. Only present when ``bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `bias_init`). + + .. seealso:: + :class:`TRNNCell` + + Examples:: + + >>> rnn = TRNN(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> h0 = torch.zeros(2, 3, 20) # (num_layers, batch, hidden_size) + >>> output, hn = rnn(input, h0) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(TRNN, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(TRNNCell, **kwargs) + + +class TRNNCell(SingleStateCellBase): + r"""A strongly typed recurrent neural network (T-RNN) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{z}(t) &= \mathbf{W}_{ih}^{z}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{z}, \\[4pt] + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{f}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{f} + \bigr), \\[4pt] + \mathbf{h}(t) &= \mathbf{f}(t)\circ\mathbf{h}(t-1) + + \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{z}(t), + + where :math:`\circ` is element-wise product. Following the "strongly + typed" design, there is no weight matrix acting on :math:`\mathbf{h}(t-1)`: + it enters the update only through the elementwise mix with the gate + :math:`\mathbf{f}(t)`, which itself depends only on the current input. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use biases. + Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. + Default: :func:`torch.sigmoid`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, h_0 + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + + If **h_0** is not provided, it defaults to zero. + + Outputs: h_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(2*hidden_size, input_size)`` (``z, f`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(2*hidden_size)`` if ``bias=True``. + + Examples:: + + >>> cell = TRNNCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h = cell(x[t], h) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias"] + + weight_ih: Tensor + bias_ih: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + gate_nonlinearity="sigmoid", + kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=False, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + + self._register_tensors( + { + "weight_ih": ((2 * hidden_size, input_size), True), + "bias_ih": ((2 * hidden_size,), self.bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward(self, inp: Tensor, state: Optional[Tensor] = None) -> Tensor: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + else: + b_state = state.unsqueeze(0) if (not is_batched and state.dim() == 1) else state + + latent, forget_pre = (b_inp @ self.weight_ih.t() + self.bias_ih).chunk(2, 1) + forget_gate = self.gate_act(forget_pre) + new_state = forget_gate * b_state + (1.0 - forget_gate) * latent + + if not is_batched: + new_state = new_state.squeeze(0) + + return new_state + + +class TGRU(DoubleStateRecurrentLayerBase): + r"""Multi-layer strongly typed gated recurrent unit (T-GRU). + + [`arXiv `_] + + Each layer consists of a :class:`TGRUCell`, which updates the hidden + state according to: + + .. math:: + \begin{aligned} + z_t &= W_{ih}^z x_t + b_{ih}^z + W_{hh}^z x_{t-1} + b_{hh}^z, \\ + f_t &= \sigma(W_{ih}^f x_t + b_{ih}^f + W_{hh}^f x_{t-1} + b_{hh}^f), \\ + o_t &= \tau(W_{ih}^o x_t + b_{ih}^o + W_{hh}^o x_{t-1} + b_{hh}^o), \\ + h_t &= f_t \circ h_{t-1} + z_t \circ o_t + \end{aligned} + + where :math:`\sigma` is the sigmoid function, :math:`\tau` is a pointwise + nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + (Hadamard) product. As in T-RNN, no weight matrix acts on the hidden + state: the recurrent weights :math:`W_{hh}^{*}` act on the *previous + input* :math:`x_{t-1}` rather than on :math:`h_{t-1}`, keeping the type + of every linear map consistent with the "strongly typed" design. + + In a multilayer TGRU, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden state `h`. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two TGRU layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. Default: + :func:`torch.sigmoid` + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh` + kernel_init: Initializer for `W_{ih}`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `W_{hh}`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (h_0, x_{-1}) + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **h_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial hidden state. Defaults to zeros if not + provided. + - **x_{-1}**: tuple of :attr:`num_layers` tensors containing the + "previous input" fed to each layer before the sequence starts. The + :math:`k`-th tensor has shape :math:`(N, \text{input\_size})` for + `k = 0` and :math:`(N, H_{out})` for `k \ge 1`, since layers beyond + the first receive the previous layer's hidden state as their + input. A single stacked tensor of shape + :math:`(\text{num_layers}, N, H_{in})` is also accepted whenever + `input_size == hidden_size`. Defaults to zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, (h_n, x_n) + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the TGRU, for each `t`. + - **h_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final hidden state for each element in the sequence. + - **x_n**: tuple of :attr:`num_layers` tensors, with the same + per-layer shapes as **x_{-1}**, containing the last input seen by + each layer. Returned as a tuple (not stacked into one tensor) + because layer 0's input dimension may differ from every other + layer's. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(3*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(3*hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable weights applied to the previous input of + the :math:`k`-th layer, of the same shape as `weight_ih`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(3*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable biases paired with `weight_hh` of the + :math:`k`-th layer, of shape `(3*hidden_size)`. Only present when + ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. note:: + Unlike every other multi-layer recurrent layer in this package, + TGRU does not stack its per-layer state into a single tensor for the + "previous input" component, since layer 0's input dimension can + differ from every subsequent layer's hidden dimension. This layer + therefore overrides the generic double-state forward loop. + + .. seealso:: + :class:`TGRUCell` + + Examples:: + + >>> rnn = TGRU(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> output, (hn, xn) = rnn(input) # default zero state + >>> xn[0].shape # layer 0: input_size-shaped + torch.Size([3, 10]) + >>> xn[1].shape # layer 1: hidden_size-shaped + torch.Size([3, 20]) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(TGRU, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(TGRUCell, **kwargs) + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tuple[Tensor, ...]]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tuple[Tensor, ...]]]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + h = [ + torch.zeros( + batch_size, self.hidden_size, dtype=inp.dtype, device=inp.device + ) + for _ in range(self.num_layers) + ] + x_prev = [ + torch.zeros(batch_size, cell.input_size, dtype=inp.dtype, device=inp.device) + for cell in self.cells + ] + else: + h0, x_prev0 = state + h = [h0[layer_idx] for layer_idx in range(self.num_layers)] + x_prev = list(x_prev0) + + outputs = [] + for t in range(seq_len): + x = inp[t] + new_h = [] + new_x_prev = [] + + for layer_idx, cell in enumerate(self.cells): + h_i, x_prev_i = cell(x, (h[layer_idx], x_prev[layer_idx])) + new_h.append(h_i) + new_x_prev.append(x_prev_i) + x = h_i + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + h = new_h + x_prev = new_x_prev + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + + h_n = torch.stack(h, dim=0) + x_n = tuple(x_prev) + return out, (h_n, x_n) + + +class TGRUCell(DoubleStateCellBase): + r"""A strongly typed gated recurrent unit (T-GRU) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{z}(t) &= \mathbf{W}_{ih}^{z}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{z} + + \mathbf{W}_{hh}^{z}\,\mathbf{x}(t-1) + \mathbf{b}_{hh}^{z}, \\[4pt] + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{W}_{ih}^{f}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{f} + + \mathbf{W}_{hh}^{f}\,\mathbf{x}(t-1) + \mathbf{b}_{hh}^{f} + \bigr), \\[4pt] + \mathbf{o}(t) &= \tau\bigl( + \mathbf{W}_{ih}^{o}\,\mathbf{x}(t) + \mathbf{b}_{ih}^{o} + + \mathbf{W}_{hh}^{o}\,\mathbf{x}(t-1) + \mathbf{b}_{hh}^{o} + \bigr), \\[4pt] + \mathbf{h}(t) &= \mathbf{f}(t)\circ\mathbf{h}(t-1) + + \mathbf{z}(t)\circ\mathbf{o}(t), + + where :math:`\circ` is element-wise product and :math:`\tau` is a + pointwise nonlinearity (e.g., tanh). The weights :math:`\mathbf{W}_{hh}^{*}` + act on the previous *input* :math:`\mathbf{x}(t-1)`, not on + :math:`\mathbf{h}(t-1)`, per the "strongly typed" design. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden state ``h``. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use the biases + paired with ``W_{hh}``. Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. + Default: :func:`torch.sigmoid`. + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh`. + kernel_init: Initializer for ``W_{ih}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``W_{hh}``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (h_0, x_{-1}) + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **h_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial hidden state. + - **x_{-1}** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing the "previous input". + + If not provided, both default to zero. + + Outputs: h_1, x_1 + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next hidden state. + - **x_1**: the current input ``x``, carried forward unchanged to + serve as :math:`\mathbf{x}(t-1)` on the following call. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + weight_hh: The learnable weights applied to the previous input, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(3*hidden_size)`` if ``bias=True``. + bias_hh: The learnable biases paired with ``weight_hh``, + of shape ``(3*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = TGRUCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> h = torch.zeros(3, 20) # (batch, hidden_size) + >>> x_prev = torch.zeros(3, 10) # (batch, input_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h, x_prev = cell(x[t], (h, x_prev)) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + gate_nonlinearity="sigmoid", + output_nonlinearity="tanh", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.out_act = resolve_activation(output_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((3 * hidden_size, input_size), True), + "weight_hh": ((3 * hidden_size, input_size), True), + "bias_ih": ((3 * hidden_size,), self.bias), + "bias_hh": ((3 * hidden_size,), self.recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tensor]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_state = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + b_prev_inp = torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + else: + h0, prev_inp0 = state + b_state = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if h0 is None + else (h0.unsqueeze(0) if (not is_batched and h0.dim() == 1) else h0) + ) + b_prev_inp = ( + torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + if prev_inp0 is None + else ( + prev_inp0.unsqueeze(0) + if (not is_batched and prev_inp0.dim() == 1) + else prev_inp0 + ) + ) + + gates_ih = b_inp @ self.weight_ih.t() + self.bias_ih + gates_hh = b_prev_inp @ self.weight_hh.t() + self.bias_hh + latent, forget_pre, out_pre = (gates_ih + gates_hh).chunk(3, 1) + + forget_gate = self.gate_act(forget_pre) + candidate = self.out_act(out_pre) + new_state = forget_gate * b_state + latent * candidate + new_prev_inp = b_inp + + if not is_batched: + new_state = new_state.squeeze(0) + new_prev_inp = new_prev_inp.squeeze(0) + + return new_state, new_prev_inp + + +class TLSTM(DecoupledDoubleStateRecurrentLayerBase): + r"""Multi-layer strongly typed long short-term memory (T-LSTM). + + [`arXiv `_] + + Each layer consists of a :class:`TLSTMCell`, which updates the cell + state according to: + + .. math:: + \begin{aligned} + z_t &= V^z x_{t-1} + W^z x_t + b^z, \\ + f_t &= \sigma(V^f x_{t-1} + W^f x_t + b^f), \\ + o_t &= \tau(V^o x_{t-1} + W^o x_t + b^o), \\ + c_t &= f_t \circ c_{t-1} + (1 - f_t) \circ z_t, \\ + h_t &= c_t \circ o_t + \end{aligned} + + where :math:`\sigma` is the sigmoid function, :math:`\tau` is a pointwise + nonlinearity (e.g., tanh), and :math:`\circ` denotes elementwise + (Hadamard) product. As in T-GRU, the recurrent weights act on the + *previous input* :math:`x_{t-1}` rather than on a hidden state. As with + :class:`MCLSTM` and :class:`IntersectionRNN`, the returned output + :math:`h_t` is *not* part of the recurring state at all: only + :math:`(c_t, x_t)` carry forward to the next timestep (:math:`x_t` + becoming :math:`x_{t-1}` there). :math:`h_t` is a pure function of + quantities that are otherwise discarded after being used, so it is free + to specialize as an output without affecting the recurrence. + + In a multilayer TLSTM, the input :math:`x^{(l)}_t` of the :math:`l`-th + layer (:math:`l \ge 2`) is the *output* :math:`h^{(l-1)}_t` of the + previous layer multiplied by dropout :math:`\delta^{(l-1)}_t`, where each + :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is 0 with + probability :attr:`dropout`. + + Args: + input_size: The number of expected features in the input `x`. + hidden_size: The number of features in the hidden and cell states. + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would + mean stacking two TLSTM layers, with the second receiving the outputs of + the first. Default: 1 + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + batch_first: If ``True``, then the input and output tensors are provided as + `(batch, seq, feature)` instead of `(seq, batch, feature)`. Default: False + bias: If ``False``, then the layer does not use input-side biases. + Default: True + recurrent_bias: If ``False``, then the layer does not use recurrent biases. + Default: True + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. Default: + :func:`torch.sigmoid` + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh` + kernel_init: Initializer for `W`. Default: + :func:`torch.nn.init.xavier_uniform_` + recurrent_kernel_init: Initializer for `V`. Default: + :func:`torch.nn.init.xavier_uniform_` + bias_init: Initializer for `b_{ih}`. Default: + :func:`torch.nn.init.zeros_` + recurrent_bias_init: Initializer for `b_{hh}`. Default: + :func:`torch.nn.init.zeros_` + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (c_0, x_{-1}) + - **input**: tensor of shape :math:`(L, N, H_{in})` when + ``batch_first=False`` or :math:`(N, L, H_{in})` when + ``batch_first=True`` containing the features of the input sequence. + - **c_0**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the initial cell state. Defaults to zeros if not + provided. + - **x_{-1}**: tuple of :attr:`num_layers` tensors containing the + "previous input" fed to each layer before the sequence starts. The + :math:`k`-th tensor has shape :math:`(N, \text{input\_size})` for + `k = 0` and :math:`(N, H_{out})` for `k \ge 1`, since layers beyond + the first receive the previous layer's output as their input. A + single stacked tensor of shape :math:`(\text{num_layers}, N, + H_{in})` is also accepted whenever `input_size == hidden_size`. + Defaults to zeros if not provided. + + where: + + .. math:: + \begin{aligned} + N ={} & \text{batch size} \\ + L ={} & \text{sequence length} \\ + H_{in} ={} & \text{input\_size} \\ + H_{out} ={} & \text{hidden\_size} + \end{aligned} + + Outputs: output, (c_n, x_n) + - **output**: tensor of shape :math:`(L, N, H_{out})` when + ``batch_first=False`` or :math:`(N, L, H_{out})` when + ``batch_first=True`` containing the output features `(h_t)` from + the last layer of the TLSTM, for each `t`. + - **c_n**: tensor of shape :math:`(\text{num_layers}, N, H_{out})` + containing the final cell state for each element in the sequence. + - **x_n**: tuple of :attr:`num_layers` tensors, with the same + per-layer shapes as **x_{-1}**, containing the last input seen by + each layer. Returned as a tuple (not stacked into one tensor) + because layer 0's input dimension may differ from every other + layer's. + + Attributes: + cells.{k}.weight_ih : the learnable input-hidden weights of the :math:`k`-th + layer, of shape `(3*hidden_size, input_size)` for `k = 0`. Otherwise, the + shape is `(3*hidden_size, hidden_size)`. + cells.{k}.weight_hh : the learnable weights applied to the previous input of + the :math:`k`-th layer, of the same shape as `weight_ih`. + cells.{k}.bias_ih : the learnable input-hidden biases of the :math:`k`-th + layer, of shape `(3*hidden_size)`. Only present when ``bias=True``. + cells.{k}.bias_hh : the learnable biases paired with `weight_hh` of the + :math:`k`-th layer, of shape `(3*hidden_size)`. Only present when + ``recurrent_bias=True``. + + .. note:: + All the weights and biases are initialized according to the provided + initializers (`kernel_init`, `recurrent_kernel_init`, etc.). + + .. note:: + Like TGRU, this layer overrides the generic decoupled double-state + forward loop: the "previous input" component of the state cannot be + stacked into a single tensor across layers when `input_size != + hidden_size`, and (unlike TGRU) the returned output is not part of + the recurring state at all. + + .. seealso:: + :class:`TLSTMCell` + + Examples:: + + >>> rnn = TLSTM(10, 20, num_layers=2, dropout=0.1) + >>> input = torch.randn(5, 3, 10) # (seq_len, batch, input_size) + >>> output, (cn, xn) = rnn(input) # default zero state + >>> xn[0].shape # layer 0: input_size-shaped + torch.Size([3, 10]) + >>> xn[1].shape # layer 1: hidden_size-shaped + torch.Size([3, 20]) + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_layers: int = 1, + dropout: float = 0.0, + batch_first: bool = False, + **kwargs, + ): + super(TLSTM, self).__init__( + input_size, hidden_size, num_layers, dropout, batch_first + ) + self.initialize_cells(TLSTMCell, **kwargs) + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tuple[Tensor, ...]]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tuple[Tensor, ...]]]: + if self.batch_first: + inp = inp.transpose(0, 1) + + seq_len, batch_size, _ = inp.size() + + if state is None: + c = [ + torch.zeros( + batch_size, self.hidden_size, dtype=inp.dtype, device=inp.device + ) + for _ in range(self.num_layers) + ] + x_prev = [ + torch.zeros(batch_size, cell.input_size, dtype=inp.dtype, device=inp.device) + for cell in self.cells + ] + else: + c0, x_prev0 = state + c = [c0[layer_idx] for layer_idx in range(self.num_layers)] + x_prev = list(x_prev0) + + outputs = [] + for t in range(seq_len): + x = inp[t] + new_c = [] + new_x_prev = [] + + for layer_idx, cell in enumerate(self.cells): + h, (c_i, x_prev_i) = cell(x, (c[layer_idx], x_prev[layer_idx])) + new_c.append(c_i) + new_x_prev.append(x_prev_i) + x = h + if self.dropout_layer is not None and layer_idx < self.num_layers - 1: + x = self.dropout_layer(x) + + c = new_c + x_prev = new_x_prev + outputs.append(x) + + out = torch.stack(outputs, dim=0) + if self.batch_first: + out = out.transpose(0, 1) + + c_n = torch.stack(c, dim=0) + x_n = tuple(x_prev) + return out, (c_n, x_n) + + +class TLSTMCell(DecoupledDoubleStateCellBase): + r"""A strongly typed long short-term memory (T-LSTM) cell. + + [`arXiv `_] + + .. math:: + + \mathbf{z}(t) &= \mathbf{V}^{z}\,\mathbf{x}(t-1) + + \mathbf{W}^{z}\,\mathbf{x}(t) + \mathbf{b}^{z}, \\[4pt] + \mathbf{f}(t) &= \sigma\bigl( + \mathbf{V}^{f}\,\mathbf{x}(t-1) + \mathbf{W}^{f}\,\mathbf{x}(t) + \mathbf{b}^{f} + \bigr), \\[4pt] + \mathbf{o}(t) &= \tau\bigl( + \mathbf{V}^{o}\,\mathbf{x}(t-1) + \mathbf{W}^{o}\,\mathbf{x}(t) + \mathbf{b}^{o} + \bigr), \\[4pt] + \mathbf{c}(t) &= \mathbf{f}(t)\circ\mathbf{c}(t-1) + + \bigl(1 - \mathbf{f}(t)\bigr)\circ\mathbf{z}(t), \\[4pt] + \mathbf{h}(t) &= \mathbf{c}(t)\circ\mathbf{o}(t), + + where :math:`\circ` is element-wise product and :math:`\tau` is a + pointwise nonlinearity (e.g., tanh). The recurrent weights act on the + previous input :math:`\mathbf{x}(t-1)`, not on a hidden state, per the + "strongly typed" design. The recurring state is + :math:`(\mathbf{c}(t), \mathbf{x}(t))`; the returned output + :math:`\mathbf{h}(t)` plays no role in any subsequent computation. + + Args: + input_size: The number of expected features in the input ``x``. + hidden_size: The number of features in the hidden and cell states. + bias: If ``False``, the layer does not use input-side biases. + Default: ``True``. + recurrent_bias: If ``False``, the layer does not use the biases + paired with ``V``. Default: ``True``. + gate_nonlinearity: Activation :math:`\sigma` for the forget gate. + Default: :func:`torch.sigmoid`. + output_nonlinearity: Activation :math:`\tau` for the output candidate. + Default: :func:`torch.tanh`. + kernel_init: Initializer for ``W``. + Default: :func:`torch.nn.init.xavier_uniform_`. + recurrent_kernel_init: Initializer for ``V``. + Default: :func:`torch.nn.init.xavier_uniform_`. + bias_init: Initializer for ``b_{ih}`` when ``bias=True``. + Default: :func:`torch.nn.init.zeros_`. + recurrent_bias_init: Initializer for ``b_{hh}`` when + ``recurrent_bias=True``. Default: :func:`torch.nn.init.zeros_`. + device: The desired device of parameters. + dtype: The desired floating point type of parameters. + + Inputs: input, (c_0, x_{-1}) + - **input** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing input features. + - **c_0** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the initial cell state. + - **x_{-1}** of shape ``(batch, input_size)`` or ``(input_size,)``: + Tensor containing the "previous input". + + If not provided, both default to zero. + + Outputs: h_1, (c_1, x_1) + - **h_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the output. + - **c_1** of shape ``(batch, hidden_size)`` or ``(hidden_size,)``: + Tensor containing the next cell state. + - **x_1**: the current input ``x``, carried forward unchanged to + serve as :math:`\mathbf{x}(t-1)` on the following call. + + Variables: + weight_ih: The learnable input-hidden weights, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + weight_hh: The learnable weights applied to the previous input, + of shape ``(3*hidden_size, input_size)`` (``z, f, o`` parts). + bias_ih: The learnable input-hidden biases, + of shape ``(3*hidden_size)`` if ``bias=True``. + bias_hh: The learnable biases paired with ``weight_hh``, + of shape ``(3*hidden_size)`` if ``recurrent_bias=True``. + + Examples:: + + >>> cell = TLSTMCell(10, 20) + >>> x = torch.randn(5, 3, 10) # (time_steps, batch, input_size) + >>> c = torch.zeros(3, 20) # (batch, hidden_size) + >>> x_prev = torch.zeros(3, 10) # (batch, input_size) + >>> out = [] + >>> for t in range(x.size(0)): + ... h, (c, x_prev) = cell(x[t], (c, x_prev)) + ... out.append(h) + >>> out = torch.stack(out, dim=0) # (time_steps, batch, hidden_size) + """ + + __constants__ = ["input_size", "hidden_size", "bias", "recurrent_bias"] + + weight_ih: Tensor + weight_hh: Tensor + bias_ih: Tensor + bias_hh: Tensor + + def __init__( + self, + input_size: int, + hidden_size: int, + bias: bool = True, + recurrent_bias: bool = True, + gate_nonlinearity="sigmoid", + output_nonlinearity="tanh", + kernel_init=nn.init.xavier_uniform_, + recurrent_kernel_init=nn.init.xavier_uniform_, + bias_init=nn.init.zeros_, + recurrent_bias_init=nn.init.zeros_, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__( + input_size=input_size, + hidden_size=hidden_size, + bias=bias, + recurrent_bias=recurrent_bias, + device=device, + dtype=dtype, + ) + self.gate_act = resolve_activation(gate_nonlinearity) + self.out_act = resolve_activation(output_nonlinearity) + self.init_cfg["kernel"] = resolve_init_name(kernel_init, self.init_cfg["kernel"]) + self.init_cfg["recurrent_kernel"] = resolve_init_name( + recurrent_kernel_init, self.init_cfg["recurrent_kernel"] + ) + self.init_cfg["bias"] = resolve_init_name(bias_init, self.init_cfg["bias"]) + self.init_cfg["recurrent_bias"] = resolve_init_name( + recurrent_bias_init, self.init_cfg["recurrent_bias"] + ) + + self._register_tensors( + { + "weight_ih": ((3 * hidden_size, input_size), True), + "weight_hh": ((3 * hidden_size, input_size), True), + "bias_ih": ((3 * hidden_size,), self.bias), + "bias_hh": ((3 * hidden_size,), self.recurrent_bias), + } + ) + self.reset_parameters() + self._cleanup_non_scriptable() + + def forward( + self, inp: Tensor, state: Optional[Tuple[Tensor, Tensor]] = None + ) -> Tuple[Tensor, Tuple[Tensor, Tensor]]: + self._validate_input(inp) + b_inp, is_batched = self._as_batched(inp) + + if state is None: + b_c = self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + b_prev_inp = torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + else: + c0, prev_inp0 = state + b_c = ( + self._zeros_state(b_inp.size(0), b_inp.device, b_inp.dtype) + if c0 is None + else (c0.unsqueeze(0) if (not is_batched and c0.dim() == 1) else c0) + ) + b_prev_inp = ( + torch.zeros( + b_inp.size(0), self.input_size, device=b_inp.device, dtype=b_inp.dtype + ) + if prev_inp0 is None + else ( + prev_inp0.unsqueeze(0) + if (not is_batched and prev_inp0.dim() == 1) + else prev_inp0 + ) + ) + + gates_ih = b_inp @ self.weight_ih.t() + self.bias_ih + gates_hh = b_prev_inp @ self.weight_hh.t() + self.bias_hh + latent, forget_pre, out_pre = (gates_ih + gates_hh).chunk(3, 1) + + forget_gate = self.gate_act(forget_pre) + output_candidate = self.out_act(out_pre) + new_c = forget_gate * b_c + (1.0 - forget_gate) * latent + new_h = new_c * output_candidate + new_prev_inp = b_inp + + if not is_batched: + new_h = new_h.squeeze(0) + new_c = new_c.squeeze(0) + new_prev_inp = new_prev_inp.squeeze(0) + + return new_h, (new_c, new_prev_inp)