Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/layers/basic.jl
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,13 @@ function (dl::DelayLayer)(inp::AbstractVecOrMat, ps, st::NamedTuple)
return inp_with_delay, (history = history, clock = clock, rng = rng)
end

function __reset_delay(rng::AbstractRNG, dl::DelayLayer, st, init_carry)
history = init_carry === nothing ? nothing : init_carry(rng, dl.in_dims, dl.num_delays)
return merge(st, (; history, clock = 0))
end

__reset_delay(::AbstractRNG, _, st, init_carry) = st

function Base.show(io::IO, dl::DelayLayer)
print(io, "DelayLayer(", dl.in_dims)
sep = "; "
Expand Down
50 changes: 43 additions & 7 deletions src/layers/deep_reservoir.jl
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,29 @@ __carry_dimensions(cell::RMNCell) = (
__cell_out_dims(cell.linear_reservoir),
)
__carry_dimensions(cell::LocalInformationFlow) = __carry_dimensions(cell.cell)
__carry_dimensions(res::SciMLProblemReservoir) = (length(res.prob.u0),)

__carry_cell(layer) = layer.cell
__carry_cell(res::AbstractSciMLProblemReservoir) = res

__init_carry_state(::Any, init_carry, rng, dim) = init_carry(rng, dim, 1)
__init_carry_state(::AbstractSciMLProblemReservoir, init_carry, rng, dim) =
vec(init_carry(rng, dim, 1))
function __init_carry_state(res::SciMLProblemReservoir, init_carry, rng, dim)
return convert(Vector{eltype(res.prob.u0)}, vec(init_carry(rng, dim, 1)))
end

function __reset_deep_delays(rng::AbstractRNG, mods, st_mods, init_carry, n_layers)
mods === nothing && return st_mods
initializers = Tuple(__asvec(init_carry, n_layers))
return ntuple(n_layers) do idx
mods[idx] === nothing ? st_mods[idx] :
map(
(layer, st_layer) -> __reset_delay(rng, layer, st_layer, initializers[idx]),
mods[idx], st_mods[idx]
)
end
end

function __reset_deep_carry(
rng::AbstractRNG, layer::StatefulLayer, st, initializer
Expand All @@ -194,14 +217,14 @@ function __reset_deep_carry(
dimensions = st.carry === nothing ? __carry_dimensions(layer.cell) :
map(state -> size(state, 1), st.carry)
carry = map(dim -> __asvec(initializer(rng, dim)), dimensions)
return merge(st, (; carry))
return __reset_window(layer, merge(st, (; carry)), carry)
end

__reset_deep_carry(::AbstractRNG, ::AbstractLuxLayer, st, ::Nothing) = st
function __reset_deep_carry(
::AbstractRNG, ::StatefulLayer, st, ::Nothing
::AbstractRNG, layer::StatefulLayer, st, ::Nothing
)
return merge(st, (; carry = nothing))
return __reset_window(layer, merge(st, (; carry = nothing)), nothing)
end
function __reset_deep_carry(::AbstractRNG, layer::AbstractLuxLayer, _, initializer)
throw(
Expand All @@ -212,16 +235,29 @@ function __reset_deep_carry(::AbstractRNG, layer::AbstractLuxLayer, _, initializ
)
end

"""
resetcarry!(rng, dres::DeepReservoir, st; init_carry=nothing)

Reset the carry of every stateful cell of a [`DeepReservoir`](@ref), and the history
and clock of every [`DelayLayer`](@ref) among the per-layer `state_modifiers`.

`init_carry` is `nothing`, a function, or a tuple or vector with one entry per layer
(`nothing` or a function). Entry `i` sets the carry of cell `i`, called as
`f(rng, dim)`, and the delay layers of layer `i`'s modifiers, called as
`f(rng, in_dims, num_delays)`, so the initializers must accept both calls (the
[WeightInitializers.jl](https://lux.csail.mit.edu/stable/api/Building_Blocks/WeightInitializers)
functions do). A `nothing` entry clears the carry and the delay history of that layer,
which is then rebuilt from the layer's own initializers on the next call.
"""
function resetcarry!(rng::AbstractRNG, dres::DeepReservoir, st; init_carry = nothing)
n_layers = length(dres.cells)
initializers = Tuple(__asvec(init_carry, n_layers))
new_cells = ntuple(n_layers) do idx
__reset_deep_carry(rng, dres.cells[idx], st.cells[idx], initializers[idx])
end

return (;
cells = new_cells,
state_modifiers = st.state_modifiers,
readout = st.readout,
new_mods = __reset_deep_delays(
rng, dres.state_modifiers, st.state_modifiers, init_carry, n_layers
)
return (; cells = new_cells, state_modifiers = new_mods, readout = st.readout)
end
16 changes: 11 additions & 5 deletions src/layers/lif_wrapper.jl
Original file line number Diff line number Diff line change
Expand Up @@ -80,14 +80,14 @@ Created by `initialparameters(rng, lif)`:
Created by `initialstates(rng, lif)`:

- `rng`: an owned copy of the RNG given to `initialstates`. The first call samples
`start_states` and `input_buffer` from a copy of it and returns the advanced copy,
so the RNG of the input state is never advanced.
`start_states` and `input_buffer` from a copy of it and leaves it unchanged, so a
later re-initialization (as after [`resetcarry!`](@ref)) draws the same values.
- `input_buffer`: a length `lookback_horizon-1` history of past inputs, stored as
copies; initialized on first call using `init_buffer`.
- `start_states`: the state tuple every window starts from; sampled on the first call
with the wrapped cell's `init_state`. A state saved without this entry, as by earlier
versions, is accepted: the next call samples it from a copy of `rng`, keeps the saved
`input_buffer`, and returns the advanced `rng`.
versions, is accepted: the next call samples it from a copy of `rng` and keeps the saved
`input_buffer`.
- `cell`: the wrapped cell’s own internal state (NamedTuple).
"""
@concrete struct LocalInformationFlow <: AbstractEchoStateNetworkCell
Expand Down Expand Up @@ -144,6 +144,12 @@ function __init_input_buffer(
return buf
end

__reset_window(_, st_res, _) = st_res

function __reset_window(::StatefulLayer{<:LocalInformationFlow}, st_res, start_states)
return merge(st_res, (; cell = merge(st_res.cell, (; input_buffer = nothing, start_states))))
end

function init_hidden_states(rng::AbstractRNG, lif::LocalInformationFlow, inp::AbstractArray)
return init_hidden_states(rng, lif.cell, inp)
end
Expand All @@ -163,7 +169,7 @@ function __init_window_states(lif::LocalInformationFlow, inp::AbstractArray, st:
input_buffer = __init_input_buffer(rng, lif, inp)
end

return merge(st, (; rng, input_buffer, start_states))
return merge(st, (; input_buffer, start_states))
end

function (lif::LocalInformationFlow)((inp, _)::Tuple, ps, st::NamedTuple)
Expand Down
4 changes: 4 additions & 0 deletions src/layers/lux_layers.jl
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@ end

(c::ReservoirChain)(x, ps, st::NamedTuple) = applychain(c.layers, x, ps, st)

function __reset_delay(rng::AbstractRNG, c::ReservoirChain, st, init_carry)
return map((layer, st_layer) -> __reset_delay(rng, layer, st_layer, init_carry), c.layers, st)
end

@generated function applychain(
layers::NamedTuple{fields}, x, ps, st::NamedTuple{fields}
) where {fields}
Expand Down
7 changes: 7 additions & 0 deletions src/layers/spiking.jl
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,13 @@ function initialstates(rng::AbstractRNG, cell::LSMCell)
)
end

__carry_dimensions(cell::LSMCell) = (2 * cell.out_dims,)

function __init_carry_state(::LSMCell, init_carry, rng, dim)
membrane = vec(init_carry(rng, dim ÷ 2, 1))
return vcat(membrane, zero(membrane))
end

__init_encoder_st(::AbstractRNG, ::CurrentInjection) = NamedTuple()
function __init_encoder_st(rng::AbstractRNG, ::PoissonRateEncoder)
return (rng = __owned_replicate(rng),)
Expand Down
21 changes: 17 additions & 4 deletions src/models/esn_deep.jl
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,20 @@ function (desn::DeepESN)(inp, ps, st)
return inp_t, merge(new_st, (readout = st_ro,))
end

"""
resetcarry!(rng, desn::DeepESN, st; init_carry=nothing)

Reset the carry of every cell of a [`DeepESN`](@ref), and the history and clock of
every [`DelayLayer`](@ref) among the per-layer `state_modifiers`.

`init_carry` is `nothing`, a function, or a tuple or vector with one entry per layer
(`nothing` or a function). Entry `i` sets the carry of cell `i`, called as
`f(rng, dim)`, and the delay layers of layer `i`'s modifiers, called as
`f(rng, in_dims, num_delays)`, so the initializers must accept both calls (the
[WeightInitializers.jl](https://lux.csail.mit.edu/stable/api/Building_Blocks/WeightInitializers)
functions do). A `nothing` entry clears the carry and the delay history of that layer,
which is then rebuilt from the layer's own initializers on the next call.
"""
function resetcarry!(rng::AbstractRNG, desn::DeepESN, st; init_carry = nothing)
n_layers = length(desn.cells)

Expand Down Expand Up @@ -255,11 +269,10 @@ function resetcarry!(rng::AbstractRNG, desn::DeepESN, st; init_carry = nothing)
end, n_layers
)

return (;
cells = new_cells,
state_modifiers = st.state_modifiers,
readout = st.readout,
new_mods = __reset_deep_delays(
rng, desn.state_modifiers, st.state_modifiers, init_carry, n_layers
)
return (; cells = new_cells, state_modifiers = new_mods, readout = st.readout)
end

function collectstates(desn::DeepESN, data::AbstractMatrix, ps, st::NamedTuple)
Expand Down
34 changes: 26 additions & 8 deletions src/models/ngrc.jl
Original file line number Diff line number Diff line change
Expand Up @@ -108,17 +108,35 @@ function __infer_ngrc_ro_dims(mods::Tuple, n_taps::Int, input_dims::Int)
end
end

"""
resetcarry!(rng, rc::NGRC, st; init_carry=nothing)
resetcarry!(rng, rc::NGRC, ps, st; init_carry=nothing)

Reset the delay history of the internal [`DelayLayer`](@ref) of an [`NGRC`](@ref)
and of every [`DelayLayer`](@ref) among its `state_modifiers`, and restart their
clocks. The readout state and the states of other modifiers are returned unchanged.

## Keyword arguments

- `init_carry`:
+ `nothing` (default): clear the history, so that the next call rebuilds
it from the `init_delay` initializers, exactly as for a fresh `setup`.
+ `f`: a function following the
[WeightInitializers.jl](https://lux.csail.mit.edu/stable/api/Building_Blocks/WeightInitializers)
convention. The history is set to `f(rng, in_dims, num_delays)`, one column
per delay.

## Returns

- `resetcarry!(rng, rc, st; ...) -> st′` with the reset `st′.reservoir` and
`st′.state_modifiers`.
- `resetcarry!(rng, rc, ps, st; ...) -> (ps, st′)`.
"""
function resetcarry!(
rng::AbstractRNG, rc::NGRC, st; init_carry = nothing
)
carry = get(st.reservoir, :carry, nothing)
@warn(
"""
Next generation reservoir computing has no internal state to reset.
Returning untouched model states.
"""
)
return st
st_res = __reset_delay(rng, rc.reservoir, st.reservoir, init_carry)
return __reset_modifier_delays(rng, rc, merge(st, (reservoir = st_res,)), init_carry)
end

@doc raw"""
Expand Down
59 changes: 46 additions & 13 deletions src/reservoircomputer.jl
Original file line number Diff line number Diff line change
Expand Up @@ -232,11 +232,34 @@ end

Reset (or set) the hidden-state carry of a model in the echo state network family.

When a function is supplied as `init_carry`, an existing carry provides its leading
dimension; otherwise the reservoir output size is used. When `init_carry=nothing`,
When a function is supplied as `init_carry`, it is called as `init_carry(rng, dim, 1)`
once per state held by the cell (one for most cells, a hidden state and a memory
state for [`RMNCell`](@ref)). An existing carry provides the leading dimensions;
otherwise they are taken from the cell. When `init_carry=nothing`,
the carry is cleared and the cell's initializer is used on the next call. This does
not require the cell to expose an output dimension.

For continuous-time reservoirs ([`ContinuousESN`](@ref), [`LSM`](@ref) and
[`SciMLProblemReservoir`](@ref)) the carry is the ODE state vector, and `init_carry` is
called as `init_carry(rng, dim, 1)` and flattened to a vector. `dim` is `out_dims` for
`ContinuousESN`, `length(prob.u0)` for `SciMLProblemReservoir`, and `out_dims` for `LSM`,
where `init_carry` sets only the membrane potentials (the first half of the state)
and the remaining half of the state is set to zero.

The history and clock of every [`DelayLayer`](@ref) held by the model, in
`st.input_delay` or in `st.state_modifiers`, are reset as well: with
`init_carry=nothing` the history is cleared and rebuilt from the layer's `init_delay`
on the next call, and with a function the history is set to
`init_carry(rng, in_dims, num_delays)`. The clock restarts at zero. Other modifier
and readout states are unchanged.

For [`LIFESN`](@ref) the cached `start_states` and the input buffer of the
[`LocalInformationFlow`](@ref) cell are reset together with the carry: with
`init_carry=nothing` both are cleared and sampled again from the cell's `init_state` and
`init_buffer` on the next call, and with a function `start_states` is set to the same
states as the carry. The cell's `rng` is never advanced, so a random `init_state` and
`init_buffer` are sampled again with the same values as in a fresh state.

## Arguments

- `rng`: Random number generator (used if a new carry is sampled/created).
Expand All @@ -255,7 +278,8 @@ not require the cell to expose an output dimension.
## Returns

- `resetcarry!(rng, rc, st; ...) -> st′`:
Updated states with `st′.cell.carry` set to `nothing` or `(h0,)`.
Updated states with `st′.reservoir.carry` set to `nothing` or a tuple with one
state per recurrent state of the cell.
- `resetcarry!(rng, rc, ps, st; ...) -> (ps, st′)`:
Same as above, but also returns the unchanged `ps` for convenience.

Expand All @@ -267,17 +291,26 @@ function resetcarry!(
if init_carry === nothing
new_state = nothing
else
if carry === nothing
sz = __cell_out_dims(rc.reservoir.cell)
else
state = first(carry)
sz = size(state, 1)
end
new_state = init_carry(rng, sz, 1)
new_state = (new_state,)
cell = __carry_cell(rc.reservoir)
dims = carry === nothing ? __carry_dimensions(cell) :
map(state -> size(state, 1), carry)
new_state = map(dim -> __init_carry_state(cell, init_carry, rng, dim), dims)
end
new_cell = merge(st.reservoir, (; carry = new_state))
return merge(st, (reservoir = new_cell,))
new_res = __reset_window(rc.reservoir, merge(st.reservoir, (; carry = new_state)), new_state)
new_st = __reset_modifier_delays(rng, rc, merge(st, (reservoir = new_res,)), init_carry)
return haskey(st, :input_delay) ?
merge(
new_st,
(input_delay = __reset_delay(rng, rc.input_delay, st.input_delay, init_carry),)
) : new_st
end

function __reset_modifier_delays(rng::AbstractRNG, rc, st, init_carry)
new_mods = map(
(layer, st_layer) -> __reset_delay(rng, layer, st_layer, init_carry),
rc.state_modifiers, st.state_modifiers
)
return merge(st, (state_modifiers = new_mods,))
end

function resetcarry!(
Expand Down
4 changes: 4 additions & 0 deletions src/states.jl
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,10 @@ function (ex::Extend)(inp, ps, st::NamedTuple)
return vcat(inp, state), (; op = st_op)
end

function __reset_delay(rng::AbstractRNG, ex::Extend, st, init_carry)
return (; op = __reset_delay(rng, ex.op, st.op, init_carry))
end

@inline function __apply_state_modifier(ex::Extend, features, inp, ps, st)
state, st_op = apply(ex.op, features, ps.op, st.op)
return vcat(inp, state), (; op = st_op)
Expand Down
23 changes: 23 additions & 0 deletions test/Extensions/continuous_esn_tests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,29 @@ begin
@test eltype(ps.reservoir.reservoir_matrix) == Float64
end

@testset "ContinuousESN: resetcarry! with init_carry" begin
n, T_steps = 12, 8
esn = ContinuousESN(2, n, 1, (0.0, 1.0), Tsit5(); init_state = zeros32)
ps, st0 = setup(MersenneTwister(2), esn)
data = randn(MersenneTwister(1), Float32, 2, T_steps)
const03 = (rng, d...) -> fill(0.3f0, d...)
run = st -> first(collectstates(esn, data, ps, deepcopy(st)))
base = run(st0)
_, st1 = collectstates(esn, data, ps, deepcopy(st0))
manual = merge(st0, (reservoir = (carry = (fill(0.3f0, n),),),))
for st in (st0, st1)
st_zero = resetcarry!(MersenneTwister(3), esn, deepcopy(st); init_carry = zeros32)
@test first(st_zero.reservoir.carry) isa Vector
@test run(st_zero) == base
st_const = resetcarry!(MersenneTwister(3), esn, deepcopy(st); init_carry = const03)
@test first(st_const.reservoir.carry) == fill(0.3f0, n)
@test run(st_const) == run(manual)
@test run(st_const) != base
st_clear = resetcarry!(MersenneTwister(3), esn, deepcopy(st))
@test run(st_clear) == base
end
end

@testset "ContinuousESN: autoregressive predict uses carry" begin
rng = MersenneTwister(99)
dim, res_dim, T_steps, steps = 2, 16, 20, 5
Expand Down
Loading
Loading