diff --git a/src/layers/basic.jl b/src/layers/basic.jl index da510180c..e88d5584b 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -438,6 +438,13 @@ function (dl::DelayLayer)(inp::AbstractVecOrMat, ps, st::NamedTuple) return inp_with_delay, (history = history, clock = clock, rng = rng) end +function __reset_delay(rng::AbstractRNG, dl::DelayLayer, st, init_carry) + history = init_carry === nothing ? nothing : init_carry(rng, dl.in_dims, dl.num_delays) + return merge(st, (; history, clock = 0)) +end + +__reset_delay(::AbstractRNG, _, st, init_carry) = st + function Base.show(io::IO, dl::DelayLayer) print(io, "DelayLayer(", dl.in_dims) sep = "; " diff --git a/src/layers/deep_reservoir.jl b/src/layers/deep_reservoir.jl index 3b52e6e86..a2f56ff57 100644 --- a/src/layers/deep_reservoir.jl +++ b/src/layers/deep_reservoir.jl @@ -184,6 +184,29 @@ __carry_dimensions(cell::RMNCell) = ( __cell_out_dims(cell.linear_reservoir), ) __carry_dimensions(cell::LocalInformationFlow) = __carry_dimensions(cell.cell) +__carry_dimensions(res::SciMLProblemReservoir) = (length(res.prob.u0),) + +__carry_cell(layer) = layer.cell +__carry_cell(res::AbstractSciMLProblemReservoir) = res + +__init_carry_state(::Any, init_carry, rng, dim) = init_carry(rng, dim, 1) +__init_carry_state(::AbstractSciMLProblemReservoir, init_carry, rng, dim) = + vec(init_carry(rng, dim, 1)) +function __init_carry_state(res::SciMLProblemReservoir, init_carry, rng, dim) + return convert(Vector{eltype(res.prob.u0)}, vec(init_carry(rng, dim, 1))) +end + +function __reset_deep_delays(rng::AbstractRNG, mods, st_mods, init_carry, n_layers) + mods === nothing && return st_mods + initializers = Tuple(__asvec(init_carry, n_layers)) + return ntuple(n_layers) do idx + mods[idx] === nothing ? st_mods[idx] : + map( + (layer, st_layer) -> __reset_delay(rng, layer, st_layer, initializers[idx]), + mods[idx], st_mods[idx] + ) + end +end function __reset_deep_carry( rng::AbstractRNG, layer::StatefulLayer, st, initializer @@ -194,14 +217,14 @@ function __reset_deep_carry( dimensions = st.carry === nothing ? __carry_dimensions(layer.cell) : map(state -> size(state, 1), st.carry) carry = map(dim -> __asvec(initializer(rng, dim)), dimensions) - return merge(st, (; carry)) + return __reset_window(layer, merge(st, (; carry)), carry) end __reset_deep_carry(::AbstractRNG, ::AbstractLuxLayer, st, ::Nothing) = st function __reset_deep_carry( - ::AbstractRNG, ::StatefulLayer, st, ::Nothing + ::AbstractRNG, layer::StatefulLayer, st, ::Nothing ) - return merge(st, (; carry = nothing)) + return __reset_window(layer, merge(st, (; carry = nothing)), nothing) end function __reset_deep_carry(::AbstractRNG, layer::AbstractLuxLayer, _, initializer) throw( @@ -212,6 +235,20 @@ function __reset_deep_carry(::AbstractRNG, layer::AbstractLuxLayer, _, initializ ) end +""" + resetcarry!(rng, dres::DeepReservoir, st; init_carry=nothing) + +Reset the carry of every stateful cell of a [`DeepReservoir`](@ref), and the history +and clock of every [`DelayLayer`](@ref) among the per-layer `state_modifiers`. + +`init_carry` is `nothing`, a function, or a tuple or vector with one entry per layer +(`nothing` or a function). Entry `i` sets the carry of cell `i`, called as +`f(rng, dim)`, and the delay layers of layer `i`'s modifiers, called as +`f(rng, in_dims, num_delays)`, so the initializers must accept both calls (the +[WeightInitializers.jl](https://lux.csail.mit.edu/stable/api/Building_Blocks/WeightInitializers) +functions do). A `nothing` entry clears the carry and the delay history of that layer, +which is then rebuilt from the layer's own initializers on the next call. +""" function resetcarry!(rng::AbstractRNG, dres::DeepReservoir, st; init_carry = nothing) n_layers = length(dres.cells) initializers = Tuple(__asvec(init_carry, n_layers)) @@ -219,9 +256,8 @@ function resetcarry!(rng::AbstractRNG, dres::DeepReservoir, st; init_carry = not __reset_deep_carry(rng, dres.cells[idx], st.cells[idx], initializers[idx]) end - return (; - cells = new_cells, - state_modifiers = st.state_modifiers, - readout = st.readout, + new_mods = __reset_deep_delays( + rng, dres.state_modifiers, st.state_modifiers, init_carry, n_layers ) + return (; cells = new_cells, state_modifiers = new_mods, readout = st.readout) end diff --git a/src/layers/lif_wrapper.jl b/src/layers/lif_wrapper.jl index 2ec5ce829..6cc8c2571 100644 --- a/src/layers/lif_wrapper.jl +++ b/src/layers/lif_wrapper.jl @@ -80,14 +80,14 @@ Created by `initialparameters(rng, lif)`: Created by `initialstates(rng, lif)`: - `rng`: an owned copy of the RNG given to `initialstates`. The first call samples - `start_states` and `input_buffer` from a copy of it and returns the advanced copy, - so the RNG of the input state is never advanced. + `start_states` and `input_buffer` from a copy of it and leaves it unchanged, so a + later re-initialization (as after [`resetcarry!`](@ref)) draws the same values. - `input_buffer`: a length `lookback_horizon-1` history of past inputs, stored as copies; initialized on first call using `init_buffer`. - `start_states`: the state tuple every window starts from; sampled on the first call with the wrapped cell's `init_state`. A state saved without this entry, as by earlier - versions, is accepted: the next call samples it from a copy of `rng`, keeps the saved - `input_buffer`, and returns the advanced `rng`. + versions, is accepted: the next call samples it from a copy of `rng` and keeps the saved + `input_buffer`. - `cell`: the wrapped cell’s own internal state (NamedTuple). """ @concrete struct LocalInformationFlow <: AbstractEchoStateNetworkCell @@ -144,6 +144,12 @@ function __init_input_buffer( return buf end +__reset_window(_, st_res, _) = st_res + +function __reset_window(::StatefulLayer{<:LocalInformationFlow}, st_res, start_states) + return merge(st_res, (; cell = merge(st_res.cell, (; input_buffer = nothing, start_states)))) +end + function init_hidden_states(rng::AbstractRNG, lif::LocalInformationFlow, inp::AbstractArray) return init_hidden_states(rng, lif.cell, inp) end @@ -163,7 +169,7 @@ function __init_window_states(lif::LocalInformationFlow, inp::AbstractArray, st: input_buffer = __init_input_buffer(rng, lif, inp) end - return merge(st, (; rng, input_buffer, start_states)) + return merge(st, (; input_buffer, start_states)) end function (lif::LocalInformationFlow)((inp, _)::Tuple, ps, st::NamedTuple) diff --git a/src/layers/lux_layers.jl b/src/layers/lux_layers.jl index 107abf051..ccae71317 100644 --- a/src/layers/lux_layers.jl +++ b/src/layers/lux_layers.jl @@ -198,6 +198,10 @@ end (c::ReservoirChain)(x, ps, st::NamedTuple) = applychain(c.layers, x, ps, st) +function __reset_delay(rng::AbstractRNG, c::ReservoirChain, st, init_carry) + return map((layer, st_layer) -> __reset_delay(rng, layer, st_layer, init_carry), c.layers, st) +end + @generated function applychain( layers::NamedTuple{fields}, x, ps, st::NamedTuple{fields} ) where {fields} diff --git a/src/layers/spiking.jl b/src/layers/spiking.jl index b90ff5eb2..a2e04b525 100644 --- a/src/layers/spiking.jl +++ b/src/layers/spiking.jl @@ -403,6 +403,13 @@ function initialstates(rng::AbstractRNG, cell::LSMCell) ) end +__carry_dimensions(cell::LSMCell) = (2 * cell.out_dims,) + +function __init_carry_state(::LSMCell, init_carry, rng, dim) + membrane = vec(init_carry(rng, dim ÷ 2, 1)) + return vcat(membrane, zero(membrane)) +end + __init_encoder_st(::AbstractRNG, ::CurrentInjection) = NamedTuple() function __init_encoder_st(rng::AbstractRNG, ::PoissonRateEncoder) return (rng = __owned_replicate(rng),) diff --git a/src/models/esn_deep.jl b/src/models/esn_deep.jl index 9ab0aeda1..3f77c9ba7 100644 --- a/src/models/esn_deep.jl +++ b/src/models/esn_deep.jl @@ -220,6 +220,20 @@ function (desn::DeepESN)(inp, ps, st) return inp_t, merge(new_st, (readout = st_ro,)) end +""" + resetcarry!(rng, desn::DeepESN, st; init_carry=nothing) + +Reset the carry of every cell of a [`DeepESN`](@ref), and the history and clock of +every [`DelayLayer`](@ref) among the per-layer `state_modifiers`. + +`init_carry` is `nothing`, a function, or a tuple or vector with one entry per layer +(`nothing` or a function). Entry `i` sets the carry of cell `i`, called as +`f(rng, dim)`, and the delay layers of layer `i`'s modifiers, called as +`f(rng, in_dims, num_delays)`, so the initializers must accept both calls (the +[WeightInitializers.jl](https://lux.csail.mit.edu/stable/api/Building_Blocks/WeightInitializers) +functions do). A `nothing` entry clears the carry and the delay history of that layer, +which is then rebuilt from the layer's own initializers on the next call. +""" function resetcarry!(rng::AbstractRNG, desn::DeepESN, st; init_carry = nothing) n_layers = length(desn.cells) @@ -255,11 +269,10 @@ function resetcarry!(rng::AbstractRNG, desn::DeepESN, st; init_carry = nothing) end, n_layers ) - return (; - cells = new_cells, - state_modifiers = st.state_modifiers, - readout = st.readout, + new_mods = __reset_deep_delays( + rng, desn.state_modifiers, st.state_modifiers, init_carry, n_layers ) + return (; cells = new_cells, state_modifiers = new_mods, readout = st.readout) end function collectstates(desn::DeepESN, data::AbstractMatrix, ps, st::NamedTuple) diff --git a/src/models/ngrc.jl b/src/models/ngrc.jl index 29011f160..789ad18d0 100644 --- a/src/models/ngrc.jl +++ b/src/models/ngrc.jl @@ -108,17 +108,35 @@ function __infer_ngrc_ro_dims(mods::Tuple, n_taps::Int, input_dims::Int) end end +""" + resetcarry!(rng, rc::NGRC, st; init_carry=nothing) + resetcarry!(rng, rc::NGRC, ps, st; init_carry=nothing) + +Reset the delay history of the internal [`DelayLayer`](@ref) of an [`NGRC`](@ref) +and of every [`DelayLayer`](@ref) among its `state_modifiers`, and restart their +clocks. The readout state and the states of other modifiers are returned unchanged. + +## Keyword arguments + + - `init_carry`: + + `nothing` (default): clear the history, so that the next call rebuilds + it from the `init_delay` initializers, exactly as for a fresh `setup`. + + `f`: a function following the + [WeightInitializers.jl](https://lux.csail.mit.edu/stable/api/Building_Blocks/WeightInitializers) + convention. The history is set to `f(rng, in_dims, num_delays)`, one column + per delay. + +## Returns + + - `resetcarry!(rng, rc, st; ...) -> st′` with the reset `st′.reservoir` and + `st′.state_modifiers`. + - `resetcarry!(rng, rc, ps, st; ...) -> (ps, st′)`. +""" function resetcarry!( rng::AbstractRNG, rc::NGRC, st; init_carry = nothing ) - carry = get(st.reservoir, :carry, nothing) - @warn( - """ - Next generation reservoir computing has no internal state to reset. - Returning untouched model states. - """ - ) - return st + st_res = __reset_delay(rng, rc.reservoir, st.reservoir, init_carry) + return __reset_modifier_delays(rng, rc, merge(st, (reservoir = st_res,)), init_carry) end @doc raw""" diff --git a/src/reservoircomputer.jl b/src/reservoircomputer.jl index bbfb59ff3..711cae6a8 100644 --- a/src/reservoircomputer.jl +++ b/src/reservoircomputer.jl @@ -232,11 +232,34 @@ end Reset (or set) the hidden-state carry of a model in the echo state network family. -When a function is supplied as `init_carry`, an existing carry provides its leading -dimension; otherwise the reservoir output size is used. When `init_carry=nothing`, +When a function is supplied as `init_carry`, it is called as `init_carry(rng, dim, 1)` +once per state held by the cell (one for most cells, a hidden state and a memory +state for [`RMNCell`](@ref)). An existing carry provides the leading dimensions; +otherwise they are taken from the cell. When `init_carry=nothing`, the carry is cleared and the cell's initializer is used on the next call. This does not require the cell to expose an output dimension. +For continuous-time reservoirs ([`ContinuousESN`](@ref), [`LSM`](@ref) and +[`SciMLProblemReservoir`](@ref)) the carry is the ODE state vector, and `init_carry` is +called as `init_carry(rng, dim, 1)` and flattened to a vector. `dim` is `out_dims` for +`ContinuousESN`, `length(prob.u0)` for `SciMLProblemReservoir`, and `out_dims` for `LSM`, +where `init_carry` sets only the membrane potentials (the first half of the state) +and the remaining half of the state is set to zero. + +The history and clock of every [`DelayLayer`](@ref) held by the model, in +`st.input_delay` or in `st.state_modifiers`, are reset as well: with +`init_carry=nothing` the history is cleared and rebuilt from the layer's `init_delay` +on the next call, and with a function the history is set to +`init_carry(rng, in_dims, num_delays)`. The clock restarts at zero. Other modifier +and readout states are unchanged. + +For [`LIFESN`](@ref) the cached `start_states` and the input buffer of the +[`LocalInformationFlow`](@ref) cell are reset together with the carry: with +`init_carry=nothing` both are cleared and sampled again from the cell's `init_state` and +`init_buffer` on the next call, and with a function `start_states` is set to the same +states as the carry. The cell's `rng` is never advanced, so a random `init_state` and +`init_buffer` are sampled again with the same values as in a fresh state. + ## Arguments - `rng`: Random number generator (used if a new carry is sampled/created). @@ -255,7 +278,8 @@ not require the cell to expose an output dimension. ## Returns - `resetcarry!(rng, rc, st; ...) -> st′`: - Updated states with `st′.cell.carry` set to `nothing` or `(h0,)`. + Updated states with `st′.reservoir.carry` set to `nothing` or a tuple with one + state per recurrent state of the cell. - `resetcarry!(rng, rc, ps, st; ...) -> (ps, st′)`: Same as above, but also returns the unchanged `ps` for convenience. @@ -267,17 +291,26 @@ function resetcarry!( if init_carry === nothing new_state = nothing else - if carry === nothing - sz = __cell_out_dims(rc.reservoir.cell) - else - state = first(carry) - sz = size(state, 1) - end - new_state = init_carry(rng, sz, 1) - new_state = (new_state,) + cell = __carry_cell(rc.reservoir) + dims = carry === nothing ? __carry_dimensions(cell) : + map(state -> size(state, 1), carry) + new_state = map(dim -> __init_carry_state(cell, init_carry, rng, dim), dims) end - new_cell = merge(st.reservoir, (; carry = new_state)) - return merge(st, (reservoir = new_cell,)) + new_res = __reset_window(rc.reservoir, merge(st.reservoir, (; carry = new_state)), new_state) + new_st = __reset_modifier_delays(rng, rc, merge(st, (reservoir = new_res,)), init_carry) + return haskey(st, :input_delay) ? + merge( + new_st, + (input_delay = __reset_delay(rng, rc.input_delay, st.input_delay, init_carry),) + ) : new_st +end + +function __reset_modifier_delays(rng::AbstractRNG, rc, st, init_carry) + new_mods = map( + (layer, st_layer) -> __reset_delay(rng, layer, st_layer, init_carry), + rc.state_modifiers, st.state_modifiers + ) + return merge(st, (state_modifiers = new_mods,)) end function resetcarry!( diff --git a/src/states.jl b/src/states.jl index ada249bc8..e71fc3ce6 100644 --- a/src/states.jl +++ b/src/states.jl @@ -149,6 +149,10 @@ function (ex::Extend)(inp, ps, st::NamedTuple) return vcat(inp, state), (; op = st_op) end +function __reset_delay(rng::AbstractRNG, ex::Extend, st, init_carry) + return (; op = __reset_delay(rng, ex.op, st.op, init_carry)) +end + @inline function __apply_state_modifier(ex::Extend, features, inp, ps, st) state, st_op = apply(ex.op, features, ps.op, st.op) return vcat(inp, state), (; op = st_op) diff --git a/test/Extensions/continuous_esn_tests.jl b/test/Extensions/continuous_esn_tests.jl index 70c468e06..99aae3983 100644 --- a/test/Extensions/continuous_esn_tests.jl +++ b/test/Extensions/continuous_esn_tests.jl @@ -291,6 +291,29 @@ begin @test eltype(ps.reservoir.reservoir_matrix) == Float64 end + @testset "ContinuousESN: resetcarry! with init_carry" begin + n, T_steps = 12, 8 + esn = ContinuousESN(2, n, 1, (0.0, 1.0), Tsit5(); init_state = zeros32) + ps, st0 = setup(MersenneTwister(2), esn) + data = randn(MersenneTwister(1), Float32, 2, T_steps) + const03 = (rng, d...) -> fill(0.3f0, d...) + run = st -> first(collectstates(esn, data, ps, deepcopy(st))) + base = run(st0) + _, st1 = collectstates(esn, data, ps, deepcopy(st0)) + manual = merge(st0, (reservoir = (carry = (fill(0.3f0, n),),),)) + for st in (st0, st1) + st_zero = resetcarry!(MersenneTwister(3), esn, deepcopy(st); init_carry = zeros32) + @test first(st_zero.reservoir.carry) isa Vector + @test run(st_zero) == base + st_const = resetcarry!(MersenneTwister(3), esn, deepcopy(st); init_carry = const03) + @test first(st_const.reservoir.carry) == fill(0.3f0, n) + @test run(st_const) == run(manual) + @test run(st_const) != base + st_clear = resetcarry!(MersenneTwister(3), esn, deepcopy(st)) + @test run(st_clear) == base + end + end + @testset "ContinuousESN: autoregressive predict uses carry" begin rng = MersenneTwister(99) dim, res_dim, T_steps, steps = 2, 16, 20, 5 diff --git a/test/Extensions/lsm_tests.jl b/test/Extensions/lsm_tests.jl index d7b6847d0..9d5df34fe 100644 --- a/test/Extensions/lsm_tests.jl +++ b/test/Extensions/lsm_tests.jl @@ -499,6 +499,40 @@ begin @test lsm.reservoir.feature_map isa ExponentialSpikeFilter end + @testset "resetcarry! with init_carry" begin + n, n_win = 6, 5 + lif = LIFNeuron(; + tau_m = 0.02, r_m = 1.0, v_rest = 0.0, v_th = 10.0, + v_reset = 0.0, tau_ref = 0.002, tau_syn = 0.005, + ) + build = init_state -> _lsm_f64( + 1, n, 1, (0.0, 0.05), Tsit5(); neuron = lif, init_state, + feature_map = MembraneVoltageFeature(), reltol = 1.0e-10, abstol = 1.0e-12, + ) + const03 = (rng, d...) -> fill(0.3, d...) + lsm = build(f64_zeros) + lsm03 = build(const03) + ps, st0 = setup(MersenneTwister(5), lsm) + ps03, st03 = setup(MersenneTwister(5), lsm03) + data = zeros(Float64, 1, n_win) + run = (m, p, st) -> first(collectstates(m, data, p, deepcopy(st))) + base = run(lsm, ps, st0) + ref03 = run(lsm03, ps03, st03) + @test ref03 != base + _, st1 = collectstates(lsm, data, ps, deepcopy(st0)) + for st in (st0, st1) + st_zero = resetcarry!(MersenneTwister(3), lsm, deepcopy(st); init_carry = f64_zeros) + @test length(first(st_zero.reservoir.carry)) == 2n + @test run(lsm, ps, st_zero) == base + st_const = resetcarry!(MersenneTwister(3), lsm, deepcopy(st); init_carry = const03) + u = first(st_const.reservoir.carry) + @test u == vcat(fill(0.3, n), zeros(n)) + @test run(lsm, ps, st_const) == ref03 + st_clear = resetcarry!(MersenneTwister(3), lsm, deepcopy(st)) + @test run(lsm, ps, st_clear) == base + end + end + @testset "init_state sets membrane IC" begin n = 4 tau_m, V0, T_end, n_win = 0.02, 0.35, 0.05, 5 diff --git a/test/Extensions/ode_reservoir_ext_tests.jl b/test/Extensions/ode_reservoir_ext_tests.jl index ded07ee39..a4f88de69 100644 --- a/test/Extensions/ode_reservoir_ext_tests.jl +++ b/test/Extensions/ode_reservoir_ext_tests.jl @@ -442,6 +442,41 @@ begin # 12. Non-NamedTuple `prob.p` rejected with a clear error # --------------------------------------------------------------------------- + @testset "SciMLProblemReservoir: resetcarry! with init_carry" begin + n, T_steps = 3, 6 + function decay_rhs!(dx, x, p, t) + return dx .= .-x .+ sum(p.input(t)) + end + build = u0 -> ReservoirComputer( + SciMLProblemReservoir( + ODEProblem(decay_rhs!, u0, (0.0, 1.0), (;)), + TerminalStateSampling(), (0.0, 1.0), Tsit5(), + ), + LinearReadout(n => 1), + ) + rc = build(zeros(n)) + rc03 = build(fill(0.3, n)) + ps, st0 = setup(MersenneTwister(0), rc) + ps03, st03 = setup(MersenneTwister(0), rc03) + data = ones(Float64, 1, T_steps) + const03 = (rng, d...) -> fill(0.3, d...) + run = (m, p, st) -> first(collectstates(m, data, p, deepcopy(st))) + base = run(rc, ps, st0) + ref03 = run(rc03, ps03, st03) + @test ref03 != base + _, st1 = collectstates(rc, data, ps, deepcopy(st0)) + for st in (st0, st1) + st_zero = resetcarry!(MersenneTwister(1), rc, deepcopy(st); init_carry = zeros32) + @test first(st_zero.reservoir.carry) isa Vector{Float64} + @test length(first(st_zero.reservoir.carry)) == n + @test run(rc, ps, st_zero) == base + st_const = resetcarry!(MersenneTwister(1), rc, deepcopy(st); init_carry = const03) + @test run(rc, ps, st_const) == ref03 + st_clear = resetcarry!(MersenneTwister(1), rc, deepcopy(st)) + @test run(rc, ps, st_clear) == base + end + end + @testset "Non-NamedTuple prob.p errors clearly" begin function rhs!(dx, x, p, t) return dx .= .-x diff --git a/test/Layers/lif_tests.jl b/test/Layers/lif_tests.jl index 351a9c7b2..971a059b6 100644 --- a/test/Layers/lif_tests.jl +++ b/test/Layers/lif_tests.jl @@ -279,7 +279,7 @@ begin end end - @testset "lazy initialization does not advance the RNG of the input state" begin + @testset "lazy initialization does not advance the RNG of the state" begin x = Float32[0.3] for make_rng in _RNG_MAKERS, ctor in cell_ctors @@ -298,7 +298,7 @@ begin @test st_1.input_buffer == st_2.input_buffer @test st_1.start_states == st_2.start_states @test st_1.rng == st_2.rng - @test st_1.rng !== st.rng + @test st_1.rng === st.rng (_, _), st_3 = lif(x, ps, st_1) @test st_3.rng === st_1.rng @@ -358,7 +358,7 @@ begin @test old_layout.rng == rng_before @test haskey(st_old, :start_states) @test st_old.start_states == start_states - @test st_old.rng == rng_after + @test st_old.rng == rng_before @test st_old.input_buffer == st_new.input_buffer @test y_old == y_new @test carry_old == carry_new diff --git a/test/Layers/rmncell_tests.jl b/test/Layers/rmncell_tests.jl index d99e55cfa..fcb739b22 100644 --- a/test/Layers/rmncell_tests.jl +++ b/test/Layers/rmncell_tests.jl @@ -4,7 +4,8 @@ begin using Random using LinearAlgebra using ReservoirComputing - using LuxCore: initialparameters, initialstates + using LuxCore: initialparameters, initialstates, setup + using WeightInitializers: zeros32 using Static const _I32 = (m, n) -> Matrix{Float32}(I, m, n) @@ -362,4 +363,32 @@ begin end end + @testset "resetcarry! keeps the RMNCell carry structure" begin + for Model in (RMNESN, RMNResESN) + mdl = Model( + 3, 12, 20, 2, tanh; init_state = zeros32, init_memory_state = zeros32, + ) + ps, st = setup(MersenneTwister(7), mdl) + x = rand(MersenneTwister(8), Float32, 3, 1) + carry_dims = carry -> map(state -> size(state, 1), carry) + + st_pre = resetcarry!(MersenneTwister(1), mdl, st; init_carry = zeros32) + @test carry_dims(st_pre.reservoir.carry) == (20, 12) + y_pre, _ = mdl(x, ps, st_pre) + @test size(y_pre) == (2, 1) + + _, st1 = mdl(x, ps, st) + st_post = resetcarry!(MersenneTwister(1), mdl, st1; init_carry = zeros32) + @test carry_dims(st_post.reservoir.carry) == (20, 12) + y_post, _ = mdl(x, ps, st_post) + @test size(y_post) == (2, 1) + + st_clear = resetcarry!(MersenneTwister(1), mdl, st1) + @test st_clear.reservoir.carry === nothing + y_clear, _ = mdl(x, ps, st_clear) + y_fresh, _ = mdl(x, ps, st) + @test y_clear == y_fresh + end + end + end diff --git a/test/Models/deep_reservoir_tests.jl b/test/Models/deep_reservoir_tests.jl index a3986f422..fea6ed92e 100644 --- a/test/Models/deep_reservoir_tests.jl +++ b/test/Models/deep_reservoir_tests.jl @@ -185,4 +185,100 @@ begin @test length(trained_st.cells) == 1 end end + + @testset "resetcarry! resets DelayLayers among the per-layer modifiers" begin + res_dims = 12 + X1 = randn(MersenneTwister(5), Float32, 3, 10) + X2 = randn(MersenneTwister(6), Float32, 3, 10) + ones_init = (rng, dims...) -> ones(Float32, dims...) + models = ( + DeepReservoir( + ( + ESNCell(3 => res_dims; init_state = zeros32), + ESNCell(res_dims => res_dims; init_state = zeros32), + ), + LinearReadout(3 * res_dims => 2); + state_modifiers = (nothing, (DelayLayer(res_dims; num_delays = 2),)), + ), + ) + for m in models + ps, st0 = setup(MersenneTwister(11), m) + run = st -> first(collectstates(m, X2, ps, deepcopy(st))) + ref = run(st0) + _, st1 = collectstates(m, X1, ps, deepcopy(st0)) + @test run(st1) != ref + @test run(resetcarry!(MersenneTwister(1), m, deepcopy(st1))) == ref + st_zero = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = (nothing, zeros32) + ) + @test run(st_zero) == ref + st_ones = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = (nothing, ones_init) + ) + delay_st = first(st_ones.state_modifiers[2]) + delay_st = haskey(delay_st, :op) ? delay_st.op : delay_st + @test all(==(1), delay_st.history) + @test delay_st.clock == 0 + @test run(st_ones) != ref + st_skip = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = (nothing, nothing) + ) + @test run(st_skip) == ref + end + end + + @testset "resetcarry! restarts LIFESNCell layers" begin + res_dims = 12 + X1 = randn(MersenneTwister(5), Float32, 3, 10) + X2 = randn(MersenneTwister(6), Float32, 3, 10) + ones_init = (rng, dims...) -> ones(Float32, dims...) + for init_state in (zeros32, randn32) + lif(in_dims) = LIFESNCell( + in_dims => res_dims; lookback_horizon = 3, init_state = init_state + ) + esn(in_dims) = ESNCell(in_dims => res_dims; init_state = zeros32) + stacks = ( + ((lif(3),), 1), + ((esn(3), lif(res_dims)), 2), + ((lif(3), esn(res_dims)), 1), + ) + for (cells, lif_idx) in stacks + n = length(cells) + m = DeepReservoir(cells, LinearReadout(n * res_dims => 2)) + ps, st0 = setup(MersenneTwister(11), m) + run = st -> first(collectstates(m, X2, ps, deepcopy(st))) + ref = run(st0) + _, st1 = collectstates(m, X1, ps, deepcopy(st0)) + @test run(st1) != ref + + st_nothing = resetcarry!(MersenneTwister(1), m, deepcopy(st1)) + @test st_nothing.cells[lif_idx].cell.start_states === nothing + @test st_nothing.cells[lif_idx].cell.input_buffer === nothing + @test run(st_nothing) == ref + + if init_state === zeros32 + st_zero = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = zeros32 + ) + @test run(st_zero) == ref + + per_layer = ntuple(i -> i == lif_idx ? zeros32 : nothing, n) + st_layer = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = per_layer + ) + @test run(st_layer) == ref + + per_layer_ones = ntuple(i -> i == lif_idx ? ones_init : nothing, n) + st_ones = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = per_layer_ones + ) + lif_st = st_ones.cells[lif_idx] + @test all(==(1), first(lif_st.cell.start_states)) + @test lif_st.cell.start_states == lif_st.carry + @test run(st_ones) != ref + end + end + end + end + end diff --git a/test/Models/esn_deep_tests.jl b/test/Models/esn_deep_tests.jl index d087a4816..d1dafc326 100644 --- a/test/Models/esn_deep_tests.jl +++ b/test/Models/esn_deep_tests.jl @@ -271,4 +271,47 @@ begin end end + @testset "resetcarry! resets DelayLayers among the per-layer modifiers" begin + res_dims = 12 + X1 = randn(MersenneTwister(5), Float32, 3, 10) + X2 = randn(MersenneTwister(6), Float32, 3, 10) + ones_init = (rng, dims...) -> ones(Float32, dims...) + models = ( + DeepESN( + 3, [res_dims, res_dims], 2; init_state = zeros32, + state_modifiers = (nothing, DelayLayer(res_dims; num_delays = 2)), + readout_in_dims = 3 * res_dims, + ), + DeepESN( + 3, [res_dims, res_dims], 2; init_state = zeros32, + state_modifiers = (nothing, Extend(DelayLayer(res_dims; num_delays = 2))), + readout_in_dims = 4 * res_dims, + ), + ) + for m in models + ps, st0 = setup(MersenneTwister(11), m) + run = st -> first(collectstates(m, X2, ps, deepcopy(st))) + ref = run(st0) + _, st1 = collectstates(m, X1, ps, deepcopy(st0)) + @test run(st1) != ref + @test run(resetcarry!(MersenneTwister(1), m, deepcopy(st1))) == ref + st_zero = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = (nothing, zeros32) + ) + @test run(st_zero) == ref + st_ones = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = (nothing, ones_init) + ) + delay_st = first(st_ones.state_modifiers[2]) + delay_st = haskey(delay_st, :op) ? delay_st.op : delay_st + @test all(==(1), delay_st.history) + @test delay_st.clock == 0 + @test run(st_ones) != ref + st_skip = resetcarry!( + MersenneTwister(1), m, deepcopy(st1); init_carry = (nothing, nothing) + ) + @test run(st_skip) == ref + end + end + end diff --git a/test/Models/esn_delay_tests.jl b/test/Models/esn_delay_tests.jl index a54ae79a7..c2ee37176 100644 --- a/test/Models/esn_delay_tests.jl +++ b/test/Models/esn_delay_tests.jl @@ -351,6 +351,85 @@ begin end end + @testset "resetcarry! restores the delay state" begin + res_dims = 12 + delay = DelayLayer(res_dims; num_delays = 2) + models = ( + InputDelayESN(3, res_dims, 2; num_delays = 2, init_state = zeros32), + StateDelayESN(3, res_dims, 2; num_delays = 2, init_state = zeros32), + DelayESN( + 3, res_dims, 2; num_input_delays = 2, num_state_delays = 2, + init_state = zeros32, + ), + ESN( + 3, res_dims, 2; init_state = zeros32, state_modifiers = (delay,), + readout_in_dims = 3 * res_dims, + ), + ) + X1 = randn(MersenneTwister(5), Float32, 3, 10) + X2 = randn(MersenneTwister(6), Float32, 3, 10) + ones_init = (rng, dims...) -> ones(Float32, dims...) + delay_states = (m, st) -> filter( + s -> haskey(s, :history), + [haskey(st, :input_delay) ? st.input_delay : (;), st.state_modifiers...], + ) + for m in models + ps, st0 = setup(MersenneTwister(11), m) + run = st -> first(collectstates(m, X2, ps, deepcopy(st))) + ref = run(st0) + _, st1 = collectstates(m, X1, ps, deepcopy(st0)) + @test run(st1) != ref + @test run(resetcarry!(MersenneTwister(1), m, deepcopy(st1))) == ref + st_zero = resetcarry!(MersenneTwister(1), m, deepcopy(st1); init_carry = zeros32) + @test run(st_zero) == ref + st_ones = resetcarry!(MersenneTwister(1), m, deepcopy(st1); init_carry = ones_init) + for s in delay_states(m, st_ones) + @test all(==(1), s.history) + @test s.clock == 0 + end + @test st_ones.readout == st1.readout + @test run(st_ones) != ref + _, st_ps = resetcarry!(MersenneTwister(1), m, ps, deepcopy(st1)) + @test run(st_ps) == ref + end + end + + @testset "resetcarry! resets DelayLayers inside wrapper modifiers" begin + res_dims = 12 + models = ( + ESN( + 3, res_dims, 2; init_state = zeros32, + state_modifiers = (Extend(DelayLayer(res_dims; num_delays = 2)),), + readout_in_dims = 4 * res_dims, + ), + ESN( + 3, res_dims, 2; init_state = zeros32, + state_modifiers = (ReservoirChain(DelayLayer(res_dims; num_delays = 2)),), + readout_in_dims = 3 * res_dims, + ), + ) + X1 = randn(MersenneTwister(5), Float32, 3, 10) + X2 = randn(MersenneTwister(6), Float32, 3, 10) + ones_init = (rng, dims...) -> ones(Float32, dims...) + histories = st -> st isa NamedTuple && haskey(st, :history) ? [st] : + reduce(vcat, map(histories, collect(values(st))); init = Any[]) + for m in models + ps, st0 = setup(MersenneTwister(11), m) + run = st -> first(collectstates(m, X2, ps, deepcopy(st))) + ref = run(st0) + _, st1 = collectstates(m, X1, ps, deepcopy(st0)) + @test run(st1) != ref + @test run(resetcarry!(MersenneTwister(1), m, deepcopy(st1))) == ref + st_zero = resetcarry!(MersenneTwister(1), m, deepcopy(st1); init_carry = zeros32) + @test run(st_zero) == ref + st_ones = resetcarry!(MersenneTwister(1), m, deepcopy(st1); init_carry = ones_init) + found = histories(st_ones.state_modifiers) + @test length(found) == 1 + @test all(h -> all(==(1), h.history) && h.clock == 0, found) + @test run(st_ones) != ref + end + end + @testset "StateDelayESN forward pass & training" begin rng = MersenneTwister(123) in_dims, res_dims, out_dims, num_delays = 3, 40, 2, 2 diff --git a/test/Models/lifesn_tests.jl b/test/Models/lifesn_tests.jl index c51a517fe..c4eafd12c 100644 --- a/test/Models/lifesn_tests.jl +++ b/test/Models/lifesn_tests.jl @@ -329,4 +329,57 @@ begin @test length(first(st3.reservoir.carry)) == res_dims end + @testset "LIFESN: resetcarry! restarts the window replay" begin + const_state = (rng, m, B) -> fill(0.3f0, m, B) + X1 = randn(MersenneTwister(1), Float32, 3, 12) + X2 = randn(MersenneTwister(2), Float32, 3, 10) + build(init_state; kwargs...) = LIFESN( + 3, 20, 3, tanh; lookback_horizon = 3, init_state = init_state, kwargs... + ) + run_seq(model, ps, st, mode) = mode == :teacher ? predict(model, X2, ps, st) : + predict(model, 7, ps, st; initialdata = X2[:, 1]) + + for mode in (:teacher, :autoregressive), init_state in (zeros32, const_state, randn32) + model = build(init_state) + ps, st = setup(MersenneTwister(7), model) + expected, _ = run_seq(model, ps, deepcopy(st), mode) + _, st_used = predict(model, X1, ps, deepcopy(st)) + + for st_in in (deepcopy(st), st_used) + st_reset = resetcarry!(MersenneTwister(7), model, deepcopy(st_in)) + @test st_reset.reservoir.cell.start_states === nothing + @test st_reset.reservoir.cell.input_buffer === nothing + @test first(run_seq(model, ps, st_reset, mode)) == expected + end + + init_state === randn32 && continue + + other = build(randn32) + ps_o, st_o = setup(MersenneTwister(7), other) + ref_model = build(init_state) + expected_f, _ = run_seq(ref_model, ps_o, deepcopy(st), mode) + _, st_o_used = predict(other, X1, ps_o, deepcopy(st_o)) + st_f = resetcarry!( + MersenneTwister(7), other, st_o_used; init_carry = init_state + ) + @test first(st_f.reservoir.cell.start_states) == st_f.reservoir.carry[1] + @test first(run_seq(other, ps_o, st_f, mode)) == expected_f + end + end + + @testset "LIFESN: resetcarry! keeps readout states and resets delays" begin + model = LIFESN( + 3, 20, 3, tanh; lookback_horizon = 3, init_state = zeros32, + state_modifiers = (DelayLayer(20; num_delays = 2),), + readout_in_dims = 60, + ) + X = randn(MersenneTwister(1), Float32, 3, 10) + ps, st = setup(MersenneTwister(7), model) + expected, _ = predict(model, X, ps, deepcopy(st)) + _, st_used = predict(model, X, ps, deepcopy(st)) + st_reset = resetcarry!(MersenneTwister(7), model, deepcopy(st_used)) + @test st_reset.readout == st_used.readout + @test first(predict(model, X, ps, st_reset)) == expected + end + end diff --git a/test/Models/ngrc_tests.jl b/test/Models/ngrc_tests.jl index d80683262..5bf3d6fac 100644 --- a/test/Models/ngrc_tests.jl +++ b/test/Models/ngrc_tests.jl @@ -100,6 +100,74 @@ begin @test propertynames(st2) == propertynames(st) end + @testset "resetcarry! restores the initial delay state" begin + ngrc = NGRC( + 2, 1; num_delays = 2, features = (square_feature,), include_input = True(), + ) + ps, st = setup(MersenneTwister(2), ngrc) + X = rand(MersenneTwister(3), Float32, 2, 30) + Y = rand(MersenneTwister(4), Float32, 1, 30) + ps_tr, st_tr = train( + ngrc, X, Y, ps, st; objective = RidgeRegression(1.0e-4), + ) + Xt = rand(MersenneTwister(5), Float32, 2, 10) + _, st_new = setup(MersenneTwister(2), ngrc) + st_fresh = merge(st_tr, (reservoir = st_new.reservoir,)) + y_fresh, _ = predict(ngrc, Xt, ps_tr, st_fresh) + + y_carried, _ = predict(ngrc, Xt, ps_tr, deepcopy(st_tr)) + @test y_carried != y_fresh + + st_reset = resetcarry!(rng, ngrc, deepcopy(st_tr)) + @test st_reset.reservoir.history === nothing + @test st_reset.reservoir.clock == 0 + @test st_reset.readout == st_tr.readout + y_reset, _ = predict(ngrc, Xt, ps_tr, st_reset) + @test y_reset == y_fresh + + st_zero = resetcarry!(rng, ngrc, deepcopy(st_tr); init_carry = zeros32) + y_zero, _ = predict(ngrc, Xt, ps_tr, st_zero) + @test y_zero == y_fresh + + ones_init = (rng, dims...) -> ones(Float32, dims...) + st_ones = resetcarry!(rng, ngrc, deepcopy(st_tr); init_carry = ones_init) + @test st_ones.reservoir.history == ones(Float32, 2, 2) + @test st_ones.reservoir.clock == 0 + _, st_ones_ps = resetcarry!(rng, ngrc, ps_tr, st_tr; init_carry = ones_init) + @test st_ones_ps.reservoir.history == ones(Float32, 2, 2) + y_ones, _ = predict(ngrc, Xt, ps_tr, st_ones) + @test size(y_ones) == size(y_fresh) + end + + @testset "resetcarry! resets DelayLayers among the state modifiers" begin + ngrc = NGRC( + 2, 1; num_delays = 2, include_input = True(), + state_modifiers = (DelayLayer(6; num_delays = 2),), + ) + ps, st0 = setup(MersenneTwister(2), ngrc) + X1 = rand(MersenneTwister(3), Float32, 2, 10) + X2 = rand(MersenneTwister(4), Float32, 2, 10) + run = st -> first(predict(ngrc, X2, ps, deepcopy(st))) + ref = run(st0) + _, st1 = predict(ngrc, X1, ps, deepcopy(st0)) + @test run(st1) != ref + + st_reset = resetcarry!(rng, ngrc, deepcopy(st1)) + @test st_reset.state_modifiers[end].history === nothing + @test st_reset.state_modifiers[end].clock == 0 + @test run(st_reset) == ref + + st_zero = resetcarry!(rng, ngrc, deepcopy(st1); init_carry = zeros32) + @test run(st_zero) == ref + + ones_init = (rng, dims...) -> ones(Float32, dims...) + st_ones = resetcarry!(rng, ngrc, deepcopy(st1); init_carry = ones_init) + @test st_ones.state_modifiers[end].history == ones(Float32, 6, 2) + @test st_ones.state_modifiers[end].clock == 0 + @test st_ones.readout == st1.readout + @test run(st_ones) != ref + end + @testset "simple 1D linear system learning" begin # x_{t+1} = a * x_t with a = 0.8 a = 0.8f0