diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 4426b6667..02e7345da 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -541,19 +541,17 @@ function initialstates(rng::AbstractRNG, l::NonlinearFeaturesLayer) end function (nfl::NonlinearFeaturesLayer)(inp::AbstractVector, ps, st) - feature_vector = Any[] - if known(nfl.include_input) === true - push!(feature_vector, inp) - end - for func in nfl.features - nonlin_feat = func(inp) - push!(feature_vector, nonlin_feat) - end - out = reduce(vcat, feature_vector) + feats = map(func -> func(inp), nfl.features) + out = known(nfl.include_input) === true ? reduce(vcat, feats; init = inp) : reduce(vcat, feats) return out, st end +function (nfl::NonlinearFeaturesLayer)(inp::AbstractMatrix, ps, st) + cols = map(col -> first(nfl(col, ps, st)), eachcol(inp)) + return reduce(hcat, cols), st +end + function Base.show(io::IO, nfl::NonlinearFeaturesLayer) print(io, "NonlinearFeaturesLayer(") diff --git a/src/layers/eusn_cell.jl b/src/layers/eusn_cell.jl index 90860df19..a8b521339 100644 --- a/src/layers/eusn_cell.jl +++ b/src/layers/eusn_cell.jl @@ -100,9 +100,9 @@ function (esn::EuSNCell)((inp, (hidden_state,))::InputType, ps, st::NamedTuple) bias = safe_getproperty(ps, Val(:bias)) t_lc = T(esn.leak_coefficient) t_diff = T(esn.diffusion) - asynm_matrix = compute_asym_recurrent(ps.reservoir_matrix, t_diff) - win_inp = dense_bias(ps.input_matrix, inp, nothing) - w_state = dense_bias(asynm_matrix, hidden_state, bias) + win_inp = dense_bias(ps.input_matrix, inp, bias) + w_state = dense_bias(ps.reservoir_matrix, hidden_state, nothing) .- + transpose(ps.reservoir_matrix) * hidden_state .- t_diff .* hidden_state candidate_h = esn.activation.(win_inp .+ w_state) h_new = hidden_state .+ t_lc .* candidate_h return (h_new, (h_new,)), st @@ -120,8 +120,3 @@ function Base.show(io::IO, esn::EuSNCell) has_bias(esn) && print(io, ", use_bias=true") return print(io, ")") end - -function compute_asym_recurrent(weight_hh::AbstractMatrix, gamma::AbstractFloat) - return weight_hh .- transpose(weight_hh) .- - gamma .* Matrix{eltype(weight_hh)}(I, size(weight_hh, 1), size(weight_hh, 1)) -end diff --git a/test/Layers/esncell_tests.jl b/test/Layers/esncell_tests.jl index b48e82270..c6688f5fc 100644 --- a/test/Layers/esncell_tests.jl +++ b/test/Layers/esncell_tests.jl @@ -401,6 +401,35 @@ begin end end + @testset "EuSNCell: matches (W - Wᵀ - γI) h + b formula" begin + for use_bias in (false, true) + cell = EuSNCell(3 => 7; use_bias, diffusion = 0.3, leak_coefficient = 0.4) + ps = initialparameters(MersenneTwister(5), cell) + if use_bias + ps = merge(ps, (; bias = rand(MersenneTwister(6), Float32, 7))) + end + x = rand(MersenneTwister(7), Float32, 3) + h = rand(MersenneTwister(8), Float32, 7) + (y, _), _ = cell((x, (h,)), ps, NamedTuple()) + W = ps.reservoir_matrix + A = W .- transpose(W) .- 0.3f0 .* Matrix{Float32}(I, 7, 7) + pre = ps.input_matrix * x .+ A * h + use_bias && (pre = pre .+ ps.bias) + @test y ≈ h .+ 0.4f0 .* tanh.(pre) rtol = 1.0f-5 + end + end + + @testset "EuSNCell: bias is not absorbed by the recurrent term" begin + cell = EuSNCell( + 1 => 1, identity; use_bias = true, diffusion = 1.0, leak_coefficient = 1.0, + init_input = _W_ZZ, init_reservoir = (rng, m, n) -> Float32[1.0f8;;] + ) + ps = initialparameters(MersenneTwister(0), cell) + ps = merge(ps, (; bias = Float32[1])) + (y, _), _ = cell((Float32[0], (Float32[1],)), ps, NamedTuple()) + @test y == Float32[1] + end + @testset "EuSN: the default initial state is zero" begin cell = EuSNCell(3 => 30) ps = initialparameters(MersenneTwister(0), cell) diff --git a/test/basic_tests.jl b/test/basic_tests.jl index 1fc4396ba..9a4eff9b8 100644 --- a/test/basic_tests.jl +++ b/test/basic_tests.jl @@ -382,6 +382,34 @@ begin @test ii_id === true end + @testset "matrix input applies features per column" begin + kr = x -> kron(x, x) + nfl = NonlinearFeaturesLayer(kr, x -> x .^ 2) + X = rand(MersenneTwister(1), Float32, 3, 4) + Y, _ = nfl(X, NamedTuple(), NamedTuple()) + @test size(Y) == (3 + 9 + 3, 4) + for j in 1:4 + @test Y[:, j] == first(nfl(X[:, j], NamedTuple(), NamedTuple())) + end + Y1, _ = nfl(X[:, 1:1], NamedTuple(), NamedTuple()) + @test Y1 == Y[:, 1:1] + end + + @testset "inside a ReservoirChain after a reservoir cell" begin + chain = ReservoirChain( + StatefulLayer(ESNCell(3 => 5)), + NonlinearFeaturesLayer(x -> x .^ 2), + LinearReadout(10 => 2) + ) + ps, st = setup(MersenneTwister(2), chain) + X = rand(MersenneTwister(3), Float32, 3, 12) + states, _ = collectstates(chain, X, ps, st) + @test size(states) == (10, 12) + (ps_t, st_t) = train(chain, X, X[1:2, :], ps, st; objective = RidgeRegression(Float32, 1.0f-6)) + out, _ = predict(chain, X, ps_t, st_t) + @test size(out) == (2, 12) + end + @testset "initialparameters / initialstates" begin sq = x -> x .^ 2 nfl = NonlinearFeaturesLayer(sq; include_input = true)