Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions src/layers/basic.jl
Original file line number Diff line number Diff line change
Expand Up @@ -541,19 +541,17 @@ function initialstates(rng::AbstractRNG, l::NonlinearFeaturesLayer)
end

function (nfl::NonlinearFeaturesLayer)(inp::AbstractVector, ps, st)
feature_vector = Any[]
if known(nfl.include_input) === true
push!(feature_vector, inp)
end
for func in nfl.features
nonlin_feat = func(inp)
push!(feature_vector, nonlin_feat)
end
out = reduce(vcat, feature_vector)
feats = map(func -> func(inp), nfl.features)
out = known(nfl.include_input) === true ? reduce(vcat, feats; init = inp) : reduce(vcat, feats)

return out, st
end

function (nfl::NonlinearFeaturesLayer)(inp::AbstractMatrix, ps, st)
cols = map(col -> first(nfl(col, ps, st)), eachcol(inp))
return reduce(hcat, cols), st
end

function Base.show(io::IO, nfl::NonlinearFeaturesLayer)
print(io, "NonlinearFeaturesLayer(")

Expand Down
11 changes: 3 additions & 8 deletions src/layers/eusn_cell.jl
Original file line number Diff line number Diff line change
Expand Up @@ -100,9 +100,9 @@ function (esn::EuSNCell)((inp, (hidden_state,))::InputType, ps, st::NamedTuple)
bias = safe_getproperty(ps, Val(:bias))
t_lc = T(esn.leak_coefficient)
t_diff = T(esn.diffusion)
asynm_matrix = compute_asym_recurrent(ps.reservoir_matrix, t_diff)
win_inp = dense_bias(ps.input_matrix, inp, nothing)
w_state = dense_bias(asynm_matrix, hidden_state, bias)
win_inp = dense_bias(ps.input_matrix, inp, bias)
w_state = dense_bias(ps.reservoir_matrix, hidden_state, nothing) .-
transpose(ps.reservoir_matrix) * hidden_state .- t_diff .* hidden_state
candidate_h = esn.activation.(win_inp .+ w_state)
h_new = hidden_state .+ t_lc .* candidate_h
return (h_new, (h_new,)), st
Expand All @@ -120,8 +120,3 @@ function Base.show(io::IO, esn::EuSNCell)
has_bias(esn) && print(io, ", use_bias=true")
return print(io, ")")
end

function compute_asym_recurrent(weight_hh::AbstractMatrix, gamma::AbstractFloat)
return weight_hh .- transpose(weight_hh) .-
gamma .* Matrix{eltype(weight_hh)}(I, size(weight_hh, 1), size(weight_hh, 1))
end
29 changes: 29 additions & 0 deletions test/Layers/esncell_tests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,35 @@ begin
end
end

@testset "EuSNCell: matches (W - Wᵀ - γI) h + b formula" begin
for use_bias in (false, true)
cell = EuSNCell(3 => 7; use_bias, diffusion = 0.3, leak_coefficient = 0.4)
ps = initialparameters(MersenneTwister(5), cell)
if use_bias
ps = merge(ps, (; bias = rand(MersenneTwister(6), Float32, 7)))
end
x = rand(MersenneTwister(7), Float32, 3)
h = rand(MersenneTwister(8), Float32, 7)
(y, _), _ = cell((x, (h,)), ps, NamedTuple())
W = ps.reservoir_matrix
A = W .- transpose(W) .- 0.3f0 .* Matrix{Float32}(I, 7, 7)
pre = ps.input_matrix * x .+ A * h
use_bias && (pre = pre .+ ps.bias)
@test y ≈ h .+ 0.4f0 .* tanh.(pre) rtol = 1.0f-5
end
end

@testset "EuSNCell: bias is not absorbed by the recurrent term" begin
cell = EuSNCell(
1 => 1, identity; use_bias = true, diffusion = 1.0, leak_coefficient = 1.0,
init_input = _W_ZZ, init_reservoir = (rng, m, n) -> Float32[1.0f8;;]
)
ps = initialparameters(MersenneTwister(0), cell)
ps = merge(ps, (; bias = Float32[1]))
(y, _), _ = cell((Float32[0], (Float32[1],)), ps, NamedTuple())
@test y == Float32[1]
end

@testset "EuSN: the default initial state is zero" begin
cell = EuSNCell(3 => 30)
ps = initialparameters(MersenneTwister(0), cell)
Expand Down
28 changes: 28 additions & 0 deletions test/basic_tests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,34 @@ begin
@test ii_id === true
end

@testset "matrix input applies features per column" begin
kr = x -> kron(x, x)
nfl = NonlinearFeaturesLayer(kr, x -> x .^ 2)
X = rand(MersenneTwister(1), Float32, 3, 4)
Y, _ = nfl(X, NamedTuple(), NamedTuple())
@test size(Y) == (3 + 9 + 3, 4)
for j in 1:4
@test Y[:, j] == first(nfl(X[:, j], NamedTuple(), NamedTuple()))
end
Y1, _ = nfl(X[:, 1:1], NamedTuple(), NamedTuple())
@test Y1 == Y[:, 1:1]
end

@testset "inside a ReservoirChain after a reservoir cell" begin
chain = ReservoirChain(
StatefulLayer(ESNCell(3 => 5)),
NonlinearFeaturesLayer(x -> x .^ 2),
LinearReadout(10 => 2)
)
ps, st = setup(MersenneTwister(2), chain)
X = rand(MersenneTwister(3), Float32, 3, 12)
states, _ = collectstates(chain, X, ps, st)
@test size(states) == (10, 12)
(ps_t, st_t) = train(chain, X, X[1:2, :], ps, st; objective = RidgeRegression(Float32, 1.0f-6))
out, _ = predict(chain, X, ps_t, st_t)
@test size(out) == (2, 12)
end

@testset "initialparameters / initialstates" begin
sq = x -> x .^ 2
nfl = NonlinearFeaturesLayer(sq; include_input = true)
Expand Down
Loading