From 8681d1dcb3ed46ea1c0cbcea74ce527c9e90cf10 Mon Sep 17 00:00:00 2001 From: yus100 <90424373+yus100@users.noreply.github.com> Date: Tue, 16 Jun 2026 22:26:50 -0700 Subject: [PATCH] fix(graph): apply SIGMOID/TANH activations in MoE layer The MoE layer's activation switch only handled GELU, GELU_ERF, RELU and SILU, with SIGMOID and TANH silently falling through to the SILU default. Both activations are valid `Activation` enum values accepted by the `moe_layer` builder and both kernels (cactus_sigmoid_f16 / cactus_tanh_f16) already exist, so a graph built with either produced wrong results. Add the two missing cases and a regression test that runs an identity-weight single-expert MoE layer for every activation and checks the output matches the corresponding standalone activation kernel. Signed-off-by: yus100 <90424373+yus100@users.noreply.github.com> Co-Authored-By: Claude Opus 4.8 (1M context) --- cactus-graph/src/ops_nn.cpp | 6 ++++ cactus-graph/tests/test_nn.cpp | 61 ++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/cactus-graph/src/ops_nn.cpp b/cactus-graph/src/ops_nn.cpp index 23afc2f20..d2c84d198 100644 --- a/cactus-graph/src/ops_nn.cpp +++ b/cactus-graph/src/ops_nn.cpp @@ -265,6 +265,12 @@ void compute_moe_layer_node(GraphNode& node, const std::vector hidden = {(__fp16)-2.0f, (__fp16)-0.5f, (__fp16)0.5f, (__fp16)2.0f}; + + std::vector<__fp16> identity(H * H, (__fp16)0.0f); + for (size_t i = 0; i < H; ++i) identity[i * H + i] = (__fp16)1.0f; + + std::vector<__fp16> routing = {(__fp16)1.0f}; + std::vector topk = {0.0f}; + + const std::vector activations = { + Activation::SILU, Activation::GELU, Activation::GELU_ERF, + Activation::RELU, Activation::SIGMOID, Activation::TANH, + }; + + for (Activation act : activations) { + CactusGraph g; + size_t hidden_id = g.input({1, H}, Precision::FP16); + size_t routing_id = g.input({1, 1}, Precision::FP16); + size_t topk_id = g.input({1, 1}, Precision::FP32); + size_t w1_id = g.input({H, H}, Precision::FP16); + size_t w2_id = g.input({H, H}, Precision::FP16); + + size_t out = g.moe_layer(hidden_id, routing_id, topk_id, + {w1_id}, {w2_id}, + 1, 1, false, 1e-6f, 1.0f, act); + + g.set_input(hidden_id, hidden.data(), Precision::FP16); + g.set_input(routing_id, routing.data(), Precision::FP16); + g.set_input(topk_id, topk.data(), Precision::FP32); + g.set_input(w1_id, identity.data(), Precision::FP16); + g.set_input(w2_id, identity.data(), Precision::FP16); + g.execute(); + + std::vector<__fp16> expected(H); + apply_activation_reference(act, hidden.data(), expected.data(), H); + + __fp16* result = static_cast<__fp16*>(g.get_output(out)); + bool ok = TestUtils::compare_arrays(result, expected.data(), H, 0.02f); + g.hard_reset(); + if (!ok) return false; + } + return true; +} + bool run_benchmarks() { auto bench = [](const char* label, auto setup, auto run) { setup(); @@ -696,6 +756,7 @@ int main() { runner.run_test("Min/Max Operations", test_min_max_operations()); runner.run_test("STFT Complex", test_stft()); runner.run_test("LayerNorm", test_layernorm()); + runner.run_test("MoE Activations", test_moe_activations()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); runner.print_summary();