diff --git a/common/common.h b/common/common.h
index 431424304011..23fc54fedb81 100644
--- a/common/common.h
+++ b/common/common.h
@@ -434,7 +434,7 @@ struct common_params_speculative {
     }
     uint32_t need_n_rs_seq() const {
         bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
-            return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
+            return t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
         });
 
         return needs_rs_seq ? draft.n_max : 0u;
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
index 26f31232f676..b8b01b2b8920 100644
--- a/ggml/include/ggml.h
+++ b/ggml/include/ggml.h
@@ -443,6 +443,7 @@ extern "C" {
     enum ggml_op_hint {
         GGML_HINT_NONE             = 0,
         GGML_HINT_SRC0_IS_HADAMARD = 1,
+        GGML_HINT_EXACT_BATCH      = 2,
     };
 
     // model file types
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index 8f40fb2f7bc3..9457c8f49156 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -2116,6 +2116,7 @@ bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op) {
     switch (op) {
         case GGML_OP_FLASH_ATTN_EXT:
         case GGML_OP_MUL_MAT_ID:
+        case GGML_OP_CONCAT:
         case GGML_OP_CUMSUM:
         case GGML_OP_ARGSORT:
         case GGML_OP_TOP_K:
diff --git a/ggml/src/ggml-cuda/argsort.cu b/ggml/src/ggml-cuda/argsort.cu
index 26af90025972..17b905fd0b69 100644
--- a/ggml/src/ggml-cuda/argsort.cu
+++ b/ggml/src/ggml-cuda/argsort.cu
@@ -1,11 +1,16 @@
 #include "argsort.cuh"
 
 #ifdef GGML_CUDA_USE_CUB
-#    include <cub/cub.cuh>
-#    if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1)
-#        define STRIDED_ITERATOR_AVAILABLE
-#        include <cuda/iterator>
-#    endif
+#    ifdef GGML_CUDA_CUB_IS_HIPCUB
+#        include <hipcub/hipcub.hpp>
+namespace cub = hipcub;
+#    else
+#        include <cub/cub.cuh>
+#        if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1)
+#            define STRIDED_ITERATOR_AVAILABLE
+#            include <cuda/iterator>
+#        endif
+#    endif  // GGML_CUDA_CUB_IS_HIPCUB
 using namespace cub;
 #endif  // GGML_CUDA_USE_CUB
 
diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu
index 2e38077bf67f..20dd6682d397 100644
--- a/ggml/src/ggml-cuda/binbcast.cu
+++ b/ggml/src/ggml-cuda/binbcast.cu
@@ -26,6 +26,43 @@ static __device__ __forceinline__ float op_div(const float a, const float b) {
     return a / b;
 }
 
+template <float (*bin_op)(const float, const float)>
+static __global__ void binary_contiguous_f32(const float * src0, const float * src1, float * dst, int64_t n) {
+    ggml_cuda_pdl_lc();
+    int64_t i = int64_t(blockIdx.x) * blockDim.x + threadIdx.x;
+    const int64_t stride = int64_t(blockDim.x) * gridDim.x;
+    ggml_cuda_pdl_sync();
+    for (; i < n; i += stride) {
+        dst[i] = bin_op(src0[i], src1[i]);
+        if (n - i <= stride) {
+            break;
+        }
+    }
+}
+
+template <float (*bin_op)(const float, const float)>
+static bool try_binary_contiguous_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+    const ggml_tensor * src0 = dst->src[0];
+    const ggml_tensor * src1 = dst->src[1];
+    if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 ||
+        src0->ne[0] != 2048 ||
+        !ggml_are_same_shape(src0, src1) || !ggml_are_same_shape(src0, dst) ||
+        !ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
+        return false;
+    }
+
+    const int threads = 256;
+    const int64_t n = ggml_nelements(dst);
+    if (n == 0) {
+        return true;
+    }
+    const int blocks = std::min<int64_t>((n - 1) / threads + 1, 65535);
+    const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream());
+    ggml_cuda_kernel_launch(binary_contiguous_f32<bin_op>, launch_params,
+        (const float *) src0->data, (const float *) src1->data, (float *) dst->data, n);
+    return true;
+}
+
 template <float (*bin_op)(const float, const float),
           typename src0_t,
           typename src1_t,
@@ -435,6 +472,9 @@ void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
 }
 
 void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+    if (try_binary_contiguous_f32<op_add>(ctx, dst)) {
+        return;
+    }
     ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
 }
 
@@ -443,6 +483,9 @@ void ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
 }
 
 void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+    if (try_binary_contiguous_f32<op_mul>(ctx, dst)) {
+        return;
+    }
     ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
 }
 
@@ -542,6 +585,219 @@ void ggml_cuda_op_fused_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst,
     }
 }
 
+#if defined(__HIP_PLATFORM_AMD__)
+static __device__ __forceinline__ float rounded_mul_f32(const float a, const float b) {
+    float result;
+    asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b));
+    return result;
+}
+
+static __device__ __forceinline__ float rounded_add_f32(const float a, const float b) {
+    float result;
+    asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b));
+    return result;
+}
+#else
+static __device__ __forceinline__ float rounded_mul_f32(const float a, const float b) {
+    return __fmul_rn(a, b);
+}
+
+static __device__ __forceinline__ float rounded_add_f32(const float a, const float b) {
+    return __fadd_rn(a, b);
+}
+#endif
+
+template <int n_expert_used, bool sync_before_store>
+static __global__ void weighted_expert_sum_f32(
+        const float * experts, const float * weights, float * dst,
+        int64_t n_embd, int64_t n_tokens, int64_t experts_s1, int64_t experts_s2,
+        int64_t weights_s1, int64_t weights_s2) {
+    const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x;
+    const int64_t n_embd4 = n_embd / 4;
+    if (index >= n_embd4 * n_tokens) {
+        return;
+    }
+
+    const int64_t token = index / n_embd4;
+    const int64_t channel = 4 * (index - token * n_embd4);
+    float4 expert = *reinterpret_cast<const float4 *>(experts + token * experts_s2 + channel);
+    const float weight = weights[token * weights_s2];
+    // Keep MUL rounding separate from ADD to match the unfused graph without forcing products through scratch memory.
+    float4 sum = {
+        rounded_mul_f32(expert.x, weight),
+        rounded_mul_f32(expert.y, weight),
+        rounded_mul_f32(expert.z, weight),
+        rounded_mul_f32(expert.w, weight),
+    };
+#pragma unroll
+    for (int i = 1; i < n_expert_used; ++i) {
+        expert = *reinterpret_cast<const float4 *>(experts + token * experts_s2 + i * experts_s1 + channel);
+        const float weight_i = weights[token * weights_s2 + i * weights_s1];
+        sum.x = rounded_add_f32(sum.x, rounded_mul_f32(expert.x, weight_i));
+        sum.y = rounded_add_f32(sum.y, rounded_mul_f32(expert.y, weight_i));
+        sum.z = rounded_add_f32(sum.z, rounded_mul_f32(expert.z, weight_i));
+        sum.w = rounded_add_f32(sum.w, rounded_mul_f32(expert.w, weight_i));
+    }
+    if constexpr (sync_before_store) {
+        // single block: all expert and weight reads of the block are complete, so dst may alias them
+        __syncthreads();
+    }
+    reinterpret_cast<float4 *>(dst)[index] = sum;
+}
+
+void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tensor * mul, ggml_tensor * dst, int n_expert_used) {
+    const ggml_tensor * experts = ggml_are_same_shape(mul, mul->src[0]) ? mul->src[0] : mul->src[1];
+    const ggml_tensor * weights = experts == mul->src[0] ? mul->src[1] : mul->src[0];
+    const int64_t n_embd = experts->ne[0];
+    const int64_t n_tokens = experts->ne[2];
+    GGML_ASSERT(n_embd % 4 == 0);
+    const int64_t n_items = n_embd / 4 * n_tokens;
+
+    // The graph allocator may place the output over the expert or weight buffers (both die inside the fused
+    // range). A single block can read everything before a barrier and then write in place; larger grids
+    // stage the sum until all reads have finished.
+    auto overlaps = [](const ggml_tensor * a, const ggml_tensor * b) {
+        const char * a0 = (const char *) a->data;
+        const char * a1 = a0 + ggml_nbytes(a);
+        const char * b0 = (const char *) b->data;
+        const char * b1 = b0 + ggml_nbytes(b);
+        return a0 < b1 && b0 < a1;
+    };
+    const bool aliased      = overlaps(dst, experts) || overlaps(dst, weights);
+    const bool single_block = n_items <= 1024;
+    const bool stage        = aliased && !single_block;
+
+    const int threads = single_block ? (int) n_items : 256;
+    const int blocks  = (int) ((n_items + threads - 1) / threads);
+    const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream());
+
+    ggml_cuda_pool_alloc<float> result(ctx.pool());
+    float * out = stage ? result.alloc(n_embd * n_tokens) : (float *) dst->data;
+
+#define launch_weighted_expert_sum(n) \
+    do { \
+        if (single_block) { \
+            ggml_cuda_kernel_launch(weighted_expert_sum_f32<n, true>, launch_params, \
+                (const float *) experts->data, (const float *) weights->data, out, \
+                n_embd, n_tokens, experts->nb[1] / sizeof(float), experts->nb[2] / sizeof(float), \
+                weights->nb[1] / sizeof(float), weights->nb[2] / sizeof(float)); \
+        } else { \
+            ggml_cuda_kernel_launch(weighted_expert_sum_f32<n, false>, launch_params, \
+                (const float *) experts->data, (const float *) weights->data, out, \
+                n_embd, n_tokens, experts->nb[1] / sizeof(float), experts->nb[2] / sizeof(float), \
+                weights->nb[1] / sizeof(float), weights->nb[2] / sizeof(float)); \
+        } \
+    } while (0)
+    switch (n_expert_used) {
+        case 2: launch_weighted_expert_sum(2); break;
+        case 3: launch_weighted_expert_sum(3); break;
+        case 4: launch_weighted_expert_sum(4); break;
+        case 5: launch_weighted_expert_sum(5); break;
+        case 6: launch_weighted_expert_sum(6); break;
+        case 7: launch_weighted_expert_sum(7); break;
+        case 8: launch_weighted_expert_sum(8); break;
+        case 9: launch_weighted_expert_sum(9); break;
+        case 10: launch_weighted_expert_sum(10); break;
+        case 11: launch_weighted_expert_sum(11); break;
+        case 12: launch_weighted_expert_sum(12); break;
+        case 13: launch_weighted_expert_sum(13); break;
+        case 14: launch_weighted_expert_sum(14); break;
+        case 15: launch_weighted_expert_sum(15); break;
+        case 16: launch_weighted_expert_sum(16); break;
+        default: GGML_ABORT("unsupported expert count");
+    }
+#undef launch_weighted_expert_sum
+
+    if (stage) {
+        CUDA_CHECK(cudaMemcpyAsync(dst->data, result.get(), ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream()));
+    }
+}
+
+static __global__ void shared_mul_add_f32(
+        const float * src, const float * gate, const float * other, const float * residual, float * dst,
+        int64_t n_embd, int64_t nelements) {
+    const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x;
+    if (index >= nelements) {
+        return;
+    }
+    volatile float product = src[index] * gate[index / n_embd];
+    float value = other[index] + product;
+    dst[index] = value + residual[index];
+}
+
+void ggml_cuda_op_shared_mul_add(
+        ggml_backend_cuda_context & ctx, ggml_tensor * mul, const ggml_tensor * other,
+        const ggml_tensor * residual, ggml_tensor * dst) {
+    const ggml_tensor * src = ggml_are_same_shape(mul, mul->src[0]) ? mul->src[0] : mul->src[1];
+    const ggml_tensor * gate = src == mul->src[0] ? mul->src[1] : mul->src[0];
+    const int threads = 256;
+    const int64_t nelements = ggml_nelements(dst);
+    const int blocks = (nelements + threads - 1) / threads;
+    const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream());
+    ggml_cuda_kernel_launch(shared_mul_add_f32, launch_params,
+        (const float *) src->data, (const float *) gate->data, (const float *) other->data,
+        (const float *) residual->data, (float *) dst->data, dst->ne[0], nelements);
+}
+
+// Qwen3.5 shared-expert gate for one token: gate = sigmoid(w . y) computed exactly like
+// mul_mat_vec_f<float, float, 1, 256> (same per-thread fma chain and the same two-level warp reduction),
+// then dst = (other + src*gate) + residual as in shared_mul_add_f32. One block, so the activation vector is
+// fully read before the reduction barrier and the output may alias it.
+static __global__ void __launch_bounds__(256, 1) shared_gate_mul_add_f32(
+        const float * w, const float * y, const float * src, const float * other, const float * residual, float * dst,
+        const int ncols2, const int64_t n_embd) {
+    constexpr int block_size = 256;
+    constexpr int warp_size  = 32;
+    __shared__ float buf_iw[warp_size];
+    __shared__ float gate_s;
+
+    const int tid = threadIdx.x;
+    if (tid < warp_size) {
+        buf_iw[tid] = 0.0f;
+    }
+    __syncthreads();
+
+    const float2 * x2 = (const float2 *) w;
+    const float2 * y2 = (const float2 *) y;
+    float sumf = 0.0f;
+    for (int col2 = tid; col2 < ncols2; col2 += block_size) {
+        const float2 tmpx = x2[col2];
+        const float2 tmpy = y2[col2];
+        ggml_cuda_mad(sumf, tmpx.x, tmpy.x);
+        ggml_cuda_mad(sumf, tmpx.y, tmpy.y);
+    }
+    sumf = warp_reduce_sum<warp_size>(sumf);
+    buf_iw[tid/warp_size] = sumf;
+    __syncthreads();
+    if (tid < warp_size) {
+        sumf = buf_iw[tid];
+        sumf = warp_reduce_sum<warp_size>(sumf);
+        if (tid == 0) {
+            gate_s = 1.0f / (1.0f + expf(-sumf));
+        }
+    }
+    __syncthreads();
+
+    const float gate = gate_s;
+    for (int64_t i = tid; i < n_embd; i += block_size) {
+        const float product = rounded_mul_f32(src[i], gate);
+        const float value   = rounded_add_f32(other[i], product);
+        dst[i] = residual ? rounded_add_f32(value, residual[i]) : value;
+    }
+}
+
+void ggml_cuda_op_shared_gate_mul_add(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * gate_mm, const ggml_tensor * src, const ggml_tensor * other,
+        const ggml_tensor * residual, ggml_tensor * dst) {
+    const ggml_tensor * w = gate_mm->src[0];
+    const ggml_tensor * y = gate_mm->src[1];
+    GGML_ASSERT(w->ne[0] % 2 == 0);
+    const ggml_cuda_kernel_launch_params launch_params(1, 256, 0, ctx.stream());
+    ggml_cuda_kernel_launch(shared_gate_mul_add_f32, launch_params,
+        (const float *) w->data, (const float *) y->data, (const float *) src->data, (const float *) other->data,
+        residual ? (const float *) residual->data : nullptr, (float *) dst->data, (int) (w->ne[0] / 2), dst->ne[0]);
+}
+
 void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     const ggml_tensor * src0 = dst->src[0];
 
diff --git a/ggml/src/ggml-cuda/binbcast.cuh b/ggml/src/ggml-cuda/binbcast.cuh
index 12624785b444..bb3a8e13f241 100644
--- a/ggml/src/ggml-cuda/binbcast.cuh
+++ b/ggml/src/ggml-cuda/binbcast.cuh
@@ -10,3 +10,9 @@ void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst
 
 void ggml_cuda_op_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse);
 void ggml_cuda_op_fused_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse);
+void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tensor * mul, ggml_tensor * dst, int n_expert_used);
+void ggml_cuda_op_shared_mul_add(ggml_backend_cuda_context & ctx, ggml_tensor * mul, const ggml_tensor * other, const ggml_tensor * residual, ggml_tensor * dst);
+
+void ggml_cuda_op_shared_gate_mul_add(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * gate_mm, const ggml_tensor * src, const ggml_tensor * other,
+        const ggml_tensor * residual, ggml_tensor * dst);
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
index e5ccd1feab14..794a88f5aa8b 100644
--- a/ggml/src/ggml-cuda/common.cuh
+++ b/ggml/src/ggml-cuda/common.cuh
@@ -111,6 +111,14 @@
 #    define GGML_CUDA_USE_CUB
 #endif  // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070
 
+// hipCUB (rocPRIM) offers the same device-wide radix sort / scan / reduce entry points as CUB. Using it lifts the
+// ncols <= 1024 limit of ARGSORT / TOP_K on HIP, which otherwise falls back to the CPU (e.g. the QSA indexer top-k
+// of qwen4exp / DeepSeek-V3.2 at contexts beyond 2k tokens).
+#if defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_CUB) && __has_include(<hipcub/hipcub.hpp>)
+#    define GGML_CUDA_USE_CUB
+#    define GGML_CUDA_CUB_IS_HIPCUB
+#endif
+
 // PDL host-side support (cudaLaunchKernelEx) requires CUDART >= 11.8.
 // However, this has been bugged in CTK < 12.3 for MSVC builds, see
 // https://github.com/ggml-org/llama.cpp/pull/22522#discussion_r3302393293
@@ -1540,6 +1548,10 @@ struct ggml_cuda_mm_fusion_args_host {
     const ggml_tensor * gate_scale = nullptr;
     ggml_glu_op glu_op;
     float glu_limit = 0.0f;
+    // RDNA3.5 fused-quantize matvec prologue (y_op == 3), see ggml_cuda_mm_fusion_args_device
+    const ggml_tensor * y_gate = nullptr;
+    const ggml_tensor * y_norm_w = nullptr;
+    float y_eps = 0.0f;
 };
 struct ggml_cuda_mm_fusion_args_device {
     const void * x_bias = nullptr;
@@ -1549,6 +1561,10 @@ struct ggml_cuda_mm_fusion_args_device {
     const void * gate_scale = nullptr;
     ggml_glu_op glu_op;
     float glu_limit = 0.0f;
+    // RDNA3.5 fused-quantize matvec prologue (y_op == 3): y' = sigmoid(y_gate) * (rms_norm_128(y) * y_norm_w)
+    const void * y_gate = nullptr;
+    const void * y_norm_w = nullptr;
+    float y_eps = 0.0f;
 };
 
 struct ggml_cuda_kernel_launch_params {
diff --git a/ggml/src/ggml-cuda/concat.cu b/ggml/src/ggml-cuda/concat.cu
index 6df89013ca79..408cbce3d219 100644
--- a/ggml/src/ggml-cuda/concat.cu
+++ b/ggml/src/ggml-cuda/concat.cu
@@ -2,6 +2,13 @@
 
 #include <stdint.h>
 
+static constexpr int concat_transposed_tile_y(int cc) {
+    return GGML_CUDA_CC_IS_RDNA3_5(cc) ? 16 : 8;
+}
+
+static_assert(concat_transposed_tile_y(GGML_CUDA_CC_RDNA3_5) == 16);
+static_assert(concat_transposed_tile_y(GGML_CUDA_CC_RDNA3) == 8);
+
 // contiguous kernels
 template <typename T, int dim>
 static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) concat_cont(const T * x,
@@ -79,6 +86,54 @@ static void concat_cont_cuda(const T * x,
     concat_cont<T, 2><<<num_blocks, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne00, ne01, ne02, ne0, ne1, ne2);
 }
 
+template <typename T>
+static __global__ void concat_transposed_src1_dim0(
+        const char * src0, const char * src1, char * dst,
+        int64_t ne00, int64_t ne10, int64_t ne11,
+        uint64_t nb00, uint64_t nb01, uint64_t nb02,
+        uint64_t nb10, uint64_t nb11, uint64_t nb12,
+        uint64_t nb0, uint64_t nb1, uint64_t nb2) {
+    constexpr int tile = 32;
+    __shared__ T values[tile][tile + 1];
+
+    const int tx = threadIdx.x;
+    const int ty = threadIdx.y;
+    const int64_t channel0 = blockIdx.x * tile;
+    const int64_t token0 = blockIdx.y * tile;
+    const int64_t i2 = blockIdx.z;
+
+    ggml_cuda_pdl_sync();
+
+    for (int j = 0; j < tile; j += blockDim.y) {
+        const int64_t token = token0 + ty + j;
+        const int64_t channel = channel0 + tx;
+        if (token < ne10 && channel < ne11) {
+            values[ty + j][tx] = *reinterpret_cast<const T *>(src1 + i2 * nb12 + channel * nb11 + token * nb10);
+        }
+    }
+    __syncthreads();
+
+    for (int j = 0; j < tile; j += blockDim.y) {
+        const int64_t token = token0 + tx;
+        const int64_t channel = channel0 + ty + j;
+        if (token < ne10 && channel < ne11) {
+            *reinterpret_cast<T *>(dst + i2 * nb2 + channel * nb1 + (ne00 + token) * nb0) = values[tx][ty + j];
+        }
+    }
+
+    if (blockIdx.y == 0) {
+        for (int j = 0; j < tile; j += blockDim.y) {
+            const int64_t channel = channel0 + ty + j;
+            if (channel < ne11) {
+                for (int64_t state = tx; state < ne00; state += tile) {
+                    *reinterpret_cast<T *>(dst + i2 * nb2 + channel * nb1 + state * nb0) =
+                        *reinterpret_cast<const T *>(src0 + i2 * nb02 + channel * nb01 + state * nb00);
+                }
+            }
+        }
+    }
+}
+
 // non-contiguous kernel (slow)
 template <typename T, int dim>
 static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE)
@@ -141,6 +196,24 @@ static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE)
 
 template <typename T>
 static void concat_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, int dim, cudaStream_t stream) {
+    if (dim == 0 && src1->ne[0] >= 32 && src0->ne[3] == 1 && src1->ne[3] == 1 && dst->ne[3] == 1 &&
+            src0->nb[0] == sizeof(T) && src0->nb[1] == src0->ne[0] * sizeof(T) &&
+            src1->nb[1] == sizeof(T) && src1->nb[0] == src1->ne[1] * sizeof(T) &&
+            dst->nb[0] == sizeof(T) && dst->nb[1] == dst->ne[0] * sizeof(T) &&
+            src0->ne[1] == src1->ne[1] && src0->ne[2] == src1->ne[2]) {
+        constexpr int tile = 32;
+        const int tile_y = concat_transposed_tile_y(ggml_cuda_info().devices[ggml_cuda_get_device()].cc);
+        const dim3 block_dims(tile, tile_y, 1);
+        const dim3 block_nums((src1->ne[1] + tile - 1) / tile, (src1->ne[0] + tile - 1) / tile, src1->ne[2]);
+        concat_transposed_src1_dim0<T><<<block_nums, block_dims, 0, stream>>>(
+            (const char *) src0->data, (const char *) src1->data, (char *) dst->data,
+            src0->ne[0], src1->ne[0], src1->ne[1],
+            src0->nb[0], src0->nb[1], src0->nb[2],
+            src1->nb[0], src1->nb[1], src1->nb[2],
+            dst->nb[0], dst->nb[1], dst->nb[2]);
+        return;
+    }
+
     if (dim != 3 && ggml_is_contiguous_to_3(src0) && ggml_is_contiguous_to_3(src1)) {
         const T * src0_d = (const T *) src0->data;
         const T * src1_d = (const T *) src1->data;
diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
index 360c614a4403..47cab5f980d4 100644
--- a/ggml/src/ggml-cuda/convert.cu
+++ b/ggml/src/ggml-cuda/convert.cu
@@ -450,8 +450,40 @@ static void convert_unary_cuda(const void * vx, dst_t * y,
         (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);
 }
 
+static __global__ void convert_f32_f16_cont(const float * x, half * y, int64_t k) {
+    const int64_t i = 2 * ((int64_t) blockDim.x * blockIdx.x + threadIdx.x);
+    if (i + 1 < k) {
+        const float2 value = ((const float2 *) x)[i / 2];
+        ((half2 *) y)[i / 2] = make_half2(value.x, value.y);
+    } else if (i < k) {
+        y[i] = x[i];
+    }
+}
+
+static __global__ void convert_f16_f32_cont(const half * x, float * y, int64_t k) {
+    const int64_t i = 2 * ((int64_t) blockDim.x * blockIdx.x + threadIdx.x);
+    if (i + 1 < k) {
+        ((float2 *) y)[i / 2] = __half22float2(((const half2 *) x)[i / 2]);
+    } else if (i < k) {
+        y[i] = x[i];
+    }
+}
+
 template <typename src_t, typename dst_t>
 static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {
+    const int nblocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE);
+    if constexpr (std::is_same<src_t, float>::value && std::is_same<dst_t, half>::value) {
+        if ((uintptr_t) vx % alignof(float2) == 0 && (uintptr_t) y % alignof(half2) == 0) {
+            convert_f32_f16_cont<<<nblocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>((const float *) vx, (half *) y, k);
+            return;
+        }
+    }
+    if constexpr (std::is_same<src_t, half>::value && std::is_same<dst_t, float>::value) {
+        if ((uintptr_t) vx % alignof(half2) == 0 && (uintptr_t) y % alignof(float2) == 0) {
+            convert_f16_f32_cont<<<nblocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>((const half *) vx, (float *) y, k);
+            return;
+        }
+    }
     convert_unary_cuda<src_t>(vx, y, k, 1, 1, 1, k, k, k, stream);
 }
 
diff --git a/ggml/src/ggml-cuda/cumsum.cu b/ggml/src/ggml-cuda/cumsum.cu
index def9c32955f2..51cfb84dab17 100644
--- a/ggml/src/ggml-cuda/cumsum.cu
+++ b/ggml/src/ggml-cuda/cumsum.cu
@@ -5,7 +5,12 @@
 #include "ggml.h"
 
 #ifdef GGML_CUDA_USE_CUB
-#   include <cub/cub.cuh>
+#   ifdef GGML_CUDA_CUB_IS_HIPCUB
+#       include <hipcub/hipcub.hpp>
+namespace cub = hipcub;
+#   else
+#       include <cub/cub.cuh>
+#   endif // GGML_CUDA_CUB_IS_HIPCUB
 #endif // GGML_CUDA_USE_CUB
 
 template<typename T, int BLOCK_SIZE>
diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh
index e67cc7fdf784..25854e94617d 100644
--- a/ggml/src/ggml-cuda/fattn-common.cuh
+++ b/ggml/src/ggml-cuda/fattn-common.cuh
@@ -1174,6 +1174,12 @@ void launch_fattn(
             }
         }
 
+        if constexpr (ncols2 == 3) {
+            if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+                parallel_blocks = std::min(18, ntiles_KV);
+            }
+        }
+
         blocks_num.x = ntiles_x;
         blocks_num.y = parallel_blocks;
         blocks_num.z = ntiles_z_gqa*K->ne[2]*Q->ne[3];
diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh
index 7f4cfd5511ff..55d7988528ca 100644
--- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh
+++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh
@@ -158,7 +158,7 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256,  8,  64, 2,  32, 128, 128, 128, 1, true);
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16,  64, 2,  32, 128, 128, 128, 1, true);
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2,  64, 128, 128,  64, 1, true);
-    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2,  64, 128, 128,  64, 1, true);
+    GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 1,  32, 128,  64,  64, 1, false);
 
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2,  32, 160, 128, 128, 1, true);
     GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2,  32, 160, 128, 128, 1, true);
@@ -1687,7 +1687,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
                 }
             }
         }
-        if (np > 1) {
+        // Warps read columns of tile_Q that were written by other warps, so a barrier is needed before tile_Q is
+        // reused: by the next nbatch_combine batch (DV/2 > nbatch_combine) or by the Q load of the next tile.
+        if (np > 1 || DV/2 > nbatch_combine) {
             __syncthreads();
         }
     }
@@ -1759,7 +1761,7 @@ static __global__ void flash_attn_ext_f16(
 #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING
 
 #if defined(AMD_WMMA_AVAILABLE)
-    if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) {
+    if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256 || (DKQ > 128 && DKQ != 256)) {
         NO_DEVICE_CODE;
         return;
     }
diff --git a/ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu b/ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu
new file mode 100644
index 000000000000..586632da20e4
--- /dev/null
+++ b/ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu
@@ -0,0 +1,12 @@
+#if defined(GGML_USE_HIP)
+
+#define GGML_CUDA_FATTN_VGPR192
+#include "fattn-tile.cuh"
+
+fattn_kernel_t ggml_cuda_fattn_tile_d256_ncols32_rdna3_5(const bool use_logit_softcap) {
+    return use_logit_softcap
+        ? flash_attn_tile<256, 256, 4, 8, true,  false>
+        : flash_attn_tile<256, 256, 4, 8, false, false>;
+}
+
+#endif // defined(GGML_USE_HIP)
diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh
index d1164b8526d3..e5be145ed5ee 100644
--- a/ggml/src/ggml-cuda/fattn-tile.cuh
+++ b/ggml/src/ggml-cuda/fattn-tile.cuh
@@ -1,6 +1,10 @@
 #include "common.cuh"
 #include "fattn-common.cuh"
 
+#if defined(GGML_USE_HIP) && defined(__clang__)
+#pragma clang diagnostic ignored "-Wpass-failed"
+#endif
+
 // nbatch_fa == number of KQ rows to process per iteration
 // nbatch_K == number of K columns to load in parallel for KQ calculation
 
@@ -213,6 +217,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2,  32,  64)
 
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2, 256, 2, 128,  64)
+    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  3,  96, 2,  32,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 256, 2,  64, 128)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 256, 2,  64, 128)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2,  32, 128)
@@ -291,6 +296,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 3,  64,  64)
 
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  2,  64, 8,  32,  64)
+    GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  3,  96, 8,  32,  64)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  4, 128, 6,  32, 256)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256,  8, 128, 6,  32, 256)
     GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 5,  32, 256)
@@ -479,14 +485,46 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile(
     ggml_cuda_unroll<5>{}(load);
 }
 
+template<int warp_size, int nwarps, int I, int J, int J_padding, bool oob_check>
+static __device__ __forceinline__ void flash_attn_tile_load_tile_q8_0(
+        const char * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int stride_KV, const int i_sup) {
+    static_assert(J % QK8_0 == 0, "bad J");
+
+    constexpr int nthreads = warp_size*nwarps;
+    constexpr int groups_per_row = J/4;
+    const int tid = threadIdx.y*warp_size + threadIdx.x;
+
+#pragma unroll
+    for (int ig = tid; ig < I*groups_per_row; ig += nthreads) {
+        const int i = ig/groups_per_row;
+        const int j = (ig % groups_per_row)*4;
+
+        half2 v0 = make_half2(0.0f, 0.0f);
+        half2 v1 = make_half2(0.0f, 0.0f);
+        if (!oob_check || i < i_sup) {
+            const block_q8_0 * row = (const block_q8_0 *) (KV + int64_t(i)*stride_KV);
+            const block_q8_0 & block = row[j/QK8_0];
+            int q;
+            ggml_cuda_memcpy_1<sizeof(q), 2>(&q, block.qs + j % QK8_0);
+            const int8_t * q8 = (const int8_t *) &q;
+            const half2 d = __half2half2(block.d);
+            v0 = d*make_half2(q8[0], q8[1]);
+            v1 = d*make_half2(q8[2], q8[3]);
+        }
+
+        tile_KV[i*(J/2 + J_padding) + j/2 + 0] = v0;
+        tile_KV[i*(J/2 + J_padding) + j/2 + 1] = v1;
+    }
+}
+
 // Function that performs a single iteration in for the KQ matrix multiplication:
 template <int warp_size, int nwarps, int ncols1, int ncols2, int DKQ, int nbatch_fa, int nbatch_K,
-    bool use_logit_softcap, bool oob_check, typename T_vec_dot>
+    bool use_logit_softcap, bool oob_check, bool q8_0_KV, typename T_vec_dot>
 static __device__ __forceinline__ void flash_attn_tile_iter_KQ(
         T_vec_dot   * const Q_tmp,
-        const half2 * const __restrict__ K_h2,
+        const char  * const __restrict__ K_data,
         T_vec_dot   * const KV_tmp,
-        const int stride_K2,
+        const int stride_K,
         const int k_VKQ_0,
         const int k_VKQ_sup,
         const int k_KQ_0,
@@ -498,8 +536,13 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ(
     constexpr int cpw   = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp
     constexpr int np    = nwarps > ncols ? nwarps/ncols : 1; // number of parallel warps per Q column
 
-    flash_attn_tile_load_tile<warp_size, nwarps, nbatch_fa, nbatch_K, cpy_ne, oob_check>
-        (K_h2 + int64_t(k_VKQ_0)*stride_K2 + k_KQ_0/2, KV_tmp, stride_K2, k_VKQ_sup);
+    if constexpr (q8_0_KV) {
+        flash_attn_tile_load_tile_q8_0<warp_size, nwarps, nbatch_fa, nbatch_K, cpy_ne, oob_check>
+            (K_data + int64_t(k_VKQ_0)*stride_K + k_KQ_0/QK8_0*sizeof(block_q8_0), KV_tmp, stride_K, k_VKQ_sup);
+    } else {
+        flash_attn_tile_load_tile<warp_size, nwarps, nbatch_fa, nbatch_K, cpy_ne, oob_check>
+            ((const half2 *) (K_data + int64_t(k_VKQ_0)*stride_K + k_KQ_0*sizeof(half)), KV_tmp, stride_K/sizeof(half2), k_VKQ_sup);
+    }
     __syncthreads();
 
 #ifdef FAST_FP16_AVAILABLE
@@ -537,13 +580,29 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ(
 #endif // FAST_FP16_AVAILABLE
         }
 
+#if defined(RDNA3_5)
+        if constexpr (DKQ == 256 && ncols == 32) {
 #pragma unroll
-        for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {
+            for (int k = 0; k < cpy_ne; ++k) {
 #pragma unroll
-            for (int jc0 = 0; jc0 < cpw; ++jc0) {
+                for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {
 #pragma unroll
-                for (int k = 0; k < cpy_ne; ++k) {
-                    ggml_cuda_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]);
+                    for (int jc0 = 0; jc0 < cpw; ++jc0) {
+                        ggml_cuda_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]);
+                    }
+                }
+            }
+        } else
+#endif // defined(RDNA3_5)
+        {
+#pragma unroll
+            for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) {
+#pragma unroll
+                for (int jc0 = 0; jc0 < cpw; ++jc0) {
+#pragma unroll
+                    for (int k = 0; k < cpy_ne; ++k) {
+                        ggml_cuda_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]);
+                    }
                 }
             }
         }
@@ -556,19 +615,19 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ(
 
 // Function that performs a single iteration of the main loop over up to nbatch_fa tokens.
 template <int warp_size, int nwarps, int ncols1, int ncols2, int DKQ, int DV, int nbatch_fa, int nbatch_K,
-    bool use_logit_softcap, bool oob_check, typename T_vec_dot, typename T_KQ, typename T_acc>
+    bool use_logit_softcap, bool oob_check, bool q8_0_KV, typename T_vec_dot, typename T_KQ, typename T_acc>
 static __device__ __forceinline__ void flash_attn_tile_iter(
         T_vec_dot * const Q_tmp,
-        const half2 * const __restrict__ K_h2,
-        const half2 * const __restrict__ V_h2,
+        const char  * const __restrict__ K_data,
+        const char  * const __restrict__ V_data,
         const half  * const __restrict__ mask,
         const uint3 ne01,
         const float logit_softcap,
         const float slope,
         T_KQ      * const KQ,
         T_vec_dot * const KV_tmp,
-        const int stride_K2,
-        const int stride_V2,
+        const int stride_K,
+        const int stride_V,
         const int stride_mask,
         float * const KQ_max,
         float * const KQ_sum,
@@ -601,19 +660,32 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
         KQ_max_new[jc0] = KQ_max[jc0];
     }
 
+#if defined(RDNA3_5)
+    constexpr bool common_mask = DKQ == 256 && DV == 256 && ncols1 == 4 && ncols2 == 8;
+    const int j_mask = common_mask ? fastmodulo(col_Q_0 + ((threadIdx.y / np)*cpw)/ncols2, ne01) : 0;
+    float mask_value[common_mask ? nbatch_fa/(np*warp_size) : 1];
+#pragma unroll
+    for (int i0 = 0; i0 < nbatch_fa; i0 += np*warp_size) {
+        mask_value[i0/(np*warp_size)] = common_mask ? slope*__half2float(mask[j_mask*stride_mask + k_VKQ_0 + i0 + (threadIdx.y % np)*warp_size + threadIdx.x]) : 0.0f;
+    }
+#else
+    constexpr bool common_mask = false;
+    const float mask_value[1] = {0.0f};
+#endif
+
     float KQ_acc[nbatch_fa/(np*warp_size) * cpw] = {0.0f}; // Accumulators for KQ matrix multiplication.
 
     // KQ = K @ Q matrix multiplication:
     constexpr int nbatch_K_last = DKQ % nbatch_K;
 #pragma unroll
     for (int k_KQ_0 = 0; k_KQ_0 < DKQ - nbatch_K_last; k_KQ_0 += nbatch_K) {
-        flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>(
-            Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);
+        flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, q8_0_KV>(
+            Q_tmp, K_data, KV_tmp, stride_K, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);
     }
-    if (nbatch_K_last > 0) {
+    if constexpr (nbatch_K_last > 0) {
         constexpr int k_KQ_0 = DKQ - nbatch_K_last;
-        flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K_last, use_logit_softcap, oob_check>(
-            Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);
+        flash_attn_tile_iter_KQ<warp_size, nwarps, ncols1, ncols2, DKQ, nbatch_fa, nbatch_K_last, use_logit_softcap, oob_check, q8_0_KV>(
+            Q_tmp, K_data, KV_tmp, stride_K, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc);
     }
 
     // Apply logit softcap + mask, update KQ_max:
@@ -636,8 +708,8 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
             }
 
             if (!oob_check || i_KQ < k_VKQ_sup) {
-                KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] += (ncols2 > 1 || mask) ?
-                    slope*__half2float(mask[j*stride_mask + k_VKQ_0 + i_KQ]) : 0.0f;
+                KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] += common_mask ? mask_value[i_KQ_0/(np*warp_size)] :
+                    (ncols2 > 1 || mask) ? slope*__half2float(mask[j*stride_mask + k_VKQ_0 + i_KQ]) : 0.0f;
 
                 KQ_max_new[jc0] = fmaxf(KQ_max_new[jc0], KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] + FATTN_KQ_MAX_OFFSET);
             }
@@ -718,8 +790,13 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
     static_assert(nbatch_V % np == 0, "bad nbatch_V");
 #pragma unroll
     for (int k0 = 0; k0 < nbatch_fa; k0 += nbatch_V) {
-        flash_attn_tile_load_tile<warp_size, nwarps, nbatch_V, DV, 0, oob_check>
-            (V_h2 + int64_t(k_VKQ_0 + k0)*stride_V2, KV_tmp, stride_V2, k_VKQ_sup - k0);
+        if constexpr (q8_0_KV) {
+            flash_attn_tile_load_tile_q8_0<warp_size, nwarps, nbatch_V, DV, 0, oob_check>
+                (V_data + int64_t(k_VKQ_0 + k0)*stride_V, KV_tmp, stride_V, k_VKQ_sup - k0);
+        } else {
+            flash_attn_tile_load_tile<warp_size, nwarps, nbatch_V, DV, 0, oob_check>
+                ((const half2 *) (V_data + int64_t(k_VKQ_0 + k0)*stride_V), KV_tmp, stride_V/sizeof(half2), k_VKQ_sup - k0);
+        }
         __syncthreads();
 
 #ifdef FAST_FP16_AVAILABLE
@@ -788,8 +865,11 @@ static __device__ __forceinline__ void flash_attn_tile_iter(
     }
 }
 
-template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap> // D == head size
+template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool q8_0_KV> // D == head size
 __launch_bounds__(ggml_cuda_fattn_tile_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_tile_get_occupancy(DKQ, DV, ncols1*ncols2))
+#if defined(RDNA3_5) && defined(GGML_CUDA_FATTN_VGPR192)
+__attribute__((amdgpu_num_vgpr(192)))
+#endif // defined(RDNA3_5) && defined(GGML_CUDA_FATTN_VGPR192)
 static __global__ void flash_attn_tile(
         const char * Q_ptr,
         const char * K_ptr,
@@ -813,6 +893,13 @@ static __global__ void flash_attn_tile(
                             const int32_t ne31, const int32_t ne32, const int32_t ne33,
                             const int32_t nb31, const int32_t nb32, const int64_t nb33) {
 #ifdef FLASH_ATTN_AVAILABLE
+#if defined(GGML_USE_HIP) && !defined(RDNA3_5)
+    if constexpr (q8_0_KV) {
+        NO_DEVICE_CODE;
+        return;
+    }
+#endif // defined(GGML_USE_HIP) && !defined(RDNA3_5)
+
     const char * GGML_CUDA_RESTRICT Q        = Q_ptr;
     const char * GGML_CUDA_RESTRICT K        = K_ptr;
     const char * GGML_CUDA_RESTRICT V        = V_ptr;
@@ -853,14 +940,12 @@ static __global__ void flash_attn_tile(
     const int sequence = blockIdx.z / (ne02/ncols2);
     const int head0 = blockIdx.z*ncols2 - sequence*ne02; // == blockIdx.z % (ne02/ncols2)
     const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.
-    const float * Q_f  = (const float *) (Q + nb03*sequence + nb02* head0);
-    const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*(head0 / gqa_ratio));
-    const half2 * V_h2 = (const half2 *) (V + nb23*sequence + nb22*(head0 / gqa_ratio)); // K and V have same shape
+    const float * Q_f    = (const float *) (Q + nb03*sequence + nb02* head0);
+    const char  * K_data = K + nb13*sequence + nb12*(head0 / gqa_ratio);
+    const char  * V_data = V + nb23*sequence + nb22*(head0 / gqa_ratio);
 
     const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33)) : nullptr;
 
-    const int stride_K2   = nb11 / sizeof(half2);
-    const int stride_V2   = nb21 / sizeof(half2);
     const int stride_mask = nb31 / sizeof(half);
 
     const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, head0, n_head_log2, m0, m1) : 1.0f;
@@ -957,24 +1042,24 @@ static __global__ void flash_attn_tile(
         int k_VKQ_0 = blockIdx.y*nbatch_fa;
         while (k_VKQ_0 < k_VKQ_max - nbatch_fa) {
             constexpr bool oob_check = false;
-            flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>
-                (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
-                stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
+            flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, q8_0_KV>
+                (Q_tmp, K_data, V_data, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
+                nb11, nb21, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
             k_VKQ_0 += gridDim.y*nbatch_fa;
         }
         if (k_VKQ_0 < k_VKQ_max) {
             constexpr bool oob_check = true;
-            flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>
-                (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
-                stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
+            flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, q8_0_KV>
+                (Q_tmp, K_data, V_data, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
+                nb11, nb21, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
         }
     } else {
         // Branch without out-of-bounds checks.
         for (int k_VKQ_0 = blockIdx.y*nbatch_fa; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nbatch_fa) {
             constexpr bool oob_check = false;
-            flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>
-                (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
-                stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
+            flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check, q8_0_KV>
+                (Q_tmp, K_data, V_data, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,
+                nb11, nb21, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0);
         }
     }
 
@@ -1145,6 +1230,38 @@ static __global__ void flash_attn_tile(
 #endif // FLASH_ATTN_AVAILABLE
 }
 
+#ifdef GGML_USE_HIP
+fattn_kernel_t ggml_cuda_fattn_tile_d256_ncols32_rdna3_5(const bool use_logit_softcap);
+#endif // GGML_USE_HIP
+
+template <int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap>
+static void launch_fattn_tile_case(
+        ggml_backend_cuda_context & ctx, ggml_tensor * dst, const int nwarps, const int nbatch_fa, const int warp_size) {
+    constexpr size_t nbytes_shared = 0;
+    bool use_q8_0_KV = false;
+    fattn_kernel_t fattn_kernel;
+#ifdef GGML_USE_HIP
+    if constexpr (DKQ == DV && (DKQ == 64 || DKQ == 128 || DKQ == 256)) {
+        use_q8_0_KV = dst->src[0]->ne[1] == 1 && dst->src[1]->type == GGML_TYPE_Q8_0 && dst->src[2]->type == GGML_TYPE_Q8_0;
+        fattn_kernel = use_q8_0_KV
+            ? flash_attn_tile<DKQ, DV, ncols1, ncols2, use_logit_softcap, true>
+            : flash_attn_tile<DKQ, DV, ncols1, ncols2, use_logit_softcap, false>;
+    } else {
+        fattn_kernel = flash_attn_tile<DKQ, DV, ncols1, ncols2, use_logit_softcap, false>;
+    }
+    if constexpr (DKQ == 256 && DV == 256 && ncols1 == 4 && ncols2 == 8) {
+        const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+        if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+            fattn_kernel = ggml_cuda_fattn_tile_d256_ncols32_rdna3_5(use_logit_softcap);
+        }
+    }
+#else
+    fattn_kernel = flash_attn_tile<DKQ, DV, ncols1, ncols2, use_logit_softcap, false>;
+#endif // GGML_USE_HIP
+    launch_fattn<DV, ncols1, ncols2>
+        (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, !use_q8_0_KV, !use_q8_0_KV, false, warp_size);
+}
+
 template <int DKQ, int DV, int ncols2, bool use_logit_softcap>
 static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
     const ggml_tensor * Q = dst->src[0];
@@ -1153,7 +1270,12 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
     const int cc        = ggml_cuda_info().devices[id].cc;
     const int warp_size = 32;
 
-    constexpr size_t nbytes_shared = 0;
+    if constexpr (ncols2 == 3) {
+        const int nwarps    = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, 3, cc) / warp_size;
+        const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, 3, cc);
+        launch_fattn_tile_case<DKQ, DV, 1, 3, use_logit_softcap>(ctx, dst, nwarps, nbatch_fa, warp_size);
+        return;
+    } else {
 
 #ifdef GGML_USE_HIP
     if constexpr (DKQ <= 128) {
@@ -1161,9 +1283,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
             constexpr int cols_per_block = 64;
             const int nwarps    = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
             const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
-            fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
-            launch_fattn<DV, cols_per_block/ncols2, ncols2>
-                (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
+            launch_fattn_tile_case<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>
+                (ctx, dst, nwarps, nbatch_fa, warp_size);
             return;
         }
     }
@@ -1177,9 +1298,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
             constexpr int cols_per_block = 32;
             const int nwarps    = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
             const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
-            fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
-            launch_fattn<DV, cols_per_block/ncols2, ncols2>
-                (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
+            launch_fattn_tile_case<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>
+                (ctx, dst, nwarps, nbatch_fa, warp_size);
             return;
         }
     }
@@ -1189,9 +1309,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
             constexpr int cols_per_block = 16;
             const int nwarps    = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
             const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
-            fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
-            launch_fattn<DV, cols_per_block/ncols2, ncols2>
-                (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
+            launch_fattn_tile_case<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>
+                (ctx, dst, nwarps, nbatch_fa, warp_size);
             return;
         }
     }
@@ -1201,9 +1320,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
             constexpr int cols_per_block = 8;
             const int nwarps    = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
             const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
-            fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
-            launch_fattn<DV, cols_per_block/ncols2, ncols2>
-                (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
+            launch_fattn_tile_case<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>
+                (ctx, dst, nwarps, nbatch_fa, warp_size);
             return;
         }
     }
@@ -1213,9 +1331,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
             constexpr int cols_per_block = 4;
             const int nwarps    = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
             const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
-            fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
-            launch_fattn<DV, cols_per_block/ncols2, ncols2>
-                (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
+            launch_fattn_tile_case<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>
+                (ctx, dst, nwarps, nbatch_fa, warp_size);
             return;
         }
     }
@@ -1224,13 +1341,13 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm
         constexpr int cols_per_block = 2;
         const int nwarps    = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size;
         const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);
-        fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;
-        launch_fattn<DV, cols_per_block/ncols2, ncols2>
-            (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);
+        launch_fattn_tile_case<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>
+            (ctx, dst, nwarps, nbatch_fa, warp_size);
         return;
     }
 
-    GGML_ABORT("fatal error");
+        GGML_ABORT("fatal error");
+    }
 }
 
 template <int DKQ, int DV, bool use_logit_softcap>
@@ -1238,6 +1355,7 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
     const ggml_tensor * KQV  = dst;
     const ggml_tensor * Q    = dst->src[0];
     const ggml_tensor * K    = dst->src[1];
+    [[maybe_unused]] const ggml_tensor * V = dst->src[2];
     const ggml_tensor * mask = dst->src[3];
 
     float max_bias = 0.0f;
@@ -1296,6 +1414,15 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
     }
 
     if constexpr (DKQ <= 512 && DKQ != 320 && DKQ != 192) {
+        if constexpr (DKQ == 256 && DV == 256) {
+#ifdef GGML_USE_HIP
+            const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+            if (use_gqa_opt && Q->ne[1] == 1 && gqa_ratio == 6 && K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_Q8_0 && GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+                launch_fattn_tile_switch_ncols1<DKQ, DV, 3, use_logit_softcap>(ctx, dst);
+                return;
+            }
+#endif // GGML_USE_HIP
+        }
         if (use_gqa_opt && gqa_ratio % 8 == 0) {
             launch_fattn_tile_switch_ncols1<DKQ, DV, 8, use_logit_softcap>(ctx, dst);
             return;
diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu
index ab7a3b297c07..574e48da826e 100644
--- a/ggml/src/ggml-cuda/fattn.cu
+++ b/ggml/src/ggml-cuda/fattn.cu
@@ -355,6 +355,20 @@ static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) {
     }
 }
 
+static bool ggml_cuda_fattn_tile_q8_0_KV_supported(const ggml_tensor * dst) {
+#ifdef GGML_USE_HIP
+    const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+    const ggml_tensor * Q = dst->src[0];
+    const ggml_tensor * K = dst->src[1];
+    const ggml_tensor * V = dst->src[2];
+    return GGML_CUDA_CC_IS_RDNA3_5(cc) && Q->ne[1] == 1 && K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_Q8_0 && Q->ne[0] == K->ne[0] && K->ne[0] == V->ne[0] &&
+        (Q->ne[0] == 64 || Q->ne[0] == 128 || Q->ne[0] == 256);
+#else
+    GGML_UNUSED(dst);
+    return false;
+#endif // GGML_USE_HIP
+}
+
 static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const ggml_tensor * dst) {
 #ifndef FLASH_ATTN_AVAILABLE
     GGML_UNUSED(device); GGML_UNUSED(dst);
@@ -488,6 +502,10 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
         gqa_ratio_eff *= 2;
     }
 
+    if (ggml_cuda_fattn_tile_q8_0_KV_supported(dst) && gqa_opt_applies && gqa_ratio_eff >= 2) {
+        return BEST_FATTN_KERNEL_TILE;
+    }
+
     if (volta_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) {
         if (can_use_vector_kernel && Q->ne[1] * gqa_ratio_eff <= 2) {
             return BEST_FATTN_KERNEL_VEC;
@@ -516,6 +534,12 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
         return BEST_FATTN_KERNEL_MMA_F16;
     }
 
+    // On RDNA3.5 the D=256 tile kernel is FMA-bound for prefill batches; the WMMA kernel with 64 columns is ~30-45% faster.
+    // For smaller batches (ncols 16/32 configs) the tile kernel is still faster.
+    if (GGML_CUDA_CC_IS_RDNA3_5(cc) && gqa_opt_applies && Q->ne[0] == 256 && V->ne[0] == 256 && Q->ne[1] * gqa_ratio_eff > 32) {
+        return BEST_FATTN_KERNEL_MMA_F16;
+    }
+
     // If there are no tensor cores available, use the generic tile kernel:
     if (can_use_vector_kernel) {
         if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) {
@@ -549,6 +573,12 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d
 
     switch (kernel) {
         case BEST_FATTN_KERNEL_TILE:
+            if (ggml_cuda_fattn_tile_q8_0_KV_supported(dst)) {
+                break;
+            }
+            need_f16_K = true;
+            need_f16_V = true;
+            break;
         case BEST_FATTN_KERNEL_MMA_F16:
             need_f16_K = true;
             need_f16_V = true;
diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu
index 1b431a724d72..ff2ac52745ea 100644
--- a/ggml/src/ggml-cuda/gated_delta_net.cu
+++ b/ggml/src/ggml-cuda/gated_delta_net.cu
@@ -1,8 +1,55 @@
 #include "gated_delta_net.cuh"
 #include "ggml-cuda/common.cuh"
+#include "unary.cuh"
+
+
+// Warp reduction without the LDS crossbar: on RDNA the generic warp_reduce_sum compiles to 5 dependent
+// ds_bpermute round trips (~100+ cycles each), which dominates the per-token latency of the recurrence.
+// Same pairing/order of additions as warp_reduce_sum<32> (xor 16, 8, 4, 2, 1), so the result is bit-identical.
+#if defined(GGML_USE_HIP) && (defined(RDNA3) || defined(RDNA4))
+template <int mask>
+static __device__ __forceinline__ float gdn_dpp_row_xmask(const float x) {
+    return __int_as_float(__builtin_amdgcn_update_dpp(0, __float_as_int(x), 0x160 | mask, 0xf, 0xf, true));
+}
+static __device__ __forceinline__ float gdn_permlanex16_swap(const float x) {
+    return __int_as_float(__builtin_amdgcn_permlanex16(__float_as_int(x), __float_as_int(x), 0x76543210, 0xFEDCBA98, true, false));
+}
+static __device__ __forceinline__ float gdn_warp_reduce_sum32(float x) {
+    x += gdn_permlanex16_swap(x);
+    x += gdn_dpp_row_xmask<8>(x);
+    x += gdn_dpp_row_xmask<4>(x);
+    x += gdn_dpp_row_xmask<2>(x);
+    x += gdn_dpp_row_xmask<1>(x);
+    return x;
+}
+#define GDN_DPP_REDUCE 1
+#endif // defined(GGML_USE_HIP) && (defined(RDNA3) || defined(RDNA4))
+
+template <int width>
+static __device__ __forceinline__ float gdn_warp_reduce_sum(const float x) {
+#if defined(GDN_DPP_REDUCE)
+    if constexpr (width == 32) {
+        return gdn_warp_reduce_sum32(x);
+    }
+#endif // defined(GDN_DPP_REDUCE)
+    return warp_reduce_sum<width>(x);
+}
+
+static constexpr int gated_delta_net_num_warps(int cc) {
+    return GGML_CUDA_CC_IS_RDNA3_5(cc) ? 32 : 4;
+}
+
+static_assert(gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3_5) == 32);
+static_assert(gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3) == 4);
+
+#if defined(RDNA3_5)
+static constexpr int gdn_num_warps = gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3_5);
+#else
+static constexpr int gdn_num_warps = gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3);
+#endif
 
 template <int S_v, bool KDA, bool keep_rs_t>
-__global__ void __launch_bounds__((ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v) * 4, 2)
+__global__ void __launch_bounds__((ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v) * gdn_num_warps, 2)
 gated_delta_net_cuda(const float * q,
                                      const float * k,
                                      const float * v,
@@ -90,7 +137,7 @@ gated_delta_net_cuda(const float * q,
             for (int r = 0; r < rows_per_lane; r++) {
                 kv_shard += s_shard[r] * k_reg[r];
             }
-            float kv_col = warp_reduce_sum<warp_size>(kv_shard);
+            float kv_col = gdn_warp_reduce_sum<warp_size>(kv_shard);
 
             // delta[col] = (v[col] - g * kv[col]) * beta
             float delta_col = (v_t[col] - g_val * kv_col) * beta_val;
@@ -104,7 +151,7 @@ gated_delta_net_cuda(const float * q,
                 attn_partial += s_shard[r] * q_reg[r];
             }
 
-            float attn_col = warp_reduce_sum<warp_size>(attn_partial);
+            float attn_col = gdn_warp_reduce_sum<warp_size>(attn_partial);
 
             if (lane == 0) {
                 attn_data[col] = attn_col * scale;
@@ -118,7 +165,7 @@ gated_delta_net_cuda(const float * q,
                 kv_shard += expf(g_t[i]) * s_shard[r] * k_reg[r];
             }
 
-            float kv_col = warp_reduce_sum<warp_size>(kv_shard);
+            float kv_col = gdn_warp_reduce_sum<warp_size>(kv_shard);
 
             // delta[col] = (v[col] - kv[col]) * beta
             float delta_col = (v_t[col] - kv_col) * beta_val;
@@ -133,7 +180,7 @@ gated_delta_net_cuda(const float * q,
                 attn_partial += s_shard[r] * q_reg[r];
             }
 
-            float attn_col = warp_reduce_sum<warp_size>(attn_partial);
+            float attn_col = gdn_warp_reduce_sum<warp_size>(attn_partial);
 
             if (lane == 0) {
                 attn_data[col] = attn_col * scale;
@@ -166,6 +213,292 @@ gated_delta_net_cuda(const float * q,
     }
 }
 
+template <int H>
+__global__ void __launch_bounds__(512, 2)
+gated_delta_net_kda_tiled_128_cuda(const float * q,
+                                   const float * k,
+                                   const float * v,
+                                   const float * g,
+                                   const float * beta,
+                                   const float * curr_state,
+                                   float *       dst,
+                                   float *       state,
+                                   int64_t       n_tokens,
+                                   int64_t       sq1,
+                                   int64_t       sq2,
+                                   int64_t       sq3,
+                                   int64_t       sv1,
+                                   int64_t       sv2,
+                                   int64_t       sv3,
+                                   int64_t       sb1,
+                                   int64_t       sb2,
+                                   int64_t       sb3,
+                                   const uint3   neqk1_magic,
+                                   const uint3   rq3_magic,
+                                   float         scale) {
+    constexpr int S_v = 128;
+    constexpr int token_tile = 16;
+    constexpr int warp_size = 32;
+    constexpr int rows_per_lane = S_v / warp_size;
+
+    __shared__ float q_shared[token_tile][S_v];
+    __shared__ float k_shared[token_tile][S_v];
+    __shared__ float g_shared[token_tile][S_v];
+    __shared__ float beta_shared[token_tile];
+
+    const int h_idx = blockIdx.x;
+    const int sequence = blockIdx.y;
+    const int lane = threadIdx.x;
+    const int col = blockIdx.z * blockDim.y + threadIdx.y;
+    const int thread = threadIdx.y * warp_size + lane;
+    const int nthreads = blockDim.y * warp_size;
+
+    const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic);
+    const uint32_t iq3 = fastdiv(sequence, rq3_magic);
+
+    const int64_t state_offset = (sequence * H + h_idx) * S_v * S_v;
+    curr_state += state_offset + col * S_v;
+    state += state_offset;
+    dst += (sequence * n_tokens * H + h_idx) * S_v;
+
+    float s_shard[rows_per_lane];
+    ggml_cuda_pdl_sync();
+#pragma unroll
+    for (int r = 0; r < rows_per_lane; ++r) {
+        s_shard[r] = curr_state[r * warp_size + lane];
+    }
+
+    for (int t0 = 0; t0 < n_tokens; t0 += token_tile) {
+        const int tile_size = min((int64_t) token_tile, n_tokens - t0);
+        for (int idx = thread; idx < tile_size * S_v; idx += nthreads) {
+            const int tt = idx / S_v;
+            const int i = idx % S_v;
+            const int t = t0 + tt;
+            const int64_t gb_offset = sequence * sb3 + t * sb2 + h_idx * sb1;
+            q_shared[tt][i] = q[iq3 * sq3 + t * sq2 + iq1 * sq1 + i];
+            k_shared[tt][i] = k[iq3 * sq3 + t * sq2 + iq1 * sq1 + i];
+            g_shared[tt][i] = g[gb_offset * S_v + i];
+        }
+        if (thread < tile_size) {
+            beta_shared[thread] = beta[sequence * sb3 + (t0 + thread) * sb2 + h_idx * sb1];
+        }
+        __syncthreads();
+
+        for (int tt = 0; tt < tile_size; ++tt) {
+            const int t = t0 + tt;
+            const float * v_t = v + sequence * sv3 + t * sv2 + h_idx * sv1;
+            float k_reg[rows_per_lane];
+            float q_reg[rows_per_lane];
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; ++r) {
+                const int i = r * warp_size + lane;
+                k_reg[r] = k_shared[tt][i];
+                q_reg[r] = q_shared[tt][i];
+            }
+
+            float kv_shard = 0.0f;
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; ++r) {
+                const int i = r * warp_size + lane;
+                kv_shard += expf(g_shared[tt][i]) * s_shard[r] * k_reg[r];
+            }
+            const float kv_col = gdn_warp_reduce_sum<warp_size>(kv_shard);
+            const float delta_col = (v_t[col] - kv_col) * beta_shared[tt];
+
+            float attn_partial = 0.0f;
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; ++r) {
+                const int i = r * warp_size + lane;
+                s_shard[r] = expf(g_shared[tt][i]) * s_shard[r] + k_reg[r] * delta_col;
+                attn_partial += s_shard[r] * q_reg[r];
+            }
+            const float attn_col = gdn_warp_reduce_sum<warp_size>(attn_partial);
+            if (lane == 0) {
+                dst[(int64_t) t * S_v * H + col] = attn_col * scale;
+            }
+        }
+        __syncthreads();
+    }
+
+#pragma unroll
+    for (int r = 0; r < rows_per_lane; ++r) {
+        state[col * S_v + r * warp_size + lane] = s_shard[r];
+    }
+}
+
+// Non-KDA variant with the per-token inputs (q, k, g, beta, v) staged in shared memory for a tile of tokens,
+// and COLS state columns per warp (k/q are loaded once per token for all columns, and the number of waves
+// competing for issue slots shrinks). The recurrence itself is unchanged: for every column the same
+// operations are executed in the same order as in gated_delta_net_cuda, so results are bit-identical.
+template <int S_v, int NUM_WARPS, int COLS, int TOKEN_TILE, bool keep_rs_t>
+__global__ void __launch_bounds__(32 * NUM_WARPS, 1)
+gated_delta_net_tiled_cuda(const float * q,
+                           const float * k,
+                           const float * v,
+                           const float * g,
+                           const float * beta,
+                           const float * curr_state,
+                           float *       dst,
+                           float *       state,
+                           int64_t       H,
+                           int64_t       n_tokens,
+                           int64_t       sq1,
+                           int64_t       sq2,
+                           int64_t       sq3,
+                           int64_t       sv1,
+                           int64_t       sv2,
+                           int64_t       sv3,
+                           int64_t       sb1,
+                           int64_t       sb2,
+                           int64_t       sb3,
+                           const uint3   neqk1_magic,
+                           const uint3   rq3_magic,
+                           float         scale,
+                           int64_t       state_slot_stride,
+                           int           K) {
+    constexpr int warp_size     = 32;
+    constexpr int rows_per_lane = S_v / warp_size;
+    constexpr int block_cols    = NUM_WARPS * COLS;
+    static_assert(S_v % warp_size == 0, "S_v must be a multiple of the warp size");
+    static_assert(S_v % block_cols == 0, "block columns must divide S_v");
+
+    __shared__ float q_shared[TOKEN_TILE][S_v];
+    __shared__ float k_shared[TOKEN_TILE][S_v];
+    __shared__ float v_shared[TOKEN_TILE][block_cols];
+    __shared__ float g_shared[TOKEN_TILE];
+    __shared__ float beta_shared[TOKEN_TILE];
+
+    const uint32_t h_idx    = blockIdx.x;
+    const uint32_t sequence = blockIdx.y;
+    const int      lane     = threadIdx.x;
+    const int      col0     = blockIdx.z * block_cols;          // first column of the block
+    const int      colw     = threadIdx.y * COLS;               // first column of this warp inside the block
+    const int      thread   = threadIdx.y * warp_size + lane;
+    constexpr int  nthreads = NUM_WARPS * warp_size;
+
+    const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic);
+    const uint32_t iq3 = fastdiv(sequence, rq3_magic);
+
+    const int64_t state_in_offset  = sequence * H * S_v * S_v + h_idx * S_v * S_v;
+    const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v;
+    state += state_out_offset;
+    curr_state += state_in_offset + (col0 + colw) * S_v;
+    float * attn_data = dst + (sequence * n_tokens * H + h_idx) * S_v + col0 + colw;
+
+    float s_shard[COLS][rows_per_lane];
+
+    ggml_cuda_pdl_sync();
+#pragma unroll
+    for (int c = 0; c < COLS; c++) {
+#pragma unroll
+        for (int r = 0; r < rows_per_lane; r++) {
+            s_shard[c][r] = curr_state[c * S_v + r * warp_size + lane];
+        }
+    }
+
+    for (int t0 = 0; t0 < n_tokens; t0 += TOKEN_TILE) {
+        const int tile_size = min((int64_t) TOKEN_TILE, n_tokens - t0);
+
+        for (int idx = thread; idx < tile_size * S_v; idx += nthreads) {
+            const int tt = idx / S_v;
+            const int i  = idx % S_v;
+            const int t  = t0 + tt;
+            q_shared[tt][i] = q[iq3 * sq3 + t * sq2 + iq1 * sq1 + i];
+            k_shared[tt][i] = k[iq3 * sq3 + t * sq2 + iq1 * sq1 + i];
+        }
+        for (int idx = thread; idx < tile_size * block_cols; idx += nthreads) {
+            const int tt = idx / block_cols;
+            const int c  = idx % block_cols;
+            const int t  = t0 + tt;
+            v_shared[tt][c] = v[sequence * sv3 + t * sv2 + h_idx * sv1 + col0 + c];
+        }
+        if (thread < tile_size) {
+            const int64_t gb_offset = sequence * sb3 + (t0 + thread) * sb2 + h_idx * sb1;
+            g_shared[thread]    = g[gb_offset];
+            beta_shared[thread] = beta[gb_offset];
+        }
+        __syncthreads();
+
+        for (int tt = 0; tt < tile_size; ++tt) {
+            const int t = t0 + tt;
+
+            float k_reg[rows_per_lane];
+            float q_reg[rows_per_lane];
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; r++) {
+                const int i = r * warp_size + lane;
+                k_reg[r] = k_shared[tt][i];
+                q_reg[r] = q_shared[tt][i];
+            }
+
+            const float g_val    = expf(g_shared[tt]);
+            const float beta_val = beta_shared[tt];
+
+            // The FMA contractions are spelled out so that the results are bit-identical to the
+            // compiler's contraction of gated_delta_net_cuda (fma chains from 0, fma(-g, kv, v)*beta,
+            // fma(g, s, k*delta)) independently of the surrounding code.
+            float attn_col[COLS];
+#pragma unroll
+            for (int c = 0; c < COLS; c++) {
+                // kv[col] = (S^T @ k)[col] = sum_i S[i][col] * k[i]
+                float kv_shard = 0.0f;
+#pragma unroll
+                for (int r = 0; r < rows_per_lane; r++) {
+                    kv_shard = fmaf(s_shard[c][r], k_reg[r], kv_shard);
+                }
+                const float kv_col = gdn_warp_reduce_sum<warp_size>(kv_shard);
+
+                // delta[col] = (v[col] - g * kv[col]) * beta
+                const float delta_col = fmaf(-g_val, kv_col, v_shared[tt][colw + c]) * beta_val;
+
+                // fused: S[i][col] = g * S[i][col] + k[i] * delta[col]
+                // attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i]
+                float attn_partial = 0.0f;
+#pragma unroll
+                for (int r = 0; r < rows_per_lane; r++) {
+                    s_shard[c][r] = fmaf(g_val, s_shard[c][r], k_reg[r] * delta_col);
+                    attn_partial  = fmaf(s_shard[c][r], q_reg[r], attn_partial);
+                }
+                attn_col[c] = gdn_warp_reduce_sum<warp_size>(attn_partial);
+            }
+
+            if (lane < COLS) {
+                float a = attn_col[0];
+#pragma unroll
+                for (int c = 1; c < COLS; c++) {
+                    a = lane == c ? attn_col[c] : a;
+                }
+                attn_data[(int64_t) t * S_v * H + lane] = a * scale;
+            }
+
+            if constexpr (keep_rs_t) {
+                const int target_slot = (int) n_tokens - 1 - t;
+                if (target_slot >= 0 && target_slot < K) {
+                    float * snapshot = state + target_slot * state_slot_stride + (col0 + colw) * S_v;
+#pragma unroll
+                    for (int c = 0; c < COLS; c++) {
+#pragma unroll
+                        for (int r = 0; r < rows_per_lane; r++) {
+                            snapshot[c * S_v + r * warp_size + lane] = s_shard[c][r];
+                        }
+                    }
+                }
+            }
+        }
+        __syncthreads();
+    }
+
+    if constexpr (!keep_rs_t) {
+#pragma unroll
+        for (int c = 0; c < COLS; c++) {
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; r++) {
+                state[(col0 + colw + c) * S_v + r * warp_size + lane] = s_shard[c][r];
+            }
+        }
+    }
+}
+
 template <bool KDA, bool keep_rs_t>
 static void launch_gated_delta_net(
         const float * q_d, const float * k_d, const float * v_d,
@@ -178,14 +511,54 @@ static void launch_gated_delta_net(
         int64_t neqk1, int64_t rq3,
         float scale, int64_t state_slot_stride, int K, cudaStream_t stream) {
     //TODO: Add chunked kernel for even faster pre-fill
-    const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
-    const int num_warps = 4;
+    const int id = ggml_cuda_get_device();
+    const int cc = ggml_cuda_info().devices[id].cc;
+    const int warp_size = ggml_cuda_info().devices[id].warp_size;
+    // Qwen3.6 PP2048 measured faster on gfx1151 on 2026-09-02. Retest if occupancy changes.
+    const int num_warps = GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && (H == 32 || H == 48 || H == 64) && !KDA ? 32 :
+        (GGML_CUDA_CC_IS_RDNA3_5(cc) ? 16 : gated_delta_net_num_warps(cc));
     dim3      grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps);
     dim3      block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1);
 
     const uint3 neqk1_magic = init_fastdiv_values(neqk1);
     const uint3 rq3_magic   = init_fastdiv_values(rq3);
 
+    if constexpr (KDA && !keep_rs_t) {
+        if (GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && (H == 16 || H == 32) && n_tokens >= 16 && n_seqs == 1) {
+            const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream);
+            auto kernel = H == 16 ? gated_delta_net_kda_tiled_128_cuda<16> : gated_delta_net_kda_tiled_128_cuda<32>;
+            ggml_cuda_kernel_launch(kernel, launch_params,
+                q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, n_tokens,
+                sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3,
+                neqk1_magic, rq3_magic, scale);
+            return;
+        }
+    }
+
+    if constexpr (!KDA) {
+        if (GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && num_warps == 32 && n_tokens >= 16) {
+            // 16 warps x 4 columns per block, 16-token tiles: best of the swept configurations on gfx1151
+            //     (2048 tokens, H=32: 5.03 ms -> 2.2 ms; H=64: 9.29 ms -> 4.5 ms).
+            const int cfg = getenv("GGML_GDN_TILE_CFG") ? atoi(getenv("GGML_GDN_TILE_CFG")) : (H == 48 ? 1 : 0);
+            auto launch_tiled = [&](auto kernel, int warps, int cols) {
+                const dim3 tiled_grid(H, n_seqs, S_v / (warps * cols));
+                const dim3 tiled_block(warp_size, warps, 1);
+                const ggml_cuda_kernel_launch_params launch_params(tiled_grid, tiled_block, 0, stream);
+                ggml_cuda_kernel_launch(kernel, launch_params,
+                    q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens,
+                    sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3,
+                    neqk1_magic, rq3_magic, scale, state_slot_stride, K);
+            };
+            switch (cfg) {
+                case 1: launch_tiled(gated_delta_net_tiled_cuda<128, 8,  8, 16, keep_rs_t>, 8,  8); break;
+                case 2: launch_tiled(gated_delta_net_tiled_cuda<128, 32, 2, 16, keep_rs_t>, 32, 2); break;
+                case 3: launch_tiled(gated_delta_net_tiled_cuda<128, 16, 4,  8, keep_rs_t>, 16, 4); break;
+                default: launch_tiled(gated_delta_net_tiled_cuda<128, 16, 4, 16, keep_rs_t>, 16, 4); break;
+            }
+            return;
+        }
+    }
+
     const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream);
     switch (S_v) {
         case 16:
@@ -325,3 +698,237 @@ void ggml_cuda_op_gated_delta_net_fused_cache(
         ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_cuda_gated_delta_net_fused_cache cache) {
     ggml_cuda_op_gated_delta_net_impl(ctx, dst, &cache);
 }
+
+// ---------------------------------------------------------------------------------------------
+// Fused Qwen3.5/3.6 Gated DeltaNet decode step (n_tokens == 1, n_seqs == 1, S == 128): one block per value
+// head computes the causal conv + SiLU of its q/k/v channels (writing the shifted conv state back into the
+// cache), the q/k L2 norms, the gate (softplus) and beta (sigmoid), the recurrence with the new state written
+// straight into the cache, and the gated RMS norm of the attention output. This replaces 14 graph nodes
+// (11 kernels plus their ~2 us launch gaps on gfx1151) that together take ~50 us per layer in decode.
+//
+// Every op is spelled with the same operation order and FMA contraction as the kernels it replaces
+// (ssm_conv_f32, l2_norm_dual_f32_s128, unary/binary elementwise, gated_delta_net_cuda, rms_norm_f32<128>),
+// so the outputs are bit-identical to the unfused graph.
+
+static __device__ __forceinline__ float gdn_decode_sigmoid(float x) {
+    return 1.0f / (1.0f + expf(-x));
+}
+
+static __device__ __forceinline__ float gdn_decode_softplus(float x) {
+    return (x > 20.0f) ? x : logf(1.0f + expf(x));
+}
+
+template <int S, int D_CONV>
+__global__ void __launch_bounds__(32 * 32, 1)
+gdn_decode_fused_cuda(const ggml_cuda_gdn_decode_args args) {
+    constexpr int warp_size     = 32;
+    constexpr int nwarps        = 32;
+    constexpr int rows_per_lane = S / warp_size;
+    constexpr int cols_per_warp = S / nwarps;
+    static_assert(S == warp_size * rows_per_lane && S == nwarps * cols_per_warp);
+
+    __shared__ float q_c[S];
+    __shared__ float k_c[S];
+    __shared__ float v_c[S];
+    __shared__ float q_n[S];
+    __shared__ float k_n[S];
+    __shared__ float attn[S];
+    __shared__ float red[4];
+
+    const int h      = blockIdx.x;                 // value head
+    const int lane   = threadIdx.x;
+    const int warp   = threadIdx.y;
+    const int thread = warp * warp_size + lane;
+    const int hk     = h % args.H_k;               // key/query head (fastmodulo(h_idx, neqk1) in gated_delta_net_cuda)
+
+    // 1. causal conv + SiLU for this head's q, k and v channels; shift the conv state (CPY conv_state_last)
+    if (thread < 3 * S) {
+        const int part = thread / S;               // 0: q, 1: k, 2: v
+        const int i    = thread % S;
+        const int c    = part == 0 ? hk * S : part == 1 ? args.H_k * S + hk * S : 2 * args.H_k * S + h * S;
+        const int ch   = c + i;
+
+        float x[D_CONV];
+        float w[D_CONV];
+#pragma unroll
+        for (int j = 0; j < D_CONV - 1; j++) {
+            x[j] = args.conv_state_in[ch * (D_CONV - 1) + j];
+        }
+        x[D_CONV - 1] = args.qkv[ch * args.qkv_stride];
+#pragma unroll
+        for (int j = 0; j < D_CONV; j++) {
+            w[j] = args.conv_w[ch * D_CONV + j];
+        }
+
+        // ssm_conv_f32: fma chain from 0, then the (zero) bias add, then SiLU
+        float sumf = fmaf(x[0], w[0], 0.0f);
+#pragma unroll
+        for (int j = 1; j < D_CONV; j++) {
+            sumf = fmaf(x[j], w[j], sumf);
+        }
+        sumf += args.conv_bias;
+        const float y = ggml_cuda_op_silu_single(sumf);
+
+        if (part == 0) {
+            q_c[i] = y;
+        } else if (part == 1) {
+            k_c[i] = y;
+        } else {
+            v_c[i] = y;
+        }
+
+        // q/k channels are shared by H_v/H_k value heads: the first one writes the conv state
+        if (part == 2 || h == hk) {
+#pragma unroll
+            for (int j = 0; j < D_CONV - 1; j++) {
+                args.conv_state_out[ch * (D_CONV - 1) + j] = x[j + 1];
+            }
+        }
+    }
+    __syncthreads();
+
+    // 2. l2_norm_dual_f32_s128 for q (warp 0) and k (warp 1)
+    if (warp < 2) {
+        const float * x = warp == 0 ? q_c : k_c;
+        float *       y = warp == 0 ? q_n : k_n;
+        float tmp = 0.0f;
+#pragma unroll
+        for (int r = 0; r < rows_per_lane; r++) {
+            const float xi = x[r * warp_size + lane];
+            tmp = fmaf(xi, xi, tmp);
+        }
+        tmp = gdn_warp_reduce_sum<warp_size>(tmp);
+        const float scale = rsqrtf(fmaxf(tmp, args.eps_l2 * args.eps_l2));
+#pragma unroll
+        for (int r = 0; r < rows_per_lane; r++) {
+            y[r * warp_size + lane] = scale * x[r * warp_size + lane];
+        }
+    }
+
+    // gate = softplus(alpha + dt_bias) * A, beta = sigmoid(beta): scalar per head, computed by every thread
+    const float alpha_biased = args.alpha[h] + args.dt_bias[h];
+    const float gate         = gdn_decode_softplus(alpha_biased) * args.ssm_a[h];
+    const float beta_val     = gdn_decode_sigmoid(args.beta[h]);
+    const float g_val        = expf(gate);
+    __syncthreads();
+
+    // 3. recurrence (gated_delta_net_cuda), cols_per_warp state columns per warp
+    {
+        const float * state_in  = args.state_cache + (int64_t) args.state_ids[0] * args.state_row_stride + (int64_t) h * S * S;
+        float *       state_out = args.state_out + (int64_t) h * S * S;
+        const int     colw      = warp * cols_per_warp;
+
+        float k_reg[rows_per_lane];
+        float q_reg[rows_per_lane];
+#pragma unroll
+        for (int r = 0; r < rows_per_lane; r++) {
+            k_reg[r] = k_n[r * warp_size + lane];
+            q_reg[r] = q_n[r * warp_size + lane];
+        }
+
+        float s_shard[cols_per_warp][rows_per_lane];
+#pragma unroll
+        for (int c = 0; c < cols_per_warp; c++) {
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; r++) {
+                s_shard[c][r] = state_in[(colw + c) * S + r * warp_size + lane];
+            }
+        }
+
+        // same explicit contractions as gated_delta_net_tiled_cuda (bit-identical to gated_delta_net_cuda)
+        float attn_col[cols_per_warp];
+#pragma unroll
+        for (int c = 0; c < cols_per_warp; c++) {
+            float kv_shard = 0.0f;
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; r++) {
+                kv_shard = fmaf(s_shard[c][r], k_reg[r], kv_shard);
+            }
+            const float kv_col = gdn_warp_reduce_sum<warp_size>(kv_shard);
+
+            const float delta_col = fmaf(-g_val, kv_col, v_c[colw + c]) * beta_val;
+
+            float attn_partial = 0.0f;
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; r++) {
+                s_shard[c][r] = fmaf(g_val, s_shard[c][r], k_reg[r] * delta_col);
+                attn_partial  = fmaf(s_shard[c][r], q_reg[r], attn_partial);
+            }
+            attn_col[c] = gdn_warp_reduce_sum<warp_size>(attn_partial);
+        }
+
+        if (lane < cols_per_warp) {
+            float a = attn_col[0];
+#pragma unroll
+            for (int c = 1; c < cols_per_warp; c++) {
+                a = lane == c ? attn_col[c] : a;
+            }
+            attn[colw + lane] = a * args.scale;
+        }
+
+#pragma unroll
+        for (int c = 0; c < cols_per_warp; c++) {
+#pragma unroll
+            for (int r = 0; r < rows_per_lane; r++) {
+                state_out[(colw + c) * S + r * warp_size + lane] = s_shard[c][r];
+            }
+        }
+    }
+    __syncthreads();
+
+    // 4. pre-norm attention output to the scratch buffer (the gated rms norm runs as a second kernel: its
+    //    destination may alias inputs of this kernel that other blocks are still reading)
+    if (thread < S) {
+        args.attn_out[(int64_t) h * S + thread] = attn[thread];
+    }
+}
+
+// rms_norm_f32<128, true>: per-warp partial sums, then the second-level butterfly of block_reduce over
+// lanes 0..3 (which reduces to (s0 + s2) + (s1 + s3)), times the norm weight
+template <int S>
+__global__ void __launch_bounds__(S, 1)
+gdn_decode_norm_cuda(const float * attn, const float * norm_w, float * out, const float eps) {
+    constexpr int warp_size = 32;
+    static_assert(S == 4 * warp_size);
+    __shared__ float red[4];
+
+    const int h    = blockIdx.x;
+    const int tid  = threadIdx.x;
+    const int warp = tid / warp_size;
+    const int lane = tid % warp_size;
+
+    const float xi   = attn[(int64_t) h * S + tid];
+    const float part = gdn_warp_reduce_sum<warp_size>(fmaf(xi, xi, 0.0f));
+    if (lane == 0) {
+        red[warp] = part;
+    }
+    __syncthreads();
+    const float tmp   = (red[0] + red[2]) + (red[1] + red[3]);
+    const float mean  = tmp / (float) S;
+    const float scale = rsqrtf(mean + eps);
+    out[(int64_t) h * S + tid] = scale * xi * norm_w[tid];
+}
+
+void ggml_cuda_op_gdn_decode_fused_prenorm(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args_in, float * attn_scratch) {
+    GGML_ASSERT(args_in.S == 128 && args_in.d_conv == 4);
+    ggml_cuda_gdn_decode_args args = args_in;
+    args.attn_out = attn_scratch;
+
+    const dim3 grid(args.H_v, 1, 1);
+    const dim3 block(32, 32, 1);
+    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid, block, 0, ctx.stream());
+    ggml_cuda_kernel_launch(gdn_decode_fused_cuda<128, 4>, launch_params, args);
+}
+
+void ggml_cuda_op_gdn_decode_fused(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args_in) {
+    GGML_ASSERT(args_in.S == 128 && args_in.d_conv == 4);
+    ggml_cuda_pool_alloc<float> attn(ctx.pool(), args_in.S * args_in.H_v);
+    ggml_cuda_gdn_decode_args args = args_in;
+    args.attn_out = attn.get();
+    ggml_cuda_op_gdn_decode_fused_prenorm(ctx, args, attn.get());
+
+    const dim3 grid(args.H_v, 1, 1);
+    const dim3 norm_block(128, 1, 1);
+    const ggml_cuda_kernel_launch_params norm_params = ggml_cuda_kernel_launch_params(grid, norm_block, 0, ctx.stream());
+    ggml_cuda_kernel_launch(gdn_decode_norm_cuda<128>, norm_params, attn.get(), args.norm_w, args.out, args.eps_rms);
+}
diff --git a/ggml/src/ggml-cuda/gated_delta_net.cuh b/ggml/src/ggml-cuda/gated_delta_net.cuh
index f9bf43706789..e2b41cf1c47f 100644
--- a/ggml/src/ggml-cuda/gated_delta_net.cuh
+++ b/ggml/src/ggml-cuda/gated_delta_net.cuh
@@ -12,3 +12,37 @@ void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor *
 // same op, but writes the snapshot(s) into the cache instead of dst (see ggml_cuda_try_gdn_cache_fusion)
 void ggml_cuda_op_gated_delta_net_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst,
                                               ggml_cuda_gated_delta_net_fused_cache cache);
+
+// Fused Qwen3.5 Gated DeltaNet decode step (see ggml_cuda_try_gdn_decode_fusion). All strides in elements.
+struct ggml_cuda_gdn_decode_args {
+    // conv: conv_state_in [d_conv-1, C] (contiguous, per channel), qkv [C] with stride, conv_w [d_conv, C]
+    const float * conv_state_in;
+    const float * qkv;
+    int64_t       qkv_stride;
+    const float * conv_w;
+    float         conv_bias;      // 0.0f (ssm_conv adds a runtime zero when there is no bias)
+    float *       conv_state_out; // [d_conv-1, C] (contiguous) cache row
+    // gate / beta
+    const float * alpha;          // [H_v]
+    const float * dt_bias;        // [H_v]
+    const float * ssm_a;          // [H_v]
+    const float * beta;           // [H_v]
+    float         eps_l2;
+    // recurrence
+    const float *   state_cache;      // [S*S*H_v, n_slots] cache, row = state_ids[0] (transposed per head: [col][row])
+    const int32_t * state_ids;
+    int64_t         state_row_stride;
+    float *       state_out;      // [S, S, H_v] cache row
+    float         scale;          // 1/sqrt(S)
+    // gated rms norm
+    const float * norm_w;         // [S]
+    const ggml_tensor * norm_w_tensor = nullptr;
+    float         eps_rms;
+    float *       out;            // [S, H_v]
+    float *       attn_out;       // [S, H_v] pre-norm scratch (set by the host)
+    int64_t       S, H_k, H_v, d_conv;
+};
+
+void ggml_cuda_op_gdn_decode_fused(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args);
+// first kernel only: writes the pre-norm attention output [S, H_v] to attn_scratch (the gated norm is applied by the consumer)
+void ggml_cuda_op_gdn_decode_fused_prenorm(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args, float * attn_scratch);
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index a6fc655c41c9..9d6df9b09141 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -59,6 +59,7 @@
 #include "ggml-cuda/wkv.cuh"
 #include "ggml-cuda/gla.cuh"
 #include "ggml-cuda/gated_delta_net.cuh"
+#include "ggml-cuda/hyperconn.cuh"
 #include "ggml-cuda/dsv4-hc.cuh"
 #include "ggml-cuda/set.cuh"
 #include "ggml-cuda/set-rows.cuh"
@@ -1835,7 +1836,8 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
     const int cc        = ggml_cuda_info().devices[ctx.device].cc;
     const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;
 
-    if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, ne11)) {
+    const int64_t mmvf_ncols = hint == GGML_HINT_EXACT_BATCH && src0->type == GGML_TYPE_BF16 ? 1 : ne11;
+    if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, mmvf_ncols)) {
         // The custom F16 vector kernel can be used over batched cuBLAS GEMM.
         // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention)
         ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);
@@ -3025,6 +3027,249 @@ static bool ggml_cuda_check_fusion_memory_ranges(const ggml_cgraph * cgraph,
     return is_ok;
 }
 
+// match the Qwen3.5 Gated DeltaNet decode chain
+//     concat(conv_state, qkv^T) -> cpy(conv state shift -> cache) -> get_rows(state cache) -> ssm_conv -> silu
+//     -> l2_norm(q) -> l2_norm(k) -> add(alpha, dt_bias) -> softplus -> mul(A) -> sigmoid(beta)
+//     -> gated_delta_net -> cpy(new state -> cache) -> rms_norm -> mul(norm weight)
+// for a single token of a single sequence with S == 128, so that one kernel can replace the 14 nodes.
+// Returns the number of nodes to skip after the concat (0 when the pattern does not match).
+static int ggml_cuda_try_gdn_decode_fusion(const ggml_cgraph * cgraph, int node_idx, ggml_cuda_gdn_decode_args & args) {
+    const ggml_tensor * concat = cgraph->nodes[node_idx];
+    if (concat->op != GGML_OP_CONCAT || concat->type != GGML_TYPE_F32 || ggml_get_op_params_i32(concat, 0) != 0) {
+        return 0;
+    }
+    const int64_t d_conv = concat->ne[0];
+    const int64_t C      = concat->ne[1];
+    if (d_conv != 4 || concat->ne[2] != 1 || concat->ne[3] != 1 || !ggml_is_contiguous(concat)) {
+        return 0;
+    }
+    const ggml_tensor * conv_state = concat->src[0]; // [d_conv-1, C, 1]
+    const ggml_tensor * qkv_t      = concat->src[1]; // [1, C, 1]
+    if (conv_state->type != GGML_TYPE_F32 || qkv_t->type != GGML_TYPE_F32 ||
+        conv_state->ne[0] != d_conv - 1 || conv_state->ne[1] != C || conv_state->ne[2] != 1 || !ggml_is_contiguous(conv_state) ||
+        qkv_t->ne[0] != 1 || qkv_t->ne[1] != C || qkv_t->ne[2] != 1 || qkv_t->nb[1] % sizeof(float) != 0) {
+        return 0;
+    }
+
+    // collect the following compute nodes
+    constexpr int n_ops = 14;
+    const ggml_op expected[n_ops] = {
+        GGML_OP_CPY, GGML_OP_GET_ROWS, GGML_OP_SSM_CONV, GGML_OP_UNARY, GGML_OP_L2_NORM, GGML_OP_L2_NORM,
+        GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_UNARY, GGML_OP_GATED_DELTA_NET, GGML_OP_CPY,
+        GGML_OP_RMS_NORM, GGML_OP_MUL,
+    };
+    const ggml_tensor * n[n_ops];
+    int idx[n_ops];
+    int found = 0;
+    int last  = node_idx;
+    for (int j = node_idx + 1; j < cgraph->n_nodes && found < n_ops; ++j) {
+        const ggml_tensor * t = cgraph->nodes[j];
+        if (ggml_cuda_is_view_or_noop(t)) {
+            continue;
+        }
+        if (t->op != expected[found] || t->type != GGML_TYPE_F32) {
+            return 0;
+        }
+        n[found]   = t;
+        idx[found] = j;
+        last       = j;
+        ++found;
+    }
+    if (found != n_ops) {
+        return 0;
+    }
+    // intermediates must not be graph outputs
+    if (concat->flags & GGML_TENSOR_FLAG_OUTPUT) {
+        return 0;
+    }
+    for (int k = 0; k < n_ops; ++k) {
+        if (k != 0 && k != 11 && k != 13 && (n[k]->flags & GGML_TENSOR_FLAG_OUTPUT)) {
+            return 0;
+        }
+    }
+
+    const ggml_tensor * cpy_conv = n[0];
+    const ggml_tensor * get_rows = n[1];
+    const ggml_tensor * ssm_conv = n[2];
+    const ggml_tensor * silu     = n[3];
+    const ggml_tensor * l2_q     = n[4];
+    const ggml_tensor * l2_k     = n[5];
+    const ggml_tensor * add      = n[6];
+    const ggml_tensor * softplus = n[7];
+    const ggml_tensor * mul_a    = n[8];
+    const ggml_tensor * sigmoid  = n[9];
+    const ggml_tensor * gdn      = n[10];
+    const ggml_tensor * rms      = n[12];
+    const ggml_tensor * mul_w    = n[13];
+
+    auto is_view_of = [](const ggml_tensor * t, const ggml_tensor * src, size_t offs) {
+        return t == src ? offs == 0 : (t->view_src == src && t->view_offs == offs);
+    };
+
+    // conv state shift: view of the concat (skipping the oldest column) copied into a contiguous cache row
+    {
+        const ggml_tensor * src = cpy_conv->src[0];
+        const ggml_tensor * dst = cpy_conv->src[1];
+        if (!is_view_of(src, concat, sizeof(float)) || src->ne[0] != d_conv - 1 || src->ne[1] != C || src->ne[2] != 1 ||
+            src->nb[1] != concat->nb[1] || dst->type != GGML_TYPE_F32 || dst->data == nullptr ||
+            ggml_nelements(dst) != (d_conv - 1) * C || !ggml_is_contiguous(dst)) {
+            return 0;
+        }
+    }
+
+    // ssm_conv(concat, weight [d_conv, C]) without bias, then silu
+    if (ssm_conv->src[0] != concat || ssm_conv->src[2] != nullptr || ssm_conv->src[1]->type != GGML_TYPE_F32 ||
+        ssm_conv->src[1]->ne[0] != d_conv || ssm_conv->src[1]->ne[1] != C || !ggml_is_contiguous(ssm_conv->src[1]) ||
+        ssm_conv->ne[0] != C || ggml_nelements(ssm_conv) != C ||
+        ggml_get_unary_op(silu) != GGML_UNARY_OP_SILU || silu->src[0] != ssm_conv || !ggml_is_contiguous(silu)) {
+        return 0;
+    }
+
+    // recurrence shapes
+    const ggml_tensor * v_view = gdn->src[2];
+    const int64_t S   = v_view->ne[0];
+    const int64_t H_v = v_view->ne[1];
+    const int64_t H_k = l2_q->ne[1];
+    if (S != 128 || H_k <= 0 || H_v % H_k != 0 || v_view->ne[2] != 1 || v_view->ne[3] != 1 ||
+        2 * S * H_k + S * H_v != C || ggml_get_op_params_i32(gdn, 0) != 1) {
+        return 0;
+    }
+
+    // q/k/v are views into the silu output: [S, H, 1, 1] with rows of S floats
+    auto qkv_view_ok = [&](const ggml_tensor * t, int64_t H, size_t offs) {
+        return t->view_src == silu && t->view_offs == offs && t->ne[0] == S && t->ne[1] == H && t->ne[2] == 1 &&
+               t->ne[3] == 1 && t->nb[0] == sizeof(float) && t->nb[1] == S * sizeof(float);
+    };
+    if (!qkv_view_ok(l2_q->src[0], H_k, 0) || !qkv_view_ok(l2_k->src[0], H_k, S * H_k * sizeof(float)) ||
+        !qkv_view_ok(v_view, H_v, 2 * S * H_k * sizeof(float)) ||
+        !ggml_are_same_shape(l2_q, l2_q->src[0]) || !ggml_are_same_shape(l2_k, l2_k->src[0]) ||
+        !ggml_is_contiguous(l2_q) || !ggml_is_contiguous(l2_k)) {
+        return 0;
+    }
+    float eps_q, eps_k;
+    memcpy(&eps_q, l2_q->op_params, sizeof(float));
+    memcpy(&eps_k, l2_k->op_params, sizeof(float));
+    if (eps_q != eps_k) {
+        return 0;
+    }
+
+    // gate = softplus(alpha + dt_bias) * A, beta = sigmoid(beta): H_v scalars
+    if (ggml_nelements(add) != H_v || add->src[0]->type != GGML_TYPE_F32 || add->src[1]->type != GGML_TYPE_F32 ||
+        ggml_nelements(add->src[0]) != H_v || ggml_nelements(add->src[1]) != H_v ||
+        !ggml_is_contiguous(add->src[0]) || !ggml_is_contiguous(add->src[1]) ||
+        ggml_get_unary_op(softplus) != GGML_UNARY_OP_SOFTPLUS || softplus->src[0] != add ||
+        mul_a->src[0] != softplus || mul_a->src[1]->type != GGML_TYPE_F32 || ggml_nelements(mul_a->src[1]) != H_v ||
+        !ggml_is_contiguous(mul_a->src[1]) || ggml_nelements(mul_a) != H_v ||
+        ggml_get_unary_op(sigmoid) != GGML_UNARY_OP_SIGMOID || sigmoid->src[0]->type != GGML_TYPE_F32 ||
+        ggml_nelements(sigmoid->src[0]) != H_v || !ggml_is_contiguous(sigmoid->src[0]) || ggml_nelements(sigmoid) != H_v) {
+        return 0;
+    }
+
+    // gated_delta_net(q, k, v, gate, beta, state) with the state gathered from the cache by get_rows
+    if (gdn->src[0] != l2_q || gdn->src[1] != l2_k || !is_view_of(gdn->src[3], mul_a, 0) || !is_view_of(gdn->src[4], sigmoid, 0) ||
+        gdn->src[3]->ne[0] != 1 || ggml_nelements(gdn->src[3]) != H_v || ggml_nelements(gdn->src[4]) != H_v ||
+        !is_view_of(gdn->src[5], get_rows, 0) || ggml_nelements(get_rows) != S * S * H_v) {
+        return 0;
+    }
+    {
+        const ggml_tensor * cache = get_rows->src[0];
+        const ggml_tensor * ids   = get_rows->src[1];
+        if (cache->type != GGML_TYPE_F32 || cache->ne[0] != S * S * H_v || cache->nb[0] != sizeof(float) ||
+            cache->nb[1] % sizeof(float) != 0 || ids->type != GGML_TYPE_I32 || ggml_nelements(ids) != 1) {
+            return 0;
+        }
+    }
+
+    // new state -> cache (same check as the standalone gated_delta_net + cpy fusion)
+    ggml_cuda_gated_delta_net_fused_cache state_cpy;
+    if (ggml_cuda_try_gdn_cache_fusion(cgraph, idx[10], state_cpy) != idx[11] - idx[10] || state_cpy.slot_stride != 0) {
+        return 0;
+    }
+
+    // gated rms norm of the attention output [S, H_v]
+    {
+        const ggml_tensor * x = rms->src[0];
+        if (x->view_src != gdn || x->view_offs != 0 || x->ne[0] != S || x->ne[1] != H_v || x->ne[2] != 1 || x->ne[3] != 1 ||
+            x->nb[0] != sizeof(float) || x->nb[1] != S * sizeof(float) || !ggml_are_same_shape(rms, x) ||
+            mul_w->src[0] != rms || mul_w->src[1]->type != GGML_TYPE_F32 || ggml_nelements(mul_w->src[1]) != S ||
+            !ggml_is_contiguous(mul_w->src[1]) || !ggml_are_same_shape(mul_w, rms) || !ggml_is_contiguous(mul_w)) {
+            return 0;
+        }
+    }
+
+    // The first kernel writes only the two cache rows: the state row may alias the gathered input row (in-place
+    // update, every block reads its head before writing it) and the conv row must not alias any external input
+    // of the range. The final output is written by a second kernel after all inputs have been consumed.
+    {
+        const ggml_tensor * out = cpy_conv->src[1];
+        for (int j = node_idx; j <= last; ++j) {
+            const ggml_tensor * t = cgraph->nodes[j];
+            for (int si = 0; si < GGML_MAX_SRC; ++si) {
+                const ggml_tensor * src = t->src[si];
+                if (!src || src->op == GGML_OP_NONE || src == out) {
+                    continue;
+                }
+                const int64_t a0 = (int64_t) out->data;
+                const int64_t a1 = a0 + ggml_backend_buft_get_alloc_size(out->buffer->buft, out);
+                const int64_t b0 = (int64_t) src->data;
+                const int64_t b1 = b0 + ggml_backend_buft_get_alloc_size(src->buffer->buft, src);
+                if ((b0 <= a0 && a0 < b1) || (a0 <= b0 && b0 < a1)) {
+                    return 0;
+                }
+            }
+        }
+    }
+
+    args.conv_state_in  = (const float *) conv_state->data;
+    args.qkv            = (const float *) qkv_t->data;
+    args.qkv_stride     = qkv_t->nb[1] / sizeof(float);
+    args.conv_w         = (const float *) ssm_conv->src[1]->data;
+    args.conv_bias      = 0.0f;
+    args.conv_state_out = (float *) cpy_conv->src[1]->data;
+    args.alpha          = (const float *) add->src[0]->data;
+    args.dt_bias        = (const float *) add->src[1]->data;
+    args.ssm_a          = (const float *) mul_a->src[1]->data;
+    args.beta           = (const float *) sigmoid->src[0]->data;
+    args.eps_l2         = eps_q;
+    args.state_cache    = (const float *) get_rows->src[0]->data;
+    args.state_ids      = (const int32_t *) get_rows->src[1]->data;
+    args.state_row_stride = get_rows->src[0]->nb[1] / sizeof(float);
+    args.state_out      = state_cpy.data;
+    args.scale          = 1.0f / sqrtf((float) S);
+    args.norm_w         = (const float *) mul_w->src[1]->data;
+    args.norm_w_tensor  = mul_w->src[1];
+    memcpy(&args.eps_rms, rms->op_params, sizeof(float));
+    args.out            = (float *) mul_w->data;
+    args.attn_out       = nullptr;
+    args.S     = S;
+    args.H_k   = H_k;
+    args.H_v   = H_v;
+    args.d_conv = d_conv;
+
+    // Every skipped node except the three outputs must have all of its consumers inside this range.
+    // Include view nodes in the count even though they are not compute nodes.
+    for (int j = node_idx; j <= last; ++j) {
+        if (j == idx[0] || j == idx[11] || j == idx[13]) {
+            continue;
+        }
+        const ggml_tensor * candidate = cgraph->nodes[j];
+        if (candidate->flags & GGML_TENSOR_FLAG_OUTPUT) {
+            return 0;
+        }
+        int internal_uses = 0;
+        for (int k = node_idx; k <= last; ++k) {
+            for (int src = 0; src < GGML_MAX_SRC; ++src) {
+                internal_uses += cgraph->nodes[k]->src[src] == candidate;
+            }
+        }
+        if (internal_uses != ggml_node_get_use_count(cgraph, j)) {
+            return 0;
+        }
+    }
+
+    return last - node_idx;
+}
+
 
 static bool ggml_cuda_can_fuse(const struct ggml_cgraph *                cgraph,
                                int                                       node_idx,
@@ -3278,6 +3523,79 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph *                cgraph,
 }
 
 // try and fuse nodes and return the number of nodes to skip
+// qwen4exp hyper-connection stream mix chain starting at node i:
+//     sigmoid(gate) -> mul(xn, .) -> reshape -> view(stream 0) -> cont -> (view(stream c) -> add) x (hc-1) -> scale
+// Fills args and ops (n_ops entries) and returns n_ops, or 0 when the nodes do not form the chain.
+static int ggml_cuda_match_hc_mix(const ggml_cgraph * cgraph, const int i, ggml_cuda_hc_mix_args & args, enum ggml_op * ops) {
+    const ggml_tensor * node = cgraph->nodes[i];
+    if (!(node->op == GGML_OP_UNARY && ggml_get_unary_op(node) == GGML_UNARY_OP_SIGMOID && node->type == GGML_TYPE_F32 &&
+        i + 5 < cgraph->n_nodes &&
+        cgraph->nodes[i + 1]->op == GGML_OP_MUL && cgraph->nodes[i + 2]->op == GGML_OP_RESHAPE &&
+        cgraph->nodes[i + 3]->op == GGML_OP_VIEW && cgraph->nodes[i + 4]->op == GGML_OP_CONT)) {
+        return 0;
+    }
+    const ggml_tensor * gate  = node->src[0];
+    const ggml_tensor * mul   = cgraph->nodes[i + 1];
+    const ggml_tensor * view0 = cgraph->nodes[i + 3];
+    const ggml_tensor * cont  = cgraph->nodes[i + 4];
+    const ggml_tensor * xn    = mul->src[0] == node ? mul->src[1] : mul->src[0];
+
+    const int64_t n_embd   = cont->ne[0];
+    const int64_t n_tokens = cont->ne[1];
+    const int64_t hc_dim   = mul->ne[0];
+    const int     hc       = n_embd > 0 ? (int) (hc_dim / n_embd) : 0;
+    const size_t  row_size = hc_dim * sizeof(float);
+
+    auto stream_view_ok = [&](const ggml_tensor * v, int c) {
+        return v->op == GGML_OP_VIEW && v->view_src == mul && v->type == GGML_TYPE_F32 &&
+            v->view_offs == (size_t) c * n_embd * sizeof(float) &&
+            v->ne[0] == n_embd && v->ne[1] == n_tokens && v->ne[2] == 1 && v->ne[3] == 1 &&
+            v->nb[0] == sizeof(float) && v->nb[1] == row_size;
+    };
+
+    bool ok = (mul->src[0] == node || mul->src[1] == node) && xn != node &&
+        mul->type == GGML_TYPE_F32 && xn->type == GGML_TYPE_F32 && gate->type == GGML_TYPE_F32 &&
+        ggml_is_contiguous(xn) && ggml_is_contiguous(gate) && ggml_are_same_shape(xn, gate) &&
+        ggml_are_same_shape(mul, xn) && ggml_are_same_shape(mul, node) &&
+        mul->ne[2] == 1 && mul->ne[3] == 1 && ggml_nrows(mul) == n_tokens &&
+        hc >= 2 && hc <= 16 && hc_dim == (int64_t) hc * n_embd &&
+        cgraph->nodes[i + 2]->view_src == mul && cont->src[0] == view0 && stream_view_ok(view0, 0) &&
+        cont->type == GGML_TYPE_F32 && ggml_is_contiguous(cont) && cont->ne[2] == 1 && cont->ne[3] == 1;
+
+    const int n_ops = ok ? 5 + 2 * (hc - 1) + 1 : 0;
+    ok = ok && i + n_ops <= cgraph->n_nodes;
+    if (!ok) {
+        return 0;
+    }
+
+    ops[0] = GGML_OP_UNARY; ops[1] = GGML_OP_MUL; ops[2] = GGML_OP_RESHAPE; ops[3] = GGML_OP_VIEW; ops[4] = GGML_OP_CONT;
+    const ggml_tensor * prev = cont;
+    for (int c = 1; ok && c < hc; ++c) {
+        const ggml_tensor * v   = cgraph->nodes[i + 5 + 2 * (c - 1)];
+        const ggml_tensor * add = cgraph->nodes[i + 6 + 2 * (c - 1)];
+        ok = stream_view_ok(v, c) && add->op == GGML_OP_ADD && add->type == GGML_TYPE_F32 &&
+            add->src[0] == prev && add->src[1] == v && ggml_are_same_shape(add, cont);
+        ops[5 + 2 * (c - 1)] = GGML_OP_VIEW;
+        ops[6 + 2 * (c - 1)] = GGML_OP_ADD;
+        prev = add;
+    }
+    ggml_tensor * scale = cgraph->nodes[i + n_ops - 1];
+    ops[n_ops - 1] = GGML_OP_SCALE;
+    ok = ok && scale->op == GGML_OP_SCALE && scale->src[0] == prev && scale->type == GGML_TYPE_F32 &&
+        ggml_is_contiguous(scale) && ggml_are_same_shape(scale, cont);
+    if (!ok) {
+        return 0;
+    }
+
+    args.xn    = xn;
+    args.gate  = gate;
+    args.dst   = scale;
+    args.hc    = hc;
+    args.scale = ggml_get_op_params_f32(scale, 0);
+    args.bias  = ggml_get_op_params_f32(scale, 1);
+    return n_ops;
+}
+
 static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) {
 
     static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));
@@ -3285,13 +3603,180 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         return 0;
     }
 
+    // debugging aid: only fuse nodes with GGML_CUDA_FUSE_MIN_NODE <= i < GGML_CUDA_FUSE_MAX_NODE
+    static const int fuse_min_node = getenv("GGML_CUDA_FUSE_MIN_NODE") ? std::atoi(getenv("GGML_CUDA_FUSE_MIN_NODE")) : 0;
+    static const int fuse_max_node = getenv("GGML_CUDA_FUSE_MAX_NODE") ? std::atoi(getenv("GGML_CUDA_FUSE_MAX_NODE")) : INT_MAX;
+    if (i < fuse_min_node || i >= fuse_max_node) {
+        return 0;
+    }
+
     ggml_tensor * node = cgraph->nodes[i];
 
+    // RDNA3.5 decode: consecutive single-column MUL_MATs that read the same activation vector (plain Q8_0/F32
+    // matvecs, or a [mul_mat, mul_mat, glu] gate/up pair) are launched as one grouped kernel. The segments are
+    // mutually independent, so the launch position of the first one is valid for all of them.
+    if (getenv("GGML_CUDA_DISABLE_MMV_GROUP") == nullptr &&
+            node->op == GGML_OP_MUL_MAT && node->src[2] == nullptr && ggml_cuda_mmv_group_seg_ok(node->src[0], node->src[1])) {
+        const ggml_tensor * y = node->src[1];
+        ggml_cuda_mmv_group_seg segs[GGML_CUDA_MMV_GROUP_MAX];
+        int out_nodes[GGML_CUDA_MMV_GROUP_MAX];
+        int nseg  = 0;
+        int j     = i;
+        int j_end = i; // index of the last node that belongs to the group
+        while (j < cgraph->n_nodes && nseg < GGML_CUDA_MMV_GROUP_MAX) {
+            ggml_tensor * n = cgraph->nodes[j];
+            if (j > i && ggml_cuda_is_view_or_noop(n)) {
+                j++;
+                continue;
+            }
+            const bool is_mmv = n->op == GGML_OP_MUL_MAT && n->src[2] == nullptr && n->src[1] == y &&
+                (n->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 && n->type == GGML_TYPE_F32 && ggml_is_contiguous(n) &&
+                ggml_cuda_mmv_group_seg_ok(n->src[0], y);
+            if (!is_mmv) {
+                break;
+            }
+            // gate/up pair followed by the GLU that combines them
+            if (j + 2 < cgraph->n_nodes && cgraph->nodes[j + 1]->op == GGML_OP_MUL_MAT && cgraph->nodes[j + 2]->op == GGML_OP_GLU) {
+                ggml_tensor * n2  = cgraph->nodes[j + 1];
+                ggml_tensor * glu = cgraph->nodes[j + 2];
+                ggml_tensor * gate_n = nullptr;
+                ggml_tensor * up_n   = nullptr;
+                if (glu->src[0] == n && glu->src[1] == n2) {
+                    gate_n = n;
+                    up_n   = n2;
+                } else if (glu->src[0] == n2 && glu->src[1] == n) {
+                    gate_n = n2;
+                    up_n   = n;
+                }
+                if (gate_n && n2->src[2] == nullptr && n2->src[1] == y && (n2->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 &&
+                        up_n->src[0]->type == GGML_TYPE_Q8_0 && gate_n->src[0]->type == GGML_TYPE_Q8_0 &&
+                        ggml_are_same_shape(up_n->src[0], gate_n->src[0]) && ggml_cuda_mmv_group_seg_ok(n2->src[0], y) &&
+                        glu->type == GGML_TYPE_F32 && ggml_is_contiguous(glu) && ggml_nelements(glu) == up_n->src[0]->ne[1] &&
+                        ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu) &&
+                        ggml_can_fuse_subgraph(cgraph, j, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, { j + 2 })) {
+                    segs[nseg].w         = up_n->src[0];
+                    segs[nseg].gate      = gate_n->src[0];
+                    segs[nseg].dst       = glu;
+                    segs[nseg].glu_op    = ggml_get_glu_op(glu);
+                    segs[nseg].glu_limit = ggml_get_op_params_f32(glu, 3);
+                    out_nodes[nseg++] = j + 2;
+                    j_end = j + 2;
+                    j += 3;
+                    continue;
+                }
+            }
+            segs[nseg].w   = n->src[0];
+            segs[nseg].dst = n;
+            out_nodes[nseg++] = j;
+            j_end = j;
+            j++;
+        }
+        if (nseg >= 2 && ggml_cuda_check_fusion_memory_ranges(cgraph, i, j_end - i + 1, out_nodes, nseg)) {
+            ggml_cuda_mmv_group(*cuda_ctx, y, segs, nseg);
+            return j_end - i;
+        }
+    }
+
+    if (node->op == GGML_OP_L2_NORM && i + 2 < cgraph->n_nodes &&
+        GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
+        ggml_tensor * k_view = cgraph->nodes[i + 1];
+        ggml_tensor * k_norm = cgraph->nodes[i + 2];
+        const ggml_tensor * q_view = node->src[0];
+        const int out_nodes[] = { i, i + 2 };
+        const bool graph_ok = k_view->op == GGML_OP_VIEW && k_norm->op == GGML_OP_L2_NORM &&
+            k_norm->src[0] == k_view && (k_view->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 &&
+            (k_norm->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 && ggml_node_get_use_count(cgraph, i + 1) == 1 &&
+            (k_view->flags & GGML_TENSOR_FLAG_OUTPUT) == 0;
+        const bool shape_ok = q_view && q_view->op == GGML_OP_VIEW && q_view->view_src &&
+            q_view->view_src == k_view->view_src && node->ne[0] == 128 && ggml_are_same_shape(node, k_norm) &&
+            ggml_are_same_shape(q_view, k_view) && q_view->type == GGML_TYPE_F32 && k_view->type == GGML_TYPE_F32 &&
+            node->type == GGML_TYPE_F32 && k_norm->type == GGML_TYPE_F32;
+        const bool layout_ok = shape_ok && ggml_is_contiguous_rows(q_view) && ggml_is_contiguous_rows(k_view) &&
+            ggml_is_contiguous(node) && ggml_is_contiguous(k_norm) &&
+            q_view->nb[1] == k_view->nb[1] && q_view->nb[2] == k_view->nb[2] && q_view->nb[3] == k_view->nb[3] &&
+            k_view->view_offs == q_view->view_offs + ggml_row_size(q_view->type, q_view->ne[0])*q_view->ne[1] &&
+            q_view->nb[2] > ggml_row_size(q_view->type, q_view->ne[0])*q_view->ne[1] && q_view->ne[2] <= UINT16_MAX/2;
+
+        if (graph_ok && layout_ok && ggml_cuda_check_fusion_memory_ranges(cgraph, i, 3, out_nodes, 2)) {
+            ggml_cuda_op_l2_norm_dual_s128(*cuda_ctx, node, k_norm);
+            return 2;
+        }
+    }
+
+    // Qwen3.5 Gated DeltaNet decode step: conv + norms + gate + recurrence + state writes + gated rms norm
+    if (node->op == GGML_OP_CONCAT && GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
+        ggml_cuda_gdn_decode_args args;
+        const int nodes_to_skip = ggml_cuda_try_gdn_decode_fusion(cgraph, i, args);
+        if (nodes_to_skip > 0) {
+            // qwen4exp: the gated norm output only feeds sigmoid(z) * . -> ssm_out matvec. Apply the norm and the
+            // gate in the prologue of the fused-quantize matvec instead of two more kernels.
+            {
+                const int norm_idx = i + nodes_to_skip; // MUL(rms_norm, w): the last node of the chain
+                ggml_tensor * norm_out = cgraph->nodes[norm_idx];
+                int j = norm_idx + 1;
+                auto next_compute = [&](int & k) -> ggml_tensor * {
+                    while (k < cgraph->n_nodes && ggml_cuda_is_view_or_noop(cgraph->nodes[k])) {
+                        k++;
+                    }
+                    return k < cgraph->n_nodes ? cgraph->nodes[k] : nullptr;
+                };
+                ggml_tensor * sig = next_compute(j);
+                const int sig_idx = j;
+                j++;
+                ggml_tensor * mul = next_compute(j);
+                const int mul_idx = j;
+                j++;
+                ggml_tensor * mm  = next_compute(j);
+                const int mm_idx  = j;
+                if (getenv("GGML_CUDA_DISABLE_GDN_GATE") == nullptr &&
+                        sig && mul && mm && sig->op == GGML_OP_UNARY && ggml_get_unary_op(sig) == GGML_UNARY_OP_SIGMOID &&
+                        mul->op == GGML_OP_MUL && mm->op == GGML_OP_MUL_MAT && mm->src[2] == nullptr &&
+                        (mm->src[1] == mul || (ggml_cuda_is_view_or_noop(mm->src[1]) && mm->src[1]->view_src == mul &&
+                            ggml_is_contiguous(mm->src[1]) && ggml_nelements(mm->src[1]) == ggml_nelements(mul))) &&
+                        ((mul->src[0] == sig && mul->src[1] == norm_out) || (mul->src[0] == norm_out && mul->src[1] == sig)) &&
+                        sig->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 && ggml_is_contiguous(mul) &&
+                        ggml_are_same_shape(sig, norm_out) && ggml_are_same_shape(mul, norm_out) &&
+                        ggml_nelements(norm_out) == args.S * args.H_v && args.S == 128 &&
+                        ggml_node_get_use_count(cgraph, norm_idx) == 1 && ggml_node_get_use_count(cgraph, sig_idx) == 1 &&
+                        ggml_node_get_use_count(cgraph, mul_idx) == 1 &&
+                        (norm_out->flags & GGML_TENSOR_FLAG_OUTPUT) == 0 && (sig->flags & GGML_TENSOR_FLAG_OUTPUT) == 0 &&
+                        (mul->flags & GGML_TENSOR_FLAG_OUTPUT) == 0 &&
+                        ggml_cuda_mul_mat_vec_q_fq_gdn_gate_ok(*cuda_ctx, mm, sig->src[0])) {
+                    ggml_cuda_pool_alloc<float> attn(cuda_ctx->pool(), args.S * args.H_v);
+                    ggml_cuda_op_gdn_decode_fused_prenorm(*cuda_ctx, args, attn.get());
+                    ggml_cuda_mul_mat_vec_q_fq_gdn_gate(*cuda_ctx, mm, attn.get(), sig->src[0], args.norm_w_tensor, args.eps_rms);
+                    return mm_idx - i;
+                }
+            }
+            ggml_cuda_op_gdn_decode_fused(*cuda_ctx, args);
+            return nodes_to_skip;
+        }
+    }
+
     // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache
     if (node->op == GGML_OP_GATED_DELTA_NET) {
         ggml_cuda_gated_delta_net_fused_cache fused_state_cpy;
         const int nodes_to_skip = ggml_cuda_try_gdn_cache_fusion(cgraph, i, fused_state_cpy);
         if (nodes_to_skip > 0) {
+            bool uses_ok = true;
+            for (int j = i; uses_ok && j <= i + nodes_to_skip; ++j) {
+                if (j == i + nodes_to_skip) {
+                    continue;
+                }
+                const ggml_tensor * candidate = cgraph->nodes[j];
+                int internal_uses = 0;
+                for (int k = i; k <= i + nodes_to_skip; ++k) {
+                    for (int src = 0; src < GGML_MAX_SRC; ++src) {
+                        internal_uses += cgraph->nodes[k]->src[src] == candidate;
+                    }
+                }
+                // The attention-score view is the one expected consumer after the snapshot copy.
+                const int allowed_external = j == i ? 1 : 0;
+                uses_ok = internal_uses + allowed_external == ggml_node_get_use_count(cgraph, j);
+            }
+            if (!uses_ok) {
+                return 0;
+            }
 #ifdef GGML_CUDA_DEBUG
             GGML_LOG_INFO("%s: fused gated_delta_net snapshot copies for %s (skipped %d nodes)\n",
                           __func__, node->name, nodes_to_skip);
@@ -3301,6 +3786,41 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         }
     }
 
+    if (node->op == GGML_OP_GLU && i + 1 < cgraph->n_nodes && ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU && node->src[1]) {
+        ggml_tensor * next = cgraph->nodes[i + 1];
+        const bool has_ids = next->op == GGML_OP_MUL_MAT_ID;
+        if (has_ids || next->op == GGML_OP_MUL_MAT) {
+            const ggml_tensor * weights = next->src[0];
+            const ggml_tensor * ids = has_ids ? next->src[2] : nullptr;
+            const ggml_tensor * gate = node->src[0];
+            const ggml_tensor * up = node->src[1];
+            const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
+            const int64_t mmq_cols = has_ids ? node->ne[2] : node->ne[1];
+            const int64_t n_experts = has_ids ? weights->ne[2] : 0;
+            const bool bad_padding_clear = ggml_backend_buffer_get_usage(weights->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE &&
+                ggml_nbytes(weights) != ggml_backend_buffer_get_alloc_size(weights->buffer, weights) && weights->view_src;
+
+            const bool target_qwen36 = node->ne[0] == 512 && next->ne[0] == 2048 &&
+                (!has_ids || (gate->ne[1] == 8 && weights->ne[2] == 256));
+            const bool target_qwen4exp = node->ne[0] == 640 && next->ne[0] == 2560 &&
+                (!has_ids || (gate->ne[1] == 10 && weights->ne[2] == 512));
+            const bool target_shape = target_qwen36 || target_qwen4exp;
+            const bool weight_type_ok = weights->type == GGML_TYPE_Q8_0 ||
+                (target_qwen4exp && weights->type == GGML_TYPE_IQ4_NL);
+            const bool shape_ok = target_shape && next->src[1] == node && gate->type == GGML_TYPE_F32 && up->type == GGML_TYPE_F32 &&
+                node->type == GGML_TYPE_F32 && next->type == GGML_TYPE_F32 && weight_type_ok &&
+                ggml_are_same_shape(gate, up) && ggml_are_same_shape(gate, node) && gate->nb[0] == sizeof(float) && up->nb[0] == sizeof(float) &&
+                gate->ne[3] == 1 && (has_ids ? gate->ne[1] == ids->ne[0] && gate->ne[2] == ids->ne[1] : gate->ne[2] == 1);
+            const bool use_mmq = shape_ok && !bad_padding_clear && GGML_CUDA_CC_IS_RDNA3_5(cc) &&
+                ggml_cuda_should_use_mmq(weights->type, cc, mmq_cols, n_experts);
+
+            if (use_mmq && ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_GLU, next->op }, { i + 1 })) {
+                ggml_cuda_mul_mat_q_swiglu(*cuda_ctx, weights, ids, next, node);
+                return 1;
+            }
+        }
+    }
+
     //topk-moe
     if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX ||
             cgraph->nodes[i]->op == GGML_OP_ARGSORT) {
@@ -3433,6 +3953,190 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         }
     }
 
+    // Shared expert gate for one token: mul_mat(w [n_embd, 1], y) -> sigmoid -> mul -> add [-> add]
+    // (Qwen3.5 adds the residual in a second add, qwen4exp folds the result into the hyper-connection instead)
+    if (node->op == GGML_OP_MUL_MAT && i + 3 < cgraph->n_nodes && node->type == GGML_TYPE_F32 &&
+        node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32 &&
+        node->src[0]->ne[0] % 2 == 0 && ggml_nelements(node->src[0]) == node->src[0]->ne[0] &&
+        ggml_nelements(node->src[1]) == node->src[0]->ne[0] &&
+        ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(node->src[1]) && ggml_nelements(node) == 1 &&
+        cgraph->nodes[i + 1]->op == GGML_OP_UNARY && ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID &&
+        cgraph->nodes[i + 1]->src[0] == node && cgraph->nodes[i + 2]->op == GGML_OP_MUL && cgraph->nodes[i + 3]->op == GGML_OP_ADD &&
+        GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
+        // the fused kernel replays mul_mat_vec_f<float, float, 1, 256>: only valid where mmvf would pick that block size
+        const int64_t ncols = node->src[0]->ne[0];
+        const int     warp_size = ggml_cuda_info().devices[cuda_ctx->device].warp_size;
+        int64_t block_size_best = warp_size;
+        int64_t niter_best      = (ncols + 2*warp_size - 1) / (2*warp_size);
+        for (int64_t block_size = 2*warp_size; block_size <= 256; block_size += warp_size) {
+            const int64_t niter = (ncols + 2*block_size - 1) / (2*block_size);
+            if (niter < niter_best) {
+                niter_best      = niter;
+                block_size_best = block_size;
+            }
+        }
+
+        const bool two_adds = i + 4 < cgraph->n_nodes && cgraph->nodes[i + 4]->op == GGML_OP_ADD &&
+            cgraph->nodes[i + 4]->src[0] == cgraph->nodes[i + 3];
+        const bool subgraph_ok = block_size_best == 256 && warp_size == 32 && (two_adds ?
+            ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_ADD }, { i + 4 }) :
+            ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_ADD }, { i + 3 }));
+
+        if (subgraph_ok) {
+            const ggml_tensor * sigmoid = cgraph->nodes[i + 1];
+            ggml_tensor * mul  = cgraph->nodes[i + 2];
+            ggml_tensor * add0 = cgraph->nodes[i + 3];
+            ggml_tensor * add1 = two_adds ? cgraph->nodes[i + 4] : nullptr;
+            ggml_tensor * out  = two_adds ? add1 : add0;
+            const ggml_tensor * gate = mul->src[0] == sigmoid ? mul->src[0] : mul->src[1];
+            const ggml_tensor * src  = gate == mul->src[0] ? mul->src[1] : mul->src[0];
+            const ggml_tensor * other = add0->src[0] == mul ? add0->src[1] : add0->src[0];
+            const ggml_tensor * residual = two_adds ? (add1->src[0] == add0 ? add1->src[1] : add1->src[0]) : nullptr;
+            const uintptr_t dst_begin = (uintptr_t) out->data;
+            const uintptr_t dst_end = dst_begin + ggml_backend_buft_get_alloc_size(out->buffer->buft, out);
+            const auto safe_output_alias = [&](const ggml_tensor * input) {
+                const uintptr_t begin = (uintptr_t) input->data;
+                const uintptr_t end = begin + ggml_backend_buft_get_alloc_size(input->buffer->buft, input);
+                return !(begin < dst_end && dst_begin < end) || (input->data == out->data && ggml_are_same_layout(input, out));
+            };
+            const bool residual_ok = !two_adds || (residual != mul && residual != add0 && residual->type == GGML_TYPE_F32 &&
+                ggml_are_same_shape(mul, residual) && ggml_is_contiguous(residual) && safe_output_alias(residual));
+            if (gate == sigmoid && ggml_nelements(sigmoid) == 1 && (add0->src[0] == mul || add0->src[1] == mul) &&
+                other != mul && residual_ok &&
+                mul->type == GGML_TYPE_F32 && src->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 &&
+                mul->ne[0] == ncols && ggml_nelements(mul) == ncols && ggml_are_same_shape(mul, src) && ggml_are_same_shape(mul, other) &&
+                ggml_are_same_shape(mul, out) &&
+                ggml_is_contiguous(src) && ggml_is_contiguous(other) && ggml_is_contiguous(out) &&
+                safe_output_alias(src) && safe_output_alias(other) && safe_output_alias(node->src[1])) {
+                ggml_cuda_op_shared_gate_mul_add(*cuda_ctx, node, src, other, residual, out);
+                return two_adds ? 4 : 3;
+            }
+        }
+    }
+
+    if (node->op == GGML_OP_MUL && node->ne[0] == 2048 && i + 2 < cgraph->n_nodes &&
+        ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_ADD }, { i + 2 })) {
+        ggml_tensor * add0 = cgraph->nodes[i + 1];
+        ggml_tensor * add1 = cgraph->nodes[i + 2];
+        const ggml_tensor * src = ggml_are_same_shape(node, node->src[0]) ? node->src[0] : node->src[1];
+        const ggml_tensor * gate = src == node->src[0] ? node->src[1] : node->src[0];
+        const ggml_tensor * other = add0->src[0];
+        const ggml_tensor * residual = add1->src[1];
+        const uintptr_t gate_begin = (uintptr_t) gate->data;
+        const uintptr_t gate_end = gate_begin + ggml_backend_buft_get_alloc_size(gate->buffer->buft, gate);
+        const uintptr_t dst_begin = (uintptr_t) add1->data;
+        const uintptr_t dst_end = dst_begin + ggml_backend_buft_get_alloc_size(add1->buffer->buft, add1);
+        const bool gate_overlap = gate_begin < dst_end && dst_begin < gate_end;
+        const auto safe_output_alias = [&](const ggml_tensor * input) {
+            const uintptr_t begin = (uintptr_t) input->data;
+            const uintptr_t end = begin + ggml_backend_buft_get_alloc_size(input->buffer->buft, input);
+            return !(begin < dst_end && dst_begin < end) || (input->data == add1->data && ggml_are_same_layout(input, add1));
+        };
+
+        if (add0->src[1] == node && add1->src[0] == add0 && other != node && residual != node && residual != add0 &&
+            node->type == GGML_TYPE_F32 &&
+            src->type == GGML_TYPE_F32 && gate->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && residual->type == GGML_TYPE_F32 &&
+            ggml_are_same_shape(node, src) && ggml_are_same_shape(node, other) && ggml_are_same_shape(node, residual) &&
+            gate->ne[0] == 1 && gate->ne[1] == node->ne[1] && gate->ne[2] == node->ne[2] && gate->ne[3] == node->ne[3] &&
+            ggml_is_contiguous(src) && ggml_is_contiguous(gate) && ggml_is_contiguous(other) &&
+            ggml_is_contiguous(residual) && ggml_is_contiguous(add1) && !gate_overlap &&
+            safe_output_alias(src) && safe_output_alias(other) && safe_output_alias(residual) &&
+            ggml_nelements(add1) <= int64_t(INT_MAX) * 256) {
+            ggml_cuda_op_shared_mul_add(*cuda_ctx, node, other, residual, add1);
+            return 2;
+        }
+    }
+
+    // qwen4exp IQ4_NL routed down projection followed by the 10-expert weighted sum. Compute all selected
+    // experts for one output row in a wave and apply their routing weights immediately, avoiding the
+    // [n_embd, n_used] intermediate and its separate reduction launch.
+    if (getenv("GGML_CUDA_DISABLE_WEIGHTED_DOWN") == nullptr &&
+            node->op == GGML_OP_MUL_MAT_ID && i + 20 < cgraph->n_nodes && cgraph->nodes[i + 1]->op == GGML_OP_MUL) {
+        constexpr int n_used = 10;
+        constexpr int n_ops  = 2 + n_used + (n_used - 1);
+        ggml_tensor * mul = cgraph->nodes[i + 1];
+        const ggml_tensor * experts = ggml_are_same_shape(mul, mul->src[0]) ? mul->src[0] : mul->src[1];
+        const ggml_tensor * weights = experts == mul->src[0] ? mul->src[1] : mul->src[0];
+        const int output_idx = i + n_ops - 1;
+        ggml_tensor * output = cgraph->nodes[output_idx];
+        bool valid = experts == node && node->ne[1] == n_used &&
+            ggml_cuda_mul_mat_id_weighted_rdna3_5_ok(experts, weights, output);
+
+        for (int j = 0; valid && j < n_used; ++j) {
+            const ggml_tensor * view = cgraph->nodes[i + 2 + j];
+            valid = view->op == GGML_OP_VIEW && view->src[0] == mul &&
+                view->ne[0] == mul->ne[0] && view->ne[1] == mul->ne[2] && view->ne[2] == 1 && view->ne[3] == 1 &&
+                view->nb[0] == sizeof(float) && view->nb[1] == mul->nb[2] && view->view_offs == size_t(j) * mul->nb[1];
+        }
+        const int add_start = i + 2 + n_used;
+        if (valid) {
+            const ggml_tensor * first = cgraph->nodes[add_start];
+            valid = first->op == GGML_OP_ADD && first->src[0] == cgraph->nodes[i + 2] && first->src[1] == cgraph->nodes[i + 3];
+        }
+        for (int j = 2; valid && j < n_used; ++j) {
+            const ggml_tensor * add = cgraph->nodes[add_start + j - 1];
+            valid = add->op == GGML_OP_ADD && add->src[0] == cgraph->nodes[add_start + j - 2] && add->src[1] == cgraph->nodes[i + 2 + j];
+        }
+        if (valid) {
+            std::vector<ggml_op> ops = { GGML_OP_MUL_MAT_ID, GGML_OP_MUL };
+            ops.insert(ops.end(), n_used, GGML_OP_VIEW);
+            ops.insert(ops.end(), n_used - 1, GGML_OP_ADD);
+            const int out_nodes[] = { output_idx };
+            if (ggml_can_fuse_subgraph(cgraph, i, n_ops, ops.data(), out_nodes, 1) &&
+                    ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) {
+                ggml_cuda_mul_mat_id_weighted_rdna3_5(*cuda_ctx, experts, weights, output);
+                return n_ops - 1;
+            }
+        }
+    }
+
+    if (node->op == GGML_OP_MUL && node->type == GGML_TYPE_F32 && node->ne[0] % 4 == 0 &&
+        node->ne[3] == 1 && node->ne[1] >= 2 && node->ne[1] <= 16) {
+        const int n_expert_used = node->ne[1];
+        const int n_ops = 2 * n_expert_used;
+        if (i + n_ops <= cgraph->n_nodes) {
+            std::vector<ggml_op> ops = { GGML_OP_MUL };
+            ops.insert(ops.end(), n_expert_used, GGML_OP_VIEW);
+            ops.insert(ops.end(), n_expert_used - 1, GGML_OP_ADD);
+            const int output_idx = i + n_ops - 1;
+
+            if (ggml_can_fuse_subgraph(cgraph, i, n_ops, ops.data(), &output_idx, 1)) {
+                const ggml_tensor * experts = ggml_are_same_shape(node, node->src[0]) ? node->src[0] : node->src[1];
+                const ggml_tensor * weights = experts == node->src[0] ? node->src[1] : node->src[0];
+                ggml_tensor * output = cgraph->nodes[output_idx];
+                bool valid = experts->op == GGML_OP_MUL_MAT_ID && experts->type == GGML_TYPE_F32 && weights->type == GGML_TYPE_F32 &&
+                    experts->ne[0] == node->ne[0] && experts->ne[1] == n_expert_used && experts->ne[2] == node->ne[2] &&
+                    experts->ne[3] == 1 && weights->ne[0] == 1 && weights->ne[1] == n_expert_used &&
+                    weights->ne[2] == node->ne[2] && weights->ne[3] == 1 &&
+                    ggml_is_contiguous(experts) && ggml_is_contiguous(weights) && ggml_is_contiguous(output) &&
+                    output->type == GGML_TYPE_F32 && output->ne[0] == node->ne[0] && output->ne[1] == node->ne[2] &&
+                    output->ne[2] == 1 && output->ne[3] == 1 && ggml_nelements(output) <= UINT32_MAX;
+
+                for (int j = 0; valid && j < n_expert_used; ++j) {
+                    const ggml_tensor * view = cgraph->nodes[i + 1 + j];
+                    valid = view->src[0] == node && view->ne[0] == node->ne[0] && view->ne[1] == node->ne[2] &&
+                        view->ne[2] == 1 && view->ne[3] == 1 && view->nb[0] == sizeof(float) &&
+                        view->nb[1] == node->nb[2] && view->view_offs == size_t(j) * node->nb[1];
+                }
+
+                const int add_start = i + 1 + n_expert_used;
+                if (valid) {
+                    const ggml_tensor * first = cgraph->nodes[add_start];
+                    valid = first->src[0] == cgraph->nodes[i + 1] && first->src[1] == cgraph->nodes[i + 2];
+                }
+                for (int j = 2; valid && j < n_expert_used; ++j) {
+                    const ggml_tensor * add = cgraph->nodes[add_start + j - 1];
+                    valid = add->src[0] == cgraph->nodes[add_start + j - 2] && add->src[1] == cgraph->nodes[i + 1 + j];
+                }
+
+                if (valid) {
+                    ggml_cuda_op_weighted_expert_sum(*cuda_ctx, node, output, n_expert_used);
+                    return n_ops - 1;
+                }
+            }
+        }
+    }
+
     // multi-(add or mul)
     if (node->op == GGML_OP_ADD || node->op == GGML_OP_MUL) {
         int     n_fuse = 0;
@@ -3469,6 +4173,39 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         }
     }
 
+    if ((node->op == GGML_OP_MUL_MAT_ID || node->op == GGML_OP_MUL_MAT) && i + 1 < cgraph->n_nodes) {
+        ggml_tensor * next = cgraph->nodes[i + 1];
+        const ggml_tensor * src0 = node->src[0];
+        const ggml_tensor * src0_next = next->src[0];
+        const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
+
+        const bool has_ids = node->op == GGML_OP_MUL_MAT_ID;
+        const bool valid_sources = next->op == node->op && src0 && src0_next && node->src[1] && next->src[1] &&
+            (!has_ids || node->src[2] && next->src[2]);
+        const bool ordinary_q8 = valid_sources && !has_ids && src0->type == GGML_TYPE_Q8_0 && src0_next->type == GGML_TYPE_Q8_0;
+        const bool shared_inputs = valid_sources && (has_ids || ordinary_q8) && node->src[1] == next->src[1] &&
+            (!has_ids || node->src[2] == next->src[2]);
+        const int64_t mmq_cols = shared_inputs ? (has_ids ? node->src[1]->ne[2] : node->src[1]->ne[1]) : 0;
+        const int64_t n_experts = shared_inputs && has_ids ? src0->ne[2] : 0;
+        const bool use_mmq = shared_inputs &&
+            ggml_cuda_should_use_mmq(src0->type, cc, mmq_cols, n_experts) &&
+            ggml_cuda_should_use_mmq(src0_next->type, cc, mmq_cols, n_experts);
+        const bool compatible = use_mmq && node->src[1]->type == GGML_TYPE_F32 &&
+            node->type == GGML_TYPE_F32 && next->type == GGML_TYPE_F32 &&
+            ggml_are_same_shape(src0, src0_next) && ggml_are_same_shape(node, next) &&
+            mmq_get_q8_1_ds_layout(src0->type) == mmq_get_q8_1_ds_layout(src0_next->type) &&
+            !blackwell_mma_available(cc);
+        const int64_t ncols_dst = has_ids ? node->ne[2] : node->ne[1];
+        const bool use_mmvq = compatible && ncols_dst <= MMVQ_MAX_BATCH_SIZE &&
+            (!has_ids || ncols_dst <= get_mmvq_mmid_max_batch(src0->type, cc) ||
+             ncols_dst <= get_mmvq_mmid_max_batch(src0_next->type, cc));
+
+        if (compatible && !use_mmvq) {
+            ggml_cuda_mul_mat_q_pair(*cuda_ctx, node, next);
+            return 1;
+        }
+    }
+
     bool fused_mul_mat_vec = false;
     int  fused_node_count  = 0;
 
@@ -3999,6 +4736,229 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         return 1;
     }
 
+    // qwen4exp hyper-connection stream mix:
+    //     sigmoid(gate) -> mul(xn, .) -> reshape -> view(stream 0) -> cont -> (view(stream c) -> add) x (hc-1) -> scale
+    // one kernel reads xn/gate once and writes the [n_embd, T] mean of the gated streams
+    {
+        ggml_cuda_hc_mix_args args;
+        enum ggml_op ops[5 + 2 * 15 + 1];
+        int n_ops = ggml_cuda_match_hc_mix(cgraph, i, args, ops);
+        if (n_ops > 0 && args.dst->ne[1] == 1) {
+            n_ops = 0; // Decode preserves bit-identical routing. Prefill uses the fused reduction.
+        }
+        if (n_ops > 0) {
+            int node_idxs[5 + 2 * 15 + 1];
+            for (int j = 0; j < n_ops; ++j) {
+                node_idxs[j] = i + j;
+            }
+            const int out_nodes[] = { i + n_ops - 1 };
+            if (ggml_can_fuse_subgraph_ext(cgraph, node_idxs, n_ops, ops, out_nodes, 1)) {
+                ggml_cuda_op_hc_mix_reduce(*cuda_ctx, args);
+                return n_ops - 1;
+            }
+        }
+    }
+
+    // qwen4exp hyper-connection combine + next grouped norm:
+    //     scale(inject) -> sigmoid -> scale -> [views] -> repeat(block_out) -> mul -> add(residual, .) -> rms_norm -> mul(gamma)
+    // one kernel: block per (stream, token), residual update + norm
+    if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 8 < cgraph->n_nodes &&
+        cgraph->nodes[i + 1]->op == GGML_OP_UNARY && ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID &&
+        cgraph->nodes[i + 1]->src[0] == node &&
+        cgraph->nodes[i + 2]->op == GGML_OP_SCALE && cgraph->nodes[i + 2]->src[0] == cgraph->nodes[i + 1]) {
+        const ggml_tensor * scale1 = node;
+        const ggml_tensor * scale2 = cgraph->nodes[i + 2];
+        const ggml_tensor * inject = scale1->src[0];
+
+        int j = i + 3;
+        while (j < cgraph->n_nodes && j < i + 7 && ggml_cuda_is_view_or_noop(cgraph->nodes[j]) && !ggml_is_empty(cgraph->nodes[j])) {
+            j++;
+        }
+
+        if (j + 4 < cgraph->n_nodes &&
+            cgraph->nodes[j]->op == GGML_OP_REPEAT && cgraph->nodes[j + 1]->op == GGML_OP_MUL && cgraph->nodes[j + 2]->op == GGML_OP_ADD &&
+            cgraph->nodes[j + 3]->op == GGML_OP_RMS_NORM && cgraph->nodes[j + 4]->op == GGML_OP_MUL) {
+            const ggml_tensor * repeat = cgraph->nodes[j];
+            const ggml_tensor * mul    = cgraph->nodes[j + 1];
+            ggml_tensor *       add    = cgraph->nodes[j + 2];
+            const ggml_tensor * rms    = cgraph->nodes[j + 3];
+            ggml_tensor *       mulg   = cgraph->nodes[j + 4];
+            const ggml_tensor * b      = repeat->src[0];
+            const ggml_tensor * w      = mul->src[0] == repeat ? mul->src[1] : mul->src[0];
+            const ggml_tensor * res    = add->src[0] == mul ? add->src[1] : add->src[0];
+            const ggml_tensor * gamma  = mulg->src[0] == rms ? mulg->src[1] : mulg->src[0];
+
+            const int64_t n_embd = add->ne[0];
+            const int64_t hc     = add->ne[1];
+            const int64_t n_tok  = add->ne[2];
+
+            const bool w_ok = (w == scale2 || (ggml_cuda_is_view_or_noop(w) && w->view_src == scale2)) &&
+                w->type == GGML_TYPE_F32 && ggml_is_contiguous(w) && ggml_nelements(w) == hc * n_tok && w->ne[0] == 1 && w->ne[1] == hc;
+            const bool b_ok = b->type == GGML_TYPE_F32 && ggml_is_contiguous(b) && ggml_nelements(b) == n_embd * n_tok &&
+                b->ne[0] == n_embd && b->ne[1] == 1;
+            const bool ok = add->ne[3] == 1 && n_tok <= 65535 &&
+                inject->type == GGML_TYPE_F32 && ggml_is_contiguous(inject) && ggml_nelements(inject) == hc * n_tok &&
+                ggml_nrows(inject) == n_tok &&
+                ggml_are_same_shape(scale1, inject) && ggml_are_same_shape(scale2, inject) &&
+                (mul->src[0] == repeat || mul->src[1] == repeat) && w != repeat &&
+                (add->src[0] == mul || add->src[1] == mul) && res != mul &&
+                rms->src[0] == add && (mulg->src[0] == rms || mulg->src[1] == rms) && gamma != rms &&
+                scale2->type == GGML_TYPE_F32 && repeat->type == GGML_TYPE_F32 &&
+                mul->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && res->type == GGML_TYPE_F32 &&
+                rms->type == GGML_TYPE_F32 && mulg->type == GGML_TYPE_F32 && gamma->type == GGML_TYPE_F32 &&
+                ggml_are_same_shape(repeat, add) && ggml_are_same_shape(mul, add) && ggml_are_same_shape(res, add) &&
+                ggml_are_same_shape(rms, add) && ggml_are_same_shape(mulg, add) &&
+                ggml_is_contiguous(res) && ggml_is_contiguous(add) && ggml_is_contiguous(mulg) && ggml_is_contiguous(gamma) &&
+                ggml_nelements(gamma) == n_embd * hc && gamma->ne[0] == n_embd && gamma->ne[1] == hc && w_ok && b_ok;
+
+            if (ok) {
+                ggml_cuda_hc_combine_norm_args args;
+                args.inject    = inject;
+                args.residual  = res;
+                args.block_out = b;
+                args.gamma     = gamma;
+                args.out_res   = add;
+                args.out_xn    = mulg;
+                args.s1        = ggml_get_op_params_f32(scale1, 0);
+                args.b1        = ggml_get_op_params_f32(scale1, 1);
+                args.s2        = ggml_get_op_params_f32(scale2, 0);
+                args.b2        = ggml_get_op_params_f32(scale2, 1);
+                memcpy(&args.eps, rms->op_params, sizeof(float));
+
+                // every block reads block_out and inject and writes its own stream of both outputs: the outputs
+                // must not overlap those, and the residual may only alias the first output in place
+                auto overlap = [](const ggml_tensor * p, const ggml_tensor * q) {
+                    const uintptr_t p0 = (uintptr_t) p->data, p1 = p0 + ggml_backend_buft_get_alloc_size(p->buffer->buft, p);
+                    const uintptr_t q0 = (uintptr_t) q->data, q1 = q0 + ggml_backend_buft_get_alloc_size(q->buffer->buft, q);
+                    return p0 < q1 && q0 < p1;
+                };
+                const bool res_inplace = res->data == add->data && ggml_are_same_layout(res, add);
+                bool alias_ok = !overlap(add, b) && !overlap(add, inject) && (res_inplace || !overlap(add, res)) &&
+                    !overlap(mulg, b) && !overlap(mulg, inject) && !overlap(mulg, res) && !overlap(mulg, add);
+                // outputs overlapping the inputs (typically xn reusing the block_out buffer, which dies at the
+                // combine): single-block variant that reads everything before it writes
+                if (!alias_ok && n_tok == 1 && hc <= 4) {
+                    args.single_block = true;
+                    alias_ok = true;
+                }
+
+                int          idxs[16];
+                enum ggml_op ops[16];
+                int          count = 0;
+                for (int k = i; k <= j + 4; ++k) {
+                    const ggml_tensor * n = cgraph->nodes[k];
+                    if (ggml_cuda_is_view_or_noop(n) && n->view_src != scale2) {
+                        continue;
+                    }
+                    idxs[count] = k;
+                    ops[count]  = n->op;
+                    count++;
+                }
+                const int out_nodes[] = { j + 2, j + 4 };
+                if (alias_ok && ggml_cuda_hc_combine_norm_supported(args, ggml_cuda_info().devices[cuda_ctx->device].warp_size) &&
+                    ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, out_nodes, 2)) {
+                    ggml_cuda_op_hc_combine_norm(*cuda_ctx, args);
+                    return j + 4 - i;
+                }
+            }
+        }
+    }
+
+    // qwen4exp hyper-connection combine:
+    //     scale(inject) -> sigmoid -> scale -> [reshape] -> repeat(block_out) -> mul -> add(residual, .)
+    // one kernel writes residual + block_out (x) (s2*sigmoid(s1*inject))
+    if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 6 < cgraph->n_nodes &&
+        cgraph->nodes[i + 1]->op == GGML_OP_UNARY && ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID &&
+        cgraph->nodes[i + 2]->op == GGML_OP_SCALE) {
+        const ggml_tensor * scale1  = node;
+        const ggml_tensor * sigmoid = cgraph->nodes[i + 1];
+        const ggml_tensor * scale2  = cgraph->nodes[i + 2];
+        const ggml_tensor * inject  = scale1->src[0];
+
+        // a few view nodes follow: the scatter weights [1, hc, T] and the block output [n_embd, 1, T]
+        int j = i + 3;
+        while (j < cgraph->n_nodes && j < i + 7 && ggml_cuda_is_view_or_noop(cgraph->nodes[j]) && !ggml_is_empty(cgraph->nodes[j])) {
+            j++;
+        }
+
+        if (j + 2 < cgraph->n_nodes &&
+            cgraph->nodes[j]->op == GGML_OP_REPEAT && cgraph->nodes[j + 1]->op == GGML_OP_MUL && cgraph->nodes[j + 2]->op == GGML_OP_ADD) {
+            const ggml_tensor * repeat = cgraph->nodes[j];
+            const ggml_tensor * mul    = cgraph->nodes[j + 1];
+            ggml_tensor *       add    = cgraph->nodes[j + 2];
+            const ggml_tensor * b      = repeat->src[0];
+            const ggml_tensor * w      = mul->src[0] == repeat ? mul->src[1] : mul->src[0];
+            const ggml_tensor * res    = add->src[0] == mul ? add->src[1] : add->src[0];
+
+            const int64_t n_embd   = add->ne[0];
+            const int64_t hc       = add->ne[1];
+            const int64_t n_tokens = add->ne[2];
+
+            const bool w_ok = (w == scale2 || (ggml_cuda_is_view_or_noop(w) && w->view_src == scale2)) &&
+                w->type == GGML_TYPE_F32 && ggml_is_contiguous(w) &&
+                w->ne[0] == 1 && w->ne[1] == hc && w->ne[2] == n_tokens && w->ne[3] == 1;
+            const bool b_ok = b->type == GGML_TYPE_F32 && ggml_is_contiguous(b) &&
+                b->ne[0] == n_embd && b->ne[1] == 1 && b->ne[2] == n_tokens && b->ne[3] == 1;
+            const bool ok = sigmoid->src[0] == scale1 && scale2->src[0] == sigmoid &&
+                (mul->src[0] == repeat || mul->src[1] == repeat) && w != repeat &&
+                (add->src[0] == mul || add->src[1] == mul) && res != mul &&
+                sigmoid->type == GGML_TYPE_F32 && scale2->type == GGML_TYPE_F32 && inject->type == GGML_TYPE_F32 &&
+                repeat->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && res->type == GGML_TYPE_F32 &&
+                ggml_is_contiguous(inject) && ggml_nelements(inject) == hc * n_tokens && ggml_nrows(inject) == n_tokens &&
+                ggml_are_same_shape(scale1, inject) && ggml_are_same_shape(sigmoid, inject) && ggml_are_same_shape(scale2, inject) &&
+                ggml_are_same_shape(repeat, add) && ggml_are_same_shape(mul, add) && ggml_are_same_shape(res, add) &&
+                add->ne[3] == 1 && ggml_is_contiguous(res) && ggml_is_contiguous(add) && w_ok && b_ok;
+
+            if (ok) {
+                // the block-output reshape views an external tensor, so it is left out of the subgraph check
+                int          idxs[12];
+                enum ggml_op ops[12];
+                int          count = 0;
+                for (int k = i; k <= j + 2; ++k) {
+                    const ggml_tensor * n = cgraph->nodes[k];
+                    if (ggml_cuda_is_view_or_noop(n) && n->view_src != scale2) {
+                        continue;
+                    }
+                    idxs[count] = k;
+                    ops[count]  = n->op;
+                    count++;
+                }
+                const int out_nodes[] = { j + 2 };
+                if (ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, out_nodes, 1)) {
+                    ggml_cuda_hc_combine_args args;
+                    args.residual  = res;
+                    args.block_out = b;
+                    args.inject    = inject;
+                    args.dst       = add;
+                    args.s1        = ggml_get_op_params_f32(scale1, 0);
+                    args.b1        = ggml_get_op_params_f32(scale1, 1);
+                    args.s2        = ggml_get_op_params_f32(scale2, 0);
+                    args.b2        = ggml_get_op_params_f32(scale2, 1);
+                    ggml_cuda_op_hc_combine(*cuda_ctx, args);
+                    return j + 2 - i;
+                }
+            }
+        }
+    }
+
+    if (node->op == GGML_OP_CONT && i + 2 < cgraph->n_nodes) {
+        ggml_tensor * unary = cgraph->nodes[i + 1];
+        ggml_tensor * mul = cgraph->nodes[i + 2];
+        const ggml_tensor * gate = node->src[0];
+        const ggml_tensor * other = mul->src[0] == unary ? mul->src[1] : mul->src[0];
+        const int out_nodes[] = { i + 2 };
+        if (unary->op == GGML_OP_UNARY && ggml_get_unary_op(unary) == GGML_UNARY_OP_SIGMOID && mul->op == GGML_OP_MUL &&
+                (mul->src[0] == unary || mul->src[1] == unary) && gate->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 &&
+                gate->nb[0] == sizeof(float) && gate->nb[1] == 2 * gate->ne[0] * sizeof(float) &&
+                gate->nb[2] == gate->nb[1] * gate->ne[1] && gate->nb[3] == gate->nb[2] * gate->ne[2] &&
+                ggml_is_contiguous(other) && ggml_are_same_shape(gate, other) &&
+                ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_CONT, GGML_OP_UNARY, GGML_OP_MUL }, { i + 2 }) &&
+                ggml_cuda_check_fusion_memory_ranges(cgraph, i, 3, out_nodes, 1)) {
+            ggml_cuda_op_cont_sigmoid_mul(*cuda_ctx, node, unary, mul);
+            return 2;
+        }
+    }
+
     if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SILU }) ||
         ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SIGMOID }) ||
         ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SOFTPLUS })) {
@@ -4016,6 +4976,34 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
         return 2;
     }
 
+    // scale -> silu/sigmoid -> quantized matvec (decode): apply the activation while the matvec quantizes it
+    if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 2 < cgraph->n_nodes &&
+        cgraph->nodes[i + 1]->op == GGML_OP_UNARY &&
+        (ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SILU || ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID) &&
+        cgraph->nodes[i + 2]->op == GGML_OP_MUL_MAT && cgraph->nodes[i + 2]->src[1] == cgraph->nodes[i + 1] &&
+        cgraph->nodes[i + 1]->src[0] == node && cgraph->nodes[i + 1]->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
+        ggml_is_contiguous(node->src[0]) && ggml_are_same_shape(node->src[0], cgraph->nodes[i + 1]) &&
+        ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_MUL_MAT }, { i + 2 }) &&
+        ggml_cuda_mul_mat_vec_q_fq_prologue_ok(*cuda_ctx, cgraph->nodes[i + 2], node->src[0])) {
+        const int y_op = ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SILU ? 1 : 2;
+        ggml_tensor * mm = cgraph->nodes[i + 2];
+
+        ggml_cuda_mul_mat_vec_q_fq_prologue(*cuda_ctx, mm, node->src[0],
+            ggml_get_op_params_f32(node, 0), ggml_get_op_params_f32(node, 1), y_op);
+        return 2;
+    }
+
+    // scale -> silu/sigmoid (e.g. the qwen4exp hyper-connection low-rank gate: silu(x / hc))
+    if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 1 < cgraph->n_nodes &&
+        cgraph->nodes[i + 1]->op == GGML_OP_UNARY &&
+        (ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SILU || ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID) &&
+        cgraph->nodes[i + 1]->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 &&
+        ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(cgraph->nodes[i + 1]) &&
+        ggml_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY })) {
+        ggml_cuda_op_scale_unary(*cuda_ctx, node, cgraph->nodes[i + 1]);
+        return 1;
+    }
+
     return 0;
 }
 
@@ -4241,6 +5229,11 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud
 static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) {
     ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key);
 
+    static const bool disable_graphs_env = getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr && atoi(getenv("GGML_CUDA_DISABLE_GRAPHS")) != 0;
+    if (disable_graphs_env) {
+        graph->disable_due_to_gpu_arch = true;
+    }
+
     if (graph->graph == nullptr) {
         if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) {
             if (!graph->disable_due_to_gpu_arch) {
diff --git a/ggml/src/ggml-cuda/hyperconn.cu b/ggml/src/ggml-cuda/hyperconn.cu
new file mode 100644
index 000000000000..b57bc966a915
--- /dev/null
+++ b/ggml/src/ggml-cuda/hyperconn.cu
@@ -0,0 +1,366 @@
+#include "hyperconn.cuh"
+#include "ggml-backend-impl.h"
+
+// Explicitly rounded mul/add: keeps the fused kernels from being contracted into FMAs so that
+// the results match the separate MUL and ADD kernels of the unfused graph bit for bit.
+#if defined(__HIP_PLATFORM_AMD__)
+static __device__ __forceinline__ float hc_mul_rn(const float a, const float b) {
+    float result;
+    asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b));
+    return result;
+}
+
+static __device__ __forceinline__ float hc_add_rn(const float a, const float b) {
+    float result;
+    asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b));
+    return result;
+}
+#else
+static __device__ __forceinline__ float hc_mul_rn(const float a, const float b) {
+    return __fmul_rn(a, b);
+}
+
+static __device__ __forceinline__ float hc_add_rn(const float a, const float b) {
+    return __fadd_rn(a, b);
+}
+#endif
+
+// same expression as op_sigmoid in unary.cu
+static __device__ __forceinline__ float hc_sigmoid(const float x) {
+    return 1.0f / (1.0f + expf(-x));
+}
+
+// mixed[e,t] = scale * ( ((xn*sig)[0] + (xn*sig)[1]) + ... ) + bias, streams summed in order 0..hc-1
+static __global__ void hc_mix_reduce_f32(
+        const float * __restrict__ xn, const float * __restrict__ gate, float * __restrict__ dst,
+        const int64_t n_embd, const int64_t n_tokens, const int hc, const float scale, const float bias) {
+    const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x;
+    if (index >= n_embd * n_tokens) {
+        return;
+    }
+
+    const int64_t t = index / n_embd;
+    const int64_t e = index - t * n_embd;
+    const int64_t hc_dim = n_embd * hc;
+    const int64_t base = t * hc_dim + e;
+
+    float acc = hc_mul_rn(xn[base], hc_sigmoid(gate[base]));
+    for (int c = 1; c < hc; ++c) {
+        const int64_t i = base + int64_t(c) * n_embd;
+        acc = hc_add_rn(acc, hc_mul_rn(xn[i], hc_sigmoid(gate[i])));
+    }
+
+    // identical expression to scale_f32 in scale.cu
+    dst[index] = scale * acc + bias;
+}
+
+// out[e,c,t] = residual[e,c,t] + block_out[e,t] * ( s2 * sigmoid(s1 * inject[c,t] + b1) + b2 )
+static __global__ void hc_combine_f32(
+        const float * __restrict__ residual, const float * __restrict__ block_out, const float * __restrict__ inject,
+        float * __restrict__ dst, const int64_t n_embd, const int hc, const int64_t n_tokens,
+        const float s1, const float b1, const float s2, const float b2) {
+    const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x;
+    const int64_t hc_dim = n_embd * hc;
+    if (index >= hc_dim * n_tokens) {
+        return;
+    }
+
+    const int64_t t   = index / hc_dim;
+    const int64_t rem = index - t * hc_dim;
+    const int64_t c   = rem / n_embd;
+    const int64_t e   = rem - c * n_embd;
+
+    // identical expressions to scale_f32 (scale.cu) and op_sigmoid (unary.cu)
+    const float x1 = s1 * inject[t * hc + c] + b1;
+    const float x2 = hc_sigmoid(x1);
+    const float w  = s2 * x2 + b2;
+
+    const float m = hc_mul_rn(block_out[t * n_embd + e], w);
+    dst[index] = hc_add_rn(residual[index], m);
+}
+
+static bool hc_ranges_overlap(const ggml_tensor * a, const ggml_tensor * b) {
+    const char * a0 = (const char *) a->data;
+    const char * a1 = a0 + ggml_backend_buft_get_alloc_size(a->buffer->buft, a);
+    const char * b0 = (const char *) b->data;
+    const char * b1 = b0 + ggml_backend_buft_get_alloc_size(b->buffer->buft, b);
+    return a0 < b1 && b0 < a1;
+}
+
+void ggml_cuda_op_hc_mix_reduce(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_mix_args & args) {
+    const ggml_tensor * xn   = args.xn;
+    const ggml_tensor * gate = args.gate;
+    ggml_tensor *       dst  = args.dst;
+
+    GGML_ASSERT(xn->type == GGML_TYPE_F32 && gate->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(ggml_is_contiguous(xn) && ggml_is_contiguous(gate) && ggml_is_contiguous(dst));
+    GGML_ASSERT(args.hc > 0);
+
+    const int64_t n_embd   = dst->ne[0];
+    const int64_t n_tokens = ggml_nrows(dst);
+    GGML_ASSERT(xn->ne[0] == n_embd * args.hc && ggml_nrows(xn) == n_tokens);
+    GGML_ASSERT(ggml_are_same_shape(xn, gate));
+
+    // The graph allocator may reuse the buffer of the (elided) sigmoid/mul intermediates, which can
+    // coincide with the gate matmul output. Every thread reads hc strided elements, so an aliased
+    // output has to be staged -- unless (single token) the output lands on a whole stream of the
+    // aliased input: element e of the output then overwrites element e of that stream, which only
+    // the same thread reads (after it has finished reading).
+    auto stream_aligned_alias = [&](const ggml_tensor * in) {
+        if (!hc_ranges_overlap(dst, in)) {
+            return true;
+        }
+        if (n_tokens != 1) {
+            return false;
+        }
+        const ptrdiff_t off = (const char *) dst->data - (const char *) in->data;
+        return off >= 0 && off % (n_embd * (ptrdiff_t) sizeof(float)) == 0 &&
+            off + (ptrdiff_t) ggml_nbytes(dst) <= (ptrdiff_t) ggml_nbytes(in);
+    };
+    const bool stage = !stream_aligned_alias(xn) || !stream_aligned_alias(gate);
+
+    ggml_cuda_pool_alloc<float> staged(ctx.pool());
+    float * out = stage ? staged.alloc(n_embd * n_tokens) : (float *) dst->data;
+
+    const int64_t n_items = n_embd * n_tokens;
+    const int     threads = 256;
+    const int     blocks  = (int) ((n_items + threads - 1) / threads);
+    const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream());
+    ggml_cuda_kernel_launch(hc_mix_reduce_f32, launch_params,
+        (const float *) xn->data, (const float *) gate->data, out, n_embd, n_tokens, args.hc, args.scale, args.bias);
+
+    if (stage) {
+        CUDA_CHECK(cudaMemcpyAsync(dst->data, out, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream()));
+    }
+}
+
+void ggml_cuda_op_hc_combine(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_args & args) {
+    const ggml_tensor * residual  = args.residual;
+    const ggml_tensor * block_out = args.block_out;
+    const ggml_tensor * inject    = args.inject;
+    ggml_tensor *       dst       = args.dst;
+
+    GGML_ASSERT(residual->type == GGML_TYPE_F32 && block_out->type == GGML_TYPE_F32 &&
+                inject->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32);
+    GGML_ASSERT(ggml_is_contiguous(residual) && ggml_is_contiguous(block_out) &&
+                ggml_is_contiguous(inject) && ggml_is_contiguous(dst));
+
+    const int64_t n_embd   = dst->ne[0];
+    const int64_t hc       = dst->ne[1];
+    const int64_t n_tokens = dst->ne[2] * dst->ne[3];
+    GGML_ASSERT(ggml_are_same_shape(residual, dst));
+    GGML_ASSERT(ggml_nelements(block_out) == n_embd * n_tokens);
+    GGML_ASSERT(ggml_nelements(inject) == hc * n_tokens);
+
+    // residual may be the in-place source of the add (same address, same layout): each thread then
+    // reads and writes its own element only. Any other overlap needs staging.
+    const bool residual_inplace = residual->data == dst->data && ggml_are_same_layout(residual, dst);
+    const bool stage = (!residual_inplace && hc_ranges_overlap(dst, residual)) ||
+                       hc_ranges_overlap(dst, block_out) || hc_ranges_overlap(dst, inject);
+
+    ggml_cuda_pool_alloc<float> staged(ctx.pool());
+    float * out = stage ? staged.alloc(ggml_nelements(dst)) : (float *) dst->data;
+
+    const int64_t n_items = ggml_nelements(dst);
+    const int     threads = 256;
+    const int     blocks  = (int) ((n_items + threads - 1) / threads);
+    const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream());
+    ggml_cuda_kernel_launch(hc_combine_f32, launch_params,
+        (const float *) residual->data, (const float *) block_out->data, (const float *) inject->data, out,
+        n_embd, (int) hc, n_tokens, args.s1, args.b1, args.s2, args.b2);
+
+    if (stage) {
+        CUDA_CHECK(cudaMemcpyAsync(dst->data, out, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream()));
+    }
+}
+
+// ---------------------------------------------------------------------------------------------------------
+// combine + grouped rms norm, one token: block per stream, 1024 threads (the rms_norm_f32<1024> layout)
+//   w        = s2 * sigmoid(s1 * inject[c] + b1) + b2
+//   res[e,c] = residual[e,c] + block_out[e] * w
+//   xn[e,c]  = rms_norm(res[:,c]) * gamma[e,c]
+
+#define HC_CN_BLOCK   1024
+#define HC_CN_MAX_EMB 3072
+
+static __global__ void __launch_bounds__(HC_CN_BLOCK, 1) hc_combine_norm_f32(
+        const float * __restrict__ inject, const float * __restrict__ residual,
+        const float * __restrict__ block_out, const float * __restrict__ gamma,
+        float * __restrict__ out_res, float * __restrict__ out_xn,
+        const int n_embd, const float s1, const float b1, const float s2, const float b2, const float eps) {
+    __shared__ float s_sum[32];
+
+    const int c   = blockIdx.x;   // stream
+    const int t   = blockIdx.y;   // token
+    const int hc  = gridDim.x;
+    const int tid = threadIdx.x;
+
+    // identical expressions to scale_f32 / op_sigmoid / scale_f32
+    const float x1 = s1 * inject[(int64_t) t * hc + c] + b1;
+    const float x2 = hc_sigmoid(x1);
+    const float w  = s2 * x2 + b2;
+
+    const int64_t row = (int64_t) t * hc + c;
+    const float * res = residual + row * n_embd;
+    float *       dst = out_res  + row * n_embd;
+    block_out += (int64_t) t * n_embd;
+    float xs[3];
+    float tmp = 0.0f;
+#pragma unroll
+    for (int k = 0; k < 3; ++k) {
+        const int col = tid + k * HC_CN_BLOCK;
+        xs[k] = 0.0f;
+        if (col < n_embd) {
+            const float m  = hc_mul_rn(block_out[col], w);
+            const float xi = hc_add_rn(res[col], m);
+            dst[col] = xi;
+            xs[k]    = xi;
+            tmp += xi * xi;
+        }
+    }
+
+    tmp = block_reduce<block_reduce_method::SUM, HC_CN_BLOCK>(tmp, s_sum);
+
+    const float mean  = tmp / n_embd;
+    const float scale = rsqrtf(mean + eps);
+
+    const float * g  = gamma  + (int64_t) c * n_embd;
+    float *       xn = out_xn + row * n_embd;
+#pragma unroll
+    for (int k = 0; k < 3; ++k) {
+        const int col = tid + k * HC_CN_BLOCK;
+        if (col < n_embd) {
+            xn[col] = scale * xs[k] * g[col];
+        }
+    }
+}
+
+// Same computation for one token with all streams in a single block, streams processed one after the other
+// with exactly the per-stream expressions and block reduction above. Every input element is loaded into
+// registers before the first store, so the outputs may alias any of the inputs (the graph allocator reuses
+// the block_out buffer for xn when block_out dies at the combine).
+#define HC_CN_SINGLE_MAX_HC 4
+
+static __global__ void __launch_bounds__(HC_CN_BLOCK, 1) hc_combine_norm_single_f32(
+        const float * inject, const float * residual, const float * block_out, const float * __restrict__ gamma,
+        float * out_res, float * out_xn,
+        const int n_embd, const int hc, const float s1, const float b1, const float s2, const float b2, const float eps) {
+    __shared__ float s_sum[HC_CN_SINGLE_MAX_HC][32];
+
+    const int t       = blockIdx.x;   // token
+    const int tid     = threadIdx.x;
+    const int warp_id = tid / WARP_SIZE;
+    const int lane_id = tid % WARP_SIZE;
+
+    // 1. every input element of the token into registers
+    float w[HC_CN_SINGLE_MAX_HC];
+#pragma unroll
+    for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) {
+        w[c] = 0.0f;
+        if (c < hc) {
+            // identical expressions to scale_f32 / op_sigmoid / scale_f32
+            const float x1 = s1 * inject[(int64_t) t * hc + c] + b1;
+            const float x2 = hc_sigmoid(x1);
+            w[c] = s2 * x2 + b2;
+        }
+    }
+
+    block_out += (int64_t) t * n_embd;
+    float xs[HC_CN_SINGLE_MAX_HC][3];
+    float tmp[HC_CN_SINGLE_MAX_HC];
+#pragma unroll
+    for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) {
+        tmp[c] = 0.0f;
+    }
+#pragma unroll
+    for (int k = 0; k < 3; ++k) {
+        const int col = tid + k * HC_CN_BLOCK;
+        const float bo = col < n_embd ? block_out[col] : 0.0f;
+#pragma unroll
+        for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) {
+            xs[c][k] = 0.0f;
+            if (c < hc && col < n_embd) {
+                const float m  = hc_mul_rn(bo, w[c]);
+                const float xi = hc_add_rn(residual[((int64_t) t * hc + c) * n_embd + col], m);
+                xs[c][k] = xi;
+                tmp[c] += xi * xi;
+            }
+        }
+    }
+    __syncthreads(); // all loads of the block have completed before the first store
+
+    // 2. per-stream block sums, same reduction tree as block_reduce<SUM, 1024> for each stream
+#pragma unroll
+    for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) {
+        tmp[c] = warp_reduce_sum(tmp[c]);
+    }
+    if (lane_id == 0) {
+#pragma unroll
+        for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) {
+            s_sum[c][warp_id] = tmp[c];
+        }
+    }
+    __syncthreads();
+#pragma unroll
+    for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) {
+        tmp[c] = lane_id < HC_CN_BLOCK / WARP_SIZE ? s_sum[c][lane_id] : 0.0f;
+        tmp[c] = warp_reduce_sum(tmp[c]);
+    }
+
+    // 3. stores
+#pragma unroll
+    for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) {
+        if (c < hc) {
+            const float mean  = tmp[c] / n_embd;
+            const float scale = rsqrtf(mean + eps);
+            const int64_t row = (int64_t) t * hc + c;
+            float *       dst = out_res + row * n_embd;
+            float *       xn  = out_xn  + row * n_embd;
+            const float * g   = gamma   + (int64_t) c * n_embd;
+#pragma unroll
+            for (int k = 0; k < 3; ++k) {
+                const int col = tid + k * HC_CN_BLOCK;
+                if (col < n_embd) {
+                    dst[col] = xs[c][k];
+                    xn[col]  = scale * xs[c][k] * g[col];
+                }
+            }
+        }
+    }
+}
+
+bool ggml_cuda_hc_combine_norm_supported(const ggml_cuda_hc_combine_norm_args & a, const int warp_size) {
+    const int64_t n_embd = a.out_res->ne[0];
+    const int64_t hc     = a.out_res->ne[1];
+    return warp_size == 32 && n_embd >= 1024 && n_embd <= HC_CN_MAX_EMB && hc >= 1 && hc <= 16 &&
+        ggml_nelements(a.gamma) == n_embd * hc;
+}
+
+void ggml_cuda_op_hc_combine_norm(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_norm_args & a) {
+    const int64_t n_embd   = a.out_res->ne[0];
+    const int64_t hc       = a.out_res->ne[1];
+    const int64_t n_tokens = a.out_res->ne[2] * a.out_res->ne[3];
+
+    GGML_ASSERT(a.out_res->ne[3] == 1);
+    GGML_ASSERT(ggml_are_same_shape(a.out_res, a.out_xn) && ggml_are_same_shape(a.out_res, a.residual));
+    GGML_ASSERT(ggml_nelements(a.block_out) == n_embd * n_tokens && ggml_nelements(a.gamma) == n_embd * hc &&
+                ggml_nelements(a.inject) == hc * n_tokens);
+
+    if (a.single_block) {
+        GGML_ASSERT(hc <= HC_CN_SINGLE_MAX_HC);
+        const ggml_cuda_kernel_launch_params launch_params(dim3((int) n_tokens, 1, 1), HC_CN_BLOCK, 0, ctx.stream());
+        ggml_cuda_kernel_launch(hc_combine_norm_single_f32, launch_params,
+            (const float *) a.inject->data, (const float *) a.residual->data,
+            (const float *) a.block_out->data, (const float *) a.gamma->data,
+            (float *) a.out_res->data, (float *) a.out_xn->data,
+            (int) n_embd, (int) hc, a.s1, a.b1, a.s2, a.b2, a.eps);
+        return;
+    }
+
+    const ggml_cuda_kernel_launch_params launch_params(dim3((int) hc, (int) n_tokens, 1), HC_CN_BLOCK, 0, ctx.stream());
+    ggml_cuda_kernel_launch(hc_combine_norm_f32, launch_params,
+        (const float *) a.inject->data, (const float *) a.residual->data,
+        (const float *) a.block_out->data, (const float *) a.gamma->data,
+        (float *) a.out_res->data, (float *) a.out_xn->data,
+        (int) n_embd, a.s1, a.b1, a.s2, a.b2, a.eps);
+}
diff --git a/ggml/src/ggml-cuda/hyperconn.cuh b/ggml/src/ggml-cuda/hyperconn.cuh
new file mode 100644
index 000000000000..b510c2d632ed
--- /dev/null
+++ b/ggml/src/ggml-cuda/hyperconn.cuh
@@ -0,0 +1,55 @@
+#pragma once
+
+#include "common.cuh"
+
+// Fused hyper-connection (DeepSeek-V4 / qwen4exp style) decode helpers.
+//
+// hc_mix_reduce:  mixed[e,t] = scale * sum_c ( xn[c*n_embd + e, t] * sigmoid(gate[c*n_embd + e, t]) ) + bias
+//   replaces: UNARY(SIGMOID) -> MUL -> RESHAPE -> VIEW -> CONT -> (VIEW -> ADD) x (hc-1) -> SCALE
+//
+// hc_combine:     out[e,c,t] = residual[e,c,t] + block_out[e,t] * ( s2 * sigmoid(s1 * inject[c,t] + b1) + b2 )
+//   replaces: SCALE -> UNARY(SIGMOID) -> SCALE -> RESHAPE* -> REPEAT -> MUL -> ADD
+//
+// Both replay the unfused rounding sequence exactly (separate mul and add roundings), so results are
+// bit-identical to the individual kernels.
+
+struct ggml_cuda_hc_mix_args {
+    const ggml_tensor * xn;     // [hc*n_embd, T] F32 contiguous
+    const ggml_tensor * gate;   // [hc*n_embd, T] F32 contiguous (pre-sigmoid)
+    ggml_tensor *       dst;    // [n_embd, T]   F32 contiguous
+    int                 hc;
+    float               scale;
+    float               bias;
+};
+
+struct ggml_cuda_hc_combine_args {
+    const ggml_tensor * residual;   // [n_embd, hc, T] F32 contiguous
+    const ggml_tensor * block_out;  // [n_embd, T]     F32 contiguous (or a reshape of it)
+    const ggml_tensor * inject;     // [hc, T]         F32 contiguous (pre-scale/sigmoid)
+    ggml_tensor *       dst;        // [n_embd, hc, T] F32 contiguous
+    float               s1, b1;     // first SCALE
+    float               s2, b2;     // second SCALE
+};
+
+void ggml_cuda_op_hc_mix_reduce(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_mix_args & args);
+void ggml_cuda_op_hc_combine(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_args & args);
+
+// Hyper-connection combine followed by the grouped RMS norm of the next hc_mix (block per stream and token):
+//   w[c]     = s2 * sigmoid(s1 * inject[c] + b1) + b2
+//   res[e,c] = residual[e,c] + block_out[e] * w[c]        -> out_res (the layer residual)
+//   xn[e,c]  = rms_norm(res[:,c]) * gamma[e,c]            -> out_xn  (next hc_norm)
+// The norm replays rms_norm_f32<1024, true> (one block of 1024 threads per stream).
+struct ggml_cuda_hc_combine_norm_args {
+    const ggml_tensor * inject;     // [hc, T]  F32 contiguous (pre-scale/sigmoid)
+    const ggml_tensor * residual;   // [n_embd, hc, T] F32 contiguous
+    const ggml_tensor * block_out;  // [n_embd, 1, T]  F32 contiguous
+    const ggml_tensor * gamma;      // [n_embd, hc] F32 contiguous
+    ggml_tensor *       out_res;    // [n_embd, hc, T]
+    ggml_tensor *       out_xn;     // [n_embd, hc, T]
+    float               s1, b1, s2, b2;
+    float               eps;
+    bool                single_block = false; // alias-proof variant: one block per token, all inputs read before any store (hc <= 4)
+};
+
+bool ggml_cuda_hc_combine_norm_supported(const ggml_cuda_hc_combine_norm_args & args, int warp_size);
+void ggml_cuda_op_hc_combine_norm(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_norm_args & args);
diff --git a/ggml/src/ggml-cuda/mean.cu b/ggml/src/ggml-cuda/mean.cu
index a8f6046e46da..55b87b375496 100644
--- a/ggml/src/ggml-cuda/mean.cu
+++ b/ggml/src/ggml-cuda/mean.cu
@@ -2,7 +2,12 @@
 #include "reduce_rows.cuh"
 
 #ifdef GGML_CUDA_USE_CUB
+#ifdef GGML_CUDA_CUB_IS_HIPCUB
+#include <hipcub/hipcub.hpp>
+namespace cub = hipcub;
+#else
 #include <cub/cub.cuh>
+#endif  // GGML_CUDA_CUB_IS_HIPCUB
 using namespace cub;
 #endif  // GGML_CUDA_USE_CUB
 
diff --git a/ggml/src/ggml-cuda/mmid.cu b/ggml/src/ggml-cuda/mmid.cu
index ed0851dcf8dd..edce236b0ca6 100644
--- a/ggml/src/ggml-cuda/mmid.cu
+++ b/ggml/src/ggml-cuda/mmid.cu
@@ -19,6 +19,57 @@ struct mm_ids_helper_store {
 };
 static_assert(sizeof(mm_ids_helper_store) == 4, "unexpected size for mm_ids_helper_store");
 
+// Qwen4exp 512-expert / 10-active fast path. The generic helper launches one warp per expert and every
+// warp scans all token slots (O(n_experts*n_tokens*n_used)). Here one block histograms the slots, computes
+// the 512 bounds, and scatters the two maps in O(n_tokens*n_used). Row ordering inside an expert is irrelevant:
+// ids_dst maps every compact MMQ row back to its exact destination and there is no cross-row reduction.
+__launch_bounds__(1024, 1)
+static __global__ void mm_ids_helper_512_10(
+        const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst,
+        int32_t * __restrict__ expert_bounds, const int n_tokens, const int nchannels_y,
+        const int si1, const int sis1, const bool write_inverse) {
+    __shared__ int counts[512];
+    __shared__ int cursors[512];
+
+    const int tid = threadIdx.x;
+    if (tid < 512) {
+        counts[tid] = 0;
+        cursors[tid] = 0;
+    }
+    __syncthreads();
+
+    const int n_slots = n_tokens * 10;
+    for (int idx = tid; idx < n_slots; idx += blockDim.x) {
+        const int token = idx / 10;
+        const int iex   = idx - token * 10;
+        atomicAdd(&counts[ids[token * si1 + iex]], 1);
+    }
+    __syncthreads();
+
+    if (tid == 0) {
+        int total = 0;
+        expert_bounds[0] = 0;
+        for (int e = 0; e < 512; ++e) {
+            total += counts[e];
+            expert_bounds[e + 1] = total;
+        }
+    }
+    __syncthreads();
+
+    for (int idx = tid; idx < n_slots; idx += blockDim.x) {
+        const int token = idx / 10;
+        const int iex   = idx - token * 10;
+        const int expert = ids[token * si1 + iex];
+        const int dst = expert_bounds[expert] + atomicAdd(&cursors[expert], 1);
+        ids_dst[dst] = idx;
+        if (write_inverse) {
+            ids_src1[idx] = dst;
+        } else {
+            ids_src1[dst] = token * sis1 + iex % nchannels_y;
+        }
+    }
+}
+
 // the generic path passes 0, which needs no padding since it never groups lanes by token
 template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2<(n + 1)/2>::value; };
 template <>      struct mm_ids_pow2<1> { static constexpr int value = 1; };
@@ -151,6 +202,12 @@ static void launch_mm_ids_helper(
 void ggml_cuda_launch_mm_ids_helper(
         const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,
         const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) {
+    const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+    if (GGML_CUDA_CC_IS_RDNA3_5(cc) && n_experts == 512 && n_expert_used == 10 && getenv("GGML_CUDA_DISABLE_MMID_512") == nullptr) {
+        mm_ids_helper_512_10<<<1, 1024, 0, stream>>>(
+            ids, ids_src1, ids_dst, expert_bounds, n_tokens, nchannels_y, si1, sis1, write_inverse);
+        return;
+    }
     switch (n_expert_used) {
         case  2:
             launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);
diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh
index 180b2d9370d1..f8a9e6350926 100644
--- a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh
+++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh
@@ -79,15 +79,15 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
     CASE(GGML_TYPE_Q8_0, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q8_0, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q8_0, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_Q8_0, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q8_0, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q8_0, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q8_0, 256, 2, 128,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q8_0, 128, 2,  64,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q8_0, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q8_0, 256, 2, 128,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q8_0, 256, 2, 128,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q8_0, 256, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
 
 // ---------------------------------------------------------------------------------------------
 
@@ -115,42 +115,42 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
 
     CASE(GGML_TYPE_Q4_K, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q4_K, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q4_K, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q4_K, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q4_K, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
 
     CASE(GGML_TYPE_Q5_K, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q5_K, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q5_K, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q5_K, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q5_K, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
 
     CASE(GGML_TYPE_Q6_K, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q6_K, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_Q6_K, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_Q6_K, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_Q6_K, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
 
 // ---------------------------------------------------------------------------------------------
 
@@ -212,7 +212,7 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
     CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_IQ3_XXS, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_IQ3_XXS, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
-    CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
+    CASE(GGML_TYPE_IQ3_XXS, 128, 2,  64,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128,  80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128,  96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
@@ -263,7 +263,7 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
     CASE(GGML_TYPE_MXFP4, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_MXFP4, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_MXFP4, 256, 2, 128,  64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
-    CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
+    CASE(GGML_TYPE_MXFP4, 128, 2,  64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
     CASE(GGML_TYPE_MXFP4, 128, 2,  64,  16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_MXFP4, 128, 2,  64,  32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
     CASE(GGML_TYPE_MXFP4, 256, 2, 128,  48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
@@ -288,3 +288,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
 
     return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
 }
+
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, false).nthreads == 256);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, false).I == 64);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, true).nthreads == 128);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, true).I == 64);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, false).nthreads == 128);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, false).I == 64);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, true).nthreads == 256);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, true).I == 128);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_MXFP4, 128, true).nthreads == 128);
+static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_MXFP4, 128, true).I == 64);
diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh
index 7f00bad943e2..40c825405e26 100644
--- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh
+++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh
@@ -532,6 +532,102 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
     }
 }
 
+// Register-staged variant of the Q8_0 tile loader used for software prefetching on RDNA3.5:
+//     the global loads for the next K iteration are issued into registers while the current tile is consumed,
+//     then written to shared memory with exactly the same layout as ggml_cuda_mmq_load_tiles_q8_0.
+template <ggml_type type, int J, bool fallback> struct ggml_cuda_mmq_x_regs_q8_0 {
+    // Only instantiated for RDNA3.5 (wave32) device code; the host pass needs constant sizes.
+    static constexpr int warp_size       = WARP_SIZE;
+    static constexpr int nwarps          = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;
+    static constexpr int I               = ggml_cuda_mmq_get_I(type, J, fallback);
+    static constexpr int threads_per_row = 32;
+    static constexpr int nrows           = warp_size / threads_per_row;
+    static constexpr int rows_per_step   = nrows*nwarps;
+    static constexpr int n_qs            = (I + rows_per_step - 1) / rows_per_step;
+    static constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0;
+    static constexpr int rows_per_warp   = warp_size / blocks_per_tile_x_row;
+    static constexpr int d_rows_per_step = nwarps*rows_per_warp;
+    static constexpr int n_d             = (I + d_rows_per_step - 1) / d_rows_per_step;
+
+    int  qs[2*n_qs];
+    half d[n_d];
+};
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0_regs(
+        const char * __restrict__ x, ggml_cuda_mmq_x_regs_q8_0<type, J, fallback> & r, const int kbx0, const int i_max, const int stride) {
+    using R = ggml_cuda_mmq_x_regs_q8_0<type, J, fallback>;
+    static_assert(R::warp_size == ggml_cuda_get_physical_warp_size(), "register-staged Q8_0 loader assumes wave32");
+
+    const int txi  = R::warp_size > R::threads_per_row ? threadIdx.x % R::threads_per_row : threadIdx.x;
+    const int kbx  = txi / QI8_0;
+    const int kqsx = txi % QI8_0;
+
+#pragma unroll
+    for (int i0 = 0; i0 < R::I; i0 += R::rows_per_step) {
+        int i = i0 + (R::nrows == 1 ? threadIdx.y : threadIdx.y*R::nrows + threadIdx.x/R::threads_per_row);
+
+        if (fallback) {
+            i = min(i, i_max);
+        }
+
+        const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;
+
+        r.qs[2*(i0/R::rows_per_step) + 0] = get_int_b2(bxi[0].qs,                   kqsx);
+        r.qs[2*(i0/R::rows_per_step) + 1] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);
+    }
+
+    const int kbxd = threadIdx.x % R::blocks_per_tile_x_row;
+
+#pragma unroll
+    for (int i0 = 0; i0 < R::I; i0 += R::d_rows_per_step) {
+        int i = i0 + threadIdx.y*R::rows_per_warp + threadIdx.x/R::blocks_per_tile_x_row;
+
+        if (fallback) {
+            i = min(i, i_max);
+        }
+
+        const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;
+
+        r.d[i0/R::d_rows_per_step] = bxi->d;
+    }
+}
+
+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_store_tiles_q8_0_regs(
+        const ggml_cuda_mmq_x_regs_q8_0<type, J, fallback> & r, int * __restrict__ x_tile, const int i_max) {
+    using R = ggml_cuda_mmq_x_regs_q8_0<type, J, fallback>;
+    constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);
+
+    int   * x_qs = (int   *)  x_tile;
+    float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K);
+
+    const int txi = R::warp_size > R::threads_per_row ? threadIdx.x % R::threads_per_row : threadIdx.x;
+
+#pragma unroll
+    for (int i0 = 0; i0 < R::I; i0 += R::rows_per_step) {
+        int i = i0 + (R::nrows == 1 ? threadIdx.y : threadIdx.y*R::nrows + threadIdx.x/R::threads_per_row);
+
+        if (fallback) {
+            i = min(i, i_max);
+        }
+
+        x_qs[i*sram_stride + 0             + txi] = r.qs[2*(i0/R::rows_per_step) + 0];
+        x_qs[i*sram_stride + MMQ_TILE_NE_K + txi] = r.qs[2*(i0/R::rows_per_step) + 1];
+    }
+
+    const int kbxd = threadIdx.x % R::blocks_per_tile_x_row;
+
+#pragma unroll
+    for (int i0 = 0; i0 < R::I; i0 += R::d_rows_per_step) {
+        int i = i0 + threadIdx.y*R::rows_per_warp + threadIdx.x/R::blocks_per_tile_x_row;
+
+        if (fallback) {
+            i = min(i, i_max);
+        }
+
+        x_df[i*sram_stride + kbxd] = r.d[i0/R::d_rows_per_step];
+    }
+}
+
 // ---------------------------------------------------------------------------------------------
 
 template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K(
diff --git a/ggml/src/ggml-cuda/mmq-q45.cu b/ggml/src/ggml-cuda/mmq-q45.cu
new file mode 100644
index 000000000000..e9774527cc61
--- /dev/null
+++ b/ggml/src/ggml-cuda/mmq-q45.cu
@@ -0,0 +1,9 @@
+#define mul_mat_q mul_mat_q_q45
+#define launch_mul_mat_q launch_mul_mat_q_q45
+#include "mmq.cuh"
+#undef launch_mul_mat_q
+#undef mul_mat_q
+
+void launch_mul_mat_q_q5_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+    launch_mul_mat_q_q45<GGML_TYPE_Q5_K, 32, false>(ctx, args, stream);
+}
diff --git a/ggml/src/ggml-cuda/mmq-q6.cu b/ggml/src/ggml-cuda/mmq-q6.cu
new file mode 100644
index 000000000000..ed9548f79608
--- /dev/null
+++ b/ggml/src/ggml-cuda/mmq-q6.cu
@@ -0,0 +1,9 @@
+#define mul_mat_q mul_mat_q_q6
+#define launch_mul_mat_q launch_mul_mat_q_q6
+#include "mmq.cuh"
+#undef launch_mul_mat_q
+#undef mul_mat_q
+
+void launch_mul_mat_q_q6_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+    launch_mul_mat_q_q6<GGML_TYPE_Q6_K, 32, false>(ctx, args, stream);
+}
diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuh
index d573433865f8..2493ebea6f24 100644
--- a/ggml/src/ggml-cuda/mmq-vec-dot.cuh
+++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh
@@ -152,8 +152,14 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma(
     constexpr int sram_stride   = ggml_cuda_mmq_get_sram_stride(type, J, fallback);
     constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);
     constexpr int ntx           = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+    constexpr int nwarps        = ggml_cuda_mmq_get_nthreads(type, J, fallback) / ggml_cuda_get_physical_warp_size();
+    constexpr bool split_j      = type == GGML_TYPE_Q8_0 && J == 128 && !fallback && I == 64 && nwarps == 8;
+    constexpr int j_group       = split_j ? J/2 : J;
 
-    y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);
+    const int warp_i = split_j ? threadIdx.y % 4 : threadIdx.y;
+    const int warp_j = split_j ? threadIdx.y / 4 : 0;
+
+    y += (warp_j*j_group + (warp_i % ntx)*tile_C::J) * MMQ_TILE_Y_K;
 
     const int   * x_qs = (const int   *) x;
     const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;
@@ -161,7 +167,7 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma(
     const float * y_df = (const float *) y;
     const half2 * y_ds = (const half2 *) y;
 
-    const int i0 = (threadIdx.y / ntx) * rows_per_warp;
+    const int i0 = (warp_i / ntx) * rows_per_warp;
 
     for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {
         const int k0 = k00 + k01;
@@ -173,7 +179,7 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma(
         }
 
 #pragma unroll
-        for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+        for (int j0 = 0; j0 < j_group; j0 += ntx*tile_C::J) {
             tile_B B;
             load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);
 
@@ -1248,4 +1254,3 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_
         }
     }
 }
-
diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
index 7fb4401489c9..70d59cee82e0 100644
--- a/ggml/src/ggml-cuda/mmq.cu
+++ b/ggml/src/ggml-cuda/mmq.cu
@@ -5,6 +5,26 @@
 
 #include <cstdint>
 
+void launch_mul_mat_q_q5_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream);
+void launch_mul_mat_q_q6_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream);
+
+static bool use_rdna3_5_q5_ling(const mmq_args & args) {
+    const int id = ggml_cuda_get_device();
+    const int cc = ggml_cuda_info().devices[id].cc;
+    return args.type_x == GGML_TYPE_Q5_K && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr &&
+        ((args.ncols_x == 2560 && args.nrows_x == 768) || (args.ncols_x == 768 && args.nrows_x == 2560)) &&
+        args.nchannels_x == 512 && args.nchannels_y > 0 &&
+        (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 32;
+}
+
+static bool use_rdna3_5_q6_ling_j32(const mmq_args & args) {
+    const int id = ggml_cuda_get_device();
+    const int cc = ggml_cuda_info().devices[id].cc;
+    return args.type_x == GGML_TYPE_Q6_K && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr &&
+        args.ncols_x == 768 && args.nrows_x == 2560 && args.nchannels_x == 512 && args.nchannels_y > 0 &&
+        (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 32;
+}
+
 static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     switch (args.type_x) {
         case GGML_TYPE_Q1_0:
@@ -39,10 +59,18 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
             mul_mat_q_case<GGML_TYPE_Q4_K>(ctx, args, stream);
             break;
         case GGML_TYPE_Q5_K:
-            mul_mat_q_case<GGML_TYPE_Q5_K>(ctx, args, stream);
+            if (use_rdna3_5_q5_ling(args)) {
+                launch_mul_mat_q_q5_k_j32(ctx, args, stream);
+            } else {
+                mul_mat_q_case<GGML_TYPE_Q5_K>(ctx, args, stream);
+            }
             break;
         case GGML_TYPE_Q6_K:
-            mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);
+            if (use_rdna3_5_q6_ling_j32(args)) {
+                launch_mul_mat_q_q6_k_j32(ctx, args, stream);
+            } else {
+                mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);
+            }
             break;
 // -----------------------------------------------------------------------
         case GGML_TYPE_IQ1_S:
@@ -82,8 +110,141 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
     }
 }
 
-void ggml_cuda_mul_mat_q(
-        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
+void ggml_cuda_mul_mat_q_pair(ggml_backend_cuda_context & ctx, ggml_tensor * dst0, ggml_tensor * dst1) {
+    GGML_ASSERT(dst0->src[1] == dst1->src[1]);
+    GGML_ASSERT(dst0->src[2] == dst1->src[2]);
+
+    const ggml_tensor * src0s[] = { dst0->src[0], dst1->src[0] };
+    ggml_tensor * dsts[] = { dst0, dst1 };
+    const ggml_tensor * src1 = dst0->src[1];
+    const ggml_tensor * ids = dst0->src[2];
+    const ggml_tensor * src0 = src0s[0];
+
+    GGML_ASSERT(src1->type == GGML_TYPE_F32);
+    if (ids == nullptr) {
+        const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+        cudaStream_t stream = ctx.stream();
+        const int64_t ne10_padded = GGML_PAD(src1->ne[0], MATRIX_ROW_PADDING);
+        size_t nbytes_src1_q8_1 = src1->ne[3] * src1->ne[2] * src1->ne[1] * ne10_padded * sizeof(block_q8_1_mmq) / QK8_1_MMQ;
+
+        for (int i = 0; i < 2; ++i) {
+            const ggml_tensor * src0_i = src0s[i];
+            ggml_tensor * dst_i = dsts[i];
+            GGML_ASSERT(dst_i->type == GGML_TYPE_F32 && dst_i->src[1] == src1);
+            GGML_ASSERT(ggml_are_same_shape(src0, src0_i) && ggml_are_same_shape(dst0, dst_i));
+            GGML_ASSERT(src0_i->ne[0] == src1->ne[0]);
+            GGML_ASSERT(mmq_get_q8_1_ds_layout(src0->type) == mmq_get_q8_1_ds_layout(src0_i->type));
+            const bool fallback = src0_i->ne[1] % 128 != 0;
+            nbytes_src1_q8_1 = std::max(nbytes_src1_q8_1,
+                src1->ne[3] * src1->ne[2] * src1->ne[1] * ne10_padded * sizeof(block_q8_1_mmq) / QK8_1_MMQ +
+                ggml_cuda_mmq_get_J_max(src0_i->type, fallback, cc, src1->ne[1]) * sizeof(block_q8_1_mmq));
+        }
+
+        ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
+        quantize_mmq_q8_1_cuda((const float *) src1->data, nullptr, src1_q8_1.get(), src0->type,
+            src1->ne[0], src1->nb[1] / sizeof(float), src1->nb[2] / sizeof(float), src1->nb[3] / sizeof(float),
+            ne10_padded, src1->ne[1], src1->ne[2], src1->ne[3], stream);
+        CUDA_CHECK(cudaGetLastError());
+
+        const int64_t s12_q = src1->ne[1] * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));
+        const int64_t s13_q = src1->ne[2] * s12_q;
+        for (int i = 0; i < 2; ++i) {
+            const ggml_tensor * src0_i = src0s[i];
+            ggml_tensor * dst_i = dsts[i];
+            GGML_ASSERT(ggml_are_same_shape(src0, src0_i) && ggml_are_same_shape(dst0, dst_i));
+            const mmq_args args = {
+                (const char *) src0_i->data, src0_i->type, (const int *) src1_q8_1.get(), nullptr, nullptr, (float *) dst_i->data,
+                nullptr,
+                src0_i->ne[0], src0_i->ne[1], src1->ne[1], (int64_t) (src0_i->nb[1] / ggml_type_size(src0_i->type)), src1->ne[1], (int64_t) (dst_i->nb[1] / sizeof(float)),
+                src0_i->ne[2], src1->ne[2], (int64_t) (src0_i->nb[2] / ggml_type_size(src0_i->type)), s12_q, (int64_t) (dst_i->nb[2] / sizeof(float)),
+                src0_i->ne[3], src1->ne[3], (int64_t) (src0_i->nb[3] / ggml_type_size(src0_i->type)), s13_q, (int64_t) (dst_i->nb[3] / sizeof(float)),
+                src1->ne[1]};
+            ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);
+        }
+        return;
+    }
+
+    GGML_ASSERT(ids->type == GGML_TYPE_I32);
+    GGML_ASSERT(src1->ne[3] == 1);
+    GGML_ASSERT(src1->nb[2] % src1->nb[1] == 0);
+    GGML_ASSERT(ids->nb[0] == ggml_element_size(ids));
+
+    const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+    cudaStream_t stream = ctx.stream();
+    const int64_t n_expert_used = ids->ne[0];
+    const int64_t n_tokens = src1->ne[2];
+    const int64_t ne_get_rows = n_tokens*n_expert_used;
+    const int64_t ne10_padded = GGML_PAD(src1->ne[0], MATRIX_ROW_PADDING);
+    const bool dedup_bcast = src1->ne[1] == 1 && n_expert_used > 1;
+
+    ggml_cuda_pool_alloc<int32_t> ids_src1(ctx.pool(), ne_get_rows);
+    ggml_cuda_pool_alloc<int32_t> ids_dst(ctx.pool(), ne_get_rows);
+    ggml_cuda_pool_alloc<int32_t> expert_bounds(ctx.pool(), src0->ne[2] + 1);
+
+    const int si1  = ids->nb[1] / ggml_element_size(ids);
+    const int sis1 = src1->nb[2] / src1->nb[1];
+    ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(),
+        src0->ne[2], n_tokens, n_expert_used, src1->ne[1], si1, sis1, /*write_inverse =*/ dedup_bcast, stream);
+    CUDA_CHECK(cudaGetLastError());
+
+    size_t nbytes_src1_q8_1 = n_tokens*n_expert_used*ne10_padded * sizeof(block_q8_1_mmq)/QK8_1_MMQ;
+    for (int i = 0; i < 2; ++i) {
+        const ggml_tensor * src0_i = src0s[i];
+        ggml_tensor * dst_i = dsts[i];
+        GGML_ASSERT(dst_i->type == GGML_TYPE_F32);
+        GGML_ASSERT(dst_i->nb[2] % dst_i->nb[1] == 0);
+        GGML_ASSERT(ggml_are_same_shape(src0, src0_i));
+        GGML_ASSERT(ggml_are_same_shape(dst0, dst_i));
+        GGML_ASSERT(mmq_get_q8_1_ds_layout(src0->type) == mmq_get_q8_1_ds_layout(src0_i->type));
+        GGML_ASSERT(src0_i->ne[0] == src1->ne[0]);
+        GGML_ASSERT(dst_i->ne[1] == n_expert_used);
+
+        const bool fallback = src0_i->ne[1] % 128 != 0;
+        nbytes_src1_q8_1 = std::max(nbytes_src1_q8_1,
+            n_tokens*n_expert_used*ne10_padded * sizeof(block_q8_1_mmq)/QK8_1_MMQ +
+            ggml_cuda_mmq_get_J_max(src0_i->type, fallback, cc, src1->ne[1]) * sizeof(block_q8_1_mmq));
+    }
+
+    ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
+    const int64_t s11 = src1->nb[1] / ggml_type_size(src1->type);
+    const int64_t s12_src = src1->nb[2] / ggml_type_size(src1->type);
+    const int64_t s13_src = src1->nb[3] / ggml_type_size(src1->type);
+    if (dedup_bcast) {
+        quantize_scatter_mmq_q8_1_cuda((const float *) src1->data, ids_src1.get(), src1_q8_1.get(), src0->type,
+            src1->ne[0], /*stride_token=*/s12_src, ne10_padded, n_tokens, ne_get_rows, n_expert_used, stream);
+    } else {
+        quantize_mmq_q8_1_cuda((const float *) src1->data, ids_src1.get(), src1_q8_1.get(), src0->type,
+            src1->ne[0], s11, s12_src, s13_src, ne10_padded, ne_get_rows, 1, 1, stream);
+    }
+    CUDA_CHECK(cudaGetLastError());
+
+    const int64_t s12_q = src1->ne[1] * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));
+    const int64_t s13_q = n_tokens*s12_q;
+    for (int i = 0; i < 2; ++i) {
+        const ggml_tensor * src0_i = src0s[i];
+        ggml_tensor * dst_i = dsts[i];
+        const size_t ts_src0 = ggml_type_size(src0_i->type);
+        const size_t ts_dst = ggml_type_size(dst_i->type);
+        const int64_t s01 = src0_i->nb[1] / ts_src0;
+        const int64_t s02 = src0_i->nb[2] / ts_src0;
+        const int64_t s03 = src0_i->nb[3] / ts_src0;
+        const int64_t s1 = dst_i->nb[1] / ts_dst;
+        const int64_t s2 = dst_i->nb[2] / ts_dst;
+        const int64_t s3 = dst_i->nb[3] / ts_dst;
+        const mmq_args args = {
+            (const char *) src0_i->data, src0_i->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), (float *) dst_i->data,
+            nullptr,
+            src0_i->ne[0], src0_i->ne[1], ne_get_rows, s01, ne_get_rows, s1,
+            src0_i->ne[2], src0_i->ne[2], s02, s12_q, s2,
+            src0_i->ne[3], src1->ne[3], s03, s13_q, s3,
+            n_tokens};
+        ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);
+    }
+}
+
+static void ggml_cuda_mul_mat_q_impl(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids,
+        ggml_tensor * dst, const ggml_tensor * swiglu) {
     GGML_ASSERT(        src1->type == GGML_TYPE_F32);
     GGML_ASSERT(        dst->type  == GGML_TYPE_F32);
     GGML_ASSERT(!ids || ids->type  == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.
@@ -102,6 +263,14 @@ void ggml_cuda_mul_mat_q(
     GGML_ASSERT(        nb0        == ts_dst);
     GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));
 
+    const ggml_tensor * gate = swiglu ? swiglu->src[0] : nullptr;
+    const ggml_tensor * up   = swiglu ? swiglu->src[1] : nullptr;
+    if (swiglu) {
+        GGML_ASSERT(src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_IQ4_NL);
+        GGML_ASSERT(gate && up && gate->type == GGML_TYPE_F32 && up->type == GGML_TYPE_F32);
+        GGML_ASSERT(ggml_are_same_shape(gate, up) && ggml_are_same_shape(gate, src1));
+    }
+
     const char  * src0_d = (const char  *) src0->data;
     const float * src1_d = (const float *) src1->data;
     float       *  dst_d = (float       *)  dst->data;
@@ -152,6 +321,12 @@ void ggml_cuda_mul_mat_q(
                 quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,
                                         ne11, ne12, ne13, stream);
 
+            } else if (swiglu) {
+                quantize_mmq_q8_1_swiglu_cuda(
+                    (const float *) gate->data, (const float *) up->data, nullptr, src1_q8_1.get(), src0->type,
+                    ne10, ne10_padded, ne11 * ne12 * ne13, /*logical_n1=*/1,
+                    gate->nb[1] / sizeof(float), gate->nb[1] / sizeof(float),
+                    up->nb[1] / sizeof(float), up->nb[1] / sizeof(float), stream);
             } else {
                 quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,
                                        ne11, ne12, ne13, stream);
@@ -229,6 +404,12 @@ void ggml_cuda_mul_mat_q(
                 quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,
                                         ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);
             }
+        } else if (swiglu) {
+            quantize_mmq_q8_1_swiglu_cuda(
+                (const float *) gate->data, (const float *) up->data, ids_src1.get(), src1_q8_1.get(), src0->type,
+                ne10, ne10_padded, ne11_flat, n_expert_used,
+                gate->nb[1] / sizeof(float), gate->nb[2] / sizeof(float),
+                up->nb[1] / sizeof(float), up->nb[2] / sizeof(float), stream);
         } else if (dedup_bcast) {
             quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,
                                     /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);
@@ -256,6 +437,17 @@ void ggml_cuda_mul_mat_q(
     ggml_cuda_mul_mat_q_switch_type(ctx, args, stream);
 }
 
+void ggml_cuda_mul_mat_q(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
+    ggml_cuda_mul_mat_q_impl(ctx, src0, src1, ids, dst, nullptr);
+}
+
+void ggml_cuda_mul_mat_q_swiglu(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * ids, ggml_tensor * dst, const ggml_tensor * swiglu) {
+    ggml_cuda_mul_mat_q_impl(ctx, src0, swiglu, ids, dst, swiglu);
+}
+
+
 bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) {
 #ifdef GGML_CUDA_FORCE_CUBLAS
     return false;
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
index c978b4421c5f..39bad397cbc4 100644
--- a/ggml/src/ggml-cuda/mmq.cuh
+++ b/ggml/src/ggml-cuda/mmq.cuh
@@ -486,18 +486,22 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(
     constexpr int I             = ggml_cuda_mmq_get_I(type, J, fallback);
     constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback);
     constexpr int ntx           = rows_per_warp/tile_C::I; // Number of x minitiles per warp.
+    constexpr bool split_j      = type == GGML_TYPE_Q8_0 && J == 128 && !fallback && I == 64 && nwarps == 8;
+    constexpr int j_group       = split_j ? J/2 : J;
 
-    const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I);
+    const int warp_i = split_j ? threadIdx.y % 4 : threadIdx.y;
+    const int warp_j = split_j ? threadIdx.y / 4 : 0;
+    const int i0 = (warp_i / ntx) * (ntx*tile_C::I);
 
     const bool y_scale_used = y_scale != nullptr;
 
 #pragma unroll
-    for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {
+    for (int j0 = 0; j0 < j_group; j0 += ntx*tile_C::J) {
 #pragma unroll
         for (int n = 0; n < ntx; ++n) {
 #pragma unroll
             for (int l = 0; l < tile_C::ne; ++l) {
-                const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l);
+                const int j = warp_j*j_group + j0 + (warp_i % ntx)*tile_C::J + tile_C::get_j(l);
 
                 if (j > j_max) {
                     continue;
@@ -870,6 +874,22 @@ static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_b
 
 // ---------------------------------------------------------------------------------------------
 
+// Software prefetch of the next K iteration into registers, see mul_mat_q_process_tile.
+template <ggml_type type, int J, bool fallback>
+static constexpr __host__ __device__ bool ggml_cuda_mmq_use_prefetch() {
+#if defined(RDNA3_5) && defined(AMD_WMMA_AVAILABLE)
+    // Whitelist: the extra registers cause spills in several other specializations.
+    return (type == GGML_TYPE_Q8_0    && (J == 48 || J == 128) && !fallback) ||
+           (type == GGML_TYPE_Q6_K    &&  J == 32)              ||
+           (type == GGML_TYPE_Q5_K    &&  J == 32)              ||
+           (type == GGML_TYPE_Q4_K    &&  J == 48)              ||
+           (type == GGML_TYPE_IQ2_S   &&  J == 128)             ||
+           (type == GGML_TYPE_IQ3_XXS &&  J == 128);
+#else
+    return false;
+#endif // defined(RDNA3_5) && defined(AMD_WMMA_AVAILABLE)
+}
+
 template <ggml_type type, int J, bool fallback, bool fixup>
 static __device__ __forceinline__ void mul_mat_q_process_tile(
         const char * __restrict__ x, const int offset_x, const int * __restrict__ y,
@@ -904,6 +924,86 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(
 
     constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int);
 
+    if constexpr (ggml_cuda_mmq_use_prefetch<type, J, fallback>()) {
+        // RDNA3.5 software prefetch: with 64 KiB of LDS per CU only 1-2 blocks are resident, so the
+        //     global load latency of every K iteration is exposed. Stage the next iteration's tiles in
+        //     registers while the current one is consumed. The arithmetic is unchanged.
+        constexpr int  nthreads  = nwarps*warp_size;
+        constexpr int  y_regs_n  = (J*MMQ_TILE_Y_K + nthreads - 1) / nthreads;
+        constexpr bool x_regs_ok = type == GGML_TYPE_Q8_0;
+        const     int  tid       = threadIdx.y*warp_size + threadIdx.x;
+
+        int yr0[y_regs_n];
+        int yr1[y_regs_n];
+        ggml_cuda_mmq_x_regs_q8_0<type, J, fallback> xr;
+        GGML_UNUSED(xr);
+
+        auto load_y_regs = [&](int (&yr)[y_regs_n], const int kb0, const int half) {
+            const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + half*sz);
+#pragma unroll
+            for (int r = 0; r < y_regs_n; ++r) {
+                yr[r] = by0[r*nthreads + tid];
+            }
+        };
+        auto store_y_regs = [&](const int (&yr)[y_regs_n]) {
+#pragma unroll
+            for (int r = 0; r < y_regs_n; ++r) {
+                tile_y[r*nthreads + tid] = yr[r];
+            }
+        };
+
+        if (kb0_start < kb0_stop) {
+            if constexpr (x_regs_ok) {
+                ggml_cuda_mmq_load_tiles_q8_0_regs<type, J, fallback>(x, xr, offset_x + kb0_start, tile_x_max_i, stride_row_x);
+            }
+            load_y_regs(yr0, kb0_start, 0);
+            load_y_regs(yr1, kb0_start, 1);
+        }
+
+        for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {
+            const bool has_next = kb0 + blocks_per_iter < kb0_stop;
+
+            if constexpr (x_regs_ok) {
+                ggml_cuda_mmq_store_tiles_q8_0_regs<type, J, fallback>(xr, tile_x, tile_x_max_i);
+            } else {
+                load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);
+            }
+            store_y_regs(yr0);
+
+            __syncthreads();
+
+            if (has_next) {
+                if constexpr (x_regs_ok) {
+                    ggml_cuda_mmq_load_tiles_q8_0_regs<type, J, fallback>(x, xr, offset_x + kb0 + blocks_per_iter, tile_x_max_i, stride_row_x);
+                }
+                load_y_regs(yr0, kb0 + blocks_per_iter, 0);
+            }
+
+            vec_dot(tile_x, tile_y, sum, 0);
+
+            __syncthreads();
+
+            store_y_regs(yr1);
+
+            __syncthreads();
+
+            if (has_next) {
+                load_y_regs(yr1, kb0 + blocks_per_iter, 1);
+            }
+
+            vec_dot(tile_x, tile_y, sum, MMQ_TILE_NE_K);
+
+            __syncthreads();
+        }
+
+        if (fixup) {
+            write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);
+        } else {
+            write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);
+        }
+        return;
+    }
+
     for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {
         load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);
         {
@@ -973,21 +1073,7 @@ static __global__ void mul_mat_q(
 
     const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y
 
-    // Initialize the ids for writing back data with just the index.
-    // For regular matrix multiplications this is never changed.
-    // For MoE the correct indices are loaded from ids_dst.
     extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory.
-#pragma unroll
-    for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
-        const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
-
-        if (j0 + nwarps*warp_size > J && j >= J) {
-            break;
-        }
-
-        ids_dst_shared[j] = j;
-    }
-    __syncthreads();
 
     if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback)) {
         const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
@@ -1036,6 +1122,18 @@ static __global__ void mul_mat_q(
                 ids_dst_shared[j] = ids_dst[col_low + jt*J + j];
             }
             __syncthreads();
+        } else {
+#pragma unroll
+            for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
+                const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
+
+                if (j0 + nwarps*warp_size > J && j >= J) {
+                    break;
+                }
+
+                ids_dst_shared[j] = j;
+            }
+            __syncthreads();
         }
 
         offset_y   += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int));
@@ -1059,6 +1157,18 @@ static __global__ void mul_mat_q(
         return;
     }
 
+#pragma unroll
+    for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {
+        const int j = j0 + threadIdx.y*warp_size + threadIdx.x;
+
+        if (j0 + nwarps*warp_size > J && j >= J) {
+            break;
+        }
+
+        ids_dst_shared[j] = j;
+    }
+    __syncthreads();
+
     constexpr int ITER_K          = ggml_cuda_mmq_get_K_vram(type, J, fallback);
     constexpr int blocks_per_iter = ITER_K / qk;
 
@@ -1236,6 +1346,101 @@ static __global__ void mul_mat_q(
          tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);
 }
 
+// routed-compact MoE tiles: one descriptor per real (expert, tile) pair; the builder runs as a single block with one
+// thread per expert
+#define MMQ_ROUTED_MAX_EXPERTS 1024
+#ifndef MMQ_IQ_ID_J_MID
+#define MMQ_IQ_ID_J_MID 64
+#endif
+
+static constexpr bool mmq_rdna3_5_id_n_experts_ok(const int64_t n_experts) {
+    return n_experts >= 8 && n_experts <= MMQ_ROUTED_MAX_EXPERTS;
+}
+
+template <int J>
+static __global__ void build_mmq_routed_descriptors(
+        const int32_t * expert_bounds, uint32_t * descriptors, int n_experts, int max_descriptors) {
+    __shared__ int tile_counts[MMQ_ROUTED_MAX_EXPERTS];
+    const int expert = threadIdx.x;
+
+    for (int i = expert; i < max_descriptors; i += blockDim.x) {
+        descriptors[i] = UINT32_MAX;
+    }
+
+    if (expert < n_experts) {
+        const int count = expert_bounds[expert + 1] - expert_bounds[expert];
+        tile_counts[expert] = (count + J - 1) / J;
+    }
+    __syncthreads();
+
+    if (expert < n_experts) {
+        int descriptor = 0;
+        for (int i = 0; i < expert; ++i) {
+            descriptor += tile_counts[i];
+        }
+        for (int jt = 0; jt < tile_counts[expert]; ++jt) {
+            descriptors[descriptor + jt] = uint32_t(expert) | (uint32_t(jt) << 16);
+        }
+    }
+}
+
+template <ggml_type type, int J, bool fallback>
+__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback), ggml_cuda_mmq_get_occupancy(type, J, fallback))
+static __global__ void mul_mat_q_routed_compact(
+        const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
+        const int32_t * __restrict__ expert_bounds, const uint32_t * __restrict__ descriptors,
+        const int max_descriptors, float * __restrict__ dst, const float * __restrict__ y_scale,
+        const uint3 blocks_per_ne00, const int nrows_x, const int stride_row_x, const int ncols_y, const int stride_col_dst,
+        const uint3 channel_ratio, const int stride_channel_x,
+        const uint3 sample_ratio, const int stride_sample_x) {
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+    constexpr int nwarps    = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;
+    constexpr int I         = ggml_cuda_mmq_get_I(type, J, fallback);
+
+    const int wt = blockIdx.z;
+    const int it = blockIdx.x;
+
+    extern __shared__ int ids_dst_shared[];
+    for (int descriptor_idx = blockIdx.y; descriptor_idx < max_descriptors; descriptor_idx += gridDim.y) {
+        const uint32_t descriptor = descriptors[descriptor_idx];
+        if (descriptor == UINT32_MAX) {
+            return;
+        }
+
+        const int zt = descriptor & 0xFFFF;
+        const int jt = descriptor >> 16;
+        const int col_low  = expert_bounds[zt + 0];
+        const int col_high = expert_bounds[zt + 1];
+        const int col_diff = col_high - col_low;
+
+        if (descriptor_idx != int(blockIdx.y)) {
+            __syncthreads();
+        }
+#pragma unroll
+        for (int j0 = 0; j0 < J; j0 += nwarps * warp_size) {
+            const int j = j0 + threadIdx.y * warp_size + threadIdx.x;
+            if (j0 + nwarps * warp_size > J && j >= J) {
+                break;
+            }
+            ids_dst_shared[j] = jt * J + j < col_diff ? ids_dst[col_low + jt * J + j] : 0;
+        }
+        __syncthreads();
+
+        const int offset_x = fastdiv(wt, sample_ratio) * stride_sample_x +
+            fastdiv(zt, channel_ratio) * stride_channel_x + it * I * stride_row_x;
+        const int offset_y = (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));
+        const int tile_x_max_i = nrows_x - it * I - 1;
+        const int tile_y_max_j = col_diff - jt * J - 1;
+        const float * y_scale_tile = y_scale ? y_scale + col_low + jt * J : nullptr;
+
+        constexpr bool fixup = false;
+        mul_mat_q_process_tile<type, J, fallback, fixup>
+            (x, offset_x, y + offset_y, ids_dst_shared, dst + it * I, nullptr, y_scale_tile,
+             stride_row_x, ncols_y, stride_col_dst,
+             tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z);
+    }
+}
+
 template <ggml_type type, int J, bool fallback>
 __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback)/2, 1)
 static __global__ void mul_mat_q_stream_k_fixup(
@@ -1390,6 +1595,82 @@ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const i
     return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int));
 }
 
+// RDNA3.5 tile selection for MoE (mul_mat_id) with 256 experts.
+// The per-expert row count is n_tokens*n_expert_used/256 (e.g. 16/32/64 for ubatch 512/1024/2048 with 8 active
+// experts), but the generic selection below only sees ncols_max == n_tokens*n_expert_used and picks J=128, so most
+// of every tile is padding and the grid contains ncols_max/J*n_experts mostly-empty blocks. Use a J matched to the
+// row count and the routed-compact kernel, which only enumerates real per-expert tiles.
+// J per type/row range was measured on gfx1151 for the 2048x512 and 3072x1024 expert shapes.
+static constexpr int mmq_rdna3_5_id_get_J(const ggml_type type, const int64_t rows_per_expert) {
+    switch (type) {
+        case GGML_TYPE_Q8_0:
+            return rows_per_expert <= 12 ? 16 : rows_per_expert <= 64 ? 48 : 128;
+        // the IQ expert types of the Unsloth qwen4exp mixes (512 experts, 10 active: 40 rows per expert at ubatch 2048)
+        case GGML_TYPE_IQ3_S:
+        case GGML_TYPE_IQ4_NL:
+        case GGML_TYPE_IQ4_XS:
+            return rows_per_expert <= 12 ? 16 : rows_per_expert <= 32 ? 48 : rows_per_expert <= 64 ? MMQ_IQ_ID_J_MID : 128;
+        case GGML_TYPE_Q4_K:
+        case GGML_TYPE_Q5_K:
+        case GGML_TYPE_Q6_K:
+            return rows_per_expert <= 12 ? 16 : rows_per_expert <= 24 ? 32 : rows_per_expert <= 48 ? 48 :
+                rows_per_expert <= 64 ? 32 : 128;
+        default:
+            return 0;
+    }
+}
+
+static constexpr bool mmq_rdna3_5_id_use_compact(const ggml_type type, const int J) {
+    switch (type) {
+        case GGML_TYPE_Q8_0:
+            return J == 16 || J == 48 || J == 128;
+        case GGML_TYPE_IQ3_S:
+        case GGML_TYPE_IQ4_NL:
+        case GGML_TYPE_IQ4_XS:
+            return J == 16 || J == 48 || J == MMQ_IQ_ID_J_MID || J == 128;
+        case GGML_TYPE_Q4_K:
+        case GGML_TYPE_Q5_K:
+        case GGML_TYPE_Q6_K:
+            return J == 16 || J == 32 || J == 48 || J == 64 || J == 96 || J == 128;
+        default:
+            return false;
+    }
+}
+
+static constexpr bool mmq_rdna3_5_id_use_j48_128e(const ggml_type type, const int64_t n_experts, const int64_t rows_per_expert) {
+    return n_experts == 128 && rows_per_expert >= 96 && rows_per_expert <= 160 &&
+        (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0);
+}
+
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0,   4) ==  16);
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0,  16) ==  48);
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0,  64) ==  48);
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 128) == 128);
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q6_K,  16) ==  32);
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_K,  32) ==  48);
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q5_K,  64) ==  32);
+static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_0,  16) ==   0);
+static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 48));
+static_assert(!mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 32));
+static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q6_K, 32));
+static_assert(!mmq_rdna3_5_id_use_compact(GGML_TYPE_IQ2_XS, 48));
+static_assert(!mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 95) &&
+              !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 95) &&
+              !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 95) &&
+              !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 95));
+static_assert(mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 96) &&
+              mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 96) &&
+              mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 96) &&
+              mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 96) &&
+              mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 160) &&
+              mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 160) &&
+              mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 160) &&
+              mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 160));
+static_assert(!mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 161) &&
+              !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 161) &&
+              !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 161) &&
+              !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 161));
+
 template <ggml_type type, int J, bool fallback>
 static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     const int id = ggml_cuda_get_device();
@@ -1424,6 +1705,23 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
     const uint3 channel_ratio_fd   = init_fastdiv_values(channel_ratio);
     const uint3 sample_ratio_fd    = init_fastdiv_values(sample_ratio);
 
+    const bool use_compact_routed = args.ids_dst != nullptr && mmq_rdna3_5_id_n_experts_ok(args.nchannels_y) && GGML_CUDA_CC_IS_RDNA3_5(cc) &&
+        !ggml_cuda_mmq_get_stream_k(type, J, fallback, cc) && mmq_rdna3_5_id_use_compact(type, J);
+    if (use_compact_routed) {
+        const int max_descriptors = (args.ncols_dst + J - 1) / J + args.nchannels_y;
+        ggml_cuda_pool_alloc<uint32_t> descriptors(ctx.pool(id), max_descriptors);
+        build_mmq_routed_descriptors<J><<<1, MMQ_ROUTED_MAX_EXPERTS, 0, stream>>>
+            (args.expert_bounds, descriptors.get(), args.nchannels_y, max_descriptors);
+
+        const int descriptor_blocks = (args.ncols_dst + J - 1) / J;
+        const dim3 block_nums_compact(nty, descriptor_blocks, args.nsamples_y);
+        mul_mat_q_routed_compact<type, J, fallback><<<block_nums_compact, block_dims, nbytes_shared, stream>>>
+            (args.x, args.y, args.ids_dst, args.expert_bounds, descriptors.get(), max_descriptors, args.dst, args.y_scale,
+             blocks_per_ne00_fd, args.nrows_x, args.stride_row_x, args.ncols_y, args.nrows_dst,
+             channel_ratio_fd, args.stride_channel_x, sample_ratio_fd, args.stride_sample_x);
+        return;
+    }
+
     if (!ggml_cuda_mmq_get_stream_k(type, J, fallback, cc)) {
         mul_mat_q<type, J, fallback><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
             (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale,
@@ -1472,6 +1770,15 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
          ntx_fd);
 }
 
+static constexpr bool mmq_use_rdna3_5_iq_id_j48(
+        ggml_type type, int cc, bool fallback, bool has_ids, int64_t ncols_dst, int64_t nchannels_y) {
+    return (type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ3_XXS) && GGML_CUDA_CC_IS_RDNA3_5(cc) &&
+        !fallback && has_ids && nchannels_y > 0 && (ncols_dst + nchannels_y - 1) / nchannels_y == 48;
+}
+
+static_assert(mmq_use_rdna3_5_iq_id_j48(GGML_TYPE_IQ2_XS, GGML_CUDA_CC_RDNA3_5, false, true, 12288, 256));
+static_assert(mmq_use_rdna3_5_iq_id_j48(GGML_TYPE_IQ3_XXS, GGML_CUDA_CC_RDNA3_5, false, true, 12288, 256));
+
 template <ggml_type type, bool fallback>
 void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     const int    id    = ggml_cuda_get_device();
@@ -1481,6 +1788,31 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args,
     int J_best        = 0;
     int ntiles_J_best = INT_MAX;
 
+    if (mmq_use_rdna3_5_iq_id_j48(type, cc, fallback, args.ids_dst != nullptr, args.ncols_dst, args.nchannels_y)) {
+        constexpr int J_rdna3_5 = 48;
+        const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J_rdna3_5, fallback, cc);
+        if (config.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(config, cc) <= smpbo) {
+            J_best = J_rdna3_5;
+            ntiles_J_best = 1;
+        }
+    } else if (GGML_CUDA_CC_IS_RDNA3_5(cc) && !fallback && args.ids_dst != nullptr && mmq_rdna3_5_id_n_experts_ok(args.nchannels_y)) {
+        const int64_t rows_per_expert = (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y;
+        const bool use_j48_128e = mmq_rdna3_5_id_use_j48_128e(type, args.nchannels_y, rows_per_expert);
+        int J_rdna3_5 = use_j48_128e ? 48 : mmq_rdna3_5_id_get_J(type, rows_per_expert);
+        if constexpr (type == GGML_TYPE_Q6_K) {
+            if (getenv("GGML_Q6_COMPACT_J")) {
+                J_rdna3_5 = atoi(getenv("GGML_Q6_COMPACT_J"));
+            }
+        }
+        if (J_rdna3_5 != 0) {
+            const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J_rdna3_5, fallback, cc);
+            if (config.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(config, cc) <= smpbo) {
+                J_best = J_rdna3_5;
+                ntiles_J_best = 1;
+            }
+        }
+    }
+
     for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {
         const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc);
         if (config.type == GGML_TYPE_COUNT) {
@@ -1600,4 +1932,9 @@ extern DECL_MMQ_CASE(GGML_TYPE_NVFP4);
 void ggml_cuda_mul_mat_q(
         ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
 
+void ggml_cuda_mul_mat_q_swiglu(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * ids, ggml_tensor * dst, const ggml_tensor * swiglu);
+
+void ggml_cuda_mul_mat_q_pair(ggml_backend_cuda_context & ctx, ggml_tensor * dst0, ggml_tensor * dst1);
+
 bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);
diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index bd5c5d421a4c..e06920980c8d 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -4,7 +4,7 @@
 #include "mmvf.cuh"
 #include "convert.cuh"
 
-template <typename T, typename type_acc, int ncols_dst, int block_size, bool has_fusion = false, bool is_multi_token_id = false>
+template <typename T, typename type_acc, int ncols_dst, int block_size, bool has_fusion = false, bool is_multi_token_id = false, bool tokens_in_block = false>
 static __global__ void mul_mat_vec_f(
         const T * x_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
         const int ncols2, const uint3 nchannels_y, const int stride_row, const int stride_col_y2, const int stride_col_dst,
@@ -27,11 +27,11 @@ static __global__ void mul_mat_vec_f(
 
     ggml_cuda_pdl_sync();
     if constexpr (is_multi_token_id) {
-        // Multi-token MUL_MAT_ID path, adding these in the normal path causes a perf regression for n_tokens=1 case
-        token_idx  = blockIdx.z;
-        channel_x  = ids[channel_dst + token_idx * ids_stride];
-        channel_y  = fastmodulo(channel_dst, nchannels_y);
-        sample_dst = 0;
+        // Multi-token path, adding these in the normal path causes a perf regression for n_tokens=1 case
+        token_idx  = tokens_in_block ? threadIdx.y : blockIdx.z;
+        channel_x  = ids ? ids[channel_dst + token_idx * ids_stride] : fastdiv((uint32_t) channel_dst, channel_ratio);
+        channel_y  = ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst;
+        sample_dst = ids ? 0 : (tokens_in_block ? blockIdx.z : 0);
     } else {
         token_idx  = ids ? blockIdx.z                                          : 0;
         channel_x  = ids ? ids[blockIdx.y + token_idx * ids_stride]            : fastdiv((uint32_t) channel_dst, channel_ratio);
@@ -99,7 +99,7 @@ static __global__ void mul_mat_vec_f(
     const float2 * y2 = (const float2 *) y;
 
     extern __shared__ char data_mmv[];
-    float * buf_iw = (float *) data_mmv;
+    float * buf_iw = (float *) data_mmv + (tokens_in_block ? threadIdx.y * warp_size : 0);
     [[maybe_unused]] float * buf_iw_gate = nullptr;
     if constexpr (has_fusion) {
         buf_iw_gate = (float *) (data_mmv + warp_size*sizeof(float));
@@ -135,7 +135,35 @@ static __global__ void mul_mat_vec_f(
             }
         }
 
-        for (int col2 = tid; col2 < ncols2; col2 += block_size) {
+        int col2 = tid;
+#if defined(GGML_USE_HIP)
+        // Issue the weight loads of four K iterations before consuming them: the runtime trip count keeps the
+        // compiler from pipelining the loop, and with one 8-byte load in flight per lane the single-column
+        // f32 matvecs (e.g. MoE routers) reach barely half the memory bandwidth. The accumulation order is
+        // unchanged, so the results are bit-identical.
+        if constexpr (ncols_dst == 1 && !has_fusion) {
+            constexpr int pf = 4;
+            for (; col2 + (pf - 1)*block_size < ncols2; col2 += pf*block_size) {
+                float2 tmpx[pf];
+                float2 tmpy[pf];
+#pragma unroll
+                for (int i = 0; i < pf; ++i) {
+                    tmpx[i] = x2[col2 + i*block_size];
+                }
+#pragma unroll
+                for (int i = 0; i < pf; ++i) {
+                    tmpy[i] = y2[col2 + i*block_size];
+                }
+#pragma unroll
+                for (int i = 0; i < pf; ++i) {
+                    ggml_cuda_mad(sumf[0], tmpx[i].x, tmpy[i].x);
+                    ggml_cuda_mad(sumf[0], tmpx[i].y, tmpy[i].y);
+                }
+            }
+        }
+#endif // defined(GGML_USE_HIP)
+
+        for (; col2 < ncols2; col2 += block_size) {
             const float2 tmpx = x2[col2];
             float2 tmpx_gate = make_float2(0.0f, 0.0f);
             if constexpr (has_fusion) {
@@ -383,7 +411,80 @@ static __global__ void mul_mat_vec_f(
     }
 }
 
-template<typename T, typename type_acc, int ncols_dst, int block_size, bool is_multi_token_id = false>
+
+#if defined(GGML_USE_HIP)
+// One wave per output row emulating mul_mat_vec_f<nv_bfloat16, float, 1, 256>: lane l accumulates the partial
+// sums of the eight virtual warps' lane l (columns 2*(32*w + l) + 256*i) in the same fma order, reduces each
+// with the same butterfly and combines the eight partials exactly like block_reduce does over lanes 0..7 (the
+// other lanes hold zero). The standard kernel launches 8 waves per row that each issue a single 4-byte load,
+// which on gfx1151 leaves K=512 MoE down projections at ~145 GB/s; here each lane keeps 8 loads in flight
+// (measured: 93 -> 67 us for 8 x 2048 rows of K=512; slower than the standard kernel for K=2048).
+template <int rows_per_block>
+__launch_bounds__(32 * rows_per_block, 1)
+static __global__ void mul_mat_vec_bf16_wave(
+        const nv_bfloat16 * x_ptr, const float * y_ptr, const int32_t * ids_ptr, float * dst_ptr,
+        const int ncols2, const int nrows, const uint3 nchannels_y, const int stride_row, const int stride_col_dst,
+        const uint3 channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
+        const uint3 sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) {
+    constexpr int warp_size = 32;
+    constexpr int nvw       = 256 / warp_size; // virtual warps of the emulated 256-thread block
+    const int lane = threadIdx.x;
+    const int row  = blockIdx.x * rows_per_block + threadIdx.y;
+    if (row >= nrows) {
+        return;
+    }
+    const int channel_dst = blockIdx.y;
+    const int channel_x   = ids_ptr ? ids_ptr[channel_dst] : (int) fastdiv((uint32_t) channel_dst, channel_ratio);
+    const int channel_y   = ids_ptr ? (int) fastmodulo(channel_dst, nchannels_y) : channel_dst;
+    const int sample_dst  = 0;
+    const int sample_x    = fastdiv((uint32_t) sample_dst, sample_ratio);
+    const int sample_y    = sample_dst;
+
+    const int   * x2  = (const int *) (x_ptr + int64_t(sample_x)*stride_sample_x + channel_x*stride_channel_x + int64_t(row)*stride_row);
+    const float2 * y2 = (const float2 *) (y_ptr + int64_t(sample_y)*stride_sample_y + channel_y*stride_channel_y);
+    float * dst = dst_ptr + int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst;
+
+    float sumf[nvw];
+#pragma unroll
+    for (int w = 0; w < nvw; ++w) {
+        sumf[w] = 0.0f;
+    }
+    for (int col0 = 0; col0 < ncols2; col0 += 256) {
+        int    tmpx[nvw];
+        float2 tmpy[nvw];
+#pragma unroll
+        for (int w = 0; w < nvw; ++w) {
+            const int col2 = col0 + w*warp_size + lane;
+            if (col2 < ncols2) {
+                tmpx[w] = x2[col2];
+                tmpy[w] = y2[col2];
+            }
+        }
+#pragma unroll
+        for (int w = 0; w < nvw; ++w) {
+            const int col2 = col0 + w*warp_size + lane;
+            if (col2 < ncols2) {
+                const float tmpx0 = ggml_cuda_cast<float>(reinterpret_cast<const nv_bfloat16 *>(&tmpx[w])[0]);
+                const float tmpx1 = ggml_cuda_cast<float>(reinterpret_cast<const nv_bfloat16 *>(&tmpx[w])[1]);
+                ggml_cuda_mad(sumf[w], tmpx0, tmpy[w].x);
+                ggml_cuda_mad(sumf[w], tmpx1, tmpy[w].y);
+            }
+        }
+    }
+#pragma unroll
+    for (int w = 0; w < nvw; ++w) {
+        // per-warp butterfly, then the two zero-lane steps (offsets 16 and 8) of the second-level reduction
+        sumf[w] = (warp_reduce_sum<warp_size>(sumf[w]) + 0.0f) + 0.0f;
+    }
+    const float value = ((sumf[0] + sumf[4]) + (sumf[2] + sumf[6])) + ((sumf[1] + sumf[5]) + (sumf[3] + sumf[7]));
+    if (lane == 0) {
+        dst[row] = value;
+    }
+    GGML_UNUSED(stride_col_dst);
+}
+#endif // defined(GGML_USE_HIP)
+
+template<typename T, typename type_acc, int ncols_dst, int block_size, bool is_multi_token_id = false, bool tokens_in_block = false>
 static void mul_mat_vec_f_switch_fusion(
         const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
         const int64_t ncols, const uint3 nchannels_y,
@@ -397,7 +498,7 @@ static void mul_mat_vec_f_switch_fusion(
     const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;
     if constexpr (ncols_dst == 1) {
         if (has_fusion) {
-            ggml_cuda_kernel_launch(mul_mat_vec_f<T, type_acc, ncols_dst, block_size, true, is_multi_token_id>, launch_params,
+            ggml_cuda_kernel_launch(mul_mat_vec_f<T, type_acc, ncols_dst, block_size, true, is_multi_token_id, tokens_in_block>, launch_params,
                 x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst,
                 channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
                 sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
@@ -407,14 +508,14 @@ static void mul_mat_vec_f_switch_fusion(
 
     GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1");
 
-    ggml_cuda_kernel_launch(mul_mat_vec_f<T, type_acc, ncols_dst, block_size, false, is_multi_token_id>, launch_params,
+    ggml_cuda_kernel_launch(mul_mat_vec_f<T, type_acc, ncols_dst, block_size, false, is_multi_token_id, tokens_in_block>, launch_params,
         x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst,
         channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
         sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);
 
 }
 
-template <typename T, typename type_acc, int ncols_dst, bool is_multi_token_id = false>
+template <typename T, typename type_acc, int ncols_dst, bool is_multi_token_id = false, bool tokens_in_block = false>
 void launch_mul_mat_vec_f_cuda(
         const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,
         const int64_t ncols, const int64_t nrows,
@@ -448,57 +549,74 @@ void launch_mul_mat_vec_f_cuda(
             block_size_best = block_size;
         }
     }
-
     const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;
 
-    const int nbytes_shared = warp_size*sizeof(float) + (has_fusion ? warp_size*sizeof(float) : 0);
-    const dim3 block_nums(nrows, nchannels_dst, nsamples_or_ntokens);
-    const dim3 block_dims(block_size_best, 1, 1);
+#if defined(GGML_USE_HIP)
+    if constexpr (std::is_same_v<T, nv_bfloat16> && ncols_dst == 1 && !is_multi_token_id && !tokens_in_block) {
+        // only where the emulated block does a single K iteration: for longer rows the 8x lower wave count loses
+        if (!has_fusion && block_size_best == 256 && ncols <= 512 && warp_size == 32 && nsamples_or_ntokens == 1 && nsamples_dst == 1 &&
+                GGML_CUDA_CC_IS_RDNA(ggml_cuda_info().devices[device].cc)) {
+            constexpr int rows_per_block = 4;
+            const dim3 block_nums((nrows + rows_per_block - 1) / rows_per_block, nchannels_dst, 1);
+            const dim3 block_dims(warp_size, rows_per_block, 1);
+            const ggml_cuda_kernel_launch_params launch_params = {block_nums, block_dims, 0, stream};
+            ggml_cuda_kernel_launch(mul_mat_vec_bf16_wave<rows_per_block>, launch_params,
+                x, y, ids, dst, (int) (ncols/2), (int) nrows, nchannels_y_fd, (int) stride_row, (int) stride_col_dst,
+                channel_ratio_fd, (int) stride_channel_x, (int) stride_channel_y, (int) stride_channel_dst,
+                sample_ratio_fd, (int) stride_sample_x, (int) stride_sample_y, (int) stride_sample_dst);
+            return;
+        }
+    }
+#endif // defined(GGML_USE_HIP)
+
+    const int nbytes_shared = (warp_size*sizeof(float) + (has_fusion ? warp_size*sizeof(float) : 0)) * (tokens_in_block ? nsamples_or_ntokens : 1);
+    const dim3 block_nums(nrows, nchannels_dst, tokens_in_block ? nsamples_dst : nsamples_or_ntokens);
+    const dim3 block_dims(block_size_best, tokens_in_block ? nsamples_or_ntokens : 1, 1);
     switch (block_size_best) {
         case   32: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 32, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 32, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
         } break;
         case   64: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 64, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 64, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
         } break;
         case   96: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 96, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 96, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
         } break;
         case  128: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 128, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 128, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
         } break;
         case  160: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 160, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 160, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
         } break;
         case  192: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 192, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 192, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
         } break;
         case  224: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 224, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 224, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
         } break;
         case  256: {
-            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 256, is_multi_token_id>
+            mul_mat_vec_f_switch_fusion<T, type_acc, ncols_dst, 256, is_multi_token_id, tokens_in_block>
                 (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst,
                  channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
                  sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream);
@@ -517,7 +635,7 @@ static void mul_mat_vec_f_cuda_switch_ncols_dst(
         const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
         const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
         const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
-        const int64_t ids_stride, cudaStream_t stream) {
+        const int64_t ids_stride, bool exact_batch, cudaStream_t stream) {
 
     const bool has_ids = ids != nullptr;
 
@@ -543,6 +661,42 @@ static void mul_mat_vec_f_cuda_switch_ncols_dst(
         return;
     }
 
+    if (exact_batch && !has_ids && ncols_dst > 1) {
+        if (ncols_dst == 2) {
+            launch_mul_mat_vec_f_cuda<T, type_acc, 2>
+                (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,
+                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,
+                 stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,
+                 nsamples_dst, ids_stride, stream);
+            return;
+        }
+        if (ncols_dst == 3) {
+            launch_mul_mat_vec_f_cuda<T, type_acc, 3>
+                (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,
+                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,
+                 stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,
+                 nsamples_dst, ids_stride, stream);
+            return;
+        }
+        if (ncols_dst == 4) {
+            launch_mul_mat_vec_f_cuda<T, type_acc, 4>
+                (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,
+                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,
+                 stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,
+                 nsamples_dst, ids_stride, stream);
+            return;
+        }
+        for (int64_t col = 0; col < ncols_dst; col += 4) {
+            const int64_t ncols_group = std::min<int64_t>(4, ncols_dst - col);
+            launch_mul_mat_vec_f_cuda<T, type_acc, 1, true, true>
+                (x, y + col*stride_col_y, ids, fusion, dst + col*stride_col_dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst,
+                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,
+                 stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst,
+                 ncols_group, ids_stride, stream);
+        }
+        return;
+    }
+
     switch (ncols_dst) {
         case 1:
             launch_mul_mat_vec_f_cuda<T, type_acc, 1>
@@ -614,21 +768,21 @@ static void mul_mat_vec_f_cuda(
         const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
         const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
         const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
-        const int64_t ids_stride, enum ggml_prec prec, cudaStream_t stream) {
+        const int64_t ids_stride, enum ggml_prec prec, bool exact_batch, cudaStream_t stream) {
 
     if constexpr(std::is_same_v<T, half>) {
         if (prec == GGML_PREC_DEFAULT) {
             mul_mat_vec_f_cuda_switch_ncols_dst<T, half>
                 (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst,
                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,
-                stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+                stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, exact_batch, stream);
             return;
         }
     }
     mul_mat_vec_f_cuda_switch_ncols_dst<T, float>
         (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst,
         nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y,
-        stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);
+        stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, exact_batch, stream);
 }
 
 void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
@@ -653,6 +807,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
 
     const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
     const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32;
+    const bool exact_batch = ggml_get_op_params_i32(dst, 1) == GGML_HINT_EXACT_BATCH;
 
     const float   * src1_d =       (const float   *) src1->data;
     const int32_t *  ids_d = ids ? (const int32_t *)  ids->data : nullptr;
@@ -709,19 +864,19 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
             const float * src0_d = (const float *) src0->data;
             mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
                 ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
-                ne03,              ne3,           s03, s13,              s3,                 ids_stride, prec, ctx.stream());
+                ne03,              ne3,           s03, s13,              s3,                 ids_stride, prec, exact_batch, ctx.stream());
         } break;
         case GGML_TYPE_F16: {
             const half * src0_d = (const half *) src0->data;
             mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
                 ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
-                ne03,              ne3,           s03, s13,              s3,                 ids_stride, prec, ctx.stream());
+                ne03,              ne3,           s03, s13,              s3,                 ids_stride, prec, exact_batch, ctx.stream());
         } break;
         case GGML_TYPE_BF16: {
             const nv_bfloat16 * src0_d = (const nv_bfloat16 *) src0->data;
             mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
                 ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
-                ne03,              ne3,           s03, s13,              s3,                 ids_stride, prec, ctx.stream());
+                ne03,              ne3,           s03, s13,              s3,                 ids_stride, prec, exact_batch, ctx.stream());
         } break;
         default:
             GGML_ABORT("unsupported type: %s", ggml_type_name(src0->type));
@@ -768,19 +923,19 @@ void ggml_cuda_op_mul_mat_vec_f(
             const float * src0_d = (const float *) src0_dd_i;
             mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst,
                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
-                nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream);
+                nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, false, stream);
         } break;
         case GGML_TYPE_F16: {
             const half * src0_d = (const half *) src0_dd_i;
             mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst,
                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
-                nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream);
+                nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, false, stream);
         } break;
         case GGML_TYPE_BF16: {
             const nv_bfloat16 * src0_d = (const nv_bfloat16 *) src0_dd_i;
             mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst,
                 nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
-                nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream);
+                nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, false, stream);
         } break;
         default:
             GGML_ABORT("unsupported type: %s", ggml_type_name(src0->type));
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index 79f7a3f6fe75..55d70f6482ca 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -1,4 +1,5 @@
 #include "mmvq.cuh"
+#include "ggml-backend-impl.h"
 #include "quantize.cuh"
 #include "unary.cuh"
 #include "vecdotq.cuh"
@@ -69,6 +70,7 @@ enum mmvq_parameter_table_id {
     MMVQ_PARAMETERS_TURING,
     MMVQ_PARAMETERS_GCN,
     MMVQ_PARAMETERS_RDNA2,
+    MMVQ_PARAMETERS_RDNA3_5,
     MMVQ_PARAMETERS_RDNA3_0,
     MMVQ_PARAMETERS_RDNA4,
     MMVQ_PARAMETERS_GB10
@@ -77,9 +79,11 @@ enum mmvq_parameter_table_id {
 static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
 #if defined(RDNA4)
     return MMVQ_PARAMETERS_RDNA4;
+#elif defined(RDNA3_5)
+    return MMVQ_PARAMETERS_RDNA3_5;
 #elif defined(RDNA3_0)
     return MMVQ_PARAMETERS_RDNA3_0;
-#elif defined(RDNA2) || defined(RDNA3_5)
+#elif defined(RDNA2)
     return MMVQ_PARAMETERS_RDNA2;
 #elif defined(GCN) || defined(CDNA)
     return MMVQ_PARAMETERS_GCN;
@@ -99,7 +103,10 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
     if (GGML_CUDA_CC_IS_RDNA3_0(cc)) {
         return MMVQ_PARAMETERS_RDNA3_0;
     }
-    if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+    if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+        return MMVQ_PARAMETERS_RDNA3_5;
+    }
+    if (GGML_CUDA_CC_IS_RDNA2(cc)) {
         return MMVQ_PARAMETERS_RDNA2;
     }
     if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {
@@ -395,6 +402,9 @@ static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {
 }
 
 static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id, bool small_k = false, bool halve_iters = false) {
+    if (table_id == MMVQ_PARAMETERS_RDNA3_5) {
+        return 1;
+    }
     if (table_id == MMVQ_PARAMETERS_GENERIC) {
         switch (ncols_dst) {
             case 1:
@@ -521,7 +531,681 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
     return 1;
 }
 
+static_assert(calc_nwarps(GGML_TYPE_Q5_1, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_Q8_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_MXFP4, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_Q4_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_Q5_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_Q6_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_Q1_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_Q2_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_Q4_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+static_assert(calc_nwarps(GGML_TYPE_IQ3_S, 1, MMVQ_PARAMETERS_RDNA3_5) == 1);
+
+static constexpr __host__ __device__ bool is_rdna3_5_q4_columns_type(ggml_type type) {
+    return type == GGML_TYPE_Q1_0 || type == GGML_TYPE_Q2_0 || type == GGML_TYPE_Q5_1 ||
+           type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K ||
+           type == GGML_TYPE_IQ3_S;
+}
+
+static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q1_0));
+static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q2_0));
+static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q5_1));
+static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q4_K));
+static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q5_K));
+static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q6_K));
+static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_IQ3_S));
+
+template <ggml_type type>
+__launch_bounds__(2 * ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_q4_columns(
+        const void * vx_ptr, const void * vy_ptr, float * dst_ptr,
+        const uint32_t ncols_x, const uint32_t stride_row_x, const uint32_t stride_col_y,
+        const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
+        const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
+        const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst) {
+    static_assert(type == GGML_TYPE_Q8_0);
+    constexpr int qk = ggml_cuda_type_traits<type>::qk;
+    constexpr int qi = ggml_cuda_type_traits<type>::qi;
+    constexpr int vdr = get_vdr_mmvq(type);
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+
+    const int row = blockIdx.x;
+    const int lane = threadIdx.x;
+    const int column0 = 0;
+    const uint32_t channel_dst = blockIdx.y;
+    const uint32_t sample_dst = blockIdx.z;
+    const uint32_t channel_x = fastdiv(channel_dst, channel_ratio);
+    const uint32_t sample_x = fastdiv(sample_dst, sample_ratio);
+
+    const void * vx = vx_ptr;
+    const block_q8_1 * y = (const block_q8_1 *) vy_ptr + sample_dst*stride_sample_y + channel_dst*stride_channel_y;
+    float * dst = dst_ptr + sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row;
+
+    const int blocks_per_row_x = ncols_x / qk;
+    const int blocks_per_iter = vdr * warp_size / qi;
+    const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row*stride_row_x;
+    float tmp[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+
+    ggml_cuda_pdl_sync();
+    for (int kbx = lane / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+        const int kby = kbx * (qk/QK8_1);
+        const int kqs = vdr * (lane % (qi/vdr));
+        const block_q8_0 * bx = (const block_q8_0 *) vx + kbx_offset + kbx;
+        int xv[VDR_Q8_0_Q8_1_MMVQ];
+#pragma unroll
+        for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
+            xv[i] = get_int_b2(bx->qs, kqs + i);
+        }
+        const float dx = __half2float(bx->d);
+#pragma unroll
+        for (int j = 0; j < 4; ++j) {
+            const block_q8_1 * by = &y[(column0 + j)*stride_col_y + kby];
+            int sumi = 0;
+#pragma unroll
+            for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {
+                sumi = ggml_cuda_dp4a(xv[i], get_int_b4(by->qs, kqs + i), sumi);
+            }
+            tmp[j] += dx * __half2float(__low2half(by->ds)) * (float) sumi;
+        }
+    }
+
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+        tmp[j] = warp_reduce_sum<warp_size>(tmp[j]);
+        if (lane == 0) {
+            dst[(column0 + j)*stride_col_dst] = tmp[j];
+        }
+    }
+}
+
+template <ggml_type type>
+__launch_bounds__(calc_nwarps(type, 1, get_device_table_id()) * ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_q4_columns_rdna3_5(
+        const void * vx_ptr, const void * vy_ptr, float * dst_ptr,
+        const uint32_t ncols_x, const uint32_t stride_row_x, const uint32_t stride_col_y,
+        const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
+        const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
+        const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst) {
+    static_assert(is_rdna3_5_q4_columns_type(type));
+    constexpr int qk        = ggml_cuda_type_traits<type>::qk;
+    constexpr int qi        = ggml_cuda_type_traits<type>::qi;
+    constexpr int vdr       = get_vdr_mmvq(type);
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+    constexpr int nwarps    = calc_nwarps(type, 1, get_device_table_id());
+
+    const int row  = blockIdx.x;
+    const int lane = threadIdx.x;
+    const int tid  = warp_size*threadIdx.y + lane;
+    const uint32_t channel_dst = blockIdx.y;
+    const uint32_t sample_dst  = blockIdx.z;
+    const uint32_t channel_x   = fastdiv(channel_dst, channel_ratio);
+    const uint32_t sample_x    = fastdiv(sample_dst, sample_ratio);
+
+    const block_q8_1 * y = (const block_q8_1 *) vy_ptr + sample_dst*stride_sample_y + channel_dst*stride_channel_y;
+    float * dst = dst_ptr + sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row;
+
+    const int blocks_per_row_x = ncols_x / qk;
+    const int blocks_per_iter  = vdr * nwarps*warp_size / qi;
+    const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row*stride_row_x;
+    float tmp[4] = {0.0f, 0.0f, 0.0f, 0.0f};
+
+    ggml_cuda_pdl_sync();
+    for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+        const int kby = kbx * (qk/QK8_1);
+        const int kqs = vdr * (tid % (qi/vdr));
+
+        if constexpr (type == GGML_TYPE_Q1_0) {
+            const block_q1_0 * bx = (const block_q1_0 *) vx_ptr + kbx_offset + kbx;
+            const int16_t * qs = (const int16_t *) bx->qs + kqs*2;
+            int xv[8];
+#pragma unroll
+            for (int i = 0; i < 2; ++i) {
+                const int q = qs[i];
+                const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0);
+                const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2);
+                const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >>  0);
+                const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >>  0);
+                const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16);
+                const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16);
+                xv[4*i+0] = __byte_perm(s0, s1, 0x5410);
+                xv[4*i+1] = __byte_perm(s0, s1, 0x7632);
+                xv[4*i+2] = __byte_perm(s2, s3, 0x5410);
+                xv[4*i+3] = __byte_perm(s2, s3, 0x7632);
+            }
+            const float dx = bx->d;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const block_q8_1 * by = &y[j*stride_col_y + kby + kqs];
+                int sumi = 0;
+#pragma unroll
+                for (int i = 0; i < 2; ++i) {
+                    sumi = ggml_cuda_dp4a(xv[4*i+0], get_int_b4(by->qs, i*4+0), sumi);
+                    sumi = ggml_cuda_dp4a(xv[4*i+1], get_int_b4(by->qs, i*4+1), sumi);
+                    sumi = ggml_cuda_dp4a(xv[4*i+2], get_int_b4(by->qs, i*4+2), sumi);
+                    sumi = ggml_cuda_dp4a(xv[4*i+3], get_int_b4(by->qs, i*4+3), sumi);
+                }
+                const float d8 = __low2float(by->ds);
+                tmp[j] += dx * d8 * sumi;
+            }
+        } else if constexpr (type == GGML_TYPE_Q2_0) {
+            const block_q2_0 * bx = (const block_q2_0 *) vx_ptr + kbx_offset + kbx;
+            const int16_t * qs = (const int16_t *) bx->qs + kqs*4;
+            int xv[4];
+            int xw[4];
+#pragma unroll
+            for (int i = 0; i < 4; ++i) {
+                const int q = qs[i];
+#if defined(GGML_USE_HIP)
+                const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);
+                const uint32_t qy_bits    = q >> 8;
+                const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);
+                xv[i] = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);
+                xw[i] = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);
+#else
+                const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);
+                const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);
+                xv[i] = __byte_perm(qe, qo, 0x5140);
+                xw[i] = __byte_perm(qe, qo, 0x7362);
+#endif // defined(GGML_USE_HIP)
+            }
+            const float dx = bx->d;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const block_q8_1 * by = &y[j*stride_col_y + kby + kqs];
+                int sumi = 0;
+#pragma unroll
+                for (int i = 0; i < 4; ++i) {
+                    sumi = ggml_cuda_dp4a(get_int_b4(by->qs, i*2+0), xv[i], sumi);
+                    sumi = ggml_cuda_dp4a(get_int_b4(by->qs, i*2+1), xw[i], sumi);
+                }
+                const float d8 = __low2float(by->ds);
+                tmp[j] += dx * d8 * sumi;
+            }
+        } else if constexpr (type == GGML_TYPE_IQ3_S) {
+            const block_iq3_s * bx = (const block_iq3_s *) vx_ptr + kbx_offset + kbx;
+            const int2 qs_packed = make_int2(get_int_b2(bx->qs, kqs + 0), get_int_b2(bx->qs, kqs + 1));
+            const uint8_t * qs = (const uint8_t *) &qs_packed;
+            const int qh = bx->qh[kqs/2];
+            const int signs_packed_32 = get_int_b2(bx->signs, kqs/2);
+            const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;
+            int2 xv[4];
+#pragma unroll
+            for (int i = 0; i < 4; ++i) {
+                const int l0 = 2*i;
+                const int2 grid_pos = make_int2(
+                    iq3s_grid[qs[l0 + 0] | ((qh << (8-l0)) & 0x100)],
+                    iq3s_grid[qs[l0 + 1] | ((qh << (7-l0)) & 0x100)]);
+                const int signs0 = __vcmpne4(((signs_packed_8[i] & 0x03) << 7) | ((signs_packed_8[i] & 0x0C) << 21), 0x00000000);
+                const int signs1 = __vcmpne4(((signs_packed_8[i] & 0x30) << 3) | ((signs_packed_8[i] & 0xC0) << 17), 0x00000000);
+                xv[i] = make_int2(__vsub4(grid_pos.x ^ signs0, signs0), __vsub4(grid_pos.y ^ signs1, signs1));
+            }
+            const int ls = 1 + 2*((bx->scales[kqs/4] >> ((kqs << 1) & 0x04)) & 0x0F);
+            const float dx = __half2float(bx->d);
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const block_q8_1 * by = &y[j*stride_col_y + kby + kqs/2];
+                int sumi = 0;
+#pragma unroll
+                for (int i = 0; i < 4; ++i) {
+                    sumi = ggml_cuda_dp4a(xv[i].x, get_int_b4(by->qs, 2*i + 0), sumi);
+                    sumi = ggml_cuda_dp4a(xv[i].y, get_int_b4(by->qs, 2*i + 1), sumi);
+                }
+                sumi *= ls;
+                const float d = dx * __low2float(by->ds);
+                tmp[j] += d * sumi;
+            }
+        } else if constexpr (type == GGML_TYPE_Q5_1) {
+            const block_q5_1 * bx = (const block_q5_1 *) vx_ptr + kbx_offset + kbx;
+            int vl[VDR_Q5_1_Q8_1_MMVQ];
+            int vh[VDR_Q5_1_Q8_1_MMVQ];
+#pragma unroll
+            for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
+                vl[i] = get_int_b4(bx->qs, kqs + i);
+                vh[i] = get_int_b4(bx->qh, 0) >> (4 * (kqs + i));
+            }
+            const half2 dm = bx->dm;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const block_q8_1 * by = &y[j*stride_col_y + kby];
+                int u[2*VDR_Q5_1_Q8_1_MMVQ];
+#pragma unroll
+                for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {
+                    u[2*i+0] = get_int_b4(by->qs, kqs + i);
+                    u[2*i+1] = get_int_b4(by->qs, kqs + i + QI5_1);
+                }
+                // ROCm 7.14 changes gfx1151 Q5_1 rounding under four-column register pressure. Keep both paths materialized until LLVM preserves this expression.
+                volatile float dot = vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, dm, by->ds);
+                tmp[j] = __fadd_rn(tmp[j], dot);
+            }
+        } else if constexpr (type == GGML_TYPE_Q4_K) {
+            const block_q4_K * bx = (const block_q4_K *) vx_ptr + kbx_offset + kbx;
+            const int bq8_offset = QR4_K * ((kqs/2) / (QI8_1/2));
+            const int * q4 = (const int *) (bx->qs + 16*bq8_offset + 4*((kqs/2)%4));
+            int xv[2] = {q4[0], q4[4]};
+            const uint16_t * scales = (const uint16_t *) bx->scales;
+            uint16_t aux[2];
+            const int is = bq8_offset/2;
+            if (is < 2) {
+                aux[0] = scales[is+0] & 0x3f3f;
+                aux[1] = scales[is+2] & 0x3f3f;
+            } else {
+                aux[0] = ((scales[is+2] >> 0) & 0x0f0f) | ((scales[is-2] & 0xc0c0) >> 2);
+                aux[1] = ((scales[is+2] >> 4) & 0x0f0f) | ((scales[is-0] & 0xc0c0) >> 2);
+            }
+            const uint8_t * sc = (const uint8_t *) aux;
+            const uint8_t * m  = sc + 2;
+            const half2 dm = bx->dm;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const block_q8_1 * by = &y[j*stride_col_y + kby];
+                int u[2*QR4_K];
+                float d8[QR4_K];
+#pragma unroll
+                for (int i = 0; i < QR4_K; ++i) {
+                    const block_q8_1 * byi = by + bq8_offset + i;
+                    d8[i] = __low2float(byi->ds);
+                    const int * q8 = (const int *) byi->qs + ((kqs/2)%4);
+                    u[2*i+0] = q8[0];
+                    u[2*i+1] = q8[4];
+                }
+                tmp[j] += vec_dot_q4_K_q8_1_impl_vmmq(xv, u, sc, m, dm, d8);
+            }
+        } else if constexpr (type == GGML_TYPE_Q5_K) {
+            const block_q5_K * bx = (const block_q5_K *) vx_ptr + kbx_offset + kbx;
+            const int bq8_offset = QR5_K * ((kqs/2) / (QI8_1/2));
+            const int * ql = (const int *) (bx->qs + 16*bq8_offset + 4*((kqs/2)%4));
+            const int * qh = (const int *) (bx->qh + 4*((kqs/2)%4));
+            int vl[2] = {ql[0], ql[4]};
+            int vh[2] = {qh[0] >> bq8_offset, qh[4] >> bq8_offset};
+            const uint16_t * scales = (const uint16_t *) bx->scales;
+            uint16_t aux[2];
+            const int is = bq8_offset/2;
+            if (is < 2) {
+                aux[0] = scales[is+0] & 0x3f3f;
+                aux[1] = scales[is+2] & 0x3f3f;
+            } else {
+                aux[0] = ((scales[is+2] >> 0) & 0x0f0f) | ((scales[is-2] & 0xc0c0) >> 2);
+                aux[1] = ((scales[is+2] >> 4) & 0x0f0f) | ((scales[is-0] & 0xc0c0) >> 2);
+            }
+            const uint8_t * sc = (const uint8_t *) aux;
+            const uint8_t * m  = sc + 2;
+            const half2 dm = bx->dm;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const block_q8_1 * by = &y[j*stride_col_y + kby];
+                int u[2*QR5_K];
+                float d8[QR5_K];
+#pragma unroll
+                for (int i = 0; i < QR5_K; ++i) {
+                    const block_q8_1 * byi = by + bq8_offset + i;
+                    d8[i] = __low2float(byi->ds);
+                    const int * q8 = (const int *) byi->qs + ((kqs/2)%4);
+                    u[2*i+0] = q8[0];
+                    u[2*i+1] = q8[4];
+                }
+                tmp[j] += vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, dm, d8);
+            }
+        } else if constexpr (type == GGML_TYPE_Q6_K) {
+            const block_q6_K * bx = (const block_q6_K *) vx_ptr + kbx_offset + kbx;
+            const int bq8_offset = 2*QR6_K*(kqs/(QI6_K/2)) + (kqs%(QI6_K/2))/(QI6_K/4);
+            const int scale_offset = (QI6_K/4)*(kqs/(QI6_K/2)) + (kqs%(QI6_K/2))/(QI6_K/8);
+            const int vh_shift = 2*((kqs%(QI6_K/2))/(QI6_K/4));
+            const int vl = get_int_b2(bx->ql, kqs);
+            const int vh = get_int_b2(bx->qh, (QI6_K/4)*(kqs/(QI6_K/2)) + kqs%(QI6_K/4)) >> vh_shift;
+            const int8_t * scales = bx->scales + scale_offset;
+            const float dx = bx->d;
+#pragma unroll
+            for (int j = 0; j < 4; ++j) {
+                const block_q8_1 * by = &y[j*stride_col_y + kby];
+                int u[QR6_K];
+                float d8[QR6_K];
+#pragma unroll
+                for (int i = 0; i < QR6_K; ++i) {
+                    u[i]  = get_int_b4(by[bq8_offset + 2*i].qs, kqs % QI8_1);
+                    d8[i] = __low2float(by[bq8_offset + 2*i].ds);
+                }
+                tmp[j] += vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, dx, d8);
+            }
+        }
+    }
+
+    __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][4][warp_size];
+    if (threadIdx.y > 0) {
+#pragma unroll
+        for (int j = 0; j < 4; ++j) {
+            tmp_shared[threadIdx.y-1][j][lane] = tmp[j];
+        }
+    }
+    __syncthreads();
+    if (threadIdx.y > 0) {
+        return;
+    }
+
+#pragma unroll
+    for (int j = 0; j < 4; ++j) {
+#pragma unroll
+        for (int i = 0; i < nwarps-1; ++i) {
+            tmp[j] += tmp_shared[i][j][lane];
+        }
+        tmp[j] = warp_reduce_sum<warp_size>(tmp[j]);
+        if (lane == 0) {
+            dst[j*stride_col_dst] = tmp[j];
+        }
+    }
+}
+
+#ifndef MMVQ_IQ3_S_ROWS_PER_BLOCK
+#define MMVQ_IQ3_S_ROWS_PER_BLOCK 1
+#endif
+
+// gfx1151 IQ3_S decode kernel. Each warp owns an independent output row, as in
+// the baseline one-warp MMVQ kernel. Grouping adjacent rows into a block keeps
+// their shared activation vector and 2 KiB IQ3 codebook hot without introducing
+// a cross-warp reduction or changing the dot-product order.
+template <bool has_gate, int rows_per_block>
+__launch_bounds__(rows_per_block * ggml_cuda_get_physical_warp_size(), 8)
+static __global__ void mul_mat_vec_iq3_s_rows_rdna3_5(
+        const void * vx_ptr, const void * vy_ptr, const int32_t * ids,
+        const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
+        const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+        const uint32_t stride_row_x, const uint32_t stride_col_dst,
+        const uint32_t stride_channel_x, const uint32_t stride_channel_y,
+        const uint32_t stride_channel_dst, const uint32_t stride_sample_x,
+        const uint32_t stride_sample_y, const uint32_t stride_sample_dst) {
+    constexpr int qk        = QK_K;
+    constexpr int qi        = ggml_cuda_type_traits<GGML_TYPE_IQ3_S>::qi;
+    constexpr int vdr       = VDR_IQ3_S_Q8_1_MMVQ;
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+
+    const int lane = threadIdx.x;
+    const int row  = rows_per_block * blockIdx.x + threadIdx.y;
+    if (row >= nrows_x) {
+        return;
+    }
+
+    const uint32_t channel_dst = blockIdx.y;
+    const uint32_t sample_dst  = blockIdx.z;
+    const uint32_t channel_x   = ids[channel_dst];
+    const uint32_t channel_y   = fastmodulo(channel_dst, nchannels_y);
+    const uint32_t sample_x    = sample_dst;
+    const uint32_t sample_y    = sample_dst;
+
+    const block_q8_1 * y = (const block_q8_1 *) vy_ptr
+        + sample_y * stride_sample_y + channel_y * stride_channel_y;
+    const int kbx_offset = sample_x * stride_sample_x
+        + channel_x * stride_channel_x + row * stride_row_x;
+    const int blocks_per_row_x = ncols_x / qk;
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+
+    float sum      = 0.0f;
+    float sum_gate = 0.0f;
+    for (int kbx = lane / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+        const int kby = kbx * (qk/QK8_1);
+        const int kqs = vdr * (lane % (qi/vdr));
+        sum += vec_dot_iq3_s_q8_1(vx_ptr, &y[kby], kbx_offset + kbx, kqs);
+        if constexpr (has_gate) {
+            sum_gate += vec_dot_iq3_s_q8_1(fusion.gate, &y[kby], kbx_offset + kbx, kqs);
+        }
+    }
+
+    sum = warp_reduce_sum<warp_size>(sum);
+    if constexpr (has_gate) {
+        sum_gate = warp_reduce_sum<warp_size>(sum_gate);
+    }
+
+    if (lane == 0) {
+        float result = sum;
+        if constexpr (has_gate) {
+            result *= ggml_cuda_op_silu_single(sum_gate);
+        }
+        dst_ptr[sample_dst * stride_sample_dst
+            + channel_dst * stride_channel_dst + row] = result;
+    }
+    GGML_UNUSED(stride_col_dst);
+}
+
+// vec_dot_iq3_s_q8_1 with the 512-entry codebook read from a caller-provided (shared memory) copy instead of
+// the __device__ table: the eight per-lane gathers per block then go to the LDS pipe instead of the
+// texture-address unit, which the direct kernel saturates (MemUnitBusy ~91% on gfx1151).
+static __device__ __forceinline__ float vec_dot_iq3_s_q8_1_grid(
+    const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs,
+    const uint32_t * __restrict__ grid) {
+
+    const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx;
+
+    const int2      qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1));
+    const uint8_t * qs        = (const uint8_t *) &qs_packed;
+
+    const int qh = bq3->qh[iqs/2];
+
+    const int       signs_packed_32 = get_int_b2(bq3->signs, iqs/2);
+    const uint8_t * signs_packed_8  = (const uint8_t *) &signs_packed_32;
+
+    int sumi = 0;
+#pragma unroll
+    for (int l0 = 0; l0 < 8; l0 += 2) {
+        const int2 grid_pos = make_int2(
+            grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)],
+            grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]);
+
+        const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);
+        const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);
+
+        const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);
+        const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);
+
+        const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+        const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+        sumi = ggml_cuda_dp4a(grid_l, u0, sumi);
+        sumi = ggml_cuda_dp4a(grid_h, u1, sumi);
+    }
+
+    sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);
+
+    const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);
+    return d * sumi;
+}
+
+// Direct kernel (same lane layout and accumulation order as mul_mat_vec_iq3_s_rows_rdna3_5) with the codebook
+// staged in shared memory once per block.
+template <bool has_gate, int rows_per_block>
+__launch_bounds__(rows_per_block * ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_iq3_s_grid_rdna3_5(
+        const void * vx_ptr, const void * vy_ptr, const int32_t * ids,
+        const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
+        const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+        const uint32_t stride_row_x, const uint32_t stride_col_dst,
+        const uint32_t stride_channel_x, const uint32_t stride_channel_y,
+        const uint32_t stride_channel_dst, const uint32_t stride_sample_x,
+        const uint32_t stride_sample_y, const uint32_t stride_sample_dst) {
+    constexpr int qk        = QK_K;
+    constexpr int qi        = ggml_cuda_type_traits<GGML_TYPE_IQ3_S>::qi;
+    constexpr int vdr       = VDR_IQ3_S_Q8_1_MMVQ;
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+
+    __shared__ uint32_t s_grid[512];
+
+    const int lane = threadIdx.x;
+    const int tid  = threadIdx.y * warp_size + lane;
+    for (int i = tid; i < 512; i += rows_per_block * warp_size) {
+        s_grid[i] = iq3s_grid[i];
+    }
+
+    const int row = rows_per_block * blockIdx.x + threadIdx.y;
+    const bool row_ok = row < nrows_x;
+
+    const uint32_t channel_dst = blockIdx.y;
+    const uint32_t sample_dst  = blockIdx.z;
+    const uint32_t channel_x   = ids[channel_dst];
+    const uint32_t channel_y   = fastmodulo(channel_dst, nchannels_y);
+    const uint32_t sample_x    = sample_dst;
+    const uint32_t sample_y    = sample_dst;
+
+    const block_q8_1 * y = (const block_q8_1 *) vy_ptr
+        + sample_y * stride_sample_y + channel_y * stride_channel_y;
+    const int kbx_offset = sample_x * stride_sample_x
+        + channel_x * stride_channel_x + (row_ok ? row : 0) * stride_row_x;
+    const int blocks_per_row_x = ncols_x / qk;
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+
+    __syncthreads();
+    if (!row_ok) {
+        return;
+    }
+
+    float sum      = 0.0f;
+    float sum_gate = 0.0f;
+    for (int kbx = lane / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+        const int kby = kbx * (qk/QK8_1);
+        const int kqs = vdr * (lane % (qi/vdr));
+        sum += vec_dot_iq3_s_q8_1_grid(vx_ptr, &y[kby], kbx_offset + kbx, kqs, s_grid);
+        if constexpr (has_gate) {
+            sum_gate += vec_dot_iq3_s_q8_1_grid(fusion.gate, &y[kby], kbx_offset + kbx, kqs, s_grid);
+        }
+    }
+
+    sum = warp_reduce_sum<warp_size>(sum);
+    if constexpr (has_gate) {
+        sum_gate = warp_reduce_sum<warp_size>(sum_gate);
+    }
+
+    if (lane == 0) {
+        float result = sum;
+        if constexpr (has_gate) {
+            result *= ggml_cuda_op_silu_single(sum_gate);
+        }
+        dst_ptr[sample_dst * stride_sample_dst
+            + channel_dst * stride_channel_dst + row] = result;
+    }
+    GGML_UNUSED(stride_col_dst);
+}
+
+// gfx1151 IQ3_S decode kernel with LDS staging, ncols == 2560 (10 blocks, 1100 B per row). The direct
+// per-lane field loads of vec_dot_iq3_s_q8_1 (about 18 vector-memory instructions per block per lane, most of
+// them for the Q8_1 activations) saturate the texture-address unit (MemUnitBusy ~91%) long before DRAM
+// bandwidth. Here each wave first copies its up (and gate) row into shared memory with 9 dword loads per
+// lane, the block copies the quantized activations once, and the dot products then read from LDS.
+// The per-lane accumulation and reduction order is the same as mul_mat_vec_iq3_s_rows_rdna3_5.
+#define MMVQ_IQ3_LDS_NCOLS      2560
+#define MMVQ_IQ3_LDS_ROW_DWORDS 275   // 10 blocks * 110 B / 4
+#define MMVQ_IQ3_LDS_ROW_SLOT   276
+#define MMVQ_IQ3_LDS_Y_DWORDS   720   // 80 Q8_1 blocks * 36 B / 4
+
+template <bool has_gate, int rows_per_block>
+__launch_bounds__(rows_per_block * ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_iq3_s_lds_rdna3_5(
+        const void * vx_ptr, const void * vy_ptr, const int32_t * ids,
+        const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
+        const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+        const uint32_t stride_row_x, const uint32_t stride_col_dst,
+        const uint32_t stride_channel_x, const uint32_t stride_channel_y,
+        const uint32_t stride_channel_dst, const uint32_t stride_sample_x,
+        const uint32_t stride_sample_y, const uint32_t stride_sample_dst) {
+    constexpr int qk        = QK_K;
+    constexpr int qi        = ggml_cuda_type_traits<GGML_TYPE_IQ3_S>::qi;
+    constexpr int vdr       = VDR_IQ3_S_Q8_1_MMVQ;
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+    constexpr int blocks_per_row  = MMVQ_IQ3_LDS_NCOLS / qk;
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+    constexpr int row_loads       = (MMVQ_IQ3_LDS_ROW_DWORDS + warp_size - 1) / warp_size; // 9
+
+    __shared__ int s_y[MMVQ_IQ3_LDS_Y_DWORDS];
+    __shared__ int s_rows[rows_per_block][has_gate ? 2 : 1][MMVQ_IQ3_LDS_ROW_SLOT];
+
+    const int lane = threadIdx.x;
+    const int wave = threadIdx.y;
+    const int tid  = wave * warp_size + lane;
+    const int row  = rows_per_block * blockIdx.x + wave;
+    const bool row_ok = row < nrows_x;
+
+    const uint32_t channel_dst = blockIdx.y;
+    const uint32_t sample_dst  = blockIdx.z;
+    const uint32_t channel_x   = ids[channel_dst];
+    const uint32_t channel_y   = fastmodulo(channel_dst, nchannels_y);
+
+    const block_q8_1 * y = (const block_q8_1 *) vy_ptr + sample_dst * stride_sample_y + channel_y * stride_channel_y;
+    const int kbx_offset = sample_dst * stride_sample_x + channel_x * stride_channel_x + (row_ok ? row : 0) * stride_row_x;
+
+    // 1. weight rows -> registers (dword loads; rows are 4 B aligned: 1100 B rows, even block offsets)
+    const int * gx = (const int *) ((const block_iq3_s *) vx_ptr + kbx_offset);
+    int rx[row_loads];
+    int rg[row_loads];
+#pragma unroll
+    for (int i = 0; i < row_loads; ++i) {
+        const int idx = i * warp_size + lane;
+        rx[i] = idx < MMVQ_IQ3_LDS_ROW_DWORDS ? gx[idx] : 0;
+    }
+    if constexpr (has_gate) {
+        const int * gg = (const int *) ((const block_iq3_s *) fusion.gate + kbx_offset);
+#pragma unroll
+        for (int i = 0; i < row_loads; ++i) {
+            const int idx = i * warp_size + lane;
+            rg[i] = idx < MMVQ_IQ3_LDS_ROW_DWORDS ? gg[idx] : 0;
+        }
+    }
+
+    // 2. quantized activations -> LDS (shared by the rows of the block)
+    {
+        const int * gy = (const int *) y;
+        for (int i = tid; i < MMVQ_IQ3_LDS_Y_DWORDS; i += rows_per_block * warp_size) {
+            s_y[i] = gy[i];
+        }
+    }
+
+    // 3. rows -> LDS
+#pragma unroll
+    for (int i = 0; i < row_loads; ++i) {
+        const int idx = i * warp_size + lane;
+        if (idx < MMVQ_IQ3_LDS_ROW_DWORDS) {
+            s_rows[wave][0][idx] = rx[i];
+            if constexpr (has_gate) {
+                s_rows[wave][1][idx] = rg[i];
+            }
+        }
+    }
+    __syncthreads();
+
+    if (!row_ok) {
+        return;
+    }
+
+    // 4. dot products from LDS, same lane -> block assignment as the direct kernel
+    const block_q8_1 * y_l = (const block_q8_1 *) s_y;
+    const void *       x_l = s_rows[wave][0];
+    float sum      = 0.0f;
+    float sum_gate = 0.0f;
+#pragma unroll
+    for (int kbx = lane / (qi/vdr); kbx < blocks_per_row; kbx += blocks_per_iter) {
+        const int kby = kbx * (qk/QK8_1);
+        const int kqs = vdr * (lane % (qi/vdr));
+        sum += vec_dot_iq3_s_q8_1(x_l, &y_l[kby], kbx, kqs);
+        if constexpr (has_gate) {
+            sum_gate += vec_dot_iq3_s_q8_1(s_rows[wave][1], &y_l[kby], kbx, kqs);
+        }
+    }
+
+    sum = warp_reduce_sum<warp_size>(sum);
+    if constexpr (has_gate) {
+        sum_gate = warp_reduce_sum<warp_size>(sum_gate);
+    }
+
+    if (lane == 0) {
+        float result = sum;
+        if constexpr (has_gate) {
+            result *= ggml_cuda_op_silu_single(sum_gate);
+        }
+        dst_ptr[sample_dst * stride_sample_dst + channel_dst * stride_channel_dst + row] = result;
+    }
+    GGML_UNUSED(stride_col_dst);
+    GGML_UNUSED(ncols_x);
+}
+
 static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {
+    if (table_id == MMVQ_PARAMETERS_RDNA3_5) {
+        return 1;
+    }
     if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) {
         switch (ncols_dst) {
             case 1:
@@ -667,12 +1351,36 @@ static __global__ void mul_mat_vec_q(
         for (int j = 0; j < ncols_dst; ++j) {
 #pragma unroll
             for (int i = 0; i < rows_per_cuda_block; ++i) {
-                tmp[j][i] += vec_dot_q_cuda(
-                    vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+                if constexpr (type == GGML_TYPE_IQ3_S && has_fusion) {
+                    if (use_gate) {
+                        float dot;
+                        float dot_gate;
+                        vec_dot_iq3_s_q8_1_pair(
+                            dot, dot_gate, vx, vgate, &y[j*stride_col_y + kby],
+                            kbx_offset + i*stride_row_x + kbx, kqs);
+                        tmp[j][i]      += dot;
+                        tmp_gate[j][i] += dot_gate;
+                        continue;
+                    }
+                }
+                if constexpr (type == GGML_TYPE_Q5_1 && table_id == MMVQ_PARAMETERS_RDNA3_5) {
+                    volatile float dot = vec_dot_q_cuda(
+                        vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+                    tmp[j][i] = __fadd_rn(tmp[j][i], dot);
+                } else {
+                    tmp[j][i] += vec_dot_q_cuda(
+                        vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+                }
                 if constexpr (has_fusion) {
                     if (use_gate) {
-                        tmp_gate[j][i] += vec_dot_q_cuda(
-                            vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+                        if constexpr (type == GGML_TYPE_Q5_1 && table_id == MMVQ_PARAMETERS_RDNA3_5) {
+                            volatile float dot_gate = vec_dot_q_cuda(
+                                vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+                            tmp_gate[j][i] = __fadd_rn(tmp_gate[j][i], dot_gate);
+                        } else {
+                            tmp_gate[j][i] += vec_dot_q_cuda(
+                                vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);
+                        }
                     }
                 }
             }
@@ -769,40 +1477,793 @@ static __global__ void mul_mat_vec_q(
     }
 }
 
-// Dedicated MoE multi-token kernel.
-// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)
-// Block: (warp_size, ncols_dst) - each warp handles one token independently.
-// No shared memory reduction needed since each warp works alone.
-template <ggml_type type, int c_rows_per_block>
-__launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)
-static __global__ void mul_mat_vec_q_moe(
-        const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr,
-        float * dst_ptr,
-        const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
-        const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,
-        const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
-        const uint32_t ncols_dst, const uint32_t ids_stride) {
-    const void    * GGML_CUDA_RESTRICT vx  = vx_ptr;
-    const void    * GGML_CUDA_RESTRICT vy  = vy_ptr;
-    const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
-    float         * GGML_CUDA_RESTRICT dst = dst_ptr;
-
-    constexpr int qk  = ggml_cuda_type_traits<type>::qk;
-    constexpr int qi  = ggml_cuda_type_traits<type>::qi;
-    constexpr int vdr = get_vdr_mmvq(type);
-    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
-
-    constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
-
-    const uint32_t token_idx   = threadIdx.y;
-    const int      row0        = c_rows_per_block*blockIdx.x;
-    const int      blocks_per_row_x = ncols_x / qk;
-    constexpr int  blocks_per_iter  = vdr * warp_size / qi;
-
-    const uint32_t channel_dst = blockIdx.y;
+// ---------------------------------------------------------------------------------------------
+// RDNA3.5 single-column MMVQ with the Q8_1 activation quantization fused into the matvec kernel.
+//
+// Token generation on gfx1151 pays a ~2 us launch gap for every kernel, so the quantize_q8_1 launch in front
+// of each matvec costs about as much as the whole quantize kernel. A block of MMVQ_FQ_NWARPS waves (one output
+// row per wave) quantizes the activation vector once into shared memory, replicating quantize_q8_1 exactly
+// (same amax, same IEEE division and roundf), and then runs the per-wave K loop of mul_mat_vec_q<type, 1> in
+// the same order against it, so the outputs are bit-identical to the two-kernel path.
+//
+// Measured on gfx1151 with Qwen3.6-35B-A3B Q8_0 decode (2026-09-03): 16 waves per block and 2 prefetched
+// K iterations; 8 waves or deeper prefetch (more VGPRs, less occupancy) were slower.
+
+#define MMVQ_FQ_NWARPS 16
+// whether the fused-quantize path also serves the IQ3_S/IQ4_NL/IQ4_XS expert matvecs (see mmvq_fq_type_ok)
+#ifndef MMVQ_FQ_IQ_TYPES
+#define MMVQ_FQ_IQ_TYPES false
+#endif
 
-    if (token_idx >= ncols_dst) {
-        return;
+static constexpr __host__ __device__ bool mmvq_fq_type_ok(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_Q8_0:
+        case GGML_TYPE_Q6_K:
+            // Q4_K/Q5_K (block sum via mmvq_fq_block_sum) are exact too but measured neutral on gfx1151
+            // with the generic, non-prefetched loop (Qwen3.6 UD-Q4_K_XL TG128 60.69 -> 60.63 t/s).
+            return true;
+        case GGML_TYPE_IQ3_S:
+        case GGML_TYPE_IQ4_NL:
+        case GGML_TYPE_IQ4_XS:
+            // MoE expert types of the Unsloth qwen4exp IQ4_XS mix: their vec_dot only reads the block scale, so
+            // the in-kernel quantization is exact; used to drop the quantize launch in front of every expert matvec
+            return MMVQ_FQ_IQ_TYPES;
+        default:
+            return false;
+    }
+}
+
+// Whether the vec_dot of the type reads the Q8_1 block sum (ds.y) in addition to the scale.
+static constexpr __host__ __device__ bool mmvq_fq_needs_sum(ggml_type type) {
+    switch (type) {
+        case GGML_TYPE_Q8_0:
+        case GGML_TYPE_Q6_K:
+        case GGML_TYPE_IQ3_S:
+        case GGML_TYPE_IQ4_NL:
+        case GGML_TYPE_IQ4_XS:
+            return false;
+        default:
+            return true;
+    }
+}
+
+// Sum of the 32 values of a Q8_1 block in exactly the order of warp_reduce_sum<32> in quantize_q8_1, where lane
+// l holds value l. Here lane (sub) of a group of 4 holds values 8*sub .. 8*sub+7 in v[].
+static __device__ __forceinline__ float mmvq_fq_block_sum(const float * v, const int width) {
+    float a[8];
+#pragma unroll
+    for (int k = 0; k < 8; ++k) {
+        // offset 16: a_i = x_i + x_{i^16}, x_{i^16} lives two lanes over
+        a[k] = v[k] + __shfl_xor_sync(0xffffffff, v[k], 2, width);
+    }
+    float b[8];
+#pragma unroll
+    for (int k = 0; k < 8; ++k) {
+        // offset 8: b_i = a_i + a_{i^8}, one lane over
+        b[k] = a[k] + __shfl_xor_sync(0xffffffff, a[k], 1, width);
+    }
+    float c[4];
+#pragma unroll
+    for (int k = 0; k < 4; ++k) {
+        c[k] = b[k] + b[k + 4]; // offset 4
+    }
+    const float d0 = c[0] + c[2]; // offset 2
+    const float d1 = c[1] + c[3];
+    return d0 + d1;               // offset 1
+}
+
+// Register-staged Q8_0 weight loads: the K loop of one wave is split into chunks of MMVQ_FQ_PF iterations whose
+// weight blocks are loaded up front, so the DRAM stream starts before the activation quantization and the
+// barrier, and each wave keeps MMVQ_FQ_PF loads in flight instead of one.
+#define MMVQ_FQ_PF 2
+// Long rows with few output rows (e.g. the [10240 x 320] hyper-connection down projections of qwen4exp) have
+// too few waves in flight to cover DRAM latency with 2 loads per wave; they use a deeper prefetch instead.
+// The accumulation order per lane is the same for any depth, so the results do not change.
+#define MMVQ_FQ_PF_LONG 8
+
+template <int pf>
+struct mmvq_fq_q8_0_chunk {
+    int  qs[pf][VDR_Q8_0_Q8_1_MMVQ];
+    half d[pf];
+};
+
+template <int pf>
+static __device__ __forceinline__ void mmvq_fq_q8_0_load(
+        mmvq_fq_q8_0_chunk<pf> & c, const block_q8_0 * GGML_CUDA_RESTRICT x, const int kbx0, const int kqs, const int blocks_per_row_x) {
+    constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * ggml_cuda_get_physical_warp_size() / QI8_0;
+#pragma unroll
+    for (int i = 0; i < pf; ++i) {
+        const int kbx = kbx0 + i*blocks_per_iter;
+        if (kbx < blocks_per_row_x) {
+#pragma unroll
+            for (int j = 0; j < VDR_Q8_0_Q8_1_MMVQ; ++j) {
+                c.qs[i][j] = get_int_b2(x[kbx].qs, kqs + j);
+            }
+            c.d[i] = x[kbx].d;
+        }
+    }
+}
+
+template <int pf>
+static __device__ __forceinline__ void mmvq_fq_q8_0_dot(
+        float & tmp, const mmvq_fq_q8_0_chunk<pf> & c, const block_q8_1 * y, const int kbx0, const int kqs, const int blocks_per_row_x) {
+    constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * ggml_cuda_get_physical_warp_size() / QI8_0;
+#pragma unroll
+    for (int i = 0; i < pf; ++i) {
+        const int kbx = kbx0 + i*blocks_per_iter;
+        if (kbx < blocks_per_row_x) {
+            const block_q8_1 * bq8_1 = &y[kbx]; // qk == QK8_1 for Q8_0
+            int u[VDR_Q8_0_Q8_1_MMVQ];
+#pragma unroll
+            for (int j = 0; j < VDR_Q8_0_Q8_1_MMVQ; ++j) {
+                u[j] = get_int_b4(bq8_1->qs, kqs + j);
+            }
+            tmp += vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMVQ>(c.qs[i], u, c.d[i], __low2half(bq8_1->ds));
+        }
+    }
+}
+
+template <ggml_type type, bool has_fusion, int nwarps, int pf>
+__launch_bounds__(nwarps * ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_q_fq(
+        const void * vx_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
+        const uint32_t ncols_x, const uint32_t nrows_x, const uint3 nchannels_y, const uint32_t stride_row_x,
+        const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,
+        const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,
+        const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,
+        const float y_scale, const float y_bias, const int y_op) {
+    static_assert(mmvq_fq_type_ok(type));
+    constexpr bool q8_0_path = type == GGML_TYPE_Q8_0;
+    const void    * GGML_CUDA_RESTRICT vx  = vx_ptr;
+    const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
+    float         * GGML_CUDA_RESTRICT dst = dst_ptr;
+
+    constexpr int qk  = ggml_cuda_type_traits<type>::qk;
+    constexpr int qi  = ggml_cuda_type_traits<type>::qi;
+    constexpr int vdr = get_vdr_mmvq(type);
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+    static_assert(!q8_0_path || qk == QK8_1);
+    [[maybe_unused]] constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
+
+    extern __shared__ char mmvq_fq_smem[];
+    block_q8_1 * y_q8 = (block_q8_1 *) mmvq_fq_smem;
+
+    const int lane = threadIdx.x;
+    const int tid  = warp_size*threadIdx.y + lane;
+
+    const uint32_t channel_dst = blockIdx.y;
+    const uint32_t sample_dst  = blockIdx.z;
+    const uint32_t channel_x   = ids ? ids[channel_dst]                     : fastdiv(channel_dst, channel_ratio);
+    const uint32_t channel_y   = ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst;
+    const uint32_t sample_x    = fastdiv(sample_dst, sample_ratio);
+    const uint32_t sample_y    = sample_dst;
+
+    const uint32_t row      = blockIdx.x*nwarps + threadIdx.y;
+    const bool     row_ok   = row < nrows_x;
+    const uint32_t row_safe = row_ok ? row : nrows_x - 1;
+
+    bool use_gate = false;
+    bool use_bias = false;
+    bool use_gate_bias = false;
+    [[maybe_unused]] const void * vgate = nullptr;
+    const float * x_bias = nullptr;
+    const float * gate_bias = nullptr;
+    ggml_glu_op active_glu;
+    float glu_limit = 0.0f;
+
+    if constexpr (has_fusion) {
+        use_gate      = fusion.gate      != nullptr;
+        use_bias      = fusion.x_bias    != nullptr;
+        use_gate_bias = fusion.gate_bias != nullptr && use_gate;
+        vgate         = fusion.gate;
+        x_bias        = (const float *) fusion.x_bias;
+        gate_bias     = (const float *) fusion.gate_bias;
+        active_glu    = fusion.glu_op;
+        glu_limit     = fusion.glu_limit;
+    }
+
+    // 1. start streaming the weights of this wave's row before anything else
+    const int blocks_per_row_x = ncols_x / qk;
+    const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row_safe*stride_row_x;
+    const int kqs  = vdr * (lane % (qi/vdr));
+    const int kbx0 = lane / (qi/vdr);
+
+    [[maybe_unused]] const block_q8_0 * x  = (const block_q8_0 *) vx + kbx_offset;
+    [[maybe_unused]] const block_q8_0 * xg = nullptr;
+
+    [[maybe_unused]] mmvq_fq_q8_0_chunk<pf> cx;
+    [[maybe_unused]] mmvq_fq_q8_0_chunk<pf> cg;
+    if constexpr (q8_0_path) {
+        mmvq_fq_q8_0_load(cx, x, kbx0, kqs, blocks_per_row_x);
+        if constexpr (has_fusion) {
+            if (use_gate) {
+                xg = (const block_q8_0 *) vgate + kbx_offset;
+                mmvq_fq_q8_0_load(cg, xg, kbx0, kqs, blocks_per_row_x);
+            }
+        }
+    }
+
+    // 2. quantize the activation vector into shared memory: 4 lanes per Q8_1 block, 8 values per lane
+    {
+        const float * y = y_ptr + sample_y*stride_sample_y + channel_y*stride_channel_y;
+        const int nby = ncols_x / QK8_1;
+        for (int i = tid; i < 4*nby; i += nwarps*warp_size) {
+            const int ib  = i >> 2;
+            const int sub = i & 3;
+            const float4 * yv = (const float4 *) (y + ib*QK8_1 + sub*8);
+            const float4 v0 = yv[0];
+            const float4 v1 = yv[1];
+            float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w};
+            if (y_op == 3) {
+                // qwen4exp GDN output: per-head (128) rms norm * weight, gated by sigmoid(z). The 16 lanes of a
+                // head are consecutive threads of one wave (4 Q8_1 blocks x 4 lanes).
+                float ss = 0.0f;
+#pragma unroll
+                for (int k = 0; k < 8; ++k) {
+                    ss = fmaf(v[k], v[k], ss);
+                }
+                ss += __shfl_xor_sync(0xffffffff, ss, 1, warp_size);
+                ss += __shfl_xor_sync(0xffffffff, ss, 2, warp_size);
+                ss += __shfl_xor_sync(0xffffffff, ss, 4, warp_size);
+                ss += __shfl_xor_sync(0xffffffff, ss, 8, warp_size);
+                const float rms_scale = rsqrtf(ss / 128.0f + fusion.y_eps);
+                const float * z = (const float *) fusion.y_gate + ib*QK8_1 + sub*8;
+                const float * w = (const float *) fusion.y_norm_w + (ib*QK8_1 + sub*8) % 128;
+                const float4 * zv = (const float4 *) z;
+                const float4 * wv = (const float4 *) w;
+                const float4 z0 = zv[0], z1 = zv[1], w0 = wv[0], w1 = wv[1];
+                const float zz[8] = {z0.x, z0.y, z0.z, z0.w, z1.x, z1.y, z1.z, z1.w};
+                const float ww[8] = {w0.x, w0.y, w0.z, w0.w, w1.x, w1.y, w1.z, w1.w};
+#pragma unroll
+                for (int k = 0; k < 8; ++k) {
+                    // rms_norm_f32<.., true>: scale * x * w ; unary_gated: sigmoid(z) * normed
+                    v[k] = (1.0f / (1.0f + expf(-zz[k]))) * (rms_scale * v[k] * ww[k]);
+                }
+            } else if (y_op != 0) {
+                // fused activation prologue (scale -> unary), same expressions as scale_f32 / op_silu / op_sigmoid
+#pragma unroll
+                for (int k = 0; k < 8; ++k) {
+                    const float t = y_scale * v[k] + y_bias;
+                    v[k] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t));
+                }
+            }
+
+            float amax = fabsf(v[0]);
+#pragma unroll
+            for (int k = 1; k < 8; ++k) {
+                amax = fmaxf(amax, fabsf(v[k]));
+            }
+            amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 1, warp_size));
+            amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 2, warp_size));
+
+            const float sum = mmvq_fq_needs_sum(type) ? mmvq_fq_block_sum(v, warp_size) : 0.0f;
+
+            const float d = amax / 127.0f;
+            int q[8];
+#pragma unroll
+            for (int k = 0; k < 8; ++k) {
+                const int8_t qk8 = amax == 0.0f ? 0 : roundf(v[k] / d);
+                q[k] = (int) qk8 & 0xff;
+            }
+            int * qs = (int *) y_q8[ib].qs;
+            qs[2*sub + 0] = q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24);
+            qs[2*sub + 1] = q[4] | (q[5] << 8) | (q[6] << 16) | (q[7] << 24);
+            if (sub == 0) {
+                y_q8[ib].ds = make_half2(d, sum);
+            }
+        }
+    }
+    __syncthreads();
+
+    if (!row_ok) {
+        return;
+    }
+
+    [[maybe_unused]] float x_biases    = 0.0f;
+    [[maybe_unused]] float gate_biases = 0.0f;
+    if constexpr (has_fusion) {
+        const uint32_t channel_bias = ids ? channel_x : channel_dst;
+        if (lane == 0) {
+            if (use_bias) {
+                x_biases = x_bias[sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row];
+            }
+            if (use_gate_bias) {
+                gate_biases = gate_bias[sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row];
+            }
+        }
+    }
+
+    // 3. exact per-wave dot products in the k order of mul_mat_vec_q<type, 1>, one chunk ahead in memory
+    float tmp      = 0.0f;
+    float tmp_gate = 0.0f;
+
+    const block_q8_1 * y = y_q8;
+    if constexpr (!q8_0_path) {
+        // generic types: the per-wave loop of mul_mat_vec_q<type, 1> against the shared-memory activations
+        for (int kbx = kbx0; kbx < blocks_per_row_x; kbx += blocks_per_iter) {
+            const int kby = kbx * (qk/QK8_1);
+            tmp += vec_dot_q_cuda(vx, &y[kby], kbx_offset + kbx, kqs);
+            if constexpr (has_fusion) {
+                if (use_gate) {
+                    tmp_gate += vec_dot_q_cuda(vgate, &y[kby], kbx_offset + kbx, kqs);
+                }
+            }
+        }
+    }
+    constexpr int chunk_blocks = pf*blocks_per_iter;
+    for (int c0 = kbx0; q8_0_path && c0 < blocks_per_row_x; c0 += chunk_blocks) {
+        mmvq_fq_q8_0_chunk<pf> nx;
+        [[maybe_unused]] mmvq_fq_q8_0_chunk<pf> ng;
+        const int c1 = c0 + chunk_blocks;
+        if (c1 < blocks_per_row_x) {
+            mmvq_fq_q8_0_load(nx, x, c1, kqs, blocks_per_row_x);
+            if constexpr (has_fusion) {
+                if (use_gate) {
+                    mmvq_fq_q8_0_load(ng, xg, c1, kqs, blocks_per_row_x);
+                }
+            }
+        }
+        mmvq_fq_q8_0_dot(tmp, cx, y, c0, kqs, blocks_per_row_x);
+        if constexpr (has_fusion) {
+            if (use_gate) {
+                mmvq_fq_q8_0_dot(tmp_gate, cg, y, c0, kqs, blocks_per_row_x);
+            }
+        }
+        if (c1 < blocks_per_row_x) {
+            cx = nx;
+            if constexpr (has_fusion) {
+                if (use_gate) {
+                    cg = ng;
+                }
+            }
+        }
+    }
+
+    tmp = warp_reduce_sum<warp_size>(tmp);
+    if constexpr (has_fusion) {
+        if (use_gate) {
+            tmp_gate = warp_reduce_sum<warp_size>(tmp_gate);
+        }
+    }
+
+    if (lane == 0) {
+        float result = tmp;
+        if constexpr (has_fusion) {
+            result += x_biases;
+            if (use_gate) {
+                float gate_value = tmp_gate;
+                gate_value += gate_biases;
+                switch (active_glu) {
+                    case GGML_GLU_OP_SWIGLU:
+                        result *= ggml_cuda_op_silu_single(gate_value);
+                        break;
+                    case GGML_GLU_OP_GEGLU:
+                        result *= ggml_cuda_op_gelu_single(gate_value);
+                        break;
+                    case GGML_GLU_OP_SWIGLU_OAI:
+                        result = ggml_cuda_op_swiglu_oai_single(gate_value, result);
+                        break;
+                    case GGML_GLU_OP_SWIGLU_CLAMP:
+                        result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);
+                        break;
+                    default:
+                        result = result * gate_value;
+                        break;
+                }
+            }
+        }
+        dst[sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row] = result;
+    }
+
+    if constexpr (!has_fusion) {
+        GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, glu_limit, gate_bias, x_bias, tmp_gate);
+    }
+    GGML_UNUSED(stride_col_dst);
+}
+
+template <ggml_type type>
+static void mul_mat_vec_q_fq_launch(
+        const void * vx, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device & fusion, float * dst,
+        const int ncols_x, const int nrows_x, const int stride_row_x, const int stride_col_dst,
+        const int nchannels_x, const int nchannels_y, const int nchannels_dst,
+        const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
+        const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
+        cudaStream_t stream, const float y_scale, const float y_bias, const int y_op) {
+    const int device    = ggml_cuda_get_device();
+    const int warp_size = ggml_cuda_info().devices[device].warp_size;
+
+    const uint3 nchannels_y_fd   = ids ? init_fastdiv_values(nchannels_y) : make_uint3(0, 0, 0);
+    const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0)              : init_fastdiv_values(nchannels_dst / nchannels_x);
+    const uint3 sample_ratio_fd  = init_fastdiv_values(nsamples_dst / nsamples_x);
+
+    // One output row per wave. Narrow matrices (e.g. the 320-row hyper-connection down projections of
+    // qwen4exp) would give only nrows/16 blocks, leaving most CUs idle while each wave streams a long row.
+    // Use fewer waves per block there so at least ~2 blocks per CU are in flight; every row is still reduced
+    // by a single wave and the activation quantization is replicated per block, so results are unchanged.
+    const int nsm = ggml_cuda_info().devices[device].nsm;
+    int nwarps = MMVQ_FQ_NWARPS;
+    while (nwarps > 4 && ((nrows_x + nwarps - 1) / nwarps) * nchannels_dst * nsamples_dst < 2 * nsm) {
+        nwarps /= 2;
+    }
+
+    const dim3 block_nums((nrows_x + nwarps - 1) / nwarps, nchannels_dst, nsamples_dst);
+    const dim3 block_dims(warp_size, nwarps, 1);
+    const int  nbytes_shared = (ncols_x / QK8_1) * sizeof(block_q8_1);
+
+    const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;
+    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);
+
+    auto launch = [&](auto kernel) {
+        ggml_cuda_kernel_launch(kernel, launch_params,
+            vx, y, ids, fusion, dst, ncols_x, nrows_x, nchannels_y_fd, stride_row_x, stride_col_dst,
+            channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+            sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, y_scale, y_bias, y_op);
+    };
+
+    // deeper weight prefetch for long rows when the grid is too small to hide DRAM latency with many waves
+    constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * 32 / QI8_0;
+    const bool long_rows = type == GGML_TYPE_Q8_0 && (ncols_x / QK8_0) >= 16 * blocks_per_iter &&
+        (int64_t) nrows_x * nchannels_dst * nsamples_dst <= 1024;
+
+    switch (nwarps) {
+        case 16:
+            if (has_fusion) { launch(mul_mat_vec_q_fq<type, true,  16, MMVQ_FQ_PF>); }
+            else            { launch(mul_mat_vec_q_fq<type, false, 16, MMVQ_FQ_PF>); }
+            break;
+        case 8:
+            if (long_rows) {
+                if (has_fusion) { launch(mul_mat_vec_q_fq<type, true,  8, MMVQ_FQ_PF_LONG>); }
+                else            { launch(mul_mat_vec_q_fq<type, false, 8, MMVQ_FQ_PF_LONG>); }
+            } else {
+                if (has_fusion) { launch(mul_mat_vec_q_fq<type, true,  8, MMVQ_FQ_PF>); }
+                else            { launch(mul_mat_vec_q_fq<type, false, 8, MMVQ_FQ_PF>); }
+            }
+            break;
+        case 4:
+            if (long_rows) {
+                if (has_fusion) { launch(mul_mat_vec_q_fq<type, true,  4, MMVQ_FQ_PF_LONG>); }
+                else            { launch(mul_mat_vec_q_fq<type, false, 4, MMVQ_FQ_PF_LONG>); }
+            } else {
+                if (has_fusion) { launch(mul_mat_vec_q_fq<type, true,  4, MMVQ_FQ_PF>); }
+                else            { launch(mul_mat_vec_q_fq<type, false, 4, MMVQ_FQ_PF>); }
+            }
+            break;
+        default:
+            GGML_ABORT("fatal error");
+    }
+}
+
+static void mul_mat_vec_q_fq_switch_type(
+        const void * vx, const ggml_type type_x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device & fusion, float * dst,
+        const int ncols_x, const int nrows_x, const int stride_row_x, const int stride_col_dst,
+        const int nchannels_x, const int nchannels_y, const int nchannels_dst,
+        const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
+        const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,
+        cudaStream_t stream, const float y_scale, const float y_bias, const int y_op) {
+#define MMVQ_FQ_LAUNCH(T) \
+        mul_mat_vec_q_fq_launch<T>(vx, y, ids, fusion, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, \
+            nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, \
+            nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, y_scale, y_bias, y_op)
+    switch (type_x) {
+        case GGML_TYPE_Q8_0:   MMVQ_FQ_LAUNCH(GGML_TYPE_Q8_0);   break;
+        case GGML_TYPE_Q6_K:   MMVQ_FQ_LAUNCH(GGML_TYPE_Q6_K);   break;
+#if MMVQ_FQ_IQ_TYPES
+        case GGML_TYPE_IQ3_S:  MMVQ_FQ_LAUNCH(GGML_TYPE_IQ3_S);  break;
+        case GGML_TYPE_IQ4_NL: MMVQ_FQ_LAUNCH(GGML_TYPE_IQ4_NL); break;
+        case GGML_TYPE_IQ4_XS: MMVQ_FQ_LAUNCH(GGML_TYPE_IQ4_XS); break;
+#endif
+        default:
+            GGML_ABORT("fatal error");
+            break;
+    }
+#undef MMVQ_FQ_LAUNCH
+}
+
+// ---------------------------------------------------------------------------------------------
+// RDNA3.5 grouped single-column matvec: several independent matvecs that share the same activation vector
+// Q8_0 matvecs are launched as one kernel. Each segment is a plain [ncols x nrows] weight matrix with an optional
+// Q8_0 gate and GLU epilogue. Blocks are assigned to segments by a prefix over the segment block counts.
+
+#define MMVQ_GROUP_MAX 4
+
+struct mmvq_group_seg_dev {
+    const void * vx;
+    const void * gate;
+    float      * dst;
+    int          nrows;
+    int          stride_row;      // in Q8_0 blocks
+    int          gate_stride_row; // in blocks
+    int          blocks_begin;
+    int          glu_op;
+    float        glu_limit;
+};
+
+struct mmvq_group_args_dev {
+    mmvq_group_seg_dev seg[MMVQ_GROUP_MAX];
+    int nseg;
+};
+
+template <int nwarps, int pf>
+__launch_bounds__(nwarps * ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_fq_group(
+        const mmvq_group_args_dev args, const float * GGML_CUDA_RESTRICT y_ptr, const int ncols_x,
+        const float y_scale, const float y_bias, const int y_op) {
+    constexpr int qk  = QK8_0;
+    constexpr int qi  = QI8_0;
+    constexpr int vdr = VDR_Q8_0_Q8_1_MMVQ;
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+
+    extern __shared__ char mmvq_fq_smem[];
+    block_q8_1 * y_q8 = (block_q8_1 *) mmvq_fq_smem;
+
+    const int lane = threadIdx.x;
+    const int tid  = warp_size*threadIdx.y + lane;
+
+    // segment lookup, uniform per block
+    int s = 0;
+#pragma unroll
+    for (int k = 1; k < MMVQ_GROUP_MAX; ++k) {
+        if (k < args.nseg && (int) blockIdx.x >= args.seg[k].blocks_begin) {
+            s = k;
+        }
+    }
+    const mmvq_group_seg_dev seg = args.seg[s];
+
+    const int  row    = ((int) blockIdx.x - seg.blocks_begin)*nwarps + threadIdx.y;
+    const bool row_ok = row < seg.nrows;
+
+    // Q8_0 path: identical to mul_mat_vec_q_fq<GGML_TYPE_Q8_0, ...>
+    const int  row_safe = row_ok ? row : seg.nrows - 1;
+    const bool use_gate = seg.gate != nullptr;
+
+    const int blocks_per_row_x = ncols_x / qk;
+    const int kqs  = vdr * (lane % (qi/vdr));
+    const int kbx0 = lane / (qi/vdr);
+
+    const block_q8_0 * x  = (const block_q8_0 *) seg.vx + (size_t) row_safe*seg.stride_row;
+    const block_q8_0 * xg = use_gate ? (const block_q8_0 *) seg.gate + (size_t) row_safe*seg.gate_stride_row : nullptr;
+
+    mmvq_fq_q8_0_chunk<pf> cx;
+    mmvq_fq_q8_0_chunk<pf> cg;
+    mmvq_fq_q8_0_load(cx, x, kbx0, kqs, blocks_per_row_x);
+    if (use_gate) {
+        mmvq_fq_q8_0_load(cg, xg, kbx0, kqs, blocks_per_row_x);
+    }
+
+    {
+        const float * y = y_ptr;
+        const int nby = ncols_x / QK8_1;
+        for (int i = tid; i < 4*nby; i += nwarps*warp_size) {
+            const int ib  = i >> 2;
+            const int sub = i & 3;
+            const float4 * yv = (const float4 *) (y + ib*QK8_1 + sub*8);
+            const float4 v0 = yv[0];
+            const float4 v1 = yv[1];
+            float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w};
+            if (y_op != 0) {
+#pragma unroll
+                for (int k = 0; k < 8; ++k) {
+                    const float t = y_scale * v[k] + y_bias;
+                    v[k] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t));
+                }
+            }
+
+            float amax = fabsf(v[0]);
+#pragma unroll
+            for (int k = 1; k < 8; ++k) {
+                amax = fmaxf(amax, fabsf(v[k]));
+            }
+            amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 1, warp_size));
+            amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 2, warp_size));
+
+            const float d = amax / 127.0f;
+            int q[8];
+#pragma unroll
+            for (int k = 0; k < 8; ++k) {
+                const int8_t qk8 = amax == 0.0f ? 0 : roundf(v[k] / d);
+                q[k] = (int) qk8 & 0xff;
+            }
+            int * qs = (int *) y_q8[ib].qs;
+            qs[2*sub + 0] = q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24);
+            qs[2*sub + 1] = q[4] | (q[5] << 8) | (q[6] << 16) | (q[7] << 24);
+            if (sub == 0) {
+                y_q8[ib].ds = make_half2(d, 0.0f);
+            }
+        }
+    }
+    __syncthreads();
+
+    if (!row_ok) {
+        return;
+    }
+
+    float tmp      = 0.0f;
+    float tmp_gate = 0.0f;
+
+    const block_q8_1 * y = y_q8;
+    constexpr int chunk_blocks = pf*blocks_per_iter;
+    for (int c0 = kbx0; c0 < blocks_per_row_x; c0 += chunk_blocks) {
+        mmvq_fq_q8_0_chunk<pf> nx;
+        mmvq_fq_q8_0_chunk<pf> ng;
+        const int c1 = c0 + chunk_blocks;
+        if (c1 < blocks_per_row_x) {
+            mmvq_fq_q8_0_load(nx, x, c1, kqs, blocks_per_row_x);
+            if (use_gate) {
+                mmvq_fq_q8_0_load(ng, xg, c1, kqs, blocks_per_row_x);
+            }
+        }
+        mmvq_fq_q8_0_dot(tmp, cx, y, c0, kqs, blocks_per_row_x);
+        if (use_gate) {
+            mmvq_fq_q8_0_dot(tmp_gate, cg, y, c0, kqs, blocks_per_row_x);
+        }
+        if (c1 < blocks_per_row_x) {
+            cx = nx;
+            if (use_gate) {
+                cg = ng;
+            }
+        }
+    }
+
+    tmp = warp_reduce_sum<warp_size>(tmp);
+    if (use_gate) {
+        tmp_gate = warp_reduce_sum<warp_size>(tmp_gate);
+    }
+
+    if (lane == 0) {
+        float result = tmp;
+        if (use_gate) {
+            switch (seg.glu_op) {
+                case GGML_GLU_OP_SWIGLU:
+                    result *= ggml_cuda_op_silu_single(tmp_gate);
+                    break;
+                case GGML_GLU_OP_GEGLU:
+                    result *= ggml_cuda_op_gelu_single(tmp_gate);
+                    break;
+                case GGML_GLU_OP_SWIGLU_OAI:
+                    result = ggml_cuda_op_swiglu_oai_single(tmp_gate, result);
+                    break;
+                case GGML_GLU_OP_SWIGLU_CLAMP:
+                    result = ggml_cuda_op_swiglu_clamp_single(tmp_gate, result, seg.glu_limit);
+                    break;
+                default:
+                    result = result * tmp_gate;
+                    break;
+            }
+        }
+        seg.dst[row] = result;
+    }
+}
+
+bool ggml_cuda_mmv_group_seg_ok(const ggml_tensor * w, const ggml_tensor * y) {
+    const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+    if (!GGML_CUDA_CC_IS_RDNA3_5(cc)) {
+        return false;
+    }
+    if (y->type != GGML_TYPE_F32 || y->ne[1] != 1 || y->ne[2] != 1 || y->ne[3] != 1 || !ggml_is_contiguous(y) ||
+            (uintptr_t) y->data % 16 != 0 || y->ne[0] != w->ne[0]) {
+        return false;
+    }
+    if (w->ne[2] != 1 || w->ne[3] != 1 || w->nb[0] != ggml_type_size(w->type) || w->ne[1] > INT32_MAX ||
+            (w->buffer && ggml_backend_buffer_get_usage(w->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE)) {
+        return false;
+    }
+    const int64_t ncols = w->ne[0];
+    if (w->type == GGML_TYPE_Q8_0) {
+        return ncols % QK8_1 == 0 && (ncols / QK8_1) * sizeof(block_q8_1) <= 16384 && w->nb[1] % sizeof(block_q8_0) == 0;
+    }
+    return false;
+}
+
+void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, const ggml_cuda_mmv_group_seg * segs, const int nseg) {
+    GGML_ASSERT(nseg >= 1 && nseg <= MMVQ_GROUP_MAX);
+    const int device    = ggml_cuda_get_device();
+    const int warp_size = ggml_cuda_info().devices[device].warp_size;
+    const int nsm       = ggml_cuda_info().devices[device].nsm;
+    GGML_ASSERT(warp_size == 32);
+
+    const int ncols = y->ne[0];
+
+    int64_t total_q8_rows = 0;
+    int64_t max_rows      = 0;
+    for (int i = 0; i < nseg; ++i) {
+        GGML_ASSERT(ggml_cuda_mmv_group_seg_ok(segs[i].w, y));
+        GGML_ASSERT(segs[i].dst->type == GGML_TYPE_F32 && ggml_is_contiguous(segs[i].dst) && ggml_nelements(segs[i].dst) == segs[i].w->ne[1]);
+        if (segs[i].gate) {
+            GGML_ASSERT(segs[i].w->type == GGML_TYPE_Q8_0 && segs[i].gate->type == GGML_TYPE_Q8_0 &&
+                ggml_are_same_shape(segs[i].w, segs[i].gate) && segs[i].gate->nb[0] == sizeof(block_q8_0) &&
+                segs[i].gate->nb[1] % sizeof(block_q8_0) == 0);
+        }
+        total_q8_rows += segs[i].w->ne[1];
+        max_rows = std::max<int64_t>(max_rows, segs[i].w->ne[1]);
+    }
+
+    // same block-size heuristic as mul_mat_vec_q_fq_launch, applied to the largest segment
+    int nwarps = MMVQ_FQ_NWARPS;
+    while (nwarps > 4 && ((max_rows + nwarps - 1) / nwarps) < 2 * nsm) {
+        nwarps /= 2;
+    }
+
+    mmvq_group_args_dev args{};
+    args.nseg = nseg;
+    int nblocks = 0;
+    for (int i = 0; i < nseg; ++i) {
+        mmvq_group_seg_dev & d = args.seg[i];
+        const ggml_tensor * w = segs[i].w;
+        d.vx           = w->data;
+        d.gate         = segs[i].gate ? segs[i].gate->data : nullptr;
+        d.dst          = (float *) segs[i].dst->data;
+        d.nrows        = (int) w->ne[1];
+        d.stride_row   = (int) (w->nb[1] / ggml_type_size(w->type));
+        d.gate_stride_row = segs[i].gate ? (int) (segs[i].gate->nb[1] / sizeof(block_q8_0)) : 0;
+        d.blocks_begin = nblocks;
+        d.glu_op       = (int) segs[i].glu_op;
+        d.glu_limit    = segs[i].glu_limit;
+        nblocks += (d.nrows + nwarps - 1) / nwarps;
+    }
+
+    const dim3 block_nums(nblocks, 1, 1);
+    const dim3 block_dims(warp_size, nwarps, 1);
+    const int  nbytes_shared = total_q8_rows > 0 ? (ncols / QK8_1) * sizeof(block_q8_1) : 0;
+    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, ctx.stream());
+
+    constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * 32 / QI8_0;
+    const bool long_rows = (ncols / QK8_0) >= 16 * blocks_per_iter && total_q8_rows <= 1024;
+
+    auto launch = [&](auto kernel) {
+        ggml_cuda_kernel_launch(kernel, launch_params, args, (const float *) y->data, ncols, 1.0f, 0.0f, 0);
+    };
+
+    switch (nwarps) {
+        case 16: launch(mul_mat_vec_fq_group<16, MMVQ_FQ_PF>); break;
+        case 8:
+            if (long_rows) { launch(mul_mat_vec_fq_group<8, MMVQ_FQ_PF_LONG>); }
+            else           { launch(mul_mat_vec_fq_group<8, MMVQ_FQ_PF>); }
+            break;
+        case 4:
+            if (long_rows) { launch(mul_mat_vec_fq_group<4, MMVQ_FQ_PF_LONG>); }
+            else           { launch(mul_mat_vec_fq_group<4, MMVQ_FQ_PF>); }
+            break;
+        default:
+            GGML_ABORT("fatal error");
+    }
+}
+
+// Dedicated MoE multi-token kernel.
+// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)
+// Block: (warp_size, ncols_dst) - each warp handles one token independently.
+// No shared memory reduction needed since each warp works alone.
+template <ggml_type type, int c_rows_per_block>
+__launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)
+static __global__ void mul_mat_vec_q_moe(
+        const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr,
+        float * dst_ptr,
+        const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,
+        const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,
+        const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,
+        const uint32_t ncols_dst, const uint32_t ids_stride) {
+    const void    * GGML_CUDA_RESTRICT vx  = vx_ptr;
+    const void    * GGML_CUDA_RESTRICT vy  = vy_ptr;
+    const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;
+    float         * GGML_CUDA_RESTRICT dst = dst_ptr;
+
+    constexpr int qk  = ggml_cuda_type_traits<type>::qk;
+    constexpr int qi  = ggml_cuda_type_traits<type>::qi;
+    constexpr int vdr = get_vdr_mmvq(type);
+    constexpr int warp_size = ggml_cuda_get_physical_warp_size();
+
+    constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);
+
+    const uint32_t token_idx   = threadIdx.y;
+    const int      row0        = c_rows_per_block*blockIdx.x;
+    const int      blocks_per_row_x = ncols_x / qk;
+    constexpr int  blocks_per_iter  = vdr * warp_size / qi;
+
+    const uint32_t channel_dst = blockIdx.y;
+
+    if (token_idx >= ncols_dst) {
+        return;
     }
 
     ggml_cuda_pdl_sync();
@@ -1010,6 +2471,53 @@ static void mul_mat_vec_q_switch_ncols_dst(
             // static, else MSVC lambda capture breaks the constexpr uses below
             static constexpr int c_ncols_dst = 1;
 
+            if constexpr (type == GGML_TYPE_IQ3_S) {
+                constexpr int c_rows = MMVQ_IQ3_S_ROWS_PER_BLOCK;
+                const bool no_extra_fusion = fusion.x_bias == nullptr && fusion.gate_bias == nullptr &&
+                    fusion.x_scale == nullptr && fusion.gate_scale == nullptr;
+                const bool gate_ok = fusion.gate == nullptr || fusion.glu_op == GGML_GLU_OP_SWIGLU;
+                if (table_id == MMVQ_PARAMETERS_RDNA3_5 && ids != nullptr && ncols_x == 2560 &&
+                        no_extra_fusion && gate_ok && nsamples_dst == 1) {
+                    static const int rows_env = getenv("GGML_IQ3_ROWS") ? atoi(getenv("GGML_IQ3_ROWS")) : c_rows;
+                    auto launch_rows = [&](auto kernel, int rows) {
+                        const dim3 block_nums((nrows_x + rows - 1) / rows, nchannels_dst, nsamples_dst);
+                        const dim3 block_dims(warp_size, rows, 1);
+                        const ggml_cuda_kernel_launch_params params(block_nums, block_dims, 0, stream);
+                        ggml_cuda_kernel_launch(kernel, params,
+                            vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, nrows_x,
+                            stride_row_x, stride_col_dst, stride_channel_x, stride_channel_y,
+                            stride_channel_dst, stride_sample_x, stride_sample_y, stride_sample_dst);
+                    };
+                    static const int lds_env  = getenv("GGML_IQ3_LDS")  ? atoi(getenv("GGML_IQ3_LDS"))  : 0;
+                    static const int grid_env = getenv("GGML_IQ3_GRID") ? atoi(getenv("GGML_IQ3_GRID")) : 4;
+                    if (fusion.gate != nullptr && grid_env > 0) {
+                        switch (grid_env) {
+                            case 1: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5<true, 1>, 1); break;
+                            case 2: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5<true, 2>, 2); break;
+                            case 8: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5<true, 8>, 8); break;
+                            default: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5<true, 4>, 4); break;
+                        }
+                    } else if (fusion.gate != nullptr && lds_env > 0) {
+                        switch (lds_env) {
+                            case 1: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5<true, 1>, 1); break;
+                            case 2: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5<true, 2>, 2); break;
+                            case 8: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5<true, 8>, 8); break;
+                            default: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5<true, 4>, 4); break;
+                        }
+                    } else if (fusion.gate != nullptr) {
+                        switch (rows_env) {
+                            case 2: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5<true, 2>, 2); break;
+                            case 4: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5<true, 4>, 4); break;
+                            case 8: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5<true, 8>, 8); break;
+                            default: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5<true, 1>, 1); break;
+                        }
+                    } else {
+                        launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5<false, c_rows>, c_rows);
+                    }
+                    break;
+                }
+            }
+
             // Tag types keep the flags compile-time, so __launch_bounds__ matches what is launched.
             const auto launch = [&](auto small_k_tag, auto halve_iters_tag) {
                 constexpr bool c_small_k = decltype(small_k_tag)::value;
@@ -1054,6 +2562,35 @@ static void mul_mat_vec_q_switch_ncols_dst(
                  dims.first, dims.second, 0, ids_stride, stream);
         } break;
         case 4: {
+            if constexpr (is_rdna3_5_q4_columns_type(type)) {
+                const bool no_fusion = fusion.gate == nullptr && fusion.x_bias == nullptr && fusion.gate_bias == nullptr &&
+                    fusion.x_scale == nullptr && fusion.gate_scale == nullptr;
+                if (table_id == MMVQ_PARAMETERS_RDNA3_5 && ids == nullptr && no_fusion) {
+                    constexpr int c_nwarps = calc_nwarps(type, 1, MMVQ_PARAMETERS_RDNA3_5);
+                    const dim3 block_nums(nrows_x, nchannels_dst, nsamples_dst);
+                    const dim3 block_dims(warp_size, c_nwarps, 1);
+                    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
+                    ggml_cuda_kernel_launch(mul_mat_vec_q4_columns_rdna3_5<type>, launch_params,
+                        vx, vy, dst, ncols_x, stride_row_x, stride_col_y, stride_col_dst,
+                        channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+                        sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst);
+                    break;
+                }
+            }
+            if constexpr (type == GGML_TYPE_Q8_0) {
+                const bool no_fusion = fusion.gate == nullptr && fusion.x_bias == nullptr && fusion.gate_bias == nullptr &&
+                    fusion.x_scale == nullptr && fusion.gate_scale == nullptr;
+                if (table_id == MMVQ_PARAMETERS_RDNA3_5 && ids == nullptr && no_fusion) {
+                    const dim3 block_nums(nrows_x, nchannels_dst, nsamples_dst);
+                    const dim3 block_dims(warp_size, 1, 1);
+                    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);
+                    ggml_cuda_kernel_launch(mul_mat_vec_q4_columns<type>, launch_params,
+                        vx, vy, dst, ncols_x, stride_row_x, stride_col_y, stride_col_dst,
+                        channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,
+                        sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst);
+                    break;
+                }
+            }
             constexpr int c_ncols_dst = 4;
             std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);
             mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,
@@ -1251,6 +2788,285 @@ static void mul_mat_vec_q_switch_type(
     }
 }
 
+// RDNA3.5 token-generation fast path: quantize the activations inside the matvec kernel. Optionally applies an
+// elementwise prologue (scale -> silu/sigmoid) to the activations first, so that the two small kernels in front
+// of the matvec disappear as well. Returns false when the shape/type is not served by this path.
+static bool mul_mat_vec_q_fq_try(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
+        const ggml_cuda_mm_fusion_args_host * fusion, const float y_scale, const float y_bias, const int y_op, const bool launch) {
+    GGML_TENSOR_BINARY_OP_LOCALS;
+
+    const size_t ts_src0 = ggml_type_size(src0->type);
+    const size_t ts_src1 = ggml_type_size(src1->type);
+    const size_t ts_dst  = ggml_type_size(dst->type);
+
+    if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || nb00 != ts_src0 || nb10 != ts_src1 || nb0 != ts_dst) {
+        return false;
+    }
+
+    const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+    const int64_t ncols_dst_fq = ids ? ne2 : ne1;
+    const bool aligned = (uintptr_t) src1->data % 16 == 0 && nb11 % 16 == 0 && nb12 % 16 == 0 && nb13 % 16 == 0;
+    const bool fusion_ok = !fusion ||
+        (fusion->x_scale == nullptr && fusion->gate_scale == nullptr &&
+         (!ids || (fusion->x_bias == nullptr && fusion->gate_bias == nullptr)));
+    if (!(GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols_dst_fq == 1 && mmvq_fq_type_ok(src0->type) && aligned && fusion_ok &&
+            ne10 % QK8_1 == 0 && (ne10 / QK8_1) * sizeof(block_q8_1) <= 16384)) {
+        return false;
+    }
+    if (!launch) {
+        return true;
+    }
+
+    cudaStream_t stream = ctx.stream();
+
+    const float   * src1_d =       (const float   *) src1->data;
+    const int32_t *  ids_d = ids ? (const int32_t *)  ids->data : nullptr;
+    float         *  dst_d =       (float         *)  dst->data;
+
+    ggml_cuda_mm_fusion_args_device fusion_local{};
+    if (fusion) {
+        GGML_ASSERT(!ids || (fusion->x_bias == nullptr && fusion->gate_bias == nullptr));
+        if (fusion->x_bias) {
+            fusion_local.x_bias = fusion->x_bias->data;
+        }
+        if (fusion->gate) {
+            fusion_local.gate = fusion->gate->data;
+        }
+        if (fusion->gate_bias) {
+            fusion_local.gate_bias = fusion->gate_bias->data;
+        }
+        fusion_local.glu_op    = fusion->glu_op;
+        fusion_local.glu_limit = fusion->glu_limit;
+        fusion_local.y_gate    = fusion->y_gate   ? fusion->y_gate->data   : nullptr;
+        fusion_local.y_norm_w  = fusion->y_norm_w ? fusion->y_norm_w->data : nullptr;
+        fusion_local.y_eps     = fusion->y_eps;
+    }
+
+    // If src0 is a temporary compute buffer, clear any potential padding.
+    if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
+        const size_t size_data  = ggml_nbytes(src0);
+        const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);
+        if (size_alloc > size_data) {
+            GGML_ASSERT(ggml_is_contiguously_allocated(src0));
+            GGML_ASSERT(!src0->view_src);
+            CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));
+        }
+    }
+
+    const int64_t s01 = src0->nb[1] / ts_src0;
+    const int64_t s02 = src0->nb[2] / ts_src0;
+    const int64_t s03 = src0->nb[3] / ts_src0;
+    const int64_t s11 = src1->nb[1] / ts_src1;
+    const int64_t s12 = src1->nb[2] / ts_src1;
+    const int64_t s13 = src1->nb[3] / ts_src1;
+    const int64_t s1  = dst->nb[1] / ts_dst;
+    const int64_t s2  = dst->nb[2] / ts_dst;
+    const int64_t s3  = dst->nb[3] / ts_dst;
+
+    // For MUL_MAT_ID the memory layout is different than for MUL_MAT:
+    const int64_t nchannels_y        = ids ? ne11 : ne12;
+    const int64_t nchannels_dst      = ids ? ne1  : ne2;
+    const int64_t stride_col_dst     = ids ? s2   : s1;
+    const int64_t stride_channel_dst = ids ? s1   : s2;
+    const int64_t stride_channel_y   = ids ? s11  : s12;
+
+    mul_mat_vec_q_fq_switch_type(
+        src0->data, src0->type, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, s01, stride_col_dst,
+        ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
+        ne03, ne3, s03, s13, s3, stream, y_scale, y_bias, y_op);
+    return true;
+}
+
+bool ggml_cuda_mul_mat_vec_q_fq_prologue_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * y) {
+    if (dst->op != GGML_OP_MUL_MAT || !ggml_is_quantized(dst->src[0]->type) || y->type != GGML_TYPE_F32 ||
+        !ggml_is_contiguous(y) || !ggml_are_same_shape(y, dst->src[1])) {
+        return false;
+    }
+    return mul_mat_vec_q_fq_try(ctx, dst->src[0], y, nullptr, const_cast<ggml_tensor *>(dst), nullptr, 1.0f, 0.0f, 0, /*launch=*/false);
+}
+
+void ggml_cuda_mul_mat_vec_q_fq_prologue(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_tensor * y,
+        const float y_scale, const float y_bias, const int y_op) {
+    const bool ok = mul_mat_vec_q_fq_try(ctx, dst->src[0], y, nullptr, dst, nullptr, y_scale, y_bias, y_op, /*launch=*/true);
+    GGML_ASSERT(ok);
+}
+
+bool ggml_cuda_mul_mat_vec_q_fq_gdn_gate_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * z) {
+    const ggml_tensor * y = dst->src[1];
+    if (dst->op != GGML_OP_MUL_MAT || !ggml_is_quantized(dst->src[0]->type) || y->type != GGML_TYPE_F32 ||
+            !ggml_is_contiguous(y) || ggml_nrows(y) != 1 || y->ne[0] % 128 != 0 ||
+            z->type != GGML_TYPE_F32 || !ggml_is_contiguous(z) || ggml_nelements(z) != y->ne[0] || (uintptr_t) z->data % 16 != 0) {
+        return false;
+    }
+    // every block reads z while the waves write dst
+    const char * d0 = (const char *) dst->data;
+    const char * d1 = d0 + ggml_backend_buft_get_alloc_size(dst->buffer->buft, dst);
+    const char * z0 = (const char *) z->data;
+    const char * z1 = z0 + ggml_backend_buft_get_alloc_size(z->buffer->buft, z);
+    if (d0 < z1 && z0 < d1) {
+        return false;
+    }
+    return mul_mat_vec_q_fq_try(ctx, dst->src[0], y, nullptr, const_cast<ggml_tensor *>(dst), nullptr, 1.0f, 0.0f, 0, /*launch=*/false);
+}
+
+void ggml_cuda_mul_mat_vec_q_fq_gdn_gate(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const float * attn,
+        const ggml_tensor * z, const ggml_tensor * norm_w, const float eps) {
+    GGML_ASSERT((uintptr_t) attn % 16 == 0 && (uintptr_t) norm_w->data % 16 == 0 && ggml_nelements(norm_w) == 128);
+    ggml_tensor y = *dst->src[1]; // same shape/strides as the (elided) gated norm output, data = pre-norm scratch
+    y.data = (void *) attn;
+    ggml_cuda_mm_fusion_args_host fusion{};
+    fusion.y_gate   = z;
+    fusion.y_norm_w = norm_w;
+    fusion.y_eps    = eps;
+    const bool ok = mul_mat_vec_q_fq_try(ctx, dst->src[0], &y, nullptr, dst, &fusion, 1.0f, 0.0f, 3, /*launch=*/true);
+    GGML_ASSERT(ok);
+}
+
+#if defined(__HIP_PLATFORM_AMD__)
+static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) {
+    float result;
+    asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b));
+    return result;
+}
+
+static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) {
+    float result;
+    asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b));
+    return result;
+}
+#else
+static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) { return __fmul_rn(a, b); }
+static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) { return __fadd_rn(a, b); }
+#endif
+
+template <int n_expert_used>
+__launch_bounds__(32, 8)
+static __global__ void mul_mat_id_iq4_nl_weighted_rdna3_5(
+        const void * vx_ptr, const block_q8_1 * y, const int32_t * ids, const float * weights, float * dst,
+        const int nrows, const int blocks_per_row, const int stride_row_x, const int stride_channel_x,
+        const int stride_y) {
+    constexpr int qi        = ggml_cuda_type_traits<GGML_TYPE_IQ4_NL>::qi;
+    constexpr int vdr       = VDR_Q4_0_Q8_1_MMVQ;
+    constexpr int warp_size = 32;
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+
+    const int lane = threadIdx.x;
+    const int row  = blockIdx.x;
+    if (row >= nrows) {
+        return;
+    }
+
+    float result = 0.0f;
+#pragma unroll
+    for (int ex = 0; ex < n_expert_used; ++ex) {
+        const int channel = ids[ex];
+        const int x_off = channel * stride_channel_x + row * stride_row_x;
+        float sum = 0.0f;
+        for (int kbx = lane / (qi/vdr); kbx < blocks_per_row; kbx += blocks_per_iter) {
+            const int kqs = vdr * (lane % (qi/vdr));
+            sum += vec_dot_iq4_nl_q8_1(vx_ptr, y + ex * stride_y + kbx, x_off + kbx, kqs);
+        }
+        sum = warp_reduce_sum<warp_size>(sum);
+        const float term = mmvq_hc_mul_rn(sum, weights[ex]);
+        result = ex == 0 ? term : mmvq_hc_add_rn(result, term);
+    }
+
+    if (lane == 0) {
+        dst[row] = result;
+    }
+}
+
+template <int n_expert_used>
+__launch_bounds__(32, 8)
+static __global__ void mul_mat_id_q8_0_weighted_rdna3_5(
+        const void * vx_ptr, const block_q8_1 * y, const int32_t * ids, const float * weights, float * dst,
+        const int nrows, const int blocks_per_row, const int stride_row_x, const int stride_channel_x,
+        const int stride_y) {
+    constexpr int qi        = QI8_0;
+    constexpr int vdr       = VDR_Q8_0_Q8_1_MMVQ;
+    constexpr int warp_size = 32;
+    constexpr int blocks_per_iter = vdr * warp_size / qi;
+
+    const int lane = threadIdx.x;
+    const int row  = blockIdx.x;
+    if (row >= nrows) {
+        return;
+    }
+
+    float result = 0.0f;
+#pragma unroll
+    for (int ex = 0; ex < n_expert_used; ++ex) {
+        const int channel = ids[ex];
+        const int x_off = channel * stride_channel_x + row * stride_row_x;
+        float sum = 0.0f;
+        for (int kbx = lane / (qi/vdr); kbx < blocks_per_row; kbx += blocks_per_iter) {
+            const int kqs = vdr * (lane % (qi/vdr));
+            sum += vec_dot_q8_0_q8_1(vx_ptr, y + ex * stride_y + kbx, x_off + kbx, kqs);
+        }
+        sum = warp_reduce_sum<warp_size>(sum);
+        const float term = mmvq_hc_mul_rn(sum, weights[ex]);
+        result = ex == 0 ? term : mmvq_hc_add_rn(result, term);
+    }
+
+    if (lane == 0) {
+        dst[row] = result;
+    }
+}
+
+bool ggml_cuda_mul_mat_id_weighted_rdna3_5_ok(
+        const ggml_tensor * experts, const ggml_tensor * weights, const ggml_tensor * dst) {
+    if (experts->op != GGML_OP_MUL_MAT_ID ||
+            (experts->src[0]->type != GGML_TYPE_IQ4_NL && experts->src[0]->type != GGML_TYPE_Q8_0) ||
+            experts->src[1]->type != GGML_TYPE_F32 || experts->src[2]->type != GGML_TYPE_I32 ||
+            experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
+        return false;
+    }
+    const ggml_tensor * w   = experts->src[0];
+    const ggml_tensor * y   = experts->src[1];
+    const ggml_tensor * ids = experts->src[2];
+    const int64_t n_used = ids->ne[0];
+    return GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[ggml_cuda_get_device()].cc) &&
+        n_used == 10 && w->ne[0] == 640 && w->ne[1] == 2560 && w->ne[2] == 512 && w->ne[3] == 1 &&
+        y->ne[0] == 640 && ggml_nelements(y) == 640 * n_used && ggml_is_contiguous(y) &&
+        ggml_nelements(ids) == n_used && ggml_is_contiguous(ids) &&
+        weights->ne[0] == 1 && weights->ne[1] == n_used && ggml_nelements(weights) == n_used && ggml_is_contiguous(weights) &&
+        ggml_nelements(experts) == 2560 * n_used && ggml_is_contiguous(experts) &&
+        ggml_nelements(dst) == 2560 && ggml_is_contiguous(dst);
+}
+
+void ggml_cuda_mul_mat_id_weighted_rdna3_5(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * experts, const ggml_tensor * weights, ggml_tensor * dst) {
+    GGML_ASSERT(ggml_cuda_mul_mat_id_weighted_rdna3_5_ok(experts, weights, dst));
+    const ggml_tensor * w   = experts->src[0];
+    const ggml_tensor * y   = experts->src[1];
+    const ggml_tensor * ids = experts->src[2];
+    constexpr int n_used = 10;
+    constexpr int ncols  = 640;
+    constexpr int nrows  = 2560;
+    constexpr int nblocks = ncols / QK8_1;
+
+    ggml_cuda_pool_alloc<block_q8_1> y_q8(ctx.pool(), n_used * nblocks);
+    quantize_row_q8_1_cuda(
+        (const float *) y->data, nullptr, y_q8.get(), w->type,
+        ncols,
+        y->nb[1] / sizeof(float), y->nb[2] / sizeof(float), y->nb[3] / sizeof(float),
+        y->ne[0], y->ne[1], y->ne[2], y->ne[3], ctx.stream());
+
+    const ggml_cuda_kernel_launch_params params(nrows, 32, 0, ctx.stream());
+    if (w->type == GGML_TYPE_IQ4_NL) {
+        ggml_cuda_kernel_launch(mul_mat_id_iq4_nl_weighted_rdna3_5<n_used>, params,
+            w->data, y_q8.get(), (const int32_t *) ids->data, (const float *) weights->data, (float *) dst->data,
+            nrows, nblocks, (int) (w->nb[1] / ggml_type_size(w->type)),
+            (int) (w->nb[2] / ggml_type_size(w->type)), nblocks);
+    } else {
+        ggml_cuda_kernel_launch(mul_mat_id_q8_0_weighted_rdna3_5<n_used>, params,
+            w->data, y_q8.get(), (const int32_t *) ids->data, (const float *) weights->data, (float *) dst->data,
+            nrows, nblocks, (int) (w->nb[1] / ggml_type_size(w->type)),
+            (int) (w->nb[2] / ggml_type_size(w->type)), nblocks);
+    }
+}
+
 void ggml_cuda_mul_mat_vec_q(
         ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,
         const ggml_cuda_mm_fusion_args_host * fusion) {
@@ -1329,6 +3145,10 @@ void ggml_cuda_mul_mat_vec_q(
         }
     }
 
+    if (mul_mat_vec_q_fq_try(ctx, src0, src1, ids, dst, fusion, 1.0f, 0.0f, 0, /*launch=*/true)) {
+        return;
+    }
+
     const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);
     ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);
     {
diff --git a/ggml/src/ggml-cuda/mmvq.cuh b/ggml/src/ggml-cuda/mmvq.cuh
index 5605bf7a4e60..c2d1da92573d 100644
--- a/ggml/src/ggml-cuda/mmvq.cuh
+++ b/ggml/src/ggml-cuda/mmvq.cuh
@@ -16,3 +16,33 @@ void ggml_cuda_op_mul_mat_vec_q(
     const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
     const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
     const int64_t src1_padded_row_size, cudaStream_t stream);
+
+// RDNA3.5 fused-quantize matvec with an activation prologue: y' = op(y_scale*y + y_bias) (op 1 = silu, 2 = sigmoid)
+// applied while the activations are quantized in-kernel. `y` replaces dst->src[1] (same shape, F32, contiguous).
+bool ggml_cuda_mul_mat_vec_q_fq_prologue_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * y);
+void ggml_cuda_mul_mat_vec_q_fq_prologue(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_tensor * y,
+        float y_scale, float y_bias, int y_op);
+
+// RDNA3.5 grouped matvec: independent single-column matvecs sharing the same F32 activation vector `y`
+// (Q8_0 rows via the fused-quantize path, optionally with a Q8_0 gate + GLU epilogue) in one launch.
+#define GGML_CUDA_MMV_GROUP_MAX 4
+struct ggml_cuda_mmv_group_seg {
+    const ggml_tensor * w    = nullptr;   // [ncols, nrows] Q8_0 or F32
+    const ggml_tensor * gate = nullptr;   // optional Q8_0 gate weights of the same shape (GLU fusion)
+    ggml_tensor *       dst  = nullptr;   // F32 [nrows]; the GLU output when gate != nullptr
+    ggml_glu_op         glu_op = GGML_GLU_OP_SWIGLU;
+    float               glu_limit = 0.0f;
+};
+bool ggml_cuda_mmv_group_seg_ok(const ggml_tensor * w, const ggml_tensor * y);
+void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, const ggml_cuda_mmv_group_seg * segs, int nseg);
+
+// RDNA3.5 qwen4exp GDN output projection with the gated per-head norm in the prologue:
+//   y' = sigmoid(z) * (rms_norm_128(attn) * norm_w), then the fused-quantize matvec of dst->src[0] with y'
+bool ggml_cuda_mul_mat_vec_q_fq_gdn_gate_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * z);
+void ggml_cuda_mul_mat_vec_q_fq_gdn_gate(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const float * attn,
+        const ggml_tensor * z, const ggml_tensor * norm_w, float eps);
+
+// RDNA3.5 one-token IQ4_NL MoE down projection with the selected-expert weighted sum in the kernel epilogue.
+bool ggml_cuda_mul_mat_id_weighted_rdna3_5_ok(const ggml_tensor * experts, const ggml_tensor * weights, const ggml_tensor * dst);
+void ggml_cuda_mul_mat_id_weighted_rdna3_5(
+        ggml_backend_cuda_context & ctx, const ggml_tensor * experts, const ggml_tensor * weights, ggml_tensor * dst);
diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu
index c3758cd50cfe..50f7367046db 100644
--- a/ggml/src/ggml-cuda/norm.cu
+++ b/ggml/src/ggml-cuda/norm.cu
@@ -277,6 +277,44 @@ static __global__ void l2_norm_f32(
     }
 }
 
+static __global__ void l2_norm_dual_f32_s128(
+        const float * x_q, float * dst_q, const int64_t stride_q_row, const int64_t stride_q_channel,
+        const int64_t stride_q_sample, const float eps_q,
+        const float * x_k, float * dst_k, const int64_t stride_k_row, const int64_t stride_k_channel,
+        const int64_t stride_k_sample, const float eps_k) {
+    const int nrows     = gridDim.x;
+    const int nchannels = gridDim.y / 2;
+    const bool is_k     = blockIdx.y >= nchannels;
+    const int row       = blockIdx.x;
+    const int channel   = blockIdx.y % nchannels;
+    const int sample    = blockIdx.z;
+    const int tid       = threadIdx.x;
+
+    const float * x = is_k ? x_k : x_q;
+    float * dst = is_k ? dst_k : dst_q;
+    const int64_t stride_row     = is_k ? stride_k_row     : stride_q_row;
+    const int64_t stride_channel = is_k ? stride_k_channel : stride_q_channel;
+    const int64_t stride_sample  = is_k ? stride_k_sample  : stride_q_sample;
+    const float eps = is_k ? eps_k : eps_q;
+
+    x   += sample*stride_sample + channel*stride_channel + row*stride_row;
+    dst += ((sample*nchannels + channel)*nrows + row)*128;
+
+    float tmp = 0.0f;
+    ggml_cuda_pdl_sync();
+    for (int col = tid; col < 128; col += WARP_SIZE) {
+        const float xi = x[col];
+        tmp += xi * xi;
+    }
+    tmp = warp_reduce_sum(tmp);
+    ggml_cuda_pdl_lc();
+
+    const float scale = rsqrtf(fmaxf(tmp, eps * eps));
+    for (int col = tid; col < 128; col += WARP_SIZE) {
+        dst[col] = scale * x[col];
+    }
+}
+
 static void norm_f32_cuda(
         const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,
         const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {
@@ -360,7 +398,14 @@ static void rms_norm_mul_f32_cuda(const float *  x,
         const uint3 mul_nrows_packed     = init_fastdiv_values(mul_nrows);
         const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
         const uint3 mul_nsamples_packed  = init_fastdiv_values(mul_nsamples);
-        if (ncols < 1024) {
+        if (ncols == 128) {
+            const dim3 block_dims(128, 1, 1);
+            const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream};
+            ggml_cuda_kernel_launch(rms_norm_f32<128, true>, launch_params,
+                x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,
+                mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
+                nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0));
+        } else if (ncols < 1024) {
             const dim3 block_dims(256, 1, 1);
             const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
             ggml_cuda_kernel_launch(rms_norm_f32<256, true>, launch_params,
@@ -696,3 +741,28 @@ void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
 
     l2_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
 }
+
+void ggml_cuda_op_l2_norm_dual_s128(ggml_backend_cuda_context & ctx, ggml_tensor * dst_q, ggml_tensor * dst_k) {
+    const ggml_tensor * src_q = dst_q->src[0];
+    const ggml_tensor * src_k = dst_k->src[0];
+
+    GGML_ASSERT(src_q->type == GGML_TYPE_F32 && src_k->type == GGML_TYPE_F32);
+    GGML_ASSERT(dst_q->type == GGML_TYPE_F32 && dst_k->type == GGML_TYPE_F32);
+    GGML_ASSERT(src_q->ne[0] == 128 && ggml_are_same_shape(src_q, src_k));
+    GGML_ASSERT(src_q->nb[0] == sizeof(float) && src_k->nb[0] == sizeof(float));
+
+    float eps_q;
+    float eps_k;
+    memcpy(&eps_q, dst_q->op_params, sizeof(float));
+    memcpy(&eps_k, dst_k->op_params, sizeof(float));
+    GGML_ASSERT(eps_q >= 0.0f && eps_k >= 0.0f);
+
+    const dim3 blocks_num(src_q->ne[1], 2*src_q->ne[2], src_q->ne[3]);
+    const dim3 block_dims(WARP_SIZE, 1, 1);
+    const ggml_cuda_kernel_launch_params launch_params = { blocks_num, block_dims, 0, ctx.stream() };
+    ggml_cuda_kernel_launch(l2_norm_dual_f32_s128, launch_params,
+        (const float *) src_q->data, (float *) dst_q->data,
+        src_q->nb[1]/sizeof(float), src_q->nb[2]/sizeof(float), src_q->nb[3]/sizeof(float), eps_q,
+        (const float *) src_k->data, (float *) dst_k->data,
+        src_k->nb[1]/sizeof(float), src_k->nb[2]/sizeof(float), src_k->nb[3]/sizeof(float), eps_k);
+}
diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh
index a74f6376720a..c09d004905a9 100644
--- a/ggml/src/ggml-cuda/norm.cuh
+++ b/ggml/src/ggml-cuda/norm.cuh
@@ -16,3 +16,5 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx,
 void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
 
 void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+
+void ggml_cuda_op_l2_norm_dual_s128(ggml_backend_cuda_context & ctx, ggml_tensor * dst_q, ggml_tensor * dst_k);
diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu
index bcc7723957b5..d4bb2af0e115 100644
--- a/ggml/src/ggml-cuda/quantize.cu
+++ b/ggml/src/ggml-cuda/quantize.cu
@@ -1,4 +1,5 @@
 #include "quantize.cuh"
+#include "unary.cuh"
 #include <cstdint>
 
 #if defined(BLACKWELL_MMA_AVAILABLE)
@@ -463,14 +464,7 @@ static __global__ void quantize_mmq_q8_1(
     constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32;
     constexpr int vals_per_sum   = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32;
 
-    const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;
-
-    if (i0 >= ne0) {
-        return;
-    }
-
-    const int64_t i00 = i0;
-    ggml_cuda_pdl_sync();
+    const int64_t first_chunk = (int64_t) blockIdx.y * CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK;
 
     int64_t base_idx;
     if constexpr (scatter) {
@@ -485,76 +479,134 @@ static __global__ void quantize_mmq_q8_1(
     const float4 * x4 = (const float4 *) x;
     block_q8_1_mmq * y = (block_q8_1_mmq *) vy;
 
-    const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel
-    const int64_t iqs     = i0 % QK8_1_MMQ; // quant index in block
-
-    // Load 4 floats per thread and calculate max. abs. value between them:
-    const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
-    float amax = fabsf(xi.x);
-    amax = fmaxf(amax, fabsf(xi.y));
-    amax = fmaxf(amax, fabsf(xi.z));
-    amax = fmaxf(amax, fabsf(xi.w));
+    ggml_cuda_pdl_sync();
 
-    // Exchange max. abs. value between vals_per_scale/4 threads.
 #pragma unroll
-    for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) {
-        amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
-    }
+    for (int chunk = 0; chunk < CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK; ++chunk) {
+        const int64_t i0 = ((first_chunk + chunk) * blockDim.x + threadIdx.x) * 4;
+        if (i0 >= ne0) {
+            break;
+        }
 
-    float sum;
-    if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
-        sum = xi.x + xi.y + xi.z + xi.w;
+        const int64_t k_block = i0 / QK8_1_MMQ;
+        const int64_t iqs     = i0 % QK8_1_MMQ;
+        const float4 xi = i0 < ne00 ? x4[(base_idx + i0)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+        float amax = fabsf(xi.x);
+        amax = fmaxf(amax, fabsf(xi.y));
+        amax = fmaxf(amax, fabsf(xi.z));
+        amax = fmaxf(amax, fabsf(xi.w));
 
-        // Calculate sums across vals_per_sum/4 threads.
 #pragma unroll
-        for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {
-            sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE);
+        for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) {
+            amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
         }
-    }
 
-    const float d_inv = 127.0f / amax;
-    char4 q;
-    q.x = roundf(xi.x*d_inv);
-    q.y = roundf(xi.y*d_inv);
-    q.z = roundf(xi.z*d_inv);
-    q.w = roundf(xi.w*d_inv);
-    const float d = 1.0f / d_inv;
-
-    // write the block once (normal) or to each of the token's compact rows (scatter)
-    const int nwrite = scatter ? n_expert_used : 1;
+        float sum;
+        if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
+            sum = xi.x + xi.y + xi.z + xi.w;
 #pragma unroll
-    for (int slot = 0; slot < nwrite; ++slot) {
-        int64_t ib;
-        if constexpr (scatter) {
-            const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];
-            ib = k_block*ne1 + i;
-        } else {
-            const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel
-            ib = ib0 + k_block*ne1 + blockIdx.x;
+            for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {
+                sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE);
+            }
         }
 
-        // Write back 4 int8 values as a single 32 bit value for better memory bandwidth:
-        char4 * yqs4 = (char4 *) y[ib].qs;
-        yqs4[iqs/4] = q;
+        const float d_inv = 127.0f / amax;
+        char4 q;
+        q.x = roundf(xi.x*d_inv);
+        q.y = roundf(xi.y*d_inv);
+        q.z = roundf(xi.z*d_inv);
+        q.w = roundf(xi.w*d_inv);
+        const float d = 1.0f / d_inv;
 
-        if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) {
-            if (iqs % 16 == 0 && iqs < 96) {
-                y[ib].d2s6[2 + iqs/16] = sum;
-                if (iqs % 64 == 0) {
-                    y[ib].d2s6[iqs/64] = d;
-                }
-            }
-        } else if (iqs % 32 == 0) {
-            if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) {
-                y[ib].ds4[iqs/32] = make_half2(d, sum);
+        const int nwrite = scatter ? n_expert_used : 1;
+#pragma unroll
+        for (int slot = 0; slot < nwrite; ++slot) {
+            int64_t ib;
+            if constexpr (scatter) {
+                const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];
+                ib = k_block*ne1 + i;
             } else {
-                y[ib].d4[iqs/32]  = d;
+                const int64_t ib0 = blockIdx.z * ((int64_t) gridDim.x * ne0 / QK8_1_MMQ);
+                ib = ib0 + k_block*ne1 + blockIdx.x;
+            }
+
+            char4 * yqs4 = (char4 *) y[ib].qs;
+            yqs4[iqs/4] = q;
+
+            if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) {
+                if (iqs % 16 == 0 && iqs < 96) {
+                    y[ib].d2s6[2 + iqs/16] = sum;
+                    if (iqs % 64 == 0) {
+                        y[ib].d2s6[iqs/64] = d;
+                    }
+                }
+            } else if (iqs % 32 == 0) {
+                if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) {
+                    y[ib].ds4[iqs/32] = make_half2(d, sum);
+                } else {
+                    y[ib].d4[iqs/32] = d;
+                }
             }
         }
     }
     GGML_UNUSED(n_expert_used);
 }
 
+static __global__ void quantize_mmq_q8_1_swiglu(
+        const float * __restrict__ gate, const float * __restrict__ up, const int32_t * __restrict__ ids,
+        void * __restrict__ vy, const int64_t ne00, const int64_t ne0, const int ne1, const int logical_n1,
+        const int64_t gate_s1, const int64_t gate_st, const int64_t up_s1, const int64_t up_st) {
+    const int64_t i0 = ((int64_t) blockDim.x * blockIdx.y + threadIdx.x) * 4;
+    if (i0 >= ne0) {
+        return;
+    }
+
+    const int64_t logical_row = ids ? ids[blockIdx.x] : blockIdx.x;
+    const int64_t slot = logical_row % logical_n1;
+    const int64_t token = logical_row / logical_n1;
+    const int64_t gate_base = token * gate_st + slot * gate_s1;
+    const int64_t up_base = token * up_st + slot * up_s1;
+
+    ggml_cuda_pdl_sync();
+    float values[4];
+#pragma unroll
+    for (int i = 0; i < 4; ++i) {
+        if (i0 + i < ne00) {
+            volatile float value = ggml_cuda_op_silu_single(gate[gate_base + i0 + i]) * up[up_base + i0 + i];
+            values[i] = value;
+        } else {
+            values[i] = 0.0f;
+        }
+    }
+    const float4 xi = make_float4(values[0], values[1], values[2], values[3]);
+
+    float amax = fabsf(xi.x);
+    amax = fmaxf(amax, fabsf(xi.y));
+    amax = fmaxf(amax, fabsf(xi.z));
+    amax = fmaxf(amax, fabsf(xi.w));
+#pragma unroll
+    for (int offset = 4; offset > 0; offset >>= 1) {
+        amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
+    }
+
+    const float d_inv = amax == 0.0f ? 0.0f : 127.0f / amax;
+    char4 q;
+    q.x = roundf(xi.x * d_inv);
+    q.y = roundf(xi.y * d_inv);
+    q.z = roundf(xi.z * d_inv);
+    q.w = roundf(xi.w * d_inv);
+    const float d = amax == 0.0f ? 0.0f : 1.0f / d_inv;
+
+    block_q8_1_mmq * y = (block_q8_1_mmq *) vy;
+    const int64_t k_block = i0 / QK8_1_MMQ;
+    const int64_t iqs = i0 % QK8_1_MMQ;
+    const int64_t ib = k_block * ne1 + blockIdx.x;
+    ((char4 *) y[ib].qs)[iqs / 4] = q;
+    if (iqs % 32 == 0) {
+        y[ib].d4[iqs / 32] = d;
+    }
+}
+
 void quantize_row_q8_1_cuda(
         const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
         const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
@@ -580,7 +632,8 @@ void quantize_mmq_q8_1_cuda(
     GGML_ASSERT(ne0 % QK8_1_MMQ == 0);
 
     // ne1 tends to assume the highest values, therefore use it as the "x" dimension of the CUDA grid:
-    const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
+    const int64_t vals_per_block = 4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ * CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK;
+    const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block;
     const dim3 num_blocks(ne1, block_num_y, ne2*ne3);
     const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
     switch (mmq_get_q8_1_ds_layout(type_src0)) {
@@ -602,6 +655,20 @@ void quantize_mmq_q8_1_cuda(
     }
 }
 
+void quantize_mmq_q8_1_swiglu_cuda(
+        const float * gate, const float * up, const int32_t * ids, void * vy, const ggml_type type_src0,
+        const int64_t ne00, const int64_t ne0, const int64_t ne1, const int logical_n1,
+        const int64_t gate_s1, const int64_t gate_st, const int64_t up_s1, const int64_t up_st, cudaStream_t stream) {
+    GGML_ASSERT(mmq_get_q8_1_ds_layout(type_src0) == MMQ_Q8_1_DS_LAYOUT_D4);
+    GGML_ASSERT(ne00 % 4 == 0);
+    GGML_ASSERT(ne0 % QK8_1_MMQ == 0);
+    const int64_t block_num_y = (ne0 + 4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
+    const dim3 num_blocks(ne1, block_num_y, 1);
+    const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
+    quantize_mmq_q8_1_swiglu<<<num_blocks, block_size, 0, stream>>>(
+        gate, up, ids, vy, ne00, ne0, ne1, logical_n1, gate_s1, gate_st, up_s1, up_st);
+}
+
 // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)
 void quantize_scatter_mmq_q8_1_cuda(
         const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
@@ -610,7 +677,8 @@ void quantize_scatter_mmq_q8_1_cuda(
     GGML_ASSERT(ne00 % 4 == 0);
     GGML_ASSERT(ne0 % QK8_1_MMQ == 0);
 
-    const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
+    const int64_t vals_per_block = 4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ * CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK;
+    const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block;
     const dim3 num_blocks(n_tokens, block_num_y, 1);
     const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
     switch (mmq_get_q8_1_ds_layout(type_src0)) {
diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh
index 5f08dcbfe331..1801da05e871 100644
--- a/ggml/src/ggml-cuda/quantize.cuh
+++ b/ggml/src/ggml-cuda/quantize.cuh
@@ -7,9 +7,11 @@
 
 #define CUDA_QUANTIZE_BLOCK_SIZE     256
 #define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128
+#define CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK 2
 
 static_assert(MATRIX_ROW_PADDING %    CUDA_QUANTIZE_BLOCK_SIZE      == 0, "Risk of out-of-bounds access.");
 static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");
+static_assert((4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) % QK8_1_MMQ == 0, "Quantization chunks must contain complete Q8_1 MMQ blocks.");
 
 typedef void (*quantize_cuda_t)(
         const float * x, const int32_t * ids, void * vy,
@@ -26,6 +28,11 @@ void quantize_mmq_q8_1_cuda(
         ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
         int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
 
+void quantize_mmq_q8_1_swiglu_cuda(
+        const float * gate, const float * up, const int32_t * ids, void * vy, ggml_type type_src0,
+        int64_t ne00, int64_t ne0, int64_t ne1, int logical_n1,
+        int64_t gate_s1, int64_t gate_st, int64_t up_s1, int64_t up_st, cudaStream_t stream);
+
 void quantize_mmq_fp4_cuda(const float *   x,
                              const int32_t * ids,
                              void *          vy,
diff --git a/ggml/src/ggml-cuda/sum.cu b/ggml/src/ggml-cuda/sum.cu
index c56257b44066..d1f938905d0b 100644
--- a/ggml/src/ggml-cuda/sum.cu
+++ b/ggml/src/ggml-cuda/sum.cu
@@ -2,7 +2,12 @@
 #include "sumrows.cuh"
 
 #ifdef GGML_CUDA_USE_CUB
+#ifdef GGML_CUDA_CUB_IS_HIPCUB
+#include <hipcub/hipcub.hpp>
+namespace cub = hipcub;
+#else
 #include <cub/cub.cuh>
+#endif  // GGML_CUDA_CUB_IS_HIPCUB
 using namespace cub;
 #endif  // GGML_CUDA_USE_CUB
 
diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu
index 9681cd293338..3febcc026043 100644
--- a/ggml/src/ggml-cuda/top-k.cu
+++ b/ggml/src/ggml-cuda/top-k.cu
@@ -2,12 +2,17 @@
 #include "top-k.cuh"
 
 #ifdef GGML_CUDA_USE_CUB
-#    include <cub/cub.cuh>
-#    if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2)
-#        define CUB_TOP_K_AVAILABLE
-#        include <cuda/iterator>
+#    ifdef GGML_CUDA_CUB_IS_HIPCUB
+#        include <hipcub/hipcub.hpp>
+namespace cub = hipcub;
+#    else
+#        include <cub/cub.cuh>
+#        if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2)
+#            define CUB_TOP_K_AVAILABLE
+#            include <cuda/iterator>
 using namespace cub;
-#    endif  // CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2
+#        endif  // CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2
+#    endif  // GGML_CUDA_CUB_IS_HIPCUB
 #endif      // GGML_CUDA_USE_CUB
 
 #ifdef CUB_TOP_K_AVAILABLE
diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu
index c8cec70bb320..183bc47ef6bf 100644
--- a/ggml/src/ggml-cuda/topk-moe.cu
+++ b/ggml/src/ggml-cuda/topk-moe.cu
@@ -87,7 +87,7 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co
 
     It is intended as fusion of softmax->top-k->get_rows pipeline for MoE models
 */
-template <int n_experts, bool has_bias>
+template <int n_experts, bool has_bias, bool small_topk = false>
 __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float *         logits,
                                                                   float *               weights,
                                                                   int32_t *             ids,
@@ -168,10 +168,11 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float *
 
     float wt_sum = 0.f;
 
-    float output_weights[experts_per_thread];
+    constexpr int output_weights_per_thread = small_topk ? 1 : experts_per_thread;
+    float output_weights[output_weights_per_thread];
 
 #pragma unroll
-    for (int i = 0; i < experts_per_thread; i++) {
+    for (int i = 0; i < output_weights_per_thread; i++) {
         output_weights[i] = 0.f;
     }
 
@@ -234,7 +235,7 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float *
         }
 
         if ((k & (WARP_SIZE - 1)) == threadIdx.x) {
-            output_weights[k / WARP_SIZE] = max_val;
+            output_weights[small_topk ? 0 : k / WARP_SIZE] = max_val;
         }
 
         if ((max_expert & (WARP_SIZE - 1)) == threadIdx.x) {
@@ -250,17 +251,17 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float *
         wt_sum              = max(wt_sum, clamp_val);
         const float inv_sum = 1.0f / wt_sum;
 
-        for (int i = 0; i < experts_per_thread; i++) {
+        for (int i = 0; i < output_weights_per_thread; i++) {
             output_weights[i] *= inv_sum;
         }
     }
 
     if (config.delayed_softmax) {
-        softmax_warp_inplace<experts_per_thread, true>(output_weights, n_expert_used, threadIdx.x);
+        softmax_warp_inplace<output_weights_per_thread, true>(output_weights, n_expert_used, threadIdx.x);
     }
 
 #pragma unroll
-    for (int i = 0; i < experts_per_thread; i++) {
+    for (int i = 0; i < output_weights_per_thread; i++) {
         const int idx = i * WARP_SIZE + threadIdx.x;
         if (idx < n_expert_used) {
             weights[idx] = output_weights[i] * scale_val;
@@ -330,8 +331,13 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx,
                 logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
             break;
         case 512:
-            ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params,
-                logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
+            if (n_expert_used <= WARP_SIZE) {
+                ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias, true>, launch_params,
+                    logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
+            } else {
+                ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params,
+                    logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config);
+            }
             break;
         case 576:
             ggml_cuda_kernel_launch(topk_moe_cuda<576, has_bias>, launch_params,
diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu
index d3e594878fcc..1508bb857a89 100644
--- a/ggml/src/ggml-cuda/unary.cu
+++ b/ggml/src/ggml-cuda/unary.cu
@@ -702,6 +702,23 @@ void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary
     }
 }
 
+void ggml_cuda_op_cont_sigmoid_mul(
+        ggml_backend_cuda_context & ctx, ggml_tensor * cont_node, ggml_tensor * unary_node, ggml_tensor * mul_node) {
+    const ggml_tensor * gate = cont_node->src[0];
+    const ggml_tensor * other = mul_node->src[0] == unary_node ? mul_node->src[1] : mul_node->src[0];
+
+    GGML_ASSERT(ggml_get_unary_op(unary_node) == GGML_UNARY_OP_SIGMOID);
+    GGML_ASSERT(gate->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && mul_node->type == GGML_TYPE_F32);
+    GGML_ASSERT(gate->nb[0] == sizeof(float) && gate->nb[2] == gate->nb[1] * gate->ne[1]);
+    GGML_ASSERT(ggml_is_contiguous(other) && ggml_is_contiguous(mul_node));
+    GGML_ASSERT(ggml_nelements(gate) == ggml_nelements(other));
+
+    const int64_t k = ggml_nelements(other);
+    const int64_t n = gate->ne[0];
+    unary_gated_cuda<op_sigmoid>((const float *) gate->data, (const float *) other->data, (float *) mul_node->data,
+        k, n, gate->nb[1] / sizeof(float), n, ctx.stream());
+}
+
 /* fused relu + sqr */
 
 void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node) {
@@ -719,3 +736,47 @@ void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_n
         unary_cuda<op_relu_sqr>((const float *)src->data, (float *)sqr_node->data, k, stream);
     }
 }
+
+/* fused scale + unary */
+
+template <float (*op)(float)>
+static __global__ void scale_unary_kernel(const float * x, float * dst, const float scale, const float bias, const int k) {
+    const int i = blockDim.x*blockIdx.x + threadIdx.x;
+
+    if (i >= k) {
+        return;
+    }
+
+    // identical expression to scale_f32 in scale.cu, followed by the unary op
+    dst[i] = op(scale * x[i] + bias);
+}
+
+template <float (*op)(float)>
+static void scale_unary_cuda(const float * x, float * dst, const float scale, const float bias, const int k, cudaStream_t stream) {
+    const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE;
+    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream);
+    ggml_cuda_kernel_launch(scale_unary_kernel<op>, launch_params, x, dst, scale, bias, k);
+}
+
+void ggml_cuda_op_scale_unary(ggml_backend_cuda_context & ctx, ggml_tensor * scale_node, ggml_tensor * unary_node) {
+    const ggml_tensor * src = scale_node->src[0];
+
+    GGML_ASSERT(src->type == GGML_TYPE_F32 && scale_node->type == GGML_TYPE_F32 && unary_node->type == GGML_TYPE_F32);
+    GGML_ASSERT(ggml_is_contiguous(src) && ggml_is_contiguous(unary_node));
+    GGML_ASSERT(unary_node->src[0] == scale_node);
+
+    const float scale = ggml_get_op_params_f32(scale_node, 0);
+    const float bias  = ggml_get_op_params_f32(scale_node, 1);
+    const int   k     = ggml_nelements(src);
+
+    switch (ggml_get_unary_op(unary_node)) {
+        case GGML_UNARY_OP_SILU:
+            scale_unary_cuda<op_silu>((const float *) src->data, (float *) unary_node->data, scale, bias, k, ctx.stream());
+            break;
+        case GGML_UNARY_OP_SIGMOID:
+            scale_unary_cuda<op_sigmoid>((const float *) src->data, (float *) unary_node->data, scale, bias, k, ctx.stream());
+            break;
+        default:
+            GGML_ABORT("unsupported unary op for fused scale+unary");
+    }
+}
diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuh
index 04f3af6443a7..f2a543cffbe2 100644
--- a/ggml/src/ggml-cuda/unary.cuh
+++ b/ggml/src/ggml-cuda/unary.cuh
@@ -93,6 +93,9 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
 
 void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node);
 
+void ggml_cuda_op_cont_sigmoid_mul(
+        ggml_backend_cuda_context & ctx, ggml_tensor * cont_node, ggml_tensor * unary_node, ggml_tensor * mul_node);
+
 void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node);
 
 __device__ __forceinline__ float ggml_cuda_op_silu_single(float x) {
@@ -121,3 +124,6 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, fl
 
     return ggml_cuda_op_silu_single(gate) * up;
 }
+
+// fused GGML_OP_SCALE + GGML_UNARY_OP_{SILU,SIGMOID}: dst = op(scale*x + bias)
+void ggml_cuda_op_scale_unary(ggml_backend_cuda_context & ctx, ggml_tensor * scale_node, ggml_tensor * unary_node);
diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh
index ec117c57dfbc..5bce2f4f02d8 100644
--- a/ggml/src/ggml-cuda/vecdotq.cuh
+++ b/ggml/src/ggml-cuda/vecdotq.cuh
@@ -1238,6 +1238,60 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(
     return d * sumi;
 }
 
+// Two IQ3_S rows against the same activation block. The fused MoE gate/up path
+// otherwise calls vec_dot_iq3_s_q8_1 twice and reloads all eight Q8_1 words.
+// Keep each integer and floating-point accumulation independent so the results
+// are bit-identical to the two scalar calls.
+static __device__ __forceinline__ void vec_dot_iq3_s_q8_1_pair(
+        float & out0, float & out1,
+        const void * __restrict__ vbq0, const void * __restrict__ vbq1,
+        const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {
+    const block_iq3_s * bq0 = (const block_iq3_s *) vbq0 + kbx;
+    const block_iq3_s * bq1 = (const block_iq3_s *) vbq1 + kbx;
+
+    const int2 qs0_packed = make_int2(get_int_b2(bq0->qs, iqs + 0), get_int_b2(bq0->qs, iqs + 1));
+    const int2 qs1_packed = make_int2(get_int_b2(bq1->qs, iqs + 0), get_int_b2(bq1->qs, iqs + 1));
+    const uint8_t * qs0 = (const uint8_t *) &qs0_packed;
+    const uint8_t * qs1 = (const uint8_t *) &qs1_packed;
+
+    const int qh0 = bq0->qh[iqs/2];
+    const int qh1 = bq1->qh[iqs/2];
+    const int signs0_packed = get_int_b2(bq0->signs, iqs/2);
+    const int signs1_packed = get_int_b2(bq1->signs, iqs/2);
+    const uint8_t * signs0 = (const uint8_t *) &signs0_packed;
+    const uint8_t * signs1 = (const uint8_t *) &signs1_packed;
+
+    int sumi0 = 0;
+    int sumi1 = 0;
+#pragma unroll
+    for (int l0 = 0; l0 < 8; l0 += 2) {
+        const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);
+        const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);
+
+        const int2 grid0 = make_int2(
+            iq3s_grid[qs0[l0 + 0] | ((qh0 << (8 - l0)) & 0x100)],
+            iq3s_grid[qs0[l0 + 1] | ((qh0 << (7 - l0)) & 0x100)]);
+        const int s00 = __vcmpne4(((signs0[l0/2] & 0x03) << 7) | ((signs0[l0/2] & 0x0C) << 21), 0x00000000);
+        const int s01 = __vcmpne4(((signs0[l0/2] & 0x30) << 3) | ((signs0[l0/2] & 0xC0) << 17), 0x00000000);
+        sumi0 = ggml_cuda_dp4a(__vsub4(grid0.x ^ s00, s00), u0, sumi0);
+        sumi0 = ggml_cuda_dp4a(__vsub4(grid0.y ^ s01, s01), u1, sumi0);
+
+        const int2 grid1 = make_int2(
+            iq3s_grid[qs1[l0 + 0] | ((qh1 << (8 - l0)) & 0x100)],
+            iq3s_grid[qs1[l0 + 1] | ((qh1 << (7 - l0)) & 0x100)]);
+        const int s10 = __vcmpne4(((signs1[l0/2] & 0x03) << 7) | ((signs1[l0/2] & 0x0C) << 21), 0x00000000);
+        const int s11 = __vcmpne4(((signs1[l0/2] & 0x30) << 3) | ((signs1[l0/2] & 0xC0) << 17), 0x00000000);
+        sumi1 = ggml_cuda_dp4a(__vsub4(grid1.x ^ s10, s10), u0, sumi1);
+        sumi1 = ggml_cuda_dp4a(__vsub4(grid1.y ^ s11, s11), u1, sumi1);
+    }
+
+    sumi0 *= 1 + 2*((bq0->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);
+    sumi1 *= 1 + 2*((bq1->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);
+    const float d8 = __low2float(bq8_1[iqs/2].ds);
+    out0 = (__half2float(bq0->d) * d8) * sumi0;
+    out1 = (__half2float(bq1->d) * d8) * sumi1;
+}
+
 #define VDR_IQ1_S_Q8_1_MMVQ 1
 #define VDR_IQ1_S_Q8_1_MMQ  1
 
diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h
index 9aa558f3f4ca..7055a4022f43 100644
--- a/ggml/src/ggml-cuda/vendors/hip.h
+++ b/ggml/src/ggml-cuda/vendors/hip.h
@@ -139,6 +139,9 @@
 #define cudaGraphExecUpdate hipGraphExecUpdate
 #define cudaStreamCaptureModeRelaxed hipStreamCaptureModeRelaxed
 #define cudaStreamBeginCapture hipStreamBeginCapture
+#define cudaStreamIsCapturing hipStreamIsCapturing
+#define cudaStreamCaptureStatus hipStreamCaptureStatus
+#define cudaStreamCaptureStatusNone hipStreamCaptureStatusNone
 #define cudaGraph_t hipGraph_t
 #define cudaStream_t hipStream_t
 #define cudaSuccess hipSuccess
diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h
index 62b76abbcec9..bcdeff6ade24 100644
--- a/ggml/src/ggml-impl.h
+++ b/ggml/src/ggml-impl.h
@@ -727,7 +727,7 @@ static inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph,
                                           const enum ggml_op *       ops,
                                           const int *                outputs,
                                           int                        num_outputs) {
-    GGML_ASSERT(count < 32);
+    GGML_ASSERT(count <= 32);
     if (node_idx + count > cgraph->n_nodes) {
         return false;
     }
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 835c98c8128d..1fbf1b744ffb 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -16,6 +16,7 @@
 #include <cinttypes>
 #include <cmath>
 #include <cstring>
+#include <cstdlib>
 #include <limits>
 #include <stdexcept>
 #include <string>
@@ -232,6 +233,10 @@ llama_context::llama_context(
     cparams.fused_gdn_ar = true;
     cparams.fused_gdn_ch = true;
     cparams.auto_fgdn    = false;
+    // LLAMA_FUSED_GDN_CH=0 selects the ggml-op chunked delta-net formulation for multi-token ubatches
+    if (const char * env = getenv("LLAMA_FUSED_GDN_CH")) {
+        cparams.fused_gdn_ch = atoi(env) != 0;
+    }
 
     cparams.fused_lid = true;
     cparams.auto_flid = false;
@@ -1359,19 +1364,21 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
     static const bool graph_timing = getenv("LLAMA_GRAPH_TIMING") && atoi(getenv("LLAMA_GRAPH_TIMING")) != 0;
     int64_t t_build = 0, t_alloc = 0, t_inputs = 0;
     bool reused = false;
+
     if (!graph_reuse_disable && res->can_reuse(gparams)) {
         reused = true;
         //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__);
 
-        // with pipeline parallelism, the previous graph_compute_async may still be running
-        // on the GPU. we must synchronize before set_inputs to avoid overwriting input tensors
-        // that the previous compute is still reading.
+        // Pipeline-parallel graphs may still read input tensors when set_inputs updates them below.
         if (cparams.pipeline_parallel) {
             ggml_backend_sched_synchronize(sched.get());
         }
 
         n_reused++;
     } else {
+        // The previous asynchronous graph may still use allocations that reset releases.
+        ggml_backend_sched_synchronize(sched.get());
+
         res->reset();
 
         ggml_backend_sched_reset(sched.get());
diff --git a/src/models/models.h b/src/models/models.h
index 168c5366b9e8..75c3ca710ed9 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2288,6 +2288,7 @@ struct llama_model_qwen4exp : public llama_model_base {
     llama_model_qwen4exp(const struct llama_model_params & params) : llama_model_base(params) {}
 
     class llm_graph_input_qsa;
+    class llm_graph_input_qsa_k;
 
     // Set when the n-gram table is left on disk (--ngram-on-disk): per_layer_tok_embd is
     // counted as created but never allocated, mapped or read, and build_ple takes its
@@ -2339,6 +2340,13 @@ struct llama_model_qwen4exp : public llama_model_base {
         // the QSA cache layout inputs do not depend on the layer, only on its compress ratio,
         // so the layers sharing a ratio share one input set
         std::map<uint32_t, llm_graph_input_qsa *> qsa_inps;
+        llm_graph_input_qsa_k * qsa_k_inp = nullptr;
+
+        // QSA: store this ubatch's raw indexer keys in the indexer cache (no selection)
+        void build_qsa_store_k(
+  const llama_memory_hybrid_idx_context * mctx_hyb,
+                    ggml_tensor * cur,
+                            int   il);
 
         // QSA: token indices this layer's queries may attend to, or nullptr for dense
         ggml_tensor * build_qsa_top_k(
diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp
index f6a71672c4d6..05a53498f395 100644
--- a/src/models/qwen35.cpp
+++ b/src/models/qwen35.cpp
@@ -221,6 +221,9 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para
 
     // LM head
     cur = build_lora_mm(model.output, cur, model.output_s);
+    if (cparams.embeddings_nextn && n_tokens <= 16 && cur->op == GGML_OP_MUL_MAT) {
+        ggml_mul_mat_set_hint(cur, GGML_HINT_EXACT_BATCH);
+    }
 
     cb(cur, "result_output", -1);
     res->t_logits = cur;
@@ -268,6 +271,9 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn(
 
     // Qwen3Next uses a single Q projection that outputs query + gate
     ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ]
+    if (cparams.embeddings_nextn && n_tokens <= 16 && Qcur_full->op == GGML_OP_MUL_MAT) {
+        ggml_mul_mat_set_hint(Qcur_full, GGML_HINT_EXACT_BATCH);
+    }
     cb(Qcur_full, "Qcur_full", il);
 
     ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,
@@ -280,9 +286,15 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn(
     cb(Qcur, "Qcur_normed", il);
 
     ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);
+    if (cparams.embeddings_nextn && n_tokens <= 16 && Kcur->op == GGML_OP_MUL_MAT) {
+        ggml_mul_mat_set_hint(Kcur, GGML_HINT_EXACT_BATCH);
+    }
     cb(Kcur, "Kcur", il);
 
     ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);
+    if (cparams.embeddings_nextn && n_tokens <= 16 && Vcur->op == GGML_OP_MUL_MAT) {
+        ggml_mul_mat_set_hint(Vcur, GGML_HINT_EXACT_BATCH);
+    }
     cb(Vcur, "Vcur", il);
 
     // Apply K normalization
diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
index 988cbd6ccd7a..abe7ff70402d 100644
--- a/src/models/qwen35moe.cpp
+++ b/src/models/qwen35moe.cpp
@@ -187,13 +187,15 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p
         cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
         cb(cur, "attn_norm", il);
 
-        ggml_build_forward_expand(gf, cur);
-
         // Determine layer type and build appropriate attention mechanism
         if (hparams.is_recr(il)) {
             // Linear attention layer (gated delta net)
             cur = build_layer_attn_linear(inp->get_recr(), cur, il);
+
+            ggml_build_forward_expand(gf, cur);
         } else {
+            ggml_build_forward_expand(gf, cur);
+
             // Full attention layer
             cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);
         }
@@ -383,14 +385,18 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear(
     ggml_tensor * qkv_mixed = qkvz.first;
     ggml_tensor * z         = qkvz.second;
 
-    ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s);
+    ggml_tensor * beta  = build_lora_mm(model.layers[il].ssm_beta,  cur, model.layers[il].ssm_beta_s);
+    ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
+
+    ggml_build_forward_expand(gf, beta);
+    ggml_build_forward_expand(gf, alpha);
+
     beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
     cb(beta, "beta", il);
 
     beta = ggml_sigmoid(ctx0, beta);
     cb(beta, "beta_sigmoid", il);
 
-    ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
     alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
     cb(alpha, "alpha", il);
 
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 9d40e8683b0d..e11058110446 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -6,6 +6,14 @@
 
 #include <algorithm>
 #include <cinttypes>
+#include <cstdint>
+#include <cstring>
+#include <cstdlib>
+
+#if defined(__linux__) || defined(__APPLE__)
+#include <sys/mman.h>
+#include <unistd.h>
+#endif
 
 void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
     ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp, false);
@@ -412,12 +420,27 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
 
     // grouped RMSNorm: reduce over one stream, then scale all streams with the [hc_dim] gamma
     // the converter folded each gamma to (1 + w)
+    //
+    // The gamma is viewed as [n_embd, hc] and applied to the [n_embd, hc, T] norm output so that the
+    // RMS_NORM and MUL nodes are adjacent in the graph (backends fuse that pair). The view of the
+    // weight is expanded first so it does not land between them.
+    ggml_tensor * w_norm_hc = ggml_reshape_2d(ctx0, w_norm, n_embd, hc);
+    ggml_build_forward_expand(gf, w_norm_hc);
+
     ggml_tensor * xn = ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps);
+    xn = ggml_mul(ctx0, xn, w_norm_hc);
     xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt);
-    xn = ggml_mul(ctx0, xn, w_norm);
     cb(xn, "hc_norm", il);
 
     ggml_tensor * lo = build_lora_mm(w_down, xn);
+    if (inject) {
+        // the inject projection reads the same xn as the down projection: emit the two matvecs back to back so
+        // that the backend can launch them as one grouped kernel (the inject result is only used by hc_combine)
+        ggml_build_forward_expand(gf, lo);
+        *inject = build_lora_mm(w_inject, xn);
+        cb(*inject, "hc_inject", il);
+        ggml_build_forward_expand(gf, *inject);
+    }
     lo = ggml_silu(ctx0, ggml_scale(ctx0, lo, 1.0f / (float) hc));
     ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(w_up, lo));
     cb(gate, "hc_gate", il);
@@ -438,11 +461,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix(
     mixed = ggml_scale(ctx0, mixed, 1.0f / (float) hc);
     cb(mixed, "hc_mixed", il);
 
-    if (inject) {
-        *inject = build_lora_mm(w_inject, xn);
-        cb(*inject, "hc_inject", il);
-    }
-
     return mixed;
 }
 
@@ -454,11 +472,24 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine(
     const int64_t hc = hparams.dsv4_hc_mult;
     const int64_t nt = residual->ne[2];
 
+    // block_out and inject die at this combine; keep their buffers from being handed to the next grouped norm
+    // output (the backend fuses combine + norm into one kernel whose blocks all read block_out/inject while
+    // writing their own stream of the norm output, so the two must not alias)
+    ggml_set_output(block_out);
+    ggml_set_output(inject);
+
     // 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add
     ggml_tensor * w = ggml_sigmoid(ctx0, ggml_scale(ctx0, inject, 1.0f / (float) hc));
     w = ggml_scale(ctx0, w, 2.0f);
     w = ggml_reshape_3d(ctx0, w, 1, hc, nt);
 
+    // Emit the block output first and the scatter-weight chain right after it, so that
+    // scale -> sigmoid -> scale -> repeat -> mul -> add is one contiguous node run that
+    // backends can fuse into a single kernel.
+    ggml_build_forward_expand(gf, residual);
+    ggml_build_forward_expand(gf, block_out);
+    ggml_build_forward_expand(gf, w);
+
     ggml_tensor * b = ggml_reshape_3d(ctx0, block_out, n_embd, 1, nt);
     b = ggml_repeat_4d(ctx0, b, n_embd, hc, nt, 1);
 
@@ -682,6 +713,54 @@ class llama_model_qwen4exp::llm_graph_input_qsa : public llm_graph_input_i {
     const bool blk_bias;
 };
 
+// indexer-key store only: the cache is written every ubatch, the selection is skipped while it fits the budget
+class llama_model_qwen4exp::llm_graph_input_qsa_k : public llm_graph_input_i {
+public:
+    llm_graph_input_qsa_k(const llama_memory_hybrid_idx_context * mctx) : mctx(mctx) {}
+    virtual ~llm_graph_input_qsa_k() = default;
+
+    void set_input(const llama_ubatch * ubatch) override {
+        mctx->get_idx()->set_input_k_idxs(k_idxs, ubatch);
+    }
+
+    bool can_reuse(const llm_graph_params & params) override {
+        mctx = static_cast<const llama_memory_hybrid_idx_context *>(params.mctx);
+
+        const auto * idx = mctx->get_idx();
+        if (idx == nullptr) {
+            return false;
+        }
+
+        return k_idxs->ne[0] == params.ubatch.n_tokens;
+    }
+
+    ggml_tensor * k_idxs = nullptr;   // I32 [n_tokens]
+
+    const llama_memory_hybrid_idx_context * mctx;
+};
+
+void llama_model_qwen4exp::graph::build_qsa_store_k(
+        const llama_memory_hybrid_idx_context * mctx_hyb,
+        ggml_tensor *                           cur,
+        int                                     il) {
+    const llama_kv_cache_context * mctx_idx = mctx_hyb->get_idx();
+
+    const int64_t idx_dim = hparams.indexer_head_size;
+
+    if (qsa_k_inp == nullptr) {
+        auto inp = std::make_unique<llm_graph_input_qsa_k>(mctx_hyb);
+        inp->k_idxs = mctx_idx->build_input_k_idxs(ctx0, ubatch);
+        qsa_k_inp = inp.get();
+        res->add_input(std::move(inp));
+    }
+
+    ggml_tensor * k_raw = build_lora_mm(model.layers[il].index_k_proj, cur);
+    k_raw = ggml_reshape_3d(ctx0, k_raw, idx_dim, 1, n_tokens);
+    cb(k_raw, "indexer_k_raw", il);
+
+    ggml_build_forward_expand(gf, mctx_idx->cpy_k(ctx0, k_raw, qsa_k_inp->k_idxs, il));
+}
+
 ggml_tensor * llama_model_qwen4exp::graph::build_qsa_top_k(
         const llama_memory_hybrid_idx_context * mctx_hyb,
         ggml_tensor *                           cur,
@@ -919,7 +998,26 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn(
     // dense, see build_layer_attn's caller in graph_mtp), no cache, or no ratio means dense
     const bool qsa = mctx_hyb != nullptr && mctx_hyb->get_idx() != nullptr && hparams.dsv4_compress_ratios[il] > 0;
 
-    ggml_tensor * top_k = qsa ? build_qsa_top_k(mctx_hyb, cur, inp_pos, inp->get_kq_mask(), sections, il) : nullptr;
+    ggml_tensor * top_k = nullptr;
+    if (qsa) {
+        // The selection keeps min(n_kv, top_k + ratio - 1) cells. While the cache holds no more
+        // cells than that, every cell is selected and the top-k mask is a no-op on the causal
+        // mask, so attend dense (same result) and only store this ubatch's indexer keys for
+        // when the context grows past the budget. Mirrors the reference "seqlen <= topk" path.
+        const int64_t r     = hparams.dsv4_compress_ratios[il];
+        const int64_t n_kv  = mctx_hyb->get_idx()->get_n_kv();
+        const int64_t width = (int64_t) hparams.indexer_top_k + r - 1;
+        // LLAMA_QSA_DENSE_SHORTCUT=0 forces the selection path (validation)
+        static const bool shortcut = [] {
+            const char * env = getenv("LLAMA_QSA_DENSE_SHORTCUT");
+            return env == nullptr || atoi(env) != 0;
+        }();
+        if (shortcut && n_kv <= width) {
+            build_qsa_store_k(mctx_hyb, cur, il);
+        } else {
+            top_k = build_qsa_top_k(mctx_hyb, cur, inp_pos, inp->get_kq_mask(), sections, il);
+        }
+    }
 
     // Qwen3Next uses a single Q projection that outputs query + gate
     ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ]
@@ -1019,13 +1117,21 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
     beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);
     cb(beta, "beta", il);
 
-    beta = ggml_sigmoid(ctx0, beta);
-    cb(beta, "beta_sigmoid", il);
-
     ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);
     alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);
     cb(alpha, "alpha", il);
 
+    // emit the two small projections ahead of the recurrence so that the elementwise chain
+    // conv -> norms -> gate -> beta -> gated_delta_net -> state copy -> norm is contiguous in the
+    // graph (the CUDA backend fuses that decode chain into a single kernel)
+    ggml_build_forward_expand(gf, qkv_mixed);
+    ggml_build_forward_expand(gf, z);
+    ggml_build_forward_expand(gf, beta);
+    ggml_build_forward_expand(gf, alpha);
+
+    beta = ggml_sigmoid(ctx0, beta);
+    cb(beta, "beta_sigmoid", il);
+
     ggml_tensor * alpha_biased   = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);
     ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);
     cb(alpha_softplus, "a_softplus", il);
@@ -1122,6 +1228,21 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear(
 ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, const int il) {
     GGML_ASSERT(model.layers[il].ffn_gate_inp != nullptr);
 
+    // Shared-expert gate/up projections first: they read the same `cur` as the router matvec that opens
+    // build_moe_ffn, so the backend can launch [up, gate, glu, router] as one grouped kernel. Same math as
+    // build_ffn(LLM_FFN_SILU, LLM_FFN_PAR) without a swiglu clamp.
+    ggml_tensor * shexp_act = nullptr;
+    if (model.layers[il].ffn_up_shexp != nullptr && model.layers[il].ffn_gate_shexp != nullptr &&
+            !(il >= 0 && hparams.swiglu_clamp_shexp[il] > 1e-6f)) {
+        ggml_tensor * up = build_lora_mm(model.layers[il].ffn_up_shexp, cur, model.layers[il].ffn_up_shexp_s);
+        cb(up, "ffn_up", il);
+        ggml_tensor * gate = build_lora_mm(model.layers[il].ffn_gate_shexp, cur, model.layers[il].ffn_gate_shexp_s);
+        cb(gate, "ffn_gate", il);
+        shexp_act = ggml_swiglu_split(ctx0, gate, up);
+        cb(shexp_act, "ffn_swiglu", il);
+        ggml_build_forward_expand(gf, shexp_act);
+    }
+
     ggml_tensor * moe_out =
         build_moe_ffn(cur,
             model.layers[il].ffn_gate_inp,
@@ -1141,7 +1262,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, co
 
     // shared experts, as in the Qwen3Next reference
     if (model.layers[il].ffn_up_shexp != nullptr) {
-        ggml_tensor * ffn_shexp =
+        ggml_tensor * ffn_shexp = shexp_act ?
+            build_lora_mm(model.layers[il].ffn_down_shexp, shexp_act, model.layers[il].ffn_down_shexp_s) :
             build_ffn(cur,
                 model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,
                 model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s,
@@ -1273,6 +1395,52 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
         }
     }
 
+    if (embd != nullptr && !pmodel.ple_disk) {
+        // host-resident table: dequantize the rows exactly as the CPU get_rows would (same to_float)
+        const ggml_tensor * tab = pmodel.per_layer_tok_embd;
+        GGML_ASSERT(tab && tab->data && rows == nullptr);
+        const size_t  row_size = ggml_row_size(tab->type, tab->ne[0]);
+        const int64_t dim      = tab->ne[0];
+        GGML_ASSERT(dim == hp.ple_head_dim);
+
+        const auto * traits = ggml_get_type_traits(tab->type);
+
+#if defined(__linux__) || defined(__APPLE__)
+        // memory-mapped table, random rows: queue every page read at once instead of taking the
+        // faults one after another (~0.2 ms each from NVMe) while dequantizing below
+        {
+            const long page = sysconf(_SC_PAGESIZE);
+            if (page > 0) {
+                const uintptr_t base = (uintptr_t) tab->data;
+                const uintptr_t mask = ~((uintptr_t) page - 1);
+                uintptr_t prev_page = 0;
+                for (const int32_t r : idx) {
+                    const uintptr_t a0 = (base + (size_t) r * row_size) & mask;
+                    const uintptr_t a1 = base + (size_t) r * row_size + row_size;
+                    if (a0 != prev_page) {
+                        madvise((void *) a0, a1 - a0, MADV_WILLNEED);
+                        prev_page = a0;
+                    }
+                }
+            }
+        }
+#endif
+
+        embd_buf.resize(idx.size() * (size_t) dim);
+        for (size_t k = 0; k < idx.size(); ++k) {
+            const char * src = (const char *) tab->data + (size_t) idx[k] * row_size;
+            float *      dst = embd_buf.data() + k * (size_t) dim;
+            if (tab->type == GGML_TYPE_F32) {
+                memcpy(dst, src, dim * sizeof(float));
+            } else {
+                GGML_ASSERT(traits->to_float != nullptr);
+                traits->to_float(src, dst, dim);
+            }
+        }
+        ggml_backend_tensor_set(embd, embd_buf.data(), 0, embd_buf.size() * sizeof(float));
+        return;
+    }
+
     if (pmodel.ple_disk) {
         // idx is already exactly what the reader wants: one global row per (token, head),
         // token-major, with ple_head_offsets folded in by the loop above. The previous
@@ -1286,6 +1454,30 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) {
         return;
     }
 
+#if defined(__linux__) || defined(__APPLE__)
+    // The table is memory-mapped and the gathered rows are effectively random, so every row is a
+    // page fault the CPU get_rows would otherwise take one after another (~0.2 ms each from NVMe).
+    // Ask the kernel to read all of them in parallel now; the gather then finds them (or the I/O in
+    // flight) in the page cache.
+    if (const ggml_tensor * tab = pmodel.per_layer_tok_embd; tab && tab->data) {
+        const long page = sysconf(_SC_PAGESIZE);
+        if (page > 0) {
+            const uintptr_t base     = (uintptr_t) tab->data;
+            const size_t    row_size = ggml_row_size(tab->type, tab->ne[0]);
+            const uintptr_t mask     = ~((uintptr_t) page - 1);
+            uintptr_t prev_page = 0;
+            for (const int32_t r : idx) {
+                const uintptr_t a0 = (base + (size_t) r * row_size) & mask;
+                const uintptr_t a1 = base + (size_t) r * row_size + row_size;
+                if (a0 != prev_page) {
+                    madvise((void *) a0, a1 - a0, MADV_WILLNEED);
+                    prev_page = a0;
+                }
+            }
+        }
+    }
+#endif
+
     ggml_backend_tensor_set(rows, idx.data(), 0, idx.size()*ggml_element_size(rows));
 }
 
@@ -1332,7 +1524,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at(
             conv_states_all->nb[1],
             kv_head * row_size);
 
-    ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst));
+    // copy the strided tail directly (no intermediate cont): this is the same node sequence as
+    // llm_build_delta_net_base::build_conv_state, which the backends' fused decode paths match
+    ggml_build_forward_expand(gf, ggml_cpy(ctx0, tail, dst));
 
     return conv_input;
 }
@@ -1348,8 +1542,16 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
     const auto & qmodel = static_cast<const llama_model_qwen4exp &>(model);
     ggml_tensor * rows = nullptr;
     ggml_tensor * emb  = nullptr;
-    if (qmodel.ple_disk) {
-        // table on disk: the gather happened in set_input and arrives as an F32 input,
+
+    // A host-resident table (memory-mapped or loaded into RAM) is gathered on the host too: a
+    // get_rows node on the CPU backend would cut the graph into three splits (GPU, CPU, GPU) with
+    // an extra graph launch and a device sync per ubatch. Only a table that lives on a device is
+    // gathered by the graph.
+    const bool host_gather = qmodel.ple_disk ||
+        (model.per_layer_tok_embd && model.per_layer_tok_embd->buffer && ggml_backend_buffer_is_host(model.per_layer_tok_embd->buffer));
+
+    if (host_gather) {
+        // the gather happens in set_input and arrives as an F32 input,
         // token-major, which reshapes to what the per-head gathers + concat produce
         ple_inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.ple_head_dim, n_heads * n_tokens);
         ggml_set_input(ple_inp->embd);
@@ -1384,11 +1586,13 @@ ggml_tensor * llama_model_qwen4exp::graph::build_ple(
 
     // both norms group over one hc stream, with a weight over the whole hc*n_embd layout
     auto grouped_norm = [&](ggml_tensor * x, ggml_tensor * w) {
+        // gamma viewed per stream so that rms_norm and mul are adjacent nodes (fusable)
+        ggml_tensor * w_hc = ggml_reshape_2d(ctx0, w, n_embd, hc);
+        ggml_build_forward_expand(gf, w_hc);
         ggml_tensor * t = ggml_reshape_3d(ctx0, x, n_embd, hc, n_tokens);
         t = ggml_rms_norm(ctx0, t, hparams.f_norm_rms_eps);
-        t = ggml_reshape_2d(ctx0, t, hc_dim, n_tokens);
-        t = ggml_mul(ctx0, t, w);
-        return ggml_reshape_3d(ctx0, t, n_embd, hc, n_tokens);
+        t = ggml_mul(ctx0, t, w_hc);
+        return t;
     };
 
     key = grouped_norm(key, model.layers[il].ple_norm_key);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 6f917f542db6..4002cb90b23c 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -3608,35 +3608,38 @@ struct test_rms_norm_back : public test_case {
     }
 };
 
-// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ADD
+// GGML_OP_RMS_NORM + GGML_OP_MUL [+ GGML_OP_ADD]
 struct test_rms_norm_mul_add : public test_case {
     const ggml_type type;
     const std::array<int64_t, 4> ne;
     const float eps;
     const bool broadcast;
     const bool multi_add; // test a sequence of adds feeding into rms_norm
+    const bool add;
+    const bool weights;
 
     std::string op_desc(ggml_tensor * t) override {
         GGML_UNUSED(t);
-        return "RMS_NORM_MUL_ADD";
+        return add ? "RMS_NORM_MUL_ADD" : "RMS_NORM_MUL";
     }
 
     bool run_whole_graph() override { return true; }
 
     std::string vars() override {
-        return VARS_TO_STR5(type, ne, eps, broadcast, multi_add);
+        return VARS_TO_STR7(type, ne, eps, broadcast, multi_add, add, weights);
     }
 
     test_rms_norm_mul_add(ggml_type type = GGML_TYPE_F32,
             std::array<int64_t, 4> ne = {64, 5, 4, 3},
-            float eps = 1e-6f, bool broadcast = false, bool multi_add = false)
-        : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add) {}
+            float eps = 1e-6f, bool broadcast = false, bool multi_add = false, bool add = true, bool weights = false)
+        : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add), add(add), weights(weights) {}
 
     ggml_tensor * build_graph(ggml_context * ctx) override {
         std::array<int64_t, 4> broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4};
+        std::array<int64_t, 4> weight_dims = {ne[0], 1, 1, 1};
 
         ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data());
-        ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data());
+        ggml_tensor * b = ggml_new_tensor(ctx, type, 4, weights ? weight_dims.data() : ne.data());
         ggml_tensor * c = ggml_new_tensor(ctx, type, 4, ne.data());
 
         ggml_set_param(a);
@@ -3651,7 +3654,10 @@ struct test_rms_norm_mul_add : public test_case {
         if (multi_add) {
             a = ggml_add(ctx, ggml_add(ctx, a, b), c);
         }
-        ggml_tensor * out = ggml_add(ctx, ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b), c);
+        ggml_tensor * out = ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b);
+        if (add) {
+            out = ggml_add(ctx, out, c);
+        }
         ggml_set_name(out, "out");
 
         return out;
@@ -3765,6 +3771,86 @@ struct test_relu_sqr : public test_case {
     }
 };
 
+static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats);
+
+struct test_weighted_expert_sum : public test_case {
+    const int64_t n_embd;
+    const int64_t n_expert_used;
+    const int64_t n_tokens;
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "WEIGHTED_EXPERT_SUM";
+    }
+
+    bool run_whole_graph() override { return true; }
+    double max_nmse_err() override { return 5e-4; }
+
+    std::string vars() override {
+        return VARS_TO_STR3(n_embd, n_expert_used, n_tokens);
+    }
+
+    test_weighted_expert_sum(int64_t n_embd, int64_t n_expert_used, int64_t n_tokens)
+        : n_embd(n_embd), n_expert_used(n_expert_used), n_tokens(n_tokens) {}
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        const int64_t k = 32;
+        const int64_t n_mats = std::max<int64_t>(16, n_expert_used);
+        ggml_tensor * expert_weights = ggml_new_tensor_3d(ctx, GGML_TYPE_Q8_0, k, n_embd, n_mats);
+        ggml_tensor * ids_all = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n_tokens);
+        ggml_set_name(ids_all, "ids");
+        ggml_tensor * ids = ggml_view_2d(ctx, ids_all, n_expert_used, n_tokens, ids_all->nb[1], 0);
+        ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_expert_used, n_tokens);
+        ggml_tensor * experts = ggml_mul_mat_id(ctx, expert_weights, input, ids);
+        ggml_tensor * weights = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, n_expert_used, n_tokens);
+        ggml_tensor * weighted = ggml_mul(ctx, experts, weights);
+        ggml_build_forward_expand(gf, weighted);
+
+        std::vector<ggml_tensor *> views;
+        views.reserve(n_expert_used);
+        for (int64_t i = 0; i < n_expert_used; ++i) {
+            ggml_tensor * view = ggml_view_2d(ctx, weighted, n_embd, n_tokens, weighted->nb[2], i * weighted->nb[1]);
+            ggml_build_forward_expand(gf, view);
+            views.push_back(view);
+        }
+
+        ggml_tensor * out = views[0];
+        for (int64_t i = 1; i < n_expert_used; ++i) {
+            out = ggml_add(ctx, out, views[i]);
+            ggml_build_forward_expand(gf, out);
+        }
+        return out;
+    }
+
+    void initialize_tensors(ggml_context * ctx) override {
+        init_mul_mat_id_tensors(ctx, std::max<int64_t>(16, n_expert_used));
+    }
+};
+
+struct test_shared_mul_add : public test_case {
+    const int64_t n_embd;
+    const int64_t n_tokens;
+
+    test_shared_mul_add(int64_t n_embd, int64_t n_tokens) : n_embd(n_embd), n_tokens(n_tokens) {}
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "SHARED_MUL_ADD";
+    }
+
+    std::string vars() override { return VARS_TO_STR2(n_embd, n_tokens); }
+    bool run_whole_graph() override { return true; }
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        ggml_tensor * src = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens);
+        ggml_tensor * gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, n_tokens);
+        ggml_tensor * other = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens);
+        ggml_tensor * residual = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens);
+        ggml_tensor * mul = ggml_mul(ctx, src, gate);
+        return ggml_add(ctx, ggml_add(ctx, other, mul), residual);
+    }
+};
+
 // GGML_OP_UNARY(SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation).
 // `layout` and `tail` are used for fallback cases where fusion must be skipped
 struct test_unary_mul : public test_case {
@@ -4660,6 +4746,72 @@ struct test_mul_mat : public test_case {
     }
 };
 
+struct test_mul_mat_exact_batch : public test_case {
+    const ggml_type type_a;
+    const int64_t m;
+    const int64_t n;
+    const int64_t k;
+
+    test_mul_mat_exact_batch(ggml_type type_a, int64_t m, int64_t n, int64_t k) : type_a(type_a), m(m), n(n), k(k) {}
+
+    std::string vars() override {
+        return VARS_TO_STR4(type_a, m, n, k);
+    }
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        ggml_tensor * a = ggml_new_tensor_2d(ctx, type_a, k, m);
+        ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n);
+        ggml_tensor * batch = ggml_mul_mat(ctx, a, b);
+        ggml_mul_mat_set_hint(batch, GGML_HINT_EXACT_BATCH);
+
+        ggml_tensor * diff = nullptr;
+        for (int64_t col = 0; col < n; ++col) {
+            ggml_tensor * b_col = ggml_view_2d(ctx, b, k, 1, b->nb[1], col*b->nb[1]);
+            ggml_tensor * single = ggml_mul_mat(ctx, a, b_col);
+            ggml_tensor * batch_col = ggml_view_2d(ctx, batch, m, 1, batch->nb[1], col*batch->nb[1]);
+            ggml_tensor * col_diff = ggml_sub(ctx, batch_col, single);
+            diff = diff == nullptr ? col_diff : ggml_concat(ctx, diff, col_diff, 1);
+        }
+        return diff;
+    }
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return ggml_op_name(GGML_OP_MUL_MAT);
+    }
+
+    double err(const float * a, const float * b, size_t n) override {
+        GGML_UNUSED(b);
+        double max_abs = 0.0;
+        for (size_t i = 0; i < n; ++i) {
+            max_abs = std::max(max_abs, std::abs((double) a[i]));
+        }
+        return max_abs;
+    }
+
+    double max_err() override { return 0.0; }
+
+    bool run_whole_graph() override { return true; }
+};
+
+struct test_mul_mat_pair : public test_case {
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "MUL_MAT_PAIR";
+    }
+
+    std::string vars() override { return {}; }
+    bool run_whole_graph() override { return true; }
+    double max_nmse_err() override { return 5e-4; }
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        ggml_tensor * a0 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, 256, 32);
+        ggml_tensor * a1 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, 256, 32);
+        ggml_tensor * b  = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 256, 129);
+        return ggml_add(ctx, ggml_mul_mat(ctx, a0, b), ggml_mul_mat(ctx, a1, b));
+    }
+};
+
 // GGML_HINT_SRC0_IS_HADAMARD
 struct test_mul_mat_hadamard : public test_mul_mat {
     test_mul_mat_hadamard(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,
@@ -4800,6 +4952,98 @@ struct test_mul_mat_id : public test_case {
     }
 };
 
+struct test_swiglu_q8_mmq : public test_case {
+    const bool use_id;
+    const int n_mats;
+    const int n_used;
+    const int64_t m;
+    const int64_t n;
+    const int64_t k;
+    const ggml_type type_down;
+    const bool zero_inputs;
+
+    test_swiglu_q8_mmq(bool use_id, int n_mats, int n_used, int64_t m, int64_t n, int64_t k,
+            ggml_type type_down = GGML_TYPE_Q8_0, bool zero_inputs = false)
+        : use_id(use_id), n_mats(n_mats), n_used(n_used), m(m), n(n), k(k), type_down(type_down), zero_inputs(zero_inputs) {}
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "SWIGLU_Q8_MMQ";
+    }
+
+    std::string vars() override {
+        return VARS_TO_STR8(use_id, n_mats, n_used, m, n, k, type_down, zero_inputs);
+    }
+
+    bool run_whole_graph() override { return true; }
+    double max_nmse_err() override { return 5e-4; }
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        if (!use_id) {
+            ggml_tensor * gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n);
+            ggml_tensor * up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n);
+            ggml_tensor * down = ggml_new_tensor_2d(ctx, type_down, k, m);
+            return ggml_mul_mat(ctx, down, ggml_swiglu_split(ctx, gate, up));
+        }
+
+        ggml_tensor * gate = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n);
+        ggml_tensor * up = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n);
+        ggml_tensor * down = ggml_new_tensor_3d(ctx, type_down, k, m, n_mats);
+        ggml_tensor * ids_all = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n);
+        ggml_set_name(ids_all, "ids");
+        ggml_tensor * ids = ggml_view_2d(ctx, ids_all, n_used, n, ids_all->nb[1], 0);
+        return ggml_mul_mat_id(ctx, down, ggml_swiglu_split(ctx, gate, up), ids);
+    }
+
+    void initialize_tensors(ggml_context * ctx) override {
+        init_mul_mat_id_tensors(ctx, n_mats);
+        if (zero_inputs) {
+            for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
+                if (t->type != GGML_TYPE_I32 && !ggml_is_view_op(t->op)) {
+                    init_tensor_uniform(t, 0.0f, 0.0f);
+                }
+            }
+        }
+    }
+};
+
+// Qwen3.8-Flash-Next decode hot path: IQ3_S gate and up projections are
+// fused with SwiGLU by ggml-cuda, with 512 experts and 10 selected experts.
+struct test_swiglu_iq3_mmvq : public test_case {
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "SWIGLU_IQ3_MMVQ";
+    }
+
+    std::string vars() override {
+        return "n_mats=512,n_used=10,m=640,n=1,k=2560";
+    }
+
+    bool run_whole_graph() override { return true; }
+    double max_nmse_err() override { return 5e-4; }
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        constexpr int n_mats = 512;
+        constexpr int n_used = 10;
+        constexpr int64_t m = 640;
+        constexpr int64_t n = 1;
+        constexpr int64_t k = 2560;
+
+        ggml_tensor * gate_w = ggml_new_tensor_3d(ctx, GGML_TYPE_IQ3_S, k, m, n_mats);
+        ggml_tensor * up_w   = ggml_new_tensor_3d(ctx, GGML_TYPE_IQ3_S, k, m, n_mats);
+        ggml_tensor * input  = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n);
+        ggml_tensor * ids_all = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n);
+        ggml_set_name(ids_all, "ids");
+        ggml_tensor * ids = ggml_view_2d(ctx, ids_all, n_used, n, ids_all->nb[1], 0);
+        ggml_tensor * gate = ggml_mul_mat_id(ctx, gate_w, input, ids);
+        ggml_tensor * up   = ggml_mul_mat_id(ctx, up_w,   input, ids);
+        return ggml_swiglu_split(ctx, gate, up);
+    }
+
+    void initialize_tensors(ggml_context * ctx) override {
+        init_mul_mat_id_tensors(ctx, 512);
+    }
+};
+
 // GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL
 struct test_mul_mat_id_fusion : public test_case {
     const ggml_type type_a;
@@ -4882,6 +5126,107 @@ struct test_mul_mat_id_fusion : public test_case {
 };
 
 // GGML_OP_OUT_PROD
+// GGML_OP_MUL_MAT group: independent single-column matvecs on one activation vector
+// (RDNA3.5 CUDA launches independent Q8_0 segments as one grouped kernel)
+struct test_mmv_group : public test_case {
+    const int64_t k;
+    const int64_t m_q8;   // rows of a plain Q8_0 segment (0 = none)
+    const int64_t m_glu;  // rows of a Q8_0 gate/up + swiglu segment (0 = none)
+
+    std::string vars() override { return VARS_TO_STR3(k, m_q8, m_glu); }
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "MMV_GROUP";
+    }
+
+    bool run_whole_graph() override { return true; }
+
+    double max_nmse_err() override { return 5e-4; }
+
+    test_mmv_group(int64_t k, int64_t m_q8, int64_t m_glu)
+        : k(k), m_q8(m_q8), m_glu(m_glu) {}
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        ggml_tensor * y = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, 1);
+        ggml_set_name(y, "y");
+
+        ggml_tensor * out = nullptr;
+        auto append = [&](ggml_tensor * t) {
+            out = out ? ggml_concat(ctx, out, t, 0) : t;
+        };
+
+        if (m_q8 > 0) {
+            ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m_q8);
+            ggml_set_name(w, "w_q8");
+            append(ggml_mul_mat(ctx, w, y));
+        }
+        if (m_glu > 0) {
+            ggml_tensor * wg = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m_glu);
+            ggml_tensor * wu = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m_glu);
+            ggml_set_name(wg, "w_gate");
+            ggml_set_name(wu, "w_up");
+            append(ggml_swiglu_split(ctx, ggml_mul_mat(ctx, wg, y), ggml_mul_mat(ctx, wu, y)));
+        }
+        ggml_set_name(out, "out");
+        return out;
+    }
+};
+
+// qwen4exp hyper-connection up projection + stream mix chain (decode):
+//   silu(scale(lo)) -> mul_mat(W_up) -> sigmoid -> mul(xn) -> reshape -> view/cont -> adds -> scale
+struct test_hc_up_mix : public test_case {
+    const ggml_type type_w;
+    const int64_t k;       // low-rank width (columns of W_up)
+    const int64_t n_embd;
+    const int64_t hc;
+    const int64_t n_tokens;
+    const bool external_consumer;
+
+    std::string vars() override { return VARS_TO_STR6(type_w, k, n_embd, hc, n_tokens, external_consumer); }
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "HC_UP_MIX";
+    }
+
+    bool run_whole_graph() override { return true; }
+
+    double max_nmse_err() override { return 5e-4; }
+
+    test_hc_up_mix(ggml_type type_w, int64_t k, int64_t n_embd, int64_t hc, int64_t n_tokens,
+            bool external_consumer = false)
+        : type_w(type_w), k(k), n_embd(n_embd), hc(hc), n_tokens(n_tokens), external_consumer(external_consumer) {}
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        ggml_tensor * lo = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n_tokens);
+        ggml_set_name(lo, "lo");
+        ggml_tensor * xn = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hc * n_embd, n_tokens);
+        ggml_set_name(xn, "xn");
+        ggml_tensor * w = ggml_new_tensor_2d(ctx, type_w, k, hc * n_embd);
+        ggml_set_name(w, "w_up");
+
+        ggml_tensor * act  = ggml_silu(ctx, ggml_scale(ctx, lo, 1.0f / (float) hc));
+        ggml_tensor * gate = ggml_sigmoid(ctx, ggml_mul_mat(ctx, w, act));
+        ggml_tensor * gated = ggml_mul(ctx, xn, gate);
+        gated = ggml_reshape_3d(ctx, gated, n_embd, hc, n_tokens);
+        ggml_tensor * mixed = ggml_view_2d(ctx, gated, n_embd, n_tokens, ggml_row_size(gated->type, n_embd) * hc, 0);
+        mixed = ggml_cont(ctx, mixed);
+        ggml_tensor * external = external_consumer ? mixed : nullptr;
+        for (int64_t c = 1; c < hc; ++c) {
+            ggml_tensor * s = ggml_view_2d(ctx, gated, n_embd, n_tokens, ggml_row_size(gated->type, n_embd) * hc,
+                    ggml_row_size(gated->type, n_embd) * c);
+            mixed = ggml_add(ctx, mixed, s);
+        }
+        mixed = ggml_scale(ctx, mixed, 1.0f / (float) hc);
+        ggml_set_name(mixed, "mixed");
+        if (external) {
+            mixed = ggml_add(ctx, mixed, external);
+        }
+        return mixed;
+    }
+};
+
 struct test_out_prod : public test_case {
     const ggml_type type_a;
     const ggml_type type_b;
@@ -6051,7 +6396,7 @@ struct test_concat : public test_case {
     const std::array<int64_t, 4> ne_a;
     const int64_t ne_b_d;
     const int dim;
-    const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim))
+    const int v; // view (bits 0-3: non-contiguous views, bit 4: transposed b)
 
     std::string vars() override {
         return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v);
@@ -6086,7 +6431,15 @@ struct test_concat : public test_case {
             ggml_set_name(a, "a");
         }
         ggml_tensor * b;
-        if (v & 2) {
+        if (v & 16) {
+            auto ne = ne_b;
+            std::swap(ne[0], ne[1]);
+            b = ggml_new_tensor(ctx, type, 4, ne.data());
+            ggml_set_name(b, "b");
+
+            b = ggml_transpose(ctx, b);
+            ggml_set_name(b, "transposed_b");
+        } else if (v & 2) {
             auto ne = ne_b; ne[0] *= 3; ne[1] *= 2; ne[2] *= 4;
             b = ggml_new_tensor(ctx, type, 4, ne.data());
             ggml_set_name(b, "b");
@@ -6248,7 +6601,13 @@ struct test_top_k : public test_case {
                 diff += std::fabs(b[i] - ib[i]);
             }
 
-            return diff + jdst(ia.data(), ib.data(), n);
+            if (n != (size_t) (ggml_nrows(input) * k)) {
+                return nmse(a, b, n);
+            }
+            for (int64_t r = 0; r < ggml_nrows(input); ++r) {
+                diff += jdst(ia.data() + r * k, ib.data() + r * k, k);
+            }
+            return diff;
         }
     }
 
@@ -6266,8 +6625,7 @@ struct test_top_k : public test_case {
     }
 
     void initialize_tensors(ggml_context * ctx) override {
-        std::random_device rd;
-        std::default_random_engine rng(rd());
+        std::default_random_engine rng(0x5eed);
         for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
             int tie_denom = std::max(1, std::min(10, k / 2));
             for (int64_t r = 0; r < ggml_nrows(t); r++) {
@@ -6460,17 +6818,41 @@ struct test_topk_moe : public test_case {
     // Verify two outputs
     std::vector<ggml_tensor *> fusion_test_nodes() override { return { selected_experts, weights }; }
 
-    // allow output in arbitrary order
-    double err(const float * a, const float * b, size_t n) override {
-        std::vector<float> a2(n);
-        std::vector<float> b2(n);
-        for (size_t i = 0; i < n; ++i) {
-            a2[i] = a[i];
-            b2[i] = b[i];
+    void initialize_tensors(ggml_context * ctx) override {
+        std::random_device rd;
+        std::default_random_engine rng(rd());
+
+        for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
+            if (t->op != GGML_OP_NONE) {
+                continue;
+            }
+            if (strcmp(t->name, "exp_probs_b") == 0) {
+                std::vector<float> data(t->ne[0]);
+                const float amplitude =
+                    gating_func == GATING_FUNC_SOFTMAX         ? 0.1f :
+                    gating_func == GATING_FUNC_SIGMOID         ? 0.5f :
+                    gating_func == GATING_FUNC_SOFTMAX_WEIGHT  ? 4.5f : 1.0f;
+                for (int64_t i = 0; i < t->ne[0]; ++i) {
+                    data[i] = i < t->ne[0] / 2 ? amplitude : -amplitude;
+                }
+                std::shuffle(data.begin(), data.end(), rng);
+                ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t));
+                continue;
+            }
+
+            for (int64_t row = 0; row < ggml_nrows(t); ++row) {
+                std::vector<float> data(t->ne[0]);
+                for (int64_t i = 0; i < t->ne[0]; ++i) {
+                    data[i] = -4.0f + 8.0f * i / std::max<int64_t>(1, t->ne[0] - 1);
+                }
+                std::shuffle(data.begin(), data.end(), rng);
+                ggml_backend_tensor_set(t, data.data(), row * t->nb[1], t->ne[0] * sizeof(float));
+            }
         }
-        std::sort(a2.begin(), a2.end());
-        std::sort(b2.begin(), b2.end());
-        return nmse(a2.data(), b2.data(), n);
+    }
+
+    double err(const float * a, const float * b, size_t n) override {
+        return nmse(a, b, n);
     }
 };
 
@@ -6948,6 +7330,53 @@ struct test_l2_norm : public test_case {
     }
 };
 
+struct test_l2_norm_dual : public test_case {
+    const int64_t n_heads;
+    const int64_t n_tokens;
+    const int64_t n_seqs;
+    const float eps_q;
+    const float eps_k;
+
+    ggml_tensor * q_norm = nullptr;
+    ggml_tensor * k_norm = nullptr;
+
+    test_l2_norm_dual(int64_t n_heads, int64_t n_tokens, int64_t n_seqs, float eps_q, float eps_k)
+        : n_heads(n_heads), n_tokens(n_tokens), n_seqs(n_seqs), eps_q(eps_q), eps_k(eps_k) {}
+
+    std::string op_desc(ggml_tensor * t) override {
+        GGML_UNUSED(t);
+        return "L2_NORM_DUAL_S128";
+    }
+
+    std::string vars() override {
+        return VARS_TO_STR5(n_heads, n_tokens, n_seqs, eps_q, eps_k);
+    }
+
+    bool run_whole_graph() override { return true; }
+
+    std::vector<ggml_tensor *> fusion_test_nodes() override { return { q_norm, k_norm }; }
+
+    ggml_tensor * build_graph(ggml_context * ctx) override {
+        constexpr int64_t head_size = 128;
+        ggml_tensor * qkv = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 3*head_size*n_heads, n_tokens, n_seqs);
+        ggml_set_name(qkv, "qkv");
+
+        ggml_tensor * q = ggml_view_4d(ctx, qkv, head_size, n_heads, n_tokens, n_seqs,
+            head_size*sizeof(float), qkv->nb[1], qkv->nb[2], 0);
+        ggml_tensor * k = ggml_view_4d(ctx, qkv, head_size, n_heads, n_tokens, n_seqs,
+            head_size*sizeof(float), qkv->nb[1], qkv->nb[2], head_size*n_heads*sizeof(float));
+        ggml_set_name(q, "q");
+        ggml_set_name(k, "k");
+
+        q_norm = ggml_l2_norm(ctx, q, eps_q);
+        k_norm = ggml_l2_norm(ctx, k, eps_k);
+        ggml_set_name(q_norm, "q_norm");
+        ggml_set_name(k_norm, "k_norm");
+
+        return ggml_add(ctx, q_norm, k_norm);
+    }
+};
+
 // GGML_OP_ACC
 struct test_acc : public test_case {
     const ggml_type type;
@@ -9188,6 +9617,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }
 
+    test_cases.emplace_back(new test_l2_norm_dual(16, 1, 1, 1e-6f, 1e-6f));
+    test_cases.emplace_back(new test_l2_norm_dual(4, 3, 2, 1e-6f, 8.0f));
+
     // in-place tests
     test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, false, 1e-6f, true));
 
@@ -9208,6 +9640,24 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false));
     }
 
+    test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {127, 48, 1, 1}, 1e-6f, false, false, false, true));
+    test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {128, 48, 1024, 1}, 1e-6f, false, false, false, true));
+    test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {129, 48, 1, 1}, 1e-6f, false, false, false, true));
+
+    test_cases.emplace_back(new test_weighted_expert_sum(127, 2, 3));
+    test_cases.emplace_back(new test_weighted_expert_sum(128, 4, 5));
+    test_cases.emplace_back(new test_weighted_expert_sum(2048, 8, 32));
+    test_cases.emplace_back(new test_weighted_expert_sum(128, 16, 3));
+    test_cases.emplace_back(new test_shared_mul_add(127, 3));
+    test_cases.emplace_back(new test_shared_mul_add(2048, 32));
+    test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2048, 32, 512));
+    test_cases.emplace_back(new test_swiglu_q8_mmq(true, 16, 8, 2048, 32, 512));
+    test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 128, 3, 256, GGML_TYPE_Q8_0, true));
+    test_cases.emplace_back(new test_swiglu_q8_mmq(true, 16, 8, 128, 3, 256, GGML_TYPE_Q8_0, true));
+    test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2560, 128, 640));
+    test_cases.emplace_back(new test_swiglu_q8_mmq(true, 512, 10, 2560, 128, 640));
+    test_cases.emplace_back(new test_swiglu_q8_mmq(true, 512, 10, 2560, 128, 640, GGML_TYPE_IQ4_NL));
+
     for (auto multi_add : {false, true}) {
         for (auto set_rows : {false, true}) {
             for (auto broadcast : {false, true}) {
@@ -9312,6 +9762,77 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1}));
 #endif
 
+    // qwen4exp hyper-connection up projection + stream mix (fused on RDNA3.5 for one token)
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 320, 2560, 4, 1));
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 320, 2560, 4, 3));
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 256, 1000, 3, 1));
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 512, 640, 4, 1));
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_F32,  320, 2560, 4, 1));
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 64, 64, 14, 3));
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 64, 64, 16, 3));
+    test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 64, 64, 16, 3, true));
+
+    // grouped single-column matvecs on one activation vector (qwen4exp decode: GDN qkv/z/beta/alpha,
+    // hyper-connection down + inject, shared-expert gate/up + router)
+    test_cases.emplace_back(new test_mmv_group(2560, 10240, 6144));
+    test_cases.emplace_back(new test_mmv_group(2560, 1024, 6144));
+
+    // 256-expert MoE with 4/16/32/128 rows per expert, exercises the RDNA3.5 mul_mat_id tile selection
+    // Qwen3.8-Flash-Next (qwen4exp) UD-IQ4_XS experts: 512 experts, 10 active, gate/up IQ3_S [2560 -> 640], down IQ4_NL [640 -> 2560]
+    for (int n : {1, 8, 64, 512, 2048}) {
+        test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ3_S,  GGML_TYPE_F32, 512, 10, false, 640,  n, 2560));
+        test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ4_NL, GGML_TYPE_F32, 512, 10, false, 2560, n, 640));
+        test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0,   GGML_TYPE_F32, 512, 10, false, 2560, n, 640));
+    }
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 1024, 2048, 3072));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 3072, 2048, 1024));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512));
+    // 128-expert J48 dispatch boundaries. Small matrix dimensions keep CPU reference time practical.
+    for (ggml_type type_a : {GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0}) {
+        test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 128, 8, false, 64, 1521, 256));
+    }
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 64, 1520, 256));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 64, 2545, 256));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 64, 2561, 256));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 1984, 2560));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 2048, 2560));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 2112, 2560));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 2560, 2048, 768));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 512, 8, false, 2560, 1984, 768));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 512, 8, false, 2560, 2048, 768));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 512, 8, false, 2560, 2112, 768));
+
     for (ggml_type type_a : all_types) {
         for (int i = 1; i < 10; ++i) {
             test_cases.emplace_back(new test_mul_mat(type_a,    GGML_TYPE_F32, 16,  i, 1*256, { 1,  1}, {1, 1}));
@@ -9332,6 +9853,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
 
+    for (int64_t n : {127, 128, 129}) {
+        test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 128, n, 5120, {1, 1}, {1, 1}));
+    }
+    for (int64_t n : {2, 3, 4, 5, 8, 9, 16}) {
+        test_cases.emplace_back(new test_mul_mat_exact_batch(GGML_TYPE_BF16, 128, n, 5120));
+    }
+    for (ggml_type type_a : {GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0,
+                             GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ3_S}) {
+        test_cases.emplace_back(new test_mul_mat_exact_batch(type_a, 128, 4, 5120));
+    }
+
     // m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above
     for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) {
         test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1}));
@@ -9462,6 +9994,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     }
 
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 1024, 32, 1056, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1024, 32, 1056, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q2_K, GGML_TYPE_F32, 1024, 32, 1280, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8, 2, true, 128, 32, 1056));
 
     // K not a multiple of 32
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32,  65, {1, 1}, {1, 1}));
@@ -9526,6 +10062,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, k));
     }
     test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3));
+    test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8, 4, false, 512, 129, 256, 2));
+    test_cases.emplace_back(new test_mul_mat_pair());
 
     // gpt-oss issue with Vulkan mmq_id
     test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_MXFP4, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880));
@@ -9843,6 +10381,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }
 
+    // Transposed src1 path: guard boundary, Qwen validation prompt, and prefill tile edges.
+    for (int64_t n_tokens : { 31, 32, 115, 512 }) {
+        test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {64, 127, 2, 1}, n_tokens, 0, 16));
+    }
+
     for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) {
         for (uint32_t i = 4; i <= 1024*1024; i *= 2) {
             test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1}));
@@ -9875,6 +10418,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
             }
         }
     }
+    // QSA (qwen4exp / DeepSeek-V3.2 style indexer): k == n_kv while the context fits the budget, [n_kv, n_tokens, 1, n_stream]
+    for (int n : {256, 512, 1024, 2048, 2304}) {
+        for (int nt : {1, 2, 512}) {
+            test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {n, nt, 1, 1}, std::min(n, 2051)));
+        }
+    }
     for (int k : {4, 8, 16, 32}) {
         for (int nrows : {1, 8, 16}) {
             test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {202048, nrows, 1, 1}, k));
@@ -10071,6 +10620,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
             }
         }
     }
+    test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {4, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
+    test_cases.emplace_back(new test_flash_attn_ext(256, 256, 8, {4, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
+    test_cases.emplace_back(new test_flash_attn_ext(256, 256, 8, {6, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
 
     for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 320, 512, 576 }) {
         for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) {
@@ -10242,6 +10794,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
                     test_cases.emplace_back(new test_topk_moe({129, 1, 1, 1}, 128, with_norm, bias_probs, gate, scale_w));
                     test_cases.emplace_back(new test_topk_moe({160, 4, 1, 1}, 160, with_norm, bias_probs, gate, scale_w));
                     test_cases.emplace_back(new test_topk_moe({256, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w)); // Used by DeepSeek-V4
+                    test_cases.emplace_back(new test_topk_moe({512, 22, 1, 1}, 10, with_norm, bias_probs, gate, scale_w)); // Used by Qwen3.8-Flash-Next (qwen4exp)
                     test_cases.emplace_back(new test_topk_moe({288, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); // Used by StepFun 3.7
                 }
             }
@@ -10278,6 +10831,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64,  64, 1, 1, false, true));
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64,  33, 1, 1, false, true));
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true));
+    for (int n_tokens : {15, 16, 17, 64, 65, 2048}) {
+        test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, n_tokens, 1, 1, false, true));
+    }
+    // Multiple sequences must remain on the generic fallback.
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 2, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 15, 1, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 16, 1, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 17, 1, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 16, 1));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 17, 1));
 
     // K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first
     // (slot 0 = final state, slot s = state s tokens back).
@@ -10330,6 +10895,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
 static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
     std::vector<std::unique_ptr<test_case>> test_cases;
 
+    // Qwen3.8-Flash-Next decode hot path: fused gate/up uses this exact shape.
+    test_cases.emplace_back(new test_mul_mat_id(
+        GGML_TYPE_IQ3_S, GGML_TYPE_F32, 512, 10, false, 640, 1, 2560));
+    test_cases.emplace_back(new test_swiglu_iq3_mmvq());
+
     // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands
     // note: same bytes either way, so a backend that indexes them differently shows it here
     for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
@@ -10504,6 +11074,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
         }
     }
 
+    // qwen3.5/3.6-35b-a3b (256 experts, 8 used): up/gate and down projections at ubatch 512/1024/2048
+    for (int bs : {512, 1024, 2048}) {
+        for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}) {
+            test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false,  512, bs, 2048));
+            test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 2048, bs,  512));
+        }
+    }
+
     // qwen3-30b-a3b
     for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) {
         for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) {
@@ -10534,6 +11112,28 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
         }
     }
 
+    // Qwen3.6-35B-A3B prompt processing (256 experts, top-8, expert width 512, ubatch 2048):
+    //     gate/up: m=512, k=2048; down: m=2048, k=512
+    for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_Q5_K, GGML_TYPE_Q4_K, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_MXFP4}) {
+        test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false,  512, 2048, 2048));
+        test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 2048, 2048,  512));
+    }
+
+    // Qwen3.6-35B-A3B token generation (batch 1):
+    //     routed experts gate/up (fused) and down, shared expert, GDN qkv/gate/out, attention q/o, output head
+    for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_Q4_K, GGML_TYPE_BF16}) {
+        test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false,  512, 1, 2048));
+        test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, GGML_TYPE_F32, 256, 8, false,  512, 1, 2048, 1));
+        test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 2048, 1,  512));
+        test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32,  512, 1, 2048, {1, 1}, {1, 1}));
+        test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 2048, 1,  512, {1, 1}, {1, 1}));
+        test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 8192, 1, 2048, {1, 1}, {1, 1}));
+        test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, 1, 2048, {1, 1}, {1, 1}));
+        test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 2048, 1, 4096, {1, 1}, {1, 1}));
+    }
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 248320, 1, 2048, {1, 1}, {1, 1}));
+    test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32,  GGML_TYPE_F32,    256, 1, 2048, {1, 1}, {1, 1}));
+
     for (int K : {3, 5}) {
         for (int IC : {256, 2560}) {
             for (int IW_IH : {32, 64, 256}) {
@@ -10571,6 +11171,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
     test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
     test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
 
+    for (int64_t kv : {4096, 8192, 12000, 16000, 24000, 32000, 64000}) {
+        test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {8, 1}, kv, 2048, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
+        test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {8, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
+    }
+
     for (int kv : { 4096, 8192, 16384, }) {
         for (int hs : { 64, 128, }) {
             for (int nr : { 1, 4, }) {
@@ -10693,12 +11298,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1)); // PP-2048
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 48, 128, 2048, 1)); // qwen4exp H48/S128 PP-2048
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 1024, 1)); // H64/S128 PP-1024
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 2048, 1)); // H64/S128 PP-2048
     // Small model configs (fewer heads = less GPU occupancy for autoregressive)
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1));   // 4h PP-64
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1));  // 4h PP-256
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1));  // 4h PP-512
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024
     test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 1, 1, false, true)); // Ling-like KDA PP-64
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 15, 1, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 16, 1, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 17, 1, 1, false, true));
+    test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1, 1, false, true));
 
     // lightning_indexer
     for (int kv : { 256, 4096, 65536 }) {
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 81228c18cc79..71910519a837 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -3220,7 +3220,7 @@ struct server_context_impl {
                                 llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id));
 
                         if (use_ckpt_dft) {
-                            slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                            slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
                         }
 
                         slot.spec_prompt = slot.prompt.tokens.get_text_tokens();
@@ -3259,7 +3259,7 @@ struct server_context_impl {
 
             if (ctx_dft) {
                 if (use_ckpt_dft) {
-                    ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                    ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
                 }
 
                 if (!llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, ckpt.pos_max + 1, -1)) {
@@ -3278,7 +3278,7 @@ struct server_context_impl {
                 if (use_ckpt_tgt) {
                     //const int64_t t_start = ggml_time_us();
 
-                    ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                    ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
 
                     //const int64_t t_total = ggml_time_us() - t_start;
                     //printf("checkpoint total: %f ms\n", t_total / 1000.0);
@@ -3290,7 +3290,7 @@ struct server_context_impl {
                 }
 
                 if (use_ckpt_dft) {
-                    ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                    ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
                 }
             }
         });
@@ -4151,10 +4151,10 @@ struct server_context_impl {
 
                         SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size());
 
-                        ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                        ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
 
                         if (slot.ctx_dft) {
-                            ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
+                            ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE);
                         }
 
                         slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1);