From 5b6308d71a5dff94995af67a2598f230ee8178fe Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:23:16 +0200 Subject: [PATCH 01/30] hip: optimize quantized matmul on RDNA3.5 Tune MMQ and MMVQ scheduling, add routed expert kernels, share Q8_1 activation quantization, and add measured four-column kernels for quant formats that outperform the generic path on gfx1151. Assisted-by: OpenCode --- ggml/include/ggml.h | 1 + ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh | 69 ++-- ggml/src/ggml-cuda/mmq-q45.cu | 13 + ggml/src/ggml-cuda/mmq-q6.cu | 9 + ggml/src/ggml-cuda/mmq-q8.cu | 9 + ggml/src/ggml-cuda/mmq-vec-dot.cuh | 13 +- ggml/src/ggml-cuda/mmq.cu | 249 +++++++++++- ggml/src/ggml-cuda/mmq.cuh | 194 +++++++++- ggml/src/ggml-cuda/mmvf.cu | 102 +++-- ggml/src/ggml-cuda/mmvq.cu | 443 +++++++++++++++++++++- ggml/src/ggml-cuda/quantize.cu | 192 +++++++--- ggml/src/ggml-cuda/quantize.cuh | 7 + 12 files changed, 1144 insertions(+), 157 deletions(-) create mode 100644 ggml/src/ggml-cuda/mmq-q45.cu create mode 100644 ggml/src/ggml-cuda/mmq-q6.cu create mode 100644 ggml/src/ggml-cuda/mmq-q8.cu diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 26f31232f676..b8b01b2b8920 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -443,6 +443,7 @@ extern "C" { enum ggml_op_hint { GGML_HINT_NONE = 0, GGML_HINT_SRC0_IS_HADAMARD = 1, + GGML_HINT_EXACT_BATCH = 2, }; // model file types diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh index 180b2d9370d1..f8a9e6350926 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh @@ -79,15 +79,15 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); // --------------------------------------------------------------------------------------------- @@ -115,42 +115,42 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); // --------------------------------------------------------------------------------------------- @@ -212,7 +212,7 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); @@ -263,7 +263,7 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); @@ -288,3 +288,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); } + +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, false).nthreads == 256); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, false).I == 64); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, true).nthreads == 128); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 128, true).I == 64); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, false).nthreads == 128); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, false).I == 64); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, true).nthreads == 256); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_Q8_0, 48, true).I == 128); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_MXFP4, 128, true).nthreads == 128); +static_assert(ggml_cuda_mmq_get_config_rdna3_5(GGML_TYPE_MXFP4, 128, true).I == 64); diff --git a/ggml/src/ggml-cuda/mmq-q45.cu b/ggml/src/ggml-cuda/mmq-q45.cu new file mode 100644 index 000000000000..2b94d38ad70f --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-q45.cu @@ -0,0 +1,13 @@ +#define mul_mat_q mul_mat_q_q45 +#define launch_mul_mat_q launch_mul_mat_q_q45 +#include "mmq.cuh" +#undef launch_mul_mat_q +#undef mul_mat_q + +void launch_mul_mat_q_q4_k_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + launch_mul_mat_q_q45(ctx, args, stream); +} + +void launch_mul_mat_q_q5_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + launch_mul_mat_q_q45(ctx, args, stream); +} diff --git a/ggml/src/ggml-cuda/mmq-q6.cu b/ggml/src/ggml-cuda/mmq-q6.cu new file mode 100644 index 000000000000..ed9548f79608 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-q6.cu @@ -0,0 +1,9 @@ +#define mul_mat_q mul_mat_q_q6 +#define launch_mul_mat_q launch_mul_mat_q_q6 +#include "mmq.cuh" +#undef launch_mul_mat_q +#undef mul_mat_q + +void launch_mul_mat_q_q6_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + launch_mul_mat_q_q6(ctx, args, stream); +} diff --git a/ggml/src/ggml-cuda/mmq-q8.cu b/ggml/src/ggml-cuda/mmq-q8.cu new file mode 100644 index 000000000000..4cb9589c715c --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-q8.cu @@ -0,0 +1,9 @@ +#define mul_mat_q mul_mat_q_q8 +#define launch_mul_mat_q launch_mul_mat_q_q8 +#include "mmq.cuh" +#undef launch_mul_mat_q +#undef mul_mat_q + +void launch_mul_mat_q_q8_0_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { + launch_mul_mat_q_q8(ctx, args, stream); +} diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuh index d573433865f8..2493ebea6f24 100644 --- a/ggml/src/ggml-cuda/mmq-vec-dot.cuh +++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh @@ -152,8 +152,14 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / ggml_cuda_get_physical_warp_size(); + constexpr bool split_j = type == GGML_TYPE_Q8_0 && J == 128 && !fallback && I == 64 && nwarps == 8; + constexpr int j_group = split_j ? J/2 : J; - y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K); + const int warp_i = split_j ? threadIdx.y % 4 : threadIdx.y; + const int warp_j = split_j ? threadIdx.y / 4 : 0; + + y += (warp_j*j_group + (warp_i % ntx)*tile_C::J) * MMQ_TILE_Y_K; const int * x_qs = (const int *) x; const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K; @@ -161,7 +167,7 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( const float * y_df = (const float *) y; const half2 * y_ds = (const half2 *) y; - const int i0 = (threadIdx.y / ntx) * rows_per_warp; + const int i0 = (warp_i / ntx) * rows_per_warp; for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) { const int k0 = k00 + k01; @@ -173,7 +179,7 @@ static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma( } #pragma unroll - for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + for (int j0 = 0; j0 < j_group; j0 += ntx*tile_C::J) { tile_B B; load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); @@ -1248,4 +1254,3 @@ template static __device__ __forceinline_ } } } - diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 7fb4401489c9..de18e2064877 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -5,6 +5,62 @@ #include +void launch_mul_mat_q_q4_k_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); +void launch_mul_mat_q_q5_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); +void launch_mul_mat_q_q6_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); +void launch_mul_mat_q_q8_0_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); + +static bool use_rdna3_5_q45_id_narrow(const mmq_args & args) { + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + if ((args.type_x != GGML_TYPE_Q4_K && args.type_x != GGML_TYPE_Q5_K) || !GGML_CUDA_CC_IS_RDNA3_5(cc) || + args.ids_dst == nullptr || args.nchannels_y <= 0) { + return false; + } + + const int64_t rows_per_channel = (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y; + const bool ornith = ((args.ncols_x == 2048 && args.nrows_x == 512) || + (args.ncols_x == 512 && args.nrows_x == 2048)) && args.nchannels_x == 256 && rows_per_channel == 64; + const bool qwen122 = ((args.ncols_x == 3072 && args.nrows_x == 1024) || + (args.ncols_x == 1024 && args.nrows_x == 3072)) && args.nchannels_x == 256 && rows_per_channel == 64; + return ornith || qwen122; +} + +static bool use_rdna3_5_q5_ling(const mmq_args & args) { + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + return args.type_x == GGML_TYPE_Q5_K && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr && + ((args.ncols_x == 2560 && args.nrows_x == 768) || (args.ncols_x == 768 && args.nrows_x == 2560)) && + args.nchannels_x == 512 && args.nchannels_y > 0 && + (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 32; +} + +static bool use_rdna3_5_q6_ling_j32(const mmq_args & args) { + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + return args.type_x == GGML_TYPE_Q6_K && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr && + args.ncols_x == 768 && args.nrows_x == 2560 && args.nchannels_x == 512 && args.nchannels_y > 0 && + (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 32; +} + +static bool use_rdna3_5_q6_id_narrow(const mmq_args & args) { + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + return args.type_x == GGML_TYPE_Q6_K && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr && + ((args.ncols_x == 2048 && args.nrows_x == 512) || (args.ncols_x == 512 && args.nrows_x == 2048)) && + args.nchannels_x == 256 && args.nchannels_y > 0 && + (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 64; +} + +static bool use_rdna3_5_q8_id_narrow(const mmq_args & args) { + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + return args.type_x == GGML_TYPE_Q8_0 && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr && + ((args.ncols_x == 2048 && args.nrows_x == 512) || (args.ncols_x == 512 && args.nrows_x == 2048)) && + args.nchannels_x == 256 && args.nchannels_y > 0 && + (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 64; +} + static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { switch (args.type_x) { case GGML_TYPE_Q1_0: @@ -26,7 +82,11 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q8_0: - mul_mat_q_case(ctx, args, stream); + if (use_rdna3_5_q8_id_narrow(args)) { + launch_mul_mat_q_q8_0_j48(ctx, args, stream); + } else { + mul_mat_q_case(ctx, args, stream); + } break; // ----------------------------------------------------------------------- case GGML_TYPE_Q2_K: @@ -36,13 +96,27 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_K: - mul_mat_q_case(ctx, args, stream); + if (use_rdna3_5_q45_id_narrow(args)) { + launch_mul_mat_q_q4_k_j48(ctx, args, stream); + } else { + mul_mat_q_case(ctx, args, stream); + } break; case GGML_TYPE_Q5_K: - mul_mat_q_case(ctx, args, stream); + if (use_rdna3_5_q5_ling(args)) { + launch_mul_mat_q_q5_k_j32(ctx, args, stream); + } else if (use_rdna3_5_q45_id_narrow(args)) { + launch_mul_mat_q_q5_k_j32(ctx, args, stream); + } else { + mul_mat_q_case(ctx, args, stream); + } break; case GGML_TYPE_Q6_K: - mul_mat_q_case(ctx, args, stream); + if (use_rdna3_5_q6_ling_j32(args) || use_rdna3_5_q6_id_narrow(args)) { + launch_mul_mat_q_q6_k_j32(ctx, args, stream); + } else { + mul_mat_q_case(ctx, args, stream); + } break; // ----------------------------------------------------------------------- case GGML_TYPE_IQ1_S: @@ -82,8 +156,141 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con } } -void ggml_cuda_mul_mat_q( - ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { +void ggml_cuda_mul_mat_q_pair(ggml_backend_cuda_context & ctx, ggml_tensor * dst0, ggml_tensor * dst1) { + GGML_ASSERT(dst0->src[1] == dst1->src[1]); + GGML_ASSERT(dst0->src[2] == dst1->src[2]); + + const ggml_tensor * src0s[] = { dst0->src[0], dst1->src[0] }; + ggml_tensor * dsts[] = { dst0, dst1 }; + const ggml_tensor * src1 = dst0->src[1]; + const ggml_tensor * ids = dst0->src[2]; + const ggml_tensor * src0 = src0s[0]; + + GGML_ASSERT(src1->type == GGML_TYPE_F32); + if (ids == nullptr) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + cudaStream_t stream = ctx.stream(); + const int64_t ne10_padded = GGML_PAD(src1->ne[0], MATRIX_ROW_PADDING); + size_t nbytes_src1_q8_1 = src1->ne[3] * src1->ne[2] * src1->ne[1] * ne10_padded * sizeof(block_q8_1_mmq) / QK8_1_MMQ; + + for (int i = 0; i < 2; ++i) { + const ggml_tensor * src0_i = src0s[i]; + ggml_tensor * dst_i = dsts[i]; + GGML_ASSERT(dst_i->type == GGML_TYPE_F32 && dst_i->src[1] == src1); + GGML_ASSERT(ggml_are_same_shape(src0, src0_i) && ggml_are_same_shape(dst0, dst_i)); + GGML_ASSERT(src0_i->ne[0] == src1->ne[0]); + GGML_ASSERT(mmq_get_q8_1_ds_layout(src0->type) == mmq_get_q8_1_ds_layout(src0_i->type)); + const bool fallback = src0_i->ne[1] % 128 != 0; + nbytes_src1_q8_1 = std::max(nbytes_src1_q8_1, + src1->ne[3] * src1->ne[2] * src1->ne[1] * ne10_padded * sizeof(block_q8_1_mmq) / QK8_1_MMQ + + ggml_cuda_mmq_get_J_max(src0_i->type, fallback, cc, src1->ne[1]) * sizeof(block_q8_1_mmq)); + } + + ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1); + quantize_mmq_q8_1_cuda((const float *) src1->data, nullptr, src1_q8_1.get(), src0->type, + src1->ne[0], src1->nb[1] / sizeof(float), src1->nb[2] / sizeof(float), src1->nb[3] / sizeof(float), + ne10_padded, src1->ne[1], src1->ne[2], src1->ne[3], stream); + CUDA_CHECK(cudaGetLastError()); + + const int64_t s12_q = src1->ne[1] * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); + const int64_t s13_q = src1->ne[2] * s12_q; + for (int i = 0; i < 2; ++i) { + const ggml_tensor * src0_i = src0s[i]; + ggml_tensor * dst_i = dsts[i]; + GGML_ASSERT(ggml_are_same_shape(src0, src0_i) && ggml_are_same_shape(dst0, dst_i)); + const mmq_args args = { + (const char *) src0_i->data, src0_i->type, (const int *) src1_q8_1.get(), nullptr, nullptr, (float *) dst_i->data, + nullptr, + src0_i->ne[0], src0_i->ne[1], src1->ne[1], (int64_t) (src0_i->nb[1] / ggml_type_size(src0_i->type)), src1->ne[1], (int64_t) (dst_i->nb[1] / sizeof(float)), + src0_i->ne[2], src1->ne[2], (int64_t) (src0_i->nb[2] / ggml_type_size(src0_i->type)), s12_q, (int64_t) (dst_i->nb[2] / sizeof(float)), + src0_i->ne[3], src1->ne[3], (int64_t) (src0_i->nb[3] / ggml_type_size(src0_i->type)), s13_q, (int64_t) (dst_i->nb[3] / sizeof(float)), + src1->ne[1]}; + ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); + } + return; + } + + GGML_ASSERT(ids->type == GGML_TYPE_I32); + GGML_ASSERT(src1->ne[3] == 1); + GGML_ASSERT(src1->nb[2] % src1->nb[1] == 0); + GGML_ASSERT(ids->nb[0] == ggml_element_size(ids)); + + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + cudaStream_t stream = ctx.stream(); + const int64_t n_expert_used = ids->ne[0]; + const int64_t n_tokens = src1->ne[2]; + const int64_t ne_get_rows = n_tokens*n_expert_used; + const int64_t ne10_padded = GGML_PAD(src1->ne[0], MATRIX_ROW_PADDING); + const bool dedup_bcast = src1->ne[1] == 1 && n_expert_used > 1; + + ggml_cuda_pool_alloc ids_src1(ctx.pool(), ne_get_rows); + ggml_cuda_pool_alloc ids_dst(ctx.pool(), ne_get_rows); + ggml_cuda_pool_alloc expert_bounds(ctx.pool(), src0->ne[2] + 1); + + const int si1 = ids->nb[1] / ggml_element_size(ids); + const int sis1 = src1->nb[2] / src1->nb[1]; + ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(), + src0->ne[2], n_tokens, n_expert_used, src1->ne[1], si1, sis1, /*write_inverse =*/ dedup_bcast, stream); + CUDA_CHECK(cudaGetLastError()); + + size_t nbytes_src1_q8_1 = n_tokens*n_expert_used*ne10_padded * sizeof(block_q8_1_mmq)/QK8_1_MMQ; + for (int i = 0; i < 2; ++i) { + const ggml_tensor * src0_i = src0s[i]; + ggml_tensor * dst_i = dsts[i]; + GGML_ASSERT(dst_i->type == GGML_TYPE_F32); + GGML_ASSERT(dst_i->nb[2] % dst_i->nb[1] == 0); + GGML_ASSERT(ggml_are_same_shape(src0, src0_i)); + GGML_ASSERT(ggml_are_same_shape(dst0, dst_i)); + GGML_ASSERT(mmq_get_q8_1_ds_layout(src0->type) == mmq_get_q8_1_ds_layout(src0_i->type)); + GGML_ASSERT(src0_i->ne[0] == src1->ne[0]); + GGML_ASSERT(dst_i->ne[1] == n_expert_used); + + const bool fallback = src0_i->ne[1] % 128 != 0; + nbytes_src1_q8_1 = std::max(nbytes_src1_q8_1, + n_tokens*n_expert_used*ne10_padded * sizeof(block_q8_1_mmq)/QK8_1_MMQ + + ggml_cuda_mmq_get_J_max(src0_i->type, fallback, cc, src1->ne[1]) * sizeof(block_q8_1_mmq)); + } + + ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1); + const int64_t s11 = src1->nb[1] / ggml_type_size(src1->type); + const int64_t s12_src = src1->nb[2] / ggml_type_size(src1->type); + const int64_t s13_src = src1->nb[3] / ggml_type_size(src1->type); + if (dedup_bcast) { + quantize_scatter_mmq_q8_1_cuda((const float *) src1->data, ids_src1.get(), src1_q8_1.get(), src0->type, + src1->ne[0], /*stride_token=*/s12_src, ne10_padded, n_tokens, ne_get_rows, n_expert_used, stream); + } else { + quantize_mmq_q8_1_cuda((const float *) src1->data, ids_src1.get(), src1_q8_1.get(), src0->type, + src1->ne[0], s11, s12_src, s13_src, ne10_padded, ne_get_rows, 1, 1, stream); + } + CUDA_CHECK(cudaGetLastError()); + + const int64_t s12_q = src1->ne[1] * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int)); + const int64_t s13_q = n_tokens*s12_q; + for (int i = 0; i < 2; ++i) { + const ggml_tensor * src0_i = src0s[i]; + ggml_tensor * dst_i = dsts[i]; + const size_t ts_src0 = ggml_type_size(src0_i->type); + const size_t ts_dst = ggml_type_size(dst_i->type); + const int64_t s01 = src0_i->nb[1] / ts_src0; + const int64_t s02 = src0_i->nb[2] / ts_src0; + const int64_t s03 = src0_i->nb[3] / ts_src0; + const int64_t s1 = dst_i->nb[1] / ts_dst; + const int64_t s2 = dst_i->nb[2] / ts_dst; + const int64_t s3 = dst_i->nb[3] / ts_dst; + const mmq_args args = { + (const char *) src0_i->data, src0_i->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), (float *) dst_i->data, + nullptr, + src0_i->ne[0], src0_i->ne[1], ne_get_rows, s01, ne_get_rows, s1, + src0_i->ne[2], src0_i->ne[2], s02, s12_q, s2, + src0_i->ne[3], src1->ne[3], s03, s13_q, s3, + n_tokens}; + ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); + } +} + +static void ggml_cuda_mul_mat_q_impl( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, + ggml_tensor * dst, const ggml_tensor * swiglu) { GGML_ASSERT( src1->type == GGML_TYPE_F32); GGML_ASSERT( dst->type == GGML_TYPE_F32); GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. @@ -102,6 +309,14 @@ void ggml_cuda_mul_mat_q( GGML_ASSERT( nb0 == ts_dst); GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type)); + const ggml_tensor * gate = swiglu ? swiglu->src[0] : nullptr; + const ggml_tensor * up = swiglu ? swiglu->src[1] : nullptr; + if (swiglu) { + GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); + GGML_ASSERT(gate && up && gate->type == GGML_TYPE_F32 && up->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_are_same_shape(gate, up) && ggml_are_same_shape(gate, src1)); + } + const char * src0_d = (const char *) src0->data; const float * src1_d = (const float *) src1->data; float * dst_d = (float *) dst->data; @@ -152,6 +367,12 @@ void ggml_cuda_mul_mat_q( quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + } else if (swiglu) { + quantize_mmq_q8_1_swiglu_cuda( + (const float *) gate->data, (const float *) up->data, nullptr, src1_q8_1.get(), src0->type, + ne10, ne10_padded, ne11 * ne12 * ne13, /*logical_n1=*/1, + gate->nb[1] / sizeof(float), gate->nb[1] / sizeof(float), + up->nb[1] / sizeof(float), up->nb[1] / sizeof(float), stream); } else { quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); @@ -229,6 +450,12 @@ void ggml_cuda_mul_mat_q( quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); } + } else if (swiglu) { + quantize_mmq_q8_1_swiglu_cuda( + (const float *) gate->data, (const float *) up->data, ids_src1.get(), src1_q8_1.get(), src0->type, + ne10, ne10_padded, ne11_flat, n_expert_used, + gate->nb[1] / sizeof(float), gate->nb[2] / sizeof(float), + up->nb[1] / sizeof(float), up->nb[2] / sizeof(float), stream); } else if (dedup_bcast) { quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream); @@ -256,6 +483,16 @@ void ggml_cuda_mul_mat_q( ggml_cuda_mul_mat_q_switch_type(ctx, args, stream); } +void ggml_cuda_mul_mat_q( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { + ggml_cuda_mul_mat_q_impl(ctx, src0, src1, ids, dst, nullptr); +} + +void ggml_cuda_mul_mat_q_swiglu( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * ids, ggml_tensor * dst, const ggml_tensor * swiglu) { + ggml_cuda_mul_mat_q_impl(ctx, src0, swiglu, ids, dst, swiglu); +} + bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) { #ifdef GGML_CUDA_FORCE_CUBLAS return false; diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index c978b4421c5f..741c346f0666 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -486,18 +486,22 @@ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback); constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp. + constexpr bool split_j = type == GGML_TYPE_Q8_0 && J == 128 && !fallback && I == 64 && nwarps == 8; + constexpr int j_group = split_j ? J/2 : J; - const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I); + const int warp_i = split_j ? threadIdx.y % 4 : threadIdx.y; + const int warp_j = split_j ? threadIdx.y / 4 : 0; + const int i0 = (warp_i / ntx) * (ntx*tile_C::I); const bool y_scale_used = y_scale != nullptr; #pragma unroll - for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) { + for (int j0 = 0; j0 < j_group; j0 += ntx*tile_C::J) { #pragma unroll for (int n = 0; n < ntx; ++n) { #pragma unroll for (int l = 0; l < tile_C::ne; ++l) { - const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l); + const int j = warp_j*j_group + j0 + (warp_i % ntx)*tile_C::J + tile_C::get_j(l); if (j > j_max) { continue; @@ -973,21 +977,7 @@ static __global__ void mul_mat_q( const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y - // Initialize the ids for writing back data with just the index. - // For regular matrix multiplications this is never changed. - // For MoE the correct indices are loaded from ids_dst. extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory. -#pragma unroll - for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { - const int j = j0 + threadIdx.y*warp_size + threadIdx.x; - - if (j0 + nwarps*warp_size > J && j >= J) { - break; - } - - ids_dst_shared[j] = j; - } - __syncthreads(); if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback)) { const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); @@ -1036,6 +1026,18 @@ static __global__ void mul_mat_q( ids_dst_shared[j] = ids_dst[col_low + jt*J + j]; } __syncthreads(); + } else { +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { + const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + + if (j0 + nwarps*warp_size > J && j >= J) { + break; + } + + ids_dst_shared[j] = j; + } + __syncthreads(); } offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int)); @@ -1059,6 +1061,18 @@ static __global__ void mul_mat_q( return; } +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) { + const int j = j0 + threadIdx.y*warp_size + threadIdx.x; + + if (j0 + nwarps*warp_size > J && j >= J) { + break; + } + + ids_dst_shared[j] = j; + } + __syncthreads(); + constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback); constexpr int blocks_per_iter = ITER_K / qk; @@ -1236,6 +1250,90 @@ static __global__ void mul_mat_q( tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); } +template +static __global__ void build_mmq_routed_descriptors( + const int32_t * expert_bounds, uint32_t * descriptors, int n_experts, int max_descriptors) { + __shared__ int tile_counts[256]; + const int expert = threadIdx.x; + + for (int i = expert; i < max_descriptors; i += blockDim.x) { + descriptors[i] = UINT32_MAX; + } + + if (expert < n_experts) { + const int count = expert_bounds[expert + 1] - expert_bounds[expert]; + tile_counts[expert] = (count + J - 1) / J; + } + __syncthreads(); + + if (expert < n_experts) { + int descriptor = 0; + for (int i = 0; i < expert; ++i) { + descriptor += tile_counts[i]; + } + for (int jt = 0; jt < tile_counts[expert]; ++jt) { + descriptors[descriptor + jt] = uint32_t(expert) | (uint32_t(jt) << 16); + } + } +} + +template +__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback), ggml_cuda_mmq_get_occupancy(type, J, fallback)) +static __global__ void mul_mat_q_routed_compact( + const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, + const int32_t * __restrict__ expert_bounds, const uint32_t * __restrict__ descriptors, + const int max_descriptors, float * __restrict__ dst, const float * __restrict__ y_scale, + const uint3 blocks_per_ne00, const int nrows_x, const int stride_row_x, const int ncols_y, const int stride_col_dst, + const uint3 channel_ratio, const int stride_channel_x, + const uint3 sample_ratio, const int stride_sample_x) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + + const int wt = blockIdx.z; + const int it = blockIdx.x; + + extern __shared__ int ids_dst_shared[]; + for (int descriptor_idx = blockIdx.y; descriptor_idx < max_descriptors; descriptor_idx += gridDim.y) { + const uint32_t descriptor = descriptors[descriptor_idx]; + if (descriptor == UINT32_MAX) { + return; + } + + const int zt = descriptor & 0xFFFF; + const int jt = descriptor >> 16; + const int col_low = expert_bounds[zt + 0]; + const int col_high = expert_bounds[zt + 1]; + const int col_diff = col_high - col_low; + + if (descriptor_idx != int(blockIdx.y)) { + __syncthreads(); + } +#pragma unroll + for (int j0 = 0; j0 < J; j0 += nwarps * warp_size) { + const int j = j0 + threadIdx.y * warp_size + threadIdx.x; + if (j0 + nwarps * warp_size > J && j >= J) { + break; + } + ids_dst_shared[j] = ids_dst[col_low + jt * J + j]; + } + __syncthreads(); + + const int offset_x = fastdiv(wt, sample_ratio) * stride_sample_x + + fastdiv(zt, channel_ratio) * stride_channel_x + it * I * stride_row_x; + const int offset_y = (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int)); + const int tile_x_max_i = nrows_x - it * I - 1; + const int tile_y_max_j = col_diff - jt * J - 1; + const float * y_scale_tile = y_scale ? y_scale + col_low + jt * J : nullptr; + + constexpr bool fixup = false; + mul_mat_q_process_tile + (x, offset_x, y + offset_y, ids_dst_shared, dst + it * I, nullptr, y_scale_tile, + stride_row_x, ncols_y, stride_col_dst, + tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z); + } +} + template __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback)/2, 1) static __global__ void mul_mat_q_stream_k_fixup( @@ -1424,6 +1522,24 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); + const bool use_compact_routed = args.ids_dst != nullptr && args.nchannels_y == 256 && GGML_CUDA_CC_IS_RDNA3_5(cc) && + !ggml_cuda_mmq_get_stream_k(type, J, fallback, cc) && + ((type == GGML_TYPE_Q6_K && J == 32) || (type == GGML_TYPE_Q8_0 && J == 48)); + if (use_compact_routed) { + const int max_descriptors = (args.ncols_dst + J - 1) / J + args.nchannels_y; + ggml_cuda_pool_alloc descriptors(ctx.pool(id), max_descriptors); + build_mmq_routed_descriptors<<<1, 256, 0, stream>>> + (args.expert_bounds, descriptors.get(), args.nchannels_y, max_descriptors); + + const int descriptor_blocks = (args.ncols_dst + J - 1) / J; + const dim3 block_nums_compact(nty, descriptor_blocks, args.nsamples_y); + mul_mat_q_routed_compact<<>> + (args.x, args.y, args.ids_dst, args.expert_bounds, descriptors.get(), max_descriptors, args.dst, args.y_scale, + blocks_per_ne00_fd, args.nrows_x, args.stride_row_x, args.ncols_y, args.nrows_dst, + channel_ratio_fd, args.stride_channel_x, sample_ratio_fd, args.stride_sample_x); + return; + } + if (!ggml_cuda_mmq_get_stream_k(type, J, fallback, cc)) { mul_mat_q<<>> (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, @@ -1472,6 +1588,29 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a ntx_fd); } +static constexpr bool mmq_use_rdna3_5_q8_id_j48( + ggml_type type, int cc, bool fallback, bool has_ids, int64_t ncols_dst, int64_t nchannels_y) { + return type == GGML_TYPE_Q8_0 && GGML_CUDA_CC_IS_RDNA3_5(cc) && !fallback && has_ids && nchannels_y > 0 && + (ncols_dst + nchannels_y - 1) / nchannels_y == 32; +} + +static_assert(mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, true, 8192, 256)); +static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3, false, true, 8192, 256)); +static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q4_0, GGML_CUDA_CC_RDNA3_5, false, true, 8192, 256)); +static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, true, true, 8192, 256)); +static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, false, 8192, 256)); +static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, true, 7936, 256)); +static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, true, 8193, 256)); + +static constexpr bool mmq_use_rdna3_5_iq_id_j48( + ggml_type type, int cc, bool fallback, bool has_ids, int64_t ncols_dst, int64_t nchannels_y) { + return (type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ3_XXS) && GGML_CUDA_CC_IS_RDNA3_5(cc) && + !fallback && has_ids && nchannels_y > 0 && (ncols_dst + nchannels_y - 1) / nchannels_y == 48; +} + +static_assert(mmq_use_rdna3_5_iq_id_j48(GGML_TYPE_IQ2_XS, GGML_CUDA_CC_RDNA3_5, false, true, 12288, 256)); +static_assert(mmq_use_rdna3_5_iq_id_j48(GGML_TYPE_IQ3_XXS, GGML_CUDA_CC_RDNA3_5, false, true, 12288, 256)); + template void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { const int id = ggml_cuda_get_device(); @@ -1481,6 +1620,22 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, int J_best = 0; int ntiles_J_best = INT_MAX; + if (mmq_use_rdna3_5_iq_id_j48(type, cc, fallback, args.ids_dst != nullptr, args.ncols_dst, args.nchannels_y)) { + constexpr int J_rdna3_5 = 48; + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J_rdna3_5, fallback, cc); + if (config.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(config, cc) <= smpbo) { + J_best = J_rdna3_5; + ntiles_J_best = 1; + } + } else if (mmq_use_rdna3_5_q8_id_j48(type, cc, fallback, args.ids_dst != nullptr, args.ncols_dst, args.nchannels_y)) { + constexpr int J_rdna3_5 = 48; + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J_rdna3_5, fallback, cc); + if (config.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(config, cc) <= smpbo) { + J_best = J_rdna3_5; + ntiles_J_best = 1; + } + } + for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) { const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc); if (config.type == GGML_TYPE_COUNT) { @@ -1600,4 +1755,9 @@ extern DECL_MMQ_CASE(GGML_TYPE_NVFP4); void ggml_cuda_mul_mat_q( ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst); +void ggml_cuda_mul_mat_q_swiglu( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * ids, ggml_tensor * dst, const ggml_tensor * swiglu); + +void ggml_cuda_mul_mat_q_pair(ggml_backend_cuda_context & ctx, ggml_tensor * dst0, ggml_tensor * dst1); + bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu index bd5c5d421a4c..4eb457db7a6b 100644 --- a/ggml/src/ggml-cuda/mmvf.cu +++ b/ggml/src/ggml-cuda/mmvf.cu @@ -4,7 +4,7 @@ #include "mmvf.cuh" #include "convert.cuh" -template +template static __global__ void mul_mat_vec_f( const T * x_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, const int ncols2, const uint3 nchannels_y, const int stride_row, const int stride_col_y2, const int stride_col_dst, @@ -27,11 +27,11 @@ static __global__ void mul_mat_vec_f( ggml_cuda_pdl_sync(); if constexpr (is_multi_token_id) { - // Multi-token MUL_MAT_ID path, adding these in the normal path causes a perf regression for n_tokens=1 case - token_idx = blockIdx.z; - channel_x = ids[channel_dst + token_idx * ids_stride]; - channel_y = fastmodulo(channel_dst, nchannels_y); - sample_dst = 0; + // Multi-token path, adding these in the normal path causes a perf regression for n_tokens=1 case + token_idx = tokens_in_block ? threadIdx.y : blockIdx.z; + channel_x = ids ? ids[channel_dst + token_idx * ids_stride] : fastdiv((uint32_t) channel_dst, channel_ratio); + channel_y = ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst; + sample_dst = ids ? 0 : (tokens_in_block ? blockIdx.z : 0); } else { token_idx = ids ? blockIdx.z : 0; channel_x = ids ? ids[blockIdx.y + token_idx * ids_stride] : fastdiv((uint32_t) channel_dst, channel_ratio); @@ -99,7 +99,7 @@ static __global__ void mul_mat_vec_f( const float2 * y2 = (const float2 *) y; extern __shared__ char data_mmv[]; - float * buf_iw = (float *) data_mmv; + float * buf_iw = (float *) data_mmv + (tokens_in_block ? threadIdx.y * warp_size : 0); [[maybe_unused]] float * buf_iw_gate = nullptr; if constexpr (has_fusion) { buf_iw_gate = (float *) (data_mmv + warp_size*sizeof(float)); @@ -383,7 +383,7 @@ static __global__ void mul_mat_vec_f( } } -template +template static void mul_mat_vec_f_switch_fusion( const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, const int64_t ncols, const uint3 nchannels_y, @@ -397,7 +397,7 @@ static void mul_mat_vec_f_switch_fusion( const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; if constexpr (ncols_dst == 1) { if (has_fusion) { - ggml_cuda_kernel_launch(mul_mat_vec_f, launch_params, + ggml_cuda_kernel_launch(mul_mat_vec_f, launch_params, x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); @@ -407,14 +407,14 @@ static void mul_mat_vec_f_switch_fusion( GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1"); - ggml_cuda_kernel_launch(mul_mat_vec_f, launch_params, + ggml_cuda_kernel_launch(mul_mat_vec_f, launch_params, x, y, ids, fusion, dst, ncols, nchannels_y, stride_row, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); } -template +template void launch_mul_mat_vec_f_cuda( const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, const int64_t ncols, const int64_t nrows, @@ -448,57 +448,56 @@ void launch_mul_mat_vec_f_cuda( block_size_best = block_size; } } - const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; - const int nbytes_shared = warp_size*sizeof(float) + (has_fusion ? warp_size*sizeof(float) : 0); - const dim3 block_nums(nrows, nchannels_dst, nsamples_or_ntokens); - const dim3 block_dims(block_size_best, 1, 1); + const int nbytes_shared = (warp_size*sizeof(float) + (has_fusion ? warp_size*sizeof(float) : 0)) * (tokens_in_block ? nsamples_or_ntokens : 1); + const dim3 block_nums(nrows, nchannels_dst, tokens_in_block ? nsamples_dst : nsamples_or_ntokens); + const dim3 block_dims(block_size_best, tokens_in_block ? nsamples_or_ntokens : 1, 1); switch (block_size_best) { case 32: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; case 64: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; case 96: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; case 128: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; case 160: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; case 192: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; case 224: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); } break; case 256: { - mul_mat_vec_f_switch_fusion + mul_mat_vec_f_switch_fusion (x, y, ids, fusion, dst, ncols/2, nchannels_y_fd, stride_row, stride_col_y/2, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, block_dims, block_nums, nbytes_shared, ids_stride, stream); @@ -517,7 +516,7 @@ static void mul_mat_vec_f_cuda_switch_ncols_dst( const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x, const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, - const int64_t ids_stride, cudaStream_t stream) { + const int64_t ids_stride, bool exact_batch, cudaStream_t stream) { const bool has_ids = ids != nullptr; @@ -543,6 +542,42 @@ static void mul_mat_vec_f_cuda_switch_ncols_dst( return; } + if (exact_batch && !has_ids && ncols_dst > 1) { + if (ncols_dst == 2) { + launch_mul_mat_vec_f_cuda + (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, + stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, + nsamples_dst, ids_stride, stream); + return; + } + if (ncols_dst == 3) { + launch_mul_mat_vec_f_cuda + (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, + stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, + nsamples_dst, ids_stride, stream); + return; + } + if (ncols_dst == 4) { + launch_mul_mat_vec_f_cuda + (x, y, ids, fusion, dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, + stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, + nsamples_dst, ids_stride, stream); + return; + } + for (int64_t col = 0; col < ncols_dst; col += 4) { + const int64_t ncols_group = std::min(4, ncols_dst - col); + launch_mul_mat_vec_f_cuda + (x, y + col*stride_col_y, ids, fusion, dst + col*stride_col_dst, ncols, nrows, stride_row, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, + stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, + ncols_group, ids_stride, stream); + } + return; + } + switch (ncols_dst) { case 1: launch_mul_mat_vec_f_cuda @@ -614,21 +649,21 @@ static void mul_mat_vec_f_cuda( const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x, const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, - const int64_t ids_stride, enum ggml_prec prec, cudaStream_t stream) { + const int64_t ids_stride, enum ggml_prec prec, bool exact_batch, cudaStream_t stream) { if constexpr(std::is_same_v) { if (prec == GGML_PREC_DEFAULT) { mul_mat_vec_f_cuda_switch_ncols_dst (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, - stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, exact_batch, stream); return; } } mul_mat_vec_f_cuda_switch_ncols_dst (x, y, ids, fusion, dst, ncols, nrows, ncols_dst, stride_row, stride_col_y, stride_col_dst, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, - stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, exact_batch, stream); } void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, @@ -653,6 +688,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const enum ggml_prec prec = fast_fp16_available(cc) ? ggml_prec(dst->op_params[0]) : GGML_PREC_F32; + const bool exact_batch = ggml_get_op_params_i32(dst, 1) == GGML_HINT_EXACT_BATCH; const float * src1_d = (const float *) src1->data; const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr; @@ -709,19 +745,19 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor const float * src0_d = (const float *) src0->data; mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, - ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream()); + ne03, ne3, s03, s13, s3, ids_stride, prec, exact_batch, ctx.stream()); } break; case GGML_TYPE_F16: { const half * src0_d = (const half *) src0->data; mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, - ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream()); + ne03, ne3, s03, s13, s3, ids_stride, prec, exact_batch, ctx.stream()); } break; case GGML_TYPE_BF16: { const nv_bfloat16 * src0_d = (const nv_bfloat16 *) src0->data; mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst, ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, - ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream()); + ne03, ne3, s03, s13, s3, ids_stride, prec, exact_batch, ctx.stream()); } break; default: GGML_ABORT("unsupported type: %s", ggml_type_name(src0->type)); @@ -768,19 +804,19 @@ void ggml_cuda_op_mul_mat_vec_f( const float * src0_d = (const float *) src0_dd_i; mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, - nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream); + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, false, stream); } break; case GGML_TYPE_F16: { const half * src0_d = (const half *) src0_dd_i; mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, - nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream); + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, false, stream); } break; case GGML_TYPE_BF16: { const nv_bfloat16 * src0_d = (const nv_bfloat16 *) src0_dd_i; mul_mat_vec_f_cuda(src0_d, src1_ddf_i, nullptr, empty, dst_dd_i, ne00, row_diff, src1_ncols, stride_row, stride_col_y, stride_col_dst, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, - nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, stream); + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, 0, prec, false, stream); } break; default: GGML_ABORT("unsupported type: %s", ggml_type_name(src0->type)); diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 79f7a3f6fe75..b93b9b6c0520 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -69,6 +69,7 @@ enum mmvq_parameter_table_id { MMVQ_PARAMETERS_TURING, MMVQ_PARAMETERS_GCN, MMVQ_PARAMETERS_RDNA2, + MMVQ_PARAMETERS_RDNA3_5, MMVQ_PARAMETERS_RDNA3_0, MMVQ_PARAMETERS_RDNA4, MMVQ_PARAMETERS_GB10 @@ -77,9 +78,11 @@ enum mmvq_parameter_table_id { static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { #if defined(RDNA4) return MMVQ_PARAMETERS_RDNA4; +#elif defined(RDNA3_5) + return MMVQ_PARAMETERS_RDNA3_5; #elif defined(RDNA3_0) return MMVQ_PARAMETERS_RDNA3_0; -#elif defined(RDNA2) || defined(RDNA3_5) +#elif defined(RDNA2) return MMVQ_PARAMETERS_RDNA2; #elif defined(GCN) || defined(CDNA) return MMVQ_PARAMETERS_GCN; @@ -99,7 +102,10 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { if (GGML_CUDA_CC_IS_RDNA3_0(cc)) { return MMVQ_PARAMETERS_RDNA3_0; } - if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) { + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return MMVQ_PARAMETERS_RDNA3_5; + } + if (GGML_CUDA_CC_IS_RDNA2(cc)) { return MMVQ_PARAMETERS_RDNA2; } if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) { @@ -395,6 +401,23 @@ static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() { } static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id, bool small_k = false, bool halve_iters = false) { + if (table_id == MMVQ_PARAMETERS_RDNA3_5) { + // Measured on gfx1151 with 4096x14336 MMVQ on 2026-09-01. Wider blocks regress the other types. + if (ncols_dst == 1) { + switch (type) { + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return 4; + case GGML_TYPE_Q5_1: + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_K: + return 2; + default: + break; + } + } + return 1; + } if (table_id == MMVQ_PARAMETERS_GENERIC) { switch (ncols_dst) { case 1: @@ -521,7 +544,374 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d return 1; } +static_assert(calc_nwarps(GGML_TYPE_Q5_1, 1, MMVQ_PARAMETERS_RDNA3_5) == 2); +static_assert(calc_nwarps(GGML_TYPE_Q8_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); +static_assert(calc_nwarps(GGML_TYPE_MXFP4, 1, MMVQ_PARAMETERS_RDNA3_5) == 2); +static_assert(calc_nwarps(GGML_TYPE_Q4_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 2); +static_assert(calc_nwarps(GGML_TYPE_Q5_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 4); +static_assert(calc_nwarps(GGML_TYPE_Q6_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 4); +static_assert(calc_nwarps(GGML_TYPE_Q1_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); +static_assert(calc_nwarps(GGML_TYPE_Q2_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); +static_assert(calc_nwarps(GGML_TYPE_Q4_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); +static_assert(calc_nwarps(GGML_TYPE_IQ3_S, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); + +static constexpr __host__ __device__ bool is_rdna3_5_q4_columns_type(ggml_type type) { + return type == GGML_TYPE_Q1_0 || type == GGML_TYPE_Q2_0 || type == GGML_TYPE_Q5_1 || + type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || + type == GGML_TYPE_IQ3_S; +} + +static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q1_0)); +static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q2_0)); +static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q5_1)); +static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q4_K)); +static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q5_K)); +static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_Q6_K)); +static_assert(is_rdna3_5_q4_columns_type(GGML_TYPE_IQ3_S)); + +template +__launch_bounds__(2 * ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_q4_columns( + const void * vx_ptr, const void * vy_ptr, float * dst_ptr, + const uint32_t ncols_x, const uint32_t stride_row_x, const uint32_t stride_col_y, + const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, + const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, + const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { + static_assert(type == GGML_TYPE_Q8_0); + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = get_vdr_mmvq(type); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + + const int row = blockIdx.x; + const int lane = threadIdx.x; + const int column0 = 0; + const uint32_t channel_dst = blockIdx.y; + const uint32_t sample_dst = blockIdx.z; + const uint32_t channel_x = fastdiv(channel_dst, channel_ratio); + const uint32_t sample_x = fastdiv(sample_dst, sample_ratio); + + const void * vx = vx_ptr; + const block_q8_1 * y = (const block_q8_1 *) vy_ptr + sample_dst*stride_sample_y + channel_dst*stride_channel_y; + float * dst = dst_ptr + sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row; + + const int blocks_per_row_x = ncols_x / qk; + const int blocks_per_iter = vdr * warp_size / qi; + const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row*stride_row_x; + float tmp[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + + ggml_cuda_pdl_sync(); + for (int kbx = lane / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); + const int kqs = vdr * (lane % (qi/vdr)); + const block_q8_0 * bx = (const block_q8_0 *) vx + kbx_offset + kbx; + int xv[VDR_Q8_0_Q8_1_MMVQ]; +#pragma unroll + for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) { + xv[i] = get_int_b2(bx->qs, kqs + i); + } + const float dx = __half2float(bx->d); +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[(column0 + j)*stride_col_y + kby]; + int sumi = 0; +#pragma unroll + for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) { + sumi = ggml_cuda_dp4a(xv[i], get_int_b4(by->qs, kqs + i), sumi); + } + tmp[j] += dx * __half2float(__low2half(by->ds)) * (float) sumi; + } + } + +#pragma unroll + for (int j = 0; j < 4; ++j) { + tmp[j] = warp_reduce_sum(tmp[j]); + if (lane == 0) { + dst[(column0 + j)*stride_col_dst] = tmp[j]; + } + } +} + +template +__launch_bounds__(calc_nwarps(type, 1, get_device_table_id()) * ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_q4_columns_rdna3_5( + const void * vx_ptr, const void * vy_ptr, float * dst_ptr, + const uint32_t ncols_x, const uint32_t stride_row_x, const uint32_t stride_col_y, + const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, + const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, + const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { + static_assert(is_rdna3_5_q4_columns_type(type)); + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = get_vdr_mmvq(type); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = calc_nwarps(type, 1, get_device_table_id()); + + const int row = blockIdx.x; + const int lane = threadIdx.x; + const int tid = warp_size*threadIdx.y + lane; + const uint32_t channel_dst = blockIdx.y; + const uint32_t sample_dst = blockIdx.z; + const uint32_t channel_x = fastdiv(channel_dst, channel_ratio); + const uint32_t sample_x = fastdiv(sample_dst, sample_ratio); + + const block_q8_1 * y = (const block_q8_1 *) vy_ptr + sample_dst*stride_sample_y + channel_dst*stride_channel_y; + float * dst = dst_ptr + sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row; + + const int blocks_per_row_x = ncols_x / qk; + const int blocks_per_iter = vdr * nwarps*warp_size / qi; + const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row*stride_row_x; + float tmp[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + + ggml_cuda_pdl_sync(); + for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); + const int kqs = vdr * (tid % (qi/vdr)); + + if constexpr (type == GGML_TYPE_Q1_0) { + const block_q1_0 * bx = (const block_q1_0 *) vx_ptr + kbx_offset + kbx; + const int16_t * qs = (const int16_t *) bx->qs + kqs*2; + int xv[8]; +#pragma unroll + for (int i = 0; i < 2; ++i) { + const int q = qs[i]; + const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); + const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); + const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0); + const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0); + const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16); + const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16); + xv[4*i+0] = __byte_perm(s0, s1, 0x5410); + xv[4*i+1] = __byte_perm(s0, s1, 0x7632); + xv[4*i+2] = __byte_perm(s2, s3, 0x5410); + xv[4*i+3] = __byte_perm(s2, s3, 0x7632); + } + const float dx = bx->d; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[j*stride_col_y + kby + kqs]; + int sumi = 0; +#pragma unroll + for (int i = 0; i < 2; ++i) { + sumi = ggml_cuda_dp4a(xv[4*i+0], get_int_b4(by->qs, i*4+0), sumi); + sumi = ggml_cuda_dp4a(xv[4*i+1], get_int_b4(by->qs, i*4+1), sumi); + sumi = ggml_cuda_dp4a(xv[4*i+2], get_int_b4(by->qs, i*4+2), sumi); + sumi = ggml_cuda_dp4a(xv[4*i+3], get_int_b4(by->qs, i*4+3), sumi); + } + const float d8 = __low2float(by->ds); + tmp[j] += dx * d8 * sumi; + } + } else if constexpr (type == GGML_TYPE_Q2_0) { + const block_q2_0 * bx = (const block_q2_0 *) vx_ptr + kbx_offset + kbx; + const int16_t * qs = (const int16_t *) bx->qs + kqs*4; + int xv[4]; + int xw[4]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const int q = qs[i]; +#if defined(GGML_USE_HIP) + const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18); + const uint32_t qy_bits = q >> 8; + const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18); + xv[i] = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices); + xw[i] = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices); +#else + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + xv[i] = __byte_perm(qe, qo, 0x5140); + xw[i] = __byte_perm(qe, qo, 0x7362); +#endif // defined(GGML_USE_HIP) + } + const float dx = bx->d; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[j*stride_col_y + kby + kqs]; + int sumi = 0; +#pragma unroll + for (int i = 0; i < 4; ++i) { + sumi = ggml_cuda_dp4a(get_int_b4(by->qs, i*2+0), xv[i], sumi); + sumi = ggml_cuda_dp4a(get_int_b4(by->qs, i*2+1), xw[i], sumi); + } + const float d8 = __low2float(by->ds); + tmp[j] += dx * d8 * sumi; + } + } else if constexpr (type == GGML_TYPE_IQ3_S) { + const block_iq3_s * bx = (const block_iq3_s *) vx_ptr + kbx_offset + kbx; + const int2 qs_packed = make_int2(get_int_b2(bx->qs, kqs + 0), get_int_b2(bx->qs, kqs + 1)); + const uint8_t * qs = (const uint8_t *) &qs_packed; + const int qh = bx->qh[kqs/2]; + const int signs_packed_32 = get_int_b2(bx->signs, kqs/2); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + int2 xv[4]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + const int l0 = 2*i; + const int2 grid_pos = make_int2( + iq3s_grid[qs[l0 + 0] | ((qh << (8-l0)) & 0x100)], + iq3s_grid[qs[l0 + 1] | ((qh << (7-l0)) & 0x100)]); + const int signs0 = __vcmpne4(((signs_packed_8[i] & 0x03) << 7) | ((signs_packed_8[i] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[i] & 0x30) << 3) | ((signs_packed_8[i] & 0xC0) << 17), 0x00000000); + xv[i] = make_int2(__vsub4(grid_pos.x ^ signs0, signs0), __vsub4(grid_pos.y ^ signs1, signs1)); + } + const int ls = 1 + 2*((bx->scales[kqs/4] >> ((kqs << 1) & 0x04)) & 0x0F); + const float dx = __half2float(bx->d); +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[j*stride_col_y + kby + kqs/2]; + int sumi = 0; +#pragma unroll + for (int i = 0; i < 4; ++i) { + sumi = ggml_cuda_dp4a(xv[i].x, get_int_b4(by->qs, 2*i + 0), sumi); + sumi = ggml_cuda_dp4a(xv[i].y, get_int_b4(by->qs, 2*i + 1), sumi); + } + sumi *= ls; + const float d = dx * __low2float(by->ds); + tmp[j] += d * sumi; + } + } else if constexpr (type == GGML_TYPE_Q5_1) { + const block_q5_1 * bx = (const block_q5_1 *) vx_ptr + kbx_offset + kbx; + int vl[VDR_Q5_1_Q8_1_MMVQ]; + int vh[VDR_Q5_1_Q8_1_MMVQ]; +#pragma unroll + for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) { + vl[i] = get_int_b4(bx->qs, kqs + i); + vh[i] = get_int_b4(bx->qh, 0) >> (4 * (kqs + i)); + } + const half2 dm = bx->dm; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[j*stride_col_y + kby]; + int u[2*VDR_Q5_1_Q8_1_MMVQ]; +#pragma unroll + for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) { + u[2*i+0] = get_int_b4(by->qs, kqs + i); + u[2*i+1] = get_int_b4(by->qs, kqs + i + QI5_1); + } + // ROCm 7.14 changes gfx1151 Q5_1 rounding under four-column register pressure. Keep both paths materialized until LLVM preserves this expression. + volatile float dot = vec_dot_q5_1_q8_1_impl(vl, vh, u, dm, by->ds); + tmp[j] = __fadd_rn(tmp[j], dot); + } + } else if constexpr (type == GGML_TYPE_Q4_K) { + const block_q4_K * bx = (const block_q4_K *) vx_ptr + kbx_offset + kbx; + const int bq8_offset = QR4_K * ((kqs/2) / (QI8_1/2)); + const int * q4 = (const int *) (bx->qs + 16*bq8_offset + 4*((kqs/2)%4)); + int xv[2] = {q4[0], q4[4]}; + const uint16_t * scales = (const uint16_t *) bx->scales; + uint16_t aux[2]; + const int is = bq8_offset/2; + if (is < 2) { + aux[0] = scales[is+0] & 0x3f3f; + aux[1] = scales[is+2] & 0x3f3f; + } else { + aux[0] = ((scales[is+2] >> 0) & 0x0f0f) | ((scales[is-2] & 0xc0c0) >> 2); + aux[1] = ((scales[is+2] >> 4) & 0x0f0f) | ((scales[is-0] & 0xc0c0) >> 2); + } + const uint8_t * sc = (const uint8_t *) aux; + const uint8_t * m = sc + 2; + const half2 dm = bx->dm; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[j*stride_col_y + kby]; + int u[2*QR4_K]; + float d8[QR4_K]; +#pragma unroll + for (int i = 0; i < QR4_K; ++i) { + const block_q8_1 * byi = by + bq8_offset + i; + d8[i] = __low2float(byi->ds); + const int * q8 = (const int *) byi->qs + ((kqs/2)%4); + u[2*i+0] = q8[0]; + u[2*i+1] = q8[4]; + } + tmp[j] += vec_dot_q4_K_q8_1_impl_vmmq(xv, u, sc, m, dm, d8); + } + } else if constexpr (type == GGML_TYPE_Q5_K) { + const block_q5_K * bx = (const block_q5_K *) vx_ptr + kbx_offset + kbx; + const int bq8_offset = QR5_K * ((kqs/2) / (QI8_1/2)); + const int * ql = (const int *) (bx->qs + 16*bq8_offset + 4*((kqs/2)%4)); + const int * qh = (const int *) (bx->qh + 4*((kqs/2)%4)); + int vl[2] = {ql[0], ql[4]}; + int vh[2] = {qh[0] >> bq8_offset, qh[4] >> bq8_offset}; + const uint16_t * scales = (const uint16_t *) bx->scales; + uint16_t aux[2]; + const int is = bq8_offset/2; + if (is < 2) { + aux[0] = scales[is+0] & 0x3f3f; + aux[1] = scales[is+2] & 0x3f3f; + } else { + aux[0] = ((scales[is+2] >> 0) & 0x0f0f) | ((scales[is-2] & 0xc0c0) >> 2); + aux[1] = ((scales[is+2] >> 4) & 0x0f0f) | ((scales[is-0] & 0xc0c0) >> 2); + } + const uint8_t * sc = (const uint8_t *) aux; + const uint8_t * m = sc + 2; + const half2 dm = bx->dm; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[j*stride_col_y + kby]; + int u[2*QR5_K]; + float d8[QR5_K]; +#pragma unroll + for (int i = 0; i < QR5_K; ++i) { + const block_q8_1 * byi = by + bq8_offset + i; + d8[i] = __low2float(byi->ds); + const int * q8 = (const int *) byi->qs + ((kqs/2)%4); + u[2*i+0] = q8[0]; + u[2*i+1] = q8[4]; + } + tmp[j] += vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, dm, d8); + } + } else if constexpr (type == GGML_TYPE_Q6_K) { + const block_q6_K * bx = (const block_q6_K *) vx_ptr + kbx_offset + kbx; + const int bq8_offset = 2*QR6_K*(kqs/(QI6_K/2)) + (kqs%(QI6_K/2))/(QI6_K/4); + const int scale_offset = (QI6_K/4)*(kqs/(QI6_K/2)) + (kqs%(QI6_K/2))/(QI6_K/8); + const int vh_shift = 2*((kqs%(QI6_K/2))/(QI6_K/4)); + const int vl = get_int_b2(bx->ql, kqs); + const int vh = get_int_b2(bx->qh, (QI6_K/4)*(kqs/(QI6_K/2)) + kqs%(QI6_K/4)) >> vh_shift; + const int8_t * scales = bx->scales + scale_offset; + const float dx = bx->d; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const block_q8_1 * by = &y[j*stride_col_y + kby]; + int u[QR6_K]; + float d8[QR6_K]; +#pragma unroll + for (int i = 0; i < QR6_K; ++i) { + u[i] = get_int_b4(by[bq8_offset + 2*i].qs, kqs % QI8_1); + d8[i] = __low2float(by[bq8_offset + 2*i].ds); + } + tmp[j] += vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, dx, d8); + } + } + } + + __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][4][warp_size]; + if (threadIdx.y > 0) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + tmp_shared[threadIdx.y-1][j][lane] = tmp[j]; + } + } + __syncthreads(); + if (threadIdx.y > 0) { + return; + } + +#pragma unroll + for (int j = 0; j < 4; ++j) { +#pragma unroll + for (int i = 0; i < nwarps-1; ++i) { + tmp[j] += tmp_shared[i][j][lane]; + } + tmp[j] = warp_reduce_sum(tmp[j]); + if (lane == 0) { + dst[j*stride_col_dst] = tmp[j]; + } + } +} + static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { + if (table_id == MMVQ_PARAMETERS_RDNA3_5) { + return 1; + } if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) { switch (ncols_dst) { case 1: @@ -667,12 +1057,24 @@ static __global__ void mul_mat_vec_q( for (int j = 0; j < ncols_dst; ++j) { #pragma unroll for (int i = 0; i < rows_per_cuda_block; ++i) { - tmp[j][i] += vec_dot_q_cuda( - vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs); + if constexpr (type == GGML_TYPE_Q5_1 && table_id == MMVQ_PARAMETERS_RDNA3_5) { + volatile float dot = vec_dot_q_cuda( + vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs); + tmp[j][i] = __fadd_rn(tmp[j][i], dot); + } else { + tmp[j][i] += vec_dot_q_cuda( + vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs); + } if constexpr (has_fusion) { if (use_gate) { - tmp_gate[j][i] += vec_dot_q_cuda( - vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs); + if constexpr (type == GGML_TYPE_Q5_1 && table_id == MMVQ_PARAMETERS_RDNA3_5) { + volatile float dot_gate = vec_dot_q_cuda( + vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs); + tmp_gate[j][i] = __fadd_rn(tmp_gate[j][i], dot_gate); + } else { + tmp_gate[j][i] += vec_dot_q_cuda( + vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs); + } } } } @@ -1054,6 +1456,35 @@ static void mul_mat_vec_q_switch_ncols_dst( dims.first, dims.second, 0, ids_stride, stream); } break; case 4: { + if constexpr (is_rdna3_5_q4_columns_type(type)) { + const bool no_fusion = fusion.gate == nullptr && fusion.x_bias == nullptr && fusion.gate_bias == nullptr && + fusion.x_scale == nullptr && fusion.gate_scale == nullptr; + if (table_id == MMVQ_PARAMETERS_RDNA3_5 && ids == nullptr && no_fusion) { + constexpr int c_nwarps = calc_nwarps(type, 1, MMVQ_PARAMETERS_RDNA3_5); + const dim3 block_nums(nrows_x, nchannels_dst, nsamples_dst); + const dim3 block_dims(warp_size, c_nwarps, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(mul_mat_vec_q4_columns_rdna3_5, launch_params, + vx, vy, dst, ncols_x, stride_row_x, stride_col_y, stride_col_dst, + channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, + sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst); + break; + } + } + if constexpr (type == GGML_TYPE_Q8_0) { + const bool no_fusion = fusion.gate == nullptr && fusion.x_bias == nullptr && fusion.gate_bias == nullptr && + fusion.x_scale == nullptr && fusion.gate_scale == nullptr; + if (table_id == MMVQ_PARAMETERS_RDNA3_5 && ids == nullptr && no_fusion) { + const dim3 block_nums(nrows_x, nchannels_dst, nsamples_dst); + const dim3 block_dims(warp_size, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(mul_mat_vec_q4_columns, launch_params, + vx, vy, dst, ncols_x, stride_row_x, stride_col_y, stride_col_dst, + channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, + sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst); + break; + } + } constexpr int c_ncols_dst = 4; std::pair dims = calc_launch_params(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id); mul_mat_vec_q_switch_fusion(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst, diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index bcc7723957b5..3923b19b0e2b 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -1,4 +1,5 @@ #include "quantize.cuh" +#include "unary.cuh" #include #if defined(BLACKWELL_MMA_AVAILABLE) @@ -463,14 +464,7 @@ static __global__ void quantize_mmq_q8_1( constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32; constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32; - const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4; - - if (i0 >= ne0) { - return; - } - - const int64_t i00 = i0; - ggml_cuda_pdl_sync(); + const int64_t first_chunk = (int64_t) blockIdx.y * CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK; int64_t base_idx; if constexpr (scatter) { @@ -485,76 +479,134 @@ static __global__ void quantize_mmq_q8_1( const float4 * x4 = (const float4 *) x; block_q8_1_mmq * y = (block_q8_1_mmq *) vy; - const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel - const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block - - // Load 4 floats per thread and calculate max. abs. value between them: - const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f); - float amax = fabsf(xi.x); - amax = fmaxf(amax, fabsf(xi.y)); - amax = fmaxf(amax, fabsf(xi.z)); - amax = fmaxf(amax, fabsf(xi.w)); + ggml_cuda_pdl_sync(); - // Exchange max. abs. value between vals_per_scale/4 threads. #pragma unroll - for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) { - amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); - } + for (int chunk = 0; chunk < CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK; ++chunk) { + const int64_t i0 = ((first_chunk + chunk) * blockDim.x + threadIdx.x) * 4; + if (i0 >= ne0) { + break; + } - float sum; - if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { - sum = xi.x + xi.y + xi.z + xi.w; + const int64_t k_block = i0 / QK8_1_MMQ; + const int64_t iqs = i0 % QK8_1_MMQ; + const float4 xi = i0 < ne00 ? x4[(base_idx + i0)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f); + float amax = fabsf(xi.x); + amax = fmaxf(amax, fabsf(xi.y)); + amax = fmaxf(amax, fabsf(xi.z)); + amax = fmaxf(amax, fabsf(xi.w)); - // Calculate sums across vals_per_sum/4 threads. #pragma unroll - for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { - sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); + for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) { + amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); } - } - const float d_inv = 127.0f / amax; - char4 q; - q.x = roundf(xi.x*d_inv); - q.y = roundf(xi.y*d_inv); - q.z = roundf(xi.z*d_inv); - q.w = roundf(xi.w*d_inv); - const float d = 1.0f / d_inv; - - // write the block once (normal) or to each of the token's compact rows (scatter) - const int nwrite = scatter ? n_expert_used : 1; + float sum; + if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + sum = xi.x + xi.y + xi.z + xi.w; #pragma unroll - for (int slot = 0; slot < nwrite; ++slot) { - int64_t ib; - if constexpr (scatter) { - const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; - ib = k_block*ne1 + i; - } else { - const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel - ib = ib0 + k_block*ne1 + blockIdx.x; + for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { + sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); + } } - // Write back 4 int8 values as a single 32 bit value for better memory bandwidth: - char4 * yqs4 = (char4 *) y[ib].qs; - yqs4[iqs/4] = q; + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x*d_inv); + q.y = roundf(xi.y*d_inv); + q.z = roundf(xi.z*d_inv); + q.w = roundf(xi.w*d_inv); + const float d = 1.0f / d_inv; - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) { - if (iqs % 16 == 0 && iqs < 96) { - y[ib].d2s6[2 + iqs/16] = sum; - if (iqs % 64 == 0) { - y[ib].d2s6[iqs/64] = d; - } - } - } else if (iqs % 32 == 0) { - if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) { - y[ib].ds4[iqs/32] = make_half2(d, sum); + const int nwrite = scatter ? n_expert_used : 1; +#pragma unroll + for (int slot = 0; slot < nwrite; ++slot) { + int64_t ib; + if constexpr (scatter) { + const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + ib = k_block*ne1 + i; } else { - y[ib].d4[iqs/32] = d; + const int64_t ib0 = blockIdx.z * ((int64_t) gridDim.x * ne0 / QK8_1_MMQ); + ib = ib0 + k_block*ne1 + blockIdx.x; + } + + char4 * yqs4 = (char4 *) y[ib].qs; + yqs4[iqs/4] = q; + + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) { + if (iqs % 16 == 0 && iqs < 96) { + y[ib].d2s6[2 + iqs/16] = sum; + if (iqs % 64 == 0) { + y[ib].d2s6[iqs/64] = d; + } + } + } else if (iqs % 32 == 0) { + if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) { + y[ib].ds4[iqs/32] = make_half2(d, sum); + } else { + y[ib].d4[iqs/32] = d; + } } } } GGML_UNUSED(n_expert_used); } +static __global__ void quantize_mmq_q8_1_swiglu( + const float * __restrict__ gate, const float * __restrict__ up, const int32_t * __restrict__ ids, + void * __restrict__ vy, const int64_t ne00, const int64_t ne0, const int ne1, const int logical_n1, + const int64_t gate_s1, const int64_t gate_st, const int64_t up_s1, const int64_t up_st) { + const int64_t i0 = ((int64_t) blockDim.x * blockIdx.y + threadIdx.x) * 4; + if (i0 >= ne0) { + return; + } + + const int64_t logical_row = ids ? ids[blockIdx.x] : blockIdx.x; + const int64_t slot = logical_row % logical_n1; + const int64_t token = logical_row / logical_n1; + const int64_t gate_base = token * gate_st + slot * gate_s1; + const int64_t up_base = token * up_st + slot * up_s1; + + ggml_cuda_pdl_sync(); + float values[4]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + if (i0 + i < ne00) { + volatile float value = ggml_cuda_op_silu_single(gate[gate_base + i0 + i]) * up[up_base + i0 + i]; + values[i] = value; + } else { + values[i] = 0.0f; + } + } + const float4 xi = make_float4(values[0], values[1], values[2], values[3]); + + float amax = fabsf(xi.x); + amax = fmaxf(amax, fabsf(xi.y)); + amax = fmaxf(amax, fabsf(xi.z)); + amax = fmaxf(amax, fabsf(xi.w)); +#pragma unroll + for (int offset = 4; offset > 0; offset >>= 1) { + amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); + } + + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x * d_inv); + q.y = roundf(xi.y * d_inv); + q.z = roundf(xi.z * d_inv); + q.w = roundf(xi.w * d_inv); + const float d = 1.0f / d_inv; + + block_q8_1_mmq * y = (block_q8_1_mmq *) vy; + const int64_t k_block = i0 / QK8_1_MMQ; + const int64_t iqs = i0 % QK8_1_MMQ; + const int64_t ib = k_block * ne1 + blockIdx.x; + ((char4 *) y[ib].qs)[iqs / 4] = q; + if (iqs % 32 == 0) { + y[ib].d4[iqs / 32] = d; + } +} + void quantize_row_q8_1_cuda( const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, @@ -580,7 +632,8 @@ void quantize_mmq_q8_1_cuda( GGML_ASSERT(ne0 % QK8_1_MMQ == 0); // ne1 tends to assume the highest values, therefore use it as the "x" dimension of the CUDA grid: - const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const int64_t vals_per_block = 4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ * CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK; + const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block; const dim3 num_blocks(ne1, block_num_y, ne2*ne3); const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); switch (mmq_get_q8_1_ds_layout(type_src0)) { @@ -602,6 +655,20 @@ void quantize_mmq_q8_1_cuda( } } +void quantize_mmq_q8_1_swiglu_cuda( + const float * gate, const float * up, const int32_t * ids, void * vy, const ggml_type type_src0, + const int64_t ne00, const int64_t ne0, const int64_t ne1, const int logical_n1, + const int64_t gate_s1, const int64_t gate_st, const int64_t up_s1, const int64_t up_st, cudaStream_t stream) { + GGML_ASSERT(mmq_get_q8_1_ds_layout(type_src0) == MMQ_Q8_1_DS_LAYOUT_D4); + GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ne0 % QK8_1_MMQ == 0); + const int64_t block_num_y = (ne0 + 4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const dim3 num_blocks(ne1, block_num_y, 1); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + quantize_mmq_q8_1_swiglu<<>>( + gate, up, ids, vy, ne00, ne0, ne1, logical_n1, gate_s1, gate_st, up_s1, up_st); +} + // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row) void quantize_scatter_mmq_q8_1_cuda( const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0, @@ -610,7 +677,8 @@ void quantize_scatter_mmq_q8_1_cuda( GGML_ASSERT(ne00 % 4 == 0); GGML_ASSERT(ne0 % QK8_1_MMQ == 0); - const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const int64_t vals_per_block = 4 * CUDA_QUANTIZE_BLOCK_SIZE_MMQ * CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK; + const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block; const dim3 num_blocks(n_tokens, block_num_y, 1); const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); switch (mmq_get_q8_1_ds_layout(type_src0)) { diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh index 5f08dcbfe331..1801da05e871 100644 --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -7,9 +7,11 @@ #define CUDA_QUANTIZE_BLOCK_SIZE 256 #define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128 +#define CUDA_QUANTIZE_MMQ_CHUNKS_PER_BLOCK 2 static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access."); static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access."); +static_assert((4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) % QK8_1_MMQ == 0, "Quantization chunks must contain complete Q8_1 MMQ blocks."); typedef void (*quantize_cuda_t)( const float * x, const int32_t * ids, void * vy, @@ -26,6 +28,11 @@ void quantize_mmq_q8_1_cuda( ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); +void quantize_mmq_q8_1_swiglu_cuda( + const float * gate, const float * up, const int32_t * ids, void * vy, ggml_type type_src0, + int64_t ne00, int64_t ne0, int64_t ne1, int logical_n1, + int64_t gate_s1, int64_t gate_st, int64_t up_s1, int64_t up_st, cudaStream_t stream); + void quantize_mmq_fp4_cuda(const float * x, const int32_t * ids, void * vy, From 87a4f9b8d7c3c033b65b84b30d5cdc4a15deb28d Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:23:35 +0200 Subject: [PATCH 02/30] hip: fuse common Qwen inference operations Add gfx1151 paths for paired matmuls, SwiGLU quantization, expert aggregation, contiguous binary operations, transposed concat, normalization, and conversion operations. Assisted-by: OpenCode --- ggml/src/ggml-cuda/binbcast.cu | 123 +++++++++++++++++++++ ggml/src/ggml-cuda/binbcast.cuh | 2 + ggml/src/ggml-cuda/concat.cu | 73 ++++++++++++ ggml/src/ggml-cuda/convert.cu | 32 ++++++ ggml/src/ggml-cuda/ggml-cuda.cu | 190 +++++++++++++++++++++++++++++++- ggml/src/ggml-cuda/norm.cu | 72 +++++++++++- ggml/src/ggml-cuda/norm.cuh | 2 + ggml/src/ggml-cuda/unary.cu | 17 +++ ggml/src/ggml-cuda/unary.cuh | 3 + 9 files changed, 512 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu index 2e38077bf67f..83361ab75182 100644 --- a/ggml/src/ggml-cuda/binbcast.cu +++ b/ggml/src/ggml-cuda/binbcast.cu @@ -26,6 +26,43 @@ static __device__ __forceinline__ float op_div(const float a, const float b) { return a / b; } +template +static __global__ void binary_contiguous_f32(const float * src0, const float * src1, float * dst, int64_t n) { + ggml_cuda_pdl_lc(); + int64_t i = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; + const int64_t stride = int64_t(blockDim.x) * gridDim.x; + ggml_cuda_pdl_sync(); + for (; i < n; i += stride) { + dst[i] = bin_op(src0[i], src1[i]); + if (n - i <= stride) { + break; + } + } +} + +template +static bool try_binary_contiguous_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + if (src0->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || + src0->ne[0] != 2048 || + !ggml_are_same_shape(src0, src1) || !ggml_are_same_shape(src0, dst) || + !ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) { + return false; + } + + const int threads = 256; + const int64_t n = ggml_nelements(dst); + if (n == 0) { + return true; + } + const int blocks = std::min((n - 1) / threads + 1, 65535); + const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream()); + ggml_cuda_kernel_launch(binary_contiguous_f32, launch_params, + (const float *) src0->data, (const float *) src1->data, (float *) dst->data, n); + return true; +} + template (ctx, dst)) { + return; + } ggml_cuda_op_bin_bcast>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream()); } @@ -443,6 +483,9 @@ void ggml_cuda_op_sub(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { } void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + if (try_binary_contiguous_f32(ctx, dst)) { + return; + } ggml_cuda_op_bin_bcast>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream()); } @@ -542,6 +585,86 @@ void ggml_cuda_op_fused_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst, } } +template +static __global__ void weighted_expert_sum_f32( + const float * experts, const float * weights, float * dst, + int64_t n_embd, int64_t n_tokens, int64_t experts_s1, int64_t experts_s2, + int64_t weights_s1, int64_t weights_s2) { + const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= n_embd * n_tokens) { + return; + } + + const int64_t token = index / n_embd; + const int64_t channel = index - token * n_embd; + // Keep MUL rounding separate from ADD to match the unfused graph. + volatile float product = experts[token * experts_s2 + channel] * weights[token * weights_s2]; + float sum = product; +#pragma unroll + for (int i = 1; i < n_expert_used; ++i) { + product = experts[token * experts_s2 + i * experts_s1 + channel] * weights[token * weights_s2 + i * weights_s1]; + sum += product; + } + dst[index] = sum; +} + +void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tensor * mul, ggml_tensor * dst, int n_expert_used) { + const ggml_tensor * experts = ggml_are_same_shape(mul, mul->src[0]) ? mul->src[0] : mul->src[1]; + const ggml_tensor * weights = experts == mul->src[0] ? mul->src[1] : mul->src[0]; + const int64_t n_embd = experts->ne[0]; + const int64_t n_tokens = experts->ne[2]; + const int threads = 256; + const int blocks = (n_embd * n_tokens + threads - 1) / threads; + const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream()); + // MUL output may alias experts, so stage until all expert reads finish. + ggml_cuda_pool_alloc result(ctx.pool(), n_embd * n_tokens); + +#define launch_weighted_expert_sum(n) \ + ggml_cuda_kernel_launch(weighted_expert_sum_f32, launch_params, \ + (const float *) experts->data, (const float *) weights->data, result.get(), \ + n_embd, n_tokens, experts->nb[1] / sizeof(float), experts->nb[2] / sizeof(float), \ + weights->nb[1] / sizeof(float), weights->nb[2] / sizeof(float)) + switch (n_expert_used) { + case 2: launch_weighted_expert_sum(2); break; + case 3: launch_weighted_expert_sum(3); break; + case 4: launch_weighted_expert_sum(4); break; + case 5: launch_weighted_expert_sum(5); break; + case 6: launch_weighted_expert_sum(6); break; + case 7: launch_weighted_expert_sum(7); break; + case 8: launch_weighted_expert_sum(8); break; + default: GGML_ABORT("unsupported expert count"); + } +#undef launch_weighted_expert_sum + + CUDA_CHECK(cudaMemcpyAsync(dst->data, result.get(), ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream())); +} + +static __global__ void shared_mul_add_f32( + const float * src, const float * gate, const float * other, const float * residual, float * dst, + int64_t n_embd, int64_t nelements) { + const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= nelements) { + return; + } + volatile float product = src[index] * gate[index / n_embd]; + float value = other[index] + product; + dst[index] = value + residual[index]; +} + +void ggml_cuda_op_shared_mul_add( + ggml_backend_cuda_context & ctx, ggml_tensor * mul, const ggml_tensor * other, + const ggml_tensor * residual, ggml_tensor * dst) { + const ggml_tensor * src = ggml_are_same_shape(mul, mul->src[0]) ? mul->src[0] : mul->src[1]; + const ggml_tensor * gate = src == mul->src[0] ? mul->src[1] : mul->src[0]; + const int threads = 256; + const int64_t nelements = ggml_nelements(dst); + const int blocks = (nelements + threads - 1) / threads; + const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream()); + ggml_cuda_kernel_launch(shared_mul_add_f32, launch_params, + (const float *) src->data, (const float *) gate->data, (const float *) other->data, + (const float *) residual->data, (float *) dst->data, dst->ne[0], nelements); +} + void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; diff --git a/ggml/src/ggml-cuda/binbcast.cuh b/ggml/src/ggml-cuda/binbcast.cuh index 12624785b444..916c5ac34931 100644 --- a/ggml/src/ggml-cuda/binbcast.cuh +++ b/ggml/src/ggml-cuda/binbcast.cuh @@ -10,3 +10,5 @@ void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst void ggml_cuda_op_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse); void ggml_cuda_op_fused_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse); +void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tensor * mul, ggml_tensor * dst, int n_expert_used); +void ggml_cuda_op_shared_mul_add(ggml_backend_cuda_context & ctx, ggml_tensor * mul, const ggml_tensor * other, const ggml_tensor * residual, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/concat.cu b/ggml/src/ggml-cuda/concat.cu index 6df89013ca79..408cbce3d219 100644 --- a/ggml/src/ggml-cuda/concat.cu +++ b/ggml/src/ggml-cuda/concat.cu @@ -2,6 +2,13 @@ #include +static constexpr int concat_transposed_tile_y(int cc) { + return GGML_CUDA_CC_IS_RDNA3_5(cc) ? 16 : 8; +} + +static_assert(concat_transposed_tile_y(GGML_CUDA_CC_RDNA3_5) == 16); +static_assert(concat_transposed_tile_y(GGML_CUDA_CC_RDNA3) == 8); + // contiguous kernels template static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) concat_cont(const T * x, @@ -79,6 +86,54 @@ static void concat_cont_cuda(const T * x, concat_cont<<>>(x, y, dst, ne00, ne01, ne02, ne0, ne1, ne2); } +template +static __global__ void concat_transposed_src1_dim0( + const char * src0, const char * src1, char * dst, + int64_t ne00, int64_t ne10, int64_t ne11, + uint64_t nb00, uint64_t nb01, uint64_t nb02, + uint64_t nb10, uint64_t nb11, uint64_t nb12, + uint64_t nb0, uint64_t nb1, uint64_t nb2) { + constexpr int tile = 32; + __shared__ T values[tile][tile + 1]; + + const int tx = threadIdx.x; + const int ty = threadIdx.y; + const int64_t channel0 = blockIdx.x * tile; + const int64_t token0 = blockIdx.y * tile; + const int64_t i2 = blockIdx.z; + + ggml_cuda_pdl_sync(); + + for (int j = 0; j < tile; j += blockDim.y) { + const int64_t token = token0 + ty + j; + const int64_t channel = channel0 + tx; + if (token < ne10 && channel < ne11) { + values[ty + j][tx] = *reinterpret_cast(src1 + i2 * nb12 + channel * nb11 + token * nb10); + } + } + __syncthreads(); + + for (int j = 0; j < tile; j += blockDim.y) { + const int64_t token = token0 + tx; + const int64_t channel = channel0 + ty + j; + if (token < ne10 && channel < ne11) { + *reinterpret_cast(dst + i2 * nb2 + channel * nb1 + (ne00 + token) * nb0) = values[tx][ty + j]; + } + } + + if (blockIdx.y == 0) { + for (int j = 0; j < tile; j += blockDim.y) { + const int64_t channel = channel0 + ty + j; + if (channel < ne11) { + for (int64_t state = tx; state < ne00; state += tile) { + *reinterpret_cast(dst + i2 * nb2 + channel * nb1 + state * nb0) = + *reinterpret_cast(src0 + i2 * nb02 + channel * nb01 + state * nb00); + } + } + } + } +} + // non-contiguous kernel (slow) template static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) @@ -141,6 +196,24 @@ static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) template static void concat_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, int dim, cudaStream_t stream) { + if (dim == 0 && src1->ne[0] >= 32 && src0->ne[3] == 1 && src1->ne[3] == 1 && dst->ne[3] == 1 && + src0->nb[0] == sizeof(T) && src0->nb[1] == src0->ne[0] * sizeof(T) && + src1->nb[1] == sizeof(T) && src1->nb[0] == src1->ne[1] * sizeof(T) && + dst->nb[0] == sizeof(T) && dst->nb[1] == dst->ne[0] * sizeof(T) && + src0->ne[1] == src1->ne[1] && src0->ne[2] == src1->ne[2]) { + constexpr int tile = 32; + const int tile_y = concat_transposed_tile_y(ggml_cuda_info().devices[ggml_cuda_get_device()].cc); + const dim3 block_dims(tile, tile_y, 1); + const dim3 block_nums((src1->ne[1] + tile - 1) / tile, (src1->ne[0] + tile - 1) / tile, src1->ne[2]); + concat_transposed_src1_dim0<<>>( + (const char *) src0->data, (const char *) src1->data, (char *) dst->data, + src0->ne[0], src1->ne[0], src1->ne[1], + src0->nb[0], src0->nb[1], src0->nb[2], + src1->nb[0], src1->nb[1], src1->nb[2], + dst->nb[0], dst->nb[1], dst->nb[2]); + return; + } + if (dim != 3 && ggml_is_contiguous_to_3(src0) && ggml_is_contiguous_to_3(src1)) { const T * src0_d = (const T *) src0->data; const T * src1_d = (const T *) src1->data; diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 360c614a4403..47cab5f980d4 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -450,8 +450,40 @@ static void convert_unary_cuda(const void * vx, dst_t * y, (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03); } +static __global__ void convert_f32_f16_cont(const float * x, half * y, int64_t k) { + const int64_t i = 2 * ((int64_t) blockDim.x * blockIdx.x + threadIdx.x); + if (i + 1 < k) { + const float2 value = ((const float2 *) x)[i / 2]; + ((half2 *) y)[i / 2] = make_half2(value.x, value.y); + } else if (i < k) { + y[i] = x[i]; + } +} + +static __global__ void convert_f16_f32_cont(const half * x, float * y, int64_t k) { + const int64_t i = 2 * ((int64_t) blockDim.x * blockIdx.x + threadIdx.x); + if (i + 1 < k) { + ((float2 *) y)[i / 2] = __half22float2(((const half2 *) x)[i / 2]); + } else if (i < k) { + y[i] = x[i]; + } +} + template static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nblocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE); + if constexpr (std::is_same::value && std::is_same::value) { + if ((uintptr_t) vx % alignof(float2) == 0 && (uintptr_t) y % alignof(half2) == 0) { + convert_f32_f16_cont<<>>((const float *) vx, (half *) y, k); + return; + } + } + if constexpr (std::is_same::value && std::is_same::value) { + if ((uintptr_t) vx % alignof(half2) == 0 && (uintptr_t) y % alignof(float2) == 0) { + convert_f16_f32_cont<<>>((const half *) vx, (float *) y, k); + return; + } + } convert_unary_cuda(vx, y, k, 1, 1, 1, k, k, k, stream); } diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 7c2d3cad20a8..e1c156af2fea 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1834,7 +1834,8 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor const int cc = ggml_cuda_info().devices[ctx.device].cc; const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size; - if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, ne11)) { + const int64_t mmvf_ncols = hint == GGML_HINT_EXACT_BATCH && src0->type == GGML_TYPE_BF16 ? 1 : ne11; + if (ggml_cuda_should_use_mmvf(src0->type, cc, src0->ne, src0->nb, mmvf_ncols)) { // The custom F16 vector kernel can be used over batched cuBLAS GEMM. // But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention) ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); @@ -3286,6 +3287,32 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + if (node->op == GGML_OP_L2_NORM && i + 2 < cgraph->n_nodes && + GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + ggml_tensor * k_view = cgraph->nodes[i + 1]; + ggml_tensor * k_norm = cgraph->nodes[i + 2]; + const ggml_tensor * q_view = node->src[0]; + const int out_nodes[] = { i, i + 2 }; + const bool graph_ok = k_view->op == GGML_OP_VIEW && k_norm->op == GGML_OP_L2_NORM && + k_norm->src[0] == k_view && (k_view->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 && + (k_norm->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 && ggml_node_get_use_count(cgraph, i + 1) == 1 && + (k_view->flags & GGML_TENSOR_FLAG_OUTPUT) == 0; + const bool shape_ok = q_view && q_view->op == GGML_OP_VIEW && q_view->view_src && + q_view->view_src == k_view->view_src && node->ne[0] == 128 && ggml_are_same_shape(node, k_norm) && + ggml_are_same_shape(q_view, k_view) && q_view->type == GGML_TYPE_F32 && k_view->type == GGML_TYPE_F32 && + node->type == GGML_TYPE_F32 && k_norm->type == GGML_TYPE_F32; + const bool layout_ok = shape_ok && ggml_is_contiguous_rows(q_view) && ggml_is_contiguous_rows(k_view) && + ggml_is_contiguous(node) && ggml_is_contiguous(k_norm) && + q_view->nb[1] == k_view->nb[1] && q_view->nb[2] == k_view->nb[2] && q_view->nb[3] == k_view->nb[3] && + k_view->view_offs == q_view->view_offs + ggml_row_size(q_view->type, q_view->ne[0])*q_view->ne[1] && + q_view->nb[2] > ggml_row_size(q_view->type, q_view->ne[0])*q_view->ne[1] && q_view->ne[2] <= UINT16_MAX/2; + + if (graph_ok && layout_ok && ggml_cuda_check_fusion_memory_ranges(cgraph, i, 3, out_nodes, 2)) { + ggml_cuda_op_l2_norm_dual_s128(*cuda_ctx, node, k_norm); + return 2; + } + } + // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache if (node->op == GGML_OP_GATED_DELTA_NET) { ggml_cuda_gated_delta_net_fused_cache fused_state_cpy; @@ -3300,6 +3327,36 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } + if (node->op == GGML_OP_GLU && i + 1 < cgraph->n_nodes && ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU && node->src[1]) { + ggml_tensor * next = cgraph->nodes[i + 1]; + const bool has_ids = next->op == GGML_OP_MUL_MAT_ID; + if (has_ids || next->op == GGML_OP_MUL_MAT) { + const ggml_tensor * weights = next->src[0]; + const ggml_tensor * ids = has_ids ? next->src[2] : nullptr; + const ggml_tensor * gate = node->src[0]; + const ggml_tensor * up = node->src[1]; + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + const int64_t mmq_cols = has_ids ? node->ne[2] : node->ne[1]; + const int64_t n_experts = has_ids ? weights->ne[2] : 0; + const bool bad_padding_clear = ggml_backend_buffer_get_usage(weights->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && + ggml_nbytes(weights) != ggml_backend_buffer_get_alloc_size(weights->buffer, weights) && weights->view_src; + + const bool target_shape = node->ne[0] == 512 && next->ne[0] == 2048 && + (!has_ids || (gate->ne[1] == 8 && weights->ne[2] == 256)); + const bool shape_ok = target_shape && next->src[1] == node && gate->type == GGML_TYPE_F32 && up->type == GGML_TYPE_F32 && + node->type == GGML_TYPE_F32 && next->type == GGML_TYPE_F32 && weights->type == GGML_TYPE_Q8_0 && + ggml_are_same_shape(gate, up) && ggml_are_same_shape(gate, node) && gate->nb[0] == sizeof(float) && up->nb[0] == sizeof(float) && + gate->ne[3] == 1 && (has_ids ? gate->ne[1] == ids->ne[0] && gate->ne[2] == ids->ne[1] : gate->ne[2] == 1); + const bool use_mmq = shape_ok && !bad_padding_clear && GGML_CUDA_CC_IS_RDNA3_5(cc) && + ggml_cuda_should_use_mmq(weights->type, cc, mmq_cols, n_experts); + + if (use_mmq && ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_GLU, next->op }, { i + 1 })) { + ggml_cuda_mul_mat_q_swiglu(*cuda_ctx, weights, ids, next, node); + return 1; + } + } + } + //topk-moe if (cgraph->nodes[i]->op == GGML_OP_UNARY || cgraph->nodes[i]->op == GGML_OP_SOFT_MAX || cgraph->nodes[i]->op == GGML_OP_ARGSORT) { @@ -3432,6 +3489,86 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } + if (node->op == GGML_OP_MUL && node->ne[0] == 2048 && i + 2 < cgraph->n_nodes && + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_ADD }, { i + 2 })) { + ggml_tensor * add0 = cgraph->nodes[i + 1]; + ggml_tensor * add1 = cgraph->nodes[i + 2]; + const ggml_tensor * src = ggml_are_same_shape(node, node->src[0]) ? node->src[0] : node->src[1]; + const ggml_tensor * gate = src == node->src[0] ? node->src[1] : node->src[0]; + const ggml_tensor * other = add0->src[0]; + const ggml_tensor * residual = add1->src[1]; + const uintptr_t gate_begin = (uintptr_t) gate->data; + const uintptr_t gate_end = gate_begin + ggml_backend_buft_get_alloc_size(gate->buffer->buft, gate); + const uintptr_t dst_begin = (uintptr_t) add1->data; + const uintptr_t dst_end = dst_begin + ggml_backend_buft_get_alloc_size(add1->buffer->buft, add1); + const bool gate_overlap = gate_begin < dst_end && dst_begin < gate_end; + const auto safe_output_alias = [&](const ggml_tensor * input) { + const uintptr_t begin = (uintptr_t) input->data; + const uintptr_t end = begin + ggml_backend_buft_get_alloc_size(input->buffer->buft, input); + return !(begin < dst_end && dst_begin < end) || (input->data == add1->data && ggml_are_same_layout(input, add1)); + }; + + if (add0->src[1] == node && add1->src[0] == add0 && other != node && residual != node && residual != add0 && + node->type == GGML_TYPE_F32 && + src->type == GGML_TYPE_F32 && gate->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && residual->type == GGML_TYPE_F32 && + ggml_are_same_shape(node, src) && ggml_are_same_shape(node, other) && ggml_are_same_shape(node, residual) && + gate->ne[0] == 1 && gate->ne[1] == node->ne[1] && gate->ne[2] == node->ne[2] && gate->ne[3] == node->ne[3] && + ggml_is_contiguous(src) && ggml_is_contiguous(gate) && ggml_is_contiguous(other) && + ggml_is_contiguous(residual) && ggml_is_contiguous(add1) && !gate_overlap && + safe_output_alias(src) && safe_output_alias(other) && safe_output_alias(residual) && + ggml_nelements(add1) <= int64_t(INT_MAX) * 256) { + ggml_cuda_op_shared_mul_add(*cuda_ctx, node, other, residual, add1); + return 2; + } + } + + if (node->op == GGML_OP_MUL && node->type == GGML_TYPE_F32 && node->ne[0] == 2048 && + node->ne[3] == 1 && node->ne[1] == 8) { + const int n_expert_used = node->ne[1]; + const int n_ops = 2 * n_expert_used; + if (i + n_ops <= cgraph->n_nodes) { + std::vector ops = { GGML_OP_MUL }; + ops.insert(ops.end(), n_expert_used, GGML_OP_VIEW); + ops.insert(ops.end(), n_expert_used - 1, GGML_OP_ADD); + const int output_idx = i + n_ops - 1; + + if (ggml_can_fuse_subgraph(cgraph, i, n_ops, ops.data(), &output_idx, 1)) { + const ggml_tensor * experts = ggml_are_same_shape(node, node->src[0]) ? node->src[0] : node->src[1]; + const ggml_tensor * weights = experts == node->src[0] ? node->src[1] : node->src[0]; + ggml_tensor * output = cgraph->nodes[output_idx]; + bool valid = experts->op == GGML_OP_MUL_MAT_ID && experts->type == GGML_TYPE_F32 && weights->type == GGML_TYPE_F32 && + experts->ne[0] == node->ne[0] && experts->ne[1] == n_expert_used && experts->ne[2] == node->ne[2] && + experts->ne[3] == 1 && weights->ne[0] == 1 && weights->ne[1] == n_expert_used && + weights->ne[2] == node->ne[2] && weights->ne[3] == 1 && + ggml_is_contiguous(experts) && ggml_is_contiguous(weights) && ggml_is_contiguous(output) && + output->type == GGML_TYPE_F32 && output->ne[0] == node->ne[0] && output->ne[1] == node->ne[2] && + output->ne[2] == 1 && output->ne[3] == 1 && ggml_nelements(output) <= UINT32_MAX; + + for (int j = 0; valid && j < n_expert_used; ++j) { + const ggml_tensor * view = cgraph->nodes[i + 1 + j]; + valid = view->src[0] == node && view->ne[0] == node->ne[0] && view->ne[1] == node->ne[2] && + view->ne[2] == 1 && view->ne[3] == 1 && view->nb[0] == sizeof(float) && + view->nb[1] == node->nb[2] && view->view_offs == size_t(j) * node->nb[1]; + } + + const int add_start = i + 1 + n_expert_used; + if (valid) { + const ggml_tensor * first = cgraph->nodes[add_start]; + valid = first->src[0] == cgraph->nodes[i + 1] && first->src[1] == cgraph->nodes[i + 2]; + } + for (int j = 2; valid && j < n_expert_used; ++j) { + const ggml_tensor * add = cgraph->nodes[add_start + j - 1]; + valid = add->src[0] == cgraph->nodes[add_start + j - 2] && add->src[1] == cgraph->nodes[i + 1 + j]; + } + + if (valid) { + ggml_cuda_op_weighted_expert_sum(*cuda_ctx, node, output, n_expert_used); + return n_ops - 1; + } + } + } + } + // multi-(add or mul) if (node->op == GGML_OP_ADD || node->op == GGML_OP_MUL) { int n_fuse = 0; @@ -3468,6 +3605,39 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } + if ((node->op == GGML_OP_MUL_MAT_ID || node->op == GGML_OP_MUL_MAT) && i + 1 < cgraph->n_nodes) { + ggml_tensor * next = cgraph->nodes[i + 1]; + const ggml_tensor * src0 = node->src[0]; + const ggml_tensor * src0_next = next->src[0]; + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + + const bool has_ids = node->op == GGML_OP_MUL_MAT_ID; + const bool valid_sources = next->op == node->op && src0 && src0_next && node->src[1] && next->src[1] && + (!has_ids || node->src[2] && next->src[2]); + const bool ordinary_q8 = valid_sources && !has_ids && src0->type == GGML_TYPE_Q8_0 && src0_next->type == GGML_TYPE_Q8_0; + const bool shared_inputs = valid_sources && (has_ids || ordinary_q8) && node->src[1] == next->src[1] && + (!has_ids || node->src[2] == next->src[2]); + const int64_t mmq_cols = shared_inputs ? (has_ids ? node->src[1]->ne[2] : node->src[1]->ne[1]) : 0; + const int64_t n_experts = shared_inputs && has_ids ? src0->ne[2] : 0; + const bool use_mmq = shared_inputs && + ggml_cuda_should_use_mmq(src0->type, cc, mmq_cols, n_experts) && + ggml_cuda_should_use_mmq(src0_next->type, cc, mmq_cols, n_experts); + const bool compatible = use_mmq && node->src[1]->type == GGML_TYPE_F32 && + node->type == GGML_TYPE_F32 && next->type == GGML_TYPE_F32 && + ggml_are_same_shape(src0, src0_next) && ggml_are_same_shape(node, next) && + mmq_get_q8_1_ds_layout(src0->type) == mmq_get_q8_1_ds_layout(src0_next->type) && + !blackwell_mma_available(cc); + const int64_t ncols_dst = has_ids ? node->ne[2] : node->ne[1]; + const bool use_mmvq = compatible && ncols_dst <= MMVQ_MAX_BATCH_SIZE && + (!has_ids || ncols_dst <= get_mmvq_mmid_max_batch(src0->type, cc) || + ncols_dst <= get_mmvq_mmid_max_batch(src0_next->type, cc)); + + if (compatible && !use_mmvq) { + ggml_cuda_mul_mat_q_pair(*cuda_ctx, node, next); + return 1; + } + } + bool fused_mul_mat_vec = false; int fused_node_count = 0; @@ -3998,6 +4168,24 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 1; } + if (node->op == GGML_OP_CONT && i + 2 < cgraph->n_nodes) { + ggml_tensor * unary = cgraph->nodes[i + 1]; + ggml_tensor * mul = cgraph->nodes[i + 2]; + const ggml_tensor * gate = node->src[0]; + const ggml_tensor * other = mul->src[0] == unary ? mul->src[1] : mul->src[0]; + const int out_nodes[] = { i + 2 }; + if (unary->op == GGML_OP_UNARY && ggml_get_unary_op(unary) == GGML_UNARY_OP_SIGMOID && mul->op == GGML_OP_MUL && + (mul->src[0] == unary || mul->src[1] == unary) && gate->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && + gate->nb[0] == sizeof(float) && gate->nb[1] == 2 * gate->ne[0] * sizeof(float) && + gate->nb[2] == gate->nb[1] * gate->ne[1] && gate->nb[3] == gate->nb[2] * gate->ne[2] && + ggml_is_contiguous(other) && ggml_are_same_shape(gate, other) && + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_CONT, GGML_OP_UNARY, GGML_OP_MUL }, { i + 2 }) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, 3, out_nodes, 1)) { + ggml_cuda_op_cont_sigmoid_mul(*cuda_ctx, node, unary, mul); + return 2; + } + } + if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SILU }) || ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SIGMOID }) || ggml_cuda_can_fuse(cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL }, { GGML_UNARY_OP_SOFTPLUS })) { diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index c3758cd50cfe..50f7367046db 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -277,6 +277,44 @@ static __global__ void l2_norm_f32( } } +static __global__ void l2_norm_dual_f32_s128( + const float * x_q, float * dst_q, const int64_t stride_q_row, const int64_t stride_q_channel, + const int64_t stride_q_sample, const float eps_q, + const float * x_k, float * dst_k, const int64_t stride_k_row, const int64_t stride_k_channel, + const int64_t stride_k_sample, const float eps_k) { + const int nrows = gridDim.x; + const int nchannels = gridDim.y / 2; + const bool is_k = blockIdx.y >= nchannels; + const int row = blockIdx.x; + const int channel = blockIdx.y % nchannels; + const int sample = blockIdx.z; + const int tid = threadIdx.x; + + const float * x = is_k ? x_k : x_q; + float * dst = is_k ? dst_k : dst_q; + const int64_t stride_row = is_k ? stride_k_row : stride_q_row; + const int64_t stride_channel = is_k ? stride_k_channel : stride_q_channel; + const int64_t stride_sample = is_k ? stride_k_sample : stride_q_sample; + const float eps = is_k ? eps_k : eps_q; + + x += sample*stride_sample + channel*stride_channel + row*stride_row; + dst += ((sample*nchannels + channel)*nrows + row)*128; + + float tmp = 0.0f; + ggml_cuda_pdl_sync(); + for (int col = tid; col < 128; col += WARP_SIZE) { + const float xi = x[col]; + tmp += xi * xi; + } + tmp = warp_reduce_sum(tmp); + ggml_cuda_pdl_lc(); + + const float scale = rsqrtf(fmaxf(tmp, eps * eps)); + for (int col = tid; col < 128; col += WARP_SIZE) { + dst[col] = scale * x[col]; + } +} + static void norm_f32_cuda( const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) { @@ -360,7 +398,14 @@ static void rms_norm_mul_f32_cuda(const float * x, const uint3 mul_nrows_packed = init_fastdiv_values(mul_nrows); const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); - if (ncols < 1024) { + if (ncols == 128) { + const dim3 block_dims(128, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream}; + ggml_cuda_kernel_launch(rms_norm_f32<128, true>, launch_params, + x, dst, ncols, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel, + mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0)); + } else if (ncols < 1024) { const dim3 block_dims(256, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream}; ggml_cuda_kernel_launch(rms_norm_f32<256, true>, launch_params, @@ -696,3 +741,28 @@ void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { l2_norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); } + +void ggml_cuda_op_l2_norm_dual_s128(ggml_backend_cuda_context & ctx, ggml_tensor * dst_q, ggml_tensor * dst_k) { + const ggml_tensor * src_q = dst_q->src[0]; + const ggml_tensor * src_k = dst_k->src[0]; + + GGML_ASSERT(src_q->type == GGML_TYPE_F32 && src_k->type == GGML_TYPE_F32); + GGML_ASSERT(dst_q->type == GGML_TYPE_F32 && dst_k->type == GGML_TYPE_F32); + GGML_ASSERT(src_q->ne[0] == 128 && ggml_are_same_shape(src_q, src_k)); + GGML_ASSERT(src_q->nb[0] == sizeof(float) && src_k->nb[0] == sizeof(float)); + + float eps_q; + float eps_k; + memcpy(&eps_q, dst_q->op_params, sizeof(float)); + memcpy(&eps_k, dst_k->op_params, sizeof(float)); + GGML_ASSERT(eps_q >= 0.0f && eps_k >= 0.0f); + + const dim3 blocks_num(src_q->ne[1], 2*src_q->ne[2], src_q->ne[3]); + const dim3 block_dims(WARP_SIZE, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = { blocks_num, block_dims, 0, ctx.stream() }; + ggml_cuda_kernel_launch(l2_norm_dual_f32_s128, launch_params, + (const float *) src_q->data, (float *) dst_q->data, + src_q->nb[1]/sizeof(float), src_q->nb[2]/sizeof(float), src_q->nb[3]/sizeof(float), eps_q, + (const float *) src_k->data, (float *) dst_k->data, + src_k->nb[1]/sizeof(float), src_k->nb[2]/sizeof(float), src_k->nb[3]/sizeof(float), eps_k); +} diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index a74f6376720a..c09d004905a9 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -16,3 +16,5 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_op_l2_norm_dual_s128(ggml_backend_cuda_context & ctx, ggml_tensor * dst_q, ggml_tensor * dst_k); diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index d3e594878fcc..71a7d24c5552 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -702,6 +702,23 @@ void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary } } +void ggml_cuda_op_cont_sigmoid_mul( + ggml_backend_cuda_context & ctx, ggml_tensor * cont_node, ggml_tensor * unary_node, ggml_tensor * mul_node) { + const ggml_tensor * gate = cont_node->src[0]; + const ggml_tensor * other = mul_node->src[0] == unary_node ? mul_node->src[1] : mul_node->src[0]; + + GGML_ASSERT(ggml_get_unary_op(unary_node) == GGML_UNARY_OP_SIGMOID); + GGML_ASSERT(gate->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && mul_node->type == GGML_TYPE_F32); + GGML_ASSERT(gate->nb[0] == sizeof(float) && gate->nb[2] == gate->nb[1] * gate->ne[1]); + GGML_ASSERT(ggml_is_contiguous(other) && ggml_is_contiguous(mul_node)); + GGML_ASSERT(ggml_nelements(gate) == ggml_nelements(other)); + + const int64_t k = ggml_nelements(other); + const int64_t n = gate->ne[0]; + unary_gated_cuda((const float *) gate->data, (const float *) other->data, (float *) mul_node->data, + k, n, gate->nb[1] / sizeof(float), n, ctx.stream()); +} + /* fused relu + sqr */ void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node) { diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuh index 04f3af6443a7..291fe16eaa33 100644 --- a/ggml/src/ggml-cuda/unary.cuh +++ b/ggml/src/ggml-cuda/unary.cuh @@ -93,6 +93,9 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node); +void ggml_cuda_op_cont_sigmoid_mul( + ggml_backend_cuda_context & ctx, ggml_tensor * cont_node, ggml_tensor * unary_node, ggml_tensor * mul_node); + void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node); __device__ __forceinline__ float ggml_cuda_op_silu_single(float x) { From 3262315ad92fd83caca108b683144e657706ee9b Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:23:48 +0200 Subject: [PATCH 03/30] hip: optimize Flash Attention on RDNA3.5 Dequantize Q8 KV data directly in tiles, specialize GQA decode grouping, reduce D256 register pressure, and reuse masks across grouped heads on gfx1151. Assisted-by: OpenCode --- ggml/src/ggml-cuda/fattn-common.cuh | 6 + ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu | 12 ++ ggml/src/ggml-cuda/fattn-tile.cuh | 232 +++++++++++++++++------ ggml/src/ggml-cuda/fattn.cu | 23 +++ 4 files changed, 215 insertions(+), 58 deletions(-) create mode 100644 ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh index e67cc7fdf784..25854e94617d 100644 --- a/ggml/src/ggml-cuda/fattn-common.cuh +++ b/ggml/src/ggml-cuda/fattn-common.cuh @@ -1174,6 +1174,12 @@ void launch_fattn( } } + if constexpr (ncols2 == 3) { + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + parallel_blocks = std::min(18, ntiles_KV); + } + } + blocks_num.x = ntiles_x; blocks_num.y = parallel_blocks; blocks_num.z = ntiles_z_gqa*K->ne[2]*Q->ne[3]; diff --git a/ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu b/ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu new file mode 100644 index 000000000000..586632da20e4 --- /dev/null +++ b/ggml/src/ggml-cuda/fattn-tile-rdna3-5.cu @@ -0,0 +1,12 @@ +#if defined(GGML_USE_HIP) + +#define GGML_CUDA_FATTN_VGPR192 +#include "fattn-tile.cuh" + +fattn_kernel_t ggml_cuda_fattn_tile_d256_ncols32_rdna3_5(const bool use_logit_softcap) { + return use_logit_softcap + ? flash_attn_tile<256, 256, 4, 8, true, false> + : flash_attn_tile<256, 256, 4, 8, false, false>; +} + +#endif // defined(GGML_USE_HIP) diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index d1164b8526d3..53029262aec4 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -213,6 +213,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 256, 2, 128, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 3, 96, 8, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 256, 2, 64, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128) @@ -291,6 +292,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 3, 64, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 64, 8, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 3, 96, 8, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 128, 6, 32, 256) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 128, 6, 32, 256) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 5, 32, 256) @@ -479,14 +481,46 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile( ggml_cuda_unroll<5>{}(load); } +template +static __device__ __forceinline__ void flash_attn_tile_load_tile_q8_0( + const char * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int stride_KV, const int i_sup) { + static_assert(J % QK8_0 == 0, "bad J"); + + constexpr int nthreads = warp_size*nwarps; + constexpr int groups_per_row = J/4; + const int tid = threadIdx.y*warp_size + threadIdx.x; + +#pragma unroll + for (int ig = tid; ig < I*groups_per_row; ig += nthreads) { + const int i = ig/groups_per_row; + const int j = (ig % groups_per_row)*4; + + half2 v0 = make_half2(0.0f, 0.0f); + half2 v1 = make_half2(0.0f, 0.0f); + if (!oob_check || i < i_sup) { + const block_q8_0 * row = (const block_q8_0 *) (KV + int64_t(i)*stride_KV); + const block_q8_0 & block = row[j/QK8_0]; + int q; + ggml_cuda_memcpy_1(&q, block.qs + j % QK8_0); + const int8_t * q8 = (const int8_t *) &q; + const half2 d = __half2half2(block.d); + v0 = d*make_half2(q8[0], q8[1]); + v1 = d*make_half2(q8[2], q8[3]); + } + + tile_KV[i*(J/2 + J_padding) + j/2 + 0] = v0; + tile_KV[i*(J/2 + J_padding) + j/2 + 1] = v1; + } +} + // Function that performs a single iteration in for the KQ matrix multiplication: template + bool use_logit_softcap, bool oob_check, bool q8_0_KV, typename T_vec_dot> static __device__ __forceinline__ void flash_attn_tile_iter_KQ( T_vec_dot * const Q_tmp, - const half2 * const __restrict__ K_h2, + const char * const __restrict__ K_data, T_vec_dot * const KV_tmp, - const int stride_K2, + const int stride_K, const int k_VKQ_0, const int k_VKQ_sup, const int k_KQ_0, @@ -498,8 +532,13 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ( constexpr int cpw = ncols > nwarps ? ncols/nwarps : 1; // Q columns per warp constexpr int np = nwarps > ncols ? nwarps/ncols : 1; // number of parallel warps per Q column - flash_attn_tile_load_tile - (K_h2 + int64_t(k_VKQ_0)*stride_K2 + k_KQ_0/2, KV_tmp, stride_K2, k_VKQ_sup); + if constexpr (q8_0_KV) { + flash_attn_tile_load_tile_q8_0 + (K_data + int64_t(k_VKQ_0)*stride_K + k_KQ_0/QK8_0*sizeof(block_q8_0), KV_tmp, stride_K, k_VKQ_sup); + } else { + flash_attn_tile_load_tile + ((const half2 *) (K_data + int64_t(k_VKQ_0)*stride_K + k_KQ_0*sizeof(half)), KV_tmp, stride_K/sizeof(half2), k_VKQ_sup); + } __syncthreads(); #ifdef FAST_FP16_AVAILABLE @@ -537,13 +576,29 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ( #endif // FAST_FP16_AVAILABLE } +#if defined(RDNA3_5) + if constexpr (DKQ == 256 && ncols == 32) { #pragma unroll - for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) { + for (int k = 0; k < cpy_ne; ++k) { #pragma unroll - for (int jc0 = 0; jc0 < cpw; ++jc0) { + for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) { #pragma unroll - for (int k = 0; k < cpy_ne; ++k) { - ggml_cuda_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]); + for (int jc0 = 0; jc0 < cpw; ++jc0) { + ggml_cuda_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]); + } + } + } + } else +#endif // defined(RDNA3_5) + { +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < nbatch_fa; i_KQ_0 += np*warp_size) { +#pragma unroll + for (int jc0 = 0; jc0 < cpw; ++jc0) { +#pragma unroll + for (int k = 0; k < cpy_ne; ++k) { + ggml_cuda_mad(KQ_acc[i_KQ_0/(np*warp_size)*cpw + jc0], K_k[i_KQ_0/(np*warp_size)][k], Q_k[jc0][k]); + } } } } @@ -556,19 +611,19 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ( // Function that performs a single iteration of the main loop over up to nbatch_fa tokens. template + bool use_logit_softcap, bool oob_check, bool q8_0_KV, typename T_vec_dot, typename T_KQ, typename T_acc> static __device__ __forceinline__ void flash_attn_tile_iter( T_vec_dot * const Q_tmp, - const half2 * const __restrict__ K_h2, - const half2 * const __restrict__ V_h2, + const char * const __restrict__ K_data, + const char * const __restrict__ V_data, const half * const __restrict__ mask, const uint3 ne01, const float logit_softcap, const float slope, T_KQ * const KQ, T_vec_dot * const KV_tmp, - const int stride_K2, - const int stride_V2, + const int stride_K, + const int stride_V, const int stride_mask, float * const KQ_max, float * const KQ_sum, @@ -601,19 +656,32 @@ static __device__ __forceinline__ void flash_attn_tile_iter( KQ_max_new[jc0] = KQ_max[jc0]; } +#if defined(RDNA3_5) + constexpr bool common_mask = DKQ == 256 && DV == 256 && ncols1 == 4 && ncols2 == 8; + const int j_mask = common_mask ? fastmodulo(col_Q_0 + ((threadIdx.y / np)*cpw)/ncols2, ne01) : 0; + float mask_value[common_mask ? nbatch_fa/(np*warp_size) : 1]; +#pragma unroll + for (int i0 = 0; i0 < nbatch_fa; i0 += np*warp_size) { + mask_value[i0/(np*warp_size)] = common_mask ? slope*__half2float(mask[j_mask*stride_mask + k_VKQ_0 + i0 + (threadIdx.y % np)*warp_size + threadIdx.x]) : 0.0f; + } +#else + constexpr bool common_mask = false; + const float mask_value[1] = {0.0f}; +#endif + float KQ_acc[nbatch_fa/(np*warp_size) * cpw] = {0.0f}; // Accumulators for KQ matrix multiplication. // KQ = K @ Q matrix multiplication: constexpr int nbatch_K_last = DKQ % nbatch_K; #pragma unroll for (int k_KQ_0 = 0; k_KQ_0 < DKQ - nbatch_K_last; k_KQ_0 += nbatch_K) { - flash_attn_tile_iter_KQ( - Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc); + flash_attn_tile_iter_KQ( + Q_tmp, K_data, KV_tmp, stride_K, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc); } - if (nbatch_K_last > 0) { + if constexpr (nbatch_K_last > 0) { constexpr int k_KQ_0 = DKQ - nbatch_K_last; - flash_attn_tile_iter_KQ( - Q_tmp, K_h2, KV_tmp, stride_K2, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc); + flash_attn_tile_iter_KQ( + Q_tmp, K_data, KV_tmp, stride_K, k_VKQ_0, k_VKQ_sup, k_KQ_0, KQ_acc); } // Apply logit softcap + mask, update KQ_max: @@ -636,8 +704,8 @@ static __device__ __forceinline__ void flash_attn_tile_iter( } if (!oob_check || i_KQ < k_VKQ_sup) { - KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] += (ncols2 > 1 || mask) ? - slope*__half2float(mask[j*stride_mask + k_VKQ_0 + i_KQ]) : 0.0f; + KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] += common_mask ? mask_value[i_KQ_0/(np*warp_size)] : + (ncols2 > 1 || mask) ? slope*__half2float(mask[j*stride_mask + k_VKQ_0 + i_KQ]) : 0.0f; KQ_max_new[jc0] = fmaxf(KQ_max_new[jc0], KQ_acc[(i_KQ_0/(np*warp_size))*cpw + jc0] + FATTN_KQ_MAX_OFFSET); } @@ -718,8 +786,13 @@ static __device__ __forceinline__ void flash_attn_tile_iter( static_assert(nbatch_V % np == 0, "bad nbatch_V"); #pragma unroll for (int k0 = 0; k0 < nbatch_fa; k0 += nbatch_V) { - flash_attn_tile_load_tile - (V_h2 + int64_t(k_VKQ_0 + k0)*stride_V2, KV_tmp, stride_V2, k_VKQ_sup - k0); + if constexpr (q8_0_KV) { + flash_attn_tile_load_tile_q8_0 + (V_data + int64_t(k_VKQ_0 + k0)*stride_V, KV_tmp, stride_V, k_VKQ_sup - k0); + } else { + flash_attn_tile_load_tile + ((const half2 *) (V_data + int64_t(k_VKQ_0 + k0)*stride_V), KV_tmp, stride_V/sizeof(half2), k_VKQ_sup - k0); + } __syncthreads(); #ifdef FAST_FP16_AVAILABLE @@ -788,8 +861,11 @@ static __device__ __forceinline__ void flash_attn_tile_iter( } } -template // D == head size +template // D == head size __launch_bounds__(ggml_cuda_fattn_tile_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_tile_get_occupancy(DKQ, DV, ncols1*ncols2)) +#if defined(RDNA3_5) && defined(GGML_CUDA_FATTN_VGPR192) +__attribute__((amdgpu_num_vgpr(192))) +#endif // defined(RDNA3_5) && defined(GGML_CUDA_FATTN_VGPR192) static __global__ void flash_attn_tile( const char * Q_ptr, const char * K_ptr, @@ -853,14 +929,12 @@ static __global__ void flash_attn_tile( const int sequence = blockIdx.z / (ne02/ncols2); const int head0 = blockIdx.z*ncols2 - sequence*ne02; // == blockIdx.z % (ne02/ncols2) const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float * Q_f = (const float *) (Q + nb03*sequence + nb02* head0); - const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*(head0 / gqa_ratio)); - const half2 * V_h2 = (const half2 *) (V + nb23*sequence + nb22*(head0 / gqa_ratio)); // K and V have same shape + const float * Q_f = (const float *) (Q + nb03*sequence + nb02* head0); + const char * K_data = K + nb13*sequence + nb12*(head0 / gqa_ratio); + const char * V_data = V + nb23*sequence + nb22*(head0 / gqa_ratio); const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33)) : nullptr; - const int stride_K2 = nb11 / sizeof(half2); - const int stride_V2 = nb21 / sizeof(half2); const int stride_mask = nb31 / sizeof(half); const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, head0, n_head_log2, m0, m1) : 1.0f; @@ -957,24 +1031,24 @@ static __global__ void flash_attn_tile( int k_VKQ_0 = blockIdx.y*nbatch_fa; while (k_VKQ_0 < k_VKQ_max - nbatch_fa) { constexpr bool oob_check = false; - flash_attn_tile_iter - (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, - stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); + flash_attn_tile_iter + (Q_tmp, K_data, V_data, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, + nb11, nb21, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); k_VKQ_0 += gridDim.y*nbatch_fa; } if (k_VKQ_0 < k_VKQ_max) { constexpr bool oob_check = true; - flash_attn_tile_iter - (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, - stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); + flash_attn_tile_iter + (Q_tmp, K_data, V_data, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, + nb11, nb21, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); } } else { // Branch without out-of-bounds checks. for (int k_VKQ_0 = blockIdx.y*nbatch_fa; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*nbatch_fa) { constexpr bool oob_check = false; - flash_attn_tile_iter - (Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, - stride_K2, stride_V2, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); + flash_attn_tile_iter + (Q_tmp, K_data, V_data, maskh, ne01, logit_softcap, slope, KQ, KV_tmp, + nb11, nb21, stride_mask, KQ_max, KQ_sum, VKQ, k_VKQ_0, k_VKQ_max, col_Q_0); } } @@ -1145,6 +1219,38 @@ static __global__ void flash_attn_tile( #endif // FLASH_ATTN_AVAILABLE } +#ifdef GGML_USE_HIP +fattn_kernel_t ggml_cuda_fattn_tile_d256_ncols32_rdna3_5(const bool use_logit_softcap); +#endif // GGML_USE_HIP + +template +static void launch_fattn_tile_case( + ggml_backend_cuda_context & ctx, ggml_tensor * dst, const int nwarps, const int nbatch_fa, const int warp_size) { + constexpr size_t nbytes_shared = 0; + bool use_q8_0_KV = false; + fattn_kernel_t fattn_kernel; +#ifdef GGML_USE_HIP + if constexpr (DKQ == DV && (DKQ == 64 || DKQ == 128 || DKQ == 256)) { + use_q8_0_KV = dst->src[0]->ne[1] == 1 && dst->src[1]->type == GGML_TYPE_Q8_0 && dst->src[2]->type == GGML_TYPE_Q8_0; + fattn_kernel = use_q8_0_KV + ? flash_attn_tile + : flash_attn_tile; + } else { + fattn_kernel = flash_attn_tile; + } + if constexpr (DKQ == 256 && DV == 256 && ncols1 == 4 && ncols2 == 8) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + fattn_kernel = ggml_cuda_fattn_tile_d256_ncols32_rdna3_5(use_logit_softcap); + } + } +#else + fattn_kernel = flash_attn_tile; +#endif // GGML_USE_HIP + launch_fattn + (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, !use_q8_0_KV, !use_q8_0_KV, false, warp_size); +} + template static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * Q = dst->src[0]; @@ -1153,7 +1259,12 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm const int cc = ggml_cuda_info().devices[id].cc; const int warp_size = 32; - constexpr size_t nbytes_shared = 0; + if constexpr (ncols2 == 3) { + const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, 3, cc) / warp_size; + const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, 3, cc); + launch_fattn_tile_case(ctx, dst, nwarps, nbatch_fa, warp_size); + return; + } else { #ifdef GGML_USE_HIP if constexpr (DKQ <= 128) { @@ -1161,9 +1272,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm constexpr int cols_per_block = 64; const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); - fattn_kernel_t fattn_kernel = flash_attn_tile; - launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + launch_fattn_tile_case + (ctx, dst, nwarps, nbatch_fa, warp_size); return; } } @@ -1177,9 +1287,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm constexpr int cols_per_block = 32; const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); - fattn_kernel_t fattn_kernel = flash_attn_tile; - launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + launch_fattn_tile_case + (ctx, dst, nwarps, nbatch_fa, warp_size); return; } } @@ -1189,9 +1298,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm constexpr int cols_per_block = 16; const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); - fattn_kernel_t fattn_kernel = flash_attn_tile; - launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + launch_fattn_tile_case + (ctx, dst, nwarps, nbatch_fa, warp_size); return; } } @@ -1201,9 +1309,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm constexpr int cols_per_block = 8; const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); - fattn_kernel_t fattn_kernel = flash_attn_tile; - launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + launch_fattn_tile_case + (ctx, dst, nwarps, nbatch_fa, warp_size); return; } } @@ -1213,9 +1320,8 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm constexpr int cols_per_block = 4; const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); - fattn_kernel_t fattn_kernel = flash_attn_tile; - launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + launch_fattn_tile_case + (ctx, dst, nwarps, nbatch_fa, warp_size); return; } } @@ -1224,13 +1330,13 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggm constexpr int cols_per_block = 2; const int nwarps = ggml_cuda_fattn_tile_get_nthreads (DKQ, DV, cols_per_block, cc) / warp_size; const int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc); - fattn_kernel_t fattn_kernel = flash_attn_tile; - launch_fattn - (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size); + launch_fattn_tile_case + (ctx, dst, nwarps, nbatch_fa, warp_size); return; } - GGML_ABORT("fatal error"); + GGML_ABORT("fatal error"); + } } template @@ -1238,6 +1344,7 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm const ggml_tensor * KQV = dst; const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; float max_bias = 0.0f; @@ -1296,6 +1403,15 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm } if constexpr (DKQ <= 512 && DKQ != 320 && DKQ != 192) { + if constexpr (DKQ == 256 && DV == 256) { +#ifdef GGML_USE_HIP + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (use_gqa_opt && Q->ne[1] == 1 && gqa_ratio == 6 && K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_Q8_0 && GGML_CUDA_CC_IS_RDNA3_5(cc)) { + launch_fattn_tile_switch_ncols1(ctx, dst); + return; + } +#endif // GGML_USE_HIP + } if (use_gqa_opt && gqa_ratio % 8 == 0) { launch_fattn_tile_switch_ncols1(ctx, dst); return; diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index ab7a3b297c07..ef6bf7adea4c 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -355,6 +355,19 @@ static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) { } } +static bool ggml_cuda_fattn_tile_q8_0_KV_supported(const ggml_tensor * dst) { +#ifdef GGML_USE_HIP + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + return Q->ne[1] == 1 && K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_Q8_0 && Q->ne[0] == K->ne[0] && K->ne[0] == V->ne[0] && + (Q->ne[0] == 64 || Q->ne[0] == 128 || Q->ne[0] == 256); +#else + GGML_UNUSED(dst); + return false; +#endif // GGML_USE_HIP +} + static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { #ifndef FLASH_ATTN_AVAILABLE GGML_UNUSED(device); GGML_UNUSED(dst); @@ -488,6 +501,10 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const gqa_ratio_eff *= 2; } + if (ggml_cuda_fattn_tile_q8_0_KV_supported(dst) && gqa_opt_applies && gqa_ratio_eff >= 2) { + return BEST_FATTN_KERNEL_TILE; + } + if (volta_mma_available(cc) && Q->ne[0] != 40 && Q->ne[0] != 72) { if (can_use_vector_kernel && Q->ne[1] * gqa_ratio_eff <= 2) { return BEST_FATTN_KERNEL_VEC; @@ -549,6 +566,12 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d switch (kernel) { case BEST_FATTN_KERNEL_TILE: + if (ggml_cuda_fattn_tile_q8_0_KV_supported(dst)) { + break; + } + need_f16_K = true; + need_f16_V = true; + break; case BEST_FATTN_KERNEL_MMA_F16: need_f16_K = true; need_f16_V = true; From 39765ff5c435ebf1cc29da198ecabbe02d361062 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:23:58 +0200 Subject: [PATCH 04/30] hip: tune Qwen Gated DeltaNet execution Increase gfx1151 column parallelism, tile KDA inputs in LDS, preserve recurrent graph ordering, and schedule compatible Qwen projections together. Assisted-by: OpenCode --- ggml/src/ggml-cuda/gated_delta_net.cu | 146 +++++++++++++++++++++++++- src/models/qwen35.cpp | 12 +++ src/models/qwen35moe.cpp | 14 ++- 3 files changed, 165 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index 1b431a724d72..8fab7a3e5dfb 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -1,8 +1,21 @@ #include "gated_delta_net.cuh" #include "ggml-cuda/common.cuh" +static constexpr int gated_delta_net_num_warps(int cc) { + return GGML_CUDA_CC_IS_RDNA3_5(cc) ? 32 : 4; +} + +static_assert(gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3_5) == 32); +static_assert(gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3) == 4); + +#if defined(RDNA3_5) +static constexpr int gdn_num_warps = gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3_5); +#else +static constexpr int gdn_num_warps = gated_delta_net_num_warps(GGML_CUDA_CC_RDNA3); +#endif + template -__global__ void __launch_bounds__((ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v) * 4, 2) +__global__ void __launch_bounds__((ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v) * gdn_num_warps, 2) gated_delta_net_cuda(const float * q, const float * k, const float * v, @@ -166,6 +179,119 @@ gated_delta_net_cuda(const float * q, } } +__global__ void __launch_bounds__(512, 2) +gated_delta_net_kda_tiled_128_cuda(const float * q, + const float * k, + const float * v, + const float * g, + const float * beta, + const float * curr_state, + float * dst, + float * state, + int64_t n_tokens, + int64_t sq1, + int64_t sq2, + int64_t sq3, + int64_t sv1, + int64_t sv2, + int64_t sv3, + int64_t sb1, + int64_t sb2, + int64_t sb3, + const uint3 neqk1_magic, + const uint3 rq3_magic, + float scale) { + constexpr int S_v = 128; + constexpr int H = 32; + constexpr int token_tile = 16; + constexpr int warp_size = 32; + constexpr int rows_per_lane = S_v / warp_size; + + __shared__ float q_shared[token_tile][S_v]; + __shared__ float k_shared[token_tile][S_v]; + __shared__ float g_shared[token_tile][S_v]; + __shared__ float beta_shared[token_tile]; + + const int h_idx = blockIdx.x; + const int sequence = blockIdx.y; + const int lane = threadIdx.x; + const int col = blockIdx.z * blockDim.y + threadIdx.y; + const int thread = threadIdx.y * warp_size + lane; + const int nthreads = blockDim.y * warp_size; + + const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); + const uint32_t iq3 = fastdiv(sequence, rq3_magic); + + const int64_t state_offset = (sequence * H + h_idx) * S_v * S_v; + curr_state += state_offset + col * S_v; + state += state_offset; + dst += (sequence * n_tokens * H + h_idx) * S_v; + + float s_shard[rows_per_lane]; + ggml_cuda_pdl_sync(); +#pragma unroll + for (int r = 0; r < rows_per_lane; ++r) { + s_shard[r] = curr_state[r * warp_size + lane]; + } + + for (int t0 = 0; t0 < n_tokens; t0 += token_tile) { + const int tile_size = min((int64_t) token_tile, n_tokens - t0); + for (int idx = thread; idx < tile_size * S_v; idx += nthreads) { + const int tt = idx / S_v; + const int i = idx % S_v; + const int t = t0 + tt; + const int64_t gb_offset = sequence * sb3 + t * sb2 + h_idx * sb1; + q_shared[tt][i] = q[iq3 * sq3 + t * sq2 + iq1 * sq1 + i]; + k_shared[tt][i] = k[iq3 * sq3 + t * sq2 + iq1 * sq1 + i]; + g_shared[tt][i] = g[gb_offset * S_v + i]; + } + if (thread < tile_size) { + beta_shared[thread] = beta[sequence * sb3 + (t0 + thread) * sb2 + h_idx * sb1]; + } + __syncthreads(); + + for (int tt = 0; tt < tile_size; ++tt) { + const int t = t0 + tt; + const float * v_t = v + sequence * sv3 + t * sv2 + h_idx * sv1; + float k_reg[rows_per_lane]; + float q_reg[rows_per_lane]; +#pragma unroll + for (int r = 0; r < rows_per_lane; ++r) { + const int i = r * warp_size + lane; + k_reg[r] = k_shared[tt][i]; + q_reg[r] = q_shared[tt][i]; + } + + float kv_shard = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; ++r) { + const int i = r * warp_size + lane; + kv_shard += expf(g_shared[tt][i]) * s_shard[r] * k_reg[r]; + } + const float kv_col = warp_reduce_sum(kv_shard); + const float delta_col = (v_t[col] - kv_col) * beta_shared[tt]; + + float attn_partial = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; ++r) { + const int i = r * warp_size + lane; + s_shard[r] = expf(g_shared[tt][i]) * s_shard[r] + k_reg[r] * delta_col; + attn_partial += s_shard[r] * q_reg[r]; + } + const float attn_col = warp_reduce_sum(attn_partial); + if (lane == 0) { + dst[t * S_v * H + col] = attn_col * scale; + } + } + __syncthreads(); + } + +#pragma unroll + for (int r = 0; r < rows_per_lane; ++r) { + state[col * S_v + r * warp_size + lane] = s_shard[r]; + } +} + template static void launch_gated_delta_net( const float * q_d, const float * k_d, const float * v_d, @@ -178,14 +304,28 @@ static void launch_gated_delta_net( int64_t neqk1, int64_t rq3, float scale, int64_t state_slot_stride, int K, cudaStream_t stream) { //TODO: Add chunked kernel for even faster pre-fill - const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; - const int num_warps = 4; + const int id = ggml_cuda_get_device(); + const int cc = ggml_cuda_info().devices[id].cc; + const int warp_size = ggml_cuda_info().devices[id].warp_size; + const int num_warps = GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && H == 64 && !KDA ? 32 : + (GGML_CUDA_CC_IS_RDNA3_5(cc) ? 16 : gated_delta_net_num_warps(cc)); dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps); dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1); const uint3 neqk1_magic = init_fastdiv_values(neqk1); const uint3 rq3_magic = init_fastdiv_values(rq3); + if constexpr (KDA && !keep_rs_t) { + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && H == 32 && n_tokens >= 16 && n_seqs == 1) { + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); + ggml_cuda_kernel_launch(gated_delta_net_kda_tiled_128_cuda, launch_params, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, n_tokens, + sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, + neqk1_magic, rq3_magic, scale); + return; + } + } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); switch (S_v) { case 16: diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 309dd432447c..696c85811a50 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -221,6 +221,9 @@ llama_model_qwen35::graph::graph(const llama_model & model, const llm_graph_para // LM head cur = build_lora_mm(model.output, cur, model.output_s); + if (cparams.embeddings_nextn && n_tokens <= 16 && cur->op == GGML_OP_MUL_MAT) { + ggml_mul_mat_set_hint(cur, GGML_HINT_EXACT_BATCH); + } cb(cur, "result_output", -1); res->t_logits = cur; @@ -268,6 +271,9 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( // Qwen3Next uses a single Q projection that outputs query + gate ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ] + if (cparams.embeddings_nextn && n_tokens <= 16 && Qcur_full->op == GGML_OP_MUL_MAT) { + ggml_mul_mat_set_hint(Qcur_full, GGML_HINT_EXACT_BATCH); + } cb(Qcur_full, "Qcur_full", il); ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens, @@ -280,9 +286,15 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn( cb(Qcur, "Qcur_normed", il); ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s); + if (cparams.embeddings_nextn && n_tokens <= 16 && Kcur->op == GGML_OP_MUL_MAT) { + ggml_mul_mat_set_hint(Kcur, GGML_HINT_EXACT_BATCH); + } cb(Kcur, "Kcur", il); ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s); + if (cparams.embeddings_nextn && n_tokens <= 16 && Vcur->op == GGML_OP_MUL_MAT) { + ggml_mul_mat_set_hint(Vcur, GGML_HINT_EXACT_BATCH); + } cb(Vcur, "Vcur", il); // Apply K normalization diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 38f2a57985a9..d091e857e00a 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -187,13 +187,15 @@ llama_model_qwen35moe::graph::graph(const llama_model & model, const llm_graph_p cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "attn_norm", il); - ggml_build_forward_expand(gf, cur); - // Determine layer type and build appropriate attention mechanism if (hparams.is_recr(il)) { // Linear attention layer (gated delta net) cur = build_layer_attn_linear(inp->get_recr(), cur, il); + + ggml_build_forward_expand(gf, cur); } else { + ggml_build_forward_expand(gf, cur); + // Full attention layer cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il); } @@ -383,14 +385,18 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear( ggml_tensor * qkv_mixed = qkvz.first; ggml_tensor * z = qkvz.second; - ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s); + ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s); + ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s); + + ggml_build_forward_expand(gf, beta); + ggml_build_forward_expand(gf, alpha); + beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs); cb(beta, "beta", il); beta = ggml_sigmoid(ctx0, beta); cb(beta, "beta_sigmoid", il); - ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s); alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs); cb(alpha, "alpha", il); From f169e0b7d2ddb48f7a433ef0deeb7d2de6b9471e Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:24:08 +0200 Subject: [PATCH 05/30] server: keep speculative checkpoints on device Use full MTP rollback checkpoints and retain speculative sequence state on the device to avoid unnecessary host transfers. Assisted-by: OpenCode --- common/common.h | 2 +- tools/server/server-context.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/common.h b/common/common.h index 0162a5f1e638..ebb60949b740 100644 --- a/common/common.h +++ b/common/common.h @@ -415,7 +415,7 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; }); return needs_rs_seq ? draft.n_max : 0u; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 90501bcb7f73..fce5f9b5f4b1 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3021,7 +3021,7 @@ struct server_context_impl { llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id)); if (use_ckpt_dft) { - slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } slot.spec_prompt = slot.prompt.tokens.get_text_tokens(); @@ -3060,7 +3060,7 @@ struct server_context_impl { if (ctx_dft) { if (use_ckpt_dft) { - ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } if (!llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, ckpt.pos_max + 1, -1)) { @@ -3079,7 +3079,7 @@ struct server_context_impl { if (use_ckpt_tgt) { //const int64_t t_start = ggml_time_us(); - ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); //const int64_t t_total = ggml_time_us() - t_start; //printf("checkpoint total: %f ms\n", t_total / 1000.0); @@ -3091,7 +3091,7 @@ struct server_context_impl { } if (use_ckpt_dft) { - ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } } }); @@ -3943,10 +3943,10 @@ struct server_context_impl { SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size()); - ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); if (slot.ctx_dft) { - ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1); From 1ac05934afa25096ece8c7cdd8f6a3210084eb4c Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:24:20 +0200 Subject: [PATCH 06/30] backend: allow padded concat allocations Permit backends to allocate quantized CONCAT outputs with row padding under the expanded allocation-size invariant. Assisted-by: OpenCode --- ggml/src/ggml-backend.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index fec7d7c92bf3..407dd5c5db98 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -2115,6 +2115,7 @@ bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op) { switch (op) { case GGML_OP_FLASH_ATTN_EXT: case GGML_OP_MUL_MAT_ID: + case GGML_OP_CONCAT: case GGML_OP_CUMSUM: case GGML_OP_ARGSORT: case GGML_OP_TOP_K: From c52f22bb6258b7a2d55b49da1329a2ff25fee341 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:24:32 +0200 Subject: [PATCH 07/30] tests: cover RDNA3.5 inference optimizations Add exact-batch, quantized matmul, routed expert, Flash Attention, concat, normalization, and Gated DeltaNet regression coverage for the optimized gfx1151 paths. Assisted-by: OpenCode --- tests/test-backend-ops.cpp | 339 ++++++++++++++++++++++++++++++++++++- 1 file changed, 330 insertions(+), 9 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 996b88db296e..7e6c8c383101 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3608,35 +3608,38 @@ struct test_rms_norm_back : public test_case { } }; -// GGML_OP_RMS_NORM + GGML_OP_MUL + GGML_OP_ADD +// GGML_OP_RMS_NORM + GGML_OP_MUL [+ GGML_OP_ADD] struct test_rms_norm_mul_add : public test_case { const ggml_type type; const std::array ne; const float eps; const bool broadcast; const bool multi_add; // test a sequence of adds feeding into rms_norm + const bool add; + const bool weights; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); - return "RMS_NORM_MUL_ADD"; + return add ? "RMS_NORM_MUL_ADD" : "RMS_NORM_MUL"; } bool run_whole_graph() override { return true; } std::string vars() override { - return VARS_TO_STR5(type, ne, eps, broadcast, multi_add); + return VARS_TO_STR7(type, ne, eps, broadcast, multi_add, add, weights); } test_rms_norm_mul_add(ggml_type type = GGML_TYPE_F32, std::array ne = {64, 5, 4, 3}, - float eps = 1e-6f, bool broadcast = false, bool multi_add = false) - : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add) {} + float eps = 1e-6f, bool broadcast = false, bool multi_add = false, bool add = true, bool weights = false) + : type(type), ne(ne), eps(eps), broadcast(broadcast), multi_add(multi_add), add(add), weights(weights) {} ggml_tensor * build_graph(ggml_context * ctx) override { std::array broadcast_dims = {ne[0]*2, ne[1]*3, ne[2]*3, ne[3]*4}; + std::array weight_dims = {ne[0], 1, 1, 1}; ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * b = ggml_new_tensor(ctx, type, 4, weights ? weight_dims.data() : ne.data()); ggml_tensor * c = ggml_new_tensor(ctx, type, 4, ne.data()); ggml_set_param(a); @@ -3651,7 +3654,10 @@ struct test_rms_norm_mul_add : public test_case { if (multi_add) { a = ggml_add(ctx, ggml_add(ctx, a, b), c); } - ggml_tensor * out = ggml_add(ctx, ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b), c); + ggml_tensor * out = ggml_mul(ctx, ggml_rms_norm(ctx, a, eps), b); + if (add) { + out = ggml_add(ctx, out, c); + } ggml_set_name(out, "out"); return out; @@ -3765,6 +3771,72 @@ struct test_relu_sqr : public test_case { } }; +struct test_weighted_expert_sum : public test_case { + const int64_t n_embd; + const int64_t n_expert_used; + const int64_t n_tokens; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "WEIGHTED_EXPERT_SUM"; + } + + bool run_whole_graph() override { return true; } + + std::string vars() override { + return VARS_TO_STR3(n_embd, n_expert_used, n_tokens); + } + + test_weighted_expert_sum(int64_t n_embd, int64_t n_expert_used, int64_t n_tokens) + : n_embd(n_embd), n_expert_used(n_expert_used), n_tokens(n_tokens) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * experts = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, n_expert_used, n_tokens); + ggml_tensor * weights = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, n_expert_used, n_tokens); + ggml_tensor * weighted = ggml_mul(ctx, experts, weights); + ggml_build_forward_expand(gf, weighted); + + std::vector views; + views.reserve(n_expert_used); + for (int64_t i = 0; i < n_expert_used; ++i) { + ggml_tensor * view = ggml_view_2d(ctx, weighted, n_embd, n_tokens, weighted->nb[2], i * weighted->nb[1]); + ggml_build_forward_expand(gf, view); + views.push_back(view); + } + + ggml_tensor * out = views[0]; + for (int64_t i = 1; i < n_expert_used; ++i) { + out = ggml_add(ctx, out, views[i]); + ggml_build_forward_expand(gf, out); + } + return out; + } +}; + +struct test_shared_mul_add : public test_case { + const int64_t n_embd; + const int64_t n_tokens; + + test_shared_mul_add(int64_t n_embd, int64_t n_tokens) : n_embd(n_embd), n_tokens(n_tokens) {} + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SHARED_MUL_ADD"; + } + + std::string vars() override { return VARS_TO_STR2(n_embd, n_tokens); } + bool run_whole_graph() override { return true; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * src = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_tensor * gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, n_tokens); + ggml_tensor * other = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_tensor * residual = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_tensor * mul = ggml_mul(ctx, src, gate); + return ggml_add(ctx, ggml_add(ctx, other, mul), residual); + } +}; + // GGML_OP_UNARY(SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation). // `layout` and `tail` are used for fallback cases where fusion must be skipped struct test_unary_mul : public test_case { @@ -4660,6 +4732,72 @@ struct test_mul_mat : public test_case { } }; +struct test_mul_mat_exact_batch : public test_case { + const ggml_type type_a; + const int64_t m; + const int64_t n; + const int64_t k; + + test_mul_mat_exact_batch(ggml_type type_a, int64_t m, int64_t n, int64_t k) : type_a(type_a), m(m), n(n), k(k) {} + + std::string vars() override { + return VARS_TO_STR4(type_a, m, n, k); + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_2d(ctx, type_a, k, m); + ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); + ggml_tensor * batch = ggml_mul_mat(ctx, a, b); + ggml_mul_mat_set_hint(batch, GGML_HINT_EXACT_BATCH); + + ggml_tensor * diff = nullptr; + for (int64_t col = 0; col < n; ++col) { + ggml_tensor * b_col = ggml_view_2d(ctx, b, k, 1, b->nb[1], col*b->nb[1]); + ggml_tensor * single = ggml_mul_mat(ctx, a, b_col); + ggml_tensor * batch_col = ggml_view_2d(ctx, batch, m, 1, batch->nb[1], col*batch->nb[1]); + ggml_tensor * col_diff = ggml_sub(ctx, batch_col, single); + diff = diff == nullptr ? col_diff : ggml_concat(ctx, diff, col_diff, 1); + } + return diff; + } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return ggml_op_name(GGML_OP_MUL_MAT); + } + + double err(const float * a, const float * b, size_t n) override { + GGML_UNUSED(b); + double max_abs = 0.0; + for (size_t i = 0; i < n; ++i) { + max_abs = std::max(max_abs, std::abs((double) a[i])); + } + return max_abs; + } + + double max_err() override { return 0.0; } + + bool run_whole_graph() override { return true; } +}; + +struct test_mul_mat_pair : public test_case { + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MUL_MAT_PAIR"; + } + + std::string vars() override { return {}; } + bool run_whole_graph() override { return true; } + double max_nmse_err() override { return 5e-4; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a0 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, 256, 32); + ggml_tensor * a1 = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, 256, 32); + ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 256, 129); + return ggml_add(ctx, ggml_mul_mat(ctx, a0, b), ggml_mul_mat(ctx, a1, b)); + } +}; + // GGML_HINT_SRC0_IS_HADAMARD struct test_mul_mat_hadamard : public test_mul_mat { test_mul_mat_hadamard(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, @@ -4800,6 +4938,51 @@ struct test_mul_mat_id : public test_case { } }; +struct test_swiglu_q8_mmq : public test_case { + const bool use_id; + const int n_mats; + const int n_used; + const int64_t m; + const int64_t n; + const int64_t k; + + test_swiglu_q8_mmq(bool use_id, int n_mats, int n_used, int64_t m, int64_t n, int64_t k) + : use_id(use_id), n_mats(n_mats), n_used(n_used), m(m), n(n), k(k) {} + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SWIGLU_Q8_MMQ"; + } + + std::string vars() override { + return VARS_TO_STR6(use_id, n_mats, n_used, m, n, k); + } + + bool run_whole_graph() override { return true; } + double max_nmse_err() override { return 5e-4; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + if (!use_id) { + ggml_tensor * gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); + ggml_tensor * up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); + ggml_tensor * down = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m); + return ggml_mul_mat(ctx, down, ggml_swiglu_split(ctx, gate, up)); + } + + ggml_tensor * gate = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n); + ggml_tensor * up = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n); + ggml_tensor * down = ggml_new_tensor_3d(ctx, GGML_TYPE_Q8_0, k, m, n_mats); + ggml_tensor * ids_all = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); + ggml_set_name(ids_all, "ids"); + ggml_tensor * ids = ggml_view_2d(ctx, ids_all, n_used, n, ids_all->nb[1], 0); + return ggml_mul_mat_id(ctx, down, ggml_swiglu_split(ctx, gate, up), ids); + } + + void initialize_tensors(ggml_context * ctx) override { + init_mul_mat_id_tensors(ctx, n_mats); + } +}; + // GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL struct test_mul_mat_id_fusion : public test_case { const ggml_type type_a; @@ -6051,7 +6234,7 @@ struct test_concat : public test_case { const std::array ne_a; const int64_t ne_b_d; const int dim; - const int v; // view (1 << 0: non-cont a (first 3 dim), 1 << 1: non-cont b (first 3 dim), 1 << 2: non-cont a (last 2 dim), 1 << 3: non-cont b (last 2 dim)) + const int v; // view (bits 0-3: non-contiguous views, bit 4: transposed b) std::string vars() override { return VARS_TO_STR5(type, ne_a, ne_b_d, dim, v); @@ -6086,7 +6269,15 @@ struct test_concat : public test_case { ggml_set_name(a, "a"); } ggml_tensor * b; - if (v & 2) { + if (v & 16) { + auto ne = ne_b; + std::swap(ne[0], ne[1]); + b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_set_name(b, "b"); + + b = ggml_transpose(ctx, b); + ggml_set_name(b, "transposed_b"); + } else if (v & 2) { auto ne = ne_b; ne[0] *= 3; ne[1] *= 2; ne[2] *= 4; b = ggml_new_tensor(ctx, type, 4, ne.data()); ggml_set_name(b, "b"); @@ -6867,6 +7058,53 @@ struct test_l2_norm : public test_case { } }; +struct test_l2_norm_dual : public test_case { + const int64_t n_heads; + const int64_t n_tokens; + const int64_t n_seqs; + const float eps_q; + const float eps_k; + + ggml_tensor * q_norm = nullptr; + ggml_tensor * k_norm = nullptr; + + test_l2_norm_dual(int64_t n_heads, int64_t n_tokens, int64_t n_seqs, float eps_q, float eps_k) + : n_heads(n_heads), n_tokens(n_tokens), n_seqs(n_seqs), eps_q(eps_q), eps_k(eps_k) {} + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "L2_NORM_DUAL_S128"; + } + + std::string vars() override { + return VARS_TO_STR5(n_heads, n_tokens, n_seqs, eps_q, eps_k); + } + + bool run_whole_graph() override { return true; } + + std::vector fusion_test_nodes() override { return { q_norm, k_norm }; } + + ggml_tensor * build_graph(ggml_context * ctx) override { + constexpr int64_t head_size = 128; + ggml_tensor * qkv = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 3*head_size*n_heads, n_tokens, n_seqs); + ggml_set_name(qkv, "qkv"); + + ggml_tensor * q = ggml_view_4d(ctx, qkv, head_size, n_heads, n_tokens, n_seqs, + head_size*sizeof(float), qkv->nb[1], qkv->nb[2], 0); + ggml_tensor * k = ggml_view_4d(ctx, qkv, head_size, n_heads, n_tokens, n_seqs, + head_size*sizeof(float), qkv->nb[1], qkv->nb[2], head_size*n_heads*sizeof(float)); + ggml_set_name(q, "q"); + ggml_set_name(k, "k"); + + q_norm = ggml_l2_norm(ctx, q, eps_q); + k_norm = ggml_l2_norm(ctx, k, eps_k); + ggml_set_name(q_norm, "q_norm"); + ggml_set_name(k_norm, "k_norm"); + + return ggml_add(ctx, q_norm, k_norm); + } +}; + // GGML_OP_ACC struct test_acc : public test_case { const ggml_type type; @@ -9107,6 +9345,9 @@ static std::vector> make_test_cases_eval() { } } + test_cases.emplace_back(new test_l2_norm_dual(16, 1, 1, 1e-6f, 1e-6f)); + test_cases.emplace_back(new test_l2_norm_dual(4, 3, 2, 1e-6f, 8.0f)); + // in-place tests test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, {64, 5, 4, 3}, false, 1e-6f, true)); @@ -9127,6 +9368,18 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false)); } + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {127, 48, 1, 1}, 1e-6f, false, false, false, true)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {128, 48, 1024, 1}, 1e-6f, false, false, false, true)); + test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, {129, 48, 1, 1}, 1e-6f, false, false, false, true)); + + test_cases.emplace_back(new test_weighted_expert_sum(127, 2, 3)); + test_cases.emplace_back(new test_weighted_expert_sum(128, 4, 5)); + test_cases.emplace_back(new test_weighted_expert_sum(2048, 8, 32)); + test_cases.emplace_back(new test_shared_mul_add(127, 3)); + test_cases.emplace_back(new test_shared_mul_add(2048, 32)); + test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2048, 32, 512)); + test_cases.emplace_back(new test_swiglu_q8_mmq(true, 16, 8, 2048, 32, 512)); + for (auto multi_add : {false, true}) { for (auto set_rows : {false, true}) { for (auto broadcast : {false, true}) { @@ -9231,6 +9484,32 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); #endif + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 1024, 2048, 3072)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 3072, 2048, 1024)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 1984, 2560)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 2048, 2560)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 2112, 2560)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 2560, 2048, 768)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 512, 8, false, 2560, 1984, 768)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 512, 8, false, 2560, 2048, 768)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 512, 8, false, 2560, 2112, 768)); + for (ggml_type type_a : all_types) { for (int i = 1; i < 10; ++i) { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, i, 1*256, { 1, 1}, {1, 1})); @@ -9247,6 +9526,17 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); + for (int64_t n : {127, 128, 129}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 128, n, 5120, {1, 1}, {1, 1})); + } + for (int64_t n : {2, 3, 4, 5, 8, 9, 16}) { + test_cases.emplace_back(new test_mul_mat_exact_batch(GGML_TYPE_BF16, 128, n, 5120)); + } + for (ggml_type type_a : {GGML_TYPE_Q1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, + GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ3_S}) { + test_cases.emplace_back(new test_mul_mat_exact_batch(type_a, 128, 4, 5120)); + } + // m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1})); @@ -9377,6 +9667,10 @@ static std::vector> make_test_cases_eval() { } test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 1024, 32, 1056, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1024, 32, 1056, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q2_K, GGML_TYPE_F32, 1024, 32, 1280, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8, 2, true, 128, 32, 1056)); // K not a multiple of 32 test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 64, 32, 65, {1, 1}, {1, 1})); @@ -9441,6 +9735,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, k)); } test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3)); + test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8, 4, false, 512, 129, 256, 2)); + test_cases.emplace_back(new test_mul_mat_pair()); // gpt-oss issue with Vulkan mmq_id test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_MXFP4, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880)); @@ -9758,6 +10054,11 @@ static std::vector> make_test_cases_eval() { } } + // Transposed src1 path: guard boundary, Qwen validation prompt, and prefill tile edges. + for (int64_t n_tokens : { 31, 32, 115, 512 }) { + test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {64, 127, 2, 1}, n_tokens, 0, 16)); + } + for (ggml_sort_order order : {GGML_SORT_ORDER_ASC, GGML_SORT_ORDER_DESC}) { for (uint32_t i = 4; i <= 1024*1024; i *= 2) { test_cases.emplace_back(new test_argsort(GGML_TYPE_F32, {i-1, 1, 1, 1})); @@ -9970,6 +10271,9 @@ static std::vector> make_test_cases_eval() { } } } + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {4, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 8, {4, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 8, {6, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0)); for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 320, 512, 576 }) { for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) { @@ -10177,6 +10481,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 64, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 15, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 16, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 17, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 16, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 17, 1)); // K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first // (slot 0 = final state, slot s = state s tokens back). @@ -10470,6 +10780,11 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 10000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 20000, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int64_t kv : {4096, 8192, 12000, 16000, 24000, 32000, 64000}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {8, 1}, kv, 2048, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {8, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + } + for (int kv : { 4096, 8192, 16384, }) { for (int hs : { 64, 128, }) { for (int nr : { 1, 4, }) { @@ -10592,12 +10907,18 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 1024, 1)); // H64/S128 PP-1024 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 2048, 1)); // H64/S128 PP-2048 // Small model configs (fewer heads = less GPU occupancy for autoregressive) test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1)); // 4h PP-64 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1)); // 4h PP-256 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 15, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 16, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 17, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1, 1, false, true)); // lightning_indexer for (int kv : { 256, 4096, 65536 }) { From 551ce30fe5c371258333d5681be76b8f3f440d65 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Wed, 2 Sep 2026 00:20:42 +0200 Subject: [PATCH 08/30] hip: preserve exact RDNA3.5 MMVQ reductions Keep batch-one MMVQ on one wave for all quant types. Wider wave counts changed FP32 reduction order, causing non-identical logits for Q2-Q5 and deterministic decode divergence in the four-corpus correctness matrix. Assisted-by: OpenCode --- ggml/src/ggml-cuda/mmvq.cu | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index b93b9b6c0520..ec1855f22526 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -402,20 +402,6 @@ static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() { static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id, bool small_k = false, bool halve_iters = false) { if (table_id == MMVQ_PARAMETERS_RDNA3_5) { - // Measured on gfx1151 with 4096x14336 MMVQ on 2026-09-01. Wider blocks regress the other types. - if (ncols_dst == 1) { - switch (type) { - case GGML_TYPE_Q5_K: - case GGML_TYPE_Q6_K: - return 4; - case GGML_TYPE_Q5_1: - case GGML_TYPE_MXFP4: - case GGML_TYPE_Q4_K: - return 2; - default: - break; - } - } return 1; } if (table_id == MMVQ_PARAMETERS_GENERIC) { @@ -544,12 +530,12 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d return 1; } -static_assert(calc_nwarps(GGML_TYPE_Q5_1, 1, MMVQ_PARAMETERS_RDNA3_5) == 2); +static_assert(calc_nwarps(GGML_TYPE_Q5_1, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); static_assert(calc_nwarps(GGML_TYPE_Q8_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); -static_assert(calc_nwarps(GGML_TYPE_MXFP4, 1, MMVQ_PARAMETERS_RDNA3_5) == 2); -static_assert(calc_nwarps(GGML_TYPE_Q4_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 2); -static_assert(calc_nwarps(GGML_TYPE_Q5_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 4); -static_assert(calc_nwarps(GGML_TYPE_Q6_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 4); +static_assert(calc_nwarps(GGML_TYPE_MXFP4, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); +static_assert(calc_nwarps(GGML_TYPE_Q4_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); +static_assert(calc_nwarps(GGML_TYPE_Q5_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); +static_assert(calc_nwarps(GGML_TYPE_Q6_K, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); static_assert(calc_nwarps(GGML_TYPE_Q1_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); static_assert(calc_nwarps(GGML_TYPE_Q2_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); static_assert(calc_nwarps(GGML_TYPE_Q4_0, 1, MMVQ_PARAMETERS_RDNA3_5) == 1); From fcae791cc6e48f90fc2ae3025ce512d10d9a33da Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:18:47 +0200 Subject: [PATCH 09/30] hip: widen Qwen3.6 GDN blocks on RDNA3.5 Use 32 waves for the production non-KDA H32/S128 Gated DeltaNet shape on gfx1151. This reduces kernel latency by 1.34% and improves Qwen3.6 IQ3_XXS PP2048 by up to 0.97% while preserving exact four-corpus logits and decode output. Assisted-by: OpenCode --- ggml/src/ggml-cuda/gated_delta_net.cu | 3 ++- tests/test-backend-ops.cpp | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index 8fab7a3e5dfb..bc0593f847ae 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -307,7 +307,8 @@ static void launch_gated_delta_net( const int id = ggml_cuda_get_device(); const int cc = ggml_cuda_info().devices[id].cc; const int warp_size = ggml_cuda_info().devices[id].warp_size; - const int num_warps = GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && H == 64 && !KDA ? 32 : + // Qwen3.6 PP2048 measured faster on gfx1151 on 2026-09-02. Retest if occupancy changes. + const int num_warps = GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && (H == 32 || H == 64) && !KDA ? 32 : (GGML_CUDA_CC_IS_RDNA3_5(cc) ? 16 : gated_delta_net_num_warps(cc)); dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps); dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 7e6c8c383101..5c1f0b7f7673 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10485,6 +10485,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 16, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 17, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 16, 1)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 17, 1)); @@ -10907,6 +10908,7 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1)); // PP-2048 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 1024, 1)); // H64/S128 PP-1024 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 2048, 1)); // H64/S128 PP-2048 // Small model configs (fewer heads = less GPU occupancy for autoregressive) From 90ad6cd26753b1eab62ff3ee39e17bfdcca3b6b5 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:20:43 +0200 Subject: [PATCH 10/30] hip: prefetch MMQ tiles into registers on RDNA3.5 With 64 KiB of LDS per CU only 1-2 MMQ blocks are resident on gfx1151, so the global-load latency of every 256-wide K iteration is exposed (PMC: VALU busy 6%, 53% of wave-cycles waiting on data). Stage the next iteration's activation tile (and, for Q8_0, the weight tile) in registers while the current one is consumed. Same loads, same LDS layout, same arithmetic: outputs are bit-identical to the previous kernels. Whitelisted to the specializations where the extra registers do not spill: Q8_0 J48/J128, Q6_K J32, Q5_K J32, Q4_K J48, IQ2_S/IQ3_XXS J128 (all others keep the original loop; no spill regression in any kernel). test-backend-ops perf, MUL_MAT_ID 256 experts top-8, 2048 tokens (includes the quantize/ids overhead): Q8_0 gate/up 2717 -> 2551 us, Q6_K 3600 -> 3481, Q5_K 2571 -> 2510, Q4_K 2776 -> 2701, IQ2_S 3830 -> 3743, IQ3_XXS 3544 -> 3457. Also adds these production shapes to the perf test list. --- ggml/src/ggml-cuda/mmq-load-tiles.cuh | 96 +++++++++++++++++++++++++++ ggml/src/ggml-cuda/mmq.cuh | 96 +++++++++++++++++++++++++++ tests/test-backend-ops.cpp | 7 ++ 3 files changed, 199 insertions(+) diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index 7f00bad943e2..40c825405e26 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -532,6 +532,102 @@ template static __device__ __forceinline_ } } +// Register-staged variant of the Q8_0 tile loader used for software prefetching on RDNA3.5: +// the global loads for the next K iteration are issued into registers while the current tile is consumed, +// then written to shared memory with exactly the same layout as ggml_cuda_mmq_load_tiles_q8_0. +template struct ggml_cuda_mmq_x_regs_q8_0 { + // Only instantiated for RDNA3.5 (wave32) device code; the host pass needs constant sizes. + static constexpr int warp_size = WARP_SIZE; + static constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + static constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + static constexpr int threads_per_row = 32; + static constexpr int nrows = warp_size / threads_per_row; + static constexpr int rows_per_step = nrows*nwarps; + static constexpr int n_qs = (I + rows_per_step - 1) / rows_per_step; + static constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0; + static constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + static constexpr int d_rows_per_step = nwarps*rows_per_warp; + static constexpr int n_d = (I + d_rows_per_step - 1) / d_rows_per_step; + + int qs[2*n_qs]; + half d[n_d]; +}; + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0_regs( + const char * __restrict__ x, ggml_cuda_mmq_x_regs_q8_0 & r, const int kbx0, const int i_max, const int stride) { + using R = ggml_cuda_mmq_x_regs_q8_0; + static_assert(R::warp_size == ggml_cuda_get_physical_warp_size(), "register-staged Q8_0 loader assumes wave32"); + + const int txi = R::warp_size > R::threads_per_row ? threadIdx.x % R::threads_per_row : threadIdx.x; + const int kbx = txi / QI8_0; + const int kqsx = txi % QI8_0; + +#pragma unroll + for (int i0 = 0; i0 < R::I; i0 += R::rows_per_step) { + int i = i0 + (R::nrows == 1 ? threadIdx.y : threadIdx.y*R::nrows + threadIdx.x/R::threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx; + + r.qs[2*(i0/R::rows_per_step) + 0] = get_int_b2(bxi[0].qs, kqsx); + r.qs[2*(i0/R::rows_per_step) + 1] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx); + } + + const int kbxd = threadIdx.x % R::blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < R::I; i0 += R::d_rows_per_step) { + int i = i0 + threadIdx.y*R::rows_per_warp + threadIdx.x/R::blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd; + + r.d[i0/R::d_rows_per_step] = bxi->d; + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_store_tiles_q8_0_regs( + const ggml_cuda_mmq_x_regs_q8_0 & r, int * __restrict__ x_tile, const int i_max) { + using R = ggml_cuda_mmq_x_regs_q8_0; + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K); + + const int txi = R::warp_size > R::threads_per_row ? threadIdx.x % R::threads_per_row : threadIdx.x; + +#pragma unroll + for (int i0 = 0; i0 < R::I; i0 += R::rows_per_step) { + int i = i0 + (R::nrows == 1 ? threadIdx.y : threadIdx.y*R::nrows + threadIdx.x/R::threads_per_row); + + if (fallback) { + i = min(i, i_max); + } + + x_qs[i*sram_stride + 0 + txi] = r.qs[2*(i0/R::rows_per_step) + 0]; + x_qs[i*sram_stride + MMQ_TILE_NE_K + txi] = r.qs[2*(i0/R::rows_per_step) + 1]; + } + + const int kbxd = threadIdx.x % R::blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < R::I; i0 += R::d_rows_per_step) { + int i = i0 + threadIdx.y*R::rows_per_warp + threadIdx.x/R::blocks_per_tile_x_row; + + if (fallback) { + i = min(i, i_max); + } + + x_df[i*sram_stride + kbxd] = r.d[i0/R::d_rows_per_step]; + } +} + // --------------------------------------------------------------------------------------------- template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K( diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 741c346f0666..a9fbe95e28bc 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -874,6 +874,22 @@ static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_b // --------------------------------------------------------------------------------------------- +// Software prefetch of the next K iteration into registers, see mul_mat_q_process_tile. +template +static constexpr __host__ __device__ bool ggml_cuda_mmq_use_prefetch() { +#if defined(RDNA3_5) && defined(AMD_WMMA_AVAILABLE) + // Whitelist: the extra registers cause spills in several other specializations. + return (type == GGML_TYPE_Q8_0 && (J == 48 || J == 128) && !fallback) || + (type == GGML_TYPE_Q6_K && J == 32) || + (type == GGML_TYPE_Q5_K && J == 32) || + (type == GGML_TYPE_Q4_K && J == 48) || + (type == GGML_TYPE_IQ2_S && J == 128) || + (type == GGML_TYPE_IQ3_XXS && J == 128); +#else + return false; +#endif // defined(RDNA3_5) && defined(AMD_WMMA_AVAILABLE) +} + template static __device__ __forceinline__ void mul_mat_q_process_tile( const char * __restrict__ x, const int offset_x, const int * __restrict__ y, @@ -908,6 +924,86 @@ static __device__ __forceinline__ void mul_mat_q_process_tile( constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int); + if constexpr (ggml_cuda_mmq_use_prefetch()) { + // RDNA3.5 software prefetch: with 64 KiB of LDS per CU only 1-2 blocks are resident, so the + // global load latency of every K iteration is exposed. Stage the next iteration's tiles in + // registers while the current one is consumed. The arithmetic is unchanged. + constexpr int nthreads = nwarps*warp_size; + constexpr int y_regs_n = (J*MMQ_TILE_Y_K + nthreads - 1) / nthreads; + constexpr bool x_regs_ok = type == GGML_TYPE_Q8_0; + const int tid = threadIdx.y*warp_size + threadIdx.x; + + int yr0[y_regs_n]; + int yr1[y_regs_n]; + ggml_cuda_mmq_x_regs_q8_0 xr; + GGML_UNUSED(xr); + + auto load_y_regs = [&](int (&yr)[y_regs_n], const int kb0, const int half) { + const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + half*sz); +#pragma unroll + for (int r = 0; r < y_regs_n; ++r) { + yr[r] = by0[r*nthreads + tid]; + } + }; + auto store_y_regs = [&](const int (&yr)[y_regs_n]) { +#pragma unroll + for (int r = 0; r < y_regs_n; ++r) { + tile_y[r*nthreads + tid] = yr[r]; + } + }; + + if (kb0_start < kb0_stop) { + if constexpr (x_regs_ok) { + ggml_cuda_mmq_load_tiles_q8_0_regs(x, xr, offset_x + kb0_start, tile_x_max_i, stride_row_x); + } + load_y_regs(yr0, kb0_start, 0); + load_y_regs(yr1, kb0_start, 1); + } + + for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) { + const bool has_next = kb0 + blocks_per_iter < kb0_stop; + + if constexpr (x_regs_ok) { + ggml_cuda_mmq_store_tiles_q8_0_regs(xr, tile_x, tile_x_max_i); + } else { + load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x); + } + store_y_regs(yr0); + + __syncthreads(); + + if (has_next) { + if constexpr (x_regs_ok) { + ggml_cuda_mmq_load_tiles_q8_0_regs(x, xr, offset_x + kb0 + blocks_per_iter, tile_x_max_i, stride_row_x); + } + load_y_regs(yr0, kb0 + blocks_per_iter, 0); + } + + vec_dot(tile_x, tile_y, sum, 0); + + __syncthreads(); + + store_y_regs(yr1); + + __syncthreads(); + + if (has_next) { + load_y_regs(yr1, kb0 + blocks_per_iter, 1); + } + + vec_dot(tile_x, tile_y, sum, MMQ_TILE_NE_K); + + __syncthreads(); + } + + if (fixup) { + write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J); + } else { + write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j); + } + return; + } + for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) { load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x); { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 5c1f0b7f7673..9478c2056b44 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10744,6 +10744,13 @@ static std::vector> make_test_cases_perf() { } } + // Qwen3.6-35B-A3B prompt processing (256 experts, top-8, expert width 512, ubatch 2048): + // gate/up: m=512, k=2048; down: m=2048, k=512 + for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_Q5_K, GGML_TYPE_Q4_K, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_MXFP4}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512)); + } + for (int K : {3, 5}) { for (int IC : {256, 2560}) { for (int IW_IH : {32, 64, 256}) { From 8ab5a837394fa51c55dfb217afe5551840f1242e Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:20:43 +0200 Subject: [PATCH 11/30] hip: DPP reductions and multi-column tiled GDN kernel on RDNA3.5 Two exact-preserving changes to the gated delta net recurrence: * warp_reduce_sum compiled to 5 dependent ds_bpermute round trips per reduction (10 per token); on RDNA3/4 use v_permlanex16 + DPP row_xmask with the same xor-16/8/4/2/1 pairing, so results are bit-identical. * For the non-KDA S=128 prompt-processing path add a tiled kernel: q/k/g/ beta/v of a 16-token tile are loaded cooperatively into LDS and each warp owns 4 state columns, so k/q are read once per token for 4 columns and 4x fewer waves compete for issue slots (the kernel was issue-bound with ~50 VALU per token per wave for 4 state elements). FMA contractions are spelled out to match the original kernel's code generation. Bit-identical to the previous kernels on H=32/64, T=1/17/64/2048 (op-level dump comparison); test-backend-ops GATED_DELTA_NET 43/43. gfx1151, test-backend-ops perf: H32 T2048 5029 -> 2350 us, H64 T2048 9289 -> 4533 us, H32 T64 158 -> 93 us, T=1 13.3 -> 12.6 us. Together with the MMQ prefetch, Qwen3.6-35B-A3B PP2048 (llama-bench -b 2048 -ub 2048): Q8_0 +10.9%, UD-Q6_K_XL +9.4%, UD-Q4_K_XL +9.2%, UD-IQ3_XXS +7.6%. --- ggml/src/ggml-cuda/gated_delta_net.cu | 236 +++++++++++++++++++++++++- 1 file changed, 230 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index bc0593f847ae..ce9a5c5e8ad7 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -1,6 +1,39 @@ #include "gated_delta_net.cuh" #include "ggml-cuda/common.cuh" + +// Warp reduction without the LDS crossbar: on RDNA the generic warp_reduce_sum compiles to 5 dependent +// ds_bpermute round trips (~100+ cycles each), which dominates the per-token latency of the recurrence. +// Same pairing/order of additions as warp_reduce_sum<32> (xor 16, 8, 4, 2, 1), so the result is bit-identical. +#if defined(GGML_USE_HIP) && (defined(RDNA3) || defined(RDNA4)) +template +static __device__ __forceinline__ float gdn_dpp_row_xmask(const float x) { + return __int_as_float(__builtin_amdgcn_update_dpp(0, __float_as_int(x), 0x160 | mask, 0xf, 0xf, true)); +} +static __device__ __forceinline__ float gdn_permlanex16_swap(const float x) { + return __int_as_float(__builtin_amdgcn_permlanex16(__float_as_int(x), __float_as_int(x), 0x76543210, 0xFEDCBA98, true, false)); +} +static __device__ __forceinline__ float gdn_warp_reduce_sum32(float x) { + x += gdn_permlanex16_swap(x); + x += gdn_dpp_row_xmask<8>(x); + x += gdn_dpp_row_xmask<4>(x); + x += gdn_dpp_row_xmask<2>(x); + x += gdn_dpp_row_xmask<1>(x); + return x; +} +#define GDN_DPP_REDUCE 1 +#endif // defined(GGML_USE_HIP) && (defined(RDNA3) || defined(RDNA4)) + +template +static __device__ __forceinline__ float gdn_warp_reduce_sum(const float x) { +#if defined(GDN_DPP_REDUCE) + if constexpr (width == 32) { + return gdn_warp_reduce_sum32(x); + } +#endif // defined(GDN_DPP_REDUCE) + return warp_reduce_sum(x); +} + static constexpr int gated_delta_net_num_warps(int cc) { return GGML_CUDA_CC_IS_RDNA3_5(cc) ? 32 : 4; } @@ -103,7 +136,7 @@ gated_delta_net_cuda(const float * q, for (int r = 0; r < rows_per_lane; r++) { kv_shard += s_shard[r] * k_reg[r]; } - float kv_col = warp_reduce_sum(kv_shard); + float kv_col = gdn_warp_reduce_sum(kv_shard); // delta[col] = (v[col] - g * kv[col]) * beta float delta_col = (v_t[col] - g_val * kv_col) * beta_val; @@ -117,7 +150,7 @@ gated_delta_net_cuda(const float * q, attn_partial += s_shard[r] * q_reg[r]; } - float attn_col = warp_reduce_sum(attn_partial); + float attn_col = gdn_warp_reduce_sum(attn_partial); if (lane == 0) { attn_data[col] = attn_col * scale; @@ -131,7 +164,7 @@ gated_delta_net_cuda(const float * q, kv_shard += expf(g_t[i]) * s_shard[r] * k_reg[r]; } - float kv_col = warp_reduce_sum(kv_shard); + float kv_col = gdn_warp_reduce_sum(kv_shard); // delta[col] = (v[col] - kv[col]) * beta float delta_col = (v_t[col] - kv_col) * beta_val; @@ -146,7 +179,7 @@ gated_delta_net_cuda(const float * q, attn_partial += s_shard[r] * q_reg[r]; } - float attn_col = warp_reduce_sum(attn_partial); + float attn_col = gdn_warp_reduce_sum(attn_partial); if (lane == 0) { attn_data[col] = attn_col * scale; @@ -268,7 +301,7 @@ gated_delta_net_kda_tiled_128_cuda(const float * q, const int i = r * warp_size + lane; kv_shard += expf(g_shared[tt][i]) * s_shard[r] * k_reg[r]; } - const float kv_col = warp_reduce_sum(kv_shard); + const float kv_col = gdn_warp_reduce_sum(kv_shard); const float delta_col = (v_t[col] - kv_col) * beta_shared[tt]; float attn_partial = 0.0f; @@ -278,7 +311,7 @@ gated_delta_net_kda_tiled_128_cuda(const float * q, s_shard[r] = expf(g_shared[tt][i]) * s_shard[r] + k_reg[r] * delta_col; attn_partial += s_shard[r] * q_reg[r]; } - const float attn_col = warp_reduce_sum(attn_partial); + const float attn_col = gdn_warp_reduce_sum(attn_partial); if (lane == 0) { dst[t * S_v * H + col] = attn_col * scale; } @@ -292,6 +325,179 @@ gated_delta_net_kda_tiled_128_cuda(const float * q, } } +// Non-KDA variant with the per-token inputs (q, k, g, beta, v) staged in shared memory for a tile of tokens, +// and COLS state columns per warp (k/q are loaded once per token for all columns, and the number of waves +// competing for issue slots shrinks). The recurrence itself is unchanged: for every column the same +// operations are executed in the same order as in gated_delta_net_cuda, so results are bit-identical. +template +__global__ void __launch_bounds__(32 * NUM_WARPS, 1) +gated_delta_net_tiled_cuda(const float * q, + const float * k, + const float * v, + const float * g, + const float * beta, + const float * curr_state, + float * dst, + float * state, + int64_t H, + int64_t n_tokens, + int64_t sq1, + int64_t sq2, + int64_t sq3, + int64_t sv1, + int64_t sv2, + int64_t sv3, + int64_t sb1, + int64_t sb2, + int64_t sb3, + const uint3 neqk1_magic, + const uint3 rq3_magic, + float scale, + int64_t state_slot_stride, + int K) { + constexpr int warp_size = 32; + constexpr int rows_per_lane = S_v / warp_size; + constexpr int block_cols = NUM_WARPS * COLS; + static_assert(S_v % warp_size == 0, "S_v must be a multiple of the warp size"); + static_assert(S_v % block_cols == 0, "block columns must divide S_v"); + + __shared__ float q_shared[TOKEN_TILE][S_v]; + __shared__ float k_shared[TOKEN_TILE][S_v]; + __shared__ float v_shared[TOKEN_TILE][block_cols]; + __shared__ float g_shared[TOKEN_TILE]; + __shared__ float beta_shared[TOKEN_TILE]; + + const uint32_t h_idx = blockIdx.x; + const uint32_t sequence = blockIdx.y; + const int lane = threadIdx.x; + const int col0 = blockIdx.z * block_cols; // first column of the block + const int colw = threadIdx.y * COLS; // first column of this warp inside the block + const int thread = threadIdx.y * warp_size + lane; + constexpr int nthreads = NUM_WARPS * warp_size; + + const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); + const uint32_t iq3 = fastdiv(sequence, rq3_magic); + + const int64_t state_in_offset = sequence * H * S_v * S_v + h_idx * S_v * S_v; + const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v; + state += state_out_offset; + curr_state += state_in_offset + (col0 + colw) * S_v; + float * attn_data = dst + (sequence * n_tokens * H + h_idx) * S_v + col0 + colw; + + float s_shard[COLS][rows_per_lane]; + + ggml_cuda_pdl_sync(); +#pragma unroll + for (int c = 0; c < COLS; c++) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + s_shard[c][r] = curr_state[c * S_v + r * warp_size + lane]; + } + } + + for (int t0 = 0; t0 < n_tokens; t0 += TOKEN_TILE) { + const int tile_size = min((int64_t) TOKEN_TILE, n_tokens - t0); + + for (int idx = thread; idx < tile_size * S_v; idx += nthreads) { + const int tt = idx / S_v; + const int i = idx % S_v; + const int t = t0 + tt; + q_shared[tt][i] = q[iq3 * sq3 + t * sq2 + iq1 * sq1 + i]; + k_shared[tt][i] = k[iq3 * sq3 + t * sq2 + iq1 * sq1 + i]; + } + for (int idx = thread; idx < tile_size * block_cols; idx += nthreads) { + const int tt = idx / block_cols; + const int c = idx % block_cols; + const int t = t0 + tt; + v_shared[tt][c] = v[sequence * sv3 + t * sv2 + h_idx * sv1 + col0 + c]; + } + if (thread < tile_size) { + const int64_t gb_offset = sequence * sb3 + (t0 + thread) * sb2 + h_idx * sb1; + g_shared[thread] = g[gb_offset]; + beta_shared[thread] = beta[gb_offset]; + } + __syncthreads(); + + for (int tt = 0; tt < tile_size; ++tt) { + const int t = t0 + tt; + + float k_reg[rows_per_lane]; + float q_reg[rows_per_lane]; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + k_reg[r] = k_shared[tt][i]; + q_reg[r] = q_shared[tt][i]; + } + + const float g_val = expf(g_shared[tt]); + const float beta_val = beta_shared[tt]; + + // The FMA contractions are spelled out so that the results are bit-identical to the + // compiler's contraction of gated_delta_net_cuda (fma chains from 0, fma(-g, kv, v)*beta, + // fma(g, s, k*delta)) independently of the surrounding code. + float attn_col[COLS]; +#pragma unroll + for (int c = 0; c < COLS; c++) { + // kv[col] = (S^T @ k)[col] = sum_i S[i][col] * k[i] + float kv_shard = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + kv_shard = fmaf(s_shard[c][r], k_reg[r], kv_shard); + } + const float kv_col = gdn_warp_reduce_sum(kv_shard); + + // delta[col] = (v[col] - g * kv[col]) * beta + const float delta_col = fmaf(-g_val, kv_col, v_shared[tt][colw + c]) * beta_val; + + // fused: S[i][col] = g * S[i][col] + k[i] * delta[col] + // attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i] + float attn_partial = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + s_shard[c][r] = fmaf(g_val, s_shard[c][r], k_reg[r] * delta_col); + attn_partial = fmaf(s_shard[c][r], q_reg[r], attn_partial); + } + attn_col[c] = gdn_warp_reduce_sum(attn_partial); + } + + if (lane < COLS) { + float a = attn_col[0]; +#pragma unroll + for (int c = 1; c < COLS; c++) { + a = lane == c ? attn_col[c] : a; + } + attn_data[(int64_t) t * S_v * H + lane] = a * scale; + } + + if constexpr (keep_rs_t) { + const int target_slot = (int) n_tokens - 1 - t; + if (target_slot >= 0 && target_slot < K) { + float * snapshot = state + target_slot * state_slot_stride + (col0 + colw) * S_v; +#pragma unroll + for (int c = 0; c < COLS; c++) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + snapshot[c * S_v + r * warp_size + lane] = s_shard[c][r]; + } + } + } + } + } + __syncthreads(); + } + + if constexpr (!keep_rs_t) { +#pragma unroll + for (int c = 0; c < COLS; c++) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + state[(col0 + colw + c) * S_v + r * warp_size + lane] = s_shard[c][r]; + } + } + } +} + template static void launch_gated_delta_net( const float * q_d, const float * k_d, const float * v_d, @@ -327,6 +533,24 @@ static void launch_gated_delta_net( } } + if constexpr (!KDA) { + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && num_warps == 32 && n_tokens >= 16) { + // 16 warps x 4 columns per block, 16-token tiles: best of the swept configurations on gfx1151 + // (2048 tokens, H=32: 5.03 ms -> 2.2 ms; H=64: 9.29 ms -> 4.5 ms). + constexpr int tiled_warps = 16; + constexpr int tiled_cols = 4; + constexpr int tiled_tile = 16; + const dim3 tiled_grid(H, n_seqs, S_v / (tiled_warps*tiled_cols)); + const dim3 tiled_block(warp_size, tiled_warps, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(tiled_grid, tiled_block, 0, stream); + ggml_cuda_kernel_launch(gated_delta_net_tiled_cuda<128, tiled_warps, tiled_cols, tiled_tile, keep_rs_t>, launch_params, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, + sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, + neqk1_magic, rq3_magic, scale, state_slot_stride, K); + return; + } + } + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); switch (S_v) { case 16: From d305b3d2a54deea7eb7595b8096f6163cf2cac8f Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:14:08 +0200 Subject: [PATCH 12/30] hip: vectorize exact expert aggregation on AMD The fused weighted expert sum used a volatile product to prevent FMA contraction and preserve the unfused graph's rounding. On AMD this spills every product to scratch. Use explicit v_mul_f32/v_add_f32 instructions to retain the exact operation order without scratch traffic, and process four adjacent channels per thread to amortize token indexing and weight loads. The fusion is restricted to contiguous 2048-wide Qwen tensors, so all float4 accesses are aligned. A direct old/new kernel comparison across 1,048,576 random outputs is bit-identical; MUL and ADD backend tests pass. gfx1151 weighted_expert_sum_f32<8>: 813 -> 689 us (-15.3%), scratch 8 -> 0 bytes. Qwen3.6-35B-A3B PP2048 improves by 0.35-1.36% across Q8_0, Q6_K_XL, Q4_K_XL, and IQ3_XXS. --- ggml/src/ggml-cuda/binbcast.cu | 54 +++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu index 83361ab75182..d3486588ed03 100644 --- a/ggml/src/ggml-cuda/binbcast.cu +++ b/ggml/src/ggml-cuda/binbcast.cu @@ -585,27 +585,60 @@ void ggml_cuda_op_fused_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst, } } +#if defined(__HIP_PLATFORM_AMD__) +static __device__ __forceinline__ float rounded_mul_f32(const float a, const float b) { + float result; + asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +static __device__ __forceinline__ float rounded_add_f32(const float a, const float b) { + float result; + asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} +#else +static __device__ __forceinline__ float rounded_mul_f32(const float a, const float b) { + return __fmul_rn(a, b); +} + +static __device__ __forceinline__ float rounded_add_f32(const float a, const float b) { + return __fadd_rn(a, b); +} +#endif + template static __global__ void weighted_expert_sum_f32( const float * experts, const float * weights, float * dst, int64_t n_embd, int64_t n_tokens, int64_t experts_s1, int64_t experts_s2, int64_t weights_s1, int64_t weights_s2) { const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; - if (index >= n_embd * n_tokens) { + const int64_t n_embd4 = n_embd / 4; + if (index >= n_embd4 * n_tokens) { return; } - const int64_t token = index / n_embd; - const int64_t channel = index - token * n_embd; - // Keep MUL rounding separate from ADD to match the unfused graph. - volatile float product = experts[token * experts_s2 + channel] * weights[token * weights_s2]; - float sum = product; + const int64_t token = index / n_embd4; + const int64_t channel = 4 * (index - token * n_embd4); + float4 expert = *reinterpret_cast(experts + token * experts_s2 + channel); + const float weight = weights[token * weights_s2]; + // Keep MUL rounding separate from ADD to match the unfused graph without forcing products through scratch memory. + float4 sum = { + rounded_mul_f32(expert.x, weight), + rounded_mul_f32(expert.y, weight), + rounded_mul_f32(expert.z, weight), + rounded_mul_f32(expert.w, weight), + }; #pragma unroll for (int i = 1; i < n_expert_used; ++i) { - product = experts[token * experts_s2 + i * experts_s1 + channel] * weights[token * weights_s2 + i * weights_s1]; - sum += product; + expert = *reinterpret_cast(experts + token * experts_s2 + i * experts_s1 + channel); + const float weight_i = weights[token * weights_s2 + i * weights_s1]; + sum.x = rounded_add_f32(sum.x, rounded_mul_f32(expert.x, weight_i)); + sum.y = rounded_add_f32(sum.y, rounded_mul_f32(expert.y, weight_i)); + sum.z = rounded_add_f32(sum.z, rounded_mul_f32(expert.z, weight_i)); + sum.w = rounded_add_f32(sum.w, rounded_mul_f32(expert.w, weight_i)); } - dst[index] = sum; + reinterpret_cast(dst)[index] = sum; } void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tensor * mul, ggml_tensor * dst, int n_expert_used) { @@ -614,7 +647,8 @@ void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tens const int64_t n_embd = experts->ne[0]; const int64_t n_tokens = experts->ne[2]; const int threads = 256; - const int blocks = (n_embd * n_tokens + threads - 1) / threads; + GGML_ASSERT(n_embd % 4 == 0); + const int blocks = (n_embd / 4 * n_tokens + threads - 1) / threads; const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream()); // MUL output may alias experts, so stage until all expert reads finish. ggml_cuda_pool_alloc result(ctx.pool(), n_embd * n_tokens); From ab55b8fdc393d7afa83a47354bf2a24939286946 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 08:31:02 +0200 Subject: [PATCH 13/30] hip: fuse Q8_1 activation quantization into the Q8_0 decode matvec on RDNA3.5 Token generation on gfx1151 launches ~1480 kernels per token with a ~2 us gap between each, so the quantize_q8_1 kernel in front of every MMVQ costs about as much as the quantization itself. Add mul_mat_vec_q_fq for the single-column Q8_0 path: a block of 16 waves (one output row per wave) quantizes the activation vector once into shared memory, replicating quantize_q8_1 exactly (same amax, IEEE division and roundf), and then runs the per-wave K loop of mul_mat_vec_q in the same order against it, with two K iterations of weight loads issued before the quantization and barrier. The two-kernel path is kept for other types, batched decode, and unaligned activations. Dense and MUL_MAT_ID (fused SWIGLU and down) outputs at all Qwen3.6 decode shapes are byte-identical to the previous path; MUL_MAT and MUL_MAT_ID backend tests pass. 16 waves per block and a prefetch depth of 2 were the fastest of the measured variants (8/32 waves, depth 1/4/8/16). Qwen3.6-35B-A3B TG128 on gfx1151: Q8_0 48.83 -> 50.99 t/s (+4.4%) at depth 0 and 42.88 -> 44.47 (+3.7%) at 32K; UD-Q8_K_XL 48.04 -> 49.61 (+3.3%) and 42.24 -> 43.38 (+2.7%). Adds Qwen3.6 batch-1 perf cases to test-backend-ops. Assisted-by: OpenCode --- ggml/src/ggml-cuda/mmvq.cu | 396 +++++++++++++++++++++++++++++++++++++ tests/test-backend-ops.cpp | 15 ++ 2 files changed, 411 insertions(+) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index ec1855f22526..5ef2696b8140 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -1157,6 +1157,369 @@ static __global__ void mul_mat_vec_q( } } +// --------------------------------------------------------------------------------------------- +// RDNA3.5 single-column MMVQ with the Q8_1 activation quantization fused into the matvec kernel. +// +// Token generation on gfx1151 pays a ~2 us launch gap for every kernel, so the quantize_q8_1 launch in front +// of each matvec costs about as much as the whole quantize kernel. A block of MMVQ_FQ_NWARPS waves (one output +// row per wave) quantizes the activation vector once into shared memory, replicating quantize_q8_1 exactly +// (same amax, same IEEE division and roundf), and then runs the per-wave K loop of mul_mat_vec_q in +// the same order against it, so the outputs are bit-identical to the two-kernel path. +// +// Measured on gfx1151 with Qwen3.6-35B-A3B Q8_0 decode (2026-09-03): 16 waves per block and 2 prefetched +// K iterations; 8 waves or deeper prefetch (more VGPRs, less occupancy) were slower. + +#define MMVQ_FQ_NWARPS 16 + +static constexpr __host__ __device__ bool mmvq_fq_type_ok(ggml_type type) { + switch (type) { + case GGML_TYPE_Q8_0: + return true; + default: + return false; + } +} + +// Whether the vec_dot of the type reads the Q8_1 block sum (ds.y) in addition to the scale. +static constexpr __host__ __device__ bool mmvq_fq_needs_sum(ggml_type type) { + switch (type) { + case GGML_TYPE_Q8_0: + return false; + default: + return true; + } +} + +// Sum of the 32 values of a Q8_1 block in exactly the order of warp_reduce_sum<32> in quantize_q8_1, where lane +// l holds value l. Here lane (sub) of a group of 4 holds values 8*sub .. 8*sub+7 in v[]. +static __device__ __forceinline__ float mmvq_fq_block_sum(const float * v, const int width) { + float a[8]; +#pragma unroll + for (int k = 0; k < 8; ++k) { + // offset 16: a_i = x_i + x_{i^16}, x_{i^16} lives two lanes over + a[k] = v[k] + __shfl_xor_sync(0xffffffff, v[k], 2, width); + } + float b[8]; +#pragma unroll + for (int k = 0; k < 8; ++k) { + // offset 8: b_i = a_i + a_{i^8}, one lane over + b[k] = a[k] + __shfl_xor_sync(0xffffffff, a[k], 1, width); + } + float c[4]; +#pragma unroll + for (int k = 0; k < 4; ++k) { + c[k] = b[k] + b[k + 4]; // offset 4 + } + const float d0 = c[0] + c[2]; // offset 2 + const float d1 = c[1] + c[3]; + return d0 + d1; // offset 1 +} + +// Register-staged Q8_0 weight loads: the K loop of one wave is split into chunks of MMVQ_FQ_PF iterations whose +// weight blocks are loaded up front, so the DRAM stream starts before the activation quantization and the +// barrier, and each wave keeps MMVQ_FQ_PF loads in flight instead of one. +#define MMVQ_FQ_PF 2 + +struct mmvq_fq_q8_0_chunk { + int qs[MMVQ_FQ_PF][VDR_Q8_0_Q8_1_MMVQ]; + half d[MMVQ_FQ_PF]; +}; + +static __device__ __forceinline__ void mmvq_fq_q8_0_load( + mmvq_fq_q8_0_chunk & c, const block_q8_0 * GGML_CUDA_RESTRICT x, const int kbx0, const int kqs, const int blocks_per_row_x) { + constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * ggml_cuda_get_physical_warp_size() / QI8_0; +#pragma unroll + for (int i = 0; i < MMVQ_FQ_PF; ++i) { + const int kbx = kbx0 + i*blocks_per_iter; + if (kbx < blocks_per_row_x) { +#pragma unroll + for (int j = 0; j < VDR_Q8_0_Q8_1_MMVQ; ++j) { + c.qs[i][j] = get_int_b2(x[kbx].qs, kqs + j); + } + c.d[i] = x[kbx].d; + } + } +} + +static __device__ __forceinline__ void mmvq_fq_q8_0_dot( + float & tmp, const mmvq_fq_q8_0_chunk & c, const block_q8_1 * y, const int kbx0, const int kqs, const int blocks_per_row_x) { + constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * ggml_cuda_get_physical_warp_size() / QI8_0; +#pragma unroll + for (int i = 0; i < MMVQ_FQ_PF; ++i) { + const int kbx = kbx0 + i*blocks_per_iter; + if (kbx < blocks_per_row_x) { + const block_q8_1 * bq8_1 = &y[kbx]; // qk == QK8_1 for Q8_0 + int u[VDR_Q8_0_Q8_1_MMVQ]; +#pragma unroll + for (int j = 0; j < VDR_Q8_0_Q8_1_MMVQ; ++j) { + u[j] = get_int_b4(bq8_1->qs, kqs + j); + } + tmp += vec_dot_q8_0_q8_1_impl(c.qs[i], u, c.d[i], __low2half(bq8_1->ds)); + } + } +} + +template +__launch_bounds__(nwarps * ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_q_fq( + const void * vx_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, + const uint32_t ncols_x, const uint32_t nrows_x, const uint3 nchannels_y, const uint32_t stride_row_x, + const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, + const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, + const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { + static_assert(type == GGML_TYPE_Q8_0, "fused activation quantization is only implemented for Q8_0"); + const void * GGML_CUDA_RESTRICT vx = vx_ptr; + const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; + + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = get_vdr_mmvq(type); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int blocks_per_iter = vdr * warp_size / qi; + static_assert(qk == QK8_1); + + extern __shared__ char mmvq_fq_smem[]; + block_q8_1 * y_q8 = (block_q8_1 *) mmvq_fq_smem; + + const int lane = threadIdx.x; + const int tid = warp_size*threadIdx.y + lane; + + const uint32_t channel_dst = blockIdx.y; + const uint32_t sample_dst = blockIdx.z; + const uint32_t channel_x = ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio); + const uint32_t channel_y = ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst; + const uint32_t sample_x = fastdiv(sample_dst, sample_ratio); + const uint32_t sample_y = sample_dst; + + const uint32_t row = blockIdx.x*nwarps + threadIdx.y; + const bool row_ok = row < nrows_x; + const uint32_t row_safe = row_ok ? row : nrows_x - 1; + + bool use_gate = false; + bool use_bias = false; + bool use_gate_bias = false; + [[maybe_unused]] const void * vgate = nullptr; + const float * x_bias = nullptr; + const float * gate_bias = nullptr; + ggml_glu_op active_glu; + float glu_limit = 0.0f; + + if constexpr (has_fusion) { + use_gate = fusion.gate != nullptr; + use_bias = fusion.x_bias != nullptr; + use_gate_bias = fusion.gate_bias != nullptr && use_gate; + vgate = fusion.gate; + x_bias = (const float *) fusion.x_bias; + gate_bias = (const float *) fusion.gate_bias; + active_glu = fusion.glu_op; + glu_limit = fusion.glu_limit; + } + + // 1. start streaming the weights of this wave's row before anything else + const int blocks_per_row_x = ncols_x / qk; + const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row_safe*stride_row_x; + const int kqs = vdr * (lane % (qi/vdr)); + const int kbx0 = lane / (qi/vdr); + + const block_q8_0 * x = (const block_q8_0 *) vx + kbx_offset; + [[maybe_unused]] const block_q8_0 * xg = nullptr; + + mmvq_fq_q8_0_chunk cx; + [[maybe_unused]] mmvq_fq_q8_0_chunk cg; + mmvq_fq_q8_0_load(cx, x, kbx0, kqs, blocks_per_row_x); + if constexpr (has_fusion) { + if (use_gate) { + xg = (const block_q8_0 *) vgate + kbx_offset; + mmvq_fq_q8_0_load(cg, xg, kbx0, kqs, blocks_per_row_x); + } + } + + // 2. quantize the activation vector into shared memory: 4 lanes per Q8_1 block, 8 values per lane + { + const float * y = y_ptr + sample_y*stride_sample_y + channel_y*stride_channel_y; + const int nby = ncols_x / QK8_1; + for (int i = tid; i < 4*nby; i += nwarps*warp_size) { + const int ib = i >> 2; + const int sub = i & 3; + const float4 * yv = (const float4 *) (y + ib*QK8_1 + sub*8); + const float4 v0 = yv[0]; + const float4 v1 = yv[1]; + const float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + + float amax = fabsf(v[0]); +#pragma unroll + for (int k = 1; k < 8; ++k) { + amax = fmaxf(amax, fabsf(v[k])); + } + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 1, warp_size)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 2, warp_size)); + + const float sum = mmvq_fq_needs_sum(type) ? mmvq_fq_block_sum(v, warp_size) : 0.0f; + + const float d = amax / 127.0f; + int q[8]; +#pragma unroll + for (int k = 0; k < 8; ++k) { + const int8_t qk8 = amax == 0.0f ? 0 : roundf(v[k] / d); + q[k] = (int) qk8 & 0xff; + } + int * qs = (int *) y_q8[ib].qs; + qs[2*sub + 0] = q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24); + qs[2*sub + 1] = q[4] | (q[5] << 8) | (q[6] << 16) | (q[7] << 24); + if (sub == 0) { + y_q8[ib].ds = make_half2(d, sum); + } + } + } + __syncthreads(); + + if (!row_ok) { + return; + } + + [[maybe_unused]] float x_biases = 0.0f; + [[maybe_unused]] float gate_biases = 0.0f; + if constexpr (has_fusion) { + const uint32_t channel_bias = ids ? channel_x : channel_dst; + if (lane == 0) { + if (use_bias) { + x_biases = x_bias[sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row]; + } + if (use_gate_bias) { + gate_biases = gate_bias[sample_dst*stride_sample_dst + channel_bias*stride_channel_dst + row]; + } + } + } + + // 3. exact per-wave dot products in the k order of mul_mat_vec_q, one chunk ahead in memory + float tmp = 0.0f; + float tmp_gate = 0.0f; + + const block_q8_1 * y = y_q8; + constexpr int chunk_blocks = MMVQ_FQ_PF*blocks_per_iter; + for (int c0 = kbx0; c0 < blocks_per_row_x; c0 += chunk_blocks) { + mmvq_fq_q8_0_chunk nx; + [[maybe_unused]] mmvq_fq_q8_0_chunk ng; + const int c1 = c0 + chunk_blocks; + if (c1 < blocks_per_row_x) { + mmvq_fq_q8_0_load(nx, x, c1, kqs, blocks_per_row_x); + if constexpr (has_fusion) { + if (use_gate) { + mmvq_fq_q8_0_load(ng, xg, c1, kqs, blocks_per_row_x); + } + } + } + mmvq_fq_q8_0_dot(tmp, cx, y, c0, kqs, blocks_per_row_x); + if constexpr (has_fusion) { + if (use_gate) { + mmvq_fq_q8_0_dot(tmp_gate, cg, y, c0, kqs, blocks_per_row_x); + } + } + cx = nx; + if constexpr (has_fusion) { + if (use_gate) { + cg = ng; + } + } + } + + tmp = warp_reduce_sum(tmp); + if constexpr (has_fusion) { + if (use_gate) { + tmp_gate = warp_reduce_sum(tmp_gate); + } + } + + if (lane == 0) { + float result = tmp; + if constexpr (has_fusion) { + result += x_biases; + if (use_gate) { + float gate_value = tmp_gate; + gate_value += gate_biases; + switch (active_glu) { + case GGML_GLU_OP_SWIGLU: + result *= ggml_cuda_op_silu_single(gate_value); + break; + case GGML_GLU_OP_GEGLU: + result *= ggml_cuda_op_gelu_single(gate_value); + break; + case GGML_GLU_OP_SWIGLU_OAI: + result = ggml_cuda_op_swiglu_oai_single(gate_value, result); + break; + case GGML_GLU_OP_SWIGLU_CLAMP: + result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit); + break; + default: + result = result * gate_value; + break; + } + } + } + dst[sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row] = result; + } + + if constexpr (!has_fusion) { + GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, active_glu, glu_limit, gate_bias, x_bias, tmp_gate); + } + GGML_UNUSED(stride_col_dst); +} + +template +static void mul_mat_vec_q_fq_launch( + const void * vx, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device & fusion, float * dst, + const int ncols_x, const int nrows_x, const int stride_row_x, const int stride_col_dst, + const int nchannels_x, const int nchannels_y, const int nchannels_dst, + const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, + const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, + cudaStream_t stream) { + const int device = ggml_cuda_get_device(); + const int warp_size = ggml_cuda_info().devices[device].warp_size; + + const uint3 nchannels_y_fd = ids ? init_fastdiv_values(nchannels_y) : make_uint3(0, 0, 0); + const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0) : init_fastdiv_values(nchannels_dst / nchannels_x); + const uint3 sample_ratio_fd = init_fastdiv_values(nsamples_dst / nsamples_x); + + constexpr int nwarps = MMVQ_FQ_NWARPS; + const dim3 block_nums((nrows_x + nwarps - 1) / nwarps, nchannels_dst, nsamples_dst); + const dim3 block_dims(warp_size, nwarps, 1); + const int nbytes_shared = (ncols_x / QK8_1) * sizeof(block_q8_1); + + const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream); + if (has_fusion) { + ggml_cuda_kernel_launch(mul_mat_vec_q_fq, launch_params, + vx, y, ids, fusion, dst, ncols_x, nrows_x, nchannels_y_fd, stride_row_x, stride_col_dst, + channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, + sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst); + } else { + ggml_cuda_kernel_launch(mul_mat_vec_q_fq, launch_params, + vx, y, ids, fusion, dst, ncols_x, nrows_x, nchannels_y_fd, stride_row_x, stride_col_dst, + channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, + sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst); + } +} + +static void mul_mat_vec_q_fq_switch_type( + const void * vx, const ggml_type type_x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device & fusion, float * dst, + const int ncols_x, const int nrows_x, const int stride_row_x, const int stride_col_dst, + const int nchannels_x, const int nchannels_y, const int nchannels_dst, + const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, + const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, + cudaStream_t stream) { + switch (type_x) { + case GGML_TYPE_Q8_0: + mul_mat_vec_q_fq_launch(vx, y, ids, fusion, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream); + break; + default: + GGML_ABORT("fatal error"); + break; + } +} + // Dedicated MoE multi-token kernel. // Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst) // Block: (warp_size, ncols_dst) - each warp handles one token independently. @@ -1746,6 +2109,39 @@ void ggml_cuda_mul_mat_vec_q( } } + { + // RDNA3.5 token generation: quantize the activations inside the matvec kernel (bit-identical, one launch less). + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int64_t ncols_dst_fq = ids ? ne2 : ne1; + const bool aligned = (uintptr_t) src1_d % 16 == 0 && nb11 % 16 == 0 && nb12 % 16 == 0 && nb13 % 16 == 0; + const bool fusion_ok = !fusion || (fusion->x_scale == nullptr && fusion->gate_scale == nullptr); + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols_dst_fq == 1 && mmvq_fq_type_ok(src0->type) && aligned && fusion_ok && + ne10 % QK8_1 == 0 && (ne10 / QK8_1) * sizeof(block_q8_1) <= 16384) { + const int64_t s01 = src0->nb[1] / ts_src0; + const int64_t s02 = src0->nb[2] / ts_src0; + const int64_t s03 = src0->nb[3] / ts_src0; + const int64_t s11 = src1->nb[1] / ts_src1; + const int64_t s12 = src1->nb[2] / ts_src1; + const int64_t s13 = src1->nb[3] / ts_src1; + const int64_t s1 = dst->nb[1] / ts_dst; + const int64_t s2 = dst->nb[2] / ts_dst; + const int64_t s3 = dst->nb[3] / ts_dst; + + // For MUL_MAT_ID the memory layout is different than for MUL_MAT: + const int64_t nchannels_y = ids ? ne11 : ne12; + const int64_t nchannels_dst = ids ? ne1 : ne2; + const int64_t stride_col_dst = ids ? s2 : s1; + const int64_t stride_channel_dst = ids ? s1 : s2; + const int64_t stride_channel_y = ids ? s11 : s12; + + mul_mat_vec_q_fq_switch_type( + src0->data, src0->type, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, s01, stride_col_dst, + ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, + ne03, ne3, s03, s13, s3, stream); + return; + } + } + const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING); ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1); { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 9478c2056b44..e90fad598e38 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10751,6 +10751,21 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512)); } + // Qwen3.6-35B-A3B token generation (batch 1): + // routed experts gate/up (fused) and down, shared expert, GDN qkv/gate/out, attention q/o, output head + for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_Q4_K, GGML_TYPE_BF16}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 512, 1, 2048)); + test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, GGML_TYPE_F32, 256, 8, false, 512, 1, 2048, 1)); + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 2048, 1, 512)); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 512, 1, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 2048, 1, 512, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 8192, 1, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, 1, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 2048, 1, 4096, {1, 1}, {1, 1})); + } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 248320, 1, 2048, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 256, 1, 2048, {1, 1}, {1, 1})); + for (int K : {3, 5}) { for (int IC : {256, 2560}) { for (int IW_IH : {32, 64, 256}) { From 1cf1594e6aa63260bc27543d2a933869ce5e78d9 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:16:09 +0200 Subject: [PATCH 14/30] hip: fuse the Qwen3.5 Gated DeltaNet decode step on AMD For a single decode token, every linear-attention layer runs 14 graph nodes between the qkv projection and the output gate: concat with the conv state, conv state shift into the cache, state gather, causal conv, SiLU, two L2 norms, alpha bias/softplus/A gate, beta sigmoid, the recurrence, the state copy into the cache, and the gated RMS norm. On gfx1151 these are 11 small kernels (~32 us) separated by ~2 us launch gaps, about 50 us per layer. Match this chain in the CUDA backend and run it as two kernels: one block per value head computes the conv + SiLU of its q/k/v channels (shifting the conv state straight into the cache), the L2 norms, gate and beta, and the recurrence with the state read from and written to the cache in place; a second tiny kernel applies the gated RMS norm (its destination may alias inputs of the first kernel that other blocks are still reading). Every op is spelled with the operation order and FMA contraction of the kernel it replaces, so the outputs, conv state and recurrent state are byte-identical to the unfused graph (checked in place and across cache rows). The fusion requires n_tokens == n_seqs == 1 and S == 128 and falls back otherwise, including the first token of a sequence where the state reset sits inside the chain. Qwen3.6-35B-A3B TG128 on gfx1151 (with the fused activation quantization): Q8_0 48.74 -> 54.18 t/s (+11.2%) at depth 0 and 42.62 -> 46.62 (+9.4%) at 32K; UD-Q8_K_XL 48.02 -> 52.69 (+9.7%) and 42.23 -> 45.71 (+8.2%). Assisted-by: OpenCode --- ggml/src/ggml-cuda/gated_delta_net.cu | 227 ++++++++++++++++++++++++ ggml/src/ggml-cuda/gated_delta_net.cuh | 31 ++++ ggml/src/ggml-cuda/ggml-cuda.cu | 231 +++++++++++++++++++++++++ 3 files changed, 489 insertions(+) diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index ce9a5c5e8ad7..4873b853c742 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -1,5 +1,6 @@ #include "gated_delta_net.cuh" #include "ggml-cuda/common.cuh" +#include "unary.cuh" // Warp reduction without the LDS crossbar: on RDNA the generic warp_reduce_sum compiles to 5 dependent @@ -690,3 +691,229 @@ void ggml_cuda_op_gated_delta_net_fused_cache( ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_cuda_gated_delta_net_fused_cache cache) { ggml_cuda_op_gated_delta_net_impl(ctx, dst, &cache); } + +// --------------------------------------------------------------------------------------------- +// Fused Qwen3.5/3.6 Gated DeltaNet decode step (n_tokens == 1, n_seqs == 1, S == 128): one block per value +// head computes the causal conv + SiLU of its q/k/v channels (writing the shifted conv state back into the +// cache), the q/k L2 norms, the gate (softplus) and beta (sigmoid), the recurrence with the new state written +// straight into the cache, and the gated RMS norm of the attention output. This replaces 14 graph nodes +// (11 kernels plus their ~2 us launch gaps on gfx1151) that together take ~50 us per layer in decode. +// +// Every op is spelled with the same operation order and FMA contraction as the kernels it replaces +// (ssm_conv_f32, l2_norm_dual_f32_s128, unary/binary elementwise, gated_delta_net_cuda, rms_norm_f32<128>), +// so the outputs are bit-identical to the unfused graph. + +static __device__ __forceinline__ float gdn_decode_sigmoid(float x) { + return 1.0f / (1.0f + expf(-x)); +} + +static __device__ __forceinline__ float gdn_decode_softplus(float x) { + return (x > 20.0f) ? x : logf(1.0f + expf(x)); +} + +template +__global__ void __launch_bounds__(32 * 32, 1) +gdn_decode_fused_cuda(const ggml_cuda_gdn_decode_args args) { + constexpr int warp_size = 32; + constexpr int nwarps = 32; + constexpr int rows_per_lane = S / warp_size; + constexpr int cols_per_warp = S / nwarps; + static_assert(S == warp_size * rows_per_lane && S == nwarps * cols_per_warp); + + __shared__ float q_c[S]; + __shared__ float k_c[S]; + __shared__ float v_c[S]; + __shared__ float q_n[S]; + __shared__ float k_n[S]; + __shared__ float attn[S]; + __shared__ float red[4]; + + const int h = blockIdx.x; // value head + const int lane = threadIdx.x; + const int warp = threadIdx.y; + const int thread = warp * warp_size + lane; + const int hk = h % args.H_k; // key/query head (fastmodulo(h_idx, neqk1) in gated_delta_net_cuda) + + // 1. causal conv + SiLU for this head's q, k and v channels; shift the conv state (CPY conv_state_last) + if (thread < 3 * S) { + const int part = thread / S; // 0: q, 1: k, 2: v + const int i = thread % S; + const int c = part == 0 ? hk * S : part == 1 ? args.H_k * S + hk * S : 2 * args.H_k * S + h * S; + const int ch = c + i; + + float x[D_CONV]; + float w[D_CONV]; +#pragma unroll + for (int j = 0; j < D_CONV - 1; j++) { + x[j] = args.conv_state_in[ch * (D_CONV - 1) + j]; + } + x[D_CONV - 1] = args.qkv[ch * args.qkv_stride]; +#pragma unroll + for (int j = 0; j < D_CONV; j++) { + w[j] = args.conv_w[ch * D_CONV + j]; + } + + // ssm_conv_f32: fma chain from 0, then the (zero) bias add, then SiLU + float sumf = fmaf(x[0], w[0], 0.0f); +#pragma unroll + for (int j = 1; j < D_CONV; j++) { + sumf = fmaf(x[j], w[j], sumf); + } + sumf += args.conv_bias; + const float y = ggml_cuda_op_silu_single(sumf); + + if (part == 0) { + q_c[i] = y; + } else if (part == 1) { + k_c[i] = y; + } else { + v_c[i] = y; + } + + // q/k channels are shared by H_v/H_k value heads: the first one writes the conv state + if (part == 2 || h == hk) { +#pragma unroll + for (int j = 0; j < D_CONV - 1; j++) { + args.conv_state_out[ch * (D_CONV - 1) + j] = x[j + 1]; + } + } + } + __syncthreads(); + + // 2. l2_norm_dual_f32_s128 for q (warp 0) and k (warp 1) + if (warp < 2) { + const float * x = warp == 0 ? q_c : k_c; + float * y = warp == 0 ? q_n : k_n; + float tmp = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const float xi = x[r * warp_size + lane]; + tmp = fmaf(xi, xi, tmp); + } + tmp = gdn_warp_reduce_sum(tmp); + const float scale = rsqrtf(fmaxf(tmp, args.eps_l2 * args.eps_l2)); +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + y[r * warp_size + lane] = scale * x[r * warp_size + lane]; + } + } + + // gate = softplus(alpha + dt_bias) * A, beta = sigmoid(beta): scalar per head, computed by every thread + const float alpha_biased = args.alpha[h] + args.dt_bias[h]; + const float gate = gdn_decode_softplus(alpha_biased) * args.ssm_a[h]; + const float beta_val = gdn_decode_sigmoid(args.beta[h]); + const float g_val = expf(gate); + __syncthreads(); + + // 3. recurrence (gated_delta_net_cuda), cols_per_warp state columns per warp + { + const float * state_in = args.state_cache + (int64_t) args.state_ids[0] * args.state_row_stride + (int64_t) h * S * S; + float * state_out = args.state_out + (int64_t) h * S * S; + const int colw = warp * cols_per_warp; + + float k_reg[rows_per_lane]; + float q_reg[rows_per_lane]; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + k_reg[r] = k_n[r * warp_size + lane]; + q_reg[r] = q_n[r * warp_size + lane]; + } + + float s_shard[cols_per_warp][rows_per_lane]; +#pragma unroll + for (int c = 0; c < cols_per_warp; c++) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + s_shard[c][r] = state_in[(colw + c) * S + r * warp_size + lane]; + } + } + + // same explicit contractions as gated_delta_net_tiled_cuda (bit-identical to gated_delta_net_cuda) + float attn_col[cols_per_warp]; +#pragma unroll + for (int c = 0; c < cols_per_warp; c++) { + float kv_shard = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + kv_shard = fmaf(s_shard[c][r], k_reg[r], kv_shard); + } + const float kv_col = gdn_warp_reduce_sum(kv_shard); + + const float delta_col = fmaf(-g_val, kv_col, v_c[colw + c]) * beta_val; + + float attn_partial = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + s_shard[c][r] = fmaf(g_val, s_shard[c][r], k_reg[r] * delta_col); + attn_partial = fmaf(s_shard[c][r], q_reg[r], attn_partial); + } + attn_col[c] = gdn_warp_reduce_sum(attn_partial); + } + + if (lane < cols_per_warp) { + float a = attn_col[0]; +#pragma unroll + for (int c = 1; c < cols_per_warp; c++) { + a = lane == c ? attn_col[c] : a; + } + attn[colw + lane] = a * args.scale; + } + +#pragma unroll + for (int c = 0; c < cols_per_warp; c++) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + state_out[(colw + c) * S + r * warp_size + lane] = s_shard[c][r]; + } + } + } + __syncthreads(); + + // 4. pre-norm attention output to the scratch buffer (the gated rms norm runs as a second kernel: its + // destination may alias inputs of this kernel that other blocks are still reading) + if (thread < S) { + args.attn_out[(int64_t) h * S + thread] = attn[thread]; + } +} + +// rms_norm_f32<128, true>: per-warp partial sums, then the second-level butterfly of block_reduce over +// lanes 0..3 (which reduces to (s0 + s2) + (s1 + s3)), times the norm weight +template +__global__ void __launch_bounds__(S, 1) +gdn_decode_norm_cuda(const float * attn, const float * norm_w, float * out, const float eps) { + constexpr int warp_size = 32; + static_assert(S == 4 * warp_size); + __shared__ float red[4]; + + const int h = blockIdx.x; + const int tid = threadIdx.x; + const int warp = tid / warp_size; + const int lane = tid % warp_size; + + const float xi = attn[(int64_t) h * S + tid]; + const float part = gdn_warp_reduce_sum(fmaf(xi, xi, 0.0f)); + if (lane == 0) { + red[warp] = part; + } + __syncthreads(); + const float tmp = (red[0] + red[2]) + (red[1] + red[3]); + const float mean = tmp / (float) S; + const float scale = rsqrtf(mean + eps); + out[(int64_t) h * S + tid] = scale * xi * norm_w[tid]; +} + +void ggml_cuda_op_gdn_decode_fused(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args_in) { + GGML_ASSERT(args_in.S == 128 && args_in.d_conv == 4); + ggml_cuda_pool_alloc attn(ctx.pool(), args_in.S * args_in.H_v); + ggml_cuda_gdn_decode_args args = args_in; + args.attn_out = attn.get(); + + const dim3 grid(args.H_v, 1, 1); + const dim3 block(32, 32, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid, block, 0, ctx.stream()); + ggml_cuda_kernel_launch(gdn_decode_fused_cuda<128, 4>, launch_params, args); + + const dim3 norm_block(128, 1, 1); + const ggml_cuda_kernel_launch_params norm_params = ggml_cuda_kernel_launch_params(grid, norm_block, 0, ctx.stream()); + ggml_cuda_kernel_launch(gdn_decode_norm_cuda<128>, norm_params, attn.get(), args.norm_w, args.out, args.eps_rms); +} diff --git a/ggml/src/ggml-cuda/gated_delta_net.cuh b/ggml/src/ggml-cuda/gated_delta_net.cuh index f9bf43706789..b2be8c4f6b7a 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cuh +++ b/ggml/src/ggml-cuda/gated_delta_net.cuh @@ -12,3 +12,34 @@ void ggml_cuda_op_gated_delta_net(ggml_backend_cuda_context & ctx, ggml_tensor * // same op, but writes the snapshot(s) into the cache instead of dst (see ggml_cuda_try_gdn_cache_fusion) void ggml_cuda_op_gated_delta_net_fused_cache(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_cuda_gated_delta_net_fused_cache cache); + +// Fused Qwen3.5 Gated DeltaNet decode step (see ggml_cuda_try_gdn_decode_fusion). All strides in elements. +struct ggml_cuda_gdn_decode_args { + // conv: conv_state_in [d_conv-1, C] (contiguous, per channel), qkv [C] with stride, conv_w [d_conv, C] + const float * conv_state_in; + const float * qkv; + int64_t qkv_stride; + const float * conv_w; + float conv_bias; // 0.0f (ssm_conv adds a runtime zero when there is no bias) + float * conv_state_out; // [d_conv-1, C] (contiguous) cache row + // gate / beta + const float * alpha; // [H_v] + const float * dt_bias; // [H_v] + const float * ssm_a; // [H_v] + const float * beta; // [H_v] + float eps_l2; + // recurrence + const float * state_cache; // [S*S*H_v, n_slots] cache, row = state_ids[0] (transposed per head: [col][row]) + const int32_t * state_ids; + int64_t state_row_stride; + float * state_out; // [S, S, H_v] cache row + float scale; // 1/sqrt(S) + // gated rms norm + const float * norm_w; // [S] + float eps_rms; + float * out; // [S, H_v] + float * attn_out; // [S, H_v] pre-norm scratch (set by the host) + int64_t S, H_k, H_v, d_conv; +}; + +void ggml_cuda_op_gdn_decode_fused(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index e1c156af2fea..aa35efbe25ec 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3025,6 +3025,227 @@ static bool ggml_cuda_check_fusion_memory_ranges(const ggml_cgraph * cgraph, return is_ok; } +// match the Qwen3.5 Gated DeltaNet decode chain +// concat(conv_state, qkv^T) -> cpy(conv state shift -> cache) -> get_rows(state cache) -> ssm_conv -> silu +// -> l2_norm(q) -> l2_norm(k) -> add(alpha, dt_bias) -> softplus -> mul(A) -> sigmoid(beta) +// -> gated_delta_net -> cpy(new state -> cache) -> rms_norm -> mul(norm weight) +// for a single token of a single sequence with S == 128, so that one kernel can replace the 14 nodes. +// Returns the number of nodes to skip after the concat (0 when the pattern does not match). +static int ggml_cuda_try_gdn_decode_fusion(const ggml_cgraph * cgraph, int node_idx, ggml_cuda_gdn_decode_args & args) { + const ggml_tensor * concat = cgraph->nodes[node_idx]; + if (concat->op != GGML_OP_CONCAT || concat->type != GGML_TYPE_F32 || ggml_get_op_params_i32(concat, 0) != 0) { + return 0; + } + const int64_t d_conv = concat->ne[0]; + const int64_t C = concat->ne[1]; + if (d_conv != 4 || concat->ne[2] != 1 || concat->ne[3] != 1 || !ggml_is_contiguous(concat)) { + return 0; + } + const ggml_tensor * conv_state = concat->src[0]; // [d_conv-1, C, 1] + const ggml_tensor * qkv_t = concat->src[1]; // [1, C, 1] + if (conv_state->type != GGML_TYPE_F32 || qkv_t->type != GGML_TYPE_F32 || + conv_state->ne[0] != d_conv - 1 || conv_state->ne[1] != C || conv_state->ne[2] != 1 || !ggml_is_contiguous(conv_state) || + qkv_t->ne[0] != 1 || qkv_t->ne[1] != C || qkv_t->ne[2] != 1 || qkv_t->nb[1] % sizeof(float) != 0) { + return 0; + } + + // collect the following compute nodes + constexpr int n_ops = 14; + const ggml_op expected[n_ops] = { + GGML_OP_CPY, GGML_OP_GET_ROWS, GGML_OP_SSM_CONV, GGML_OP_UNARY, GGML_OP_L2_NORM, GGML_OP_L2_NORM, + GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_UNARY, GGML_OP_GATED_DELTA_NET, GGML_OP_CPY, + GGML_OP_RMS_NORM, GGML_OP_MUL, + }; + const ggml_tensor * n[n_ops]; + int idx[n_ops]; + int found = 0; + int last = node_idx; + for (int j = node_idx + 1; j < cgraph->n_nodes && found < n_ops; ++j) { + const ggml_tensor * t = cgraph->nodes[j]; + if (ggml_cuda_is_view_or_noop(t)) { + continue; + } + if (t->op != expected[found] || t->type != GGML_TYPE_F32) { + return 0; + } + n[found] = t; + idx[found] = j; + last = j; + ++found; + } + if (found != n_ops) { + return 0; + } + // intermediates must not be graph outputs + if (concat->flags & GGML_TENSOR_FLAG_OUTPUT) { + return 0; + } + for (int k = 0; k < n_ops; ++k) { + if (k != 0 && k != 11 && k != 13 && (n[k]->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + } + + const ggml_tensor * cpy_conv = n[0]; + const ggml_tensor * get_rows = n[1]; + const ggml_tensor * ssm_conv = n[2]; + const ggml_tensor * silu = n[3]; + const ggml_tensor * l2_q = n[4]; + const ggml_tensor * l2_k = n[5]; + const ggml_tensor * add = n[6]; + const ggml_tensor * softplus = n[7]; + const ggml_tensor * mul_a = n[8]; + const ggml_tensor * sigmoid = n[9]; + const ggml_tensor * gdn = n[10]; + const ggml_tensor * rms = n[12]; + const ggml_tensor * mul_w = n[13]; + + auto is_view_of = [](const ggml_tensor * t, const ggml_tensor * src, size_t offs) { + return t == src ? offs == 0 : (t->view_src == src && t->view_offs == offs); + }; + + // conv state shift: view of the concat (skipping the oldest column) copied into a contiguous cache row + { + const ggml_tensor * src = cpy_conv->src[0]; + const ggml_tensor * dst = cpy_conv->src[1]; + if (!is_view_of(src, concat, sizeof(float)) || src->ne[0] != d_conv - 1 || src->ne[1] != C || src->ne[2] != 1 || + src->nb[1] != concat->nb[1] || dst->type != GGML_TYPE_F32 || dst->data == nullptr || + ggml_nelements(dst) != (d_conv - 1) * C || !ggml_is_contiguous(dst)) { + return 0; + } + } + + // ssm_conv(concat, weight [d_conv, C]) without bias, then silu + if (ssm_conv->src[0] != concat || ssm_conv->src[2] != nullptr || ssm_conv->src[1]->type != GGML_TYPE_F32 || + ssm_conv->src[1]->ne[0] != d_conv || ssm_conv->src[1]->ne[1] != C || !ggml_is_contiguous(ssm_conv->src[1]) || + ssm_conv->ne[0] != C || ggml_nelements(ssm_conv) != C || + ggml_get_unary_op(silu) != GGML_UNARY_OP_SILU || silu->src[0] != ssm_conv || !ggml_is_contiguous(silu)) { + return 0; + } + + // recurrence shapes + const ggml_tensor * v_view = gdn->src[2]; + const int64_t S = v_view->ne[0]; + const int64_t H_v = v_view->ne[1]; + const int64_t H_k = l2_q->ne[1]; + if (S != 128 || H_k <= 0 || H_v % H_k != 0 || v_view->ne[2] != 1 || v_view->ne[3] != 1 || + 2 * S * H_k + S * H_v != C || ggml_get_op_params_i32(gdn, 0) != 1) { + return 0; + } + + // q/k/v are views into the silu output: [S, H, 1, 1] with rows of S floats + auto qkv_view_ok = [&](const ggml_tensor * t, int64_t H, size_t offs) { + return t->view_src == silu && t->view_offs == offs && t->ne[0] == S && t->ne[1] == H && t->ne[2] == 1 && + t->ne[3] == 1 && t->nb[0] == sizeof(float) && t->nb[1] == S * sizeof(float); + }; + if (!qkv_view_ok(l2_q->src[0], H_k, 0) || !qkv_view_ok(l2_k->src[0], H_k, S * H_k * sizeof(float)) || + !qkv_view_ok(v_view, H_v, 2 * S * H_k * sizeof(float)) || + !ggml_are_same_shape(l2_q, l2_q->src[0]) || !ggml_are_same_shape(l2_k, l2_k->src[0]) || + !ggml_is_contiguous(l2_q) || !ggml_is_contiguous(l2_k)) { + return 0; + } + float eps_q, eps_k; + memcpy(&eps_q, l2_q->op_params, sizeof(float)); + memcpy(&eps_k, l2_k->op_params, sizeof(float)); + if (eps_q != eps_k) { + return 0; + } + + // gate = softplus(alpha + dt_bias) * A, beta = sigmoid(beta): H_v scalars + if (ggml_nelements(add) != H_v || add->src[0]->type != GGML_TYPE_F32 || add->src[1]->type != GGML_TYPE_F32 || + ggml_nelements(add->src[0]) != H_v || ggml_nelements(add->src[1]) != H_v || + !ggml_is_contiguous(add->src[0]) || !ggml_is_contiguous(add->src[1]) || + ggml_get_unary_op(softplus) != GGML_UNARY_OP_SOFTPLUS || softplus->src[0] != add || + mul_a->src[0] != softplus || mul_a->src[1]->type != GGML_TYPE_F32 || ggml_nelements(mul_a->src[1]) != H_v || + !ggml_is_contiguous(mul_a->src[1]) || ggml_nelements(mul_a) != H_v || + ggml_get_unary_op(sigmoid) != GGML_UNARY_OP_SIGMOID || sigmoid->src[0]->type != GGML_TYPE_F32 || + ggml_nelements(sigmoid->src[0]) != H_v || !ggml_is_contiguous(sigmoid->src[0]) || ggml_nelements(sigmoid) != H_v) { + return 0; + } + + // gated_delta_net(q, k, v, gate, beta, state) with the state gathered from the cache by get_rows + if (gdn->src[0] != l2_q || gdn->src[1] != l2_k || !is_view_of(gdn->src[3], mul_a, 0) || !is_view_of(gdn->src[4], sigmoid, 0) || + gdn->src[3]->ne[0] != 1 || ggml_nelements(gdn->src[3]) != H_v || ggml_nelements(gdn->src[4]) != H_v || + !is_view_of(gdn->src[5], get_rows, 0) || ggml_nelements(get_rows) != S * S * H_v) { + return 0; + } + { + const ggml_tensor * cache = get_rows->src[0]; + const ggml_tensor * ids = get_rows->src[1]; + if (cache->type != GGML_TYPE_F32 || cache->ne[0] != S * S * H_v || cache->nb[0] != sizeof(float) || + cache->nb[1] % sizeof(float) != 0 || ids->type != GGML_TYPE_I32 || ggml_nelements(ids) != 1) { + return 0; + } + } + + // new state -> cache (same check as the standalone gated_delta_net + cpy fusion) + ggml_cuda_gated_delta_net_fused_cache state_cpy; + if (ggml_cuda_try_gdn_cache_fusion(cgraph, idx[10], state_cpy) != idx[11] - idx[10] || state_cpy.slot_stride != 0) { + return 0; + } + + // gated rms norm of the attention output [S, H_v] + { + const ggml_tensor * x = rms->src[0]; + if (x->view_src != gdn || x->view_offs != 0 || x->ne[0] != S || x->ne[1] != H_v || x->ne[2] != 1 || x->ne[3] != 1 || + x->nb[0] != sizeof(float) || x->nb[1] != S * sizeof(float) || !ggml_are_same_shape(rms, x) || + mul_w->src[0] != rms || mul_w->src[1]->type != GGML_TYPE_F32 || ggml_nelements(mul_w->src[1]) != S || + !ggml_is_contiguous(mul_w->src[1]) || !ggml_are_same_shape(mul_w, rms) || !ggml_is_contiguous(mul_w)) { + return 0; + } + } + + // The first kernel writes only the two cache rows: the state row may alias the gathered input row (in-place + // update, every block reads its head before writing it) and the conv row must not alias any external input + // of the range. The final output is written by a second kernel after all inputs have been consumed. + { + const ggml_tensor * out = cpy_conv->src[1]; + for (int j = node_idx; j <= last; ++j) { + const ggml_tensor * t = cgraph->nodes[j]; + for (int si = 0; si < GGML_MAX_SRC; ++si) { + const ggml_tensor * src = t->src[si]; + if (!src || src->op == GGML_OP_NONE || src == out) { + continue; + } + const int64_t a0 = (int64_t) out->data; + const int64_t a1 = a0 + ggml_backend_buft_get_alloc_size(out->buffer->buft, out); + const int64_t b0 = (int64_t) src->data; + const int64_t b1 = b0 + ggml_backend_buft_get_alloc_size(src->buffer->buft, src); + if ((b0 <= a0 && a0 < b1) || (a0 <= b0 && b0 < a1)) { + return 0; + } + } + } + } + + args.conv_state_in = (const float *) conv_state->data; + args.qkv = (const float *) qkv_t->data; + args.qkv_stride = qkv_t->nb[1] / sizeof(float); + args.conv_w = (const float *) ssm_conv->src[1]->data; + args.conv_bias = 0.0f; + args.conv_state_out = (float *) cpy_conv->src[1]->data; + args.alpha = (const float *) add->src[0]->data; + args.dt_bias = (const float *) add->src[1]->data; + args.ssm_a = (const float *) mul_a->src[1]->data; + args.beta = (const float *) sigmoid->src[0]->data; + args.eps_l2 = eps_q; + args.state_cache = (const float *) get_rows->src[0]->data; + args.state_ids = (const int32_t *) get_rows->src[1]->data; + args.state_row_stride = get_rows->src[0]->nb[1] / sizeof(float); + args.state_out = state_cpy.data; + args.scale = 1.0f / sqrtf((float) S); + args.norm_w = (const float *) mul_w->src[1]->data; + memcpy(&args.eps_rms, rms->op_params, sizeof(float)); + args.out = (float *) mul_w->data; + args.attn_out = nullptr; + args.S = S; + args.H_k = H_k; + args.H_v = H_v; + args.d_conv = d_conv; + + return last - node_idx; +} + static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, @@ -3313,6 +3534,16 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } + // Qwen3.5 Gated DeltaNet decode step: conv + norms + gate + recurrence + state writes + gated rms norm + if (node->op == GGML_OP_CONCAT && GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + ggml_cuda_gdn_decode_args args; + const int nodes_to_skip = ggml_cuda_try_gdn_decode_fusion(cgraph, i, args); + if (nodes_to_skip > 0) { + ggml_cuda_op_gdn_decode_fused(*cuda_ctx, args); + return nodes_to_skip; + } + } + // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache if (node->op == GGML_OP_GATED_DELTA_NET) { ggml_cuda_gated_delta_net_fused_cache fused_state_cpy; From 3a10a0f1cab494a4dfc19813b66be66a3ca7137b Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:22:38 +0200 Subject: [PATCH 15/30] hip: write the fused expert sum in place for single-block decode ggml_cuda_op_weighted_expert_sum always staged its result in a pool buffer and copied it into the destination, because the graph allocator places the output over the expert or routing-weight buffers that die inside the fused range (in Qwen3.6 decode the eight routing weights sit inside the output). That copy is one more launch per layer in a decode step that is dominated by launch gaps. When the whole sum fits in one block (n_embd/4 * n_tokens <= 1024, i.e. decode), launch a single block and place a barrier between the reads and the store, so the output can be written in place whatever it aliases. The per-thread arithmetic is unchanged; larger grids keep the staging copy. Greedy 120-token generations with Qwen3.6-35B-A3B are identical. Qwen3.6-35B-A3B UD-Q8_K_XL TG128 on gfx1151: 52.69 -> 52.91 t/s. Assisted-by: OpenCode --- ggml/src/ggml-cuda/binbcast.cu | 52 +++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu index d3486588ed03..30fa5fde96fb 100644 --- a/ggml/src/ggml-cuda/binbcast.cu +++ b/ggml/src/ggml-cuda/binbcast.cu @@ -607,7 +607,7 @@ static __device__ __forceinline__ float rounded_add_f32(const float a, const flo } #endif -template +template static __global__ void weighted_expert_sum_f32( const float * experts, const float * weights, float * dst, int64_t n_embd, int64_t n_tokens, int64_t experts_s1, int64_t experts_s2, @@ -638,6 +638,10 @@ static __global__ void weighted_expert_sum_f32( sum.z = rounded_add_f32(sum.z, rounded_mul_f32(expert.z, weight_i)); sum.w = rounded_add_f32(sum.w, rounded_mul_f32(expert.w, weight_i)); } + if constexpr (sync_before_store) { + // single block: all expert and weight reads of the block are complete, so dst may alias them + __syncthreads(); + } reinterpret_cast(dst)[index] = sum; } @@ -646,18 +650,44 @@ void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tens const ggml_tensor * weights = experts == mul->src[0] ? mul->src[1] : mul->src[0]; const int64_t n_embd = experts->ne[0]; const int64_t n_tokens = experts->ne[2]; - const int threads = 256; GGML_ASSERT(n_embd % 4 == 0); - const int blocks = (n_embd / 4 * n_tokens + threads - 1) / threads; + const int64_t n_items = n_embd / 4 * n_tokens; + + // The graph allocator may place the output over the expert or weight buffers (both die inside the fused + // range). A single block can read everything before a barrier and then write in place; larger grids + // stage the sum until all reads have finished. + auto overlaps = [](const ggml_tensor * a, const ggml_tensor * b) { + const char * a0 = (const char *) a->data; + const char * a1 = a0 + ggml_nbytes(a); + const char * b0 = (const char *) b->data; + const char * b1 = b0 + ggml_nbytes(b); + return a0 < b1 && b0 < a1; + }; + const bool aliased = overlaps(dst, experts) || overlaps(dst, weights); + const bool single_block = n_items <= 1024; + const bool stage = aliased && !single_block; + + const int threads = single_block ? (int) n_items : 256; + const int blocks = (int) ((n_items + threads - 1) / threads); const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream()); - // MUL output may alias experts, so stage until all expert reads finish. - ggml_cuda_pool_alloc result(ctx.pool(), n_embd * n_tokens); + + ggml_cuda_pool_alloc result(ctx.pool()); + float * out = stage ? result.alloc(n_embd * n_tokens) : (float *) dst->data; #define launch_weighted_expert_sum(n) \ - ggml_cuda_kernel_launch(weighted_expert_sum_f32, launch_params, \ - (const float *) experts->data, (const float *) weights->data, result.get(), \ - n_embd, n_tokens, experts->nb[1] / sizeof(float), experts->nb[2] / sizeof(float), \ - weights->nb[1] / sizeof(float), weights->nb[2] / sizeof(float)) + do { \ + if (single_block) { \ + ggml_cuda_kernel_launch(weighted_expert_sum_f32, launch_params, \ + (const float *) experts->data, (const float *) weights->data, out, \ + n_embd, n_tokens, experts->nb[1] / sizeof(float), experts->nb[2] / sizeof(float), \ + weights->nb[1] / sizeof(float), weights->nb[2] / sizeof(float)); \ + } else { \ + ggml_cuda_kernel_launch(weighted_expert_sum_f32, launch_params, \ + (const float *) experts->data, (const float *) weights->data, out, \ + n_embd, n_tokens, experts->nb[1] / sizeof(float), experts->nb[2] / sizeof(float), \ + weights->nb[1] / sizeof(float), weights->nb[2] / sizeof(float)); \ + } \ + } while (0) switch (n_expert_used) { case 2: launch_weighted_expert_sum(2); break; case 3: launch_weighted_expert_sum(3); break; @@ -670,7 +700,9 @@ void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tens } #undef launch_weighted_expert_sum - CUDA_CHECK(cudaMemcpyAsync(dst->data, result.get(), ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream())); + if (stage) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, result.get(), ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream())); + } } static __global__ void shared_mul_add_f32( From 3a67cce8f76d3bf567d6b66fc07ed21e7009c34f Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:49:33 +0200 Subject: [PATCH 16/30] hip: fuse the Qwen3.5 shared-expert gate for decode on AMD The shared expert output is gated by sigmoid(w . y) with a 2048-wide f32 weight vector: a one-row mul_mat_vec_f, a sigmoid and the already fused mul + add + add, i.e. three launches per layer in decode. Match the five nodes and run them in one single-block kernel: the dot product repeats the per-thread fma chain and two-level warp reduction of mul_mat_vec_f<256>, the sigmoid is the same expression, and the elementwise tail uses explicit v_mul_f32/v_add_f32 like shared_mul_add_f32 (plain __fmul_rn/__fadd_rn are contracted into an fma by the HIP compiler). One block reads the whole activation vector before the reduction barrier, so the output may alias it. Outputs are byte-identical to the unfused graph. Qwen3.6-35B-A3B UD-Q8_K_XL TG128 on gfx1151: 52.91 -> 53.42 t/s. Assisted-by: OpenCode --- ggml/src/ggml-cuda/binbcast.cu | 59 +++++++++++++++++++++++++++++++++ ggml/src/ggml-cuda/binbcast.cuh | 4 +++ ggml/src/ggml-cuda/ggml-cuda.cu | 35 +++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu index 30fa5fde96fb..e04868e5ccf6 100644 --- a/ggml/src/ggml-cuda/binbcast.cu +++ b/ggml/src/ggml-cuda/binbcast.cu @@ -731,6 +731,65 @@ void ggml_cuda_op_shared_mul_add( (const float *) residual->data, (float *) dst->data, dst->ne[0], nelements); } +// Qwen3.5 shared-expert gate for one token: gate = sigmoid(w . y) computed exactly like +// mul_mat_vec_f (same per-thread fma chain and the same two-level warp reduction), +// then dst = (other + src*gate) + residual as in shared_mul_add_f32. One block, so the activation vector is +// fully read before the reduction barrier and the output may alias it. +static __global__ void __launch_bounds__(256, 1) shared_gate_mul_add_f32( + const float * w, const float * y, const float * src, const float * other, const float * residual, float * dst, + const int ncols2, const int64_t n_embd) { + constexpr int block_size = 256; + constexpr int warp_size = 32; + __shared__ float buf_iw[warp_size]; + __shared__ float gate_s; + + const int tid = threadIdx.x; + if (tid < warp_size) { + buf_iw[tid] = 0.0f; + } + __syncthreads(); + + const float2 * x2 = (const float2 *) w; + const float2 * y2 = (const float2 *) y; + float sumf = 0.0f; + for (int col2 = tid; col2 < ncols2; col2 += block_size) { + const float2 tmpx = x2[col2]; + const float2 tmpy = y2[col2]; + ggml_cuda_mad(sumf, tmpx.x, tmpy.x); + ggml_cuda_mad(sumf, tmpx.y, tmpy.y); + } + sumf = warp_reduce_sum(sumf); + buf_iw[tid/warp_size] = sumf; + __syncthreads(); + if (tid < warp_size) { + sumf = buf_iw[tid]; + sumf = warp_reduce_sum(sumf); + if (tid == 0) { + gate_s = 1.0f / (1.0f + expf(-sumf)); + } + } + __syncthreads(); + + const float gate = gate_s; + for (int64_t i = tid; i < n_embd; i += block_size) { + const float product = rounded_mul_f32(src[i], gate); + const float value = rounded_add_f32(other[i], product); + dst[i] = rounded_add_f32(value, residual[i]); + } +} + +void ggml_cuda_op_shared_gate_mul_add( + ggml_backend_cuda_context & ctx, const ggml_tensor * gate_mm, const ggml_tensor * src, const ggml_tensor * other, + const ggml_tensor * residual, ggml_tensor * dst) { + const ggml_tensor * w = gate_mm->src[0]; + const ggml_tensor * y = gate_mm->src[1]; + GGML_ASSERT(w->ne[0] % 2 == 0); + const ggml_cuda_kernel_launch_params launch_params(1, 256, 0, ctx.stream()); + ggml_cuda_kernel_launch(shared_gate_mul_add_f32, launch_params, + (const float *) w->data, (const float *) y->data, (const float *) src->data, (const float *) other->data, + (const float *) residual->data, (float *) dst->data, (int) (w->ne[0] / 2), dst->ne[0]); +} + void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; diff --git a/ggml/src/ggml-cuda/binbcast.cuh b/ggml/src/ggml-cuda/binbcast.cuh index 916c5ac34931..bb3a8e13f241 100644 --- a/ggml/src/ggml-cuda/binbcast.cuh +++ b/ggml/src/ggml-cuda/binbcast.cuh @@ -12,3 +12,7 @@ void ggml_cuda_op_fused_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst, void ggml_cuda_op_fused_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst, int n_fuse); void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tensor * mul, ggml_tensor * dst, int n_expert_used); void ggml_cuda_op_shared_mul_add(ggml_backend_cuda_context & ctx, ggml_tensor * mul, const ggml_tensor * other, const ggml_tensor * residual, ggml_tensor * dst); + +void ggml_cuda_op_shared_gate_mul_add( + ggml_backend_cuda_context & ctx, const ggml_tensor * gate_mm, const ggml_tensor * src, const ggml_tensor * other, + const ggml_tensor * residual, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index aa35efbe25ec..50c343a1c0d4 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3720,6 +3720,41 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } + // Qwen3.5 shared expert gate for one token: mul_mat(w [2048, 1], y) -> sigmoid -> mul -> add -> add + if (node->op == GGML_OP_MUL_MAT && i + 4 < cgraph->n_nodes && node->type == GGML_TYPE_F32 && + node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32 && + node->src[0]->ne[0] == 2048 && ggml_nelements(node->src[0]) == 2048 && ggml_nelements(node->src[1]) == 2048 && + ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(node->src[1]) && ggml_nelements(node) == 1 && + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_ADD }, { i + 4 }) && + ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID && cgraph->nodes[i + 1]->src[0] == node && + GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + const ggml_tensor * sigmoid = cgraph->nodes[i + 1]; + ggml_tensor * mul = cgraph->nodes[i + 2]; + ggml_tensor * add0 = cgraph->nodes[i + 3]; + ggml_tensor * add1 = cgraph->nodes[i + 4]; + const ggml_tensor * gate = mul->src[0] == sigmoid ? mul->src[0] : mul->src[1]; + const ggml_tensor * src = gate == mul->src[0] ? mul->src[1] : mul->src[0]; + const ggml_tensor * other = add0->src[0]; + const ggml_tensor * residual = add1->src[1]; + const uintptr_t dst_begin = (uintptr_t) add1->data; + const uintptr_t dst_end = dst_begin + ggml_backend_buft_get_alloc_size(add1->buffer->buft, add1); + const auto safe_output_alias = [&](const ggml_tensor * input) { + const uintptr_t begin = (uintptr_t) input->data; + const uintptr_t end = begin + ggml_backend_buft_get_alloc_size(input->buffer->buft, input); + return !(begin < dst_end && dst_begin < end) || (input->data == add1->data && ggml_are_same_layout(input, add1)); + }; + if (gate == sigmoid && ggml_nelements(sigmoid) == 1 && add0->src[1] == mul && add1->src[0] == add0 && + other != mul && residual != mul && residual != add0 && + mul->type == GGML_TYPE_F32 && src->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && residual->type == GGML_TYPE_F32 && + mul->ne[0] == 2048 && ggml_nelements(mul) == 2048 && ggml_are_same_shape(mul, src) && ggml_are_same_shape(mul, other) && + ggml_are_same_shape(mul, residual) && ggml_are_same_shape(mul, add1) && + ggml_is_contiguous(src) && ggml_is_contiguous(other) && ggml_is_contiguous(residual) && ggml_is_contiguous(add1) && + safe_output_alias(src) && safe_output_alias(other) && safe_output_alias(residual)) { + ggml_cuda_op_shared_gate_mul_add(*cuda_ctx, node, src, other, residual, add1); + return 4; + } + } + if (node->op == GGML_OP_MUL && node->ne[0] == 2048 && i + 2 < cgraph->n_nodes && ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_ADD, GGML_OP_ADD }, { i + 2 })) { ggml_tensor * add0 = cgraph->nodes[i + 1]; From f8524a5033018e93a8aec856a93ba8c054b468b9 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:05:10 +0200 Subject: [PATCH 17/30] hip: pipeline the single-column f32 matvec K loop mul_mat_vec_f loops over K with a runtime trip count, so the compiler keeps one 8-byte load per lane in flight and waits for it every iteration. The f32 MoE router (2048 x 256) of Qwen3.6 ran at 131 GB/s on gfx1151 because of this. For the single-column, unfused f32 path on HIP, load four K iterations of weights and activations before consuming them, in the same accumulation order; the remainder loop is unchanged. Outputs are byte-identical. gfx1151 Qwen3.6-35B-A3B decode: router matvec 16.0 -> 10.5 us, alpha/beta projections 3.2 -> 2.6 us; UD-Q8_K_XL TG128 53.42 -> 54.16 t/s. Assisted-by: OpenCode --- ggml/src/ggml-cuda/mmvf.cu | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu index 4eb457db7a6b..e245ab4f2731 100644 --- a/ggml/src/ggml-cuda/mmvf.cu +++ b/ggml/src/ggml-cuda/mmvf.cu @@ -135,7 +135,35 @@ static __global__ void mul_mat_vec_f( } } - for (int col2 = tid; col2 < ncols2; col2 += block_size) { + int col2 = tid; +#if defined(GGML_USE_HIP) + // Issue the weight loads of four K iterations before consuming them: the runtime trip count keeps the + // compiler from pipelining the loop, and with one 8-byte load in flight per lane the single-column + // f32 matvecs (e.g. MoE routers) reach barely half the memory bandwidth. The accumulation order is + // unchanged, so the results are bit-identical. + if constexpr (ncols_dst == 1 && !has_fusion) { + constexpr int pf = 4; + for (; col2 + (pf - 1)*block_size < ncols2; col2 += pf*block_size) { + float2 tmpx[pf]; + float2 tmpy[pf]; +#pragma unroll + for (int i = 0; i < pf; ++i) { + tmpx[i] = x2[col2 + i*block_size]; + } +#pragma unroll + for (int i = 0; i < pf; ++i) { + tmpy[i] = y2[col2 + i*block_size]; + } +#pragma unroll + for (int i = 0; i < pf; ++i) { + ggml_cuda_mad(sumf[0], tmpx[i].x, tmpy[i].x); + ggml_cuda_mad(sumf[0], tmpx[i].y, tmpy[i].y); + } + } + } +#endif // defined(GGML_USE_HIP) + + for (; col2 < ncols2; col2 += block_size) { const float2 tmpx = x2[col2]; float2 tmpx_gate = make_float2(0.0f, 0.0f); if constexpr (has_fusion) { From 5437ff81214efbafab89f50459f9a3d30efb4436 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:17:33 +0200 Subject: [PATCH 18/30] hip: one wave per row for short bf16 decode matvecs on RDNA mul_mat_vec_f picks a 256-thread block for K=512, so every wave issues a single 4-byte bf16 load and then reduces; on gfx1151 the bf16 MoE down projections of Unsloth UD quants (8 x 2048 rows of K=512) run at ~145 GB/s. Add a wave-per-row kernel that emulates the 256-thread block: lane l accumulates the eight virtual warps' lane l in the same fma order, reduces each with the same butterfly and combines them exactly like block_reduce over lanes 0..7 (including the zero-lane adds), with eight loads in flight per lane. Used on RDNA for unfused single-column bf16 with K <= 512; for longer rows the lower wave count loses, so the standard kernel stays. Outputs are byte-identical (dense and MUL_MAT_ID, odd shapes included). gfx1151 Qwen3.6-35B-A3B UD-Q8_K_XL: bf16 down projection 93 -> 67 us, TG128 54.16 -> 54.54 t/s. Assisted-by: OpenCode --- ggml/src/ggml-cuda/mmvf.cu | 91 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu index e245ab4f2731..e06920980c8d 100644 --- a/ggml/src/ggml-cuda/mmvf.cu +++ b/ggml/src/ggml-cuda/mmvf.cu @@ -411,6 +411,79 @@ static __global__ void mul_mat_vec_f( } } + +#if defined(GGML_USE_HIP) +// One wave per output row emulating mul_mat_vec_f: lane l accumulates the partial +// sums of the eight virtual warps' lane l (columns 2*(32*w + l) + 256*i) in the same fma order, reduces each +// with the same butterfly and combines the eight partials exactly like block_reduce does over lanes 0..7 (the +// other lanes hold zero). The standard kernel launches 8 waves per row that each issue a single 4-byte load, +// which on gfx1151 leaves K=512 MoE down projections at ~145 GB/s; here each lane keeps 8 loads in flight +// (measured: 93 -> 67 us for 8 x 2048 rows of K=512; slower than the standard kernel for K=2048). +template +__launch_bounds__(32 * rows_per_block, 1) +static __global__ void mul_mat_vec_bf16_wave( + const nv_bfloat16 * x_ptr, const float * y_ptr, const int32_t * ids_ptr, float * dst_ptr, + const int ncols2, const int nrows, const uint3 nchannels_y, const int stride_row, const int stride_col_dst, + const uint3 channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, + const uint3 sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) { + constexpr int warp_size = 32; + constexpr int nvw = 256 / warp_size; // virtual warps of the emulated 256-thread block + const int lane = threadIdx.x; + const int row = blockIdx.x * rows_per_block + threadIdx.y; + if (row >= nrows) { + return; + } + const int channel_dst = blockIdx.y; + const int channel_x = ids_ptr ? ids_ptr[channel_dst] : (int) fastdiv((uint32_t) channel_dst, channel_ratio); + const int channel_y = ids_ptr ? (int) fastmodulo(channel_dst, nchannels_y) : channel_dst; + const int sample_dst = 0; + const int sample_x = fastdiv((uint32_t) sample_dst, sample_ratio); + const int sample_y = sample_dst; + + const int * x2 = (const int *) (x_ptr + int64_t(sample_x)*stride_sample_x + channel_x*stride_channel_x + int64_t(row)*stride_row); + const float2 * y2 = (const float2 *) (y_ptr + int64_t(sample_y)*stride_sample_y + channel_y*stride_channel_y); + float * dst = dst_ptr + int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst; + + float sumf[nvw]; +#pragma unroll + for (int w = 0; w < nvw; ++w) { + sumf[w] = 0.0f; + } + for (int col0 = 0; col0 < ncols2; col0 += 256) { + int tmpx[nvw]; + float2 tmpy[nvw]; +#pragma unroll + for (int w = 0; w < nvw; ++w) { + const int col2 = col0 + w*warp_size + lane; + if (col2 < ncols2) { + tmpx[w] = x2[col2]; + tmpy[w] = y2[col2]; + } + } +#pragma unroll + for (int w = 0; w < nvw; ++w) { + const int col2 = col0 + w*warp_size + lane; + if (col2 < ncols2) { + const float tmpx0 = ggml_cuda_cast(reinterpret_cast(&tmpx[w])[0]); + const float tmpx1 = ggml_cuda_cast(reinterpret_cast(&tmpx[w])[1]); + ggml_cuda_mad(sumf[w], tmpx0, tmpy[w].x); + ggml_cuda_mad(sumf[w], tmpx1, tmpy[w].y); + } + } + } +#pragma unroll + for (int w = 0; w < nvw; ++w) { + // per-warp butterfly, then the two zero-lane steps (offsets 16 and 8) of the second-level reduction + sumf[w] = (warp_reduce_sum(sumf[w]) + 0.0f) + 0.0f; + } + const float value = ((sumf[0] + sumf[4]) + (sumf[2] + sumf[6])) + ((sumf[1] + sumf[5]) + (sumf[3] + sumf[7])); + if (lane == 0) { + dst[row] = value; + } + GGML_UNUSED(stride_col_dst); +} +#endif // defined(GGML_USE_HIP) + template static void mul_mat_vec_f_switch_fusion( const T * x, const float * y, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, @@ -478,6 +551,24 @@ void launch_mul_mat_vec_f_cuda( } const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; +#if defined(GGML_USE_HIP) + if constexpr (std::is_same_v && ncols_dst == 1 && !is_multi_token_id && !tokens_in_block) { + // only where the emulated block does a single K iteration: for longer rows the 8x lower wave count loses + if (!has_fusion && block_size_best == 256 && ncols <= 512 && warp_size == 32 && nsamples_or_ntokens == 1 && nsamples_dst == 1 && + GGML_CUDA_CC_IS_RDNA(ggml_cuda_info().devices[device].cc)) { + constexpr int rows_per_block = 4; + const dim3 block_nums((nrows + rows_per_block - 1) / rows_per_block, nchannels_dst, 1); + const dim3 block_dims(warp_size, rows_per_block, 1); + const ggml_cuda_kernel_launch_params launch_params = {block_nums, block_dims, 0, stream}; + ggml_cuda_kernel_launch(mul_mat_vec_bf16_wave, launch_params, + x, y, ids, dst, (int) (ncols/2), (int) nrows, nchannels_y_fd, (int) stride_row, (int) stride_col_dst, + channel_ratio_fd, (int) stride_channel_x, (int) stride_channel_y, (int) stride_channel_dst, + sample_ratio_fd, (int) stride_sample_x, (int) stride_sample_y, (int) stride_sample_dst); + return; + } + } +#endif // defined(GGML_USE_HIP) + const int nbytes_shared = (warp_size*sizeof(float) + (has_fusion ? warp_size*sizeof(float) : 0)) * (tokens_in_block ? nsamples_or_ntokens : 1); const dim3 block_nums(nrows, nchannels_dst, tokens_in_block ? nsamples_dst : nsamples_or_ntokens); const dim3 block_dims(block_size_best, tokens_in_block ? nsamples_or_ntokens : 1, 1); From 37f02eb14ec5d58461596fb9f0ce6314cf6fb2d4 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:31:41 +0200 Subject: [PATCH 19/30] hip: fuse the activation quantization for Q6_K decode matvecs too Extend mul_mat_vec_q_fq beyond Q8_0 with a generic path that runs the per-wave vec_dot loop of mul_mat_vec_q against the shared-memory activations, and replicate the Q8_1 block sum of quantize_q8_1 in the exact butterfly order for types that read it. Q6_K (scale only) is enabled: it is byte-identical (dense and MUL_MAT_ID) and measured +0.8% on Qwen3.6-35B-A3B UD-Q6_K_XL TG128 (56.64 -> 57.09 t/s). Q4_K/Q5_K are exact as well but neutral without weight prefetch, so they stay on the two-kernel path. Assisted-by: OpenCode --- ggml/src/ggml-cuda/mmvq.cu | 45 +++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 5ef2696b8140..dc00b2f7e305 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -1174,6 +1174,9 @@ static __global__ void mul_mat_vec_q( static constexpr __host__ __device__ bool mmvq_fq_type_ok(ggml_type type) { switch (type) { case GGML_TYPE_Q8_0: + case GGML_TYPE_Q6_K: + // Q4_K/Q5_K (block sum via mmvq_fq_block_sum) are exact too but measured neutral on gfx1151 + // with the generic, non-prefetched loop (Qwen3.6 UD-Q4_K_XL TG128 60.69 -> 60.63 t/s). return true; default: return false; @@ -1184,6 +1187,7 @@ static constexpr __host__ __device__ bool mmvq_fq_type_ok(ggml_type type) { static constexpr __host__ __device__ bool mmvq_fq_needs_sum(ggml_type type) { switch (type) { case GGML_TYPE_Q8_0: + case GGML_TYPE_Q6_K: return false; default: return true; @@ -1267,7 +1271,8 @@ static __global__ void mul_mat_vec_q_fq( const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { - static_assert(type == GGML_TYPE_Q8_0, "fused activation quantization is only implemented for Q8_0"); + static_assert(mmvq_fq_type_ok(type)); + constexpr bool q8_0_path = type == GGML_TYPE_Q8_0; const void * GGML_CUDA_RESTRICT vx = vx_ptr; const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; float * GGML_CUDA_RESTRICT dst = dst_ptr; @@ -1277,7 +1282,8 @@ static __global__ void mul_mat_vec_q_fq( constexpr int vdr = get_vdr_mmvq(type); constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int blocks_per_iter = vdr * warp_size / qi; - static_assert(qk == QK8_1); + static_assert(!q8_0_path || qk == QK8_1); + [[maybe_unused]] constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); extern __shared__ char mmvq_fq_smem[]; block_q8_1 * y_q8 = (block_q8_1 *) mmvq_fq_smem; @@ -1322,16 +1328,18 @@ static __global__ void mul_mat_vec_q_fq( const int kqs = vdr * (lane % (qi/vdr)); const int kbx0 = lane / (qi/vdr); - const block_q8_0 * x = (const block_q8_0 *) vx + kbx_offset; + [[maybe_unused]] const block_q8_0 * x = (const block_q8_0 *) vx + kbx_offset; [[maybe_unused]] const block_q8_0 * xg = nullptr; - mmvq_fq_q8_0_chunk cx; + [[maybe_unused]] mmvq_fq_q8_0_chunk cx; [[maybe_unused]] mmvq_fq_q8_0_chunk cg; - mmvq_fq_q8_0_load(cx, x, kbx0, kqs, blocks_per_row_x); - if constexpr (has_fusion) { - if (use_gate) { - xg = (const block_q8_0 *) vgate + kbx_offset; - mmvq_fq_q8_0_load(cg, xg, kbx0, kqs, blocks_per_row_x); + if constexpr (q8_0_path) { + mmvq_fq_q8_0_load(cx, x, kbx0, kqs, blocks_per_row_x); + if constexpr (has_fusion) { + if (use_gate) { + xg = (const block_q8_0 *) vgate + kbx_offset; + mmvq_fq_q8_0_load(cg, xg, kbx0, kqs, blocks_per_row_x); + } } } @@ -1397,8 +1405,20 @@ static __global__ void mul_mat_vec_q_fq( float tmp_gate = 0.0f; const block_q8_1 * y = y_q8; + if constexpr (!q8_0_path) { + // generic types: the per-wave loop of mul_mat_vec_q against the shared-memory activations + for (int kbx = kbx0; kbx < blocks_per_row_x; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); + tmp += vec_dot_q_cuda(vx, &y[kby], kbx_offset + kbx, kqs); + if constexpr (has_fusion) { + if (use_gate) { + tmp_gate += vec_dot_q_cuda(vgate, &y[kby], kbx_offset + kbx, kqs); + } + } + } + } constexpr int chunk_blocks = MMVQ_FQ_PF*blocks_per_iter; - for (int c0 = kbx0; c0 < blocks_per_row_x; c0 += chunk_blocks) { + for (int c0 = kbx0; q8_0_path && c0 < blocks_per_row_x; c0 += chunk_blocks) { mmvq_fq_q8_0_chunk nx; [[maybe_unused]] mmvq_fq_q8_0_chunk ng; const int c1 = c0 + chunk_blocks; @@ -1514,6 +1534,11 @@ static void mul_mat_vec_q_fq_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream); break; + case GGML_TYPE_Q6_K: + mul_mat_vec_q_fq_launch(vx, y, ids, fusion, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream); + break; default: GGML_ABORT("fatal error"); break; From a01b5d731752ee511794f3058ffc95d837ac984c Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:48:31 +0200 Subject: [PATCH 20/30] hip: narrow routed-compact MMQ tiles for 256-expert MoE prefill on RDNA3.5 For mul_mat_id the MMQ tile width J is chosen from ncols_max, which is the total number of token-expert pairs, so the generic path always picks J=128. With 256 experts and 8 active the per-expert row count is only n_tokens/32 (16/32/64 rows for ubatch 512/1024/2048), so 50-87% of every tile was padding. The RDNA3.5 specializations that fixed this were gated on exact shapes and on 32/64 rows per expert, i.e. only ubatch 1024/2048, and only Q8_0/Q6_K used the routed-compact kernel. Replace the shape-specific paths with a per-type rule on rows per expert, and route all of them through mul_mat_q_routed_compact: Q8_0: 16/32/64 rows -> J=48 Q4_K/Q5_K/Q6_K: 16 rows -> J=32, 32 rows -> J=48, 64 rows -> J=32 Values measured on gfx1151 for the 2048x512 and 3072x1024 expert shapes; the previous choices (Q8_0 J48, Q6_K J32 at 64 rows) are unchanged, Q4_K at 64 rows moves from tiled J48 to compact J32 (-7% up-proj, -31% down-proj), Q5_K at 64 rows from tiled J32 to compact J32 (-31% down-proj). MUL_MAT_ID kernel time, 256 experts, 512x2048 / 2048x512, n=512 tokens: Q8_0 2347/2490 us -> 1517/1518 us Q6_K 3918/3931 us -> 1344/1300 us Q5_K 2606/2607 us -> 1087/1084 us Q4_K 2561/2538 us -> 1026/ 991 us n=1024: Q6_K -54%, Q5_K -50%, Q4_K -54% (Q8_0 unchanged). Qwen3.6-35B-A3B on gfx1151, llama-bench -b 2048 -ub 2048: UD-Q8_K_XL PP512 1437 -> 1722 t/s (+19.8%); PP1024/2048/4096 unchanged UD-Q4_K_XL PP512 1439 -> 2086 t/s (+44.9%), PP1024 +25.5%, PP2048 +3.5% UD-Q8_K_XL default ubatch 512: PP512 +22.1%, PP2048 +22.4% TG unchanged. test-backend-ops MUL_MAT_ID: 906/906 passed (adds 16/32 rows-per-expert eval cases and qwen3.6 perf cases). --- ggml/src/ggml-cuda/mmq-q45.cu | 4 --- ggml/src/ggml-cuda/mmq-q8.cu | 9 ----- ggml/src/ggml-cuda/mmq.cu | 52 ++------------------------- ggml/src/ggml-cuda/mmq.cuh | 66 +++++++++++++++++++++++------------ tests/test-backend-ops.cpp | 17 +++++++++ 5 files changed, 64 insertions(+), 84 deletions(-) delete mode 100644 ggml/src/ggml-cuda/mmq-q8.cu diff --git a/ggml/src/ggml-cuda/mmq-q45.cu b/ggml/src/ggml-cuda/mmq-q45.cu index 2b94d38ad70f..e9774527cc61 100644 --- a/ggml/src/ggml-cuda/mmq-q45.cu +++ b/ggml/src/ggml-cuda/mmq-q45.cu @@ -4,10 +4,6 @@ #undef launch_mul_mat_q #undef mul_mat_q -void launch_mul_mat_q_q4_k_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { - launch_mul_mat_q_q45(ctx, args, stream); -} - void launch_mul_mat_q_q5_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { launch_mul_mat_q_q45(ctx, args, stream); } diff --git a/ggml/src/ggml-cuda/mmq-q8.cu b/ggml/src/ggml-cuda/mmq-q8.cu deleted file mode 100644 index 4cb9589c715c..000000000000 --- a/ggml/src/ggml-cuda/mmq-q8.cu +++ /dev/null @@ -1,9 +0,0 @@ -#define mul_mat_q mul_mat_q_q8 -#define launch_mul_mat_q launch_mul_mat_q_q8 -#include "mmq.cuh" -#undef launch_mul_mat_q -#undef mul_mat_q - -void launch_mul_mat_q_q8_0_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { - launch_mul_mat_q_q8(ctx, args, stream); -} diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index de18e2064877..592f36591af0 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -5,26 +5,8 @@ #include -void launch_mul_mat_q_q4_k_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); void launch_mul_mat_q_q5_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); void launch_mul_mat_q_q6_k_j32(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); -void launch_mul_mat_q_q8_0_j48(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream); - -static bool use_rdna3_5_q45_id_narrow(const mmq_args & args) { - const int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - if ((args.type_x != GGML_TYPE_Q4_K && args.type_x != GGML_TYPE_Q5_K) || !GGML_CUDA_CC_IS_RDNA3_5(cc) || - args.ids_dst == nullptr || args.nchannels_y <= 0) { - return false; - } - - const int64_t rows_per_channel = (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y; - const bool ornith = ((args.ncols_x == 2048 && args.nrows_x == 512) || - (args.ncols_x == 512 && args.nrows_x == 2048)) && args.nchannels_x == 256 && rows_per_channel == 64; - const bool qwen122 = ((args.ncols_x == 3072 && args.nrows_x == 1024) || - (args.ncols_x == 1024 && args.nrows_x == 3072)) && args.nchannels_x == 256 && rows_per_channel == 64; - return ornith || qwen122; -} static bool use_rdna3_5_q5_ling(const mmq_args & args) { const int id = ggml_cuda_get_device(); @@ -43,24 +25,6 @@ static bool use_rdna3_5_q6_ling_j32(const mmq_args & args) { (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 32; } -static bool use_rdna3_5_q6_id_narrow(const mmq_args & args) { - const int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - return args.type_x == GGML_TYPE_Q6_K && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr && - ((args.ncols_x == 2048 && args.nrows_x == 512) || (args.ncols_x == 512 && args.nrows_x == 2048)) && - args.nchannels_x == 256 && args.nchannels_y > 0 && - (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 64; -} - -static bool use_rdna3_5_q8_id_narrow(const mmq_args & args) { - const int id = ggml_cuda_get_device(); - const int cc = ggml_cuda_info().devices[id].cc; - return args.type_x == GGML_TYPE_Q8_0 && GGML_CUDA_CC_IS_RDNA3_5(cc) && args.ids_dst != nullptr && - ((args.ncols_x == 2048 && args.nrows_x == 512) || (args.ncols_x == 512 && args.nrows_x == 2048)) && - args.nchannels_x == 256 && args.nchannels_y > 0 && - (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y == 64; -} - static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { switch (args.type_x) { case GGML_TYPE_Q1_0: @@ -82,11 +46,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q8_0: - if (use_rdna3_5_q8_id_narrow(args)) { - launch_mul_mat_q_q8_0_j48(ctx, args, stream); - } else { - mul_mat_q_case(ctx, args, stream); - } + mul_mat_q_case(ctx, args, stream); break; // ----------------------------------------------------------------------- case GGML_TYPE_Q2_K: @@ -96,23 +56,17 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q4_K: - if (use_rdna3_5_q45_id_narrow(args)) { - launch_mul_mat_q_q4_k_j48(ctx, args, stream); - } else { - mul_mat_q_case(ctx, args, stream); - } + mul_mat_q_case(ctx, args, stream); break; case GGML_TYPE_Q5_K: if (use_rdna3_5_q5_ling(args)) { launch_mul_mat_q_q5_k_j32(ctx, args, stream); - } else if (use_rdna3_5_q45_id_narrow(args)) { - launch_mul_mat_q_q5_k_j32(ctx, args, stream); } else { mul_mat_q_case(ctx, args, stream); } break; case GGML_TYPE_Q6_K: - if (use_rdna3_5_q6_ling_j32(args) || use_rdna3_5_q6_id_narrow(args)) { + if (use_rdna3_5_q6_ling_j32(args)) { launch_mul_mat_q_q6_k_j32(ctx, args, stream); } else { mul_mat_q_case(ctx, args, stream); diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index a9fbe95e28bc..f7ac408b22fe 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1584,6 +1584,40 @@ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const i return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int)); } +// RDNA3.5 tile selection for MoE (mul_mat_id) with 256 experts. +// The per-expert row count is n_tokens*n_expert_used/256 (16/32/64 for ubatch 512/1024/2048 with 8 active experts), +// but the generic selection below only sees ncols_max == n_tokens*n_expert_used and picks J=128, so most of every +// tile is padding. Use a narrow J and the routed-compact kernel, which only enumerates real per-expert tiles. +// J per type/row count was measured on gfx1151 for the 2048x512 and 3072x1024 expert shapes. +static constexpr int mmq_rdna3_5_id_get_J(const ggml_type type, const int64_t rows_per_expert) { + switch (type) { + case GGML_TYPE_Q8_0: + return rows_per_expert == 16 || rows_per_expert == 32 || rows_per_expert == 64 ? 48 : 0; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return rows_per_expert == 16 || rows_per_expert == 64 ? 32 : rows_per_expert == 32 ? 48 : 0; + default: + return 0; + } +} + +static constexpr bool mmq_rdna3_5_id_use_compact(const ggml_type type, const int J) { + return (type == GGML_TYPE_Q8_0 && J == 48) || + ((type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K) && (J == 32 || J == 48)); +} + +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 16) == 48); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q6_K, 16) == 32); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_K, 32) == 48); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q5_K, 64) == 32); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 48) == 0); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_0, 16) == 0); +static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 48)); +static_assert(!mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 32)); +static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q6_K, 32)); +static_assert(!mmq_rdna3_5_id_use_compact(GGML_TYPE_IQ2_XS, 48)); + template static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { const int id = ggml_cuda_get_device(); @@ -1619,8 +1653,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); const bool use_compact_routed = args.ids_dst != nullptr && args.nchannels_y == 256 && GGML_CUDA_CC_IS_RDNA3_5(cc) && - !ggml_cuda_mmq_get_stream_k(type, J, fallback, cc) && - ((type == GGML_TYPE_Q6_K && J == 32) || (type == GGML_TYPE_Q8_0 && J == 48)); + !ggml_cuda_mmq_get_stream_k(type, J, fallback, cc) && mmq_rdna3_5_id_use_compact(type, J); if (use_compact_routed) { const int max_descriptors = (args.ncols_dst + J - 1) / J + args.nchannels_y; ggml_cuda_pool_alloc descriptors(ctx.pool(id), max_descriptors); @@ -1684,20 +1717,6 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a ntx_fd); } -static constexpr bool mmq_use_rdna3_5_q8_id_j48( - ggml_type type, int cc, bool fallback, bool has_ids, int64_t ncols_dst, int64_t nchannels_y) { - return type == GGML_TYPE_Q8_0 && GGML_CUDA_CC_IS_RDNA3_5(cc) && !fallback && has_ids && nchannels_y > 0 && - (ncols_dst + nchannels_y - 1) / nchannels_y == 32; -} - -static_assert(mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, true, 8192, 256)); -static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3, false, true, 8192, 256)); -static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q4_0, GGML_CUDA_CC_RDNA3_5, false, true, 8192, 256)); -static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, true, true, 8192, 256)); -static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, false, 8192, 256)); -static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, true, 7936, 256)); -static_assert(!mmq_use_rdna3_5_q8_id_j48(GGML_TYPE_Q8_0, GGML_CUDA_CC_RDNA3_5, false, true, 8193, 256)); - static constexpr bool mmq_use_rdna3_5_iq_id_j48( ggml_type type, int cc, bool fallback, bool has_ids, int64_t ncols_dst, int64_t nchannels_y) { return (type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ3_XXS) && GGML_CUDA_CC_IS_RDNA3_5(cc) && @@ -1723,12 +1742,15 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, J_best = J_rdna3_5; ntiles_J_best = 1; } - } else if (mmq_use_rdna3_5_q8_id_j48(type, cc, fallback, args.ids_dst != nullptr, args.ncols_dst, args.nchannels_y)) { - constexpr int J_rdna3_5 = 48; - const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J_rdna3_5, fallback, cc); - if (config.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(config, cc) <= smpbo) { - J_best = J_rdna3_5; - ntiles_J_best = 1; + } else if (GGML_CUDA_CC_IS_RDNA3_5(cc) && !fallback && args.ids_dst != nullptr && args.nchannels_y == 256) { + const int64_t rows_per_expert = (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y; + const int J_rdna3_5 = mmq_rdna3_5_id_get_J(type, rows_per_expert); + if (J_rdna3_5 != 0) { + const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J_rdna3_5, fallback, cc); + if (config.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(config, cc) <= smpbo) { + J_best = J_rdna3_5; + ntiles_J_best = 1; + } } } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e90fad598e38..b5130fb7708d 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9484,6 +9484,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); #endif + // 256-expert MoE with 16/32 rows per expert (ubatch 512/1024), exercises the narrow RDNA3.5 mul_mat_id tiles + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048)); @@ -10714,6 +10723,14 @@ static std::vector> make_test_cases_perf() { } } + // qwen3.5/3.6-35b-a3b (256 experts, 8 used): up/gate and down projections at ubatch 512/1024/2048 + for (int bs : {512, 1024, 2048}) { + for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 512, bs, 2048)); + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 256, 8, false, 2048, bs, 512)); + } + } + // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) { From d7550c14bf36e2b6a48265c1a07d399ce07fce9e Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:24:25 +0200 Subject: [PATCH 21/30] hip: select RDNA3.5 mul_mat_id tile width by rows-per-expert range The previous rule only matched exactly 16/32/64 rows per expert (ubatch 512/1024/2048 with 8 active experts); any other ubatch size, and the remainder ubatch of every prompt, still fell back to the generic J=128 tiled grid, which launches ncols_max/J*n_experts mostly-empty blocks. Use ranges instead, and allow the routed-compact kernel for every selected J: Q8_0: rows <= 12 -> J16, <= 64 -> J48, else J128 Q4_K/Q5_K/Q6_K: rows <= 12 -> J16, <= 24 -> J32, <= 48 -> J48, <= 64 -> J32, else J128 The 16/32/64 choices are unchanged from the previous commit. Ranges were picked from a J sweep over 64..4096 tokens on gfx1151 and validated end-to-end. Qwen3.6-35B-A3B on gfx1151, -b 2048 -ub 2048 unless noted, vs cutoff: PP128 PP256 PP384 PP768 PP1536 PP4096@ub4096 UD-Q8_K_XL +39.7% +61.1% +21.5% +17.2% +12.4% +3.9% UD-Q4_K_XL +93.5% +75.9% +60.1% +39.1% +18.4% +2.4% (PP512/1024/2048 as in the previous commit.) test-backend-ops MUL_MAT_ID: 914/914 passed (adds 4- and 128-row cases). --- ggml/src/ggml-cuda/mmq.cuh | 40 +++++++++++++++++++++++++------------- tests/test-backend-ops.cpp | 10 +++++++++- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index f7ac408b22fe..1c46a8e0e870 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1585,34 +1585,46 @@ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const i } // RDNA3.5 tile selection for MoE (mul_mat_id) with 256 experts. -// The per-expert row count is n_tokens*n_expert_used/256 (16/32/64 for ubatch 512/1024/2048 with 8 active experts), -// but the generic selection below only sees ncols_max == n_tokens*n_expert_used and picks J=128, so most of every -// tile is padding. Use a narrow J and the routed-compact kernel, which only enumerates real per-expert tiles. -// J per type/row count was measured on gfx1151 for the 2048x512 and 3072x1024 expert shapes. +// The per-expert row count is n_tokens*n_expert_used/256 (e.g. 16/32/64 for ubatch 512/1024/2048 with 8 active +// experts), but the generic selection below only sees ncols_max == n_tokens*n_expert_used and picks J=128, so most +// of every tile is padding and the grid contains ncols_max/J*n_experts mostly-empty blocks. Use a J matched to the +// row count and the routed-compact kernel, which only enumerates real per-expert tiles. +// J per type/row range was measured on gfx1151 for the 2048x512 and 3072x1024 expert shapes. static constexpr int mmq_rdna3_5_id_get_J(const ggml_type type, const int64_t rows_per_expert) { switch (type) { case GGML_TYPE_Q8_0: - return rows_per_expert == 16 || rows_per_expert == 32 || rows_per_expert == 64 ? 48 : 0; + return rows_per_expert <= 12 ? 16 : rows_per_expert <= 64 ? 48 : 128; case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: - return rows_per_expert == 16 || rows_per_expert == 64 ? 32 : rows_per_expert == 32 ? 48 : 0; + return rows_per_expert <= 12 ? 16 : rows_per_expert <= 24 ? 32 : rows_per_expert <= 48 ? 48 : + rows_per_expert <= 64 ? 32 : 128; default: return 0; } } static constexpr bool mmq_rdna3_5_id_use_compact(const ggml_type type, const int J) { - return (type == GGML_TYPE_Q8_0 && J == 48) || - ((type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K) && (J == 32 || J == 48)); + switch (type) { + case GGML_TYPE_Q8_0: + return J == 16 || J == 48 || J == 128; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return J == 16 || J == 32 || J == 48 || J == 128; + default: + return false; + } } -static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 16) == 48); -static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q6_K, 16) == 32); -static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_K, 32) == 48); -static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q5_K, 64) == 32); -static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 48) == 0); -static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_0, 16) == 0); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 4) == 16); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 16) == 48); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 64) == 48); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 128) == 128); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q6_K, 16) == 32); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_K, 32) == 48); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q5_K, 64) == 32); +static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q4_0, 16) == 0); static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 48)); static_assert(!mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 32)); static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q6_K, 32)); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index b5130fb7708d..3d8401f52fc7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9484,15 +9484,23 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); #endif - // 256-expert MoE with 16/32 rows per expert (ubatch 512/1024), exercises the narrow RDNA3.5 mul_mat_id tiles + // 256-expert MoE with 4/16/32/128 rows per expert, exercises the RDNA3.5 mul_mat_id tile selection + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q6_K, GGML_TYPE_F32, 256, 8, false, 512, 4096, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2016, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048)); From 7018546eb2e3ec105acbec0bf624659856f7d70e Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:13:37 +0200 Subject: [PATCH 22/30] CUDA/HIP: use WMMA flash attention for D=256 prefill on RDNA3.5 On gfx1151 the D=256 flash-attention path always fell back to the tile kernel, which is FMA-bound (~8 TFLOPS) and dominates prefill time at deep context (68% of PP time at 32k, 81% at 64k for Qwen3.5-0.8B). - Enable AMD WMMA device code for DKQ=256 (previously DKQ>128 was excluded). - Add a RDNA ncols=64 D=256 config tuned for the 256-VGPR budget: nwarps=8, nstages=1, nbatch_fa=32, nbatch_K2=128, nbatch_V2=64, nbatch_combine=64. Naive settings spill heavily and lose to the tile kernel. - Dispatch to MMA only when Q->ne[1]*gqa_ratio_eff > 32; the tile kernel is still faster for small batches (nb 4/8). - Fix a latent tile_Q reuse race in the combine loop: when DV/2 > nbatch_combine, warps read tile_Q columns written by other warps and the buffer is rewritten by the next batch without a barrier. Qwen3.5-0.8B UD-Q8_K_XL, gfx1151, PP512: depth 0: 9361 -> 9473 t/s (+1.2%) depth 32768: 3211 -> 4105 t/s (+27.9%) depth 65536: 1927 -> 2632 t/s (+36.6%) TG unchanged. test-backend-ops FLASH_ATTN_EXT: 2939/2939 passed. --- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 8 +++++--- ggml/src/ggml-cuda/fattn.cu | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 7f4cfd5511ff..55d7988528ca 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -158,7 +158,7 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 8, 64, 2, 32, 128, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 16, 64, 2, 32, 128, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 32, 128, 2, 64, 128, 128, 64, 1, true); - GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 128, 2, 64, 128, 128, 64, 1, true); + GGML_CUDA_FATTN_MMA_CONFIG_CASE(256, 256, 64, 256, 1, 32, 128, 64, 64, 1, false); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 32, 128, 2, 32, 160, 128, 128, 1, true); GGML_CUDA_FATTN_MMA_CONFIG_CASE(320, 256, 64, 128, 2, 32, 160, 128, 128, 1, true); @@ -1687,7 +1687,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile( } } } - if (np > 1) { + // Warps read columns of tile_Q that were written by other warps, so a barrier is needed before tile_Q is + // reused: by the next nbatch_combine batch (DV/2 > nbatch_combine) or by the Q load of the next tile. + if (np > 1 || DV/2 > nbatch_combine) { __syncthreads(); } } @@ -1759,7 +1761,7 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 128) { + if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256 || (DKQ > 128 && DKQ != 256)) { NO_DEVICE_CODE; return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index ef6bf7adea4c..3a5faa5fbaad 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -533,6 +533,12 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const return BEST_FATTN_KERNEL_MMA_F16; } + // On RDNA3.5 the D=256 tile kernel is FMA-bound for prefill batches; the WMMA kernel with 64 columns is ~30-45% faster. + // For smaller batches (ncols 16/32 configs) the tile kernel is still faster. + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && gqa_opt_applies && Q->ne[0] == 256 && V->ne[0] == 256 && Q->ne[1] * gqa_ratio_eff > 32) { + return BEST_FATTN_KERNEL_MMA_F16; + } + // If there are no tensor cores available, use the generic tile kernel: if (can_use_vector_kernel) { if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { From c3f0ac2bfbfa6edbc3b8ab34a384a8bfb5aa9824 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:04:18 +0200 Subject: [PATCH 23/30] llama: synchronize before mutating graph state --- src/llama-context.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index ca387021d221..02202f8c5693 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1348,17 +1348,14 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll static const bool graph_timing = getenv("LLAMA_GRAPH_TIMING") && atoi(getenv("LLAMA_GRAPH_TIMING")) != 0; int64_t t_build = 0, t_alloc = 0, t_inputs = 0; bool reused = false; + + // The previous asynchronous graph may still read inputs or use graph allocations. + ggml_backend_sched_synchronize(sched.get()); + if (!graph_reuse_disable && res->can_reuse(gparams)) { reused = true; //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__); - // with pipeline parallelism, the previous graph_compute_async may still be running - // on the GPU. we must synchronize before set_inputs to avoid overwriting input tensors - // that the previous compute is still reading. - if (cparams.pipeline_parallel) { - ggml_backend_sched_synchronize(sched.get()); - } - n_reused++; } else { res->reset(); From ff9c3b29987f75398a43d1e2464378639fa9808e Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:04:18 +0200 Subject: [PATCH 24/30] tests: avoid ambiguous top-k MoE scores --- tests/test-backend-ops.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3d8401f52fc7..1e87615b57a4 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -6570,6 +6570,39 @@ struct test_topk_moe : public test_case { // Verify two outputs std::vector fusion_test_nodes() override { return { selected_experts, weights }; } + void initialize_tensors(ggml_context * ctx) override { + std::random_device rd; + std::default_random_engine rng(rd()); + + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->op != GGML_OP_NONE) { + continue; + } + if (strcmp(t->name, "exp_probs_b") == 0) { + std::vector data(t->ne[0]); + const float amplitude = + gating_func == GATING_FUNC_SOFTMAX ? 0.1f : + gating_func == GATING_FUNC_SIGMOID ? 0.5f : + gating_func == GATING_FUNC_SOFTMAX_WEIGHT ? 4.5f : 1.0f; + for (int64_t i = 0; i < t->ne[0]; ++i) { + data[i] = i < t->ne[0] / 2 ? amplitude : -amplitude; + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t)); + continue; + } + + for (int64_t row = 0; row < ggml_nrows(t); ++row) { + std::vector data(t->ne[0]); + for (int64_t i = 0; i < t->ne[0]; ++i) { + data[i] = -4.0f + 8.0f * i / std::max(1, t->ne[0] - 1); + } + std::shuffle(data.begin(), data.end(), rng); + ggml_backend_tensor_set(t, data.data(), row * t->nb[1], t->ne[0] * sizeof(float)); + } + } + } + // allow output in arbitrary order double err(const float * a, const float * b, size_t n) override { std::vector a2(n); From 6130b7262ae97d353556903e0175b8993db77bef Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:02:14 +0200 Subject: [PATCH 25/30] hip: optimize RDNA3.5 MoE inference paths --- ggml/src/ggml-cuda/argsort.cu | 15 +- ggml/src/ggml-cuda/binbcast.cu | 12 +- ggml/src/ggml-cuda/common.cuh | 16 + ggml/src/ggml-cuda/cumsum.cu | 7 +- ggml/src/ggml-cuda/gated_delta_net.cu | 42 +- ggml/src/ggml-cuda/gated_delta_net.cuh | 3 + ggml/src/ggml-cuda/ggml-cuda.cu | 649 ++++++++++- ggml/src/ggml-cuda/hyperconn.cu | 366 ++++++ ggml/src/ggml-cuda/hyperconn.cuh | 55 + ggml/src/ggml-cuda/mean.cu | 5 + ggml/src/ggml-cuda/mmid.cu | 56 + ggml/src/ggml-cuda/mmq.cu | 3 +- ggml/src/ggml-cuda/mmq.cuh | 38 +- ggml/src/ggml-cuda/mmvq.cu | 1456 ++++++++++++++++++++++-- ggml/src/ggml-cuda/mmvq.cuh | 36 + ggml/src/ggml-cuda/sum.cu | 5 + ggml/src/ggml-cuda/top-k.cu | 15 +- ggml/src/ggml-cuda/topk-moe.cu | 24 +- ggml/src/ggml-cuda/unary.cu | 44 + ggml/src/ggml-cuda/unary.cuh | 3 + ggml/src/ggml-cuda/vecdotq.cuh | 54 + ggml/src/ggml-cuda/vendors/hip.h | 3 + src/llama-context.cpp | 5 + src/models/models.h | 8 + src/models/qwen4exp.cpp | 238 +++- tests/test-backend-ops.cpp | 197 +++- 26 files changed, 3141 insertions(+), 214 deletions(-) create mode 100644 ggml/src/ggml-cuda/hyperconn.cu create mode 100644 ggml/src/ggml-cuda/hyperconn.cuh diff --git a/ggml/src/ggml-cuda/argsort.cu b/ggml/src/ggml-cuda/argsort.cu index 26af90025972..17b905fd0b69 100644 --- a/ggml/src/ggml-cuda/argsort.cu +++ b/ggml/src/ggml-cuda/argsort.cu @@ -1,11 +1,16 @@ #include "argsort.cuh" #ifdef GGML_CUDA_USE_CUB -# include -# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1) -# define STRIDED_ITERATOR_AVAILABLE -# include -# endif +# ifdef GGML_CUDA_CUB_IS_HIPCUB +# include +namespace cub = hipcub; +# else +# include +# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 1) +# define STRIDED_ITERATOR_AVAILABLE +# include +# endif +# endif // GGML_CUDA_CUB_IS_HIPCUB using namespace cub; #endif // GGML_CUDA_USE_CUB diff --git a/ggml/src/ggml-cuda/binbcast.cu b/ggml/src/ggml-cuda/binbcast.cu index e04868e5ccf6..20dd6682d397 100644 --- a/ggml/src/ggml-cuda/binbcast.cu +++ b/ggml/src/ggml-cuda/binbcast.cu @@ -696,6 +696,14 @@ void ggml_cuda_op_weighted_expert_sum(ggml_backend_cuda_context & ctx, ggml_tens case 6: launch_weighted_expert_sum(6); break; case 7: launch_weighted_expert_sum(7); break; case 8: launch_weighted_expert_sum(8); break; + case 9: launch_weighted_expert_sum(9); break; + case 10: launch_weighted_expert_sum(10); break; + case 11: launch_weighted_expert_sum(11); break; + case 12: launch_weighted_expert_sum(12); break; + case 13: launch_weighted_expert_sum(13); break; + case 14: launch_weighted_expert_sum(14); break; + case 15: launch_weighted_expert_sum(15); break; + case 16: launch_weighted_expert_sum(16); break; default: GGML_ABORT("unsupported expert count"); } #undef launch_weighted_expert_sum @@ -774,7 +782,7 @@ static __global__ void __launch_bounds__(256, 1) shared_gate_mul_add_f32( for (int64_t i = tid; i < n_embd; i += block_size) { const float product = rounded_mul_f32(src[i], gate); const float value = rounded_add_f32(other[i], product); - dst[i] = rounded_add_f32(value, residual[i]); + dst[i] = residual ? rounded_add_f32(value, residual[i]) : value; } } @@ -787,7 +795,7 @@ void ggml_cuda_op_shared_gate_mul_add( const ggml_cuda_kernel_launch_params launch_params(1, 256, 0, ctx.stream()); ggml_cuda_kernel_launch(shared_gate_mul_add_f32, launch_params, (const float *) w->data, (const float *) y->data, (const float *) src->data, (const float *) other->data, - (const float *) residual->data, (float *) dst->data, (int) (w->ne[0] / 2), dst->ne[0]); + residual ? (const float *) residual->data : nullptr, (float *) dst->data, (int) (w->ne[0] / 2), dst->ne[0]); } void ggml_cuda_op_repeat_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index e5ccd1feab14..794a88f5aa8b 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -111,6 +111,14 @@ # define GGML_CUDA_USE_CUB #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11070 +// hipCUB (rocPRIM) offers the same device-wide radix sort / scan / reduce entry points as CUB. Using it lifts the +// ncols <= 1024 limit of ARGSORT / TOP_K on HIP, which otherwise falls back to the CPU (e.g. the QSA indexer top-k +// of qwen4exp / DeepSeek-V3.2 at contexts beyond 2k tokens). +#if defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_CUB) && __has_include() +# define GGML_CUDA_USE_CUB +# define GGML_CUDA_CUB_IS_HIPCUB +#endif + // PDL host-side support (cudaLaunchKernelEx) requires CUDART >= 11.8. // However, this has been bugged in CTK < 12.3 for MSVC builds, see // https://github.com/ggml-org/llama.cpp/pull/22522#discussion_r3302393293 @@ -1540,6 +1548,10 @@ struct ggml_cuda_mm_fusion_args_host { const ggml_tensor * gate_scale = nullptr; ggml_glu_op glu_op; float glu_limit = 0.0f; + // RDNA3.5 fused-quantize matvec prologue (y_op == 3), see ggml_cuda_mm_fusion_args_device + const ggml_tensor * y_gate = nullptr; + const ggml_tensor * y_norm_w = nullptr; + float y_eps = 0.0f; }; struct ggml_cuda_mm_fusion_args_device { const void * x_bias = nullptr; @@ -1549,6 +1561,10 @@ struct ggml_cuda_mm_fusion_args_device { const void * gate_scale = nullptr; ggml_glu_op glu_op; float glu_limit = 0.0f; + // RDNA3.5 fused-quantize matvec prologue (y_op == 3): y' = sigmoid(y_gate) * (rms_norm_128(y) * y_norm_w) + const void * y_gate = nullptr; + const void * y_norm_w = nullptr; + float y_eps = 0.0f; }; struct ggml_cuda_kernel_launch_params { diff --git a/ggml/src/ggml-cuda/cumsum.cu b/ggml/src/ggml-cuda/cumsum.cu index def9c32955f2..51cfb84dab17 100644 --- a/ggml/src/ggml-cuda/cumsum.cu +++ b/ggml/src/ggml-cuda/cumsum.cu @@ -5,7 +5,12 @@ #include "ggml.h" #ifdef GGML_CUDA_USE_CUB -# include +# ifdef GGML_CUDA_CUB_IS_HIPCUB +# include +namespace cub = hipcub; +# else +# include +# endif // GGML_CUDA_CUB_IS_HIPCUB #endif // GGML_CUDA_USE_CUB template diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index 4873b853c742..9f0e8f75cc14 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -515,7 +515,7 @@ static void launch_gated_delta_net( const int cc = ggml_cuda_info().devices[id].cc; const int warp_size = ggml_cuda_info().devices[id].warp_size; // Qwen3.6 PP2048 measured faster on gfx1151 on 2026-09-02. Retest if occupancy changes. - const int num_warps = GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && (H == 32 || H == 64) && !KDA ? 32 : + const int num_warps = GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && (H == 32 || H == 48 || H == 64) && !KDA ? 32 : (GGML_CUDA_CC_IS_RDNA3_5(cc) ? 16 : gated_delta_net_num_warps(cc)); dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps); dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1); @@ -538,16 +538,22 @@ static void launch_gated_delta_net( if (GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && num_warps == 32 && n_tokens >= 16) { // 16 warps x 4 columns per block, 16-token tiles: best of the swept configurations on gfx1151 // (2048 tokens, H=32: 5.03 ms -> 2.2 ms; H=64: 9.29 ms -> 4.5 ms). - constexpr int tiled_warps = 16; - constexpr int tiled_cols = 4; - constexpr int tiled_tile = 16; - const dim3 tiled_grid(H, n_seqs, S_v / (tiled_warps*tiled_cols)); - const dim3 tiled_block(warp_size, tiled_warps, 1); - const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(tiled_grid, tiled_block, 0, stream); - ggml_cuda_kernel_launch(gated_delta_net_tiled_cuda<128, tiled_warps, tiled_cols, tiled_tile, keep_rs_t>, launch_params, - q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, - sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, - neqk1_magic, rq3_magic, scale, state_slot_stride, K); + const int cfg = getenv("GGML_GDN_TILE_CFG") ? atoi(getenv("GGML_GDN_TILE_CFG")) : (H == 48 ? 1 : 0); + auto launch_tiled = [&](auto kernel, int warps, int cols) { + const dim3 tiled_grid(H, n_seqs, S_v / (warps * cols)); + const dim3 tiled_block(warp_size, warps, 1); + const ggml_cuda_kernel_launch_params launch_params(tiled_grid, tiled_block, 0, stream); + ggml_cuda_kernel_launch(kernel, launch_params, + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, + sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, + neqk1_magic, rq3_magic, scale, state_slot_stride, K); + }; + switch (cfg) { + case 1: launch_tiled(gated_delta_net_tiled_cuda<128, 8, 8, 16, keep_rs_t>, 8, 8); break; + case 2: launch_tiled(gated_delta_net_tiled_cuda<128, 32, 2, 16, keep_rs_t>, 32, 2); break; + case 3: launch_tiled(gated_delta_net_tiled_cuda<128, 16, 4, 8, keep_rs_t>, 16, 4); break; + default: launch_tiled(gated_delta_net_tiled_cuda<128, 16, 4, 16, keep_rs_t>, 16, 4); break; + } return; } } @@ -902,17 +908,25 @@ gdn_decode_norm_cuda(const float * attn, const float * norm_w, float * out, cons out[(int64_t) h * S + tid] = scale * xi * norm_w[tid]; } -void ggml_cuda_op_gdn_decode_fused(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args_in) { +void ggml_cuda_op_gdn_decode_fused_prenorm(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args_in, float * attn_scratch) { GGML_ASSERT(args_in.S == 128 && args_in.d_conv == 4); - ggml_cuda_pool_alloc attn(ctx.pool(), args_in.S * args_in.H_v); ggml_cuda_gdn_decode_args args = args_in; - args.attn_out = attn.get(); + args.attn_out = attn_scratch; const dim3 grid(args.H_v, 1, 1); const dim3 block(32, 32, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid, block, 0, ctx.stream()); ggml_cuda_kernel_launch(gdn_decode_fused_cuda<128, 4>, launch_params, args); +} +void ggml_cuda_op_gdn_decode_fused(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args_in) { + GGML_ASSERT(args_in.S == 128 && args_in.d_conv == 4); + ggml_cuda_pool_alloc attn(ctx.pool(), args_in.S * args_in.H_v); + ggml_cuda_gdn_decode_args args = args_in; + args.attn_out = attn.get(); + ggml_cuda_op_gdn_decode_fused_prenorm(ctx, args, attn.get()); + + const dim3 grid(args.H_v, 1, 1); const dim3 norm_block(128, 1, 1); const ggml_cuda_kernel_launch_params norm_params = ggml_cuda_kernel_launch_params(grid, norm_block, 0, ctx.stream()); ggml_cuda_kernel_launch(gdn_decode_norm_cuda<128>, norm_params, attn.get(), args.norm_w, args.out, args.eps_rms); diff --git a/ggml/src/ggml-cuda/gated_delta_net.cuh b/ggml/src/ggml-cuda/gated_delta_net.cuh index b2be8c4f6b7a..e2b41cf1c47f 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cuh +++ b/ggml/src/ggml-cuda/gated_delta_net.cuh @@ -36,6 +36,7 @@ struct ggml_cuda_gdn_decode_args { float scale; // 1/sqrt(S) // gated rms norm const float * norm_w; // [S] + const ggml_tensor * norm_w_tensor = nullptr; float eps_rms; float * out; // [S, H_v] float * attn_out; // [S, H_v] pre-norm scratch (set by the host) @@ -43,3 +44,5 @@ struct ggml_cuda_gdn_decode_args { }; void ggml_cuda_op_gdn_decode_fused(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args); +// first kernel only: writes the pre-norm attention output [S, H_v] to attn_scratch (the gated norm is applied by the consumer) +void ggml_cuda_op_gdn_decode_fused_prenorm(ggml_backend_cuda_context & ctx, const ggml_cuda_gdn_decode_args & args, float * attn_scratch); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 50c343a1c0d4..088a66735c6c 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -59,6 +59,7 @@ #include "ggml-cuda/wkv.cuh" #include "ggml-cuda/gla.cuh" #include "ggml-cuda/gated_delta_net.cuh" +#include "ggml-cuda/hyperconn.cuh" #include "ggml-cuda/dsv4-hc.cuh" #include "ggml-cuda/set.cuh" #include "ggml-cuda/set-rows.cuh" @@ -725,17 +726,71 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { // cuda buffer +#if defined(GGML_USE_HIP) && defined(__linux__) +#include +#include +#include +// Integrated GPUs (UMA): large buffers can be backed by transparent-huge-page host memory that is registered +// with the device. The GPU then maps 2 MiB fragments instead of the 4 KiB pages of hipMalloc'd GTT memory, +// which removes most of the TLB misses of the small-matrix weight streams during token generation +// (gfx1151: 3.5 MiB Q8_0 matvec 22 -> 18 us). Enabled with GGML_CUDA_THP_BUFFERS=1. +static bool ggml_cuda_thp_buffers_enabled() { + static const bool enabled = getenv("GGML_CUDA_THP_BUFFERS") != nullptr && atoi(getenv("GGML_CUDA_THP_BUFFERS")) != 0; + return enabled; +} + +static void * ggml_cuda_thp_alloc(size_t size, size_t * alloc_size_out) { + const size_t huge = 2u << 20; + const size_t asz = (size + huge - 1) / huge * huge; + void * h = mmap(nullptr, asz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (h == MAP_FAILED) { + return nullptr; + } + if (madvise(h, asz, MADV_HUGEPAGE) != 0) { + GGML_LOG_WARN("%s: madvise(MADV_HUGEPAGE) failed: %s\n", __func__, strerror(errno)); + } + // fault the pages in on the host side (THP allocation happens here) before the GPU mapping is built + for (size_t off = 0; off < asz; off += huge) { + ((volatile char *) h)[off] = 0; + } + if (hipHostRegister(h, asz, hipHostRegisterDefault) != hipSuccess) { + (void) hipGetLastError(); + GGML_LOG_WARN("%s: hipHostRegister of %.2f MiB failed, falling back to hipMalloc\n", __func__, asz / 1024.0 / 1024.0); + munmap(h, asz); + return nullptr; + } + void * dptr = nullptr; + if (hipHostGetDevicePointer(&dptr, h, 0) != hipSuccess || dptr == nullptr) { + (void) hipGetLastError(); + hipHostUnregister(h); + munmap(h, asz); + return nullptr; + } + GGML_ASSERT(dptr == h); // UMA: the device pointer is the host pointer, tensor data is copied with plain memcpys + *alloc_size_out = asz; + return h; +} +#endif + struct ggml_backend_cuda_buffer_context { int device; void * dev_ptr = nullptr; std::string name; + size_t thp_size = 0; // != 0: dev_ptr is registered THP host memory of this size (see ggml_cuda_thp_alloc) - ggml_backend_cuda_buffer_context(int device, void * dev_ptr) : + ggml_backend_cuda_buffer_context(int device, void * dev_ptr, size_t thp_size = 0) : device(device), dev_ptr(dev_ptr), - name(GGML_CUDA_NAME + std::to_string(device)) { + name(GGML_CUDA_NAME + std::to_string(device)), thp_size(thp_size) { } ~ggml_backend_cuda_buffer_context() { +#if defined(GGML_USE_HIP) && defined(__linux__) + if (thp_size != 0) { + CUDA_CHECK(hipHostUnregister(dev_ptr)); + munmap(dev_ptr, thp_size); + return; + } +#endif CUDA_CHECK(cudaFree(dev_ptr)); } }; @@ -888,6 +943,16 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac ggml_cuda_set_device(buft_ctx->device); void * dev_ptr; +#if defined(GGML_USE_HIP) && defined(__linux__) + if (ggml_cuda_thp_buffers_enabled() && ggml_cuda_info().devices[buft_ctx->device].integrated && size >= (64u << 20)) { + size_t thp_size = 0; + dev_ptr = ggml_cuda_thp_alloc(size, &thp_size); + if (dev_ptr != nullptr) { + ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr, thp_size); + return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size); + } + } +#endif cudaError_t err = ggml_cuda_device_malloc(&dev_ptr, size, buft_ctx->device); if (err != cudaSuccess) { // clear the error @@ -3235,6 +3300,7 @@ static int ggml_cuda_try_gdn_decode_fusion(const ggml_cgraph * cgraph, int node_ args.state_out = state_cpy.data; args.scale = 1.0f / sqrtf((float) S); args.norm_w = (const float *) mul_w->src[1]->data; + args.norm_w_tensor = mul_w->src[1]; memcpy(&args.eps_rms, rms->op_params, sizeof(float)); args.out = (float *) mul_w->data; args.attn_out = nullptr; @@ -3499,6 +3565,79 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, } // try and fuse nodes and return the number of nodes to skip +// qwen4exp hyper-connection stream mix chain starting at node i: +// sigmoid(gate) -> mul(xn, .) -> reshape -> view(stream 0) -> cont -> (view(stream c) -> add) x (hc-1) -> scale +// Fills args and ops (n_ops entries) and returns n_ops, or 0 when the nodes do not form the chain. +static int ggml_cuda_match_hc_mix(const ggml_cgraph * cgraph, const int i, ggml_cuda_hc_mix_args & args, enum ggml_op * ops) { + const ggml_tensor * node = cgraph->nodes[i]; + if (!(node->op == GGML_OP_UNARY && ggml_get_unary_op(node) == GGML_UNARY_OP_SIGMOID && node->type == GGML_TYPE_F32 && + i + 5 < cgraph->n_nodes && + cgraph->nodes[i + 1]->op == GGML_OP_MUL && cgraph->nodes[i + 2]->op == GGML_OP_RESHAPE && + cgraph->nodes[i + 3]->op == GGML_OP_VIEW && cgraph->nodes[i + 4]->op == GGML_OP_CONT)) { + return 0; + } + const ggml_tensor * gate = node->src[0]; + const ggml_tensor * mul = cgraph->nodes[i + 1]; + const ggml_tensor * view0 = cgraph->nodes[i + 3]; + const ggml_tensor * cont = cgraph->nodes[i + 4]; + const ggml_tensor * xn = mul->src[0] == node ? mul->src[1] : mul->src[0]; + + const int64_t n_embd = cont->ne[0]; + const int64_t n_tokens = cont->ne[1]; + const int64_t hc_dim = mul->ne[0]; + const int hc = n_embd > 0 ? (int) (hc_dim / n_embd) : 0; + const size_t row_size = hc_dim * sizeof(float); + + auto stream_view_ok = [&](const ggml_tensor * v, int c) { + return v->op == GGML_OP_VIEW && v->view_src == mul && v->type == GGML_TYPE_F32 && + v->view_offs == (size_t) c * n_embd * sizeof(float) && + v->ne[0] == n_embd && v->ne[1] == n_tokens && v->ne[2] == 1 && v->ne[3] == 1 && + v->nb[0] == sizeof(float) && v->nb[1] == row_size; + }; + + bool ok = (mul->src[0] == node || mul->src[1] == node) && xn != node && + mul->type == GGML_TYPE_F32 && xn->type == GGML_TYPE_F32 && gate->type == GGML_TYPE_F32 && + ggml_is_contiguous(xn) && ggml_is_contiguous(gate) && ggml_are_same_shape(xn, gate) && + ggml_are_same_shape(mul, xn) && ggml_are_same_shape(mul, node) && + mul->ne[2] == 1 && mul->ne[3] == 1 && ggml_nrows(mul) == n_tokens && + hc >= 2 && hc <= 16 && hc_dim == (int64_t) hc * n_embd && + cgraph->nodes[i + 2]->view_src == mul && cont->src[0] == view0 && stream_view_ok(view0, 0) && + cont->type == GGML_TYPE_F32 && ggml_is_contiguous(cont) && cont->ne[2] == 1 && cont->ne[3] == 1; + + const int n_ops = ok ? 5 + 2 * (hc - 1) + 1 : 0; + ok = ok && i + n_ops <= cgraph->n_nodes; + if (!ok) { + return 0; + } + + ops[0] = GGML_OP_UNARY; ops[1] = GGML_OP_MUL; ops[2] = GGML_OP_RESHAPE; ops[3] = GGML_OP_VIEW; ops[4] = GGML_OP_CONT; + const ggml_tensor * prev = cont; + for (int c = 1; ok && c < hc; ++c) { + const ggml_tensor * v = cgraph->nodes[i + 5 + 2 * (c - 1)]; + const ggml_tensor * add = cgraph->nodes[i + 6 + 2 * (c - 1)]; + ok = stream_view_ok(v, c) && add->op == GGML_OP_ADD && add->type == GGML_TYPE_F32 && + add->src[0] == prev && add->src[1] == v && ggml_are_same_shape(add, cont); + ops[5 + 2 * (c - 1)] = GGML_OP_VIEW; + ops[6 + 2 * (c - 1)] = GGML_OP_ADD; + prev = add; + } + ggml_tensor * scale = cgraph->nodes[i + n_ops - 1]; + ops[n_ops - 1] = GGML_OP_SCALE; + ok = ok && scale->op == GGML_OP_SCALE && scale->src[0] == prev && scale->type == GGML_TYPE_F32 && + ggml_is_contiguous(scale) && ggml_are_same_shape(scale, cont); + if (!ok) { + return 0; + } + + args.xn = xn; + args.gate = gate; + args.dst = scale; + args.hc = hc; + args.scale = ggml_get_op_params_f32(scale, 0); + args.bias = ggml_get_op_params_f32(scale, 1); + return n_ops; +} + static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION")); @@ -3506,8 +3645,80 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 0; } + // debugging aid: only fuse nodes with GGML_CUDA_FUSE_MIN_NODE <= i < GGML_CUDA_FUSE_MAX_NODE + static const int fuse_min_node = getenv("GGML_CUDA_FUSE_MIN_NODE") ? std::atoi(getenv("GGML_CUDA_FUSE_MIN_NODE")) : 0; + static const int fuse_max_node = getenv("GGML_CUDA_FUSE_MAX_NODE") ? std::atoi(getenv("GGML_CUDA_FUSE_MAX_NODE")) : INT_MAX; + if (i < fuse_min_node || i >= fuse_max_node) { + return 0; + } + ggml_tensor * node = cgraph->nodes[i]; + // RDNA3.5 decode: consecutive single-column MUL_MATs that read the same activation vector (plain Q8_0/F32 + // matvecs, or a [mul_mat, mul_mat, glu] gate/up pair) are launched as one grouped kernel. The segments are + // mutually independent, so the launch position of the first one is valid for all of them. + if (getenv("GGML_CUDA_DISABLE_MMV_GROUP") == nullptr && + node->op == GGML_OP_MUL_MAT && node->src[2] == nullptr && ggml_cuda_mmv_group_seg_ok(node->src[0], node->src[1])) { + const ggml_tensor * y = node->src[1]; + ggml_cuda_mmv_group_seg segs[GGML_CUDA_MMV_GROUP_MAX]; + int out_nodes[GGML_CUDA_MMV_GROUP_MAX]; + int nseg = 0; + int j = i; + int j_end = i; // index of the last node that belongs to the group + while (j < cgraph->n_nodes && nseg < GGML_CUDA_MMV_GROUP_MAX) { + ggml_tensor * n = cgraph->nodes[j]; + if (j > i && ggml_cuda_is_view_or_noop(n)) { + j++; + continue; + } + const bool is_mmv = n->op == GGML_OP_MUL_MAT && n->src[2] == nullptr && n->src[1] == y && + (n->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 && n->type == GGML_TYPE_F32 && ggml_is_contiguous(n) && + ggml_cuda_mmv_group_seg_ok(n->src[0], y); + if (!is_mmv) { + break; + } + // gate/up pair followed by the GLU that combines them + if (j + 2 < cgraph->n_nodes && cgraph->nodes[j + 1]->op == GGML_OP_MUL_MAT && cgraph->nodes[j + 2]->op == GGML_OP_GLU) { + ggml_tensor * n2 = cgraph->nodes[j + 1]; + ggml_tensor * glu = cgraph->nodes[j + 2]; + ggml_tensor * gate_n = nullptr; + ggml_tensor * up_n = nullptr; + if (glu->src[0] == n && glu->src[1] == n2) { + gate_n = n; + up_n = n2; + } else if (glu->src[0] == n2 && glu->src[1] == n) { + gate_n = n2; + up_n = n; + } + if (gate_n && n2->src[2] == nullptr && n2->src[1] == y && (n2->flags & GGML_TENSOR_FLAG_COMPUTE) != 0 && + up_n->src[0]->type == GGML_TYPE_Q8_0 && gate_n->src[0]->type == GGML_TYPE_Q8_0 && + ggml_are_same_shape(up_n->src[0], gate_n->src[0]) && ggml_cuda_mmv_group_seg_ok(n2->src[0], y) && + glu->type == GGML_TYPE_F32 && ggml_is_contiguous(glu) && ggml_nelements(glu) == up_n->src[0]->ne[1] && + ggml_cuda_should_fuse_mul_mat(up_n, gate_n, glu) && + ggml_can_fuse_subgraph(cgraph, j, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, { j + 2 })) { + segs[nseg].w = up_n->src[0]; + segs[nseg].gate = gate_n->src[0]; + segs[nseg].dst = glu; + segs[nseg].glu_op = ggml_get_glu_op(glu); + segs[nseg].glu_limit = ggml_get_op_params_f32(glu, 3); + out_nodes[nseg++] = j + 2; + j_end = j + 2; + j += 3; + continue; + } + } + segs[nseg].w = n->src[0]; + segs[nseg].dst = n; + out_nodes[nseg++] = j; + j_end = j; + j++; + } + if (nseg >= 2 && ggml_cuda_check_fusion_memory_ranges(cgraph, i, j_end - i + 1, out_nodes, nseg)) { + ggml_cuda_mmv_group(*cuda_ctx, y, segs, nseg); + return j_end - i; + } + } + if (node->op == GGML_OP_L2_NORM && i + 2 < cgraph->n_nodes && GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) { ggml_tensor * k_view = cgraph->nodes[i + 1]; @@ -3539,6 +3750,46 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_cuda_gdn_decode_args args; const int nodes_to_skip = ggml_cuda_try_gdn_decode_fusion(cgraph, i, args); if (nodes_to_skip > 0) { + // qwen4exp: the gated norm output only feeds sigmoid(z) * . -> ssm_out matvec. Apply the norm and the + // gate in the prologue of the fused-quantize matvec instead of two more kernels. + { + const int norm_idx = i + nodes_to_skip; // MUL(rms_norm, w): the last node of the chain + ggml_tensor * norm_out = cgraph->nodes[norm_idx]; + int j = norm_idx + 1; + auto next_compute = [&](int & k) -> ggml_tensor * { + while (k < cgraph->n_nodes && ggml_cuda_is_view_or_noop(cgraph->nodes[k])) { + k++; + } + return k < cgraph->n_nodes ? cgraph->nodes[k] : nullptr; + }; + ggml_tensor * sig = next_compute(j); + const int sig_idx = j; + j++; + ggml_tensor * mul = next_compute(j); + const int mul_idx = j; + j++; + ggml_tensor * mm = next_compute(j); + const int mm_idx = j; + if (getenv("GGML_CUDA_DISABLE_GDN_GATE") == nullptr && + sig && mul && mm && sig->op == GGML_OP_UNARY && ggml_get_unary_op(sig) == GGML_UNARY_OP_SIGMOID && + mul->op == GGML_OP_MUL && mm->op == GGML_OP_MUL_MAT && mm->src[2] == nullptr && + (mm->src[1] == mul || (ggml_cuda_is_view_or_noop(mm->src[1]) && mm->src[1]->view_src == mul && + ggml_is_contiguous(mm->src[1]) && ggml_nelements(mm->src[1]) == ggml_nelements(mul))) && + ((mul->src[0] == sig && mul->src[1] == norm_out) || (mul->src[0] == norm_out && mul->src[1] == sig)) && + sig->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 && ggml_is_contiguous(mul) && + ggml_are_same_shape(sig, norm_out) && ggml_are_same_shape(mul, norm_out) && + ggml_nelements(norm_out) == args.S * args.H_v && args.S == 128 && + ggml_node_get_use_count(cgraph, norm_idx) == 1 && ggml_node_get_use_count(cgraph, sig_idx) == 1 && + ggml_node_get_use_count(cgraph, mul_idx) == 1 && + (norm_out->flags & GGML_TENSOR_FLAG_OUTPUT) == 0 && (sig->flags & GGML_TENSOR_FLAG_OUTPUT) == 0 && + (mul->flags & GGML_TENSOR_FLAG_OUTPUT) == 0 && + ggml_cuda_mul_mat_vec_q_fq_gdn_gate_ok(*cuda_ctx, mm, sig->src[0])) { + ggml_cuda_pool_alloc attn(cuda_ctx->pool(), args.S * args.H_v); + ggml_cuda_op_gdn_decode_fused_prenorm(*cuda_ctx, args, attn.get()); + ggml_cuda_mul_mat_vec_q_fq_gdn_gate(*cuda_ctx, mm, attn.get(), sig->src[0], args.norm_w_tensor, args.eps_rms); + return mm_idx - i; + } + } ggml_cuda_op_gdn_decode_fused(*cuda_ctx, args); return nodes_to_skip; } @@ -3572,10 +3823,15 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const bool bad_padding_clear = ggml_backend_buffer_get_usage(weights->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE && ggml_nbytes(weights) != ggml_backend_buffer_get_alloc_size(weights->buffer, weights) && weights->view_src; - const bool target_shape = node->ne[0] == 512 && next->ne[0] == 2048 && + const bool target_qwen36 = node->ne[0] == 512 && next->ne[0] == 2048 && (!has_ids || (gate->ne[1] == 8 && weights->ne[2] == 256)); + const bool target_qwen4exp = node->ne[0] == 640 && next->ne[0] == 2560 && + (!has_ids || (gate->ne[1] == 10 && weights->ne[2] == 512)); + const bool target_shape = target_qwen36 || target_qwen4exp; + const bool weight_type_ok = weights->type == GGML_TYPE_Q8_0 || + (target_qwen4exp && weights->type == GGML_TYPE_IQ4_NL); const bool shape_ok = target_shape && next->src[1] == node && gate->type == GGML_TYPE_F32 && up->type == GGML_TYPE_F32 && - node->type == GGML_TYPE_F32 && next->type == GGML_TYPE_F32 && weights->type == GGML_TYPE_Q8_0 && + node->type == GGML_TYPE_F32 && next->type == GGML_TYPE_F32 && weight_type_ok && ggml_are_same_shape(gate, up) && ggml_are_same_shape(gate, node) && gate->nb[0] == sizeof(float) && up->nb[0] == sizeof(float) && gate->ne[3] == 1 && (has_ids ? gate->ne[1] == ids->ne[0] && gate->ne[2] == ids->ne[1] : gate->ne[2] == 1); const bool use_mmq = shape_ok && !bad_padding_clear && GGML_CUDA_CC_IS_RDNA3_5(cc) && @@ -3720,38 +3976,64 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } - // Qwen3.5 shared expert gate for one token: mul_mat(w [2048, 1], y) -> sigmoid -> mul -> add -> add - if (node->op == GGML_OP_MUL_MAT && i + 4 < cgraph->n_nodes && node->type == GGML_TYPE_F32 && + // Shared expert gate for one token: mul_mat(w [n_embd, 1], y) -> sigmoid -> mul -> add [-> add] + // (Qwen3.5 adds the residual in a second add, qwen4exp folds the result into the hyper-connection instead) + if (node->op == GGML_OP_MUL_MAT && i + 3 < cgraph->n_nodes && node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32 && - node->src[0]->ne[0] == 2048 && ggml_nelements(node->src[0]) == 2048 && ggml_nelements(node->src[1]) == 2048 && + node->src[0]->ne[0] % 2 == 0 && ggml_nelements(node->src[0]) == node->src[0]->ne[0] && + ggml_nelements(node->src[1]) == node->src[0]->ne[0] && ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(node->src[1]) && ggml_nelements(node) == 1 && - ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_ADD }, { i + 4 }) && - ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID && cgraph->nodes[i + 1]->src[0] == node && + cgraph->nodes[i + 1]->op == GGML_OP_UNARY && ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID && + cgraph->nodes[i + 1]->src[0] == node && cgraph->nodes[i + 2]->op == GGML_OP_MUL && cgraph->nodes[i + 3]->op == GGML_OP_ADD && GGML_CUDA_CC_IS_AMD(ggml_cuda_info().devices[cuda_ctx->device].cc)) { - const ggml_tensor * sigmoid = cgraph->nodes[i + 1]; - ggml_tensor * mul = cgraph->nodes[i + 2]; - ggml_tensor * add0 = cgraph->nodes[i + 3]; - ggml_tensor * add1 = cgraph->nodes[i + 4]; - const ggml_tensor * gate = mul->src[0] == sigmoid ? mul->src[0] : mul->src[1]; - const ggml_tensor * src = gate == mul->src[0] ? mul->src[1] : mul->src[0]; - const ggml_tensor * other = add0->src[0]; - const ggml_tensor * residual = add1->src[1]; - const uintptr_t dst_begin = (uintptr_t) add1->data; - const uintptr_t dst_end = dst_begin + ggml_backend_buft_get_alloc_size(add1->buffer->buft, add1); - const auto safe_output_alias = [&](const ggml_tensor * input) { - const uintptr_t begin = (uintptr_t) input->data; - const uintptr_t end = begin + ggml_backend_buft_get_alloc_size(input->buffer->buft, input); - return !(begin < dst_end && dst_begin < end) || (input->data == add1->data && ggml_are_same_layout(input, add1)); - }; - if (gate == sigmoid && ggml_nelements(sigmoid) == 1 && add0->src[1] == mul && add1->src[0] == add0 && - other != mul && residual != mul && residual != add0 && - mul->type == GGML_TYPE_F32 && src->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && residual->type == GGML_TYPE_F32 && - mul->ne[0] == 2048 && ggml_nelements(mul) == 2048 && ggml_are_same_shape(mul, src) && ggml_are_same_shape(mul, other) && - ggml_are_same_shape(mul, residual) && ggml_are_same_shape(mul, add1) && - ggml_is_contiguous(src) && ggml_is_contiguous(other) && ggml_is_contiguous(residual) && ggml_is_contiguous(add1) && - safe_output_alias(src) && safe_output_alias(other) && safe_output_alias(residual)) { - ggml_cuda_op_shared_gate_mul_add(*cuda_ctx, node, src, other, residual, add1); - return 4; + // the fused kernel replays mul_mat_vec_f: only valid where mmvf would pick that block size + const int64_t ncols = node->src[0]->ne[0]; + const int warp_size = ggml_cuda_info().devices[cuda_ctx->device].warp_size; + int64_t block_size_best = warp_size; + int64_t niter_best = (ncols + 2*warp_size - 1) / (2*warp_size); + for (int64_t block_size = 2*warp_size; block_size <= 256; block_size += warp_size) { + const int64_t niter = (ncols + 2*block_size - 1) / (2*block_size); + if (niter < niter_best) { + niter_best = niter; + block_size_best = block_size; + } + } + + const bool two_adds = i + 4 < cgraph->n_nodes && cgraph->nodes[i + 4]->op == GGML_OP_ADD && + cgraph->nodes[i + 4]->src[0] == cgraph->nodes[i + 3]; + const bool subgraph_ok = block_size_best == 256 && warp_size == 32 && (two_adds ? + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_ADD }, { i + 4 }) : + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_UNARY, GGML_OP_MUL, GGML_OP_ADD }, { i + 3 })); + + if (subgraph_ok) { + const ggml_tensor * sigmoid = cgraph->nodes[i + 1]; + ggml_tensor * mul = cgraph->nodes[i + 2]; + ggml_tensor * add0 = cgraph->nodes[i + 3]; + ggml_tensor * add1 = two_adds ? cgraph->nodes[i + 4] : nullptr; + ggml_tensor * out = two_adds ? add1 : add0; + const ggml_tensor * gate = mul->src[0] == sigmoid ? mul->src[0] : mul->src[1]; + const ggml_tensor * src = gate == mul->src[0] ? mul->src[1] : mul->src[0]; + const ggml_tensor * other = add0->src[0] == mul ? add0->src[1] : add0->src[0]; + const ggml_tensor * residual = two_adds ? (add1->src[0] == add0 ? add1->src[1] : add1->src[0]) : nullptr; + const uintptr_t dst_begin = (uintptr_t) out->data; + const uintptr_t dst_end = dst_begin + ggml_backend_buft_get_alloc_size(out->buffer->buft, out); + const auto safe_output_alias = [&](const ggml_tensor * input) { + const uintptr_t begin = (uintptr_t) input->data; + const uintptr_t end = begin + ggml_backend_buft_get_alloc_size(input->buffer->buft, input); + return !(begin < dst_end && dst_begin < end) || (input->data == out->data && ggml_are_same_layout(input, out)); + }; + const bool residual_ok = !two_adds || (residual != mul && residual != add0 && residual->type == GGML_TYPE_F32 && + ggml_are_same_shape(mul, residual) && ggml_is_contiguous(residual) && safe_output_alias(residual)); + if (gate == sigmoid && ggml_nelements(sigmoid) == 1 && (add0->src[0] == mul || add0->src[1] == mul) && + other != mul && residual_ok && + mul->type == GGML_TYPE_F32 && src->type == GGML_TYPE_F32 && other->type == GGML_TYPE_F32 && + mul->ne[0] == ncols && ggml_nelements(mul) == ncols && ggml_are_same_shape(mul, src) && ggml_are_same_shape(mul, other) && + ggml_are_same_shape(mul, out) && + ggml_is_contiguous(src) && ggml_is_contiguous(other) && ggml_is_contiguous(out) && + safe_output_alias(src) && safe_output_alias(other) && safe_output_alias(node->src[1])) { + ggml_cuda_op_shared_gate_mul_add(*cuda_ctx, node, src, other, residual, out); + return two_adds ? 4 : 3; + } } } @@ -3788,8 +4070,51 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph } } - if (node->op == GGML_OP_MUL && node->type == GGML_TYPE_F32 && node->ne[0] == 2048 && - node->ne[3] == 1 && node->ne[1] == 8) { + // qwen4exp IQ4_NL routed down projection followed by the 10-expert weighted sum. Compute all selected + // experts for one output row in a wave and apply their routing weights immediately, avoiding the + // [n_embd, n_used] intermediate and its separate reduction launch. + if (getenv("GGML_CUDA_DISABLE_WEIGHTED_DOWN") == nullptr && + node->op == GGML_OP_MUL_MAT_ID && i + 20 < cgraph->n_nodes && cgraph->nodes[i + 1]->op == GGML_OP_MUL) { + constexpr int n_used = 10; + constexpr int n_ops = 2 + n_used + (n_used - 1); + ggml_tensor * mul = cgraph->nodes[i + 1]; + const ggml_tensor * experts = ggml_are_same_shape(mul, mul->src[0]) ? mul->src[0] : mul->src[1]; + const ggml_tensor * weights = experts == mul->src[0] ? mul->src[1] : mul->src[0]; + const int output_idx = i + n_ops - 1; + ggml_tensor * output = cgraph->nodes[output_idx]; + bool valid = experts == node && node->ne[1] == n_used && + ggml_cuda_mul_mat_id_weighted_rdna3_5_ok(experts, weights, output); + + for (int j = 0; valid && j < n_used; ++j) { + const ggml_tensor * view = cgraph->nodes[i + 2 + j]; + valid = view->op == GGML_OP_VIEW && view->src[0] == mul && + view->ne[0] == mul->ne[0] && view->ne[1] == mul->ne[2] && view->ne[2] == 1 && view->ne[3] == 1 && + view->nb[0] == sizeof(float) && view->nb[1] == mul->nb[2] && view->view_offs == size_t(j) * mul->nb[1]; + } + const int add_start = i + 2 + n_used; + if (valid) { + const ggml_tensor * first = cgraph->nodes[add_start]; + valid = first->op == GGML_OP_ADD && first->src[0] == cgraph->nodes[i + 2] && first->src[1] == cgraph->nodes[i + 3]; + } + for (int j = 2; valid && j < n_used; ++j) { + const ggml_tensor * add = cgraph->nodes[add_start + j - 1]; + valid = add->op == GGML_OP_ADD && add->src[0] == cgraph->nodes[add_start + j - 2] && add->src[1] == cgraph->nodes[i + 2 + j]; + } + if (valid) { + std::vector ops = { GGML_OP_MUL_MAT_ID, GGML_OP_MUL }; + ops.insert(ops.end(), n_used, GGML_OP_VIEW); + ops.insert(ops.end(), n_used - 1, GGML_OP_ADD); + const int out_nodes[] = { output_idx }; + if (ggml_can_fuse_subgraph(cgraph, i, n_ops, ops.data(), out_nodes, 1) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, n_ops, out_nodes, 1)) { + ggml_cuda_mul_mat_id_weighted_rdna3_5(*cuda_ctx, experts, weights, output); + return n_ops - 1; + } + } + } + + if (node->op == GGML_OP_MUL && node->type == GGML_TYPE_F32 && node->ne[0] % 4 == 0 && + node->ne[3] == 1 && node->ne[1] >= 2 && node->ne[1] <= 16) { const int n_expert_used = node->ne[1]; const int n_ops = 2 * n_expert_used; if (i + n_ops <= cgraph->n_nodes) { @@ -4434,6 +4759,207 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 1; } + // qwen4exp hyper-connection stream mix: + // sigmoid(gate) -> mul(xn, .) -> reshape -> view(stream 0) -> cont -> (view(stream c) -> add) x (hc-1) -> scale + // one kernel reads xn/gate once and writes the [n_embd, T] mean of the gated streams + { + ggml_cuda_hc_mix_args args; + enum ggml_op ops[5 + 2 * 15 + 1]; + int n_ops = ggml_cuda_match_hc_mix(cgraph, i, args, ops); + if (n_ops > 0 && args.dst->ne[1] == 1 && getenv("GGML_CUDA_ENABLE_HCMIX") == nullptr) { + n_ops = 0; // decode: preserve bit-identical routing; prefill retains the exact fused reduction + } + if (n_ops > 0) { + const int out_nodes[] = { i + n_ops - 1 }; + if (ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1)) { + ggml_cuda_op_hc_mix_reduce(*cuda_ctx, args); + return n_ops - 1; + } + } + } + + // qwen4exp hyper-connection combine + next grouped norm: + // scale(inject) -> sigmoid -> scale -> [views] -> repeat(block_out) -> mul -> add(residual, .) -> rms_norm -> mul(gamma) + // one kernel: block per (stream, token), residual update + norm + if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 8 < cgraph->n_nodes && + cgraph->nodes[i + 1]->op == GGML_OP_UNARY && ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID && + cgraph->nodes[i + 1]->src[0] == node && + cgraph->nodes[i + 2]->op == GGML_OP_SCALE && cgraph->nodes[i + 2]->src[0] == cgraph->nodes[i + 1]) { + const ggml_tensor * scale1 = node; + const ggml_tensor * scale2 = cgraph->nodes[i + 2]; + const ggml_tensor * inject = scale1->src[0]; + + int j = i + 3; + while (j < cgraph->n_nodes && j < i + 7 && ggml_cuda_is_view_or_noop(cgraph->nodes[j]) && !ggml_is_empty(cgraph->nodes[j])) { + j++; + } + + if (j + 4 < cgraph->n_nodes && + cgraph->nodes[j]->op == GGML_OP_REPEAT && cgraph->nodes[j + 1]->op == GGML_OP_MUL && cgraph->nodes[j + 2]->op == GGML_OP_ADD && + cgraph->nodes[j + 3]->op == GGML_OP_RMS_NORM && cgraph->nodes[j + 4]->op == GGML_OP_MUL) { + const ggml_tensor * repeat = cgraph->nodes[j]; + const ggml_tensor * mul = cgraph->nodes[j + 1]; + ggml_tensor * add = cgraph->nodes[j + 2]; + const ggml_tensor * rms = cgraph->nodes[j + 3]; + ggml_tensor * mulg = cgraph->nodes[j + 4]; + const ggml_tensor * b = repeat->src[0]; + const ggml_tensor * w = mul->src[0] == repeat ? mul->src[1] : mul->src[0]; + const ggml_tensor * res = add->src[0] == mul ? add->src[1] : add->src[0]; + const ggml_tensor * gamma = mulg->src[0] == rms ? mulg->src[1] : mulg->src[0]; + + const int64_t n_embd = add->ne[0]; + const int64_t hc = add->ne[1]; + const int64_t n_tok = add->ne[2]; + + const bool w_ok = (w == scale2 || (ggml_cuda_is_view_or_noop(w) && w->view_src == scale2)) && + w->type == GGML_TYPE_F32 && ggml_is_contiguous(w) && ggml_nelements(w) == hc * n_tok && w->ne[0] == 1 && w->ne[1] == hc; + const bool b_ok = b->type == GGML_TYPE_F32 && ggml_is_contiguous(b) && ggml_nelements(b) == n_embd * n_tok && + b->ne[0] == n_embd && b->ne[1] == 1; + const bool ok = add->ne[3] == 1 && n_tok <= 65535 && + inject->type == GGML_TYPE_F32 && ggml_is_contiguous(inject) && ggml_nelements(inject) == hc * n_tok && + ggml_nrows(inject) == n_tok && + ggml_are_same_shape(scale1, inject) && ggml_are_same_shape(scale2, inject) && + (mul->src[0] == repeat || mul->src[1] == repeat) && w != repeat && + (add->src[0] == mul || add->src[1] == mul) && res != mul && + rms->src[0] == add && (mulg->src[0] == rms || mulg->src[1] == rms) && gamma != rms && + scale2->type == GGML_TYPE_F32 && repeat->type == GGML_TYPE_F32 && + mul->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && res->type == GGML_TYPE_F32 && + rms->type == GGML_TYPE_F32 && mulg->type == GGML_TYPE_F32 && gamma->type == GGML_TYPE_F32 && + ggml_are_same_shape(repeat, add) && ggml_are_same_shape(mul, add) && ggml_are_same_shape(res, add) && + ggml_are_same_shape(rms, add) && ggml_are_same_shape(mulg, add) && + ggml_is_contiguous(res) && ggml_is_contiguous(add) && ggml_is_contiguous(mulg) && ggml_is_contiguous(gamma) && + ggml_nelements(gamma) == n_embd * hc && gamma->ne[0] == n_embd && gamma->ne[1] == hc && w_ok && b_ok; + + if (ok) { + ggml_cuda_hc_combine_norm_args args; + args.inject = inject; + args.residual = res; + args.block_out = b; + args.gamma = gamma; + args.out_res = add; + args.out_xn = mulg; + args.s1 = ggml_get_op_params_f32(scale1, 0); + args.b1 = ggml_get_op_params_f32(scale1, 1); + args.s2 = ggml_get_op_params_f32(scale2, 0); + args.b2 = ggml_get_op_params_f32(scale2, 1); + memcpy(&args.eps, rms->op_params, sizeof(float)); + + // every block reads block_out and inject and writes its own stream of both outputs: the outputs + // must not overlap those, and the residual may only alias the first output in place + auto overlap = [](const ggml_tensor * p, const ggml_tensor * q) { + const uintptr_t p0 = (uintptr_t) p->data, p1 = p0 + ggml_backend_buft_get_alloc_size(p->buffer->buft, p); + const uintptr_t q0 = (uintptr_t) q->data, q1 = q0 + ggml_backend_buft_get_alloc_size(q->buffer->buft, q); + return p0 < q1 && q0 < p1; + }; + const bool res_inplace = res->data == add->data && ggml_are_same_layout(res, add); + bool alias_ok = !overlap(add, b) && !overlap(add, inject) && (res_inplace || !overlap(add, res)) && + !overlap(mulg, b) && !overlap(mulg, inject) && !overlap(mulg, res) && !overlap(mulg, add); + // outputs overlapping the inputs (typically xn reusing the block_out buffer, which dies at the + // combine): single-block variant that reads everything before it writes + if (!alias_ok && n_tok == 1 && hc <= 4) { + args.single_block = true; + alias_ok = true; + } + + int idxs[16]; + enum ggml_op ops[16]; + int count = 0; + for (int k = i; k <= j + 4; ++k) { + const ggml_tensor * n = cgraph->nodes[k]; + if (ggml_cuda_is_view_or_noop(n) && n->view_src != scale2) { + continue; + } + idxs[count] = k; + ops[count] = n->op; + count++; + } + const int out_nodes[] = { j + 2, j + 4 }; + if (alias_ok && ggml_cuda_hc_combine_norm_supported(args, ggml_cuda_info().devices[cuda_ctx->device].warp_size) && + ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, out_nodes, 2)) { + ggml_cuda_op_hc_combine_norm(*cuda_ctx, args); + return j + 4 - i; + } + } + } + } + + // qwen4exp hyper-connection combine: + // scale(inject) -> sigmoid -> scale -> [reshape] -> repeat(block_out) -> mul -> add(residual, .) + // one kernel writes residual + block_out (x) (s2*sigmoid(s1*inject)) + if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 6 < cgraph->n_nodes && + cgraph->nodes[i + 1]->op == GGML_OP_UNARY && ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID && + cgraph->nodes[i + 2]->op == GGML_OP_SCALE) { + const ggml_tensor * scale1 = node; + const ggml_tensor * sigmoid = cgraph->nodes[i + 1]; + const ggml_tensor * scale2 = cgraph->nodes[i + 2]; + const ggml_tensor * inject = scale1->src[0]; + + // a few view nodes follow: the scatter weights [1, hc, T] and the block output [n_embd, 1, T] + int j = i + 3; + while (j < cgraph->n_nodes && j < i + 7 && ggml_cuda_is_view_or_noop(cgraph->nodes[j]) && !ggml_is_empty(cgraph->nodes[j])) { + j++; + } + + if (j + 2 < cgraph->n_nodes && + cgraph->nodes[j]->op == GGML_OP_REPEAT && cgraph->nodes[j + 1]->op == GGML_OP_MUL && cgraph->nodes[j + 2]->op == GGML_OP_ADD) { + const ggml_tensor * repeat = cgraph->nodes[j]; + const ggml_tensor * mul = cgraph->nodes[j + 1]; + ggml_tensor * add = cgraph->nodes[j + 2]; + const ggml_tensor * b = repeat->src[0]; + const ggml_tensor * w = mul->src[0] == repeat ? mul->src[1] : mul->src[0]; + const ggml_tensor * res = add->src[0] == mul ? add->src[1] : add->src[0]; + + const int64_t n_embd = add->ne[0]; + const int64_t hc = add->ne[1]; + const int64_t n_tokens = add->ne[2]; + + const bool w_ok = (w == scale2 || (ggml_cuda_is_view_or_noop(w) && w->view_src == scale2)) && + w->type == GGML_TYPE_F32 && ggml_is_contiguous(w) && + w->ne[0] == 1 && w->ne[1] == hc && w->ne[2] == n_tokens && w->ne[3] == 1; + const bool b_ok = b->type == GGML_TYPE_F32 && ggml_is_contiguous(b) && + b->ne[0] == n_embd && b->ne[1] == 1 && b->ne[2] == n_tokens && b->ne[3] == 1; + const bool ok = sigmoid->src[0] == scale1 && scale2->src[0] == sigmoid && + (mul->src[0] == repeat || mul->src[1] == repeat) && w != repeat && + (add->src[0] == mul || add->src[1] == mul) && res != mul && + sigmoid->type == GGML_TYPE_F32 && scale2->type == GGML_TYPE_F32 && inject->type == GGML_TYPE_F32 && + repeat->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && res->type == GGML_TYPE_F32 && + ggml_is_contiguous(inject) && ggml_nelements(inject) == hc * n_tokens && ggml_nrows(inject) == n_tokens && + ggml_are_same_shape(scale1, inject) && ggml_are_same_shape(sigmoid, inject) && ggml_are_same_shape(scale2, inject) && + ggml_are_same_shape(repeat, add) && ggml_are_same_shape(mul, add) && ggml_are_same_shape(res, add) && + add->ne[3] == 1 && ggml_is_contiguous(res) && ggml_is_contiguous(add) && w_ok && b_ok; + + if (ok) { + // the block-output reshape views an external tensor, so it is left out of the subgraph check + int idxs[12]; + enum ggml_op ops[12]; + int count = 0; + for (int k = i; k <= j + 2; ++k) { + const ggml_tensor * n = cgraph->nodes[k]; + if (ggml_cuda_is_view_or_noop(n) && n->view_src != scale2) { + continue; + } + idxs[count] = k; + ops[count] = n->op; + count++; + } + const int out_nodes[] = { j + 2 }; + if (ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, out_nodes, 1)) { + ggml_cuda_hc_combine_args args; + args.residual = res; + args.block_out = b; + args.inject = inject; + args.dst = add; + args.s1 = ggml_get_op_params_f32(scale1, 0); + args.b1 = ggml_get_op_params_f32(scale1, 1); + args.s2 = ggml_get_op_params_f32(scale2, 0); + args.b2 = ggml_get_op_params_f32(scale2, 1); + ggml_cuda_op_hc_combine(*cuda_ctx, args); + return j + 2 - i; + } + } + } + } + if (node->op == GGML_OP_CONT && i + 2 < cgraph->n_nodes) { ggml_tensor * unary = cgraph->nodes[i + 1]; ggml_tensor * mul = cgraph->nodes[i + 2]; @@ -4469,6 +4995,54 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph return 2; } + // scale -> silu/sigmoid -> quantized matvec (decode): apply the activation while the matvec quantizes it + if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 2 < cgraph->n_nodes && + cgraph->nodes[i + 1]->op == GGML_OP_UNARY && + (ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SILU || ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID) && + cgraph->nodes[i + 2]->op == GGML_OP_MUL_MAT && cgraph->nodes[i + 2]->src[1] == cgraph->nodes[i + 1] && + cgraph->nodes[i + 1]->src[0] == node && cgraph->nodes[i + 1]->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(node->src[0]) && ggml_are_same_shape(node->src[0], cgraph->nodes[i + 1]) && + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_MUL_MAT }, { i + 2 }) && + ggml_cuda_mul_mat_vec_q_fq_prologue_ok(*cuda_ctx, cgraph->nodes[i + 2], node->src[0])) { + const int y_op = ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SILU ? 1 : 2; + ggml_tensor * mm = cgraph->nodes[i + 2]; + + // qwen4exp hyper-connection: the matvec is the up projection whose result only feeds the stream mix + // (sigmoid -> mul(xn) -> ... -> scale): one kernel computes the gate rows of each element and the mix + if (getenv("GGML_CUDA_ENABLE_HCMIX") != nullptr && + i + 3 < cgraph->n_nodes && cgraph->nodes[i + 3]->src[0] == mm) { + ggml_cuda_hc_mix_args mix; + enum ggml_op ops[3 + 5 + 2 * 15 + 1]; + const int n_mix = ggml_cuda_match_hc_mix(cgraph, i + 3, mix, ops + 3); + if (n_mix > 0 && mix.gate == mm && + ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(mm->src[0], node->src[0], mix.xn, mix.dst, mix.hc)) { + ops[0] = GGML_OP_SCALE; ops[1] = GGML_OP_UNARY; ops[2] = GGML_OP_MUL_MAT; + const int out_nodes[] = { i + 3 + n_mix - 1 }; + // (aliasing of the output with y / xn is checked by ggml_cuda_mul_mat_vec_q_fq_hcmix_ok) + if (ggml_can_fuse_subgraph(cgraph, i, 3 + n_mix, ops, out_nodes, 1)) { + ggml_cuda_mul_mat_vec_q_fq_hcmix(*cuda_ctx, mm->src[0], node->src[0], mix.xn, mix.dst, mix.hc, + ggml_get_op_params_f32(node, 0), ggml_get_op_params_f32(node, 1), y_op, mix.scale, mix.bias); + return 2 + n_mix; + } + } + } + + ggml_cuda_mul_mat_vec_q_fq_prologue(*cuda_ctx, mm, node->src[0], + ggml_get_op_params_f32(node, 0), ggml_get_op_params_f32(node, 1), y_op); + return 2; + } + + // scale -> silu/sigmoid (e.g. the qwen4exp hyper-connection low-rank gate: silu(x / hc)) + if (node->op == GGML_OP_SCALE && node->type == GGML_TYPE_F32 && i + 1 < cgraph->n_nodes && + cgraph->nodes[i + 1]->op == GGML_OP_UNARY && + (ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SILU || ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SIGMOID) && + cgraph->nodes[i + 1]->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && + ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(cgraph->nodes[i + 1]) && + ggml_can_fuse(cgraph, i, { GGML_OP_SCALE, GGML_OP_UNARY })) { + ggml_cuda_op_scale_unary(*cuda_ctx, node, cgraph->nodes[i + 1]); + return 1; + } + return 0; } @@ -4694,6 +5268,11 @@ static void ggml_cuda_graph_evaluate_and_capture(ggml_backend_cuda_context * cud static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, const void * graph_key) { ggml_cuda_graph * graph = cuda_ctx->cuda_graph(graph_key); + static const bool disable_graphs_env = getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr && atoi(getenv("GGML_CUDA_DISABLE_GRAPHS")) != 0; + if (disable_graphs_env) { + graph->disable_due_to_gpu_arch = true; + } + if (graph->graph == nullptr) { if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) { if (!graph->disable_due_to_gpu_arch) { diff --git a/ggml/src/ggml-cuda/hyperconn.cu b/ggml/src/ggml-cuda/hyperconn.cu new file mode 100644 index 000000000000..b57bc966a915 --- /dev/null +++ b/ggml/src/ggml-cuda/hyperconn.cu @@ -0,0 +1,366 @@ +#include "hyperconn.cuh" +#include "ggml-backend-impl.h" + +// Explicitly rounded mul/add: keeps the fused kernels from being contracted into FMAs so that +// the results match the separate MUL and ADD kernels of the unfused graph bit for bit. +#if defined(__HIP_PLATFORM_AMD__) +static __device__ __forceinline__ float hc_mul_rn(const float a, const float b) { + float result; + asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +static __device__ __forceinline__ float hc_add_rn(const float a, const float b) { + float result; + asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} +#else +static __device__ __forceinline__ float hc_mul_rn(const float a, const float b) { + return __fmul_rn(a, b); +} + +static __device__ __forceinline__ float hc_add_rn(const float a, const float b) { + return __fadd_rn(a, b); +} +#endif + +// same expression as op_sigmoid in unary.cu +static __device__ __forceinline__ float hc_sigmoid(const float x) { + return 1.0f / (1.0f + expf(-x)); +} + +// mixed[e,t] = scale * ( ((xn*sig)[0] + (xn*sig)[1]) + ... ) + bias, streams summed in order 0..hc-1 +static __global__ void hc_mix_reduce_f32( + const float * __restrict__ xn, const float * __restrict__ gate, float * __restrict__ dst, + const int64_t n_embd, const int64_t n_tokens, const int hc, const float scale, const float bias) { + const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= n_embd * n_tokens) { + return; + } + + const int64_t t = index / n_embd; + const int64_t e = index - t * n_embd; + const int64_t hc_dim = n_embd * hc; + const int64_t base = t * hc_dim + e; + + float acc = hc_mul_rn(xn[base], hc_sigmoid(gate[base])); + for (int c = 1; c < hc; ++c) { + const int64_t i = base + int64_t(c) * n_embd; + acc = hc_add_rn(acc, hc_mul_rn(xn[i], hc_sigmoid(gate[i]))); + } + + // identical expression to scale_f32 in scale.cu + dst[index] = scale * acc + bias; +} + +// out[e,c,t] = residual[e,c,t] + block_out[e,t] * ( s2 * sigmoid(s1 * inject[c,t] + b1) + b2 ) +static __global__ void hc_combine_f32( + const float * __restrict__ residual, const float * __restrict__ block_out, const float * __restrict__ inject, + float * __restrict__ dst, const int64_t n_embd, const int hc, const int64_t n_tokens, + const float s1, const float b1, const float s2, const float b2) { + const int64_t index = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; + const int64_t hc_dim = n_embd * hc; + if (index >= hc_dim * n_tokens) { + return; + } + + const int64_t t = index / hc_dim; + const int64_t rem = index - t * hc_dim; + const int64_t c = rem / n_embd; + const int64_t e = rem - c * n_embd; + + // identical expressions to scale_f32 (scale.cu) and op_sigmoid (unary.cu) + const float x1 = s1 * inject[t * hc + c] + b1; + const float x2 = hc_sigmoid(x1); + const float w = s2 * x2 + b2; + + const float m = hc_mul_rn(block_out[t * n_embd + e], w); + dst[index] = hc_add_rn(residual[index], m); +} + +static bool hc_ranges_overlap(const ggml_tensor * a, const ggml_tensor * b) { + const char * a0 = (const char *) a->data; + const char * a1 = a0 + ggml_backend_buft_get_alloc_size(a->buffer->buft, a); + const char * b0 = (const char *) b->data; + const char * b1 = b0 + ggml_backend_buft_get_alloc_size(b->buffer->buft, b); + return a0 < b1 && b0 < a1; +} + +void ggml_cuda_op_hc_mix_reduce(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_mix_args & args) { + const ggml_tensor * xn = args.xn; + const ggml_tensor * gate = args.gate; + ggml_tensor * dst = args.dst; + + GGML_ASSERT(xn->type == GGML_TYPE_F32 && gate->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(xn) && ggml_is_contiguous(gate) && ggml_is_contiguous(dst)); + GGML_ASSERT(args.hc > 0); + + const int64_t n_embd = dst->ne[0]; + const int64_t n_tokens = ggml_nrows(dst); + GGML_ASSERT(xn->ne[0] == n_embd * args.hc && ggml_nrows(xn) == n_tokens); + GGML_ASSERT(ggml_are_same_shape(xn, gate)); + + // The graph allocator may reuse the buffer of the (elided) sigmoid/mul intermediates, which can + // coincide with the gate matmul output. Every thread reads hc strided elements, so an aliased + // output has to be staged -- unless (single token) the output lands on a whole stream of the + // aliased input: element e of the output then overwrites element e of that stream, which only + // the same thread reads (after it has finished reading). + auto stream_aligned_alias = [&](const ggml_tensor * in) { + if (!hc_ranges_overlap(dst, in)) { + return true; + } + if (n_tokens != 1) { + return false; + } + const ptrdiff_t off = (const char *) dst->data - (const char *) in->data; + return off >= 0 && off % (n_embd * (ptrdiff_t) sizeof(float)) == 0 && + off + (ptrdiff_t) ggml_nbytes(dst) <= (ptrdiff_t) ggml_nbytes(in); + }; + const bool stage = !stream_aligned_alias(xn) || !stream_aligned_alias(gate); + + ggml_cuda_pool_alloc staged(ctx.pool()); + float * out = stage ? staged.alloc(n_embd * n_tokens) : (float *) dst->data; + + const int64_t n_items = n_embd * n_tokens; + const int threads = 256; + const int blocks = (int) ((n_items + threads - 1) / threads); + const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream()); + ggml_cuda_kernel_launch(hc_mix_reduce_f32, launch_params, + (const float *) xn->data, (const float *) gate->data, out, n_embd, n_tokens, args.hc, args.scale, args.bias); + + if (stage) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, out, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream())); + } +} + +void ggml_cuda_op_hc_combine(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_args & args) { + const ggml_tensor * residual = args.residual; + const ggml_tensor * block_out = args.block_out; + const ggml_tensor * inject = args.inject; + ggml_tensor * dst = args.dst; + + GGML_ASSERT(residual->type == GGML_TYPE_F32 && block_out->type == GGML_TYPE_F32 && + inject->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(residual) && ggml_is_contiguous(block_out) && + ggml_is_contiguous(inject) && ggml_is_contiguous(dst)); + + const int64_t n_embd = dst->ne[0]; + const int64_t hc = dst->ne[1]; + const int64_t n_tokens = dst->ne[2] * dst->ne[3]; + GGML_ASSERT(ggml_are_same_shape(residual, dst)); + GGML_ASSERT(ggml_nelements(block_out) == n_embd * n_tokens); + GGML_ASSERT(ggml_nelements(inject) == hc * n_tokens); + + // residual may be the in-place source of the add (same address, same layout): each thread then + // reads and writes its own element only. Any other overlap needs staging. + const bool residual_inplace = residual->data == dst->data && ggml_are_same_layout(residual, dst); + const bool stage = (!residual_inplace && hc_ranges_overlap(dst, residual)) || + hc_ranges_overlap(dst, block_out) || hc_ranges_overlap(dst, inject); + + ggml_cuda_pool_alloc staged(ctx.pool()); + float * out = stage ? staged.alloc(ggml_nelements(dst)) : (float *) dst->data; + + const int64_t n_items = ggml_nelements(dst); + const int threads = 256; + const int blocks = (int) ((n_items + threads - 1) / threads); + const ggml_cuda_kernel_launch_params launch_params(blocks, threads, 0, ctx.stream()); + ggml_cuda_kernel_launch(hc_combine_f32, launch_params, + (const float *) residual->data, (const float *) block_out->data, (const float *) inject->data, out, + n_embd, (int) hc, n_tokens, args.s1, args.b1, args.s2, args.b2); + + if (stage) { + CUDA_CHECK(cudaMemcpyAsync(dst->data, out, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, ctx.stream())); + } +} + +// --------------------------------------------------------------------------------------------------------- +// combine + grouped rms norm, one token: block per stream, 1024 threads (the rms_norm_f32<1024> layout) +// w = s2 * sigmoid(s1 * inject[c] + b1) + b2 +// res[e,c] = residual[e,c] + block_out[e] * w +// xn[e,c] = rms_norm(res[:,c]) * gamma[e,c] + +#define HC_CN_BLOCK 1024 +#define HC_CN_MAX_EMB 3072 + +static __global__ void __launch_bounds__(HC_CN_BLOCK, 1) hc_combine_norm_f32( + const float * __restrict__ inject, const float * __restrict__ residual, + const float * __restrict__ block_out, const float * __restrict__ gamma, + float * __restrict__ out_res, float * __restrict__ out_xn, + const int n_embd, const float s1, const float b1, const float s2, const float b2, const float eps) { + __shared__ float s_sum[32]; + + const int c = blockIdx.x; // stream + const int t = blockIdx.y; // token + const int hc = gridDim.x; + const int tid = threadIdx.x; + + // identical expressions to scale_f32 / op_sigmoid / scale_f32 + const float x1 = s1 * inject[(int64_t) t * hc + c] + b1; + const float x2 = hc_sigmoid(x1); + const float w = s2 * x2 + b2; + + const int64_t row = (int64_t) t * hc + c; + const float * res = residual + row * n_embd; + float * dst = out_res + row * n_embd; + block_out += (int64_t) t * n_embd; + float xs[3]; + float tmp = 0.0f; +#pragma unroll + for (int k = 0; k < 3; ++k) { + const int col = tid + k * HC_CN_BLOCK; + xs[k] = 0.0f; + if (col < n_embd) { + const float m = hc_mul_rn(block_out[col], w); + const float xi = hc_add_rn(res[col], m); + dst[col] = xi; + xs[k] = xi; + tmp += xi * xi; + } + } + + tmp = block_reduce(tmp, s_sum); + + const float mean = tmp / n_embd; + const float scale = rsqrtf(mean + eps); + + const float * g = gamma + (int64_t) c * n_embd; + float * xn = out_xn + row * n_embd; +#pragma unroll + for (int k = 0; k < 3; ++k) { + const int col = tid + k * HC_CN_BLOCK; + if (col < n_embd) { + xn[col] = scale * xs[k] * g[col]; + } + } +} + +// Same computation for one token with all streams in a single block, streams processed one after the other +// with exactly the per-stream expressions and block reduction above. Every input element is loaded into +// registers before the first store, so the outputs may alias any of the inputs (the graph allocator reuses +// the block_out buffer for xn when block_out dies at the combine). +#define HC_CN_SINGLE_MAX_HC 4 + +static __global__ void __launch_bounds__(HC_CN_BLOCK, 1) hc_combine_norm_single_f32( + const float * inject, const float * residual, const float * block_out, const float * __restrict__ gamma, + float * out_res, float * out_xn, + const int n_embd, const int hc, const float s1, const float b1, const float s2, const float b2, const float eps) { + __shared__ float s_sum[HC_CN_SINGLE_MAX_HC][32]; + + const int t = blockIdx.x; // token + const int tid = threadIdx.x; + const int warp_id = tid / WARP_SIZE; + const int lane_id = tid % WARP_SIZE; + + // 1. every input element of the token into registers + float w[HC_CN_SINGLE_MAX_HC]; +#pragma unroll + for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) { + w[c] = 0.0f; + if (c < hc) { + // identical expressions to scale_f32 / op_sigmoid / scale_f32 + const float x1 = s1 * inject[(int64_t) t * hc + c] + b1; + const float x2 = hc_sigmoid(x1); + w[c] = s2 * x2 + b2; + } + } + + block_out += (int64_t) t * n_embd; + float xs[HC_CN_SINGLE_MAX_HC][3]; + float tmp[HC_CN_SINGLE_MAX_HC]; +#pragma unroll + for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) { + tmp[c] = 0.0f; + } +#pragma unroll + for (int k = 0; k < 3; ++k) { + const int col = tid + k * HC_CN_BLOCK; + const float bo = col < n_embd ? block_out[col] : 0.0f; +#pragma unroll + for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) { + xs[c][k] = 0.0f; + if (c < hc && col < n_embd) { + const float m = hc_mul_rn(bo, w[c]); + const float xi = hc_add_rn(residual[((int64_t) t * hc + c) * n_embd + col], m); + xs[c][k] = xi; + tmp[c] += xi * xi; + } + } + } + __syncthreads(); // all loads of the block have completed before the first store + + // 2. per-stream block sums, same reduction tree as block_reduce for each stream +#pragma unroll + for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) { + tmp[c] = warp_reduce_sum(tmp[c]); + } + if (lane_id == 0) { +#pragma unroll + for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) { + s_sum[c][warp_id] = tmp[c]; + } + } + __syncthreads(); +#pragma unroll + for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) { + tmp[c] = lane_id < HC_CN_BLOCK / WARP_SIZE ? s_sum[c][lane_id] : 0.0f; + tmp[c] = warp_reduce_sum(tmp[c]); + } + + // 3. stores +#pragma unroll + for (int c = 0; c < HC_CN_SINGLE_MAX_HC; ++c) { + if (c < hc) { + const float mean = tmp[c] / n_embd; + const float scale = rsqrtf(mean + eps); + const int64_t row = (int64_t) t * hc + c; + float * dst = out_res + row * n_embd; + float * xn = out_xn + row * n_embd; + const float * g = gamma + (int64_t) c * n_embd; +#pragma unroll + for (int k = 0; k < 3; ++k) { + const int col = tid + k * HC_CN_BLOCK; + if (col < n_embd) { + dst[col] = xs[c][k]; + xn[col] = scale * xs[c][k] * g[col]; + } + } + } + } +} + +bool ggml_cuda_hc_combine_norm_supported(const ggml_cuda_hc_combine_norm_args & a, const int warp_size) { + const int64_t n_embd = a.out_res->ne[0]; + const int64_t hc = a.out_res->ne[1]; + return warp_size == 32 && n_embd >= 1024 && n_embd <= HC_CN_MAX_EMB && hc >= 1 && hc <= 16 && + ggml_nelements(a.gamma) == n_embd * hc; +} + +void ggml_cuda_op_hc_combine_norm(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_norm_args & a) { + const int64_t n_embd = a.out_res->ne[0]; + const int64_t hc = a.out_res->ne[1]; + const int64_t n_tokens = a.out_res->ne[2] * a.out_res->ne[3]; + + GGML_ASSERT(a.out_res->ne[3] == 1); + GGML_ASSERT(ggml_are_same_shape(a.out_res, a.out_xn) && ggml_are_same_shape(a.out_res, a.residual)); + GGML_ASSERT(ggml_nelements(a.block_out) == n_embd * n_tokens && ggml_nelements(a.gamma) == n_embd * hc && + ggml_nelements(a.inject) == hc * n_tokens); + + if (a.single_block) { + GGML_ASSERT(hc <= HC_CN_SINGLE_MAX_HC); + const ggml_cuda_kernel_launch_params launch_params(dim3((int) n_tokens, 1, 1), HC_CN_BLOCK, 0, ctx.stream()); + ggml_cuda_kernel_launch(hc_combine_norm_single_f32, launch_params, + (const float *) a.inject->data, (const float *) a.residual->data, + (const float *) a.block_out->data, (const float *) a.gamma->data, + (float *) a.out_res->data, (float *) a.out_xn->data, + (int) n_embd, (int) hc, a.s1, a.b1, a.s2, a.b2, a.eps); + return; + } + + const ggml_cuda_kernel_launch_params launch_params(dim3((int) hc, (int) n_tokens, 1), HC_CN_BLOCK, 0, ctx.stream()); + ggml_cuda_kernel_launch(hc_combine_norm_f32, launch_params, + (const float *) a.inject->data, (const float *) a.residual->data, + (const float *) a.block_out->data, (const float *) a.gamma->data, + (float *) a.out_res->data, (float *) a.out_xn->data, + (int) n_embd, a.s1, a.b1, a.s2, a.b2, a.eps); +} diff --git a/ggml/src/ggml-cuda/hyperconn.cuh b/ggml/src/ggml-cuda/hyperconn.cuh new file mode 100644 index 000000000000..b510c2d632ed --- /dev/null +++ b/ggml/src/ggml-cuda/hyperconn.cuh @@ -0,0 +1,55 @@ +#pragma once + +#include "common.cuh" + +// Fused hyper-connection (DeepSeek-V4 / qwen4exp style) decode helpers. +// +// hc_mix_reduce: mixed[e,t] = scale * sum_c ( xn[c*n_embd + e, t] * sigmoid(gate[c*n_embd + e, t]) ) + bias +// replaces: UNARY(SIGMOID) -> MUL -> RESHAPE -> VIEW -> CONT -> (VIEW -> ADD) x (hc-1) -> SCALE +// +// hc_combine: out[e,c,t] = residual[e,c,t] + block_out[e,t] * ( s2 * sigmoid(s1 * inject[c,t] + b1) + b2 ) +// replaces: SCALE -> UNARY(SIGMOID) -> SCALE -> RESHAPE* -> REPEAT -> MUL -> ADD +// +// Both replay the unfused rounding sequence exactly (separate mul and add roundings), so results are +// bit-identical to the individual kernels. + +struct ggml_cuda_hc_mix_args { + const ggml_tensor * xn; // [hc*n_embd, T] F32 contiguous + const ggml_tensor * gate; // [hc*n_embd, T] F32 contiguous (pre-sigmoid) + ggml_tensor * dst; // [n_embd, T] F32 contiguous + int hc; + float scale; + float bias; +}; + +struct ggml_cuda_hc_combine_args { + const ggml_tensor * residual; // [n_embd, hc, T] F32 contiguous + const ggml_tensor * block_out; // [n_embd, T] F32 contiguous (or a reshape of it) + const ggml_tensor * inject; // [hc, T] F32 contiguous (pre-scale/sigmoid) + ggml_tensor * dst; // [n_embd, hc, T] F32 contiguous + float s1, b1; // first SCALE + float s2, b2; // second SCALE +}; + +void ggml_cuda_op_hc_mix_reduce(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_mix_args & args); +void ggml_cuda_op_hc_combine(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_args & args); + +// Hyper-connection combine followed by the grouped RMS norm of the next hc_mix (block per stream and token): +// w[c] = s2 * sigmoid(s1 * inject[c] + b1) + b2 +// res[e,c] = residual[e,c] + block_out[e] * w[c] -> out_res (the layer residual) +// xn[e,c] = rms_norm(res[:,c]) * gamma[e,c] -> out_xn (next hc_norm) +// The norm replays rms_norm_f32<1024, true> (one block of 1024 threads per stream). +struct ggml_cuda_hc_combine_norm_args { + const ggml_tensor * inject; // [hc, T] F32 contiguous (pre-scale/sigmoid) + const ggml_tensor * residual; // [n_embd, hc, T] F32 contiguous + const ggml_tensor * block_out; // [n_embd, 1, T] F32 contiguous + const ggml_tensor * gamma; // [n_embd, hc] F32 contiguous + ggml_tensor * out_res; // [n_embd, hc, T] + ggml_tensor * out_xn; // [n_embd, hc, T] + float s1, b1, s2, b2; + float eps; + bool single_block = false; // alias-proof variant: one block per token, all inputs read before any store (hc <= 4) +}; + +bool ggml_cuda_hc_combine_norm_supported(const ggml_cuda_hc_combine_norm_args & args, int warp_size); +void ggml_cuda_op_hc_combine_norm(ggml_backend_cuda_context & ctx, const ggml_cuda_hc_combine_norm_args & args); diff --git a/ggml/src/ggml-cuda/mean.cu b/ggml/src/ggml-cuda/mean.cu index a8f6046e46da..55b87b375496 100644 --- a/ggml/src/ggml-cuda/mean.cu +++ b/ggml/src/ggml-cuda/mean.cu @@ -2,7 +2,12 @@ #include "reduce_rows.cuh" #ifdef GGML_CUDA_USE_CUB +#ifdef GGML_CUDA_CUB_IS_HIPCUB +#include +namespace cub = hipcub; +#else #include +#endif // GGML_CUDA_CUB_IS_HIPCUB using namespace cub; #endif // GGML_CUDA_USE_CUB diff --git a/ggml/src/ggml-cuda/mmid.cu b/ggml/src/ggml-cuda/mmid.cu index ed0851dcf8dd..ec0b687d38d2 100644 --- a/ggml/src/ggml-cuda/mmid.cu +++ b/ggml/src/ggml-cuda/mmid.cu @@ -19,6 +19,57 @@ struct mm_ids_helper_store { }; static_assert(sizeof(mm_ids_helper_store) == 4, "unexpected size for mm_ids_helper_store"); +// Qwen4exp 512-expert / 10-active fast path. The generic helper launches one warp per expert and every +// warp scans all token slots (O(n_experts*n_tokens*n_used)). Here one block histograms the slots, computes +// the 512 bounds, and scatters the two maps in O(n_tokens*n_used). Row ordering inside an expert is irrelevant: +// ids_dst maps every compact MMQ row back to its exact destination and there is no cross-row reduction. +__launch_bounds__(1024, 1) +static __global__ void mm_ids_helper_512_10( + const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, + int32_t * __restrict__ expert_bounds, const int n_tokens, const int nchannels_y, + const int si1, const int sis1, const bool write_inverse) { + __shared__ int counts[512]; + __shared__ int cursors[512]; + + const int tid = threadIdx.x; + if (tid < 512) { + counts[tid] = 0; + cursors[tid] = 0; + } + __syncthreads(); + + const int n_slots = n_tokens * 10; + for (int idx = tid; idx < n_slots; idx += blockDim.x) { + const int token = idx / 10; + const int iex = idx - token * 10; + atomicAdd(&counts[ids[token * si1 + iex]], 1); + } + __syncthreads(); + + if (tid == 0) { + int total = 0; + expert_bounds[0] = 0; + for (int e = 0; e < 512; ++e) { + total += counts[e]; + expert_bounds[e + 1] = total; + } + } + __syncthreads(); + + for (int idx = tid; idx < n_slots; idx += blockDim.x) { + const int token = idx / 10; + const int iex = idx - token * 10; + const int expert = ids[token * si1 + iex]; + const int dst = expert_bounds[expert] + atomicAdd(&cursors[expert], 1); + ids_dst[dst] = idx; + if (write_inverse) { + ids_src1[idx] = dst; + } else { + ids_src1[dst] = token * sis1 + iex % nchannels_y; + } + } +} + // the generic path passes 0, which needs no padding since it never groups lanes by token template struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2<(n + 1)/2>::value; }; template <> struct mm_ids_pow2<1> { static constexpr int value = 1; }; @@ -151,6 +202,11 @@ static void launch_mm_ids_helper( void ggml_cuda_launch_mm_ids_helper( const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { + if (n_experts == 512 && n_expert_used == 10 && getenv("GGML_CUDA_DISABLE_MMID_512") == nullptr) { + mm_ids_helper_512_10<<<1, 1024, 0, stream>>>( + ids, ids_src1, ids_dst, expert_bounds, n_tokens, nchannels_y, si1, sis1, write_inverse); + return; + } switch (n_expert_used) { case 2: launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream); diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 592f36591af0..70d59cee82e0 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -266,7 +266,7 @@ static void ggml_cuda_mul_mat_q_impl( const ggml_tensor * gate = swiglu ? swiglu->src[0] : nullptr; const ggml_tensor * up = swiglu ? swiglu->src[1] : nullptr; if (swiglu) { - GGML_ASSERT(src0->type == GGML_TYPE_Q8_0); + GGML_ASSERT(src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_IQ4_NL); GGML_ASSERT(gate && up && gate->type == GGML_TYPE_F32 && up->type == GGML_TYPE_F32); GGML_ASSERT(ggml_are_same_shape(gate, up) && ggml_are_same_shape(gate, src1)); } @@ -447,6 +447,7 @@ void ggml_cuda_mul_mat_q_swiglu( ggml_cuda_mul_mat_q_impl(ctx, src0, swiglu, ids, dst, swiglu); } + bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) { #ifdef GGML_CUDA_FORCE_CUBLAS return false; diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 1c46a8e0e870..73b71dc22967 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1346,10 +1346,21 @@ static __global__ void mul_mat_q( tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop); } +// routed-compact MoE tiles: one descriptor per real (expert, tile) pair; the builder runs as a single block with one +// thread per expert +#define MMQ_ROUTED_MAX_EXPERTS 1024 +#ifndef MMQ_IQ_ID_J_MID +#define MMQ_IQ_ID_J_MID 64 +#endif + +static constexpr bool mmq_rdna3_5_id_n_experts_ok(const int64_t n_experts) { + return n_experts >= 8 && n_experts <= MMQ_ROUTED_MAX_EXPERTS; +} + template static __global__ void build_mmq_routed_descriptors( const int32_t * expert_bounds, uint32_t * descriptors, int n_experts, int max_descriptors) { - __shared__ int tile_counts[256]; + __shared__ int tile_counts[MMQ_ROUTED_MAX_EXPERTS]; const int expert = threadIdx.x; for (int i = expert; i < max_descriptors; i += blockDim.x) { @@ -1594,6 +1605,11 @@ static constexpr int mmq_rdna3_5_id_get_J(const ggml_type type, const int64_t ro switch (type) { case GGML_TYPE_Q8_0: return rows_per_expert <= 12 ? 16 : rows_per_expert <= 64 ? 48 : 128; + // the IQ expert types of the Unsloth qwen4exp mixes (512 experts, 10 active: 40 rows per expert at ubatch 2048) + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return rows_per_expert <= 12 ? 16 : rows_per_expert <= 32 ? 48 : rows_per_expert <= 64 ? MMQ_IQ_ID_J_MID : 128; case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: @@ -1608,10 +1624,14 @@ static constexpr bool mmq_rdna3_5_id_use_compact(const ggml_type type, const int switch (type) { case GGML_TYPE_Q8_0: return J == 16 || J == 48 || J == 128; + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + return J == 16 || J == 48 || J == MMQ_IQ_ID_J_MID || J == 128; case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: - return J == 16 || J == 32 || J == 48 || J == 128; + return J == 16 || J == 32 || J == 48 || J == 64 || J == 96 || J == 128; default: return false; } @@ -1664,12 +1684,12 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); - const bool use_compact_routed = args.ids_dst != nullptr && args.nchannels_y == 256 && GGML_CUDA_CC_IS_RDNA3_5(cc) && + const bool use_compact_routed = args.ids_dst != nullptr && mmq_rdna3_5_id_n_experts_ok(args.nchannels_y) && GGML_CUDA_CC_IS_RDNA3_5(cc) && !ggml_cuda_mmq_get_stream_k(type, J, fallback, cc) && mmq_rdna3_5_id_use_compact(type, J); if (use_compact_routed) { const int max_descriptors = (args.ncols_dst + J - 1) / J + args.nchannels_y; ggml_cuda_pool_alloc descriptors(ctx.pool(id), max_descriptors); - build_mmq_routed_descriptors<<<1, 256, 0, stream>>> + build_mmq_routed_descriptors<<<1, MMQ_ROUTED_MAX_EXPERTS, 0, stream>>> (args.expert_bounds, descriptors.get(), args.nchannels_y, max_descriptors); const int descriptor_blocks = (args.ncols_dst + J - 1) / J; @@ -1754,9 +1774,15 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, J_best = J_rdna3_5; ntiles_J_best = 1; } - } else if (GGML_CUDA_CC_IS_RDNA3_5(cc) && !fallback && args.ids_dst != nullptr && args.nchannels_y == 256) { + } else if (GGML_CUDA_CC_IS_RDNA3_5(cc) && !fallback && args.ids_dst != nullptr && mmq_rdna3_5_id_n_experts_ok(args.nchannels_y)) { const int64_t rows_per_expert = (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y; - const int J_rdna3_5 = mmq_rdna3_5_id_get_J(type, rows_per_expert); + int J_rdna3_5 = type == GGML_TYPE_Q6_K && args.nchannels_y == 128 && rows_per_expert >= 96 && rows_per_expert <= 160 ? + 48 : mmq_rdna3_5_id_get_J(type, rows_per_expert); + if constexpr (type == GGML_TYPE_Q6_K) { + if (getenv("GGML_Q6_COMPACT_J")) { + J_rdna3_5 = atoi(getenv("GGML_Q6_COMPACT_J")); + } + } if (J_rdna3_5 != 0) { const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J_rdna3_5, fallback, cc); if (config.type != GGML_TYPE_COUNT && mmq_get_nbytes_shared(config, cc) <= smpbo) { diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index dc00b2f7e305..3a15f9657ae6 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -1,4 +1,5 @@ #include "mmvq.cuh" +#include "ggml-backend-impl.h" #include "quantize.cuh" #include "unary.cuh" #include "vecdotq.cuh" @@ -894,6 +895,313 @@ static __global__ void mul_mat_vec_q4_columns_rdna3_5( } } +#ifndef MMVQ_IQ3_S_ROWS_PER_BLOCK +#define MMVQ_IQ3_S_ROWS_PER_BLOCK 1 +#endif + +// gfx1151 IQ3_S decode kernel. Each warp owns an independent output row, as in +// the baseline one-warp MMVQ kernel. Grouping adjacent rows into a block keeps +// their shared activation vector and 2 KiB IQ3 codebook hot without introducing +// a cross-warp reduction or changing the dot-product order. +template +__launch_bounds__(rows_per_block * ggml_cuda_get_physical_warp_size(), 8) +static __global__ void mul_mat_vec_iq3_s_rows_rdna3_5( + const void * vx_ptr, const void * vy_ptr, const int32_t * ids, + const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, + const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, + const uint32_t stride_row_x, const uint32_t stride_col_dst, + const uint32_t stride_channel_x, const uint32_t stride_channel_y, + const uint32_t stride_channel_dst, const uint32_t stride_sample_x, + const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { + constexpr int qk = QK_K; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = VDR_IQ3_S_Q8_1_MMVQ; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + + const int lane = threadIdx.x; + const int row = rows_per_block * blockIdx.x + threadIdx.y; + if (row >= nrows_x) { + return; + } + + const uint32_t channel_dst = blockIdx.y; + const uint32_t sample_dst = blockIdx.z; + const uint32_t channel_x = ids[channel_dst]; + const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y); + const uint32_t sample_x = sample_dst; + const uint32_t sample_y = sample_dst; + + const block_q8_1 * y = (const block_q8_1 *) vy_ptr + + sample_y * stride_sample_y + channel_y * stride_channel_y; + const int kbx_offset = sample_x * stride_sample_x + + channel_x * stride_channel_x + row * stride_row_x; + const int blocks_per_row_x = ncols_x / qk; + constexpr int blocks_per_iter = vdr * warp_size / qi; + + float sum = 0.0f; + float sum_gate = 0.0f; + for (int kbx = lane / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); + const int kqs = vdr * (lane % (qi/vdr)); + sum += vec_dot_iq3_s_q8_1(vx_ptr, &y[kby], kbx_offset + kbx, kqs); + if constexpr (has_gate) { + sum_gate += vec_dot_iq3_s_q8_1(fusion.gate, &y[kby], kbx_offset + kbx, kqs); + } + } + + sum = warp_reduce_sum(sum); + if constexpr (has_gate) { + sum_gate = warp_reduce_sum(sum_gate); + } + + if (lane == 0) { + float result = sum; + if constexpr (has_gate) { + result *= ggml_cuda_op_silu_single(sum_gate); + } + dst_ptr[sample_dst * stride_sample_dst + + channel_dst * stride_channel_dst + row] = result; + } + GGML_UNUSED(stride_col_dst); +} + +// vec_dot_iq3_s_q8_1 with the 512-entry codebook read from a caller-provided (shared memory) copy instead of +// the __device__ table: the eight per-lane gathers per block then go to the LDS pipe instead of the +// texture-address unit, which the direct kernel saturates (MemUnitBusy ~91% on gfx1151). +static __device__ __forceinline__ float vec_dot_iq3_s_q8_1_grid( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs, + const uint32_t * __restrict__ grid) { + + const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx; + + const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1)); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bq3->qh[iqs/2]; + + const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2); + const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32; + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int2 grid_pos = make_int2( + grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)], + grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]); + + const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000); + const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000); + + const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0); + const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1); + + const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1); + + sumi = ggml_cuda_dp4a(grid_l, u0, sumi); + sumi = ggml_cuda_dp4a(grid_h, u1, sumi); + } + + sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F); + + const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds); + return d * sumi; +} + +// Direct kernel (same lane layout and accumulation order as mul_mat_vec_iq3_s_rows_rdna3_5) with the codebook +// staged in shared memory once per block. +template +__launch_bounds__(rows_per_block * ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_iq3_s_grid_rdna3_5( + const void * vx_ptr, const void * vy_ptr, const int32_t * ids, + const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, + const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, + const uint32_t stride_row_x, const uint32_t stride_col_dst, + const uint32_t stride_channel_x, const uint32_t stride_channel_y, + const uint32_t stride_channel_dst, const uint32_t stride_sample_x, + const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { + constexpr int qk = QK_K; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = VDR_IQ3_S_Q8_1_MMVQ; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + + __shared__ uint32_t s_grid[512]; + + const int lane = threadIdx.x; + const int tid = threadIdx.y * warp_size + lane; + for (int i = tid; i < 512; i += rows_per_block * warp_size) { + s_grid[i] = iq3s_grid[i]; + } + + const int row = rows_per_block * blockIdx.x + threadIdx.y; + const bool row_ok = row < nrows_x; + + const uint32_t channel_dst = blockIdx.y; + const uint32_t sample_dst = blockIdx.z; + const uint32_t channel_x = ids[channel_dst]; + const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y); + const uint32_t sample_x = sample_dst; + const uint32_t sample_y = sample_dst; + + const block_q8_1 * y = (const block_q8_1 *) vy_ptr + + sample_y * stride_sample_y + channel_y * stride_channel_y; + const int kbx_offset = sample_x * stride_sample_x + + channel_x * stride_channel_x + (row_ok ? row : 0) * stride_row_x; + const int blocks_per_row_x = ncols_x / qk; + constexpr int blocks_per_iter = vdr * warp_size / qi; + + __syncthreads(); + if (!row_ok) { + return; + } + + float sum = 0.0f; + float sum_gate = 0.0f; + for (int kbx = lane / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); + const int kqs = vdr * (lane % (qi/vdr)); + sum += vec_dot_iq3_s_q8_1_grid(vx_ptr, &y[kby], kbx_offset + kbx, kqs, s_grid); + if constexpr (has_gate) { + sum_gate += vec_dot_iq3_s_q8_1_grid(fusion.gate, &y[kby], kbx_offset + kbx, kqs, s_grid); + } + } + + sum = warp_reduce_sum(sum); + if constexpr (has_gate) { + sum_gate = warp_reduce_sum(sum_gate); + } + + if (lane == 0) { + float result = sum; + if constexpr (has_gate) { + result *= ggml_cuda_op_silu_single(sum_gate); + } + dst_ptr[sample_dst * stride_sample_dst + + channel_dst * stride_channel_dst + row] = result; + } + GGML_UNUSED(stride_col_dst); +} + +// gfx1151 IQ3_S decode kernel with LDS staging, ncols == 2560 (10 blocks, 1100 B per row). The direct +// per-lane field loads of vec_dot_iq3_s_q8_1 (about 18 vector-memory instructions per block per lane, most of +// them for the Q8_1 activations) saturate the texture-address unit (MemUnitBusy ~91%) long before DRAM +// bandwidth. Here each wave first copies its up (and gate) row into shared memory with 9 dword loads per +// lane, the block copies the quantized activations once, and the dot products then read from LDS. +// The per-lane accumulation and reduction order is the same as mul_mat_vec_iq3_s_rows_rdna3_5. +#define MMVQ_IQ3_LDS_NCOLS 2560 +#define MMVQ_IQ3_LDS_ROW_DWORDS 275 // 10 blocks * 110 B / 4 +#define MMVQ_IQ3_LDS_ROW_SLOT 276 +#define MMVQ_IQ3_LDS_Y_DWORDS 720 // 80 Q8_1 blocks * 36 B / 4 + +template +__launch_bounds__(rows_per_block * ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_iq3_s_lds_rdna3_5( + const void * vx_ptr, const void * vy_ptr, const int32_t * ids, + const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, + const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, + const uint32_t stride_row_x, const uint32_t stride_col_dst, + const uint32_t stride_channel_x, const uint32_t stride_channel_y, + const uint32_t stride_channel_dst, const uint32_t stride_sample_x, + const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { + constexpr int qk = QK_K; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = VDR_IQ3_S_Q8_1_MMVQ; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int blocks_per_row = MMVQ_IQ3_LDS_NCOLS / qk; + constexpr int blocks_per_iter = vdr * warp_size / qi; + constexpr int row_loads = (MMVQ_IQ3_LDS_ROW_DWORDS + warp_size - 1) / warp_size; // 9 + + __shared__ int s_y[MMVQ_IQ3_LDS_Y_DWORDS]; + __shared__ int s_rows[rows_per_block][has_gate ? 2 : 1][MMVQ_IQ3_LDS_ROW_SLOT]; + + const int lane = threadIdx.x; + const int wave = threadIdx.y; + const int tid = wave * warp_size + lane; + const int row = rows_per_block * blockIdx.x + wave; + const bool row_ok = row < nrows_x; + + const uint32_t channel_dst = blockIdx.y; + const uint32_t sample_dst = blockIdx.z; + const uint32_t channel_x = ids[channel_dst]; + const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y); + + const block_q8_1 * y = (const block_q8_1 *) vy_ptr + sample_dst * stride_sample_y + channel_y * stride_channel_y; + const int kbx_offset = sample_dst * stride_sample_x + channel_x * stride_channel_x + (row_ok ? row : 0) * stride_row_x; + + // 1. weight rows -> registers (dword loads; rows are 4 B aligned: 1100 B rows, even block offsets) + const int * gx = (const int *) ((const block_iq3_s *) vx_ptr + kbx_offset); + int rx[row_loads]; + int rg[row_loads]; +#pragma unroll + for (int i = 0; i < row_loads; ++i) { + const int idx = i * warp_size + lane; + rx[i] = idx < MMVQ_IQ3_LDS_ROW_DWORDS ? gx[idx] : 0; + } + if constexpr (has_gate) { + const int * gg = (const int *) ((const block_iq3_s *) fusion.gate + kbx_offset); +#pragma unroll + for (int i = 0; i < row_loads; ++i) { + const int idx = i * warp_size + lane; + rg[i] = idx < MMVQ_IQ3_LDS_ROW_DWORDS ? gg[idx] : 0; + } + } + + // 2. quantized activations -> LDS (shared by the rows of the block) + { + const int * gy = (const int *) y; + for (int i = tid; i < MMVQ_IQ3_LDS_Y_DWORDS; i += rows_per_block * warp_size) { + s_y[i] = gy[i]; + } + } + + // 3. rows -> LDS +#pragma unroll + for (int i = 0; i < row_loads; ++i) { + const int idx = i * warp_size + lane; + if (idx < MMVQ_IQ3_LDS_ROW_DWORDS) { + s_rows[wave][0][idx] = rx[i]; + if constexpr (has_gate) { + s_rows[wave][1][idx] = rg[i]; + } + } + } + __syncthreads(); + + if (!row_ok) { + return; + } + + // 4. dot products from LDS, same lane -> block assignment as the direct kernel + const block_q8_1 * y_l = (const block_q8_1 *) s_y; + const void * x_l = s_rows[wave][0]; + float sum = 0.0f; + float sum_gate = 0.0f; +#pragma unroll + for (int kbx = lane / (qi/vdr); kbx < blocks_per_row; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); + const int kqs = vdr * (lane % (qi/vdr)); + sum += vec_dot_iq3_s_q8_1(x_l, &y_l[kby], kbx, kqs); + if constexpr (has_gate) { + sum_gate += vec_dot_iq3_s_q8_1(s_rows[wave][1], &y_l[kby], kbx, kqs); + } + } + + sum = warp_reduce_sum(sum); + if constexpr (has_gate) { + sum_gate = warp_reduce_sum(sum_gate); + } + + if (lane == 0) { + float result = sum; + if constexpr (has_gate) { + result *= ggml_cuda_op_silu_single(sum_gate); + } + dst_ptr[sample_dst * stride_sample_dst + channel_dst * stride_channel_dst + row] = result; + } + GGML_UNUSED(stride_col_dst); + GGML_UNUSED(ncols_x); +} + static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) { if (table_id == MMVQ_PARAMETERS_RDNA3_5) { return 1; @@ -1043,6 +1351,18 @@ static __global__ void mul_mat_vec_q( for (int j = 0; j < ncols_dst; ++j) { #pragma unroll for (int i = 0; i < rows_per_cuda_block; ++i) { + if constexpr (type == GGML_TYPE_IQ3_S && has_fusion) { + if (use_gate) { + float dot; + float dot_gate; + vec_dot_iq3_s_q8_1_pair( + dot, dot_gate, vx, vgate, &y[j*stride_col_y + kby], + kbx_offset + i*stride_row_x + kbx, kqs); + tmp[j][i] += dot; + tmp_gate[j][i] += dot_gate; + continue; + } + } if constexpr (type == GGML_TYPE_Q5_1 && table_id == MMVQ_PARAMETERS_RDNA3_5) { volatile float dot = vec_dot_q_cuda( vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs); @@ -1170,6 +1490,10 @@ static __global__ void mul_mat_vec_q( // K iterations; 8 waves or deeper prefetch (more VGPRs, less occupancy) were slower. #define MMVQ_FQ_NWARPS 16 +// whether the fused-quantize path also serves the IQ3_S/IQ4_NL/IQ4_XS expert matvecs (see mmvq_fq_type_ok) +#ifndef MMVQ_FQ_IQ_TYPES +#define MMVQ_FQ_IQ_TYPES false +#endif static constexpr __host__ __device__ bool mmvq_fq_type_ok(ggml_type type) { switch (type) { @@ -1178,6 +1502,12 @@ static constexpr __host__ __device__ bool mmvq_fq_type_ok(ggml_type type) { // Q4_K/Q5_K (block sum via mmvq_fq_block_sum) are exact too but measured neutral on gfx1151 // with the generic, non-prefetched loop (Qwen3.6 UD-Q4_K_XL TG128 60.69 -> 60.63 t/s). return true; + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + // MoE expert types of the Unsloth qwen4exp IQ4_XS mix: their vec_dot only reads the block scale, so + // the in-kernel quantization is exact; used to drop the quantize launch in front of every expert matvec + return MMVQ_FQ_IQ_TYPES; default: return false; } @@ -1188,6 +1518,9 @@ static constexpr __host__ __device__ bool mmvq_fq_needs_sum(ggml_type type) { switch (type) { case GGML_TYPE_Q8_0: case GGML_TYPE_Q6_K: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: return false; default: return true; @@ -1223,17 +1556,23 @@ static __device__ __forceinline__ float mmvq_fq_block_sum(const float * v, const // weight blocks are loaded up front, so the DRAM stream starts before the activation quantization and the // barrier, and each wave keeps MMVQ_FQ_PF loads in flight instead of one. #define MMVQ_FQ_PF 2 +// Long rows with few output rows (e.g. the [10240 x 320] hyper-connection down projections of qwen4exp) have +// too few waves in flight to cover DRAM latency with 2 loads per wave; they use a deeper prefetch instead. +// The accumulation order per lane is the same for any depth, so the results do not change. +#define MMVQ_FQ_PF_LONG 8 +template struct mmvq_fq_q8_0_chunk { - int qs[MMVQ_FQ_PF][VDR_Q8_0_Q8_1_MMVQ]; - half d[MMVQ_FQ_PF]; + int qs[pf][VDR_Q8_0_Q8_1_MMVQ]; + half d[pf]; }; +template static __device__ __forceinline__ void mmvq_fq_q8_0_load( - mmvq_fq_q8_0_chunk & c, const block_q8_0 * GGML_CUDA_RESTRICT x, const int kbx0, const int kqs, const int blocks_per_row_x) { + mmvq_fq_q8_0_chunk & c, const block_q8_0 * GGML_CUDA_RESTRICT x, const int kbx0, const int kqs, const int blocks_per_row_x) { constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * ggml_cuda_get_physical_warp_size() / QI8_0; #pragma unroll - for (int i = 0; i < MMVQ_FQ_PF; ++i) { + for (int i = 0; i < pf; ++i) { const int kbx = kbx0 + i*blocks_per_iter; if (kbx < blocks_per_row_x) { #pragma unroll @@ -1245,11 +1584,12 @@ static __device__ __forceinline__ void mmvq_fq_q8_0_load( } } +template static __device__ __forceinline__ void mmvq_fq_q8_0_dot( - float & tmp, const mmvq_fq_q8_0_chunk & c, const block_q8_1 * y, const int kbx0, const int kqs, const int blocks_per_row_x) { + float & tmp, const mmvq_fq_q8_0_chunk & c, const block_q8_1 * y, const int kbx0, const int kqs, const int blocks_per_row_x) { constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * ggml_cuda_get_physical_warp_size() / QI8_0; #pragma unroll - for (int i = 0; i < MMVQ_FQ_PF; ++i) { + for (int i = 0; i < pf; ++i) { const int kbx = kbx0 + i*blocks_per_iter; if (kbx < blocks_per_row_x) { const block_q8_1 * bq8_1 = &y[kbx]; // qk == QK8_1 for Q8_0 @@ -1263,14 +1603,15 @@ static __device__ __forceinline__ void mmvq_fq_q8_0_dot( } } -template +template __launch_bounds__(nwarps * ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q_fq( const void * vx_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, const uint32_t ncols_x, const uint32_t nrows_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, - const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst) { + const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst, + const float y_scale, const float y_bias, const int y_op) { static_assert(mmvq_fq_type_ok(type)); constexpr bool q8_0_path = type == GGML_TYPE_Q8_0; const void * GGML_CUDA_RESTRICT vx = vx_ptr; @@ -1331,8 +1672,8 @@ static __global__ void mul_mat_vec_q_fq( [[maybe_unused]] const block_q8_0 * x = (const block_q8_0 *) vx + kbx_offset; [[maybe_unused]] const block_q8_0 * xg = nullptr; - [[maybe_unused]] mmvq_fq_q8_0_chunk cx; - [[maybe_unused]] mmvq_fq_q8_0_chunk cg; + [[maybe_unused]] mmvq_fq_q8_0_chunk cx; + [[maybe_unused]] mmvq_fq_q8_0_chunk cg; if constexpr (q8_0_path) { mmvq_fq_q8_0_load(cx, x, kbx0, kqs, blocks_per_row_x); if constexpr (has_fusion) { @@ -1353,7 +1694,40 @@ static __global__ void mul_mat_vec_q_fq( const float4 * yv = (const float4 *) (y + ib*QK8_1 + sub*8); const float4 v0 = yv[0]; const float4 v1 = yv[1]; - const float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + if (y_op == 3) { + // qwen4exp GDN output: per-head (128) rms norm * weight, gated by sigmoid(z). The 16 lanes of a + // head are consecutive threads of one wave (4 Q8_1 blocks x 4 lanes). + float ss = 0.0f; +#pragma unroll + for (int k = 0; k < 8; ++k) { + ss = fmaf(v[k], v[k], ss); + } + ss += __shfl_xor_sync(0xffffffff, ss, 1, warp_size); + ss += __shfl_xor_sync(0xffffffff, ss, 2, warp_size); + ss += __shfl_xor_sync(0xffffffff, ss, 4, warp_size); + ss += __shfl_xor_sync(0xffffffff, ss, 8, warp_size); + const float rms_scale = rsqrtf(ss / 128.0f + fusion.y_eps); + const float * z = (const float *) fusion.y_gate + ib*QK8_1 + sub*8; + const float * w = (const float *) fusion.y_norm_w + (ib*QK8_1 + sub*8) % 128; + const float4 * zv = (const float4 *) z; + const float4 * wv = (const float4 *) w; + const float4 z0 = zv[0], z1 = zv[1], w0 = wv[0], w1 = wv[1]; + const float zz[8] = {z0.x, z0.y, z0.z, z0.w, z1.x, z1.y, z1.z, z1.w}; + const float ww[8] = {w0.x, w0.y, w0.z, w0.w, w1.x, w1.y, w1.z, w1.w}; +#pragma unroll + for (int k = 0; k < 8; ++k) { + // rms_norm_f32<.., true>: scale * x * w ; unary_gated: sigmoid(z) * normed + v[k] = (1.0f / (1.0f + expf(-zz[k]))) * (rms_scale * v[k] * ww[k]); + } + } else if (y_op != 0) { + // fused activation prologue (scale -> unary), same expressions as scale_f32 / op_silu / op_sigmoid +#pragma unroll + for (int k = 0; k < 8; ++k) { + const float t = y_scale * v[k] + y_bias; + v[k] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t)); + } + } float amax = fabsf(v[0]); #pragma unroll @@ -1417,10 +1791,10 @@ static __global__ void mul_mat_vec_q_fq( } } } - constexpr int chunk_blocks = MMVQ_FQ_PF*blocks_per_iter; + constexpr int chunk_blocks = pf*blocks_per_iter; for (int c0 = kbx0; q8_0_path && c0 < blocks_per_row_x; c0 += chunk_blocks) { - mmvq_fq_q8_0_chunk nx; - [[maybe_unused]] mmvq_fq_q8_0_chunk ng; + mmvq_fq_q8_0_chunk nx; + [[maybe_unused]] mmvq_fq_q8_0_chunk ng; const int c1 = c0 + chunk_blocks; if (c1 < blocks_per_row_x) { mmvq_fq_q8_0_load(nx, x, c1, kqs, blocks_per_row_x); @@ -1493,7 +1867,7 @@ static void mul_mat_vec_q_fq_launch( const int nchannels_x, const int nchannels_y, const int nchannels_dst, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, - cudaStream_t stream) { + cudaStream_t stream, const float y_scale, const float y_bias, const int y_op) { const int device = ggml_cuda_get_device(); const int warp_size = ggml_cuda_info().devices[device].warp_size; @@ -1501,23 +1875,60 @@ static void mul_mat_vec_q_fq_launch( const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0) : init_fastdiv_values(nchannels_dst / nchannels_x); const uint3 sample_ratio_fd = init_fastdiv_values(nsamples_dst / nsamples_x); - constexpr int nwarps = MMVQ_FQ_NWARPS; + // One output row per wave. Narrow matrices (e.g. the 320-row hyper-connection down projections of + // qwen4exp) would give only nrows/16 blocks, leaving most CUs idle while each wave streams a long row. + // Use fewer waves per block there so at least ~2 blocks per CU are in flight; every row is still reduced + // by a single wave and the activation quantization is replicated per block, so results are unchanged. + const int nsm = ggml_cuda_info().devices[device].nsm; + int nwarps = MMVQ_FQ_NWARPS; + while (nwarps > 4 && ((nrows_x + nwarps - 1) / nwarps) * nchannels_dst * nsamples_dst < 2 * nsm) { + nwarps /= 2; + } + const dim3 block_nums((nrows_x + nwarps - 1) / nwarps, nchannels_dst, nsamples_dst); const dim3 block_dims(warp_size, nwarps, 1); const int nbytes_shared = (ncols_x / QK8_1) * sizeof(block_q8_1); const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr; const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream); - if (has_fusion) { - ggml_cuda_kernel_launch(mul_mat_vec_q_fq, launch_params, - vx, y, ids, fusion, dst, ncols_x, nrows_x, nchannels_y_fd, stride_row_x, stride_col_dst, - channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, - sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst); - } else { - ggml_cuda_kernel_launch(mul_mat_vec_q_fq, launch_params, + + auto launch = [&](auto kernel) { + ggml_cuda_kernel_launch(kernel, launch_params, vx, y, ids, fusion, dst, ncols_x, nrows_x, nchannels_y_fd, stride_row_x, stride_col_dst, channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, - sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst); + sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst, y_scale, y_bias, y_op); + }; + + // deeper weight prefetch for long rows when the grid is too small to hide DRAM latency with many waves + constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * 32 / QI8_0; + const bool long_rows = type == GGML_TYPE_Q8_0 && (ncols_x / QK8_0) >= 16 * blocks_per_iter && + (int64_t) nrows_x * nchannels_dst * nsamples_dst <= 1024; + + switch (nwarps) { + case 16: + if (has_fusion) { launch(mul_mat_vec_q_fq); } + else { launch(mul_mat_vec_q_fq); } + break; + case 8: + if (long_rows) { + if (has_fusion) { launch(mul_mat_vec_q_fq); } + else { launch(mul_mat_vec_q_fq); } + } else { + if (has_fusion) { launch(mul_mat_vec_q_fq); } + else { launch(mul_mat_vec_q_fq); } + } + break; + case 4: + if (long_rows) { + if (has_fusion) { launch(mul_mat_vec_q_fq); } + else { launch(mul_mat_vec_q_fq); } + } else { + if (has_fusion) { launch(mul_mat_vec_q_fq); } + else { launch(mul_mat_vec_q_fq); } + } + break; + default: + GGML_ABORT("fatal error"); } } @@ -1527,89 +1938,628 @@ static void mul_mat_vec_q_fq_switch_type( const int nchannels_x, const int nchannels_y, const int nchannels_dst, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst, - cudaStream_t stream) { + cudaStream_t stream, const float y_scale, const float y_bias, const int y_op) { +#define MMVQ_FQ_LAUNCH(T) \ + mul_mat_vec_q_fq_launch(vx, y, ids, fusion, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, \ + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, \ + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, y_scale, y_bias, y_op) switch (type_x) { - case GGML_TYPE_Q8_0: - mul_mat_vec_q_fq_launch(vx, y, ids, fusion, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, - nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, - nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream); - break; - case GGML_TYPE_Q6_K: - mul_mat_vec_q_fq_launch(vx, y, ids, fusion, dst, ncols_x, nrows_x, stride_row_x, stride_col_dst, - nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, - nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream); - break; + case GGML_TYPE_Q8_0: MMVQ_FQ_LAUNCH(GGML_TYPE_Q8_0); break; + case GGML_TYPE_Q6_K: MMVQ_FQ_LAUNCH(GGML_TYPE_Q6_K); break; +#if MMVQ_FQ_IQ_TYPES + case GGML_TYPE_IQ3_S: MMVQ_FQ_LAUNCH(GGML_TYPE_IQ3_S); break; + case GGML_TYPE_IQ4_NL: MMVQ_FQ_LAUNCH(GGML_TYPE_IQ4_NL); break; + case GGML_TYPE_IQ4_XS: MMVQ_FQ_LAUNCH(GGML_TYPE_IQ4_XS); break; +#endif default: GGML_ABORT("fatal error"); break; } +#undef MMVQ_FQ_LAUNCH } -// Dedicated MoE multi-token kernel. -// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst) -// Block: (warp_size, ncols_dst) - each warp handles one token independently. -// No shared memory reduction needed since each warp works alone. -template -__launch_bounds__(get_mmvq_mmid_max_batch_for_device()*ggml_cuda_get_physical_warp_size(), 1) -static __global__ void mul_mat_vec_q_moe( - const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, - float * dst_ptr, - const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, - const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst, - const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, - const uint32_t ncols_dst, const uint32_t ids_stride) { - const void * GGML_CUDA_RESTRICT vx = vx_ptr; - const void * GGML_CUDA_RESTRICT vy = vy_ptr; - const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; - float * GGML_CUDA_RESTRICT dst = dst_ptr; +// --------------------------------------------------------------------------------------------- +// RDNA3.5 grouped single-column matvec: several independent matvecs that share the same activation vector +// (e.g. the qwen4exp GDN qkv / gate / alpha / beta projections, or a hyper-connection down projection with its +// F32 inject rows) are launched as one kernel. Each segment is a plain [ncols x nrows] weight matrix (Q8_0, with +// an optional Q8_0 gate + GLU epilogue, or F32) with its own destination; blocks are assigned to segments by a +// prefix over the per-segment block counts. Q8_0 rows use the fused-quantize path of mul_mat_vec_q_fq (same +// per-lane accumulation order), F32 rows are one wave per row with float4 loads. + +#define MMVQ_GROUP_MAX 4 +#define MMVQ_GROUP_TYPE_Q8_0 0 +#define MMVQ_GROUP_TYPE_F32 1 + +struct mmvq_group_seg_dev { + const void * vx; + const void * gate; + float * dst; + int nrows; + int stride_row; // in blocks (Q8_0) or floats (F32) + int gate_stride_row; // in blocks + int blocks_begin; + int type; + int glu_op; + float glu_limit; +}; - constexpr int qk = ggml_cuda_type_traits::qk; - constexpr int qi = ggml_cuda_type_traits::qi; - constexpr int vdr = get_vdr_mmvq(type); +struct mmvq_group_args_dev { + mmvq_group_seg_dev seg[MMVQ_GROUP_MAX]; + int nseg; +}; + +template +__launch_bounds__(nwarps * ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_fq_group( + const mmvq_group_args_dev args, const float * GGML_CUDA_RESTRICT y_ptr, const int ncols_x, + const float y_scale, const float y_bias, const int y_op) { + constexpr int qk = QK8_0; + constexpr int qi = QI8_0; + constexpr int vdr = VDR_Q8_0_Q8_1_MMVQ; constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int blocks_per_iter = vdr * warp_size / qi; - constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); + extern __shared__ char mmvq_fq_smem[]; + block_q8_1 * y_q8 = (block_q8_1 *) mmvq_fq_smem; - const uint32_t token_idx = threadIdx.y; - const int row0 = c_rows_per_block*blockIdx.x; - const int blocks_per_row_x = ncols_x / qk; - constexpr int blocks_per_iter = vdr * warp_size / qi; + const int lane = threadIdx.x; + const int tid = warp_size*threadIdx.y + lane; - const uint32_t channel_dst = blockIdx.y; + // segment lookup, uniform per block + int s = 0; +#pragma unroll + for (int k = 1; k < MMVQ_GROUP_MAX; ++k) { + if (k < args.nseg && (int) blockIdx.x >= args.seg[k].blocks_begin) { + s = k; + } + } + const mmvq_group_seg_dev seg = args.seg[s]; - if (token_idx >= ncols_dst) { + const int row = ((int) blockIdx.x - seg.blocks_begin)*nwarps + threadIdx.y; + const bool row_ok = row < seg.nrows; + + if (seg.type == MMVQ_GROUP_TYPE_F32) { + if (!row_ok) { + return; + } + const float4 * w = (const float4 *) ((const float *) seg.vx + (size_t) row*seg.stride_row); + const float4 * y = (const float4 *) y_ptr; + float acc = 0.0f; +#pragma unroll 4 + for (int k = lane; k < ncols_x/4; k += warp_size) { + float4 w4 = w[k]; + float4 y4 = y[k]; + if (y_op != 0) { + float v[4] = {y4.x, y4.y, y4.z, y4.w}; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const float t = y_scale * v[j] + y_bias; + v[j] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t)); + } + y4 = make_float4(v[0], v[1], v[2], v[3]); + } + acc = fmaf(w4.x, y4.x, acc); + acc = fmaf(w4.y, y4.y, acc); + acc = fmaf(w4.z, y4.z, acc); + acc = fmaf(w4.w, y4.w, acc); + } + acc = warp_reduce_sum(acc); + if (lane == 0) { + seg.dst[row] = acc; + } return; } - ggml_cuda_pdl_sync(); - const uint32_t channel_x = ids[channel_dst + token_idx * ids_stride]; - const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y); - - const block_q8_1 * y = ((const block_q8_1 *) vy) + channel_y*stride_channel_y + token_idx*stride_col_y; - const int kbx_offset = channel_x*stride_channel_x + row0*stride_row_x; + // Q8_0 path: identical to mul_mat_vec_q_fq + const int row_safe = row_ok ? row : seg.nrows - 1; + const bool use_gate = seg.gate != nullptr; - // partial sum for each thread - float tmp[c_rows_per_block] = {0.0f}; + const int blocks_per_row_x = ncols_x / qk; + const int kqs = vdr * (lane % (qi/vdr)); + const int kbx0 = lane / (qi/vdr); - for (int kbx = threadIdx.x / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { - const int kby = kbx * (qk/QK8_1); - const int kqs = vdr * (threadIdx.x % (qi/vdr)); + const block_q8_0 * x = (const block_q8_0 *) seg.vx + (size_t) row_safe*seg.stride_row; + const block_q8_0 * xg = use_gate ? (const block_q8_0 *) seg.gate + (size_t) row_safe*seg.gate_stride_row : nullptr; -#pragma unroll - for (int i = 0; i < c_rows_per_block; ++i) { - tmp[i] += vec_dot_q_cuda(vx, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs); - } + mmvq_fq_q8_0_chunk cx; + mmvq_fq_q8_0_chunk cg; + mmvq_fq_q8_0_load(cx, x, kbx0, kqs, blocks_per_row_x); + if (use_gate) { + mmvq_fq_q8_0_load(cg, xg, kbx0, kqs, blocks_per_row_x); } - ggml_cuda_pdl_lc(); + { + const float * y = y_ptr; + const int nby = ncols_x / QK8_1; + for (int i = tid; i < 4*nby; i += nwarps*warp_size) { + const int ib = i >> 2; + const int sub = i & 3; + const float4 * yv = (const float4 *) (y + ib*QK8_1 + sub*8); + const float4 v0 = yv[0]; + const float4 v1 = yv[1]; + float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + if (y_op != 0) { +#pragma unroll + for (int k = 0; k < 8; ++k) { + const float t = y_scale * v[k] + y_bias; + v[k] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t)); + } + } - // Warp-level reduction only - no shared memory needed + float amax = fabsf(v[0]); #pragma unroll - for (int i = 0; i < c_rows_per_block; ++i) { - tmp[i] = warp_reduce_sum(tmp[i]); - } + for (int k = 1; k < 8; ++k) { + amax = fmaxf(amax, fabsf(v[k])); + } + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 1, warp_size)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 2, warp_size)); - // Write results + const float d = amax / 127.0f; + int q[8]; +#pragma unroll + for (int k = 0; k < 8; ++k) { + const int8_t qk8 = amax == 0.0f ? 0 : roundf(v[k] / d); + q[k] = (int) qk8 & 0xff; + } + int * qs = (int *) y_q8[ib].qs; + qs[2*sub + 0] = q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24); + qs[2*sub + 1] = q[4] | (q[5] << 8) | (q[6] << 16) | (q[7] << 24); + if (sub == 0) { + y_q8[ib].ds = make_half2(d, 0.0f); + } + } + } + __syncthreads(); + + if (!row_ok) { + return; + } + + float tmp = 0.0f; + float tmp_gate = 0.0f; + + const block_q8_1 * y = y_q8; + constexpr int chunk_blocks = pf*blocks_per_iter; + for (int c0 = kbx0; c0 < blocks_per_row_x; c0 += chunk_blocks) { + mmvq_fq_q8_0_chunk nx; + mmvq_fq_q8_0_chunk ng; + const int c1 = c0 + chunk_blocks; + if (c1 < blocks_per_row_x) { + mmvq_fq_q8_0_load(nx, x, c1, kqs, blocks_per_row_x); + if (use_gate) { + mmvq_fq_q8_0_load(ng, xg, c1, kqs, blocks_per_row_x); + } + } + mmvq_fq_q8_0_dot(tmp, cx, y, c0, kqs, blocks_per_row_x); + if (use_gate) { + mmvq_fq_q8_0_dot(tmp_gate, cg, y, c0, kqs, blocks_per_row_x); + } + cx = nx; + if (use_gate) { + cg = ng; + } + } + + tmp = warp_reduce_sum(tmp); + if (use_gate) { + tmp_gate = warp_reduce_sum(tmp_gate); + } + + if (lane == 0) { + float result = tmp; + if (use_gate) { + switch (seg.glu_op) { + case GGML_GLU_OP_SWIGLU: + result *= ggml_cuda_op_silu_single(tmp_gate); + break; + case GGML_GLU_OP_GEGLU: + result *= ggml_cuda_op_gelu_single(tmp_gate); + break; + case GGML_GLU_OP_SWIGLU_OAI: + result = ggml_cuda_op_swiglu_oai_single(tmp_gate, result); + break; + case GGML_GLU_OP_SWIGLU_CLAMP: + result = ggml_cuda_op_swiglu_clamp_single(tmp_gate, result, seg.glu_limit); + break; + default: + result = result * tmp_gate; + break; + } + } + seg.dst[row] = result; + } +} + +bool ggml_cuda_mmv_group_seg_ok(const ggml_tensor * w, const ggml_tensor * y) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (!GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return false; + } + if (y->type != GGML_TYPE_F32 || y->ne[1] != 1 || y->ne[2] != 1 || y->ne[3] != 1 || !ggml_is_contiguous(y) || + (uintptr_t) y->data % 16 != 0 || y->ne[0] != w->ne[0]) { + return false; + } + if (w->ne[2] != 1 || w->ne[3] != 1 || w->nb[0] != ggml_type_size(w->type) || w->ne[1] > INT32_MAX || + (w->buffer && ggml_backend_buffer_get_usage(w->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE)) { + return false; + } + const int64_t ncols = w->ne[0]; + if (w->type == GGML_TYPE_Q8_0) { + return ncols % QK8_1 == 0 && (ncols / QK8_1) * sizeof(block_q8_1) <= 16384 && w->nb[1] % sizeof(block_q8_0) == 0; + } + return false; +} + +void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, const ggml_cuda_mmv_group_seg * segs, const int nseg) { + GGML_ASSERT(nseg >= 1 && nseg <= MMVQ_GROUP_MAX); + const int device = ggml_cuda_get_device(); + const int warp_size = ggml_cuda_info().devices[device].warp_size; + const int nsm = ggml_cuda_info().devices[device].nsm; + GGML_ASSERT(warp_size == 32); + + const int ncols = y->ne[0]; + + int64_t total_q8_rows = 0; + int64_t max_rows = 0; + for (int i = 0; i < nseg; ++i) { + GGML_ASSERT(ggml_cuda_mmv_group_seg_ok(segs[i].w, y)); + GGML_ASSERT(segs[i].dst->type == GGML_TYPE_F32 && ggml_is_contiguous(segs[i].dst) && ggml_nelements(segs[i].dst) == segs[i].w->ne[1]); + if (segs[i].gate) { + GGML_ASSERT(segs[i].w->type == GGML_TYPE_Q8_0 && segs[i].gate->type == GGML_TYPE_Q8_0 && + ggml_are_same_shape(segs[i].w, segs[i].gate) && segs[i].gate->nb[0] == sizeof(block_q8_0) && + segs[i].gate->nb[1] % sizeof(block_q8_0) == 0); + } + if (segs[i].w->type == GGML_TYPE_Q8_0) { + total_q8_rows += segs[i].w->ne[1]; + } + max_rows = std::max(max_rows, segs[i].w->ne[1]); + } + + // same block-size heuristic as mul_mat_vec_q_fq_launch, applied to the largest segment + int nwarps = MMVQ_FQ_NWARPS; + while (nwarps > 4 && ((max_rows + nwarps - 1) / nwarps) < 2 * nsm) { + nwarps /= 2; + } + + mmvq_group_args_dev args{}; + args.nseg = nseg; + int nblocks = 0; + for (int i = 0; i < nseg; ++i) { + mmvq_group_seg_dev & d = args.seg[i]; + const ggml_tensor * w = segs[i].w; + d.vx = w->data; + d.gate = segs[i].gate ? segs[i].gate->data : nullptr; + d.dst = (float *) segs[i].dst->data; + d.nrows = (int) w->ne[1]; + d.type = w->type == GGML_TYPE_Q8_0 ? MMVQ_GROUP_TYPE_Q8_0 : MMVQ_GROUP_TYPE_F32; + d.stride_row = (int) (w->nb[1] / ggml_type_size(w->type)); + d.gate_stride_row = segs[i].gate ? (int) (segs[i].gate->nb[1] / sizeof(block_q8_0)) : 0; + d.blocks_begin = nblocks; + d.glu_op = (int) segs[i].glu_op; + d.glu_limit = segs[i].glu_limit; + nblocks += (d.nrows + nwarps - 1) / nwarps; + } + + const dim3 block_nums(nblocks, 1, 1); + const dim3 block_dims(warp_size, nwarps, 1); + const int nbytes_shared = total_q8_rows > 0 ? (ncols / QK8_1) * sizeof(block_q8_1) : 0; + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, ctx.stream()); + + constexpr int blocks_per_iter = VDR_Q8_0_Q8_1_MMVQ * 32 / QI8_0; + const bool long_rows = (ncols / QK8_0) >= 16 * blocks_per_iter && total_q8_rows <= 1024; + + auto launch = [&](auto kernel) { + ggml_cuda_kernel_launch(kernel, launch_params, args, (const float *) y->data, ncols, 1.0f, 0.0f, 0); + }; + + switch (nwarps) { + case 16: launch(mul_mat_vec_fq_group<16, MMVQ_FQ_PF>); break; + case 8: + if (long_rows) { launch(mul_mat_vec_fq_group<8, MMVQ_FQ_PF_LONG>); } + else { launch(mul_mat_vec_fq_group<8, MMVQ_FQ_PF>); } + break; + case 4: + if (long_rows) { launch(mul_mat_vec_fq_group<4, MMVQ_FQ_PF_LONG>); } + else { launch(mul_mat_vec_fq_group<4, MMVQ_FQ_PF>); } + break; + default: + GGML_ABORT("fatal error"); + } +} + +// --------------------------------------------------------------------------------------------- +// qwen4exp hyper-connection up projection + stream mix, one token (RDNA3.5): +// gate = W_up * y' (y' = op(y_scale*y + y_bias), Q8_0 weights [ncols, hc*n_embd], fused quantization) +// mixed[e] = mix_scale * ( sum_c xn[c*n_embd + e] * sigmoid(gate[c*n_embd + e]) ) + mix_bias +// One wave owns element e and computes its hc gate rows {c*n_embd + e}: the hc*blocks_per_row Q8_0 blocks +// of those rows are spread flat over the lanes (4 lanes per block, 8 blocks per pass), so the short rows +// (ncols = 320 -> 10 blocks) keep all lanes busy. The gate itself is never written; the mix uses the same +// rounding sequence as hc_mix_reduce_f32 (separate mul / add roundings, streams summed in order). + +#define MMVQ_HCMIX_MAX_PASSES 16 +#define MMVQ_HCMIX_MAX_HC 4 +#define MMVQ_HCMIX_EPW 2 // elements (mixed outputs) per wave: 2*hc adjacent-row pairs stream per wave + +#if defined(__HIP_PLATFORM_AMD__) +static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) { + float result; + asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} +static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) { + float result; + asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} +#else +static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) { return __fmul_rn(a, b); } +static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) { return __fadd_rn(a, b); } +#endif + +template +__launch_bounds__(nwarps * ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_q_fq_hcmix( + const void * GGML_CUDA_RESTRICT vx_ptr, const float * GGML_CUDA_RESTRICT y_ptr, const float * xn, float * dst, + const int ncols_x, const int n_embd, const int hc, const int stride_row_x, + const float y_scale, const float y_bias, const int y_op, const float mix_scale, const float mix_bias) { + constexpr int qi = QI8_0; + constexpr int vdr = VDR_Q8_0_Q8_1_MMVQ; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int lanes_per_row = qi / vdr; // 4 + constexpr int rows_per_wave = warp_size / lanes_per_row; // 8 = hc_max * epw + constexpr int epw = MMVQ_HCMIX_EPW; + static_assert(rows_per_wave == MMVQ_HCMIX_MAX_HC * epw, "lane layout"); + + extern __shared__ char mmvq_fq_smem[]; + block_q8_1 * y_q8 = (block_q8_1 *) mmvq_fq_smem; + + const int lane = threadIdx.x; + const int tid = warp_size*threadIdx.y + lane; + + const int e0 = (blockIdx.x*nwarps + threadIdx.y) * epw; // first element of this wave + + // lane group g = lane/4 owns wave row r = g: stream c = r / epw, element e0 + (r % epw) + const int r = lane / lanes_per_row; + const int kqs = vdr * (lane % lanes_per_row); + const int c = r / epw; + const int j = r - c*epw; + const bool row_ok = c < hc && e0 + j < n_embd; + const int row = c*n_embd + min(e0 + j, n_embd - 1); // clamped so that every lane reads valid memory + + const block_q8_0 * x = (const block_q8_0 *) vx_ptr + (size_t) (row_ok ? row : 0)*stride_row_x; + const int blocks_per_row = ncols_x / QK8_0; + + // 1. the whole row of this lane group first (the DRAM stream; everything else is tiny) + int qs[MMVQ_HCMIX_MAX_PASSES][vdr]; + half d[MMVQ_HCMIX_MAX_PASSES]; +#pragma unroll + for (int p = 0; p < MMVQ_HCMIX_MAX_PASSES; ++p) { + if (p < blocks_per_row) { +#pragma unroll + for (int jj = 0; jj < vdr; ++jj) { + qs[p][jj] = get_int_b2(x[p].qs, kqs + jj); + } + d[p] = x[p].d; + } + } + + // 2. quantize the activation vector into shared memory (same as mul_mat_vec_q_fq) + { + const int nby = ncols_x / QK8_1; + for (int i = tid; i < 4*nby; i += nwarps*warp_size) { + const int ib = i >> 2; + const int sub = i & 3; + const float4 * yv = (const float4 *) (y_ptr + ib*QK8_1 + sub*8); + const float4 v0 = yv[0]; + const float4 v1 = yv[1]; + float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; + if (y_op != 0) { +#pragma unroll + for (int k = 0; k < 8; ++k) { + const float t = y_scale * v[k] + y_bias; + v[k] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t)); + } + } + float amax = fabsf(v[0]); +#pragma unroll + for (int k = 1; k < 8; ++k) { + amax = fmaxf(amax, fabsf(v[k])); + } + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 1, warp_size)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 2, warp_size)); + const float dq = amax / 127.0f; + int q[8]; +#pragma unroll + for (int k = 0; k < 8; ++k) { + const int8_t qk8 = amax == 0.0f ? 0 : roundf(v[k] / dq); + q[k] = (int) qk8 & 0xff; + } + int * qsy = (int *) y_q8[ib].qs; + qsy[2*sub + 0] = q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24); + qsy[2*sub + 1] = q[4] | (q[5] << 8) | (q[6] << 16) | (q[7] << 24); + if (sub == 0) { + y_q8[ib].ds = make_half2(dq, 0.0f); + } + } + __syncthreads(); + } + + if (e0 >= n_embd) { + return; + } + + // 3. dot product of this lane group's row + float acc = 0.0f; +#pragma unroll + for (int p = 0; p < MMVQ_HCMIX_MAX_PASSES; ++p) { + if (p < blocks_per_row) { + const block_q8_1 * bq8_1 = &y_q8[p]; + int u[vdr]; +#pragma unroll + for (int jj = 0; jj < vdr; ++jj) { + u[jj] = get_int_b4(bq8_1->qs, kqs + jj); + } + acc += vec_dot_q8_0_q8_1_impl(qs[p], u, d[p], __low2half(bq8_1->ds)); + } + } + acc += __shfl_xor_sync(0xffffffff, acc, 1, warp_size); + acc += __shfl_xor_sync(0xffffffff, acc, 2, warp_size); + // lane 4*r now holds the gate of wave row r = c*epw + j + + float g[MMVQ_HCMIX_MAX_HC]; +#pragma unroll + for (int cc = 0; cc < MMVQ_HCMIX_MAX_HC; ++cc) { + // element j = lane (for lane < epw): its stream-cc gate sits in lane group cc*epw + lane + g[cc] = __shfl_sync(0xffffffff, acc, lanes_per_row * (cc*epw + (lane % epw)), warp_size); + } + + if (lane < epw && e0 + lane < n_embd) { + const int e = e0 + lane; + // same expressions as hc_mix_reduce_f32: sigmoid, rounded mul, streams added in order, then scale_f32 + float mixed = mmvq_hc_mul_rn(xn[e], 1.0f / (1.0f + expf(-g[0]))); +#pragma unroll + for (int cc = 1; cc < MMVQ_HCMIX_MAX_HC; ++cc) { + if (cc < hc) { + mixed = mmvq_hc_add_rn(mixed, mmvq_hc_mul_rn(xn[cc*n_embd + e], 1.0f / (1.0f + expf(-g[cc])))); + } + } + dst[e] = mix_scale * mixed + mix_bias; + } +} + +bool ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, const ggml_tensor * dst, const int hc) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (!GGML_CUDA_CC_IS_RDNA3_5(cc) || hc < 1 || hc > MMVQ_HCMIX_MAX_HC) { + return false; + } + if (w->type != GGML_TYPE_Q8_0 || w->ne[2] != 1 || w->ne[3] != 1 || w->nb[0] != sizeof(block_q8_0) || w->nb[1] % sizeof(block_q8_0) != 0 || + (w->buffer && ggml_backend_buffer_get_usage(w->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE)) { + return false; + } + const int64_t ncols = w->ne[0]; + const int64_t n_embd = dst->ne[0]; + if (ncols % QK8_0 != 0 || w->ne[1] != hc * n_embd || (ncols / QK8_0) > MMVQ_HCMIX_MAX_PASSES) { + return false; + } + if (y->type != GGML_TYPE_F32 || !ggml_is_contiguous(y) || ggml_nelements(y) != ncols || (uintptr_t) y->data % 16 != 0) { + return false; + } + if (xn->type != GGML_TYPE_F32 || !ggml_is_contiguous(xn) || ggml_nelements(xn) != hc * n_embd || + dst->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst) || ggml_nelements(dst) != n_embd) { + return false; + } + // every block reads y: dst must not overlap it + { + const char * d0 = (const char *) dst->data; + const char * d1 = d0 + ggml_backend_buft_get_alloc_size(dst->buffer->buft, dst); + const char * y0 = (const char *) y->data; + const char * y1 = y0 + ggml_backend_buft_get_alloc_size(y->buffer->buft, y); + if (d0 < y1 && y0 < d1) { + return false; + } + } + // dst may alias xn only on a whole stream (element e is then read and written by the same wave) + { + const char * d0 = (const char *) dst->data; + const char * d1 = d0 + ggml_backend_buft_get_alloc_size(dst->buffer->buft, dst); + const char * x0 = (const char *) xn->data; + const char * x1 = x0 + ggml_backend_buft_get_alloc_size(xn->buffer->buft, xn); + if (d0 < x1 && x0 < d1) { + const ptrdiff_t off = d0 - x0; + if (off < 0 || off % (n_embd * (ptrdiff_t) sizeof(float)) != 0 || off + (ptrdiff_t) ggml_nbytes(dst) > (ptrdiff_t) ggml_nbytes(xn)) { + return false; + } + } + } + return true; +} + +void ggml_cuda_mul_mat_vec_q_fq_hcmix(ggml_backend_cuda_context & ctx, const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, + ggml_tensor * dst, const int hc, const float y_scale, const float y_bias, const int y_op, const float mix_scale, const float mix_bias) { + GGML_ASSERT(ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(w, y, xn, dst, hc)); + const int device = ggml_cuda_get_device(); + const int warp_size = ggml_cuda_info().devices[device].warp_size; + GGML_ASSERT(warp_size == 32); + + const int ncols = (int) w->ne[0]; + const int n_embd = (int) dst->ne[0]; + + constexpr int nwarps = 32; + const int n_waves = (n_embd + MMVQ_HCMIX_EPW - 1) / MMVQ_HCMIX_EPW; + const dim3 block_nums((n_waves + nwarps - 1) / nwarps, 1, 1); + const dim3 block_dims(warp_size, nwarps, 1); + const int nbytes_shared = (ncols / QK8_1) * sizeof(block_q8_1); + const ggml_cuda_kernel_launch_params launch_params(block_nums, block_dims, nbytes_shared, ctx.stream()); + ggml_cuda_kernel_launch(mul_mat_vec_q_fq_hcmix, launch_params, + w->data, (const float *) y->data, (const float *) xn->data, (float *) dst->data, + ncols, n_embd, hc, (int) (w->nb[1] / sizeof(block_q8_0)), y_scale, y_bias, y_op, mix_scale, mix_bias); +} + +// Dedicated MoE multi-token kernel. +// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst) +// Block: (warp_size, ncols_dst) - each warp handles one token independently. +// No shared memory reduction needed since each warp works alone. +template +__launch_bounds__(get_mmvq_mmid_max_batch_for_device()*ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_vec_q_moe( + const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, + float * dst_ptr, + const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, + const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst, + const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, + const uint32_t ncols_dst, const uint32_t ids_stride) { + const void * GGML_CUDA_RESTRICT vx = vx_ptr; + const void * GGML_CUDA_RESTRICT vy = vy_ptr; + const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; + float * GGML_CUDA_RESTRICT dst = dst_ptr; + + constexpr int qk = ggml_cuda_type_traits::qk; + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = get_vdr_mmvq(type); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + + constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type); + + const uint32_t token_idx = threadIdx.y; + const int row0 = c_rows_per_block*blockIdx.x; + const int blocks_per_row_x = ncols_x / qk; + constexpr int blocks_per_iter = vdr * warp_size / qi; + + const uint32_t channel_dst = blockIdx.y; + + if (token_idx >= ncols_dst) { + return; + } + + ggml_cuda_pdl_sync(); + const uint32_t channel_x = ids[channel_dst + token_idx * ids_stride]; + const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y); + + const block_q8_1 * y = ((const block_q8_1 *) vy) + channel_y*stride_channel_y + token_idx*stride_col_y; + const int kbx_offset = channel_x*stride_channel_x + row0*stride_row_x; + + // partial sum for each thread + float tmp[c_rows_per_block] = {0.0f}; + + for (int kbx = threadIdx.x / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) { + const int kby = kbx * (qk/QK8_1); + const int kqs = vdr * (threadIdx.x % (qi/vdr)); + +#pragma unroll + for (int i = 0; i < c_rows_per_block; ++i) { + tmp[i] += vec_dot_q_cuda(vx, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs); + } + } + + ggml_cuda_pdl_lc(); + + // Warp-level reduction only - no shared memory needed +#pragma unroll + for (int i = 0; i < c_rows_per_block; ++i) { + tmp[i] = warp_reduce_sum(tmp[i]); + } + + // Write results if (threadIdx.x < c_rows_per_block && (c_rows_per_block == 1 || uint32_t(row0 + threadIdx.x) < nrows_x)) { dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = tmp[threadIdx.x]; } @@ -1786,6 +2736,53 @@ static void mul_mat_vec_q_switch_ncols_dst( // static, else MSVC lambda capture breaks the constexpr uses below static constexpr int c_ncols_dst = 1; + if constexpr (type == GGML_TYPE_IQ3_S) { + constexpr int c_rows = MMVQ_IQ3_S_ROWS_PER_BLOCK; + const bool no_extra_fusion = fusion.x_bias == nullptr && fusion.gate_bias == nullptr && + fusion.x_scale == nullptr && fusion.gate_scale == nullptr; + const bool gate_ok = fusion.gate == nullptr || fusion.glu_op == GGML_GLU_OP_SWIGLU; + if (table_id == MMVQ_PARAMETERS_RDNA3_5 && ids != nullptr && ncols_x == 2560 && + no_extra_fusion && gate_ok && nsamples_dst == 1) { + static const int rows_env = getenv("GGML_IQ3_ROWS") ? atoi(getenv("GGML_IQ3_ROWS")) : c_rows; + auto launch_rows = [&](auto kernel, int rows) { + const dim3 block_nums((nrows_x + rows - 1) / rows, nchannels_dst, nsamples_dst); + const dim3 block_dims(warp_size, rows, 1); + const ggml_cuda_kernel_launch_params params(block_nums, block_dims, 0, stream); + ggml_cuda_kernel_launch(kernel, params, + vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, nrows_x, + stride_row_x, stride_col_dst, stride_channel_x, stride_channel_y, + stride_channel_dst, stride_sample_x, stride_sample_y, stride_sample_dst); + }; + static const int lds_env = getenv("GGML_IQ3_LDS") ? atoi(getenv("GGML_IQ3_LDS")) : 0; + static const int grid_env = getenv("GGML_IQ3_GRID") ? atoi(getenv("GGML_IQ3_GRID")) : 4; + if (fusion.gate != nullptr && grid_env > 0) { + switch (grid_env) { + case 1: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5, 1); break; + case 2: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5, 2); break; + case 8: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5, 8); break; + default: launch_rows(mul_mat_vec_iq3_s_grid_rdna3_5, 4); break; + } + } else if (fusion.gate != nullptr && lds_env > 0) { + switch (lds_env) { + case 1: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5, 1); break; + case 2: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5, 2); break; + case 8: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5, 8); break; + default: launch_rows(mul_mat_vec_iq3_s_lds_rdna3_5, 4); break; + } + } else if (fusion.gate != nullptr) { + switch (rows_env) { + case 2: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5, 2); break; + case 4: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5, 4); break; + case 8: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5, 8); break; + default: launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5, 1); break; + } + } else { + launch_rows(mul_mat_vec_iq3_s_rows_rdna3_5, c_rows); + } + break; + } + } + // Tag types keep the flags compile-time, so __launch_bounds__ matches what is launched. const auto launch = [&](auto small_k_tag, auto halve_iters_tag) { constexpr bool c_small_k = decltype(small_k_tag)::value; @@ -2056,6 +3053,266 @@ static void mul_mat_vec_q_switch_type( } } +// RDNA3.5 token-generation fast path: quantize the activations inside the matvec kernel. Optionally applies an +// elementwise prologue (scale -> silu/sigmoid) to the activations first, so that the two small kernels in front +// of the matvec disappear as well. Returns false when the shape/type is not served by this path. +static bool mul_mat_vec_q_fq_try( + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, + const ggml_cuda_mm_fusion_args_host * fusion, const float y_scale, const float y_bias, const int y_op, const bool launch) { + GGML_TENSOR_BINARY_OP_LOCALS; + + const size_t ts_src0 = ggml_type_size(src0->type); + const size_t ts_src1 = ggml_type_size(src1->type); + const size_t ts_dst = ggml_type_size(dst->type); + + if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || nb00 != ts_src0 || nb10 != ts_src1 || nb0 != ts_dst) { + return false; + } + + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const int64_t ncols_dst_fq = ids ? ne2 : ne1; + const bool aligned = (uintptr_t) src1->data % 16 == 0 && nb11 % 16 == 0 && nb12 % 16 == 0 && nb13 % 16 == 0; + const bool fusion_ok = !fusion || (fusion->x_scale == nullptr && fusion->gate_scale == nullptr); + if (!(GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols_dst_fq == 1 && mmvq_fq_type_ok(src0->type) && aligned && fusion_ok && + ne10 % QK8_1 == 0 && (ne10 / QK8_1) * sizeof(block_q8_1) <= 16384)) { + return false; + } + if (!launch) { + return true; + } + + cudaStream_t stream = ctx.stream(); + + const float * src1_d = (const float *) src1->data; + const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr; + float * dst_d = (float *) dst->data; + + ggml_cuda_mm_fusion_args_device fusion_local{}; + if (fusion) { + GGML_ASSERT(!ids || (fusion->x_bias == nullptr && fusion->gate_bias == nullptr)); + if (fusion->x_bias) { + fusion_local.x_bias = fusion->x_bias->data; + } + if (fusion->gate) { + fusion_local.gate = fusion->gate->data; + } + if (fusion->gate_bias) { + fusion_local.gate_bias = fusion->gate_bias->data; + } + fusion_local.glu_op = fusion->glu_op; + fusion_local.glu_limit = fusion->glu_limit; + fusion_local.y_gate = fusion->y_gate ? fusion->y_gate->data : nullptr; + fusion_local.y_norm_w = fusion->y_norm_w ? fusion->y_norm_w->data : nullptr; + fusion_local.y_eps = fusion->y_eps; + } + + // If src0 is a temporary compute buffer, clear any potential padding. + if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + const size_t size_data = ggml_nbytes(src0); + const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0); + if (size_alloc > size_data) { + GGML_ASSERT(ggml_is_contiguously_allocated(src0)); + GGML_ASSERT(!src0->view_src); + CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream)); + } + } + + const int64_t s01 = src0->nb[1] / ts_src0; + const int64_t s02 = src0->nb[2] / ts_src0; + const int64_t s03 = src0->nb[3] / ts_src0; + const int64_t s11 = src1->nb[1] / ts_src1; + const int64_t s12 = src1->nb[2] / ts_src1; + const int64_t s13 = src1->nb[3] / ts_src1; + const int64_t s1 = dst->nb[1] / ts_dst; + const int64_t s2 = dst->nb[2] / ts_dst; + const int64_t s3 = dst->nb[3] / ts_dst; + + // For MUL_MAT_ID the memory layout is different than for MUL_MAT: + const int64_t nchannels_y = ids ? ne11 : ne12; + const int64_t nchannels_dst = ids ? ne1 : ne2; + const int64_t stride_col_dst = ids ? s2 : s1; + const int64_t stride_channel_dst = ids ? s1 : s2; + const int64_t stride_channel_y = ids ? s11 : s12; + + mul_mat_vec_q_fq_switch_type( + src0->data, src0->type, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, s01, stride_col_dst, + ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, + ne03, ne3, s03, s13, s3, stream, y_scale, y_bias, y_op); + return true; +} + +bool ggml_cuda_mul_mat_vec_q_fq_prologue_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * y) { + if (dst->op != GGML_OP_MUL_MAT || !ggml_is_quantized(dst->src[0]->type) || y->type != GGML_TYPE_F32 || + !ggml_is_contiguous(y) || !ggml_are_same_shape(y, dst->src[1])) { + return false; + } + return mul_mat_vec_q_fq_try(ctx, dst->src[0], y, nullptr, const_cast(dst), nullptr, 1.0f, 0.0f, 0, /*launch=*/false); +} + +void ggml_cuda_mul_mat_vec_q_fq_prologue(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_tensor * y, + const float y_scale, const float y_bias, const int y_op) { + const bool ok = mul_mat_vec_q_fq_try(ctx, dst->src[0], y, nullptr, dst, nullptr, y_scale, y_bias, y_op, /*launch=*/true); + GGML_ASSERT(ok); +} + +bool ggml_cuda_mul_mat_vec_q_fq_gdn_gate_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * z) { + const ggml_tensor * y = dst->src[1]; + if (dst->op != GGML_OP_MUL_MAT || !ggml_is_quantized(dst->src[0]->type) || y->type != GGML_TYPE_F32 || + !ggml_is_contiguous(y) || ggml_nrows(y) != 1 || y->ne[0] % 128 != 0 || + z->type != GGML_TYPE_F32 || !ggml_is_contiguous(z) || ggml_nelements(z) != y->ne[0] || (uintptr_t) z->data % 16 != 0) { + return false; + } + // every block reads z while the waves write dst + const char * d0 = (const char *) dst->data; + const char * d1 = d0 + ggml_backend_buft_get_alloc_size(dst->buffer->buft, dst); + const char * z0 = (const char *) z->data; + const char * z1 = z0 + ggml_backend_buft_get_alloc_size(z->buffer->buft, z); + if (d0 < z1 && z0 < d1) { + return false; + } + return mul_mat_vec_q_fq_try(ctx, dst->src[0], y, nullptr, const_cast(dst), nullptr, 1.0f, 0.0f, 0, /*launch=*/false); +} + +void ggml_cuda_mul_mat_vec_q_fq_gdn_gate(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const float * attn, + const ggml_tensor * z, const ggml_tensor * norm_w, const float eps) { + GGML_ASSERT((uintptr_t) attn % 16 == 0 && (uintptr_t) norm_w->data % 16 == 0 && ggml_nelements(norm_w) == 128); + ggml_tensor y = *dst->src[1]; // same shape/strides as the (elided) gated norm output, data = pre-norm scratch + y.data = (void *) attn; + ggml_cuda_mm_fusion_args_host fusion{}; + fusion.y_gate = z; + fusion.y_norm_w = norm_w; + fusion.y_eps = eps; + const bool ok = mul_mat_vec_q_fq_try(ctx, dst->src[0], &y, nullptr, dst, &fusion, 1.0f, 0.0f, 3, /*launch=*/true); + GGML_ASSERT(ok); +} + +template +__launch_bounds__(32, 8) +static __global__ void mul_mat_id_iq4_nl_weighted_rdna3_5( + const void * vx_ptr, const block_q8_1 * y, const int32_t * ids, const float * weights, float * dst, + const int nrows, const int blocks_per_row, const int stride_row_x, const int stride_channel_x, + const int stride_y) { + constexpr int qi = ggml_cuda_type_traits::qi; + constexpr int vdr = VDR_Q4_0_Q8_1_MMVQ; + constexpr int warp_size = 32; + constexpr int blocks_per_iter = vdr * warp_size / qi; + + const int lane = threadIdx.x; + const int row = blockIdx.x; + if (row >= nrows) { + return; + } + + float result = 0.0f; +#pragma unroll + for (int ex = 0; ex < n_expert_used; ++ex) { + const int channel = ids[ex]; + const int x_off = channel * stride_channel_x + row * stride_row_x; + float sum = 0.0f; + for (int kbx = lane / (qi/vdr); kbx < blocks_per_row; kbx += blocks_per_iter) { + const int kqs = vdr * (lane % (qi/vdr)); + sum += vec_dot_iq4_nl_q8_1(vx_ptr, y + ex * stride_y + kbx, x_off + kbx, kqs); + } + sum = warp_reduce_sum(sum); + const float term = mmvq_hc_mul_rn(sum, weights[ex]); + result = ex == 0 ? term : mmvq_hc_add_rn(result, term); + } + + if (lane == 0) { + dst[row] = result; + } +} + +template +__launch_bounds__(32, 8) +static __global__ void mul_mat_id_q8_0_weighted_rdna3_5( + const void * vx_ptr, const block_q8_1 * y, const int32_t * ids, const float * weights, float * dst, + const int nrows, const int blocks_per_row, const int stride_row_x, const int stride_channel_x, + const int stride_y) { + constexpr int qi = QI8_0; + constexpr int vdr = VDR_Q8_0_Q8_1_MMVQ; + constexpr int warp_size = 32; + constexpr int blocks_per_iter = vdr * warp_size / qi; + + const int lane = threadIdx.x; + const int row = blockIdx.x; + if (row >= nrows) { + return; + } + + float result = 0.0f; +#pragma unroll + for (int ex = 0; ex < n_expert_used; ++ex) { + const int channel = ids[ex]; + const int x_off = channel * stride_channel_x + row * stride_row_x; + float sum = 0.0f; + for (int kbx = lane / (qi/vdr); kbx < blocks_per_row; kbx += blocks_per_iter) { + const int kqs = vdr * (lane % (qi/vdr)); + sum += vec_dot_q8_0_q8_1(vx_ptr, y + ex * stride_y + kbx, x_off + kbx, kqs); + } + sum = warp_reduce_sum(sum); + const float term = mmvq_hc_mul_rn(sum, weights[ex]); + result = ex == 0 ? term : mmvq_hc_add_rn(result, term); + } + + if (lane == 0) { + dst[row] = result; + } +} + +bool ggml_cuda_mul_mat_id_weighted_rdna3_5_ok( + const ggml_tensor * experts, const ggml_tensor * weights, const ggml_tensor * dst) { + if (experts->op != GGML_OP_MUL_MAT_ID || + (experts->src[0]->type != GGML_TYPE_IQ4_NL && experts->src[0]->type != GGML_TYPE_Q8_0) || + experts->src[1]->type != GGML_TYPE_F32 || experts->src[2]->type != GGML_TYPE_I32 || + experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + const ggml_tensor * w = experts->src[0]; + const ggml_tensor * y = experts->src[1]; + const ggml_tensor * ids = experts->src[2]; + const int64_t n_used = ids->ne[0]; + return GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[ggml_cuda_get_device()].cc) && + n_used == 10 && w->ne[0] == 640 && w->ne[1] == 2560 && w->ne[2] == 512 && w->ne[3] == 1 && + y->ne[0] == 640 && ggml_nelements(y) == 640 * n_used && ggml_is_contiguous(y) && + ggml_nelements(ids) == n_used && ggml_is_contiguous(ids) && + weights->ne[0] == 1 && weights->ne[1] == n_used && ggml_nelements(weights) == n_used && ggml_is_contiguous(weights) && + ggml_nelements(experts) == 2560 * n_used && ggml_is_contiguous(experts) && + ggml_nelements(dst) == 2560 && ggml_is_contiguous(dst); +} + +void ggml_cuda_mul_mat_id_weighted_rdna3_5( + ggml_backend_cuda_context & ctx, const ggml_tensor * experts, const ggml_tensor * weights, ggml_tensor * dst) { + GGML_ASSERT(ggml_cuda_mul_mat_id_weighted_rdna3_5_ok(experts, weights, dst)); + const ggml_tensor * w = experts->src[0]; + const ggml_tensor * y = experts->src[1]; + const ggml_tensor * ids = experts->src[2]; + constexpr int n_used = 10; + constexpr int ncols = 640; + constexpr int nrows = 2560; + constexpr int nblocks = ncols / QK8_1; + + ggml_cuda_pool_alloc y_q8(ctx.pool(), n_used * nblocks); + quantize_row_q8_1_cuda( + (const float *) y->data, nullptr, y_q8.get(), w->type, + ncols, + y->nb[1] / sizeof(float), y->nb[2] / sizeof(float), y->nb[3] / sizeof(float), + y->ne[0], y->ne[1], y->ne[2], y->ne[3], ctx.stream()); + + const ggml_cuda_kernel_launch_params params(nrows, 32, 0, ctx.stream()); + if (w->type == GGML_TYPE_IQ4_NL) { + ggml_cuda_kernel_launch(mul_mat_id_iq4_nl_weighted_rdna3_5, params, + w->data, y_q8.get(), (const int32_t *) ids->data, (const float *) weights->data, (float *) dst->data, + nrows, nblocks, (int) (w->nb[1] / ggml_type_size(w->type)), + (int) (w->nb[2] / ggml_type_size(w->type)), nblocks); + } else { + ggml_cuda_kernel_launch(mul_mat_id_q8_0_weighted_rdna3_5, params, + w->data, y_q8.get(), (const int32_t *) ids->data, (const float *) weights->data, (float *) dst->data, + nrows, nblocks, (int) (w->nb[1] / ggml_type_size(w->type)), + (int) (w->nb[2] / ggml_type_size(w->type)), nblocks); + } +} + void ggml_cuda_mul_mat_vec_q( ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion) { @@ -2134,37 +3391,8 @@ void ggml_cuda_mul_mat_vec_q( } } - { - // RDNA3.5 token generation: quantize the activations inside the matvec kernel (bit-identical, one launch less). - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const int64_t ncols_dst_fq = ids ? ne2 : ne1; - const bool aligned = (uintptr_t) src1_d % 16 == 0 && nb11 % 16 == 0 && nb12 % 16 == 0 && nb13 % 16 == 0; - const bool fusion_ok = !fusion || (fusion->x_scale == nullptr && fusion->gate_scale == nullptr); - if (GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols_dst_fq == 1 && mmvq_fq_type_ok(src0->type) && aligned && fusion_ok && - ne10 % QK8_1 == 0 && (ne10 / QK8_1) * sizeof(block_q8_1) <= 16384) { - const int64_t s01 = src0->nb[1] / ts_src0; - const int64_t s02 = src0->nb[2] / ts_src0; - const int64_t s03 = src0->nb[3] / ts_src0; - const int64_t s11 = src1->nb[1] / ts_src1; - const int64_t s12 = src1->nb[2] / ts_src1; - const int64_t s13 = src1->nb[3] / ts_src1; - const int64_t s1 = dst->nb[1] / ts_dst; - const int64_t s2 = dst->nb[2] / ts_dst; - const int64_t s3 = dst->nb[3] / ts_dst; - - // For MUL_MAT_ID the memory layout is different than for MUL_MAT: - const int64_t nchannels_y = ids ? ne11 : ne12; - const int64_t nchannels_dst = ids ? ne1 : ne2; - const int64_t stride_col_dst = ids ? s2 : s1; - const int64_t stride_channel_dst = ids ? s1 : s2; - const int64_t stride_channel_y = ids ? s11 : s12; - - mul_mat_vec_q_fq_switch_type( - src0->data, src0->type, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, s01, stride_col_dst, - ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, - ne03, ne3, s03, s13, s3, stream); - return; - } + if (mul_mat_vec_q_fq_try(ctx, src0, src1, ids, dst, fusion, 1.0f, 0.0f, 0, /*launch=*/true)) { + return; } const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING); diff --git a/ggml/src/ggml-cuda/mmvq.cuh b/ggml/src/ggml-cuda/mmvq.cuh index 5605bf7a4e60..c050c68073b2 100644 --- a/ggml/src/ggml-cuda/mmvq.cuh +++ b/ggml/src/ggml-cuda/mmvq.cuh @@ -16,3 +16,39 @@ void ggml_cuda_op_mul_mat_vec_q( const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i, const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols, const int64_t src1_padded_row_size, cudaStream_t stream); + +// RDNA3.5 fused-quantize matvec with an activation prologue: y' = op(y_scale*y + y_bias) (op 1 = silu, 2 = sigmoid) +// applied while the activations are quantized in-kernel. `y` replaces dst->src[1] (same shape, F32, contiguous). +bool ggml_cuda_mul_mat_vec_q_fq_prologue_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * y); +void ggml_cuda_mul_mat_vec_q_fq_prologue(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const ggml_tensor * y, + float y_scale, float y_bias, int y_op); + +// RDNA3.5 grouped matvec: independent single-column matvecs sharing the same F32 activation vector `y` +// (Q8_0 rows via the fused-quantize path, optionally with a Q8_0 gate + GLU epilogue) in one launch. +#define GGML_CUDA_MMV_GROUP_MAX 4 +struct ggml_cuda_mmv_group_seg { + const ggml_tensor * w = nullptr; // [ncols, nrows] Q8_0 or F32 + const ggml_tensor * gate = nullptr; // optional Q8_0 gate weights of the same shape (GLU fusion) + ggml_tensor * dst = nullptr; // F32 [nrows]; the GLU output when gate != nullptr + ggml_glu_op glu_op = GGML_GLU_OP_SWIGLU; + float glu_limit = 0.0f; +}; +bool ggml_cuda_mmv_group_seg_ok(const ggml_tensor * w, const ggml_tensor * y); +void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, const ggml_cuda_mmv_group_seg * segs, int nseg); + +// RDNA3.5 qwen4exp hyper-connection up projection fused with the stream mix (one token): +// mixed[e] = mix_scale * sum_c xn[c*n_embd+e] * sigmoid((W_up * op(y_scale*y + y_bias))[c*n_embd+e]) + mix_bias +bool ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, const ggml_tensor * dst, int hc); +void ggml_cuda_mul_mat_vec_q_fq_hcmix(ggml_backend_cuda_context & ctx, const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, + ggml_tensor * dst, int hc, float y_scale, float y_bias, int y_op, float mix_scale, float mix_bias); + +// RDNA3.5 qwen4exp GDN output projection with the gated per-head norm in the prologue: +// y' = sigmoid(z) * (rms_norm_128(attn) * norm_w), then the fused-quantize matvec of dst->src[0] with y' +bool ggml_cuda_mul_mat_vec_q_fq_gdn_gate_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * z); +void ggml_cuda_mul_mat_vec_q_fq_gdn_gate(ggml_backend_cuda_context & ctx, ggml_tensor * dst, const float * attn, + const ggml_tensor * z, const ggml_tensor * norm_w, float eps); + +// RDNA3.5 one-token IQ4_NL MoE down projection with the selected-expert weighted sum in the kernel epilogue. +bool ggml_cuda_mul_mat_id_weighted_rdna3_5_ok(const ggml_tensor * experts, const ggml_tensor * weights, const ggml_tensor * dst); +void ggml_cuda_mul_mat_id_weighted_rdna3_5( + ggml_backend_cuda_context & ctx, const ggml_tensor * experts, const ggml_tensor * weights, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/sum.cu b/ggml/src/ggml-cuda/sum.cu index c56257b44066..d1f938905d0b 100644 --- a/ggml/src/ggml-cuda/sum.cu +++ b/ggml/src/ggml-cuda/sum.cu @@ -2,7 +2,12 @@ #include "sumrows.cuh" #ifdef GGML_CUDA_USE_CUB +#ifdef GGML_CUDA_CUB_IS_HIPCUB +#include +namespace cub = hipcub; +#else #include +#endif // GGML_CUDA_CUB_IS_HIPCUB using namespace cub; #endif // GGML_CUDA_USE_CUB diff --git a/ggml/src/ggml-cuda/top-k.cu b/ggml/src/ggml-cuda/top-k.cu index 9681cd293338..3febcc026043 100644 --- a/ggml/src/ggml-cuda/top-k.cu +++ b/ggml/src/ggml-cuda/top-k.cu @@ -2,12 +2,17 @@ #include "top-k.cuh" #ifdef GGML_CUDA_USE_CUB -# include -# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2) -# define CUB_TOP_K_AVAILABLE -# include +# ifdef GGML_CUDA_CUB_IS_HIPCUB +# include +namespace cub = hipcub; +# else +# include +# if (CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2) +# define CUB_TOP_K_AVAILABLE +# include using namespace cub; -# endif // CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2 +# endif // CCCL_MAJOR_VERSION >= 3 && CCCL_MINOR_VERSION >= 2 +# endif // GGML_CUDA_CUB_IS_HIPCUB #endif // GGML_CUDA_USE_CUB #ifdef CUB_TOP_K_AVAILABLE diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu index c8cec70bb320..183bc47ef6bf 100644 --- a/ggml/src/ggml-cuda/topk-moe.cu +++ b/ggml/src/ggml-cuda/topk-moe.cu @@ -87,7 +87,7 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co It is intended as fusion of softmax->top-k->get_rows pipeline for MoE models */ -template +template __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * logits, float * weights, int32_t * ids, @@ -168,10 +168,11 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * float wt_sum = 0.f; - float output_weights[experts_per_thread]; + constexpr int output_weights_per_thread = small_topk ? 1 : experts_per_thread; + float output_weights[output_weights_per_thread]; #pragma unroll - for (int i = 0; i < experts_per_thread; i++) { + for (int i = 0; i < output_weights_per_thread; i++) { output_weights[i] = 0.f; } @@ -234,7 +235,7 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * } if ((k & (WARP_SIZE - 1)) == threadIdx.x) { - output_weights[k / WARP_SIZE] = max_val; + output_weights[small_topk ? 0 : k / WARP_SIZE] = max_val; } if ((max_expert & (WARP_SIZE - 1)) == threadIdx.x) { @@ -250,17 +251,17 @@ __launch_bounds__(4 * WARP_SIZE, 1) __global__ void topk_moe_cuda(const float * wt_sum = max(wt_sum, clamp_val); const float inv_sum = 1.0f / wt_sum; - for (int i = 0; i < experts_per_thread; i++) { + for (int i = 0; i < output_weights_per_thread; i++) { output_weights[i] *= inv_sum; } } if (config.delayed_softmax) { - softmax_warp_inplace(output_weights, n_expert_used, threadIdx.x); + softmax_warp_inplace(output_weights, n_expert_used, threadIdx.x); } #pragma unroll - for (int i = 0; i < experts_per_thread; i++) { + for (int i = 0; i < output_weights_per_thread; i++) { const int idx = i * WARP_SIZE + threadIdx.x; if (idx < n_expert_used) { weights[idx] = output_weights[i] * scale_val; @@ -330,8 +331,13 @@ static void launch_topk_moe_cuda(ggml_backend_cuda_context & ctx, logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); break; case 512: - ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params, - logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); + if (n_expert_used <= WARP_SIZE) { + ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias, true>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); + } else { + ggml_cuda_kernel_launch(topk_moe_cuda<512, has_bias>, launch_params, + logits, weights, ids, bias, n_rows, n_expert_used, clamp_val, scale_val, config); + } break; case 576: ggml_cuda_kernel_launch(topk_moe_cuda<576, has_bias>, launch_params, diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index 71a7d24c5552..1508bb857a89 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -736,3 +736,47 @@ void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_n unary_cuda((const float *)src->data, (float *)sqr_node->data, k, stream); } } + +/* fused scale + unary */ + +template +static __global__ void scale_unary_kernel(const float * x, float * dst, const float scale, const float bias, const int k) { + const int i = blockDim.x*blockIdx.x + threadIdx.x; + + if (i >= k) { + return; + } + + // identical expression to scale_f32 in scale.cu, followed by the unary op + dst[i] = op(scale * x[i] + bias); +} + +template +static void scale_unary_cuda(const float * x, float * dst, const float scale, const float bias, const int k, cudaStream_t stream) { + const int num_blocks = (k + CUDA_NEG_BLOCK_SIZE - 1) / CUDA_NEG_BLOCK_SIZE; + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_NEG_BLOCK_SIZE, 0, stream); + ggml_cuda_kernel_launch(scale_unary_kernel, launch_params, x, dst, scale, bias, k); +} + +void ggml_cuda_op_scale_unary(ggml_backend_cuda_context & ctx, ggml_tensor * scale_node, ggml_tensor * unary_node) { + const ggml_tensor * src = scale_node->src[0]; + + GGML_ASSERT(src->type == GGML_TYPE_F32 && scale_node->type == GGML_TYPE_F32 && unary_node->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(src) && ggml_is_contiguous(unary_node)); + GGML_ASSERT(unary_node->src[0] == scale_node); + + const float scale = ggml_get_op_params_f32(scale_node, 0); + const float bias = ggml_get_op_params_f32(scale_node, 1); + const int k = ggml_nelements(src); + + switch (ggml_get_unary_op(unary_node)) { + case GGML_UNARY_OP_SILU: + scale_unary_cuda((const float *) src->data, (float *) unary_node->data, scale, bias, k, ctx.stream()); + break; + case GGML_UNARY_OP_SIGMOID: + scale_unary_cuda((const float *) src->data, (float *) unary_node->data, scale, bias, k, ctx.stream()); + break; + default: + GGML_ABORT("unsupported unary op for fused scale+unary"); + } +} diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuh index 291fe16eaa33..f2a543cffbe2 100644 --- a/ggml/src/ggml-cuda/unary.cuh +++ b/ggml/src/ggml-cuda/unary.cuh @@ -124,3 +124,6 @@ __device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, fl return ggml_cuda_op_silu_single(gate) * up; } + +// fused GGML_OP_SCALE + GGML_UNARY_OP_{SILU,SIGMOID}: dst = op(scale*x + bias) +void ggml_cuda_op_scale_unary(ggml_backend_cuda_context & ctx, ggml_tensor * scale_node, ggml_tensor * unary_node); diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index ec117c57dfbc..5bce2f4f02d8 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -1238,6 +1238,60 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( return d * sumi; } +// Two IQ3_S rows against the same activation block. The fused MoE gate/up path +// otherwise calls vec_dot_iq3_s_q8_1 twice and reloads all eight Q8_1 words. +// Keep each integer and floating-point accumulation independent so the results +// are bit-identical to the two scalar calls. +static __device__ __forceinline__ void vec_dot_iq3_s_q8_1_pair( + float & out0, float & out1, + const void * __restrict__ vbq0, const void * __restrict__ vbq1, + const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + const block_iq3_s * bq0 = (const block_iq3_s *) vbq0 + kbx; + const block_iq3_s * bq1 = (const block_iq3_s *) vbq1 + kbx; + + const int2 qs0_packed = make_int2(get_int_b2(bq0->qs, iqs + 0), get_int_b2(bq0->qs, iqs + 1)); + const int2 qs1_packed = make_int2(get_int_b2(bq1->qs, iqs + 0), get_int_b2(bq1->qs, iqs + 1)); + const uint8_t * qs0 = (const uint8_t *) &qs0_packed; + const uint8_t * qs1 = (const uint8_t *) &qs1_packed; + + const int qh0 = bq0->qh[iqs/2]; + const int qh1 = bq1->qh[iqs/2]; + const int signs0_packed = get_int_b2(bq0->signs, iqs/2); + const int signs1_packed = get_int_b2(bq1->signs, iqs/2); + const uint8_t * signs0 = (const uint8_t *) &signs0_packed; + const uint8_t * signs1 = (const uint8_t *) &signs1_packed; + + int sumi0 = 0; + int sumi1 = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1); + + const int2 grid0 = make_int2( + iq3s_grid[qs0[l0 + 0] | ((qh0 << (8 - l0)) & 0x100)], + iq3s_grid[qs0[l0 + 1] | ((qh0 << (7 - l0)) & 0x100)]); + const int s00 = __vcmpne4(((signs0[l0/2] & 0x03) << 7) | ((signs0[l0/2] & 0x0C) << 21), 0x00000000); + const int s01 = __vcmpne4(((signs0[l0/2] & 0x30) << 3) | ((signs0[l0/2] & 0xC0) << 17), 0x00000000); + sumi0 = ggml_cuda_dp4a(__vsub4(grid0.x ^ s00, s00), u0, sumi0); + sumi0 = ggml_cuda_dp4a(__vsub4(grid0.y ^ s01, s01), u1, sumi0); + + const int2 grid1 = make_int2( + iq3s_grid[qs1[l0 + 0] | ((qh1 << (8 - l0)) & 0x100)], + iq3s_grid[qs1[l0 + 1] | ((qh1 << (7 - l0)) & 0x100)]); + const int s10 = __vcmpne4(((signs1[l0/2] & 0x03) << 7) | ((signs1[l0/2] & 0x0C) << 21), 0x00000000); + const int s11 = __vcmpne4(((signs1[l0/2] & 0x30) << 3) | ((signs1[l0/2] & 0xC0) << 17), 0x00000000); + sumi1 = ggml_cuda_dp4a(__vsub4(grid1.x ^ s10, s10), u0, sumi1); + sumi1 = ggml_cuda_dp4a(__vsub4(grid1.y ^ s11, s11), u1, sumi1); + } + + sumi0 *= 1 + 2*((bq0->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F); + sumi1 *= 1 + 2*((bq1->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F); + const float d8 = __low2float(bq8_1[iqs/2].ds); + out0 = (__half2float(bq0->d) * d8) * sumi0; + out1 = (__half2float(bq1->d) * d8) * sumi1; +} + #define VDR_IQ1_S_Q8_1_MMVQ 1 #define VDR_IQ1_S_Q8_1_MMQ 1 diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 9aa558f3f4ca..7055a4022f43 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -139,6 +139,9 @@ #define cudaGraphExecUpdate hipGraphExecUpdate #define cudaStreamCaptureModeRelaxed hipStreamCaptureModeRelaxed #define cudaStreamBeginCapture hipStreamBeginCapture +#define cudaStreamIsCapturing hipStreamIsCapturing +#define cudaStreamCaptureStatus hipStreamCaptureStatus +#define cudaStreamCaptureStatusNone hipStreamCaptureStatusNone #define cudaGraph_t hipGraph_t #define cudaStream_t hipStream_t #define cudaSuccess hipSuccess diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 02202f8c5693..ab560e7d5e90 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -232,6 +233,10 @@ llama_context::llama_context( cparams.fused_gdn_ar = true; cparams.fused_gdn_ch = true; cparams.auto_fgdn = false; + // LLAMA_FUSED_GDN_CH=0 selects the ggml-op chunked delta-net formulation for multi-token ubatches + if (const char * env = getenv("LLAMA_FUSED_GDN_CH")) { + cparams.fused_gdn_ch = atoi(env) != 0; + } cparams.fused_lid = true; cparams.auto_flid = false; diff --git a/src/models/models.h b/src/models/models.h index f285d3d315ac..1ca6678be989 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2281,6 +2281,7 @@ struct llama_model_qwen4exp : public llama_model_base { llama_model_qwen4exp(const struct llama_model_params & params) : llama_model_base(params) {} class llm_graph_input_qsa; + class llm_graph_input_qsa_k; // Set when the n-gram table is left on disk (--ngram-on-disk): per_layer_tok_embd is // counted as created but never allocated, mapped or read, and build_ple takes its @@ -2332,6 +2333,13 @@ struct llama_model_qwen4exp : public llama_model_base { // the QSA cache layout inputs do not depend on the layer, only on its compress ratio, // so the layers sharing a ratio share one input set std::map qsa_inps; + llm_graph_input_qsa_k * qsa_k_inp = nullptr; + + // QSA: store this ubatch's raw indexer keys in the indexer cache (no selection) + void build_qsa_store_k( + const llama_memory_hybrid_idx_context * mctx_hyb, + ggml_tensor * cur, + int il); // QSA: token indices this layer's queries may attend to, or nullptr for dense ggml_tensor * build_qsa_top_k( diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index ca47c3706cae..56a1e8b67591 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -6,6 +6,14 @@ #include #include +#include +#include +#include + +#if defined(__linux__) || defined(__APPLE__) +#include +#include +#endif void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); @@ -412,12 +420,27 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( // grouped RMSNorm: reduce over one stream, then scale all streams with the [hc_dim] gamma // the converter folded each gamma to (1 + w) + // + // The gamma is viewed as [n_embd, hc] and applied to the [n_embd, hc, T] norm output so that the + // RMS_NORM and MUL nodes are adjacent in the graph (backends fuse that pair). The view of the + // weight is expanded first so it does not land between them. + ggml_tensor * w_norm_hc = ggml_reshape_2d(ctx0, w_norm, n_embd, hc); + ggml_build_forward_expand(gf, w_norm_hc); + ggml_tensor * xn = ggml_rms_norm(ctx0, x, hparams.f_norm_rms_eps); + xn = ggml_mul(ctx0, xn, w_norm_hc); xn = ggml_reshape_2d(ctx0, xn, hc_dim, nt); - xn = ggml_mul(ctx0, xn, w_norm); cb(xn, "hc_norm", il); ggml_tensor * lo = build_lora_mm(w_down, xn); + if (inject) { + // the inject projection reads the same xn as the down projection: emit the two matvecs back to back so + // that the backend can launch them as one grouped kernel (the inject result is only used by hc_combine) + ggml_build_forward_expand(gf, lo); + *inject = build_lora_mm(w_inject, xn); + cb(*inject, "hc_inject", il); + ggml_build_forward_expand(gf, *inject); + } lo = ggml_silu(ctx0, ggml_scale(ctx0, lo, 1.0f / (float) hc)); ggml_tensor * gate = ggml_sigmoid(ctx0, build_lora_mm(w_up, lo)); cb(gate, "hc_gate", il); @@ -438,11 +461,6 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_mix( mixed = ggml_scale(ctx0, mixed, 1.0f / (float) hc); cb(mixed, "hc_mixed", il); - if (inject) { - *inject = build_lora_mm(w_inject, xn); - cb(*inject, "hc_inject", il); - } - return mixed; } @@ -454,11 +472,24 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( const int64_t hc = hparams.dsv4_hc_mult; const int64_t nt = residual->ne[2]; + // block_out and inject die at this combine; keep their buffers from being handed to the next grouped norm + // output (the backend fuses combine + norm into one kernel whose blocks all read block_out/inject while + // writing their own stream of the norm output, so the two must not alias) + ggml_set_output(block_out); + ggml_set_output(inject); + // 2*sigmoid centres the scatter weights on 1, so a zero injection is a plain residual add ggml_tensor * w = ggml_sigmoid(ctx0, ggml_scale(ctx0, inject, 1.0f / (float) hc)); w = ggml_scale(ctx0, w, 2.0f); w = ggml_reshape_3d(ctx0, w, 1, hc, nt); + // Emit the block output first and the scatter-weight chain right after it, so that + // scale -> sigmoid -> scale -> repeat -> mul -> add is one contiguous node run that + // backends can fuse into a single kernel. + ggml_build_forward_expand(gf, residual); + ggml_build_forward_expand(gf, block_out); + ggml_build_forward_expand(gf, w); + ggml_tensor * b = ggml_reshape_3d(ctx0, block_out, n_embd, 1, nt); b = ggml_repeat_4d(ctx0, b, n_embd, hc, nt, 1); @@ -682,6 +713,54 @@ class llama_model_qwen4exp::llm_graph_input_qsa : public llm_graph_input_i { const bool blk_bias; }; +// indexer-key store only: the cache is written every ubatch, the selection is skipped while it fits the budget +class llama_model_qwen4exp::llm_graph_input_qsa_k : public llm_graph_input_i { +public: + llm_graph_input_qsa_k(const llama_memory_hybrid_idx_context * mctx) : mctx(mctx) {} + virtual ~llm_graph_input_qsa_k() = default; + + void set_input(const llama_ubatch * ubatch) override { + mctx->get_idx()->set_input_k_idxs(k_idxs, ubatch); + } + + bool can_reuse(const llm_graph_params & params) override { + mctx = static_cast(params.mctx); + + const auto * idx = mctx->get_idx(); + if (idx == nullptr) { + return false; + } + + return k_idxs->ne[0] == params.ubatch.n_tokens; + } + + ggml_tensor * k_idxs = nullptr; // I32 [n_tokens] + + const llama_memory_hybrid_idx_context * mctx; +}; + +void llama_model_qwen4exp::graph::build_qsa_store_k( + const llama_memory_hybrid_idx_context * mctx_hyb, + ggml_tensor * cur, + int il) { + const llama_kv_cache_context * mctx_idx = mctx_hyb->get_idx(); + + const int64_t idx_dim = hparams.indexer_head_size; + + if (qsa_k_inp == nullptr) { + auto inp = std::make_unique(mctx_hyb); + inp->k_idxs = mctx_idx->build_input_k_idxs(ctx0, ubatch); + qsa_k_inp = inp.get(); + res->add_input(std::move(inp)); + } + + ggml_tensor * k_raw = build_lora_mm(model.layers[il].index_k_proj, cur); + k_raw = ggml_reshape_3d(ctx0, k_raw, idx_dim, 1, n_tokens); + cb(k_raw, "indexer_k_raw", il); + + ggml_build_forward_expand(gf, mctx_idx->cpy_k(ctx0, k_raw, qsa_k_inp->k_idxs, il)); +} + ggml_tensor * llama_model_qwen4exp::graph::build_qsa_top_k( const llama_memory_hybrid_idx_context * mctx_hyb, ggml_tensor * cur, @@ -919,7 +998,26 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( // dense, see build_layer_attn's caller in graph_mtp), no cache, or no ratio means dense const bool qsa = mctx_hyb != nullptr && mctx_hyb->get_idx() != nullptr && hparams.dsv4_compress_ratios[il] > 0; - ggml_tensor * top_k = qsa ? build_qsa_top_k(mctx_hyb, cur, inp_pos, inp->get_kq_mask(), sections, il) : nullptr; + ggml_tensor * top_k = nullptr; + if (qsa) { + // The selection keeps min(n_kv, top_k + ratio - 1) cells. While the cache holds no more + // cells than that, every cell is selected and the top-k mask is a no-op on the causal + // mask, so attend dense (same result) and only store this ubatch's indexer keys for + // when the context grows past the budget. Mirrors the reference "seqlen <= topk" path. + const int64_t r = hparams.dsv4_compress_ratios[il]; + const int64_t n_kv = mctx_hyb->get_idx()->get_n_kv(); + const int64_t width = (int64_t) hparams.indexer_top_k + r - 1; + // LLAMA_QSA_DENSE_SHORTCUT=0 forces the selection path (validation) + static const bool shortcut = [] { + const char * env = getenv("LLAMA_QSA_DENSE_SHORTCUT"); + return env == nullptr || atoi(env) != 0; + }(); + if (shortcut && n_kv <= width) { + build_qsa_store_k(mctx_hyb, cur, il); + } else { + top_k = build_qsa_top_k(mctx_hyb, cur, inp_pos, inp->get_kq_mask(), sections, il); + } + } // Qwen3Next uses a single Q projection that outputs query + gate ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ] @@ -1019,13 +1117,21 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear( beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs); cb(beta, "beta", il); - beta = ggml_sigmoid(ctx0, beta); - cb(beta, "beta_sigmoid", il); - ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s); alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs); cb(alpha, "alpha", il); + // emit the two small projections ahead of the recurrence so that the elementwise chain + // conv -> norms -> gate -> beta -> gated_delta_net -> state copy -> norm is contiguous in the + // graph (the CUDA backend fuses that decode chain into a single kernel) + ggml_build_forward_expand(gf, qkv_mixed); + ggml_build_forward_expand(gf, z); + ggml_build_forward_expand(gf, beta); + ggml_build_forward_expand(gf, alpha); + + beta = ggml_sigmoid(ctx0, beta); + cb(beta, "beta_sigmoid", il); + ggml_tensor * alpha_biased = ggml_add(ctx0, alpha, model.layers[il].ssm_dt); ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased); cb(alpha_softplus, "a_softplus", il); @@ -1121,6 +1227,21 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn_linear( ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, const int il) { GGML_ASSERT(model.layers[il].ffn_gate_inp != nullptr); + // Shared-expert gate/up projections first: they read the same `cur` as the router matvec that opens + // build_moe_ffn, so the backend can launch [up, gate, glu, router] as one grouped kernel. Same math as + // build_ffn(LLM_FFN_SILU, LLM_FFN_PAR) without a swiglu clamp. + ggml_tensor * shexp_act = nullptr; + if (model.layers[il].ffn_up_shexp != nullptr && model.layers[il].ffn_gate_shexp != nullptr && + !(il >= 0 && hparams.swiglu_clamp_shexp[il] > 1e-6f)) { + ggml_tensor * up = build_lora_mm(model.layers[il].ffn_up_shexp, cur, model.layers[il].ffn_up_shexp_s); + cb(up, "ffn_up", il); + ggml_tensor * gate = build_lora_mm(model.layers[il].ffn_gate_shexp, cur, model.layers[il].ffn_gate_shexp_s); + cb(gate, "ffn_gate", il); + shexp_act = ggml_swiglu_split(ctx0, gate, up); + cb(shexp_act, "ffn_swiglu", il); + ggml_build_forward_expand(gf, shexp_act); + } + ggml_tensor * moe_out = build_moe_ffn(cur, model.layers[il].ffn_gate_inp, @@ -1140,7 +1261,8 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_ffn(ggml_tensor * cur, co // shared experts, as in the Qwen3Next reference if (model.layers[il].ffn_up_shexp != nullptr) { - ggml_tensor * ffn_shexp = + ggml_tensor * ffn_shexp = shexp_act ? + build_lora_mm(model.layers[il].ffn_down_shexp, shexp_act, model.layers[il].ffn_down_shexp_s) : build_ffn(cur, model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s, model.layers[il].ffn_gate_shexp, NULL, model.layers[il].ffn_gate_shexp_s, @@ -1272,6 +1394,52 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { } } + if (embd != nullptr && !pmodel.ple_disk) { + // host-resident table: dequantize the rows exactly as the CPU get_rows would (same to_float) + const ggml_tensor * tab = pmodel.per_layer_tok_embd; + GGML_ASSERT(tab && tab->data && rows == nullptr); + const size_t row_size = ggml_row_size(tab->type, tab->ne[0]); + const int64_t dim = tab->ne[0]; + GGML_ASSERT(dim == hp.ple_head_dim); + + const auto * traits = ggml_get_type_traits(tab->type); + +#if defined(__linux__) || defined(__APPLE__) + // memory-mapped table, random rows: queue every page read at once instead of taking the + // faults one after another (~0.2 ms each from NVMe) while dequantizing below + { + const long page = sysconf(_SC_PAGESIZE); + if (page > 0) { + const uintptr_t base = (uintptr_t) tab->data; + const uintptr_t mask = ~((uintptr_t) page - 1); + uintptr_t prev_page = 0; + for (const int32_t r : idx) { + const uintptr_t a0 = (base + (size_t) r * row_size) & mask; + const uintptr_t a1 = base + (size_t) r * row_size + row_size; + if (a0 != prev_page) { + madvise((void *) a0, a1 - a0, MADV_WILLNEED); + prev_page = a0; + } + } + } + } +#endif + + embd_buf.resize(idx.size() * (size_t) dim); + for (size_t k = 0; k < idx.size(); ++k) { + const char * src = (const char *) tab->data + (size_t) idx[k] * row_size; + float * dst = embd_buf.data() + k * (size_t) dim; + if (tab->type == GGML_TYPE_F32) { + memcpy(dst, src, dim * sizeof(float)); + } else { + GGML_ASSERT(traits->to_float != nullptr); + traits->to_float(src, dst, dim); + } + } + ggml_backend_tensor_set(embd, embd_buf.data(), 0, embd_buf.size() * sizeof(float)); + return; + } + if (pmodel.ple_disk) { // idx is already exactly what the reader wants: one global row per (token, head), // token-major, with ple_head_offsets folded in by the loop above. The previous @@ -1285,6 +1453,30 @@ void llm_graph_input_ple::set_input(const llama_ubatch * ubatch) { return; } +#if defined(__linux__) || defined(__APPLE__) + // The table is memory-mapped and the gathered rows are effectively random, so every row is a + // page fault the CPU get_rows would otherwise take one after another (~0.2 ms each from NVMe). + // Ask the kernel to read all of them in parallel now; the gather then finds them (or the I/O in + // flight) in the page cache. + if (const ggml_tensor * tab = pmodel.per_layer_tok_embd; tab && tab->data) { + const long page = sysconf(_SC_PAGESIZE); + if (page > 0) { + const uintptr_t base = (uintptr_t) tab->data; + const size_t row_size = ggml_row_size(tab->type, tab->ne[0]); + const uintptr_t mask = ~((uintptr_t) page - 1); + uintptr_t prev_page = 0; + for (const int32_t r : idx) { + const uintptr_t a0 = (base + (size_t) r * row_size) & mask; + const uintptr_t a1 = base + (size_t) r * row_size + row_size; + if (a0 != prev_page) { + madvise((void *) a0, a1 - a0, MADV_WILLNEED); + prev_page = a0; + } + } + } + } +#endif + ggml_backend_tensor_set(rows, idx.data(), 0, idx.size()*ggml_element_size(rows)); } @@ -1331,7 +1523,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at( conv_states_all->nb[1], kv_head * row_size); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst)); + // copy the strided tail directly (no intermediate cont): this is the same node sequence as + // llm_build_delta_net_base::build_conv_state, which the backends' fused decode paths match + ggml_build_forward_expand(gf, ggml_cpy(ctx0, tail, dst)); return conv_input; } @@ -1347,8 +1541,16 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple( const auto & qmodel = static_cast(model); ggml_tensor * rows = nullptr; ggml_tensor * emb = nullptr; - if (qmodel.ple_disk) { - // table on disk: the gather happened in set_input and arrives as an F32 input, + + // A host-resident table (memory-mapped or loaded into RAM) is gathered on the host too: a + // get_rows node on the CPU backend would cut the graph into three splits (GPU, CPU, GPU) with + // an extra graph launch and a device sync per ubatch. Only a table that lives on a device is + // gathered by the graph. + const bool host_gather = qmodel.ple_disk || + (model.per_layer_tok_embd && model.per_layer_tok_embd->buffer && ggml_backend_buffer_is_host(model.per_layer_tok_embd->buffer)); + + if (host_gather) { + // the gather happens in set_input and arrives as an F32 input, // token-major, which reshapes to what the per-head gathers + concat produce ple_inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.ple_head_dim, n_heads * n_tokens); ggml_set_input(ple_inp->embd); @@ -1383,11 +1585,13 @@ ggml_tensor * llama_model_qwen4exp::graph::build_ple( // both norms group over one hc stream, with a weight over the whole hc*n_embd layout auto grouped_norm = [&](ggml_tensor * x, ggml_tensor * w) { + // gamma viewed per stream so that rms_norm and mul are adjacent nodes (fusable) + ggml_tensor * w_hc = ggml_reshape_2d(ctx0, w, n_embd, hc); + ggml_build_forward_expand(gf, w_hc); ggml_tensor * t = ggml_reshape_3d(ctx0, x, n_embd, hc, n_tokens); t = ggml_rms_norm(ctx0, t, hparams.f_norm_rms_eps); - t = ggml_reshape_2d(ctx0, t, hc_dim, n_tokens); - t = ggml_mul(ctx0, t, w); - return ggml_reshape_3d(ctx0, t, n_embd, hc, n_tokens); + t = ggml_mul(ctx0, t, w_hc); + return t; }; key = grouped_norm(key, model.layers[il].ple_norm_key); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 1e87615b57a4..a2dbaf4e1555 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4945,9 +4945,11 @@ struct test_swiglu_q8_mmq : public test_case { const int64_t m; const int64_t n; const int64_t k; + const ggml_type type_down; - test_swiglu_q8_mmq(bool use_id, int n_mats, int n_used, int64_t m, int64_t n, int64_t k) - : use_id(use_id), n_mats(n_mats), n_used(n_used), m(m), n(n), k(k) {} + test_swiglu_q8_mmq(bool use_id, int n_mats, int n_used, int64_t m, int64_t n, int64_t k, + ggml_type type_down = GGML_TYPE_Q8_0) + : use_id(use_id), n_mats(n_mats), n_used(n_used), m(m), n(n), k(k), type_down(type_down) {} std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -4955,7 +4957,7 @@ struct test_swiglu_q8_mmq : public test_case { } std::string vars() override { - return VARS_TO_STR6(use_id, n_mats, n_used, m, n, k); + return VARS_TO_STR7(use_id, n_mats, n_used, m, n, k, type_down); } bool run_whole_graph() override { return true; } @@ -4965,13 +4967,13 @@ struct test_swiglu_q8_mmq : public test_case { if (!use_id) { ggml_tensor * gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); ggml_tensor * up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n); - ggml_tensor * down = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m); + ggml_tensor * down = ggml_new_tensor_2d(ctx, type_down, k, m); return ggml_mul_mat(ctx, down, ggml_swiglu_split(ctx, gate, up)); } ggml_tensor * gate = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n); ggml_tensor * up = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n); - ggml_tensor * down = ggml_new_tensor_3d(ctx, GGML_TYPE_Q8_0, k, m, n_mats); + ggml_tensor * down = ggml_new_tensor_3d(ctx, type_down, k, m, n_mats); ggml_tensor * ids_all = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); ggml_set_name(ids_all, "ids"); ggml_tensor * ids = ggml_view_2d(ctx, ids_all, n_used, n, ids_all->nb[1], 0); @@ -4983,6 +4985,43 @@ struct test_swiglu_q8_mmq : public test_case { } }; +// Qwen3.8-Flash-Next decode hot path: IQ3_S gate and up projections are +// fused with SwiGLU by ggml-cuda, with 512 experts and 10 selected experts. +struct test_swiglu_iq3_mmvq : public test_case { + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SWIGLU_IQ3_MMVQ"; + } + + std::string vars() override { + return "n_mats=512,n_used=10,m=640,n=1,k=2560"; + } + + bool run_whole_graph() override { return true; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + constexpr int n_mats = 512; + constexpr int n_used = 10; + constexpr int64_t m = 640; + constexpr int64_t n = 1; + constexpr int64_t k = 2560; + + ggml_tensor * gate_w = ggml_new_tensor_3d(ctx, GGML_TYPE_IQ3_S, k, m, n_mats); + ggml_tensor * up_w = ggml_new_tensor_3d(ctx, GGML_TYPE_IQ3_S, k, m, n_mats); + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_used, n); + ggml_tensor * ids_all = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n); + ggml_set_name(ids_all, "ids"); + ggml_tensor * ids = ggml_view_2d(ctx, ids_all, n_used, n, ids_all->nb[1], 0); + ggml_tensor * gate = ggml_mul_mat_id(ctx, gate_w, input, ids); + ggml_tensor * up = ggml_mul_mat_id(ctx, up_w, input, ids); + return ggml_swiglu_split(ctx, gate, up); + } + + void initialize_tensors(ggml_context * ctx) override { + init_mul_mat_id_tensors(ctx, 512); + } +}; + // GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL struct test_mul_mat_id_fusion : public test_case { const ggml_type type_a; @@ -5065,6 +5104,113 @@ struct test_mul_mat_id_fusion : public test_case { }; // GGML_OP_OUT_PROD +// GGML_OP_MUL_MAT group: independent single-column matvecs on one activation vector +// (RDNA3.5 CUDA launches [Q8_0 | Q8_0 gate/up + GLU | F32 ...] as one grouped kernel) +struct test_mmv_group : public test_case { + const int64_t k; + const int64_t m_q8; // rows of a plain Q8_0 segment (0 = none) + const int64_t m_glu; // rows of a Q8_0 gate/up + swiglu segment (0 = none) + const int64_t m_f32a; // rows of an F32 segment (0 = none) + const int64_t m_f32b; // rows of a second F32 segment (0 = none) + + std::string vars() override { return VARS_TO_STR5(k, m_q8, m_glu, m_f32a, m_f32b); } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "MMV_GROUP"; + } + + bool run_whole_graph() override { return true; } + + double max_nmse_err() override { return 5e-4; } + + test_mmv_group(int64_t k, int64_t m_q8, int64_t m_glu, int64_t m_f32a, int64_t m_f32b) + : k(k), m_q8(m_q8), m_glu(m_glu), m_f32a(m_f32a), m_f32b(m_f32b) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * y = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, 1); + ggml_set_name(y, "y"); + + ggml_tensor * out = nullptr; + auto append = [&](ggml_tensor * t) { + out = out ? ggml_concat(ctx, out, t, 0) : t; + }; + + if (m_q8 > 0) { + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m_q8); + ggml_set_name(w, "w_q8"); + append(ggml_mul_mat(ctx, w, y)); + } + if (m_glu > 0) { + ggml_tensor * wg = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m_glu); + ggml_tensor * wu = ggml_new_tensor_2d(ctx, GGML_TYPE_Q8_0, k, m_glu); + ggml_set_name(wg, "w_gate"); + ggml_set_name(wu, "w_up"); + append(ggml_swiglu_split(ctx, ggml_mul_mat(ctx, wg, y), ggml_mul_mat(ctx, wu, y))); + } + if (m_f32a > 0) { + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, m_f32a); + ggml_set_name(w, "w_f32a"); + append(ggml_mul_mat(ctx, w, y)); + } + if (m_f32b > 0) { + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, m_f32b); + ggml_set_name(w, "w_f32b"); + append(ggml_mul_mat(ctx, w, y)); + } + ggml_set_name(out, "out"); + return out; + } +}; + +// qwen4exp hyper-connection up projection + stream mix chain (decode): +// silu(scale(lo)) -> mul_mat(W_up) -> sigmoid -> mul(xn) -> reshape -> view/cont -> adds -> scale +struct test_hc_up_mix : public test_case { + const ggml_type type_w; + const int64_t k; // low-rank width (columns of W_up) + const int64_t n_embd; + const int64_t hc; + const int64_t n_tokens; + + std::string vars() override { return VARS_TO_STR5(type_w, k, n_embd, hc, n_tokens); } + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "HC_UP_MIX"; + } + + bool run_whole_graph() override { return true; } + + double max_nmse_err() override { return 5e-4; } + + test_hc_up_mix(ggml_type type_w, int64_t k, int64_t n_embd, int64_t hc, int64_t n_tokens) + : type_w(type_w), k(k), n_embd(n_embd), hc(hc), n_tokens(n_tokens) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * lo = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n_tokens); + ggml_set_name(lo, "lo"); + ggml_tensor * xn = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hc * n_embd, n_tokens); + ggml_set_name(xn, "xn"); + ggml_tensor * w = ggml_new_tensor_2d(ctx, type_w, k, hc * n_embd); + ggml_set_name(w, "w_up"); + + ggml_tensor * act = ggml_silu(ctx, ggml_scale(ctx, lo, 1.0f / (float) hc)); + ggml_tensor * gate = ggml_sigmoid(ctx, ggml_mul_mat(ctx, w, act)); + ggml_tensor * gated = ggml_mul(ctx, xn, gate); + gated = ggml_reshape_3d(ctx, gated, n_embd, hc, n_tokens); + ggml_tensor * mixed = ggml_view_2d(ctx, gated, n_embd, n_tokens, ggml_row_size(gated->type, n_embd) * hc, 0); + mixed = ggml_cont(ctx, mixed); + for (int64_t c = 1; c < hc; ++c) { + ggml_tensor * s = ggml_view_2d(ctx, gated, n_embd, n_tokens, ggml_row_size(gated->type, n_embd) * hc, + ggml_row_size(gated->type, n_embd) * c); + mixed = ggml_add(ctx, mixed, s); + } + mixed = ggml_scale(ctx, mixed, 1.0f / (float) hc); + ggml_set_name(mixed, "mixed"); + return mixed; + } +}; + struct test_out_prod : public test_case { const ggml_type type_a; const ggml_type type_b; @@ -9412,6 +9558,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_shared_mul_add(2048, 32)); test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2048, 32, 512)); test_cases.emplace_back(new test_swiglu_q8_mmq(true, 16, 8, 2048, 32, 512)); + test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2560, 128, 640)); + test_cases.emplace_back(new test_swiglu_q8_mmq(true, 512, 10, 2560, 128, 640)); + test_cases.emplace_back(new test_swiglu_q8_mmq(true, 512, 10, 2560, 128, 640, GGML_TYPE_IQ4_NL)); for (auto multi_add : {false, true}) { for (auto set_rows : {false, true}) { @@ -9517,7 +9666,32 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); #endif + // qwen4exp hyper-connection up projection + stream mix (fused on RDNA3.5 for one token) + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 320, 2560, 4, 1)); + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 320, 2560, 4, 3)); + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 256, 1000, 3, 1)); + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 512, 640, 4, 1)); + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_F32, 320, 2560, 4, 1)); + + // grouped single-column matvecs on one activation vector (qwen4exp decode: GDN qkv/z/beta/alpha, + // hyper-connection down + inject, shared-expert gate/up + router) + test_cases.emplace_back(new test_mmv_group(2560, 10240, 0, 48, 48)); + test_cases.emplace_back(new test_mmv_group(2560, 10240, 6144, 0, 0)); + test_cases.emplace_back(new test_mmv_group(2560, 1024, 6144, 48, 48)); + test_cases.emplace_back(new test_mmv_group(10240, 320, 0, 4, 0)); + test_cases.emplace_back(new test_mmv_group(2560, 0, 640, 512, 0)); + test_cases.emplace_back(new test_mmv_group(2560, 0, 640, 512, 1)); + test_cases.emplace_back(new test_mmv_group(2560, 0, 0, 48, 48)); + test_cases.emplace_back(new test_mmv_group(640, 2560, 0, 33, 0)); + test_cases.emplace_back(new test_mmv_group(256, 17, 5, 3, 129)); + // 256-expert MoE with 4/16/32/128 rows per expert, exercises the RDNA3.5 mul_mat_id tile selection + // Qwen3.8-Flash-Next (qwen4exp) UD-IQ4_XS experts: 512 experts, 10 active, gate/up IQ3_S [2560 -> 640], down IQ4_NL [640 -> 2560] + for (int n : {1, 8, 64, 512, 2048}) { + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ3_S, GGML_TYPE_F32, 512, 10, false, 640, n, 2560)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ4_NL, GGML_TYPE_F32, 512, 10, false, 2560, n, 640)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 512, 10, false, 2560, n, 640)); + } test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 128, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 512, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 1024, 2048)); @@ -10141,6 +10315,12 @@ static std::vector> make_test_cases_eval() { } } } + // QSA (qwen4exp / DeepSeek-V3.2 style indexer): k == n_kv while the context fits the budget, [n_kv, n_tokens, 1, n_stream] + for (int n : {256, 512, 1024, 2048, 2304}) { + for (int nt : {1, 2, 512}) { + test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {n, nt, 1, 1}, std::min(n, 2051))); + } + } for (int k : {4, 8, 16, 32}) { for (int nrows : {1, 8, 16}) { test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {202048, nrows, 1, 1}, k)); @@ -10495,6 +10675,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_topk_moe({129, 1, 1, 1}, 128, with_norm, bias_probs, gate, scale_w)); test_cases.emplace_back(new test_topk_moe({160, 4, 1, 1}, 160, with_norm, bias_probs, gate, scale_w)); test_cases.emplace_back(new test_topk_moe({256, 22, 1, 1}, 6, with_norm, bias_probs, gate, scale_w)); // Used by DeepSeek-V4 + test_cases.emplace_back(new test_topk_moe({512, 22, 1, 1}, 10, with_norm, bias_probs, gate, scale_w)); // Used by Qwen3.8-Flash-Next (qwen4exp) test_cases.emplace_back(new test_topk_moe({288, 22, 1, 1}, 8, with_norm, bias_probs, gate, scale_w)); // Used by StepFun 3.7 } } @@ -10590,6 +10771,11 @@ static std::vector> make_test_cases_eval() { static std::vector> make_test_cases_perf() { std::vector> test_cases; + // Qwen3.8-Flash-Next decode hot path: fused gate/up uses this exact shape. + test_cases.emplace_back(new test_mul_mat_id( + GGML_TYPE_IQ3_S, GGML_TYPE_F32, 512, 10, false, 640, 1, 2560)); + test_cases.emplace_back(new test_swiglu_iq3_mmvq()); + // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands // note: same bytes either way, so a backend that indexes them differently shows it here for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { @@ -10989,6 +11175,7 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 2048, 1)); // PP-2048 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 48, 128, 2048, 1)); // qwen4exp H48/S128 PP-2048 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 1024, 1)); // H64/S128 PP-1024 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 64, 128, 2048, 1)); // H64/S128 PP-2048 // Small model configs (fewer heads = less GPU occupancy for autoregressive) From 930a8bdad3d6a1ed7011df038962a188e7432606 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:13:04 +0200 Subject: [PATCH 26/30] hip: address PR review correctness issues Constrain architecture-specific paths, validate fused GDN use counts, narrow graph synchronization, remove disabled experiments, and strengthen fusion tests. Assisted-by: OpenCode --- ggml/src/ggml-cuda/fattn-tile.cuh | 2 +- ggml/src/ggml-cuda/fattn.cu | 3 +- ggml/src/ggml-cuda/ggml-cuda.cu | 132 +++++-------- ggml/src/ggml-cuda/mmid.cu | 3 +- ggml/src/ggml-cuda/mmvq.cu | 298 +++--------------------------- ggml/src/ggml-cuda/mmvq.cuh | 6 - src/llama-context.cpp | 11 +- tests/test-backend-ops.cpp | 69 +++---- 8 files changed, 111 insertions(+), 413 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index 53029262aec4..e6df7f1f640d 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -1344,7 +1344,7 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm const ggml_tensor * KQV = dst; const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; - const ggml_tensor * V = dst->src[2]; + [[maybe_unused]] const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; float max_bias = 0.0f; diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 3a5faa5fbaad..574e48da826e 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -357,10 +357,11 @@ static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) { static bool ggml_cuda_fattn_tile_q8_0_KV_supported(const ggml_tensor * dst) { #ifdef GGML_USE_HIP + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; - return Q->ne[1] == 1 && K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_Q8_0 && Q->ne[0] == K->ne[0] && K->ne[0] == V->ne[0] && + return GGML_CUDA_CC_IS_RDNA3_5(cc) && Q->ne[1] == 1 && K->type == GGML_TYPE_Q8_0 && V->type == GGML_TYPE_Q8_0 && Q->ne[0] == K->ne[0] && K->ne[0] == V->ne[0] && (Q->ne[0] == 64 || Q->ne[0] == 128 || Q->ne[0] == 256); #else GGML_UNUSED(dst); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 088a66735c6c..029b3fdd75af 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -726,71 +726,17 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { // cuda buffer -#if defined(GGML_USE_HIP) && defined(__linux__) -#include -#include -#include -// Integrated GPUs (UMA): large buffers can be backed by transparent-huge-page host memory that is registered -// with the device. The GPU then maps 2 MiB fragments instead of the 4 KiB pages of hipMalloc'd GTT memory, -// which removes most of the TLB misses of the small-matrix weight streams during token generation -// (gfx1151: 3.5 MiB Q8_0 matvec 22 -> 18 us). Enabled with GGML_CUDA_THP_BUFFERS=1. -static bool ggml_cuda_thp_buffers_enabled() { - static const bool enabled = getenv("GGML_CUDA_THP_BUFFERS") != nullptr && atoi(getenv("GGML_CUDA_THP_BUFFERS")) != 0; - return enabled; -} - -static void * ggml_cuda_thp_alloc(size_t size, size_t * alloc_size_out) { - const size_t huge = 2u << 20; - const size_t asz = (size + huge - 1) / huge * huge; - void * h = mmap(nullptr, asz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (h == MAP_FAILED) { - return nullptr; - } - if (madvise(h, asz, MADV_HUGEPAGE) != 0) { - GGML_LOG_WARN("%s: madvise(MADV_HUGEPAGE) failed: %s\n", __func__, strerror(errno)); - } - // fault the pages in on the host side (THP allocation happens here) before the GPU mapping is built - for (size_t off = 0; off < asz; off += huge) { - ((volatile char *) h)[off] = 0; - } - if (hipHostRegister(h, asz, hipHostRegisterDefault) != hipSuccess) { - (void) hipGetLastError(); - GGML_LOG_WARN("%s: hipHostRegister of %.2f MiB failed, falling back to hipMalloc\n", __func__, asz / 1024.0 / 1024.0); - munmap(h, asz); - return nullptr; - } - void * dptr = nullptr; - if (hipHostGetDevicePointer(&dptr, h, 0) != hipSuccess || dptr == nullptr) { - (void) hipGetLastError(); - hipHostUnregister(h); - munmap(h, asz); - return nullptr; - } - GGML_ASSERT(dptr == h); // UMA: the device pointer is the host pointer, tensor data is copied with plain memcpys - *alloc_size_out = asz; - return h; -} -#endif - struct ggml_backend_cuda_buffer_context { int device; void * dev_ptr = nullptr; std::string name; - size_t thp_size = 0; // != 0: dev_ptr is registered THP host memory of this size (see ggml_cuda_thp_alloc) - ggml_backend_cuda_buffer_context(int device, void * dev_ptr, size_t thp_size = 0) : + ggml_backend_cuda_buffer_context(int device, void * dev_ptr) : device(device), dev_ptr(dev_ptr), - name(GGML_CUDA_NAME + std::to_string(device)), thp_size(thp_size) { + name(GGML_CUDA_NAME + std::to_string(device)) { } ~ggml_backend_cuda_buffer_context() { -#if defined(GGML_USE_HIP) && defined(__linux__) - if (thp_size != 0) { - CUDA_CHECK(hipHostUnregister(dev_ptr)); - munmap(dev_ptr, thp_size); - return; - } -#endif CUDA_CHECK(cudaFree(dev_ptr)); } }; @@ -943,16 +889,6 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac ggml_cuda_set_device(buft_ctx->device); void * dev_ptr; -#if defined(GGML_USE_HIP) && defined(__linux__) - if (ggml_cuda_thp_buffers_enabled() && ggml_cuda_info().devices[buft_ctx->device].integrated && size >= (64u << 20)) { - size_t thp_size = 0; - dev_ptr = ggml_cuda_thp_alloc(size, &thp_size); - if (dev_ptr != nullptr) { - ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr, thp_size); - return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size); - } - } -#endif cudaError_t err = ggml_cuda_device_malloc(&dev_ptr, size, buft_ctx->device); if (err != cudaSuccess) { // clear the error @@ -3309,6 +3245,27 @@ static int ggml_cuda_try_gdn_decode_fusion(const ggml_cgraph * cgraph, int node_ args.H_v = H_v; args.d_conv = d_conv; + // Every skipped node except the three outputs must have all of its consumers inside this range. + // Include view nodes in the count even though they are not compute nodes. + for (int j = node_idx; j <= last; ++j) { + if (j == idx[0] || j == idx[11] || j == idx[13]) { + continue; + } + const ggml_tensor * candidate = cgraph->nodes[j]; + if (candidate->flags & GGML_TENSOR_FLAG_OUTPUT) { + return 0; + } + int internal_uses = 0; + for (int k = node_idx; k <= last; ++k) { + for (int src = 0; src < GGML_MAX_SRC; ++src) { + internal_uses += cgraph->nodes[k]->src[src] == candidate; + } + } + if (internal_uses != ggml_node_get_use_count(cgraph, j)) { + return 0; + } + } + return last - node_idx; } @@ -3800,6 +3757,25 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_cuda_gated_delta_net_fused_cache fused_state_cpy; const int nodes_to_skip = ggml_cuda_try_gdn_cache_fusion(cgraph, i, fused_state_cpy); if (nodes_to_skip > 0) { + bool uses_ok = true; + for (int j = i; uses_ok && j <= i + nodes_to_skip; ++j) { + if (j == i + nodes_to_skip) { + continue; + } + const ggml_tensor * candidate = cgraph->nodes[j]; + int internal_uses = 0; + for (int k = i; k <= i + nodes_to_skip; ++k) { + for (int src = 0; src < GGML_MAX_SRC; ++src) { + internal_uses += cgraph->nodes[k]->src[src] == candidate; + } + } + // The attention-score view is the one expected consumer after the snapshot copy. + const int allowed_external = j == i ? 1 : 0; + uses_ok = internal_uses + allowed_external == ggml_node_get_use_count(cgraph, j); + } + if (!uses_ok) { + return 0; + } #ifdef GGML_CUDA_DEBUG GGML_LOG_INFO("%s: fused gated_delta_net snapshot copies for %s (skipped %d nodes)\n", __func__, node->name, nodes_to_skip); @@ -4766,8 +4742,8 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_cuda_hc_mix_args args; enum ggml_op ops[5 + 2 * 15 + 1]; int n_ops = ggml_cuda_match_hc_mix(cgraph, i, args, ops); - if (n_ops > 0 && args.dst->ne[1] == 1 && getenv("GGML_CUDA_ENABLE_HCMIX") == nullptr) { - n_ops = 0; // decode: preserve bit-identical routing; prefill retains the exact fused reduction + if (n_ops > 0 && args.dst->ne[1] == 1) { + n_ops = 0; // Decode preserves bit-identical routing. Prefill uses the fused reduction. } if (n_ops > 0) { const int out_nodes[] = { i + n_ops - 1 }; @@ -5007,26 +4983,6 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph const int y_op = ggml_get_unary_op(cgraph->nodes[i + 1]) == GGML_UNARY_OP_SILU ? 1 : 2; ggml_tensor * mm = cgraph->nodes[i + 2]; - // qwen4exp hyper-connection: the matvec is the up projection whose result only feeds the stream mix - // (sigmoid -> mul(xn) -> ... -> scale): one kernel computes the gate rows of each element and the mix - if (getenv("GGML_CUDA_ENABLE_HCMIX") != nullptr && - i + 3 < cgraph->n_nodes && cgraph->nodes[i + 3]->src[0] == mm) { - ggml_cuda_hc_mix_args mix; - enum ggml_op ops[3 + 5 + 2 * 15 + 1]; - const int n_mix = ggml_cuda_match_hc_mix(cgraph, i + 3, mix, ops + 3); - if (n_mix > 0 && mix.gate == mm && - ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(mm->src[0], node->src[0], mix.xn, mix.dst, mix.hc)) { - ops[0] = GGML_OP_SCALE; ops[1] = GGML_OP_UNARY; ops[2] = GGML_OP_MUL_MAT; - const int out_nodes[] = { i + 3 + n_mix - 1 }; - // (aliasing of the output with y / xn is checked by ggml_cuda_mul_mat_vec_q_fq_hcmix_ok) - if (ggml_can_fuse_subgraph(cgraph, i, 3 + n_mix, ops, out_nodes, 1)) { - ggml_cuda_mul_mat_vec_q_fq_hcmix(*cuda_ctx, mm->src[0], node->src[0], mix.xn, mix.dst, mix.hc, - ggml_get_op_params_f32(node, 0), ggml_get_op_params_f32(node, 1), y_op, mix.scale, mix.bias); - return 2 + n_mix; - } - } - } - ggml_cuda_mul_mat_vec_q_fq_prologue(*cuda_ctx, mm, node->src[0], ggml_get_op_params_f32(node, 0), ggml_get_op_params_f32(node, 1), y_op); return 2; diff --git a/ggml/src/ggml-cuda/mmid.cu b/ggml/src/ggml-cuda/mmid.cu index ec0b687d38d2..edce236b0ca6 100644 --- a/ggml/src/ggml-cuda/mmid.cu +++ b/ggml/src/ggml-cuda/mmid.cu @@ -202,7 +202,8 @@ static void launch_mm_ids_helper( void ggml_cuda_launch_mm_ids_helper( const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds, const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) { - if (n_experts == 512 && n_expert_used == 10 && getenv("GGML_CUDA_DISABLE_MMID_512") == nullptr) { + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && n_experts == 512 && n_expert_used == 10 && getenv("GGML_CUDA_DISABLE_MMID_512") == nullptr) { mm_ids_helper_512_10<<<1, 1024, 0, stream>>>( ids, ids_src1, ids_dst, expert_bounds, n_tokens, nchannels_y, si1, sis1, write_inverse); return; diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 3a15f9657ae6..eeb88b09f487 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -1960,25 +1960,19 @@ static void mul_mat_vec_q_fq_switch_type( // --------------------------------------------------------------------------------------------- // RDNA3.5 grouped single-column matvec: several independent matvecs that share the same activation vector -// (e.g. the qwen4exp GDN qkv / gate / alpha / beta projections, or a hyper-connection down projection with its -// F32 inject rows) are launched as one kernel. Each segment is a plain [ncols x nrows] weight matrix (Q8_0, with -// an optional Q8_0 gate + GLU epilogue, or F32) with its own destination; blocks are assigned to segments by a -// prefix over the per-segment block counts. Q8_0 rows use the fused-quantize path of mul_mat_vec_q_fq (same -// per-lane accumulation order), F32 rows are one wave per row with float4 loads. +// Q8_0 matvecs are launched as one kernel. Each segment is a plain [ncols x nrows] weight matrix with an optional +// Q8_0 gate and GLU epilogue. Blocks are assigned to segments by a prefix over the segment block counts. #define MMVQ_GROUP_MAX 4 -#define MMVQ_GROUP_TYPE_Q8_0 0 -#define MMVQ_GROUP_TYPE_F32 1 struct mmvq_group_seg_dev { const void * vx; const void * gate; float * dst; int nrows; - int stride_row; // in blocks (Q8_0) or floats (F32) + int stride_row; // in Q8_0 blocks int gate_stride_row; // in blocks int blocks_begin; - int type; int glu_op; float glu_limit; }; @@ -2018,38 +2012,6 @@ static __global__ void mul_mat_vec_fq_group( const int row = ((int) blockIdx.x - seg.blocks_begin)*nwarps + threadIdx.y; const bool row_ok = row < seg.nrows; - if (seg.type == MMVQ_GROUP_TYPE_F32) { - if (!row_ok) { - return; - } - const float4 * w = (const float4 *) ((const float *) seg.vx + (size_t) row*seg.stride_row); - const float4 * y = (const float4 *) y_ptr; - float acc = 0.0f; -#pragma unroll 4 - for (int k = lane; k < ncols_x/4; k += warp_size) { - float4 w4 = w[k]; - float4 y4 = y[k]; - if (y_op != 0) { - float v[4] = {y4.x, y4.y, y4.z, y4.w}; -#pragma unroll - for (int j = 0; j < 4; ++j) { - const float t = y_scale * v[j] + y_bias; - v[j] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t)); - } - y4 = make_float4(v[0], v[1], v[2], v[3]); - } - acc = fmaf(w4.x, y4.x, acc); - acc = fmaf(w4.y, y4.y, acc); - acc = fmaf(w4.z, y4.z, acc); - acc = fmaf(w4.w, y4.w, acc); - } - acc = warp_reduce_sum(acc); - if (lane == 0) { - seg.dst[row] = acc; - } - return; - } - // Q8_0 path: identical to mul_mat_vec_q_fq const int row_safe = row_ok ? row : seg.nrows - 1; const bool use_gate = seg.gate != nullptr; @@ -2209,9 +2171,7 @@ void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, ggml_are_same_shape(segs[i].w, segs[i].gate) && segs[i].gate->nb[0] == sizeof(block_q8_0) && segs[i].gate->nb[1] % sizeof(block_q8_0) == 0); } - if (segs[i].w->type == GGML_TYPE_Q8_0) { - total_q8_rows += segs[i].w->ne[1]; - } + total_q8_rows += segs[i].w->ne[1]; max_rows = std::max(max_rows, segs[i].w->ne[1]); } @@ -2231,7 +2191,6 @@ void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, d.gate = segs[i].gate ? segs[i].gate->data : nullptr; d.dst = (float *) segs[i].dst->data; d.nrows = (int) w->ne[1]; - d.type = w->type == GGML_TYPE_Q8_0 ? MMVQ_GROUP_TYPE_Q8_0 : MMVQ_GROUP_TYPE_F32; d.stride_row = (int) (w->nb[1] / ggml_type_size(w->type)); d.gate_stride_row = segs[i].gate ? (int) (segs[i].gate->nb[1] / sizeof(block_q8_0)) : 0; d.blocks_begin = nblocks; @@ -2267,234 +2226,6 @@ void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, } } -// --------------------------------------------------------------------------------------------- -// qwen4exp hyper-connection up projection + stream mix, one token (RDNA3.5): -// gate = W_up * y' (y' = op(y_scale*y + y_bias), Q8_0 weights [ncols, hc*n_embd], fused quantization) -// mixed[e] = mix_scale * ( sum_c xn[c*n_embd + e] * sigmoid(gate[c*n_embd + e]) ) + mix_bias -// One wave owns element e and computes its hc gate rows {c*n_embd + e}: the hc*blocks_per_row Q8_0 blocks -// of those rows are spread flat over the lanes (4 lanes per block, 8 blocks per pass), so the short rows -// (ncols = 320 -> 10 blocks) keep all lanes busy. The gate itself is never written; the mix uses the same -// rounding sequence as hc_mix_reduce_f32 (separate mul / add roundings, streams summed in order). - -#define MMVQ_HCMIX_MAX_PASSES 16 -#define MMVQ_HCMIX_MAX_HC 4 -#define MMVQ_HCMIX_EPW 2 // elements (mixed outputs) per wave: 2*hc adjacent-row pairs stream per wave - -#if defined(__HIP_PLATFORM_AMD__) -static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) { - float result; - asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); - return result; -} -static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) { - float result; - asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); - return result; -} -#else -static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) { return __fmul_rn(a, b); } -static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) { return __fadd_rn(a, b); } -#endif - -template -__launch_bounds__(nwarps * ggml_cuda_get_physical_warp_size(), 1) -static __global__ void mul_mat_vec_q_fq_hcmix( - const void * GGML_CUDA_RESTRICT vx_ptr, const float * GGML_CUDA_RESTRICT y_ptr, const float * xn, float * dst, - const int ncols_x, const int n_embd, const int hc, const int stride_row_x, - const float y_scale, const float y_bias, const int y_op, const float mix_scale, const float mix_bias) { - constexpr int qi = QI8_0; - constexpr int vdr = VDR_Q8_0_Q8_1_MMVQ; - constexpr int warp_size = ggml_cuda_get_physical_warp_size(); - constexpr int lanes_per_row = qi / vdr; // 4 - constexpr int rows_per_wave = warp_size / lanes_per_row; // 8 = hc_max * epw - constexpr int epw = MMVQ_HCMIX_EPW; - static_assert(rows_per_wave == MMVQ_HCMIX_MAX_HC * epw, "lane layout"); - - extern __shared__ char mmvq_fq_smem[]; - block_q8_1 * y_q8 = (block_q8_1 *) mmvq_fq_smem; - - const int lane = threadIdx.x; - const int tid = warp_size*threadIdx.y + lane; - - const int e0 = (blockIdx.x*nwarps + threadIdx.y) * epw; // first element of this wave - - // lane group g = lane/4 owns wave row r = g: stream c = r / epw, element e0 + (r % epw) - const int r = lane / lanes_per_row; - const int kqs = vdr * (lane % lanes_per_row); - const int c = r / epw; - const int j = r - c*epw; - const bool row_ok = c < hc && e0 + j < n_embd; - const int row = c*n_embd + min(e0 + j, n_embd - 1); // clamped so that every lane reads valid memory - - const block_q8_0 * x = (const block_q8_0 *) vx_ptr + (size_t) (row_ok ? row : 0)*stride_row_x; - const int blocks_per_row = ncols_x / QK8_0; - - // 1. the whole row of this lane group first (the DRAM stream; everything else is tiny) - int qs[MMVQ_HCMIX_MAX_PASSES][vdr]; - half d[MMVQ_HCMIX_MAX_PASSES]; -#pragma unroll - for (int p = 0; p < MMVQ_HCMIX_MAX_PASSES; ++p) { - if (p < blocks_per_row) { -#pragma unroll - for (int jj = 0; jj < vdr; ++jj) { - qs[p][jj] = get_int_b2(x[p].qs, kqs + jj); - } - d[p] = x[p].d; - } - } - - // 2. quantize the activation vector into shared memory (same as mul_mat_vec_q_fq) - { - const int nby = ncols_x / QK8_1; - for (int i = tid; i < 4*nby; i += nwarps*warp_size) { - const int ib = i >> 2; - const int sub = i & 3; - const float4 * yv = (const float4 *) (y_ptr + ib*QK8_1 + sub*8); - const float4 v0 = yv[0]; - const float4 v1 = yv[1]; - float v[8] = {v0.x, v0.y, v0.z, v0.w, v1.x, v1.y, v1.z, v1.w}; - if (y_op != 0) { -#pragma unroll - for (int k = 0; k < 8; ++k) { - const float t = y_scale * v[k] + y_bias; - v[k] = y_op == 1 ? ggml_cuda_op_silu_single(t) : 1.0f / (1.0f + expf(-t)); - } - } - float amax = fabsf(v[0]); -#pragma unroll - for (int k = 1; k < 8; ++k) { - amax = fmaxf(amax, fabsf(v[k])); - } - amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 1, warp_size)); - amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 2, warp_size)); - const float dq = amax / 127.0f; - int q[8]; -#pragma unroll - for (int k = 0; k < 8; ++k) { - const int8_t qk8 = amax == 0.0f ? 0 : roundf(v[k] / dq); - q[k] = (int) qk8 & 0xff; - } - int * qsy = (int *) y_q8[ib].qs; - qsy[2*sub + 0] = q[0] | (q[1] << 8) | (q[2] << 16) | (q[3] << 24); - qsy[2*sub + 1] = q[4] | (q[5] << 8) | (q[6] << 16) | (q[7] << 24); - if (sub == 0) { - y_q8[ib].ds = make_half2(dq, 0.0f); - } - } - __syncthreads(); - } - - if (e0 >= n_embd) { - return; - } - - // 3. dot product of this lane group's row - float acc = 0.0f; -#pragma unroll - for (int p = 0; p < MMVQ_HCMIX_MAX_PASSES; ++p) { - if (p < blocks_per_row) { - const block_q8_1 * bq8_1 = &y_q8[p]; - int u[vdr]; -#pragma unroll - for (int jj = 0; jj < vdr; ++jj) { - u[jj] = get_int_b4(bq8_1->qs, kqs + jj); - } - acc += vec_dot_q8_0_q8_1_impl(qs[p], u, d[p], __low2half(bq8_1->ds)); - } - } - acc += __shfl_xor_sync(0xffffffff, acc, 1, warp_size); - acc += __shfl_xor_sync(0xffffffff, acc, 2, warp_size); - // lane 4*r now holds the gate of wave row r = c*epw + j - - float g[MMVQ_HCMIX_MAX_HC]; -#pragma unroll - for (int cc = 0; cc < MMVQ_HCMIX_MAX_HC; ++cc) { - // element j = lane (for lane < epw): its stream-cc gate sits in lane group cc*epw + lane - g[cc] = __shfl_sync(0xffffffff, acc, lanes_per_row * (cc*epw + (lane % epw)), warp_size); - } - - if (lane < epw && e0 + lane < n_embd) { - const int e = e0 + lane; - // same expressions as hc_mix_reduce_f32: sigmoid, rounded mul, streams added in order, then scale_f32 - float mixed = mmvq_hc_mul_rn(xn[e], 1.0f / (1.0f + expf(-g[0]))); -#pragma unroll - for (int cc = 1; cc < MMVQ_HCMIX_MAX_HC; ++cc) { - if (cc < hc) { - mixed = mmvq_hc_add_rn(mixed, mmvq_hc_mul_rn(xn[cc*n_embd + e], 1.0f / (1.0f + expf(-g[cc])))); - } - } - dst[e] = mix_scale * mixed + mix_bias; - } -} - -bool ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, const ggml_tensor * dst, const int hc) { - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - if (!GGML_CUDA_CC_IS_RDNA3_5(cc) || hc < 1 || hc > MMVQ_HCMIX_MAX_HC) { - return false; - } - if (w->type != GGML_TYPE_Q8_0 || w->ne[2] != 1 || w->ne[3] != 1 || w->nb[0] != sizeof(block_q8_0) || w->nb[1] % sizeof(block_q8_0) != 0 || - (w->buffer && ggml_backend_buffer_get_usage(w->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE)) { - return false; - } - const int64_t ncols = w->ne[0]; - const int64_t n_embd = dst->ne[0]; - if (ncols % QK8_0 != 0 || w->ne[1] != hc * n_embd || (ncols / QK8_0) > MMVQ_HCMIX_MAX_PASSES) { - return false; - } - if (y->type != GGML_TYPE_F32 || !ggml_is_contiguous(y) || ggml_nelements(y) != ncols || (uintptr_t) y->data % 16 != 0) { - return false; - } - if (xn->type != GGML_TYPE_F32 || !ggml_is_contiguous(xn) || ggml_nelements(xn) != hc * n_embd || - dst->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst) || ggml_nelements(dst) != n_embd) { - return false; - } - // every block reads y: dst must not overlap it - { - const char * d0 = (const char *) dst->data; - const char * d1 = d0 + ggml_backend_buft_get_alloc_size(dst->buffer->buft, dst); - const char * y0 = (const char *) y->data; - const char * y1 = y0 + ggml_backend_buft_get_alloc_size(y->buffer->buft, y); - if (d0 < y1 && y0 < d1) { - return false; - } - } - // dst may alias xn only on a whole stream (element e is then read and written by the same wave) - { - const char * d0 = (const char *) dst->data; - const char * d1 = d0 + ggml_backend_buft_get_alloc_size(dst->buffer->buft, dst); - const char * x0 = (const char *) xn->data; - const char * x1 = x0 + ggml_backend_buft_get_alloc_size(xn->buffer->buft, xn); - if (d0 < x1 && x0 < d1) { - const ptrdiff_t off = d0 - x0; - if (off < 0 || off % (n_embd * (ptrdiff_t) sizeof(float)) != 0 || off + (ptrdiff_t) ggml_nbytes(dst) > (ptrdiff_t) ggml_nbytes(xn)) { - return false; - } - } - } - return true; -} - -void ggml_cuda_mul_mat_vec_q_fq_hcmix(ggml_backend_cuda_context & ctx, const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, - ggml_tensor * dst, const int hc, const float y_scale, const float y_bias, const int y_op, const float mix_scale, const float mix_bias) { - GGML_ASSERT(ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(w, y, xn, dst, hc)); - const int device = ggml_cuda_get_device(); - const int warp_size = ggml_cuda_info().devices[device].warp_size; - GGML_ASSERT(warp_size == 32); - - const int ncols = (int) w->ne[0]; - const int n_embd = (int) dst->ne[0]; - - constexpr int nwarps = 32; - const int n_waves = (n_embd + MMVQ_HCMIX_EPW - 1) / MMVQ_HCMIX_EPW; - const dim3 block_nums((n_waves + nwarps - 1) / nwarps, 1, 1); - const dim3 block_dims(warp_size, nwarps, 1); - const int nbytes_shared = (ncols / QK8_1) * sizeof(block_q8_1); - const ggml_cuda_kernel_launch_params launch_params(block_nums, block_dims, nbytes_shared, ctx.stream()); - ggml_cuda_kernel_launch(mul_mat_vec_q_fq_hcmix, launch_params, - w->data, (const float *) y->data, (const float *) xn->data, (float *) dst->data, - ncols, n_embd, hc, (int) (w->nb[1] / sizeof(block_q8_0)), y_scale, y_bias, y_op, mix_scale, mix_bias); -} - // Dedicated MoE multi-token kernel. // Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst) // Block: (warp_size, ncols_dst) - each warp handles one token independently. @@ -3072,7 +2803,9 @@ static bool mul_mat_vec_q_fq_try( const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const int64_t ncols_dst_fq = ids ? ne2 : ne1; const bool aligned = (uintptr_t) src1->data % 16 == 0 && nb11 % 16 == 0 && nb12 % 16 == 0 && nb13 % 16 == 0; - const bool fusion_ok = !fusion || (fusion->x_scale == nullptr && fusion->gate_scale == nullptr); + const bool fusion_ok = !fusion || + (fusion->x_scale == nullptr && fusion->gate_scale == nullptr && + (!ids || (fusion->x_bias == nullptr && fusion->gate_bias == nullptr))); if (!(GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols_dst_fq == 1 && mmvq_fq_type_ok(src0->type) && aligned && fusion_ok && ne10 % QK8_1 == 0 && (ne10 / QK8_1) * sizeof(block_q8_1) <= 16384)) { return false; @@ -3186,6 +2919,23 @@ void ggml_cuda_mul_mat_vec_q_fq_gdn_gate(ggml_backend_cuda_context & ctx, ggml_t GGML_ASSERT(ok); } +#if defined(__HIP_PLATFORM_AMD__) +static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) { + float result; + asm("v_mul_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) { + float result; + asm("v_add_f32_e32 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} +#else +static __device__ __forceinline__ float mmvq_hc_mul_rn(const float a, const float b) { return __fmul_rn(a, b); } +static __device__ __forceinline__ float mmvq_hc_add_rn(const float a, const float b) { return __fadd_rn(a, b); } +#endif + template __launch_bounds__(32, 8) static __global__ void mul_mat_id_iq4_nl_weighted_rdna3_5( diff --git a/ggml/src/ggml-cuda/mmvq.cuh b/ggml/src/ggml-cuda/mmvq.cuh index c050c68073b2..c2d1da92573d 100644 --- a/ggml/src/ggml-cuda/mmvq.cuh +++ b/ggml/src/ggml-cuda/mmvq.cuh @@ -36,12 +36,6 @@ struct ggml_cuda_mmv_group_seg { bool ggml_cuda_mmv_group_seg_ok(const ggml_tensor * w, const ggml_tensor * y); void ggml_cuda_mmv_group(ggml_backend_cuda_context & ctx, const ggml_tensor * y, const ggml_cuda_mmv_group_seg * segs, int nseg); -// RDNA3.5 qwen4exp hyper-connection up projection fused with the stream mix (one token): -// mixed[e] = mix_scale * sum_c xn[c*n_embd+e] * sigmoid((W_up * op(y_scale*y + y_bias))[c*n_embd+e]) + mix_bias -bool ggml_cuda_mul_mat_vec_q_fq_hcmix_ok(const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, const ggml_tensor * dst, int hc); -void ggml_cuda_mul_mat_vec_q_fq_hcmix(ggml_backend_cuda_context & ctx, const ggml_tensor * w, const ggml_tensor * y, const ggml_tensor * xn, - ggml_tensor * dst, int hc, float y_scale, float y_bias, int y_op, float mix_scale, float mix_bias); - // RDNA3.5 qwen4exp GDN output projection with the gated per-head norm in the prologue: // y' = sigmoid(z) * (rms_norm_128(attn) * norm_w), then the fused-quantize matvec of dst->src[0] with y' bool ggml_cuda_mul_mat_vec_q_fq_gdn_gate_ok(ggml_backend_cuda_context & ctx, const ggml_tensor * dst, const ggml_tensor * z); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index ab560e7d5e90..0621efe7143d 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1354,15 +1354,20 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll int64_t t_build = 0, t_alloc = 0, t_inputs = 0; bool reused = false; - // The previous asynchronous graph may still read inputs or use graph allocations. - ggml_backend_sched_synchronize(sched.get()); - if (!graph_reuse_disable && res->can_reuse(gparams)) { reused = true; //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__); + // Pipeline-parallel graphs may still read input tensors when set_inputs updates them below. + if (cparams.pipeline_parallel) { + ggml_backend_sched_synchronize(sched.get()); + } + n_reused++; } else { + // The previous asynchronous graph may still use allocations that reset releases. + ggml_backend_sched_synchronize(sched.get()); + res->reset(); ggml_backend_sched_reset(sched.get()); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index a2dbaf4e1555..9b4e64d4bafd 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3771,6 +3771,8 @@ struct test_relu_sqr : public test_case { } }; +static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats); + struct test_weighted_expert_sum : public test_case { const int64_t n_embd; const int64_t n_expert_used; @@ -3782,6 +3784,7 @@ struct test_weighted_expert_sum : public test_case { } bool run_whole_graph() override { return true; } + double max_nmse_err() override { return 5e-4; } std::string vars() override { return VARS_TO_STR3(n_embd, n_expert_used, n_tokens); @@ -3791,7 +3794,14 @@ struct test_weighted_expert_sum : public test_case { : n_embd(n_embd), n_expert_used(n_expert_used), n_tokens(n_tokens) {} ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * experts = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd, n_expert_used, n_tokens); + const int64_t k = 32; + const int64_t n_mats = std::max(16, n_expert_used); + ggml_tensor * expert_weights = ggml_new_tensor_3d(ctx, GGML_TYPE_Q8_0, k, n_embd, n_mats); + ggml_tensor * ids_all = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_mats, n_tokens); + ggml_set_name(ids_all, "ids"); + ggml_tensor * ids = ggml_view_2d(ctx, ids_all, n_expert_used, n_tokens, ids_all->nb[1], 0); + ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, n_expert_used, n_tokens); + ggml_tensor * experts = ggml_mul_mat_id(ctx, expert_weights, input, ids); ggml_tensor * weights = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, n_expert_used, n_tokens); ggml_tensor * weighted = ggml_mul(ctx, experts, weights); ggml_build_forward_expand(gf, weighted); @@ -3811,6 +3821,10 @@ struct test_weighted_expert_sum : public test_case { } return out; } + + void initialize_tensors(ggml_context * ctx) override { + init_mul_mat_id_tensors(ctx, std::max(16, n_expert_used)); + } }; struct test_shared_mul_add : public test_case { @@ -5105,15 +5119,13 @@ struct test_mul_mat_id_fusion : public test_case { // GGML_OP_OUT_PROD // GGML_OP_MUL_MAT group: independent single-column matvecs on one activation vector -// (RDNA3.5 CUDA launches [Q8_0 | Q8_0 gate/up + GLU | F32 ...] as one grouped kernel) +// (RDNA3.5 CUDA launches independent Q8_0 segments as one grouped kernel) struct test_mmv_group : public test_case { const int64_t k; const int64_t m_q8; // rows of a plain Q8_0 segment (0 = none) const int64_t m_glu; // rows of a Q8_0 gate/up + swiglu segment (0 = none) - const int64_t m_f32a; // rows of an F32 segment (0 = none) - const int64_t m_f32b; // rows of a second F32 segment (0 = none) - std::string vars() override { return VARS_TO_STR5(k, m_q8, m_glu, m_f32a, m_f32b); } + std::string vars() override { return VARS_TO_STR3(k, m_q8, m_glu); } std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -5124,8 +5136,8 @@ struct test_mmv_group : public test_case { double max_nmse_err() override { return 5e-4; } - test_mmv_group(int64_t k, int64_t m_q8, int64_t m_glu, int64_t m_f32a, int64_t m_f32b) - : k(k), m_q8(m_q8), m_glu(m_glu), m_f32a(m_f32a), m_f32b(m_f32b) {} + test_mmv_group(int64_t k, int64_t m_q8, int64_t m_glu) + : k(k), m_q8(m_q8), m_glu(m_glu) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * y = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, 1); @@ -5148,16 +5160,6 @@ struct test_mmv_group : public test_case { ggml_set_name(wu, "w_up"); append(ggml_swiglu_split(ctx, ggml_mul_mat(ctx, wg, y), ggml_mul_mat(ctx, wu, y))); } - if (m_f32a > 0) { - ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, m_f32a); - ggml_set_name(w, "w_f32a"); - append(ggml_mul_mat(ctx, w, y)); - } - if (m_f32b > 0) { - ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, m_f32b); - ggml_set_name(w, "w_f32b"); - append(ggml_mul_mat(ctx, w, y)); - } ggml_set_name(out, "out"); return out; } @@ -6585,7 +6587,13 @@ struct test_top_k : public test_case { diff += std::fabs(b[i] - ib[i]); } - return diff + jdst(ia.data(), ib.data(), n); + if (n != (size_t) (ggml_nrows(input) * k)) { + return nmse(a, b, n); + } + for (int64_t r = 0; r < ggml_nrows(input); ++r) { + diff += jdst(ia.data() + r * k, ib.data() + r * k, k); + } + return diff; } } @@ -6603,8 +6611,7 @@ struct test_top_k : public test_case { } void initialize_tensors(ggml_context * ctx) override { - std::random_device rd; - std::default_random_engine rng(rd()); + std::default_random_engine rng(0x5eed); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { int tie_denom = std::max(1, std::min(10, k / 2)); for (int64_t r = 0; r < ggml_nrows(t); r++) { @@ -6749,17 +6756,8 @@ struct test_topk_moe : public test_case { } } - // allow output in arbitrary order double err(const float * a, const float * b, size_t n) override { - std::vector a2(n); - std::vector b2(n); - for (size_t i = 0; i < n; ++i) { - a2[i] = a[i]; - b2[i] = b[i]; - } - std::sort(a2.begin(), a2.end()); - std::sort(b2.begin(), b2.end()); - return nmse(a2.data(), b2.data(), n); + return nmse(a, b, n); } }; @@ -9675,15 +9673,8 @@ static std::vector> make_test_cases_eval() { // grouped single-column matvecs on one activation vector (qwen4exp decode: GDN qkv/z/beta/alpha, // hyper-connection down + inject, shared-expert gate/up + router) - test_cases.emplace_back(new test_mmv_group(2560, 10240, 0, 48, 48)); - test_cases.emplace_back(new test_mmv_group(2560, 10240, 6144, 0, 0)); - test_cases.emplace_back(new test_mmv_group(2560, 1024, 6144, 48, 48)); - test_cases.emplace_back(new test_mmv_group(10240, 320, 0, 4, 0)); - test_cases.emplace_back(new test_mmv_group(2560, 0, 640, 512, 0)); - test_cases.emplace_back(new test_mmv_group(2560, 0, 640, 512, 1)); - test_cases.emplace_back(new test_mmv_group(2560, 0, 0, 48, 48)); - test_cases.emplace_back(new test_mmv_group(640, 2560, 0, 33, 0)); - test_cases.emplace_back(new test_mmv_group(256, 17, 5, 3, 129)); + test_cases.emplace_back(new test_mmv_group(2560, 10240, 6144)); + test_cases.emplace_back(new test_mmv_group(2560, 1024, 6144)); // 256-expert MoE with 4/16/32/128 rows per expert, exercises the RDNA3.5 mul_mat_id tile selection // Qwen3.8-Flash-Next (qwen4exp) UD-IQ4_XS experts: 512 experts, 10 active, gate/up IQ3_S [2560 -> 640], down IQ4_NL [640 -> 2560] From 7c877db647c2148b0ec9213dd0d46d80a3de9412 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:10:41 +0200 Subject: [PATCH 27/30] hip: tune Ling MoE inference on RDNA3.5 --- ggml/src/ggml-cuda/gated_delta_net.cu | 9 +++++---- ggml/src/ggml-cuda/ggml-cuda.cu | 6 +++++- ggml/src/ggml-cuda/mmq.cuh | 27 ++++++++++++++++++++++--- ggml/src/ggml-impl.h | 2 +- tests/test-backend-ops.cpp | 29 ++++++++++++++++++++++++--- 5 files changed, 61 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu index 9f0e8f75cc14..ff2ac52745ea 100644 --- a/ggml/src/ggml-cuda/gated_delta_net.cu +++ b/ggml/src/ggml-cuda/gated_delta_net.cu @@ -213,6 +213,7 @@ gated_delta_net_cuda(const float * q, } } +template __global__ void __launch_bounds__(512, 2) gated_delta_net_kda_tiled_128_cuda(const float * q, const float * k, @@ -236,7 +237,6 @@ gated_delta_net_kda_tiled_128_cuda(const float * q, const uint3 rq3_magic, float scale) { constexpr int S_v = 128; - constexpr int H = 32; constexpr int token_tile = 16; constexpr int warp_size = 32; constexpr int rows_per_lane = S_v / warp_size; @@ -314,7 +314,7 @@ gated_delta_net_kda_tiled_128_cuda(const float * q, } const float attn_col = gdn_warp_reduce_sum(attn_partial); if (lane == 0) { - dst[t * S_v * H + col] = attn_col * scale; + dst[(int64_t) t * S_v * H + col] = attn_col * scale; } } __syncthreads(); @@ -524,9 +524,10 @@ static void launch_gated_delta_net( const uint3 rq3_magic = init_fastdiv_values(rq3); if constexpr (KDA && !keep_rs_t) { - if (GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && H == 32 && n_tokens >= 16 && n_seqs == 1) { + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && S_v == 128 && (H == 16 || H == 32) && n_tokens >= 16 && n_seqs == 1) { const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); - ggml_cuda_kernel_launch(gated_delta_net_kda_tiled_128_cuda, launch_params, + auto kernel = H == 16 ? gated_delta_net_kda_tiled_128_cuda<16> : gated_delta_net_kda_tiled_128_cuda<32>; + ggml_cuda_kernel_launch(kernel, launch_params, q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, n_tokens, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 029b3fdd75af..d226a919d281 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4746,8 +4746,12 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph n_ops = 0; // Decode preserves bit-identical routing. Prefill uses the fused reduction. } if (n_ops > 0) { + int node_idxs[5 + 2 * 15 + 1]; + for (int j = 0; j < n_ops; ++j) { + node_idxs[j] = i + j; + } const int out_nodes[] = { i + n_ops - 1 }; - if (ggml_can_fuse_subgraph(cgraph, i, n_ops, ops, out_nodes, 1)) { + if (ggml_can_fuse_subgraph_ext(cgraph, node_idxs, n_ops, ops, out_nodes, 1)) { ggml_cuda_op_hc_mix_reduce(*cuda_ctx, args); return n_ops - 1; } diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 73b71dc22967..39bad397cbc4 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1422,7 +1422,7 @@ static __global__ void mul_mat_q_routed_compact( if (j0 + nwarps * warp_size > J && j >= J) { break; } - ids_dst_shared[j] = ids_dst[col_low + jt * J + j]; + ids_dst_shared[j] = jt * J + j < col_diff ? ids_dst[col_low + jt * J + j] : 0; } __syncthreads(); @@ -1637,6 +1637,11 @@ static constexpr bool mmq_rdna3_5_id_use_compact(const ggml_type type, const int } } +static constexpr bool mmq_rdna3_5_id_use_j48_128e(const ggml_type type, const int64_t n_experts, const int64_t rows_per_expert) { + return n_experts == 128 && rows_per_expert >= 96 && rows_per_expert <= 160 && + (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q8_0); +} + static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 4) == 16); static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 16) == 48); static_assert(mmq_rdna3_5_id_get_J(GGML_TYPE_Q8_0, 64) == 48); @@ -1649,6 +1654,22 @@ static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 48)); static_assert(!mmq_rdna3_5_id_use_compact(GGML_TYPE_Q8_0, 32)); static_assert(mmq_rdna3_5_id_use_compact(GGML_TYPE_Q6_K, 32)); static_assert(!mmq_rdna3_5_id_use_compact(GGML_TYPE_IQ2_XS, 48)); +static_assert(!mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 95) && + !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 95) && + !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 95) && + !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 95)); +static_assert(mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 96) && + mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 96) && + mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 96) && + mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 96) && + mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 160) && + mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 160) && + mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 160) && + mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 160)); +static_assert(!mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q4_K, 128, 161) && + !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q5_K, 128, 161) && + !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q6_K, 128, 161) && + !mmq_rdna3_5_id_use_j48_128e(GGML_TYPE_Q8_0, 128, 161)); template static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { @@ -1776,8 +1797,8 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, } } else if (GGML_CUDA_CC_IS_RDNA3_5(cc) && !fallback && args.ids_dst != nullptr && mmq_rdna3_5_id_n_experts_ok(args.nchannels_y)) { const int64_t rows_per_expert = (args.ncols_dst + args.nchannels_y - 1) / args.nchannels_y; - int J_rdna3_5 = type == GGML_TYPE_Q6_K && args.nchannels_y == 128 && rows_per_expert >= 96 && rows_per_expert <= 160 ? - 48 : mmq_rdna3_5_id_get_J(type, rows_per_expert); + const bool use_j48_128e = mmq_rdna3_5_id_use_j48_128e(type, args.nchannels_y, rows_per_expert); + int J_rdna3_5 = use_j48_128e ? 48 : mmq_rdna3_5_id_get_J(type, rows_per_expert); if constexpr (type == GGML_TYPE_Q6_K) { if (getenv("GGML_Q6_COMPACT_J")) { J_rdna3_5 = atoi(getenv("GGML_Q6_COMPACT_J")); diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 62b76abbcec9..bcdeff6ade24 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -727,7 +727,7 @@ static inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph, const enum ggml_op * ops, const int * outputs, int num_outputs) { - GGML_ASSERT(count < 32); + GGML_ASSERT(count <= 32); if (node_idx + count > cgraph->n_nodes) { return false; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 9b4e64d4bafd..4bbe0e31a203 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5173,8 +5173,9 @@ struct test_hc_up_mix : public test_case { const int64_t n_embd; const int64_t hc; const int64_t n_tokens; + const bool external_consumer; - std::string vars() override { return VARS_TO_STR5(type_w, k, n_embd, hc, n_tokens); } + std::string vars() override { return VARS_TO_STR6(type_w, k, n_embd, hc, n_tokens, external_consumer); } std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -5185,8 +5186,9 @@ struct test_hc_up_mix : public test_case { double max_nmse_err() override { return 5e-4; } - test_hc_up_mix(ggml_type type_w, int64_t k, int64_t n_embd, int64_t hc, int64_t n_tokens) - : type_w(type_w), k(k), n_embd(n_embd), hc(hc), n_tokens(n_tokens) {} + test_hc_up_mix(ggml_type type_w, int64_t k, int64_t n_embd, int64_t hc, int64_t n_tokens, + bool external_consumer = false) + : type_w(type_w), k(k), n_embd(n_embd), hc(hc), n_tokens(n_tokens), external_consumer(external_consumer) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * lo = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k, n_tokens); @@ -5202,6 +5204,7 @@ struct test_hc_up_mix : public test_case { gated = ggml_reshape_3d(ctx, gated, n_embd, hc, n_tokens); ggml_tensor * mixed = ggml_view_2d(ctx, gated, n_embd, n_tokens, ggml_row_size(gated->type, n_embd) * hc, 0); mixed = ggml_cont(ctx, mixed); + ggml_tensor * external = external_consumer ? mixed : nullptr; for (int64_t c = 1; c < hc; ++c) { ggml_tensor * s = ggml_view_2d(ctx, gated, n_embd, n_tokens, ggml_row_size(gated->type, n_embd) * hc, ggml_row_size(gated->type, n_embd) * c); @@ -5209,6 +5212,9 @@ struct test_hc_up_mix : public test_case { } mixed = ggml_scale(ctx, mixed, 1.0f / (float) hc); ggml_set_name(mixed, "mixed"); + if (external) { + mixed = ggml_add(ctx, mixed, external); + } return mixed; } }; @@ -9552,6 +9558,7 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_weighted_expert_sum(127, 2, 3)); test_cases.emplace_back(new test_weighted_expert_sum(128, 4, 5)); test_cases.emplace_back(new test_weighted_expert_sum(2048, 8, 32)); + test_cases.emplace_back(new test_weighted_expert_sum(128, 16, 3)); test_cases.emplace_back(new test_shared_mul_add(127, 3)); test_cases.emplace_back(new test_shared_mul_add(2048, 32)); test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2048, 32, 512)); @@ -9670,6 +9677,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 256, 1000, 3, 1)); test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 512, 640, 4, 1)); test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_F32, 320, 2560, 4, 1)); + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 64, 64, 14, 3)); + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 64, 64, 16, 3)); + test_cases.emplace_back(new test_hc_up_mix(GGML_TYPE_Q8_0, 64, 64, 16, 3, true)); // grouped single-column matvecs on one activation vector (qwen4exp decode: GDN qkv/z/beta/alpha, // hyper-connection down + inject, shared-expert gate/up + router) @@ -9717,6 +9727,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2048, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 512, 2080, 2048)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 256, 8, false, 2048, 2048, 512)); + // 128-expert J48 dispatch boundaries. Small matrix dimensions keep CPU reference time practical. + for (ggml_type type_a : {GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_Q8_0}) { + test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 128, 8, false, 64, 1521, 256)); + } + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 64, 1520, 256)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 64, 2545, 256)); + test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 64, 2561, 256)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 1984, 2560)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 2048, 2560)); test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q5_K, GGML_TYPE_F32, 512, 8, false, 768, 2112, 2560)); @@ -10703,6 +10720,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 64, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true)); + for (int n_tokens : {15, 16, 17, 64, 65, 2048}) { + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, n_tokens, 1, 1, false, true)); + } + // Multiple sequences must remain on the generic fallback. + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 2, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 15, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 16, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 17, 1, 1, false, true)); @@ -11175,6 +11197,7 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 1, 1, false, true)); // Ling-like KDA PP-64 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 15, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 16, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 17, 1, 1, false, true)); From 312ea53c6c637cff78df25bbd5e6bbb26b3c76be Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:57:27 +0200 Subject: [PATCH 28/30] hip: fix RDNA3.5 fused quantization edge cases --- ggml/src/ggml-cuda/mmvq.cu | 18 +++++++++++------- ggml/src/ggml-cuda/quantize.cu | 4 ++-- tests/test-backend-ops.cpp | 16 +++++++++++++--- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index eeb88b09f487..55d70f6482ca 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -1810,10 +1810,12 @@ static __global__ void mul_mat_vec_q_fq( mmvq_fq_q8_0_dot(tmp_gate, cg, y, c0, kqs, blocks_per_row_x); } } - cx = nx; - if constexpr (has_fusion) { - if (use_gate) { - cg = ng; + if (c1 < blocks_per_row_x) { + cx = nx; + if constexpr (has_fusion) { + if (use_gate) { + cg = ng; + } } } } @@ -2096,9 +2098,11 @@ static __global__ void mul_mat_vec_fq_group( if (use_gate) { mmvq_fq_q8_0_dot(tmp_gate, cg, y, c0, kqs, blocks_per_row_x); } - cx = nx; - if (use_gate) { - cg = ng; + if (c1 < blocks_per_row_x) { + cx = nx; + if (use_gate) { + cg = ng; + } } } diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index 3923b19b0e2b..d4bb2af0e115 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -589,13 +589,13 @@ static __global__ void quantize_mmq_q8_1_swiglu( amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); } - const float d_inv = 127.0f / amax; + const float d_inv = amax == 0.0f ? 0.0f : 127.0f / amax; char4 q; q.x = roundf(xi.x * d_inv); q.y = roundf(xi.y * d_inv); q.z = roundf(xi.z * d_inv); q.w = roundf(xi.w * d_inv); - const float d = 1.0f / d_inv; + const float d = amax == 0.0f ? 0.0f : 1.0f / d_inv; block_q8_1_mmq * y = (block_q8_1_mmq *) vy; const int64_t k_block = i0 / QK8_1_MMQ; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 4bbe0e31a203..6d2b41c1b11d 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4960,10 +4960,11 @@ struct test_swiglu_q8_mmq : public test_case { const int64_t n; const int64_t k; const ggml_type type_down; + const bool zero_inputs; test_swiglu_q8_mmq(bool use_id, int n_mats, int n_used, int64_t m, int64_t n, int64_t k, - ggml_type type_down = GGML_TYPE_Q8_0) - : use_id(use_id), n_mats(n_mats), n_used(n_used), m(m), n(n), k(k), type_down(type_down) {} + ggml_type type_down = GGML_TYPE_Q8_0, bool zero_inputs = false) + : use_id(use_id), n_mats(n_mats), n_used(n_used), m(m), n(n), k(k), type_down(type_down), zero_inputs(zero_inputs) {} std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -4971,7 +4972,7 @@ struct test_swiglu_q8_mmq : public test_case { } std::string vars() override { - return VARS_TO_STR7(use_id, n_mats, n_used, m, n, k, type_down); + return VARS_TO_STR8(use_id, n_mats, n_used, m, n, k, type_down, zero_inputs); } bool run_whole_graph() override { return true; } @@ -4996,6 +4997,13 @@ struct test_swiglu_q8_mmq : public test_case { void initialize_tensors(ggml_context * ctx) override { init_mul_mat_id_tensors(ctx, n_mats); + if (zero_inputs) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->type != GGML_TYPE_I32 && !ggml_is_view_op(t->op)) { + init_tensor_uniform(t, 0.0f, 0.0f); + } + } + } } }; @@ -9563,6 +9571,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_shared_mul_add(2048, 32)); test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2048, 32, 512)); test_cases.emplace_back(new test_swiglu_q8_mmq(true, 16, 8, 2048, 32, 512)); + test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 128, 3, 256, GGML_TYPE_Q8_0, true)); + test_cases.emplace_back(new test_swiglu_q8_mmq(true, 16, 8, 128, 3, 256, GGML_TYPE_Q8_0, true)); test_cases.emplace_back(new test_swiglu_q8_mmq(false, 1, 1, 2560, 128, 640)); test_cases.emplace_back(new test_swiglu_q8_mmq(true, 512, 10, 2560, 128, 640)); test_cases.emplace_back(new test_swiglu_q8_mmq(true, 512, 10, 2560, 128, 640, GGML_TYPE_IQ4_NL)); From cde4bf7dd3f8bcd16dae8526c5406c645e3a4d6d Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:17:10 +0200 Subject: [PATCH 29/30] hip: fix ROCm 7.2 Flash Attention Werror build --- ggml/src/ggml-cuda/fattn-tile.cuh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index e6df7f1f640d..cc6196037b6e 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -1,6 +1,10 @@ #include "common.cuh" #include "fattn-common.cuh" +#if defined(GGML_USE_HIP) && defined(__clang__) +#pragma clang diagnostic ignored "-Wpass-failed" +#endif + // nbatch_fa == number of KQ rows to process per iteration // nbatch_K == number of K columns to load in parallel for KQ calculation @@ -213,7 +217,7 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am GGML_CUDA_FATTN_TILE_CONFIG_CASE(192, 128, 32, 256, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 2, 256, 2, 128, 64) - GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 3, 96, 8, 32, 64) + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 3, 96, 2, 32, 64) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 4, 256, 2, 64, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 8, 256, 2, 64, 128) GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 16, 256, 2, 32, 128) From 4021b991c6245044ecda10f1e9759e28308610e0 Mon Sep 17 00:00:00 2001 From: Gaetan Puleo <12990773+gaetan-puleo@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:32:37 +0200 Subject: [PATCH 30/30] hip: avoid compiling RDNA3.5 Q8 FA on CDNA --- ggml/src/ggml-cuda/fattn-tile.cuh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh index cc6196037b6e..e5be145ed5ee 100644 --- a/ggml/src/ggml-cuda/fattn-tile.cuh +++ b/ggml/src/ggml-cuda/fattn-tile.cuh @@ -893,6 +893,13 @@ static __global__ void flash_attn_tile( const int32_t ne31, const int32_t ne32, const int32_t ne33, const int32_t nb31, const int32_t nb32, const int64_t nb33) { #ifdef FLASH_ATTN_AVAILABLE +#if defined(GGML_USE_HIP) && !defined(RDNA3_5) + if constexpr (q8_0_KV) { + NO_DEVICE_CODE; + return; + } +#endif // defined(GGML_USE_HIP) && !defined(RDNA3_5) + const char * GGML_CUDA_RESTRICT Q = Q_ptr; const char * GGML_CUDA_RESTRICT K = K_ptr; const char * GGML_CUDA_RESTRICT V = V_ptr;