Skip to content

ggml : tune Volta BF16 small-batch matrix vectors #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
novkien merged 1 commit into from
Oct 3, 2026
Merged

ggml : tune Volta BF16 small-batch matrix vectors #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ggml/src/ggml-cuda/mmvf.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,14 @@ void launch_mul_mat_vec_f_cuda(

const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;

const int cc = ggml_cuda_info().devices[device].cc;
if (cc == GGML_CUDA_CC_VOLTA && std::is_same_v<T, nv_bfloat16> &&
ncols == 320 && nrows == 10240 && ncols_dst == 4 && ids == nullptr &&
!is_multi_token_id && !has_fusion) {
// Keep each output row in one warp for this small-K projection.
block_size_best = 32;
}

const int nbytes_shared = warp_size*sizeof(float) + (has_fusion ? warp_size*sizeof(float) : 0);
const dim3 block_nums(nrows, nchannels_dst, nsamples_or_ntokens);
const dim3 block_dims(block_size_best, 1, 1);
Expand Down
13 changes: 13 additions & 0 deletions tests/test-backend-ops.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10259,6 +10259,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}

// Qwen4Exp hc_gate uses BF16 weights with K=320 and M=10240.
for (int n : { 1, 3, 4, 5 }) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 10240, n, 320, { 1, 1 }, { 1, 1 }));
}
test_cases.emplace_back(new test_mul_mat(
GGML_TYPE_BF16, GGML_TYPE_F32, 10240, 4, 320, { 1, 1 }, { 1, 1 }, { 0, 1, 2, 3 }, 384));
test_cases.emplace_back(new test_mul_mat(
GGML_TYPE_BF16, GGML_TYPE_F32, 10240, 4, 320, { 2, 1 }, { 2, 1 }));

// The SYCL backend picks between one and two output rows per subgroup by row count when there
// are two destination columns (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp). Cover both
// sides of that boundary, including an odd row count above it for the row-pair tail.
Expand Down Expand Up @@ -11653,6 +11662,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}

// Qwen4Exp hc_gate MMVF shape: K=320, M=10240, W4.
test_cases.emplace_back(new test_mul_mat(
GGML_TYPE_BF16, GGML_TYPE_F32, 10240, 4, 320, {1, 1}, {1, 1}));

// qwen3-30b-a3b
for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) {
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) {
Expand Down
Loading