From bd61914ecc75f51dc347dfe423b0491ae3b86f86 Mon Sep 17 00:00:00 2001 From: LeoBorcherding Date: Thu, 17 Sep 2026 14:45:42 -0500 Subject: [PATCH 1/2] qwen4exp: load the MTP head's hc_head_norm as [n_embd, hc] The rebase onto the upstream rms_norm + mul fusion moved the trunk hyper-connection gammas to { n_embd, hc } with TENSOR_ALLOW_RESHAPE, but left the MTP head's gamma flat. build_hc_mix then multiplies a [hc_dim] weight into an [n_embd, hc, n_tokens] stream and ggml_can_repeat aborts while the draft context is reserved. --- src/models/qwen4exp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 97652d42f390..e329d13b8a7c 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -277,7 +277,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { hc_dim }, flags); layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, flags); - layer.nextn.hc_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_NORM, "weight", il), { hc_dim }, flags); + layer.nextn.hc_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_NORM, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE); layer.nextn.hc_head_down = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_DOWN, "weight", il), { hc_dim, hc_lr }, flags); layer.nextn.hc_head_up = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_UP, "weight", il), { hc_lr, hc_dim }, flags); From fafa22fb24a4d42d14415fbc63ba570c4fc68251 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 18 Sep 2026 10:04:41 +0000 Subject: [PATCH 2/2] qwen4exp: let the memory fit measure a borrowing draft head common/fit.cpp opens the draft model on its own to price it, with no_alloc set and no target attached. borrow_shared_tensor threw there, so the fit caught it, warned "failed to measure the memory of the extra model, fitting without it" and then budgeted nothing for the draft. The real load ran out of device memory by roughly the head's own size: error loading model: borrow_shared_tensor: this model is a draft head without its own 'token_embd.weight'; load it as a draft of its target model, not on its own failed to measure the memory of the extra model, fitting without it allocating 2647.04 MiB on device 0: cudaMalloc failed: out of memory Refusing there protected nobody, since the caller only writes the draft out of the budget. On a measurement load the borrowed tensor is reported to the caller instead, which stands a shape in for it so the measurement can finish and report the bytes this file does own. The shape goes on the CPU buffer type: the graph needs the dimensions to reserve correctly, but those bytes belong to the target and are counted in its own measurement, so charging them to a device would trade under-budgeting the draft for over-budgeting it by the size of the embedding. A real load always has a target by construction and keeps the hard error, so a borrowing head opened on its own still refuses rather than running on uninitialised embeddings. Measured on Qwen3.8-Flash-Next UD-Q4_K_XL with the card held at 30.7 GiB free, -c 8192, --fit on, --spec-type draft-mtp --spec-draft-n-max 2: mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf before: OOM at load after: 63.6 tok/s mtp-Qwen3.8-Flash-Next-Q8_0.gguf 61.2 tok/s, unchanged no drafter 43.8 tok/s The shared head is the one the model card recommends as fastest, and this is the first configuration in which a 16 to 32 GB card can load it. --- src/llama-model-loader.cpp | 39 ++++++++++++++++++++++++++++++++++++-- src/llama-model-loader.h | 4 +++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 5eda2f345b48..6749abc890ef 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1109,7 +1109,8 @@ bool llama_model_loader::lazy_read::add(const std::string & name, const ggml_ten // declared in llama-model.h, which this file does not include const std::vector> & llama_internal_get_tensor_map(const llama_model * model); -struct ggml_tensor * llama_model_loader::borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne) { +struct ggml_tensor * llama_model_loader::borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, + bool * borrows_without_target) { // checked first so no other tensor in any model pays a metadata lookup if (tn.tensor != LLM_TENSOR_TOKEN_EMBD && tn.tensor != LLM_TENSOR_OUTPUT && tn.tensor != LLM_TENSOR_OUTPUT_NORM) { return nullptr; @@ -1130,6 +1131,12 @@ struct ggml_tensor * llama_model_loader::borrow_shared_tensor(const LLM_TN_IMPL } if (model_shared == nullptr) { + // the fit opens the draft alone to price it; throwing there only makes it budget + // nothing for the draft, so report the borrow and let the caller stand a shape in + if (no_alloc && borrows_without_target != nullptr) { + *borrows_without_target = true; + return nullptr; + } throw std::runtime_error(format("%s: this model is a draft head without its own '%s'; " "load it as a draft of its target model, not on its own", __func__, name.c_str())); } @@ -1389,9 +1396,37 @@ struct ggml_tensor * llama_model_loader::create_tensor( } // must precede check_tensor_dims, and must win over the arch fallback that ties output to token_embd - if (ggml_tensor * shared = borrow_shared_tensor(tn, ne)) { + bool borrows_without_target = false; + if (ggml_tensor * shared = borrow_shared_tensor(tn, ne, &borrows_without_target)) { return shared; } + if (borrows_without_target) { + // shape only, on CPU: the graph needs the dimensions, but the bytes are the + // target's and are counted in its own measurement + ggml_type type = GGML_TYPE_F32; + const int64_t tid_shared = gguf_find_tensor(metadata, tn.str().c_str()); + if (tid_shared != -1) { + type = gguf_get_tensor_type(metadata, tid_shared); + } + ggml_tensor t_shared; + memset(&t_shared, 0, sizeof(ggml_tensor)); + t_shared.type = type; + for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) { + t_shared.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1; + GGML_ASSERT(t_shared.ne[dim] >= 1); + if (dim == 0) { + t_shared.nb[dim] = ggml_type_size(type); + } else if (dim == 1) { + t_shared.nb[dim] = ggml_row_size(type, t_shared.ne[dim-1]); + } else { + t_shared.nb[dim] = t_shared.nb[dim-1]*t_shared.ne[dim-1]; + } + } + ggml_set_name(&t_shared, tn.str().c_str()); + ggml_tensor * ret = ggml_dup_tensor(ctx_for_buft(ggml_backend_cpu_buffer_type()), &t_shared); + ggml_set_name(ret, tn.str().c_str()); + return ret; + } LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str()); const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED), flags & TENSOR_ALLOW_RESHAPE); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 7cf1d823cde8..0632d5e6a541 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -244,7 +244,9 @@ struct llama_model_loader { const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); // token_embd/output/output_norm from the target. null unless the file declares the flag. - struct ggml_tensor * borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne); + // borrows_without_target: borrowed, but no target attached (measurement load only) + struct ggml_tensor * borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, + bool * borrows_without_target = nullptr); void done_getting_tensors(bool partial = false) const;