Skip to content

Support standalone Qwen3.5 MTP companion GGUFs (-md ... --spec-type mtp) #43

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Thireus merged 1 commit into from
Aug 16, 2026
Merged

Support standalone Qwen3.5 MTP companion GGUFs (-md ... --spec-type mtp) #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions common/common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4072,6 +4072,17 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
return iparams;
}

// A predictor-only MTP companion has no main blocks, so it cannot serve as the target model.
// Reject it here: llama_model_load() cannot tell a -m load from a -md load, and building a
// target context over the absent blocks aborts deep inside the graph builder.
if (llama_model_mtp_package(model) == LLAMA_MTP_PACKAGE_COMPANION) {
fprintf(stderr, "%s: error: '%s' is an MTP predictor-only companion GGUF; "
"pass it with -md/--model-draft and load a complete model with -m\n",
__func__, params.model.c_str());
llama_free_model(model);
return iparams;
}

auto cparams = common_context_params_to_llama(params);

llama_context * lctx = llama_init_from_model(model, cparams);
Expand Down
29 changes: 27 additions & 2 deletions common/speculative.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ static bool common_speculative_are_compatible(
}

static bool common_speculative_target_has_appended_mtp_contract(const llama_model * model) {
return llama_model_is_step35(model) || llama_model_is_deepseek4(model);
return llama_model_is_step35(model) || llama_model_is_deepseek4(model) ||
llama_model_is_qwen35_family(model);
}

static bool common_speculative_has_recognized_mtp_companion(
Expand Down Expand Up @@ -2041,6 +2042,9 @@ bool common_speculative_prepare_mtp_runtime(
params.cparams_dft.mtp = true;
params.cparams_dft.mtp_op_type = MTP_OP_WARMUP;
params.cparams_dft.embeddings = true;
// An MTP graph never appends a pooling layer, so a pooled cparams would leave inp_mean/inp_cls
// null and abort in llama_set_inputs on the first companion decode.
params.cparams_dft.pooling_type = LLAMA_POOLING_TYPE_NONE;

return true;
}
Expand Down Expand Up @@ -2127,7 +2131,12 @@ bool common_speculative_finalize_startup(
const llama_model * companion = params.model_dft;
const bool appended_contract = common_speculative_target_has_appended_mtp_contract(model);

if (appended_contract &&
// A Qwen3.5 target may already carry its own NextN tail. In that case -md is an ordinary
// draft model for a chained draft stage and need not be a predictor-only companion.
const bool target_self_mtp = llama_model_is_qwen35_family(model) &&
llama_model_mtp_package(model) == LLAMA_MTP_PACKAGE_EMBEDDED;

if (appended_contract && !target_self_mtp &&
llama_model_mtp_package(companion) != LLAMA_MTP_PACKAGE_COMPANION) {
LOG_ERR("%s: -md for an MTP stage must be a predictor-only companion GGUF\n", __func__);
return false;
Expand Down Expand Up @@ -2162,6 +2171,22 @@ bool common_speculative_finalize_startup(
__func__, n_heads);
return false;
}
} else if (llama_model_is_qwen35_family(model) &&
llama_model_mtp_package(companion) == LLAMA_MTP_PACKAGE_COMPANION) {
const char * arch_tgt = llama_model_arch_string(model);
const char * arch_dft = llama_model_arch_string(companion);
if (arch_tgt == nullptr || arch_dft == nullptr || std::strcmp(arch_tgt, arch_dft) != 0) {
LOG_ERR("%s: Qwen3.5 MTP companion must use the same architecture as the target (target=%s, companion=%s)\n",
__func__, arch_tgt ? arch_tgt : "?", arch_dft ? arch_dft : "?");
return false;
}

const int32_t n_heads = llama_model_n_nextn_layer(companion);
if (n_heads != 1) {
LOG_ERR("%s: Qwen3.5 MTP companion requires exactly one predictor layer, got %d\n",
__func__, n_heads);
return false;
}
}

if (common_speculative_has_recognized_mtp_companion(model, companion)) {
Expand Down
3 changes: 3 additions & 0 deletions include/llama.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,9 @@ extern "C" {

LLAMA_API bool llama_model_is_step35(const struct llama_model * model);

// Returns true for Qwen3.5 dense/MoE, whose MTP tail can ship either embedded or as a separate companion GGUF
LLAMA_API bool llama_model_is_qwen35_family(const struct llama_model * model);

LLAMA_API bool llama_is_gemma4_mtp_file(const char * path);

LLAMA_API bool llama_model_is_split_mode_graph(const struct llama_model * model);
Expand Down
9 changes: 7 additions & 2 deletions src/llama-hparams.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -644,12 +644,17 @@ void llm_load_hparams(
}
}

// the +1 variants carry the MTP block (either embedded or as a standalone companion)
switch (hparams.n_layer) {
case 40:
case 41:
model.type = e_model::MODEL_35B_A3B; break;
case 48: model.type = e_model::MODEL_122B_A10B; break;
case 60: model.type = e_model::MODEL_397B_A17B; break;
case 48:
case 49:
model.type = e_model::MODEL_122B_A10B; break;
case 60:
case 61:
model.type = e_model::MODEL_397B_A17B; break;
default: model.type = e_model::MODEL_UNKNOWN;
}
} break;
Expand Down
43 changes: 37 additions & 6 deletions src/llama-load-tensors.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1704,6 +1704,16 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) {
const int64_t value_dim = head_v_dim * n_v_heads;
const int64_t conv_dim = key_dim * 2 + value_dim;

// See create_qwen35_tensors: a predictor-only companion ships only the NextN block.
const bool mtp_only = hparams.nextn_predict_layers > 0 &&
ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr;
const int trunk_flags = mtp_only
? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0;
if (mtp_only) {
LLAMA_LOG_INFO("%s: standalone MTP companion - skipping %d absent main blocks\n",
__func__, (int)(n_layer - hparams.nextn_predict_layers));
}

for (int i = 0; i < n_layer; ++i) {
const bool is_mtp_layer = hparams.nextn_predict_layers > 0 &&
static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers;
Expand All @@ -1712,7 +1722,7 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) {

auto & layer = model.layers[i];

int flags = 0;
int flags = is_mtp_layer ? 0 : trunk_flags;
if (!model.mtp && is_mtp_layer) {
flags |= llama_model_loader::TENSOR_SKIP;
}
Expand Down Expand Up @@ -1813,6 +1823,17 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) {
const int64_t value_dim = head_v_dim * n_v_heads;
const int64_t conv_dim = key_dim * 2 + value_dim;

// A predictor-only companion GGUF advertises the full block count (n_main + nextn) but ships only the
// NextN block, so blk.0 .. blk.n_main-1 are absent - they live in the target GGUF passed with -m.
const bool mtp_only = hparams.nextn_predict_layers > 0 &&
ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr;
const int trunk_flags = mtp_only
? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0;
if (mtp_only) {
LLAMA_LOG_INFO("%s: standalone MTP companion - skipping %d absent main blocks\n",
__func__, (int)(n_layer - hparams.nextn_predict_layers));
}

for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];

Expand All @@ -1821,7 +1842,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) {

ggml_context * ctx_split = ctx_for_layer_split(i);

int flags = 0;
int flags = is_mtp_layer ? 0 : trunk_flags;
// Skip loading MTP layers if the feature is disabled
if (!model.mtp) {
if (is_mtp_layer) {
Expand Down Expand Up @@ -1887,13 +1908,15 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) {
}
}

// 9B shares q_proj
if (model.mtp && hparams.nextn_predict_layers > 0) {
// 9B shares q_proj. This is only possible when the trunk is part of this GGUF: a predictor-only
// companion has no layer n_main-1 to borrow wq from, and n_main could be 0.
if (model.mtp && !mtp_only && hparams.nextn_predict_layers > 0 &&
hparams.nextn_predict_layers < (uint32_t) n_layer) {
const uint32_t n_main = n_layer - hparams.nextn_predict_layers;
auto & last_main = model.layers[n_main - 1];
for (uint32_t i = n_main; i < (uint32_t)n_layer; ++i) {
auto & mtp_layer = model.layers[i];
auto & last_main = model.layers[n_main - 1];
if (mtp_layer.wq == nullptr) {
if (mtp_layer.wq == nullptr && last_main.wq != nullptr) {
mtp_layer.wq = last_main.wq;
}
}
Expand Down Expand Up @@ -5395,6 +5418,14 @@ bool create_tensors_helper::create_tensors() {
for ([[maybe_unused]] auto mem : mem_used) LLAMA_LOG_DEBUG(" %g", mem/1024./1024.);
LLAMA_LOG_DEBUG("\n");
auto & layer = model.layers[il];
// A predictor-only MTP companion GGUF advertises the full block count but ships only the
// NextN block, so its main blocks hold no tensors at all - there is nothing to split, and
// split_recurrent_tensors() would assert on the missing ssm_in/wqkv.
if (!layer.attn_norm && !layer.wq && !layer.wqkv && !layer.ssm_in &&
!layer.wo && !layer.ffn_down && !layer.ffn_down_exps && !layer.ffn_up_gate_exps) {
LLAMA_LOG_DEBUG("%s: layer %d holds no tensors - nothing to split\n", __func__, il);
continue;
}
auto ctx_split = ctx_for_layer_split(il);
if (layer.attn_norm) {
prepare_split_tensors(-1, ctx_split, layer.attn_norm, layer.split_attn_norm, mirror, mem_used);
Expand Down
14 changes: 12 additions & 2 deletions src/llama-model.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2319,6 +2319,10 @@ bool llama_model_is_step35(const llama_model * model) {
return model && model->arch == LLM_ARCH_STEP35;
}

bool llama_model_is_qwen35_family(const llama_model * model) {
return model && (model->arch == LLM_ARCH_QWEN35 || model->arch == LLM_ARCH_QWEN35MOE);
}

enum llama_mtp_package llama_model_mtp_package(const llama_model * model) {
if (!model) {
return LLAMA_MTP_PACKAGE_INVALID;
Expand All @@ -2331,13 +2335,15 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) {
const size_t n_nextn = model->hparams.nextn_predict_layers;
const bool has_common_package_contract =
llama_model_is_step35(model) || llama_model_is_deepseek4(model) ||
llama_model_is_qwen35_family(model) ||
llama_model_is_gemma4_mtp_assistant(model);
if (!has_common_package_contract) {
return n_nextn > 0 ? LLAMA_MTP_PACKAGE_EMBEDDED : LLAMA_MTP_PACKAGE_NONE;
}

if (n_nextn == 0) {
if (llama_model_is_step35(model) || llama_model_is_deepseek4(model)) {
if (llama_model_is_step35(model) || llama_model_is_deepseek4(model) ||
llama_model_is_qwen35_family(model)) {
for (const auto & layer : model->layers) {
if (layer.attn_norm != nullptr) {
return LLAMA_MTP_PACKAGE_TARGET_ONLY;
Expand All @@ -2353,7 +2359,11 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) {
}

const size_t first = n_layers - n_nextn;
const bool has_tail = model->layers[first].nextn.eh_proj != nullptr;
// Qwen3.5 9B/4B NextN blocks have no eh_proj (the fused input is an add, not a projection),
// so enorm/hnorm - which every Qwen3.5 NextN block carries - also mark a predictor tail.
const bool has_tail = model->layers[first].nextn.eh_proj != nullptr ||
(llama_model_is_qwen35_family(model) &&
(model->layers[first].nextn.enorm != nullptr || model->layers[first].nextn.hnorm != nullptr));

bool has_trunk = false;
for (size_t il = 0; il < first; ++il) {
Expand Down
Loading