diff --git a/common/common.cpp b/common/common.cpp index f241371fbf..d02c811927 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -4072,6 +4072,17 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) { return iparams; } + // A predictor-only MTP companion has no main blocks, so it cannot serve as the target model. + // Reject it here: llama_model_load() cannot tell a -m load from a -md load, and building a + // target context over the absent blocks aborts deep inside the graph builder. + if (llama_model_mtp_package(model) == LLAMA_MTP_PACKAGE_COMPANION) { + fprintf(stderr, "%s: error: '%s' is an MTP predictor-only companion GGUF; " + "pass it with -md/--model-draft and load a complete model with -m\n", + __func__, params.model.c_str()); + llama_free_model(model); + return iparams; + } + auto cparams = common_context_params_to_llama(params); llama_context * lctx = llama_init_from_model(model, cparams); diff --git a/common/speculative.cpp b/common/speculative.cpp index b641f3ef4d..cd220ee219 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -145,7 +145,8 @@ static bool common_speculative_are_compatible( } static bool common_speculative_target_has_appended_mtp_contract(const llama_model * model) { - return llama_model_is_step35(model) || llama_model_is_deepseek4(model); + return llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model); } static bool common_speculative_has_recognized_mtp_companion( @@ -2041,6 +2042,9 @@ bool common_speculative_prepare_mtp_runtime( params.cparams_dft.mtp = true; params.cparams_dft.mtp_op_type = MTP_OP_WARMUP; params.cparams_dft.embeddings = true; + // An MTP graph never appends a pooling layer, so a pooled cparams would leave inp_mean/inp_cls + // null and abort in llama_set_inputs on the first companion decode. + params.cparams_dft.pooling_type = LLAMA_POOLING_TYPE_NONE; return true; } @@ -2127,7 +2131,12 @@ bool common_speculative_finalize_startup( const llama_model * companion = params.model_dft; const bool appended_contract = common_speculative_target_has_appended_mtp_contract(model); - if (appended_contract && + // A Qwen3.5 target may already carry its own NextN tail. In that case -md is an ordinary + // draft model for a chained draft stage and need not be a predictor-only companion. + const bool target_self_mtp = llama_model_is_qwen35_family(model) && + llama_model_mtp_package(model) == LLAMA_MTP_PACKAGE_EMBEDDED; + + if (appended_contract && !target_self_mtp && llama_model_mtp_package(companion) != LLAMA_MTP_PACKAGE_COMPANION) { LOG_ERR("%s: -md for an MTP stage must be a predictor-only companion GGUF\n", __func__); return false; @@ -2162,6 +2171,22 @@ bool common_speculative_finalize_startup( __func__, n_heads); return false; } + } else if (llama_model_is_qwen35_family(model) && + llama_model_mtp_package(companion) == LLAMA_MTP_PACKAGE_COMPANION) { + const char * arch_tgt = llama_model_arch_string(model); + const char * arch_dft = llama_model_arch_string(companion); + if (arch_tgt == nullptr || arch_dft == nullptr || std::strcmp(arch_tgt, arch_dft) != 0) { + LOG_ERR("%s: Qwen3.5 MTP companion must use the same architecture as the target (target=%s, companion=%s)\n", + __func__, arch_tgt ? arch_tgt : "?", arch_dft ? arch_dft : "?"); + return false; + } + + const int32_t n_heads = llama_model_n_nextn_layer(companion); + if (n_heads != 1) { + LOG_ERR("%s: Qwen3.5 MTP companion requires exactly one predictor layer, got %d\n", + __func__, n_heads); + return false; + } } if (common_speculative_has_recognized_mtp_companion(model, companion)) { diff --git a/include/llama.h b/include/llama.h index 177f1a7193..5cb25f0e65 100644 --- a/include/llama.h +++ b/include/llama.h @@ -713,6 +713,9 @@ extern "C" { LLAMA_API bool llama_model_is_step35(const struct llama_model * model); + // Returns true for Qwen3.5 dense/MoE, whose MTP tail can ship either embedded or as a separate companion GGUF + LLAMA_API bool llama_model_is_qwen35_family(const struct llama_model * model); + LLAMA_API bool llama_is_gemma4_mtp_file(const char * path); LLAMA_API bool llama_model_is_split_mode_graph(const struct llama_model * model); diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 84eb3b588f..e97a4c0894 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -644,12 +644,17 @@ void llm_load_hparams( } } + // the +1 variants carry the MTP block (either embedded or as a standalone companion) switch (hparams.n_layer) { case 40: case 41: model.type = e_model::MODEL_35B_A3B; break; - case 48: model.type = e_model::MODEL_122B_A10B; break; - case 60: model.type = e_model::MODEL_397B_A17B; break; + case 48: + case 49: + model.type = e_model::MODEL_122B_A10B; break; + case 60: + case 61: + model.type = e_model::MODEL_397B_A17B; break; default: model.type = e_model::MODEL_UNKNOWN; } } break; diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index 47300df048..b245f3a1d4 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -1704,6 +1704,16 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) { const int64_t value_dim = head_v_dim * n_v_heads; const int64_t conv_dim = key_dim * 2 + value_dim; + // See create_qwen35_tensors: a predictor-only companion ships only the NextN block. + const bool mtp_only = hparams.nextn_predict_layers > 0 && + ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr; + const int trunk_flags = mtp_only + ? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0; + if (mtp_only) { + LLAMA_LOG_INFO("%s: standalone MTP companion - skipping %d absent main blocks\n", + __func__, (int)(n_layer - hparams.nextn_predict_layers)); + } + for (int i = 0; i < n_layer; ++i) { const bool is_mtp_layer = hparams.nextn_predict_layers > 0 && static_cast(i) >= n_layer - hparams.nextn_predict_layers; @@ -1712,7 +1722,7 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) { auto & layer = model.layers[i]; - int flags = 0; + int flags = is_mtp_layer ? 0 : trunk_flags; if (!model.mtp && is_mtp_layer) { flags |= llama_model_loader::TENSOR_SKIP; } @@ -1813,6 +1823,17 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { const int64_t value_dim = head_v_dim * n_v_heads; const int64_t conv_dim = key_dim * 2 + value_dim; + // A predictor-only companion GGUF advertises the full block count (n_main + nextn) but ships only the + // NextN block, so blk.0 .. blk.n_main-1 are absent - they live in the target GGUF passed with -m. + const bool mtp_only = hparams.nextn_predict_layers > 0 && + ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr; + const int trunk_flags = mtp_only + ? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0; + if (mtp_only) { + LLAMA_LOG_INFO("%s: standalone MTP companion - skipping %d absent main blocks\n", + __func__, (int)(n_layer - hparams.nextn_predict_layers)); + } + for (int i = 0; i < n_layer; ++i) { auto & layer = model.layers[i]; @@ -1821,7 +1842,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { ggml_context * ctx_split = ctx_for_layer_split(i); - int flags = 0; + int flags = is_mtp_layer ? 0 : trunk_flags; // Skip loading MTP layers if the feature is disabled if (!model.mtp) { if (is_mtp_layer) { @@ -1887,13 +1908,15 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { } } - // 9B shares q_proj - if (model.mtp && hparams.nextn_predict_layers > 0) { + // 9B shares q_proj. This is only possible when the trunk is part of this GGUF: a predictor-only + // companion has no layer n_main-1 to borrow wq from, and n_main could be 0. + if (model.mtp && !mtp_only && hparams.nextn_predict_layers > 0 && + hparams.nextn_predict_layers < (uint32_t) n_layer) { const uint32_t n_main = n_layer - hparams.nextn_predict_layers; + auto & last_main = model.layers[n_main - 1]; for (uint32_t i = n_main; i < (uint32_t)n_layer; ++i) { auto & mtp_layer = model.layers[i]; - auto & last_main = model.layers[n_main - 1]; - if (mtp_layer.wq == nullptr) { + if (mtp_layer.wq == nullptr && last_main.wq != nullptr) { mtp_layer.wq = last_main.wq; } } @@ -5395,6 +5418,14 @@ bool create_tensors_helper::create_tensors() { for ([[maybe_unused]] auto mem : mem_used) LLAMA_LOG_DEBUG(" %g", mem/1024./1024.); LLAMA_LOG_DEBUG("\n"); auto & layer = model.layers[il]; + // A predictor-only MTP companion GGUF advertises the full block count but ships only the + // NextN block, so its main blocks hold no tensors at all - there is nothing to split, and + // split_recurrent_tensors() would assert on the missing ssm_in/wqkv. + if (!layer.attn_norm && !layer.wq && !layer.wqkv && !layer.ssm_in && + !layer.wo && !layer.ffn_down && !layer.ffn_down_exps && !layer.ffn_up_gate_exps) { + LLAMA_LOG_DEBUG("%s: layer %d holds no tensors - nothing to split\n", __func__, il); + continue; + } auto ctx_split = ctx_for_layer_split(il); if (layer.attn_norm) { prepare_split_tensors(-1, ctx_split, layer.attn_norm, layer.split_attn_norm, mirror, mem_used); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 119f0ca390..d99b318514 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2319,6 +2319,10 @@ bool llama_model_is_step35(const llama_model * model) { return model && model->arch == LLM_ARCH_STEP35; } +bool llama_model_is_qwen35_family(const llama_model * model) { + return model && (model->arch == LLM_ARCH_QWEN35 || model->arch == LLM_ARCH_QWEN35MOE); +} + enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { if (!model) { return LLAMA_MTP_PACKAGE_INVALID; @@ -2331,13 +2335,15 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { const size_t n_nextn = model->hparams.nextn_predict_layers; const bool has_common_package_contract = llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model) || llama_model_is_gemma4_mtp_assistant(model); if (!has_common_package_contract) { return n_nextn > 0 ? LLAMA_MTP_PACKAGE_EMBEDDED : LLAMA_MTP_PACKAGE_NONE; } if (n_nextn == 0) { - if (llama_model_is_step35(model) || llama_model_is_deepseek4(model)) { + if (llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model)) { for (const auto & layer : model->layers) { if (layer.attn_norm != nullptr) { return LLAMA_MTP_PACKAGE_TARGET_ONLY; @@ -2353,7 +2359,11 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { } const size_t first = n_layers - n_nextn; - const bool has_tail = model->layers[first].nextn.eh_proj != nullptr; + // Qwen3.5 9B/4B NextN blocks have no eh_proj (the fused input is an add, not a projection), + // so enorm/hnorm - which every Qwen3.5 NextN block carries - also mark a predictor tail. + const bool has_tail = model->layers[first].nextn.eh_proj != nullptr || + (llama_model_is_qwen35_family(model) && + (model->layers[first].nextn.enorm != nullptr || model->layers[first].nextn.hnorm != nullptr)); bool has_trunk = false; for (size_t il = 0; il < first; ++il) {