diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 5eda2f345b48..6749abc890ef 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1109,7 +1109,8 @@ bool llama_model_loader::lazy_read::add(const std::string & name, const ggml_ten // declared in llama-model.h, which this file does not include const std::vector> & llama_internal_get_tensor_map(const llama_model * model); -struct ggml_tensor * llama_model_loader::borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne) { +struct ggml_tensor * llama_model_loader::borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, + bool * borrows_without_target) { // checked first so no other tensor in any model pays a metadata lookup if (tn.tensor != LLM_TENSOR_TOKEN_EMBD && tn.tensor != LLM_TENSOR_OUTPUT && tn.tensor != LLM_TENSOR_OUTPUT_NORM) { return nullptr; @@ -1130,6 +1131,12 @@ struct ggml_tensor * llama_model_loader::borrow_shared_tensor(const LLM_TN_IMPL } if (model_shared == nullptr) { + // the fit opens the draft alone to price it; throwing there only makes it budget + // nothing for the draft, so report the borrow and let the caller stand a shape in + if (no_alloc && borrows_without_target != nullptr) { + *borrows_without_target = true; + return nullptr; + } throw std::runtime_error(format("%s: this model is a draft head without its own '%s'; " "load it as a draft of its target model, not on its own", __func__, name.c_str())); } @@ -1389,9 +1396,37 @@ struct ggml_tensor * llama_model_loader::create_tensor( } // must precede check_tensor_dims, and must win over the arch fallback that ties output to token_embd - if (ggml_tensor * shared = borrow_shared_tensor(tn, ne)) { + bool borrows_without_target = false; + if (ggml_tensor * shared = borrow_shared_tensor(tn, ne, &borrows_without_target)) { return shared; } + if (borrows_without_target) { + // shape only, on CPU: the graph needs the dimensions, but the bytes are the + // target's and are counted in its own measurement + ggml_type type = GGML_TYPE_F32; + const int64_t tid_shared = gguf_find_tensor(metadata, tn.str().c_str()); + if (tid_shared != -1) { + type = gguf_get_tensor_type(metadata, tid_shared); + } + ggml_tensor t_shared; + memset(&t_shared, 0, sizeof(ggml_tensor)); + t_shared.type = type; + for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) { + t_shared.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1; + GGML_ASSERT(t_shared.ne[dim] >= 1); + if (dim == 0) { + t_shared.nb[dim] = ggml_type_size(type); + } else if (dim == 1) { + t_shared.nb[dim] = ggml_row_size(type, t_shared.ne[dim-1]); + } else { + t_shared.nb[dim] = t_shared.nb[dim-1]*t_shared.ne[dim-1]; + } + } + ggml_set_name(&t_shared, tn.str().c_str()); + ggml_tensor * ret = ggml_dup_tensor(ctx_for_buft(ggml_backend_cpu_buffer_type()), &t_shared); + ggml_set_name(ret, tn.str().c_str()); + return ret; + } LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str()); const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED), flags & TENSOR_ALLOW_RESHAPE); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 7cf1d823cde8..0632d5e6a541 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -244,7 +244,9 @@ struct llama_model_loader { const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); // token_embd/output/output_norm from the target. null unless the file declares the flag. - struct ggml_tensor * borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne); + // borrows_without_target: borrowed, but no target attached (measurement load only) + struct ggml_tensor * borrow_shared_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, + bool * borrows_without_target = nullptr); void done_getting_tensors(bool partial = false) const; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 97652d42f390..e329d13b8a7c 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -277,7 +277,7 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { hc_dim }, flags); layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, flags); - layer.nextn.hc_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_NORM, "weight", il), { hc_dim }, flags); + layer.nextn.hc_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_NORM, "weight", il), { n_embd, hc }, flags | TENSOR_ALLOW_RESHAPE); layer.nextn.hc_head_down = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_DOWN, "weight", il), { hc_dim, hc_lr }, flags); layer.nextn.hc_head_up = create_tensor(tn(LLM_TENSOR_NEXTN_HC_HEAD_UP, "weight", il), { hc_lr, hc_dim }, flags);