Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fit: Fix memory allocation for MTP layers (#26605)

Browse Source
This commit is contained in:
Sergey Malinin
2026-08-05 13:29:45 +02:00
committed by GitHub
parent 9303cdd8d3
commit 9a688e51e6
2 changed files with 19 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/fit.cpp
+4 -1
View File
@@ -136,7 +136,10 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
devs.push_back(llama_model_get_device(model, i)); devs.push_back(llama_model_get_device(model, i));
} }
hp_ngl = llama_model_n_layer(model) + llama_model_n_layer_nextn(model); hp_ngl = llama_model_n_layer(model);
if (mparams->load_mtp) {
hp_ngl += llama_model_n_layer_nextn(model);
}
hp_n_ctx_train = llama_model_n_ctx_train(model); hp_n_ctx_train = llama_model_n_ctx_train(model);
hp_n_expert = llama_model_n_expert(model); hp_n_expert = llama_model_n_expert(model);
src/llama-model.cpp
+15
View File
@@ -2890,6 +2890,21 @@ void llama_model_base::create_tensor_qkv(llama_layer & layer, int bid,
int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_, int64_t n_embd_, int64_t n_embd_q_, int64_t n_embd_k_, int64_t n_embd_v_,
int flags) { int flags) {
const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_; const int64_t n_embd_qkv = n_embd_q_ + n_embd_k_ + n_embd_v_;
if (flags & TENSOR_SKIP) {
const int skip = TENSOR_NOT_REQUIRED | TENSOR_SKIP;
create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, skip | TENSOR_SKIP_IF_VIRTUAL);
create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", bid), {n_embd_, n_embd_q_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", bid), {n_embd_, n_embd_k_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", bid), {n_embd_, n_embd_v_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", bid), {n_embd_q_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", bid), {n_embd_k_}, skip);
create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", bid), {n_embd_v_}, skip);
return;
}
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", bid), {n_embd_, n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
if (layer.wqkv) { if (layer.wqkv) {
layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); layer.wqkv_b = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", bid), {n_embd_qkv}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);