llama : fix on-device state copy sizes for quantized KV caches (#30541)
* llama : fix on-device state copy sizes for quantized KV caches The on-device sequence state copies (LLAMA_STATE_SEQ_FLAGS_ON_DEVICE) computed their element count as size / ggml_element_size(). For a quantized type such as q8_0 or q4_0 that is the number of blocks, not elements, so only part of each range was copied: the restored state was wrong, or the copy crashed. Compute the count as size / ggml_type_size() * ggml_blck_size() for the save, the restore and the copy between differently chunked ranges, and assert that each range covers whole blocks. * llama : drop the one-line comments * llama : drop the redundant cast in the state copy size
This commit is contained in:
1 parent
3de52ac388
commit
e56f006ed6
1 file changed
+6
-4
@@ -3350,7 +3350,8 @@ public:
|
||||
for (const auto & winfo : winfos) {
|
||||
auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer);
|
||||
|
||||
const int64_t n = winfo.size/ggml_element_size(winfo.tensor);
|
||||
GGML_ASSERT(winfo.size % ggml_type_size(winfo.tensor->type) == 0);
|
||||
const int64_t n = winfo.size/ggml_type_size(winfo.tensor->type)*ggml_blck_size(winfo.tensor->type);
|
||||
|
||||
auto & mbuf = mbufs_new[buft];
|
||||
|
||||
@@ -3481,7 +3482,8 @@ public:
|
||||
for (const auto & rinfo : rinfos) {
|
||||
auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer);
|
||||
|
||||
const int64_t n = rinfo.size/ggml_element_size(rinfo.tensor);
|
||||
GGML_ASSERT(rinfo.size % ggml_type_size(rinfo.tensor->type) == 0);
|
||||
const int64_t n = rinfo.size/ggml_type_size(rinfo.tensor->type)*ggml_blck_size(rinfo.tensor->type);
|
||||
|
||||
auto & mbuf = mbufs_new[buft];
|
||||
|
||||
@@ -3553,8 +3555,8 @@ public:
|
||||
|
||||
const size_t n_copy = std::min(src_size - src_off, dst_size - dst_off);
|
||||
|
||||
const size_t el = ggml_element_size(src_t);
|
||||
const int64_t n_el = (int64_t) (n_copy / el);
|
||||
GGML_ASSERT(n_copy % ggml_type_size(src_t->type) == 0);
|
||||
const int64_t n_el = n_copy / ggml_type_size(src_t->type) * ggml_blck_size(src_t->type);
|
||||
|
||||
auto * src_v = ggml_view_1d(ctx_scratch, src_t, n_el, src_off);
|
||||
ggml_backend_view_init(src_v);
|
||||
|
||||
Reference in new issue
Block a user