cuda: Scatter kernel for set_n (#32489)

* cuda: add scatter for batched tensor writes

* update test case to mirror get_n test case

* save restore timing logs

* refactor code into batch-copy.cu file
This commit is contained in:
Bhargav Krish authored and GitHub committed 2026-11-19 20:14:47 +05:30
1 parent ac830a9cbb
commit c7468107ef
6 files changed
+117 -45

No files matched your search

+4
View File
@@ -353,6 +353,8 @@ struct server_slot {
return false;
}
const int64_t t_start = ggml_time_us();
const size_t cur_size_tgt = llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE);
const size_t cur_size_dft = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0;
@@ -374,6 +376,8 @@ struct server_slot {
return false;
}
SLT_TRC(*this, "prompt cache save took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
return true;
}