model: fix vllm d3 multi seq case (#30408)

* model: fix vllm d3 multi seq case

* add TODO
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-13 00:23:01 +02:00
1 parent 358a420ec4
commit cbe28bb3cb
5 files changed
+7 -7

No files matched your search

+1
View File
@@ -186,6 +186,7 @@ public:
llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail);
// sequence-set-wise split - each ubatch contains a single sequence-set
// TODO: extend to support multiple _full_ sequences in the ubatch
llama_ubatch split_seq(uint32_t n_ubatch);
// a helper method for creating a well-defined ubatch of tokens
+3 -1
View File
@@ -1838,6 +1838,8 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
// when computing embeddings, all tokens are output
const bool output_all = cparams.embeddings;
// non-causal attention needs the whole sequence in one ubatch
const bool embd_all = output_all || !cparams.causal_attn;
const bool has_samplers = !sampling.samplers.empty();
const uint32_t n_seq_max = cparams.kv_unified ? LLAMA_MAX_SEQ : cparams.n_seq_max;
@@ -1916,7 +1918,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
llama_memory_context_ptr mctx;
while (true) {
mctx = memory->init_batch(*balloc, cparams.n_ubatch, output_all);
mctx = memory->init_batch(*balloc, cparams.n_ubatch, embd_all);
if (!mctx) {
return -2;
}
+1 -3
View File
@@ -157,8 +157,6 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_iswa::memory_breakdo
}
llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
GGML_UNUSED(embd_all);
// first try simple split
do {
if (!unified) {
@@ -206,7 +204,7 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & ba
std::vector<llama_ubatch> ubatches;
while (true) {
auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);
auto ubatch = embd_all ? balloc.split_seq(n_ubatch) : balloc.split_equal(n_ubatch, !unified, 0);
if (ubatch.n_tokens == 0) {
break;
+1 -3
View File
@@ -721,14 +721,12 @@ llama_memory_context_ptr llama_kv_cache::init_batch(
llama_batch_allocr & balloc,
uint32_t n_ubatch,
bool embd_all) {
GGML_UNUSED(embd_all);
do {
balloc.split_reset();
std::vector<llama_ubatch> ubatches;
while (true) {
auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : embd_all ? balloc.split_seq(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
if (ubatch.n_tokens == 0) {
break;
+1
View File
@@ -85,6 +85,7 @@ struct llama_memory_i {
// split the input batch into a set of ubatches and verify that they can fit into the cache
// return a context object containing the ubatches and memory state required to process them
// check the llama_memory_context_i::get_status() for the result
// embd_all: keep all the tokens of a sequence in one ubatch (all tokens are output, or the attention is non-causal)
virtual llama_memory_context_ptr init_batch(
llama_batch_allocr & balloc,
uint32_t n_ubatch,