model: fix vllm d3 multi seq case (#30408)
* model: fix vllm d3 multi seq case * add TODO
This commit is contained in:
1 parent
358a420ec4
commit
cbe28bb3cb
5 files changed
+7
-7
No files matched your search
@@ -186,6 +186,7 @@ public:
|
||||
llama_ubatch split_equal(uint32_t n_ubatch, bool sequential, uint32_t n_keep_tail);
|
||||
|
||||
// sequence-set-wise split - each ubatch contains a single sequence-set
|
||||
// TODO: extend to support multiple _full_ sequences in the ubatch
|
||||
llama_ubatch split_seq(uint32_t n_ubatch);
|
||||
|
||||
// a helper method for creating a well-defined ubatch of tokens
|
||||
|
||||
@@ -1838,6 +1838,8 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
||||
|
||||
// when computing embeddings, all tokens are output
|
||||
const bool output_all = cparams.embeddings;
|
||||
// non-causal attention needs the whole sequence in one ubatch
|
||||
const bool embd_all = output_all || !cparams.causal_attn;
|
||||
const bool has_samplers = !sampling.samplers.empty();
|
||||
|
||||
const uint32_t n_seq_max = cparams.kv_unified ? LLAMA_MAX_SEQ : cparams.n_seq_max;
|
||||
@@ -1916,7 +1918,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
||||
llama_memory_context_ptr mctx;
|
||||
|
||||
while (true) {
|
||||
mctx = memory->init_batch(*balloc, cparams.n_ubatch, output_all);
|
||||
mctx = memory->init_batch(*balloc, cparams.n_ubatch, embd_all);
|
||||
if (!mctx) {
|
||||
return -2;
|
||||
}
|
||||
|
||||
@@ -157,8 +157,6 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache_iswa::memory_breakdo
|
||||
}
|
||||
|
||||
llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
|
||||
GGML_UNUSED(embd_all);
|
||||
|
||||
// first try simple split
|
||||
do {
|
||||
if (!unified) {
|
||||
@@ -206,7 +204,7 @@ llama_memory_context_ptr llama_kv_cache_iswa::init_batch(llama_batch_allocr & ba
|
||||
|
||||
std::vector<llama_ubatch> ubatches;
|
||||
while (true) {
|
||||
auto ubatch = balloc.split_equal(n_ubatch, !unified, 0);
|
||||
auto ubatch = embd_all ? balloc.split_seq(n_ubatch) : balloc.split_equal(n_ubatch, !unified, 0);
|
||||
|
||||
if (ubatch.n_tokens == 0) {
|
||||
break;
|
||||
|
||||
@@ -721,14 +721,12 @@ llama_memory_context_ptr llama_kv_cache::init_batch(
|
||||
llama_batch_allocr & balloc,
|
||||
uint32_t n_ubatch,
|
||||
bool embd_all) {
|
||||
GGML_UNUSED(embd_all);
|
||||
|
||||
do {
|
||||
balloc.split_reset();
|
||||
|
||||
std::vector<llama_ubatch> ubatches;
|
||||
while (true) {
|
||||
auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
|
||||
auto ubatch = n_stream == 1 ? balloc.split_simple(n_ubatch) : embd_all ? balloc.split_seq(n_ubatch) : balloc.split_equal(n_ubatch, true, 0);
|
||||
|
||||
if (ubatch.n_tokens == 0) {
|
||||
break;
|
||||
|
||||
@@ -85,6 +85,7 @@ struct llama_memory_i {
|
||||
// split the input batch into a set of ubatches and verify that they can fit into the cache
|
||||
// return a context object containing the ubatches and memory state required to process them
|
||||
// check the llama_memory_context_i::get_status() for the result
|
||||
// embd_all: keep all the tokens of a sequence in one ubatch (all tokens are output, or the attention is non-causal)
|
||||
virtual llama_memory_context_ptr init_batch(
|
||||
llama_batch_allocr & balloc,
|
||||
uint32_t n_ubatch,
|
||||
|
||||
Reference in new issue
Block a user