llama: support both embd + raw tokens in batch (#29622)
* llama: support both embd + raw tokens in batch * add to test-llama-archs * also check case llm_arch_supports_mixed_batch = false * constant graph topology * have dedicated input for mixed case * rm set_tensor_backend * is_embd --> type * consolidate m-rope pos handling into one place * nits
This commit is contained in:
1 parent
2ca15f5404
commit
0bb496dbd3
29 files changed
+506
-99
No files matched your search
+106
-39
@@ -12,7 +12,7 @@
|
||||
#include <algorithm>
|
||||
#include <sstream>
|
||||
|
||||
llama_batch_allocr::llama_batch_allocr(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {
|
||||
llama_batch_allocr::llama_batch_allocr(uint32_t n_pos_per_embd, bool allow_mixed) : n_pos_per_embd(n_pos_per_embd), allow_mixed(allow_mixed) {
|
||||
const char * LLAMA_BATCH_DEBUG = getenv("LLAMA_BATCH_DEBUG");
|
||||
debug = LLAMA_BATCH_DEBUG ? atoi(LLAMA_BATCH_DEBUG) : 0;
|
||||
|
||||
@@ -49,25 +49,49 @@ bool llama_batch_allocr::init(
|
||||
//
|
||||
// determine the content types of the batch
|
||||
// an entry can carry a token id, a token embedding, or both (e.g. MTP hook batches)
|
||||
// all entries must carry the same combination
|
||||
// all entries must carry the same combination, or be a mix of token and embd entries
|
||||
//
|
||||
|
||||
const bool has_token = batch_inp.tokens[0].id != LLAMA_TOKEN_NULL;
|
||||
const bool has_embd = batch_inp.tokens[0].has_embd;
|
||||
int32_t n_tok_only = 0;
|
||||
int32_t n_embd_only = 0;
|
||||
int32_t n_both = 0;
|
||||
|
||||
for (int32_t i = 1; i < n_tok; ++i) {
|
||||
if ((batch_inp.tokens[i].id != LLAMA_TOKEN_NULL) != has_token ||
|
||||
batch_inp.tokens[i].has_embd != has_embd) {
|
||||
LLAMA_LOG_ERROR("%s: all entries in the batch must have the same content types\n", __func__);
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
const bool is_tok = batch_inp.tokens[i].id != LLAMA_TOKEN_NULL;
|
||||
const bool is_emb = batch_inp.tokens[i].has_embd;
|
||||
|
||||
if (!is_tok && !is_emb) {
|
||||
LLAMA_LOG_ERROR("%s: entry %d has neither a token id nor an embedding\n", __func__, i);
|
||||
return false;
|
||||
}
|
||||
|
||||
n_tok_only += is_tok && !is_emb;
|
||||
n_embd_only += is_emb && !is_tok;
|
||||
n_both += is_tok && is_emb;
|
||||
}
|
||||
|
||||
if (!has_token && !has_embd) {
|
||||
LLAMA_LOG_ERROR("%s: batch has neither token ids nor embeddings\n", __func__);
|
||||
if (n_both > 0 && n_both != n_tok) {
|
||||
LLAMA_LOG_ERROR("%s: entries with both a token id and an embedding cannot be mixed with other entries\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool mixed = n_tok_only > 0 && n_embd_only > 0;
|
||||
|
||||
if (mixed && !allow_mixed) {
|
||||
LLAMA_LOG_ERROR("%s: this model or context does not support batches mixing token and embedding entries\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool has_token = n_tok_only > 0 || n_both > 0;
|
||||
const bool has_embd = n_embd_only > 0 || n_both > 0;
|
||||
|
||||
if (mixed) {
|
||||
is_embd_vec.resize(n_tok);
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
is_embd_vec[i] = batch_inp.tokens[i].has_embd ? 1 : 0;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// build flat token/embd array
|
||||
//
|
||||
@@ -75,6 +99,10 @@ bool llama_batch_allocr::init(
|
||||
if (has_token) {
|
||||
token_vec.resize(n_tok);
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
if (mixed && is_embd_vec[i]) {
|
||||
token_vec[i] = 0; // placeholder
|
||||
continue;
|
||||
}
|
||||
const llama_token id = batch_inp.tokens[i].id;
|
||||
if (id < 0 || id >= batch_inp.n_vocab) {
|
||||
LLAMA_LOG_ERROR("%s: invalid token[%d] = %d\n", __func__, i, id);
|
||||
@@ -84,29 +112,35 @@ bool llama_batch_allocr::init(
|
||||
}
|
||||
}
|
||||
|
||||
if (has_embd) {
|
||||
if (mixed) {
|
||||
embd_vec.assign((size_t) n_tok*n_embd, 0.0f);
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
if (is_embd_vec[i]) {
|
||||
const float * src = batch_inp.embd.data() + batch_inp.tokens[i].embd_off;
|
||||
std::copy(src, src + n_embd, embd_vec.data() + (size_t) i*n_embd);
|
||||
}
|
||||
}
|
||||
} else if (has_embd) {
|
||||
embd_vec = batch_inp.embd;
|
||||
}
|
||||
|
||||
//
|
||||
// build flat pos array
|
||||
// token batch: pos[i] = tokens[i].pos[0]
|
||||
// embedding batch: pos[j*n_tok + i] = tokens[i].pos[j] (section-major)
|
||||
// build flat pos array, section-major: pos[j*n_tok + i] = section j of entry i
|
||||
// token entry: [p, p, p, 0] (M-RoPE text position)
|
||||
// embd entry: tokens[i].pos as-is
|
||||
//
|
||||
|
||||
{
|
||||
const int32_t n_pos_total = has_token ? n_tok : n_tok * (int32_t) n_pos_per_embd;
|
||||
pos.resize(n_pos_total);
|
||||
if (has_token) {
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
pos[i] = batch_inp.tokens[i].pos[0];
|
||||
}
|
||||
} else {
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
for (uint32_t j = 0; j < n_pos_per_embd; ++j) {
|
||||
pos[(int32_t) j * n_tok + i] = batch_inp.tokens[i].pos[j];
|
||||
}
|
||||
pos.resize((size_t) n_tok*n_pos_per_embd);
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
const auto & tok = batch_inp.tokens[i];
|
||||
const bool expand = tok.id != LLAMA_TOKEN_NULL;
|
||||
for (uint32_t j = 0; j < n_pos_per_embd; ++j) {
|
||||
llama_pos p = tok.pos[j];
|
||||
if (expand) {
|
||||
// expand [p] to [p, p, p, 0] for M-RoPE
|
||||
p = j < 3 ? tok.pos[0] : 0;
|
||||
}
|
||||
pos[(size_t) j*n_tok + i] = p;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,6 +298,7 @@ bool llama_batch_allocr::init(
|
||||
/*.seq_id_unq =*/ this->seq_id_unq.data(),
|
||||
/*.seq_idx =*/ this->seq_idx.data(),
|
||||
/*.output =*/ batch.logits,
|
||||
/*.type =*/ is_embd_vec.empty() ? nullptr : is_embd_vec.data(),
|
||||
/*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
|
||||
/*.data =*/ {},
|
||||
};
|
||||
@@ -294,6 +329,21 @@ bool llama_batch_allocr::init(
|
||||
//
|
||||
|
||||
if (n_pos_per_embd > 1) {
|
||||
// in a mixed batch, the first entry of each seq picks the rule
|
||||
std::vector<int8_t> seq_first_embd(n_seq_max, batch.token ? 0 : 1);
|
||||
if (mixed) {
|
||||
std::vector<bool> seen(n_seq_max, false);
|
||||
for (int32_t i = 0; i < batch.n_tokens; ++i) {
|
||||
for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) {
|
||||
const llama_seq_id sid = batch.seq_id[i][s];
|
||||
if (!seen[sid]) {
|
||||
seen[sid] = true;
|
||||
seq_first_embd[sid] = is_embd_vec[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// M-RoPE case: allow position to "jump" forward only (non-continuous positions are allowed)
|
||||
for (uint32_t s = 0; s < n_seq_max; ++s) {
|
||||
if (seq_pos[s].empty()) {
|
||||
@@ -302,7 +352,7 @@ bool llama_batch_allocr::init(
|
||||
|
||||
const llama_pos p0 = mem ? mem->seq_pos_max(s) : -1;
|
||||
|
||||
if (batch.token) {
|
||||
if (!seq_first_embd[s]) {
|
||||
if (p0 >= 0 && p0 >= seq_pos_min(s)) {
|
||||
LLAMA_LOG_ERROR(
|
||||
"%s: the tokens of sequence %d in the input batch have inconsistent sequence positions:\n"
|
||||
@@ -471,6 +521,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
|
||||
/*.seq_id_unq =*/ udata->seq_id_unq.data(),
|
||||
/*.seq_idx =*/ udata->seq_idx.data(),
|
||||
/*.output =*/ udata->output.data(),
|
||||
/*.type =*/ nullptr,
|
||||
/*.decision_order =*/ nullptr,
|
||||
/*.data =*/ std::move(udata),
|
||||
};
|
||||
@@ -769,6 +820,7 @@ void llama_batch_allocr::clear() {
|
||||
|
||||
token_vec .clear();
|
||||
embd_vec .clear();
|
||||
is_embd_vec .clear();
|
||||
seq_id_data .clear();
|
||||
pos .clear();
|
||||
n_seq_id .clear();
|
||||
@@ -799,7 +851,20 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
|
||||
auto udata = std::make_shared<llama_ubatch::data_t>();
|
||||
|
||||
const int64_t n_embd_all = batch.embd ? (int64_t) n_tokens*n_embd : 0;
|
||||
const bool mixed_batch = !is_embd_vec.empty();
|
||||
|
||||
// a ubatch with a single kind of rows is emitted as a plain token or embd ubatch
|
||||
uint32_t n_embd_rows = 0;
|
||||
if (mixed_batch) {
|
||||
for (int32_t idx : idxs) {
|
||||
n_embd_rows += is_embd_vec[idx];
|
||||
}
|
||||
}
|
||||
const bool mixed = mixed_batch && n_embd_rows > 0 && n_embd_rows < n_tokens;
|
||||
const bool use_token = batch.token && !(mixed_batch && n_embd_rows == n_tokens);
|
||||
const bool use_embd = batch.embd && !(mixed_batch && n_embd_rows == 0);
|
||||
|
||||
const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0;
|
||||
const int64_t n_pos_all = (int64_t) n_tokens*n_pos_per_embd;
|
||||
|
||||
udata->token .resize(n_tokens);
|
||||
@@ -810,6 +875,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
udata->seq_id_unq.resize(0);
|
||||
udata->seq_idx .resize(LLAMA_MAX_SEQ, -1);
|
||||
udata->output .resize(n_tokens);
|
||||
udata->type .resize(mixed ? n_tokens : 0);
|
||||
udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
|
||||
|
||||
udata->batch_idxs = idxs;
|
||||
@@ -818,21 +884,20 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
seq_set_t seq_set_unq;
|
||||
|
||||
for (size_t i = 0; i < idxs.size(); ++i) {
|
||||
if (batch.token) {
|
||||
if (use_token) {
|
||||
udata->token[i] = batch.token[idxs[i]];
|
||||
}
|
||||
|
||||
if (batch.embd) {
|
||||
if (use_embd) {
|
||||
memcpy(udata->embd.data() + i*n_embd, batch.embd + (int64_t) idxs[i]*n_embd, n_embd*sizeof(float));
|
||||
}
|
||||
|
||||
if (mixed) {
|
||||
udata->type[i] = is_embd_vec[idxs[i]];
|
||||
}
|
||||
|
||||
for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {
|
||||
// if we are using M-RoPE
|
||||
// if the current batch is text, we need to broadcast the same position across all RoPE sections
|
||||
// otherwise, the input batch is image embeddings, we copy the positions as-is
|
||||
// if we are not using M-RoPE, there is only one position per token (this loop runs only once)
|
||||
size_t src_off = batch.token ? 0 : j*batch.n_tokens;
|
||||
udata->pos[j*n_tokens + i] = batch.pos[src_off + idxs[i]];
|
||||
udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]];
|
||||
}
|
||||
|
||||
udata->n_seq_id[i] = batch.n_seq_id[idxs[i]];
|
||||
@@ -875,14 +940,15 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
/*.n_seqs_unq =*/ (uint32_t) udata->seq_id_unq.size(),
|
||||
/*.n_pos =*/ n_pos_per_embd,
|
||||
|
||||
/*.token =*/ batch.token ? udata->token.data() : nullptr,
|
||||
/*.embd =*/ batch.embd ? udata->embd.data() : nullptr,
|
||||
/*.token =*/ use_token ? udata->token.data() : nullptr,
|
||||
/*.embd =*/ use_embd ? udata->embd.data() : nullptr,
|
||||
/*.pos =*/ udata->pos.data(),
|
||||
/*.n_seq_id =*/ udata->n_seq_id.data(),
|
||||
/*.seq_id =*/ udata->seq_id.data(),
|
||||
/*.seq_id_unq =*/ udata->seq_id_unq.data(),
|
||||
/*.seq_idx =*/ udata->seq_idx.data(),
|
||||
/*.output =*/ udata->output.data(),
|
||||
/*.type =*/ mixed ? udata->type.data() : nullptr,
|
||||
/*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
|
||||
/*.data =*/ std::move(udata),
|
||||
};
|
||||
@@ -933,6 +999,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
|
||||
LLAMA_LOG_DEBUG("%s: seq_id_unq = %s\n", __func__, ss_seq_id_unq.str().c_str());
|
||||
LLAMA_LOG_DEBUG("%s: seq_idx = %s\n", __func__, ss_seq_idx.str().c_str());
|
||||
LLAMA_LOG_DEBUG("%s: output = %p\n", __func__, (void *) ubatch.output);
|
||||
LLAMA_LOG_DEBUG("%s: type = %p\n", __func__, (void *) ubatch.type);
|
||||
LLAMA_LOG_DEBUG("%s: n_outputs = %d\n", __func__, n_outputs);
|
||||
|
||||
if (debug > 0) {
|
||||
@@ -963,7 +1030,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
|
||||
}
|
||||
}
|
||||
|
||||
if (ubatch.token) {
|
||||
if (ubatch.token && !(ubatch.is_mixed() && ubatch.type[i])) {
|
||||
LLAMA_LOG_DEBUG("%s: %4d: id = %6d (%16s), pos = %4d, n_seq_id = %2d, seq_id = [%s], output = %d\n",
|
||||
__func__, i, ubatch.token[i], vocab->token_to_piece(ubatch.token[i]).c_str(),
|
||||
ubatch.pos[i], ubatch.n_seq_id[i], ss.str().c_str(), ubatch.output[i]);
|
||||
|
||||
Reference in new issue
Block a user