dflash: copy hidden features via device copies (#31506)
This commit is contained in:
1 parent
29c720eaf7
commit
2cbca389f5
14 files changed
+441
-25
No files matched your search
+63
-1
@@ -52,6 +52,7 @@ bool llama_batch_allocr::init(
|
||||
// an entry can carry a token id, a token embedding, or both
|
||||
// all entries must carry the same combination, or be a mix of token and embd entries
|
||||
// a state embedding (e.g. MTP hook batches) is set on all entries or on none
|
||||
// an embd row (embd read from a device buffer) is set on all entries or on none, with no token id or embedding
|
||||
//
|
||||
|
||||
int32_t n_tok_only = 0;
|
||||
@@ -59,12 +60,24 @@ bool llama_batch_allocr::init(
|
||||
int32_t n_both = 0;
|
||||
|
||||
const bool has_state = batch_inp.tokens[0].has_state;
|
||||
const bool has_row = batch_inp.tokens[0].embd_row >= 0;
|
||||
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
const bool is_tok = batch_inp.tokens[i].id != LLAMA_TOKEN_NULL;
|
||||
const bool is_emb = batch_inp.tokens[i].has_embd;
|
||||
const bool is_row = batch_inp.tokens[i].embd_row >= 0;
|
||||
|
||||
if (!is_tok && !is_emb) {
|
||||
if (is_row != has_row) {
|
||||
LLAMA_LOG_ERROR("%s: all entries in the batch must have the same embd row presence\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (is_row) {
|
||||
if (is_tok || is_emb) {
|
||||
LLAMA_LOG_ERROR("%s: entry %d has an embd row and a token id or embedding\n", __func__, i);
|
||||
return false;
|
||||
}
|
||||
} else if (!is_tok && !is_emb) {
|
||||
LLAMA_LOG_ERROR("%s: entry %d has neither a token id nor an embedding\n", __func__, i);
|
||||
return false;
|
||||
}
|
||||
@@ -137,6 +150,13 @@ bool llama_batch_allocr::init(
|
||||
state_vec = batch_inp.state;
|
||||
}
|
||||
|
||||
if (has_row) {
|
||||
embd_row_vec.resize(n_tok);
|
||||
for (int32_t i = 0; i < n_tok; ++i) {
|
||||
embd_row_vec[i] = batch_inp.tokens[i].embd_row;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// build flat pos array, section-major: pos[j*n_tok + i] = section j of entry i
|
||||
// token entry: [p, p, p, 0] (M-RoPE text position)
|
||||
@@ -306,6 +326,7 @@ bool llama_batch_allocr::init(
|
||||
/*.token =*/ batch.token,
|
||||
/*.embd =*/ batch.embd,
|
||||
/*.embd_state =*/ state_vec.empty() ? nullptr : state_vec.data(),
|
||||
/*.embd_row =*/ embd_row_vec.empty() ? nullptr : embd_row_vec.data(),
|
||||
/*.pos =*/ batch.pos,
|
||||
/*.n_seq_id =*/ batch.n_seq_id,
|
||||
/*.seq_id =*/ batch.seq_id,
|
||||
@@ -531,6 +552,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
|
||||
/*.token =*/ udata->token.data(),
|
||||
/*.embd =*/ nullptr,
|
||||
/*.embd_state =*/ nullptr,
|
||||
/*.embd_row =*/ nullptr,
|
||||
/*.pos =*/ udata->pos.data(),
|
||||
/*.n_seq_id =*/ udata->n_seq_id.data(),
|
||||
/*.seq_id =*/ udata->seq_id.data(),
|
||||
@@ -838,6 +860,7 @@ void llama_batch_allocr::clear() {
|
||||
embd_vec .clear();
|
||||
is_embd_vec .clear();
|
||||
state_vec .clear();
|
||||
embd_row_vec.clear();
|
||||
seq_id_data .clear();
|
||||
pos .clear();
|
||||
n_seq_id .clear();
|
||||
@@ -881,6 +904,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
const bool use_token = batch.token && !(mixed_batch && n_embd_rows == n_tokens);
|
||||
const bool use_embd = batch.embd && !(mixed_batch && n_embd_rows == 0);
|
||||
const bool has_state = !state_vec.empty();
|
||||
const bool has_row = !embd_row_vec.empty();
|
||||
|
||||
const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0;
|
||||
const int64_t n_state_all = has_state ? (int64_t) n_tokens*n_embd_state : 0;
|
||||
@@ -889,6 +913,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
udata->token .resize(n_tokens);
|
||||
udata->embd .resize(n_embd_all);
|
||||
udata->embd_state.resize(n_state_all);
|
||||
udata->embd_row .resize(has_row ? n_tokens : 0);
|
||||
udata->pos .resize(n_pos_all);
|
||||
udata->n_seq_id .resize(n_tokens);
|
||||
udata->seq_id .resize(n_tokens);
|
||||
@@ -920,6 +945,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
memcpy(udata->embd_state.data() + i*n_embd_state, state_vec.data() + (int64_t) idxs[i]*n_embd_state, n_embd_state*sizeof(float));
|
||||
}
|
||||
|
||||
if (has_row) {
|
||||
udata->embd_row[i] = embd_row_vec[idxs[i]];
|
||||
}
|
||||
|
||||
for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {
|
||||
udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]];
|
||||
}
|
||||
@@ -967,6 +996,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
/*.token =*/ use_token ? udata->token.data() : nullptr,
|
||||
/*.embd =*/ use_embd ? udata->embd.data() : nullptr,
|
||||
/*.embd_state =*/ has_state ? udata->embd_state.data() : nullptr,
|
||||
/*.embd_row =*/ has_row ? udata->embd_row.data() : nullptr,
|
||||
/*.pos =*/ udata->pos.data(),
|
||||
/*.n_seq_id =*/ udata->n_seq_id.data(),
|
||||
/*.seq_id =*/ udata->seq_id.data(),
|
||||
@@ -1019,6 +1049,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
|
||||
LLAMA_LOG_DEBUG("%s: token = %p\n", __func__, (void *) ubatch.token);
|
||||
LLAMA_LOG_DEBUG("%s: embd = %p\n", __func__, (void *) ubatch.embd);
|
||||
LLAMA_LOG_DEBUG("%s: embd_state = %p\n", __func__, (void *) ubatch.embd_state);
|
||||
LLAMA_LOG_DEBUG("%s: embd_row = %p\n", __func__, (void *) ubatch.embd_row);
|
||||
LLAMA_LOG_DEBUG("%s: pos = %p\n", __func__, (void *) ubatch.pos);
|
||||
LLAMA_LOG_DEBUG("%s: n_seq_id = %p\n", __func__, (void *) ubatch.n_seq_id);
|
||||
LLAMA_LOG_DEBUG("%s: seq_id = %p\n", __func__, (void *) ubatch.seq_id);
|
||||
@@ -1303,6 +1334,26 @@ bool llama_batch_ext::set_token_state(int32_t idx, llama_embd state_in) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool llama_batch_ext::set_token_embd_row(int32_t idx, int32_t row) {
|
||||
if (idx < 0 || idx >= (int32_t) tokens.size()) {
|
||||
return false;
|
||||
}
|
||||
if (row < 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
token & t = tokens[idx];
|
||||
|
||||
if (t.id != LLAMA_TOKEN_NULL || t.has_embd || t.embd_row >= 0) {
|
||||
LLAMA_LOG_ERROR("%s: entry %d already has a token id, an embedding or an embd row\n", __func__, idx);
|
||||
return false;
|
||||
}
|
||||
|
||||
t.embd_row = row;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool llama_batch_ext::set_token_pos(int32_t idx, const llama_pos * pos_in) {
|
||||
if (idx < 0 || idx >= (int32_t) tokens.size()) {
|
||||
return false;
|
||||
@@ -1377,6 +1428,17 @@ int32_t llama_batch_ext_add_embd(llama_batch_ext * batch, llama_seq_id seq_id, l
|
||||
return idx;
|
||||
}
|
||||
|
||||
int32_t llama_batch_ext_add_embd_row(llama_batch_ext * batch, llama_seq_id seq_id, int32_t row) {
|
||||
int32_t idx = batch->add_token(seq_id);
|
||||
if (idx < 0) {
|
||||
return idx;
|
||||
}
|
||||
if (!batch->set_token_embd_row(idx, row)) {
|
||||
return -2;
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
|
||||
bool llama_batch_ext_add_seq(llama_batch_ext * batch, int32_t idx, llama_seq_id seq_id) {
|
||||
return batch->add_seq(idx, seq_id);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user