batch: migrate the rest of examples to llama_batch_ext (#29601)
* migrate the rest * test-thread-safety * rm common_batch_staged
This commit is contained in:
1 parent
05af0d2b13
commit
60e9cf7a7b
34 files changed
+513
-631
No files matched your search
@@ -5,6 +5,17 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// fill the batch with tokens at consecutive positions starting from pos_0, output logits only for the last one
|
||||
static void batch_set_tokens(llama_batch_ext * batch, const llama_token * tokens, int32_t n_tokens, llama_pos pos_0) {
|
||||
llama_batch_ext_clear(batch);
|
||||
for (int32_t i = 0; i < n_tokens; ++i) {
|
||||
const int32_t idx = llama_batch_ext_add_token(batch, 0, tokens[i]);
|
||||
const llama_pos pos = pos_0 + i;
|
||||
llama_batch_ext_set_pos(batch, idx, &pos);
|
||||
}
|
||||
llama_batch_ext_set_output_logits(batch, n_tokens - 1, true);
|
||||
}
|
||||
|
||||
static void print_usage(int, char ** argv) {
|
||||
printf("\nexample usage:\n");
|
||||
printf("\n %s -m model.gguf [-n n_predict] [-ngl n_gpu_layers] [prompt]\n", argv[0]);
|
||||
@@ -144,10 +155,13 @@ int main(int argc, char ** argv) {
|
||||
|
||||
// prepare a batch for the prompt
|
||||
|
||||
llama_batch batch = llama_batch_get_one(prompt_tokens.data(), prompt_tokens.size());
|
||||
llama_batch_ext * batch = llama_batch_ext_init(ctx);
|
||||
int n_tokens = n_prompt; // number of tokens in the current batch
|
||||
|
||||
batch_set_tokens(batch, prompt_tokens.data(), n_prompt, 0);
|
||||
|
||||
if (llama_model_has_encoder(model)) {
|
||||
if (llama_encode(ctx, batch)) {
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_ENCODE, batch)) {
|
||||
fprintf(stderr, "%s : failed to eval\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
@@ -157,7 +171,8 @@ int main(int argc, char ** argv) {
|
||||
decoder_start_token_id = llama_vocab_bos(vocab);
|
||||
}
|
||||
|
||||
batch = llama_batch_get_one(&decoder_start_token_id, 1);
|
||||
batch_set_tokens(batch, &decoder_start_token_id, 1, 0);
|
||||
n_tokens = 1;
|
||||
}
|
||||
|
||||
// main loop
|
||||
@@ -166,14 +181,14 @@ int main(int argc, char ** argv) {
|
||||
int n_decode = 0;
|
||||
llama_token new_token_id;
|
||||
|
||||
for (int n_pos = 0; n_pos + batch.n_tokens < n_prompt + n_predict; ) {
|
||||
for (int n_pos = 0; n_pos + n_tokens < n_prompt + n_predict; ) {
|
||||
// evaluate the current batch with the transformer model
|
||||
if (llama_decode(ctx, batch)) {
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch)) {
|
||||
fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, 1);
|
||||
return 1;
|
||||
}
|
||||
|
||||
n_pos += batch.n_tokens;
|
||||
n_pos += n_tokens;
|
||||
|
||||
// sample the next token
|
||||
{
|
||||
@@ -195,7 +210,8 @@ int main(int argc, char ** argv) {
|
||||
fflush(stdout);
|
||||
|
||||
// prepare the next batch with the sampled token
|
||||
batch = llama_batch_get_one(&new_token_id, 1);
|
||||
batch_set_tokens(batch, &new_token_id, 1, n_pos);
|
||||
n_tokens = 1;
|
||||
|
||||
n_decode += 1;
|
||||
}
|
||||
@@ -213,6 +229,7 @@ int main(int argc, char ** argv) {
|
||||
llama_perf_context_print(ctx);
|
||||
fprintf(stderr, "\n");
|
||||
|
||||
llama_batch_ext_free(batch);
|
||||
llama_sampler_free(smpl);
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
|
||||
Reference in new issue
Block a user