diff --git a/examples/batched.swift/Sources/main.swift b/examples/batched.swift/Sources/main.swift index fd90bbec5..d8761f265 100644 --- a/examples/batched.swift/Sources/main.swift +++ b/examples/batched.swift/Sources/main.swift @@ -87,36 +87,32 @@ for id: llama_token in tokens { print("\n") -var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1) +let batch = llama_batch_ext_init(context) defer { - llama_batch_free(batch) + llama_batch_ext_free(batch) } // evaluate the initial prompt -batch.n_tokens = Int32(tokens.count) - for (i, token) in tokens.enumerated() { - batch.token[i] = token - batch.pos[i] = Int32(i) - batch.n_seq_id[i] = 1 - // batch.seq_id[i][0] = 0 - // TODO: is this the proper way to do this? - if let seq_id = batch.seq_id[i] { - seq_id[0] = 0 + let idx = llama_batch_ext_add_token(batch, 0, token) + if idx < 0 { + print("llama_batch_ext_add_token() failed: \(idx)") + exit(1) } - batch.logits[i] = 0 + var pos = llama_pos(i) + llama_batch_ext_set_pos(batch, idx, &pos) } -// llama_decode will output logits only for the last token of the prompt -batch.logits[Int(batch.n_tokens) - 1] = 1 +// llama_process will output logits only for the last token of the prompt +llama_batch_ext_set_output_logits(batch, Int32(tokens.count) - 1, true) -if llama_decode(context, batch) != 0 { - print("llama_decode() failed") +if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 { + print("llama_process() failed") exit(1) } for i in 1 ..< n_parallel { - llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, batch.n_tokens) + llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, Int32(tokens.count)) } if n_parallel > 1 { @@ -125,16 +121,17 @@ if n_parallel > 1 { var streams: [String] = .init(repeating: "", count: n_parallel) var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel) -var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel) +var i_batch = [Int32](repeating: Int32(tokens.count) - 1, count: n_parallel) -var n_cur = batch.n_tokens +var n_cur = Int32(tokens.count) var n_decode = 0 let t_main_start = ggml_time_us() while n_cur <= n_len { // prepare the next batch - batch.n_tokens = 0 + llama_batch_ext_clear(batch) + var n_tokens: Int32 = 0 // sample the next token for each parallel sequence / stream for i in 0 ..< n_parallel { @@ -165,31 +162,32 @@ while n_cur <= n_len { streams[i] += nextStringPiece // push this new token for next evaluation - batch.token[Int(batch.n_tokens)] = new_token_id - batch.pos[Int(batch.n_tokens)] = n_cur - batch.n_seq_id[Int(batch.n_tokens)] = 1 - if let seq_id = batch.seq_id[Int(batch.n_tokens)] { - seq_id[0] = Int32(i) + let idx = llama_batch_ext_add_token(batch, Int32(i), new_token_id) + if idx < 0 { + print("llama_batch_ext_add_token() failed: \(idx)") + exit(1) } - batch.logits[Int(batch.n_tokens)] = 1 + var pos = n_cur + llama_batch_ext_set_pos(batch, idx, &pos) + llama_batch_ext_set_output_logits(batch, idx, true) - i_batch[i] = batch.n_tokens + i_batch[i] = idx - batch.n_tokens += 1 + n_tokens += 1 n_decode += 1 } // all streams are finished - if batch.n_tokens == 0 { + if n_tokens == 0 { break } n_cur += 1 // evaluate the current batch with the transformer model - if llama_decode(context, batch) != 0 { - print("llama_decode() failed") + if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 { + print("llama_process() failed") exit(1) } } diff --git a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift index dc2bafc88..e00938300 100644 --- a/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift +++ b/examples/llama.swiftui/llama.cpp.swift/LibLlama.swift @@ -5,20 +5,16 @@ enum LlamaError: Error { case couldNotInitializeContext } -func llama_batch_clear(_ batch: inout llama_batch) { - batch.n_tokens = 0 -} - -func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama_pos, _ seq_ids: [llama_seq_id], _ logits: Bool) { - batch.token [Int(batch.n_tokens)] = id - batch.pos [Int(batch.n_tokens)] = pos - batch.n_seq_id[Int(batch.n_tokens)] = Int32(seq_ids.count) - for i in 0.. Int32 { + let idx = llama_batch_ext_add_token(batch, seq_id, id) + if idx < 0 { + return idx } - batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0 - - batch.n_tokens += 1 + var pos = pos + llama_batch_ext_set_pos(batch, idx, &pos) + llama_batch_ext_set_output_logits(batch, idx, logits) + return idx } actor LlamaContext { @@ -26,7 +22,7 @@ actor LlamaContext { private var context: OpaquePointer private var vocab: OpaquePointer private var sampling: UnsafeMutablePointer - private var batch: llama_batch + private var batch: OpaquePointer private var tokens_list: [llama_token] var is_done: Bool = false @@ -42,7 +38,7 @@ actor LlamaContext { self.model = model self.context = context self.tokens_list = [] - self.batch = llama_batch_init(512, 0, 1) + self.batch = llama_batch_ext_init(context) self.temporary_invalid_cchars = [] let sparams = llama_sampler_chain_default_params() self.sampling = llama_sampler_chain_init(sparams) @@ -53,7 +49,7 @@ actor LlamaContext { deinit { llama_sampler_free(sampling) - llama_batch_free(batch) + llama_batch_ext_free(batch) llama_model_free(model) llama_free(context) llama_backend_free() @@ -110,10 +106,6 @@ actor LlamaContext { return SwiftString } - func get_n_tokens() -> Int32 { - return batch.n_tokens; - } - func completion_init(text: String) { print("attempting to complete \"\(text)\"") @@ -133,25 +125,27 @@ actor LlamaContext { print(String(cString: token_to_piece(token: id) + [0])) } - llama_batch_clear(&batch) + llama_batch_ext_clear(batch) for i1 in 0.. String { var new_token_id: llama_token = 0 - new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1) + new_token_id = llama_sampler_sample(sampling, context, -1) if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len { print("\n") @@ -178,13 +172,13 @@ actor LlamaContext { print(new_token_str) // tokens_list.append(new_token_id) - llama_batch_clear(&batch) - llama_batch_add(&batch, new_token_id, n_cur, [0], true) + llama_batch_ext_clear(batch) + llama_batch_add(batch, new_token_id, n_cur, 0, true) n_decode += 1 n_cur += 1 - if llama_decode(context, batch) != 0 { + if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 { print("failed to evaluate llama!") } @@ -201,21 +195,20 @@ actor LlamaContext { for _ in 0.. & tokens, const std::vector & labels_sparse, - llama_batch & batch, + llama_batch_ext & batch, ggml_opt_epoch_callback callback, bool train, int64_t idata_in_loop, @@ -4268,22 +4268,21 @@ void llama_context::opt_epoch_iter( memory->clear(true); for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) { - batch.n_tokens = n_batch; + batch.clear(); for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) { - batch.token [pos_batch] = tokens[pos_ctx + pos_batch]; - batch.pos [pos_batch] = pos_ctx + pos_batch; - batch.n_seq_id[pos_batch] = 1; - batch.seq_id [pos_batch][0] = 0; - batch.logits [pos_batch] = true; - } - - // TODO: use llama_batch_ext here - { - llama_batch_compat compat(this, batch); - if (!balloc->init(*compat.batch_ext, model.vocab, true)) { + const int32_t idx = batch.add_token(0); + const llama_pos pos = pos_ctx + pos_batch; + if (idx < 0 || !batch.set_token_id(idx, tokens[pos_ctx + pos_batch])) { LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__); return; } + batch.set_token_pos(idx, &pos); + batch.set_output(idx, true); + } + + if (!balloc->init(batch, model.vocab, true)) { + LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__); + return; } const uint32_t n_tokens_all = balloc->get_n_tokens(); @@ -4382,7 +4381,7 @@ void llama_context::opt_epoch( const uint32_t ubatch_per_ctx = n_ctx / n_ubatch; - struct llama_batch batch = llama_batch_init(n_batch, 0, 1); + llama_batch_ext batch(this); std::vector tokens(n_ctx); std::vector labels_sparse(n_ctx); @@ -4409,8 +4408,6 @@ void llama_context::opt_epoch( opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch, callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start); } - - llama_batch_free(batch); } // diff --git a/src/llama-context.h b/src/llama-context.h index 45eca4566..3e3921c7d 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -233,7 +233,7 @@ struct llama_context { ggml_opt_result_t result, const std::vector & tokens, const std::vector & labels_sparse, - llama_batch & batch, + llama_batch_ext & batch, ggml_opt_epoch_callback callback, bool train, int64_t idata_in_loop, diff --git a/tests/llama-context/cases-kv.cpp b/tests/llama-context/cases-kv.cpp index b970ea940..4d2970e9d 100644 --- a/tests/llama-context/cases-kv.cpp +++ b/tests/llama-context/cases-kv.cpp @@ -557,7 +557,8 @@ struct case_state_rotation : ctx_test { } llama_token token = 0; - if (llama_decode(src.get(), llama_batch_get_one(&token, 1))) { + common_batch batch = common_batch_get_one(src.get(), &token, 1); + if (llama_process(src.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) { TLOG_ERR("%s: failed to decode a token\n", __func__); success = false; break; diff --git a/tests/test-batch-alloc.cpp b/tests/test-batch-alloc.cpp index c41e91d61..42c4c5249 100644 --- a/tests/test-batch-alloc.cpp +++ b/tests/test-batch-alloc.cpp @@ -978,7 +978,9 @@ static void test_compat(testing & t) { mem.ranges[0] = {0, 9}; llama_token token[3] = { 5, 6, 7 }; - llama_batch lb = llama_batch_get_one(token, 3); + llama_batch lb = {}; + lb.n_tokens = 3; + lb.token = token; batch_builder bb(2, &mem, 4, 1, /*n_vocab*/ 100); llama_batch_compat::init(bb.b, lb); @@ -999,7 +1001,9 @@ static void test_compat(testing & t) { t.test("auto_pos_starts_at_zero_without_memory", [&](testing & t) { llama_token token[2] = { 5, 6 }; - llama_batch lb = llama_batch_get_one(token, 2); + llama_batch lb = {}; + lb.n_tokens = 2; + lb.token = token; batch_builder bb(2, nullptr, 4, 1, /*n_vocab*/ 100); llama_batch_compat::init(bb.b, lb); diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 8a5757c1e..2148ec526 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -52,6 +52,20 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri return LLAMA_TOKEN_NULL; } +static bool batch_set_tokens(llama_batch_ext * batch, const llama_token * tokens, int32_t n, llama_pos pos_0, llama_seq_id seq_id) { + llama_batch_ext_clear(batch); + for (int32_t i = 0; i < n; i++) { + const int32_t idx = llama_batch_ext_add_token(batch, seq_id, tokens[i]); + const llama_pos pos = pos_0 + i; + if (idx < 0) { + LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tokens[i], idx); + return false; + } + llama_batch_ext_set_pos(batch, idx, &pos); + } + return true; +} + static void put_bytes(std::vector & buf, const void * p, size_t n) { const char * c = (const char *) p; buf.insert(buf.end(), c, c + n); @@ -1206,18 +1220,12 @@ private: } bool decode(llama_token * tokens, int n) { - llama_batch batch = llama_batch_init(n, 0, 1); - batch.n_tokens = n; - for (int i = 0; i < n; ++i) { - batch.token[i] = tokens[i]; - batch.pos[i] = pos + i; - batch.n_seq_id[i] = 1; - batch.seq_id[i][0] = seq_id; - batch.logits[i] = i == n - 1; + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), tokens, n, pos, seq_id)) { + return false; } - const int ret = llama_decode(lctx, batch); - llama_batch_free(batch); - if (ret != 0) { + llama_batch_ext_set_output_logits(batch.get(), n - 1, true); + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n"); return false; } @@ -1330,18 +1338,14 @@ public: return 0; } const int n = std::min(n_batch, n_prompt - prompt_pos); - llama_batch batch = llama_batch_init(n, 0, 1); - for (int i = 0; i < n; i++) { - batch.token [i] = prompt_tokens[(size_t) (prompt_pos + i)]; - batch.pos [i] = pos + i; - batch.n_seq_id[i] = 1; - batch.seq_id [i][0] = seq_id; - batch.logits [i] = prompt_pos + i == n_prompt - 1; + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) { + return -1; } - batch.n_tokens = n; - const int ret = llama_decode(lctx, batch); - llama_batch_free(batch); - if (ret != 0) { + if (prompt_pos + n == n_prompt) { + llama_batch_ext_set_output_logits(batch.get(), n - 1, true); + } + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } @@ -1382,8 +1386,7 @@ public: text_done = true; speaking = true; } - llama_batch batch = llama_batch_get_one(&sampled, 1); - if (!decode(batch)) { + if (!decode(sampled)) { return 1; } } else { @@ -1626,25 +1629,17 @@ private: return true; } - bool decode(llama_batch batch) { - std::vector positions(batch.n_tokens); - std::vector n_seq(batch.n_tokens, 1); - std::vector seq(batch.n_tokens, seq_id); - std::vector seq_ptr(batch.n_tokens); - std::vector logits(batch.n_tokens, 1); - for (int i = 0; i < batch.n_tokens; i++) { - positions[i] = pos + i; - seq_ptr[i] = &seq[i]; + bool decode(llama_token token) { + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), &token, 1, pos, seq_id)) { + return false; } - batch.pos = positions.data(); - batch.n_seq_id = n_seq.data(); - batch.seq_id = seq_ptr.data(); - batch.logits = logits.data(); - if (llama_decode(lctx, batch) != 0) { + llama_batch_ext_set_output_logits(batch.get(), 0, true); + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return false; } - pos += batch.n_tokens; + pos += 1; return true; } @@ -2160,18 +2155,14 @@ public: } // small batches: every row also outputs logits const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 }); - llama_batch batch = llama_batch_init(n, 0, 1); - for (int i = 0; i < n; i++) { - batch.token [i] = prompt_tokens[(size_t) (prompt_pos + i)]; - batch.pos [i] = pos + i; - batch.n_seq_id[i] = 1; - batch.seq_id [i][0] = seq_id; - batch.logits [i] = 1; + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) { + return -1; } - batch.n_tokens = n; - const int ret = llama_decode(lctx, batch); - llama_batch_free(batch); - if (ret != 0) { + for (int i = 0; i < n; i++) { + llama_batch_ext_set_output_logits(batch.get(), i, true); + } + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } @@ -2546,17 +2537,14 @@ public: int ret; if (!seg.tokens.empty()) { - llama_batch batch = llama_batch_init(n, 0, 1); - for (int i = 0; i < n; i++) { - batch.token [i] = seg.tokens[(size_t) (seg_pos + i)]; - batch.pos [i] = pos + i; - batch.n_seq_id[i] = 1; - batch.seq_id [i][0] = seq_id; - batch.logits [i] = last && i == n - 1; + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), seg.tokens.data() + seg_pos, n, pos, seq_id)) { + return -1; } - batch.n_tokens = n; - ret = llama_decode(lctx, batch); - llama_batch_free(batch); + if (last) { + llama_batch_ext_set_output_logits(batch.get(), n - 1, true); + } + ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); } else { decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd); batch.set_position_normal(pos, seq_id); diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp index e747d32d7..201740b2e 100644 --- a/tools/server/server-realtime.cpp +++ b/tools/server/server-realtime.cpp @@ -1032,24 +1032,20 @@ bool server_realtime_slot::decode(const llama_token * tokens, const float * rows if (n_past + n >= n_ctx) { return fail("the conversation does not fit in the context", ERROR_TYPE_EXCEED_CONTEXT_SIZE); } - for (int32_t i0 = 0; i0 < n; i0 += n_batch) { - const int32_t n_cur = std::min(n_batch, n - i0); - llama_batch b = llama_batch_init(n_cur, rows ? n_embd : 0, 1); - for (int32_t i = 0; i < n_cur; i++) { - if (rows) { - memcpy(b.embd + (size_t) i * n_embd, rows + (size_t) (i0 + i) * n_embd, n_embd * sizeof(float)); - } else { - b.token[i] = tokens[i0 + i]; - } - b.pos[i] = n_past + i0 + i; - b.n_seq_id[i] = 1; - b.seq_id[i][0] = seq_id; - b.logits[i] = logits_last && i0 + i == n - 1; + common_batch batch(ctx); + for (int32_t i = 0; i < n; i++) { + const llama_pos pos = n_past + i; + const bool output = logits_last && i == n - 1; + if (rows) { + const llama_embd e = { rows + (size_t) i * n_embd, 1, (size_t) n_embd }; + batch.add_embd(e, &pos, seq_id, output); + } else { + batch.add(tokens[i], pos, seq_id, output); } - b.n_tokens = n_cur; - const int ret = llama_decode(ctx, b); - llama_batch_free(b); - if (ret != 0) { + } + for (int32_t i = 0; i < n; i += n_batch) { + const int32_t n_cur = std::min(n_batch, n - i); + if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get_sub_batch(i, n_cur)) != 0) { return fail("failed to decode the conversation"); } }