mtmd: fix silent prompt truncation on embedded NUL (#25548)
* mtmd: fix silent prompt truncation on embedded NUL mtmd_input_text carried the prompt as a bare const char* with no length, so a NUL byte in message content cut the prompt at the tokenizer boundary and dropped every later message plus the assistant marker, with no log. Add an explicit text_len and thread it through, matching llama_tokenize and the text only path. * cleanup --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
co-authored by
Xuan Son Nguyen
parent
0c4fa7a989
commit
4114ba18b2
+3
-2
@@ -809,7 +809,7 @@ void mtmd_free(mtmd_context * ctx) {
|
||||
struct mtmd_tokenizer {
|
||||
mtmd_context * ctx;
|
||||
|
||||
std::string input_text;
|
||||
std::string input_text; // note: can contain null bytes; do not use c_str()
|
||||
bool add_special;
|
||||
bool parse_special;
|
||||
const llama_vocab * vocab;
|
||||
@@ -839,9 +839,10 @@ struct mtmd_tokenizer {
|
||||
size_t n_bitmaps) : ctx(ctx) {
|
||||
add_special = text->add_special;
|
||||
parse_special = text->parse_special;
|
||||
input_text = text->text;
|
||||
vocab = ctx->vocab;
|
||||
|
||||
input_text.assign(text->text, text->text_len);
|
||||
|
||||
std::vector<const mtmd_bitmap *> bitmaps(bmps, bmps + n_bitmaps);
|
||||
auto parts_str = split_text(input_text, ctx->media_marker);
|
||||
size_t i_bm = 0;
|
||||
|
||||
Reference in New Issue
Block a user