mtmd: add mtmd_audio_stream + support voxtral-realtime (#30629)

* mtmd: add mtmd_audio_stream + support voxtral-realtime

* address review
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-16 01:46:42 +02:00
1 parent 59d5fad7c1
commit 2851b3bcbf
27 files changed
+1000 -22

No files matched your search

+47 -18
View File
@@ -3906,6 +3906,32 @@ uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) {
return (uint32_t) model->target_layer_ids.size();
}
// read rows [row0, row0 + n_rows) of a 2D tensor as F32
static void llama_tensor_rows_to_f32(const ggml_tensor * tensor, int64_t row0, int64_t n_rows, float * out) {
const size_t nelements = (size_t) n_rows * tensor->ne[0];
const size_t offset = (size_t) row0 * tensor->nb[1];
const size_t nbytes = ggml_row_size(tensor->type, tensor->ne[0]) * n_rows;
if (tensor->type == GGML_TYPE_F32) {
ggml_backend_tensor_get(tensor, out, offset, nbytes);
return;
}
std::vector<uint8_t> buf(nbytes);
ggml_backend_tensor_get(tensor, buf.data(), offset, nbytes);
const ggml_type_traits * traits = ggml_get_type_traits(tensor->type);
if (tensor->type == GGML_TYPE_F16) {
ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements);
} else if (tensor->type == GGML_TYPE_BF16) {
ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements);
} else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) {
traits->to_float(buf.data(), out, nelements);
} else {
GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type));
}
}
uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) {
if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) {
return 0;
@@ -3919,24 +3945,27 @@ uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out)
return (uint32_t) nelements;
}
if (tensor->type == GGML_TYPE_F32) {
ggml_backend_tensor_get(tensor, out, 0, nelements * sizeof(float));
return (uint32_t) nelements;
}
std::vector<uint8_t> buf(ggml_nbytes(tensor));
ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size());
const ggml_type_traits * traits = ggml_get_type_traits(tensor->type);
if (tensor->type == GGML_TYPE_F16) {
ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements);
} else if (tensor->type == GGML_TYPE_BF16) {
ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements);
} else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) {
traits->to_float(buf.data(), out, nelements);
} else {
GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type));
}
llama_tensor_rows_to_f32(tensor, 0, tensor->ne[1], out);
return (uint32_t) nelements;
}
uint32_t llama_model_n_tokens_delay(const struct llama_model * model) {
return model->hparams.n_tokens_delay;
}
uint32_t llama_model_get_tok_embd_rows(const struct llama_model * model, const llama_token * tokens, uint32_t n_tokens, float * out) {
if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr || tokens == nullptr || out == nullptr) {
return 0;
}
const ggml_tensor * tensor = model->tok_embd;
for (uint32_t i = 0; i < n_tokens; i++) {
if (tokens[i] < 0 || tokens[i] >= tensor->ne[1]) {
return 0;
}
llama_tensor_rows_to_f32(tensor, tokens[i], 1, out + (size_t) i * tensor->ne[0]);
}
return n_tokens * (uint32_t) tensor->ne[0];
}