mtmd: add mtmd_audio_stream + support voxtral-realtime (#30629)
* mtmd: add mtmd_audio_stream + support voxtral-realtime * address review
This commit is contained in:
1 parent
59d5fad7c1
commit
2851b3bcbf
27 files changed
+1000
-22
No files matched your search
+47
-18
@@ -3906,6 +3906,32 @@ uint32_t llama_model_target_layer_ids_n(const struct llama_model * model) {
|
||||
return (uint32_t) model->target_layer_ids.size();
|
||||
}
|
||||
|
||||
// read rows [row0, row0 + n_rows) of a 2D tensor as F32
|
||||
static void llama_tensor_rows_to_f32(const ggml_tensor * tensor, int64_t row0, int64_t n_rows, float * out) {
|
||||
const size_t nelements = (size_t) n_rows * tensor->ne[0];
|
||||
const size_t offset = (size_t) row0 * tensor->nb[1];
|
||||
const size_t nbytes = ggml_row_size(tensor->type, tensor->ne[0]) * n_rows;
|
||||
|
||||
if (tensor->type == GGML_TYPE_F32) {
|
||||
ggml_backend_tensor_get(tensor, out, offset, nbytes);
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> buf(nbytes);
|
||||
ggml_backend_tensor_get(tensor, buf.data(), offset, nbytes);
|
||||
|
||||
const ggml_type_traits * traits = ggml_get_type_traits(tensor->type);
|
||||
if (tensor->type == GGML_TYPE_F16) {
|
||||
ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements);
|
||||
} else if (tensor->type == GGML_TYPE_BF16) {
|
||||
ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements);
|
||||
} else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) {
|
||||
traits->to_float(buf.data(), out, nelements);
|
||||
} else {
|
||||
GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type));
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out) {
|
||||
if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr) {
|
||||
return 0;
|
||||
@@ -3919,24 +3945,27 @@ uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out)
|
||||
return (uint32_t) nelements;
|
||||
}
|
||||
|
||||
if (tensor->type == GGML_TYPE_F32) {
|
||||
ggml_backend_tensor_get(tensor, out, 0, nelements * sizeof(float));
|
||||
return (uint32_t) nelements;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> buf(ggml_nbytes(tensor));
|
||||
ggml_backend_tensor_get(tensor, buf.data(), 0, buf.size());
|
||||
|
||||
const ggml_type_traits * traits = ggml_get_type_traits(tensor->type);
|
||||
if (tensor->type == GGML_TYPE_F16) {
|
||||
ggml_fp16_to_fp32_row((const ggml_fp16_t *) buf.data(), out, nelements);
|
||||
} else if (tensor->type == GGML_TYPE_BF16) {
|
||||
ggml_bf16_to_fp32_row((const ggml_bf16_t *) buf.data(), out, nelements);
|
||||
} else if (ggml_is_quantized(tensor->type) && traits->to_float != nullptr) {
|
||||
traits->to_float(buf.data(), out, nelements);
|
||||
} else {
|
||||
GGML_ABORT("unsupported tensor type for dequantization: %s", ggml_type_name(tensor->type));
|
||||
}
|
||||
llama_tensor_rows_to_f32(tensor, 0, tensor->ne[1], out);
|
||||
|
||||
return (uint32_t) nelements;
|
||||
}
|
||||
|
||||
uint32_t llama_model_n_tokens_delay(const struct llama_model * model) {
|
||||
return model->hparams.n_tokens_delay;
|
||||
}
|
||||
|
||||
uint32_t llama_model_get_tok_embd_rows(const struct llama_model * model, const llama_token * tokens, uint32_t n_tokens, float * out) {
|
||||
if (model->vocab.n_tokens() == 0 || model->tok_embd == nullptr || tokens == nullptr || out == nullptr) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const ggml_tensor * tensor = model->tok_embd;
|
||||
for (uint32_t i = 0; i < n_tokens; i++) {
|
||||
if (tokens[i] < 0 || tokens[i] >= tensor->ne[1]) {
|
||||
return 0;
|
||||
}
|
||||
llama_tensor_rows_to_f32(tensor, tokens[i], 1, out + (size_t) i * tensor->ne[0]);
|
||||
}
|
||||
|
||||
return n_tokens * (uint32_t) tensor->ne[0];
|
||||
}
|
||||
Reference in new issue
Block a user