mtmd: add mtmd_audio_stream + support voxtral-realtime (#30629)
* mtmd: add mtmd_audio_stream + support voxtral-realtime * address review
This commit is contained in:
1 parent
59d5fad7c1
commit
2851b3bcbf
27 files changed
+1000
-22
No files matched your search
@@ -257,6 +257,99 @@ static int generate_response(mtmd_cli_context & ctx, int n_predict) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// feed an audio file to a realtime model chunk by chunk, printing the transcript as it is produced
|
||||
static int run_audio_stream(mtmd_cli_context & ctx, const std::string & fname, int n_predict) {
|
||||
auto res = mtmd_helper_bitmap_init_from_file(ctx.ctx_vision.get(), fname.c_str(), false, ctx.init_opt);
|
||||
if (!res.bitmap) {
|
||||
return 1; // error is already printed by libmtmd
|
||||
}
|
||||
mtmd::bitmap bmp(res.bitmap);
|
||||
if (!mtmd_bitmap_is_audio(bmp.ptr.get())) {
|
||||
LOG_ERR("%s: %s is not an audio file\n", __func__, fname.c_str());
|
||||
return 1;
|
||||
}
|
||||
const float * samples = (const float *) bmp.data();
|
||||
const size_t n_samples = bmp.n_bytes() / sizeof(float);
|
||||
const int rate = mtmd_get_audio_sample_rate(ctx.ctx_vision.get());
|
||||
|
||||
const auto sparams = mtmd_audio_stream_params_default();
|
||||
mtmd::audio_stream_ptr stream(mtmd_audio_stream_init(ctx.ctx_vision.get(), sparams));
|
||||
if (!stream) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const size_t n_chunk = (size_t) rate * sparams.step_ms / 1000;
|
||||
const int n_embd = llama_model_n_embd_inp(ctx.model);
|
||||
|
||||
LOG("Transcribing %.1f s of audio, fed in chunks of %d ms\n\n", (double) n_samples / rate, sparams.step_ms);
|
||||
|
||||
size_t n_fed = 0;
|
||||
int n_gen = 0;
|
||||
llama_token tok = LLAMA_TOKEN_NULL;
|
||||
const int64_t t_start = ggml_time_us();
|
||||
int64_t t_enc = 0; // time spent in the stream (mel + encoder)
|
||||
int64_t t_dec = 0; // time spent in the decoder
|
||||
while (!g_is_interrupted) {
|
||||
const int64_t t0 = ggml_time_us();
|
||||
const int32_t n = mtmd_audio_stream_step(stream.get(), tok);
|
||||
t_enc += ggml_time_us() - t0;
|
||||
if (n < 0) {
|
||||
if (n == -2) {
|
||||
LOG_ERR("%s: stream error\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
break; // end of stream
|
||||
}
|
||||
if (n == 0) {
|
||||
if (n_fed < n_samples) {
|
||||
const size_t cnt = std::min(n_chunk, n_samples - n_fed);
|
||||
mtmd_audio_stream_push(stream.get(), samples + n_fed, cnt);
|
||||
n_fed += cnt;
|
||||
} else if (mtmd_audio_stream_flush(stream.get(), true) != 0) {
|
||||
break; // already ended
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const int64_t t1 = ggml_time_us();
|
||||
const float * rows = mtmd_audio_stream_get_output_embd(stream.get());
|
||||
ctx.batch.clear();
|
||||
for (int32_t i = 0; i < n; i++) {
|
||||
const llama_pos pos[GGML_MROPE_SECTIONS] = { ctx.n_past, ctx.n_past, ctx.n_past, 0 };
|
||||
ctx.n_past++;
|
||||
ctx.batch.add_embd({ rows + (size_t) i * n_embd, 1, (size_t) n_embd }, pos, 0, i == n - 1);
|
||||
}
|
||||
if (llama_process(ctx.lctx, LLAMA_PROCESS_TYPE_DECODE, ctx.batch.get())) {
|
||||
LOG_ERR("%s: failed to decode\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
tok = common_sampler_sample(ctx.smpl, ctx.lctx, -1);
|
||||
common_sampler_accept(ctx.smpl, tok, true);
|
||||
t_dec += ggml_time_us() - t1;
|
||||
if (llama_vocab_is_eog(ctx.vocab, tok)) {
|
||||
break;
|
||||
}
|
||||
// the model emits pad and word-boundary tokens between words
|
||||
if (!llama_vocab_is_control(ctx.vocab, tok)) {
|
||||
LOG("%s", common_token_to_piece(ctx.lctx, tok).c_str());
|
||||
fflush(stdout);
|
||||
}
|
||||
if (++n_gen >= n_predict) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const double t_total = (ggml_time_us() - t_start) / 1e6;
|
||||
LOG("\n\n%s: %d tokens, %.2f s audio in %.2f s (%.2fx realtime)\n", __func__,
|
||||
n_gen, (double) n_samples / rate, t_total, (double) n_samples / rate / t_total);
|
||||
LOG("%s: encoder %.2f s, decoder %.2f s, %.1f ms per 80 ms step\n", __func__,
|
||||
t_enc / 1e6, t_dec / 1e6, (t_enc + t_dec) / 1e3 / std::max(n_gen, 1));
|
||||
llama_perf_context_print(ctx.lctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static std::string chat_add_and_format(mtmd_cli_context & ctx, common_chat_msg & new_msg) {
|
||||
LOG_DBG("chat_add_and_format: new_msg.role='%s', new_msg.content='%s'\n",
|
||||
new_msg.role.c_str(), new_msg.content.c_str());
|
||||
@@ -472,6 +565,14 @@ int main(int argc, char ** argv) {
|
||||
|
||||
if (g_is_interrupted) return 130;
|
||||
|
||||
if (mtmd_support_audio_stream(ctx.ctx_vision.get())) {
|
||||
if (params.image.size() != 1) {
|
||||
LOG_ERR("This model transcribes audio in realtime, pass exactly one --audio file\n");
|
||||
return 1;
|
||||
}
|
||||
return run_audio_stream(ctx, params.image[0], n_predict);
|
||||
}
|
||||
|
||||
auto eval_system_prompt_if_present = [&] {
|
||||
if (params.system_prompt.empty()) {
|
||||
return 0;
|
||||
|
||||
Reference in new issue
Block a user