server: add oai-compat /v1/audio/speech endpoint (#31763)

* server: add oai-compat /v1/audio/speech endpoint

* add e2e pytest

* server: share the s16 rounding, type speech form fields

The WAV writer of the audio helper truncates while pcm_f32_to_s16
rounds, so wav and pcm differ by one LSB. Both now round.

pcm_f32_to_s16 takes a pointer and a count and converts the TTS
bytes without an intermediate copy.

/tts and /v1/audio/speech parse their body with parse_tts_body, so
multipart fields such as speed get their type before the speech
conversion checks them.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son NguyenandPascal authored and GitHub committed 2026-11-09 23:58:52 +01:00
1 parent 20c2e96655
commit 08f945ffb5
13 files changed
+537 -200

No files matched your search

+213 -192
View File
@@ -5744,6 +5744,25 @@ json server_routes::get_model_info() const {
return get_res_model_info(*meta);
}
// body of /tts and /v1/audio/speech; multipart form fields arrive as strings, the numeric and boolean params get their types back
static json parse_tts_body(const server_http_req & req) {
json body = json::parse(req.body);
for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stoll(body[k].get<std::string>());
}
}
for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stof(body[k].get<std::string>());
}
}
if (body.contains("stream") && body["stream"].is_string()) {
body["stream"] = body["stream"] == "true";
}
return body;
}
void server_routes::init_routes() {
// IMPORTANT: all lambda functions must start with create_response()
// this is to ensure that the server_res_generator can handle sleeping case correctly
@@ -6504,199 +6523,14 @@ void server_routes::init_routes() {
};
this->post_tts = [this](const server_http_req & req) {
auto res = create_response();
res->set_req(&req); // will also set spipe if needed
return handle_tts_impl(req, parse_tts_body(req));
};
if (!ctx_server.has_cap_tts()) {
res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
json body = json::parse(req.body);
// multipart form fields arrive as strings, correct the types of the numeric and boolean params
for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stoll(body[k].get<std::string>());
}
}
for (const char * k : { "top_p", "temperature", "repeat_penalty" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stof(body[k].get<std::string>());
}
}
if (body.contains("stream") && body["stream"].is_string()) {
body["stream"] = body["stream"] == "true";
}
std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
if (prompt.empty()) {
res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
return res;
}
const std::string response_format = json_value(body, "response_format", std::string("wav"));
server_task task(SERVER_TASK_TYPE_TTS);
server_schema::eval_tts_schema(params, task, body);
const bool stream = task.params.stream;
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
const std::string lang = json_value(body, "lang", std::string());
const std::string voice = json_value(body, "voice", std::string());
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
// a model with predefined voices only speaks with one of them
mtmd::bitmap_ptr voice_ref;
if (!voice.empty() && info.n_voices > 0) {
voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str()));
if (!voice_ref) {
res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST));
return res;
}
}
task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
task.tts_inp.set_lang(lang);
task.tts_inp.data.stream = stream;
task.tts_inp.data.out_type = response_format == "pcm"
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
: MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
// note: -1 is clamped to 0 by the sampler, it will disable the penalty
task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
if (task.tts_inp.data.top_k > 0) {
task.params.sampling.top_k = task.tts_inp.data.top_k;
}
if (task.tts_inp.data.top_p > 0) {
task.params.sampling.top_p = task.tts_inp.data.top_p;
}
if (task.tts_inp.data.temp >= 0) {
task.params.sampling.temp = task.tts_inp.data.temp;
}
// speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
const unsigned char * speaker_ref_data = nullptr;
size_t speaker_ref_len = 0;
std::string speaker_ref_b64_decoded;
auto speaker_ref_file = req.files.find("speaker_ref");
if (speaker_ref_file != req.files.end()) {
speaker_ref_data = speaker_ref_file->second.data.data();
speaker_ref_len = speaker_ref_file->second.data.size();
} else {
std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
if (!speaker_ref_b64.empty()) {
try {
speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
} catch (const std::exception &) {
res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
return res;
}
speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
speaker_ref_len = speaker_ref_b64_decoded.size();
}
}
if (speaker_ref_len > 0) {
if (voice_ref) {
res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST));
return res;
}
if (!mtmd_support_audio(ctx_server.mctx)) {
res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST));
return res;
}
auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
if (!wrapper.bitmap) {
res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
return res;
}
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
} else if (voice_ref) {
task.tts_inp.set_speaker_ref(std::move(voice_ref));
} else if (info.n_voices > 0) {
// without a voice prompt the model produces near silence: the first voice is the default
const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0);
SRV_WRN("no voice given, speaking with \"%s\"\n", first);
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first)));
} else {
SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
}
auto & rd = res->rd;
task.id = rd.get_new_id();
rd.post_task(std::move(task));
// raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
const std::string content_type = response_format == "pcm"
? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32"
: "audio/wav";
if (!stream) {
auto result = rd.next(req.should_stop);
if (!result) {
GGML_ASSERT(req.should_stop());
return res; // connection is closed
}
if (result->is_error()) {
res->error(result->to_json());
return res;
}
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
GGML_ASSERT(tts_res != nullptr);
res->status = 200;
res->content_type = content_type;
res->data = std::move(tts_res->audio);
return res;
} else {
auto first_result = rd.next(req.should_stop);
if (!first_result) {
GGML_ASSERT(req.should_stop());
return res; // connection is closed
}
if (first_result->is_error()) {
res->error(first_result->to_json());
return res;
}
auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
GGML_ASSERT(first_tts_res != nullptr);
res->status = 200;
res->content_type = content_type;
res->data = std::move(first_tts_res->audio);
bool is_done = first_tts_res->final;
res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {
// flush buffered audio before is_done - the first result can already be final
if (!res_this->data.empty()) {
output = std::move(res_this->data);
res_this->data.clear();
return true;
}
if (is_done) {
return false;
}
if (res_this->should_stop()) {
return false;
}
server_response_reader & rd = res_this->rd;
if (!rd.has_next()) {
return false;
}
auto result = rd.next([&res_this]() { return res_this->should_stop(); });
if (!result || result->is_error()) {
return false;
}
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
GGML_ASSERT(tts_res != nullptr);
output = std::move(tts_res->audio);
is_done = tts_res->final;
return true;
});
}
return res;
this->post_speech_oai = [this](const server_http_req & req) {
json body = convert_speech_to_tts(parse_tts_body(req));
SRV_DBG("%s\n", "Request converted: OpenAI Speech -> TTS");
SRV_DBG("converted request: %s\n", body.dump().c_str());
return handle_tts_impl(req, std::move(body));
};
this->post_systemone = [this](const server_http_req & req) {
@@ -7215,6 +7049,193 @@ std::unique_ptr<server_res_generator> server_routes::handle_rerank_impl(const se
return res;
}
std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const server_http_req & req, json body) {
auto res = create_response();
res->set_req(&req); // will also set spipe if needed
if (!ctx_server.has_cap_tts()) {
res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
if (prompt.empty()) {
res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
return res;
}
const std::string response_format = json_value(body, "response_format", std::string("wav"));
const bool pcm16 = response_format == "pcm16";
server_task task(SERVER_TASK_TYPE_TTS);
server_schema::eval_tts_schema(params, task, body);
const bool stream = task.params.stream;
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
const std::string lang = json_value(body, "lang", std::string());
const std::string voice = json_value(body, "voice", std::string());
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
// a model with predefined voices only speaks with one of them
mtmd::bitmap_ptr voice_ref;
if (!voice.empty() && info.n_voices > 0) {
voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str()));
if (!voice_ref) {
res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST));
return res;
}
}
task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
task.tts_inp.set_lang(lang);
task.tts_inp.data.stream = stream;
task.tts_inp.data.out_type = response_format == "pcm" || pcm16
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
: MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
// note: -1 is clamped to 0 by the sampler, it will disable the penalty
task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
if (task.tts_inp.data.top_k > 0) {
task.params.sampling.top_k = task.tts_inp.data.top_k;
}
if (task.tts_inp.data.top_p > 0) {
task.params.sampling.top_p = task.tts_inp.data.top_p;
}
if (task.tts_inp.data.temp >= 0) {
task.params.sampling.temp = task.tts_inp.data.temp;
}
// speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
const unsigned char * speaker_ref_data = nullptr;
size_t speaker_ref_len = 0;
std::string speaker_ref_b64_decoded;
auto speaker_ref_file = req.files.find("speaker_ref");
if (speaker_ref_file != req.files.end()) {
speaker_ref_data = speaker_ref_file->second.data.data();
speaker_ref_len = speaker_ref_file->second.data.size();
} else {
std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
if (!speaker_ref_b64.empty()) {
try {
speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
} catch (const std::exception &) {
res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
return res;
}
speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
speaker_ref_len = speaker_ref_b64_decoded.size();
}
}
if (speaker_ref_len > 0) {
if (voice_ref) {
res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST));
return res;
}
if (!mtmd_support_audio(ctx_server.mctx)) {
res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST));
return res;
}
auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
if (!wrapper.bitmap) {
res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
return res;
}
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
} else if (voice_ref) {
task.tts_inp.set_speaker_ref(std::move(voice_ref));
} else if (info.n_voices > 0) {
// without a voice prompt the model produces near silence: the first voice is the default
const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0);
SRV_WRN("no voice given, speaking with \"%s\"\n", first);
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first)));
} else {
SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
}
auto & rd = res->rd;
task.id = rd.get_new_id();
rd.post_task(std::move(task));
// raw LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
const std::string pcm_type = "audio/pcm;rate=" + std::to_string(info.sample_rate);
const std::string content_type = pcm16 ? pcm_type + ";encoding=signed-int;bits=16"
: response_format == "pcm" ? pcm_type + ";encoding=float;bits=32"
: "audio/wav";
// the pipeline outputs float32 samples
auto take_audio = [pcm16](server_task_result_tts * tts_res) {
const std::string & audio = tts_res->audio;
return pcm16 ? pcm_f32_to_s16((const float *) audio.data(), audio.size() / sizeof(float)) : std::move(tts_res->audio);
};
if (!stream) {
auto result = rd.next(req.should_stop);
if (!result) {
GGML_ASSERT(req.should_stop());
return res; // connection is closed
}
if (result->is_error()) {
res->error(result->to_json());
return res;
}
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
GGML_ASSERT(tts_res != nullptr);
res->status = 200;
res->content_type = content_type;
res->data = take_audio(tts_res);
return res;
} else {
auto first_result = rd.next(req.should_stop);
if (!first_result) {
GGML_ASSERT(req.should_stop());
return res; // connection is closed
}
if (first_result->is_error()) {
res->error(first_result->to_json());
return res;
}
auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
GGML_ASSERT(first_tts_res != nullptr);
res->status = 200;
res->content_type = content_type;
res->data = take_audio(first_tts_res);
bool is_done = first_tts_res->final;
res->set_next([res_this = res.get(), is_done, take_audio](std::string & output) mutable -> bool {
// flush buffered audio before is_done - the first result can already be final
if (!res_this->data.empty()) {
output = std::move(res_this->data);
res_this->data.clear();
return true;
}
if (is_done) {
return false;
}
if (res_this->should_stop()) {
return false;
}
server_response_reader & rd = res_this->rd;
if (!rd.has_next()) {
return false;
}
auto result = rd.next([&res_this]() { return res_this->should_stop(); });
if (!result || result->is_error()) {
return false;
}
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
GGML_ASSERT(tts_res != nullptr);
output = take_audio(tts_res);
is_done = tts_res->final;
return true;
});
}
return res;
}
std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) {
auto res = create_response();
std::vector<raw_buffer> files;