server: add oai-compat /v1/audio/speech endpoint (#31763)
* server: add oai-compat /v1/audio/speech endpoint * add e2e pytest * server: share the s16 rounding, type speech form fields The WAV writer of the audio helper truncates while pcm_f32_to_s16 rounds, so wav and pcm differ by one LSB. Both now round. pcm_f32_to_s16 takes a pointer and a count and converts the TTS bytes without an intermediate copy. /tts and /v1/audio/speech parse their body with parse_tts_body, so multipart fields such as speed get their type before the speech conversion checks them. --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
1 parent
20c2e96655
commit
08f945ffb5
13 files changed
+537
-200
No files matched your search
+213
-192
@@ -5744,6 +5744,25 @@ json server_routes::get_model_info() const {
|
||||
return get_res_model_info(*meta);
|
||||
}
|
||||
|
||||
// body of /tts and /v1/audio/speech; multipart form fields arrive as strings, the numeric and boolean params get their types back
|
||||
static json parse_tts_body(const server_http_req & req) {
|
||||
json body = json::parse(req.body);
|
||||
for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
|
||||
if (body.contains(k) && body[k].is_string()) {
|
||||
body[k] = std::stoll(body[k].get<std::string>());
|
||||
}
|
||||
}
|
||||
for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed" }) {
|
||||
if (body.contains(k) && body[k].is_string()) {
|
||||
body[k] = std::stof(body[k].get<std::string>());
|
||||
}
|
||||
}
|
||||
if (body.contains("stream") && body["stream"].is_string()) {
|
||||
body["stream"] = body["stream"] == "true";
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
void server_routes::init_routes() {
|
||||
// IMPORTANT: all lambda functions must start with create_response()
|
||||
// this is to ensure that the server_res_generator can handle sleeping case correctly
|
||||
@@ -6504,199 +6523,14 @@ void server_routes::init_routes() {
|
||||
};
|
||||
|
||||
this->post_tts = [this](const server_http_req & req) {
|
||||
auto res = create_response();
|
||||
res->set_req(&req); // will also set spipe if needed
|
||||
return handle_tts_impl(req, parse_tts_body(req));
|
||||
};
|
||||
|
||||
if (!ctx_server.has_cap_tts()) {
|
||||
res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
|
||||
return res;
|
||||
}
|
||||
|
||||
const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
|
||||
|
||||
json body = json::parse(req.body);
|
||||
|
||||
// multipart form fields arrive as strings, correct the types of the numeric and boolean params
|
||||
for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) {
|
||||
if (body.contains(k) && body[k].is_string()) {
|
||||
body[k] = std::stoll(body[k].get<std::string>());
|
||||
}
|
||||
}
|
||||
for (const char * k : { "top_p", "temperature", "repeat_penalty" }) {
|
||||
if (body.contains(k) && body[k].is_string()) {
|
||||
body[k] = std::stof(body[k].get<std::string>());
|
||||
}
|
||||
}
|
||||
if (body.contains("stream") && body["stream"].is_string()) {
|
||||
body["stream"] = body["stream"] == "true";
|
||||
}
|
||||
|
||||
std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
|
||||
if (prompt.empty()) {
|
||||
res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
|
||||
const std::string response_format = json_value(body, "response_format", std::string("wav"));
|
||||
|
||||
server_task task(SERVER_TASK_TYPE_TTS);
|
||||
server_schema::eval_tts_schema(params, task, body);
|
||||
const bool stream = task.params.stream;
|
||||
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
|
||||
const std::string lang = json_value(body, "lang", std::string());
|
||||
const std::string voice = json_value(body, "voice", std::string());
|
||||
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
|
||||
// a model with predefined voices only speaks with one of them
|
||||
mtmd::bitmap_ptr voice_ref;
|
||||
if (!voice.empty() && info.n_voices > 0) {
|
||||
voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str()));
|
||||
if (!voice_ref) {
|
||||
res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
}
|
||||
task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
|
||||
task.tts_inp.set_lang(lang);
|
||||
task.tts_inp.data.stream = stream;
|
||||
task.tts_inp.data.out_type = response_format == "pcm"
|
||||
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
|
||||
: MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
||||
// note: -1 is clamped to 0 by the sampler, it will disable the penalty
|
||||
task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
|
||||
task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
|
||||
if (task.tts_inp.data.top_k > 0) {
|
||||
task.params.sampling.top_k = task.tts_inp.data.top_k;
|
||||
}
|
||||
if (task.tts_inp.data.top_p > 0) {
|
||||
task.params.sampling.top_p = task.tts_inp.data.top_p;
|
||||
}
|
||||
if (task.tts_inp.data.temp >= 0) {
|
||||
task.params.sampling.temp = task.tts_inp.data.temp;
|
||||
}
|
||||
|
||||
// speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
|
||||
const unsigned char * speaker_ref_data = nullptr;
|
||||
size_t speaker_ref_len = 0;
|
||||
std::string speaker_ref_b64_decoded;
|
||||
|
||||
auto speaker_ref_file = req.files.find("speaker_ref");
|
||||
if (speaker_ref_file != req.files.end()) {
|
||||
speaker_ref_data = speaker_ref_file->second.data.data();
|
||||
speaker_ref_len = speaker_ref_file->second.data.size();
|
||||
} else {
|
||||
std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
|
||||
if (!speaker_ref_b64.empty()) {
|
||||
try {
|
||||
speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
|
||||
} catch (const std::exception &) {
|
||||
res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
|
||||
speaker_ref_len = speaker_ref_b64_decoded.size();
|
||||
}
|
||||
}
|
||||
|
||||
if (speaker_ref_len > 0) {
|
||||
if (voice_ref) {
|
||||
res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
if (!mtmd_support_audio(ctx_server.mctx)) {
|
||||
res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
|
||||
if (!wrapper.bitmap) {
|
||||
res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
|
||||
} else if (voice_ref) {
|
||||
task.tts_inp.set_speaker_ref(std::move(voice_ref));
|
||||
} else if (info.n_voices > 0) {
|
||||
// without a voice prompt the model produces near silence: the first voice is the default
|
||||
const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0);
|
||||
SRV_WRN("no voice given, speaking with \"%s\"\n", first);
|
||||
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first)));
|
||||
} else {
|
||||
SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
|
||||
}
|
||||
|
||||
auto & rd = res->rd;
|
||||
task.id = rd.get_new_id();
|
||||
rd.post_task(std::move(task));
|
||||
|
||||
// raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
|
||||
const std::string content_type = response_format == "pcm"
|
||||
? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32"
|
||||
: "audio/wav";
|
||||
|
||||
if (!stream) {
|
||||
auto result = rd.next(req.should_stop);
|
||||
if (!result) {
|
||||
GGML_ASSERT(req.should_stop());
|
||||
return res; // connection is closed
|
||||
}
|
||||
if (result->is_error()) {
|
||||
res->error(result->to_json());
|
||||
return res;
|
||||
}
|
||||
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
|
||||
GGML_ASSERT(tts_res != nullptr);
|
||||
res->status = 200;
|
||||
res->content_type = content_type;
|
||||
res->data = std::move(tts_res->audio);
|
||||
return res;
|
||||
} else {
|
||||
auto first_result = rd.next(req.should_stop);
|
||||
if (!first_result) {
|
||||
GGML_ASSERT(req.should_stop());
|
||||
return res; // connection is closed
|
||||
}
|
||||
if (first_result->is_error()) {
|
||||
res->error(first_result->to_json());
|
||||
return res;
|
||||
}
|
||||
auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
|
||||
GGML_ASSERT(first_tts_res != nullptr);
|
||||
|
||||
res->status = 200;
|
||||
res->content_type = content_type;
|
||||
res->data = std::move(first_tts_res->audio);
|
||||
bool is_done = first_tts_res->final;
|
||||
|
||||
res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {
|
||||
// flush buffered audio before is_done - the first result can already be final
|
||||
if (!res_this->data.empty()) {
|
||||
output = std::move(res_this->data);
|
||||
res_this->data.clear();
|
||||
return true;
|
||||
}
|
||||
if (is_done) {
|
||||
return false;
|
||||
}
|
||||
if (res_this->should_stop()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
server_response_reader & rd = res_this->rd;
|
||||
if (!rd.has_next()) {
|
||||
return false;
|
||||
}
|
||||
auto result = rd.next([&res_this]() { return res_this->should_stop(); });
|
||||
if (!result || result->is_error()) {
|
||||
return false;
|
||||
}
|
||||
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
|
||||
GGML_ASSERT(tts_res != nullptr);
|
||||
output = std::move(tts_res->audio);
|
||||
is_done = tts_res->final;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
return res;
|
||||
this->post_speech_oai = [this](const server_http_req & req) {
|
||||
json body = convert_speech_to_tts(parse_tts_body(req));
|
||||
SRV_DBG("%s\n", "Request converted: OpenAI Speech -> TTS");
|
||||
SRV_DBG("converted request: %s\n", body.dump().c_str());
|
||||
return handle_tts_impl(req, std::move(body));
|
||||
};
|
||||
|
||||
this->post_systemone = [this](const server_http_req & req) {
|
||||
@@ -7215,6 +7049,193 @@ std::unique_ptr<server_res_generator> server_routes::handle_rerank_impl(const se
|
||||
return res;
|
||||
}
|
||||
|
||||
std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const server_http_req & req, json body) {
|
||||
auto res = create_response();
|
||||
res->set_req(&req); // will also set spipe if needed
|
||||
|
||||
if (!ctx_server.has_cap_tts()) {
|
||||
res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
|
||||
return res;
|
||||
}
|
||||
|
||||
const auto info = mtmd_gen_audio_get_info(ctx_server.mctx);
|
||||
|
||||
std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
|
||||
if (prompt.empty()) {
|
||||
res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
|
||||
const std::string response_format = json_value(body, "response_format", std::string("wav"));
|
||||
const bool pcm16 = response_format == "pcm16";
|
||||
|
||||
server_task task(SERVER_TASK_TYPE_TTS);
|
||||
server_schema::eval_tts_schema(params, task, body);
|
||||
const bool stream = task.params.stream;
|
||||
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
|
||||
const std::string lang = json_value(body, "lang", std::string());
|
||||
const std::string voice = json_value(body, "voice", std::string());
|
||||
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
|
||||
// a model with predefined voices only speaks with one of them
|
||||
mtmd::bitmap_ptr voice_ref;
|
||||
if (!voice.empty() && info.n_voices > 0) {
|
||||
voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str()));
|
||||
if (!voice_ref) {
|
||||
res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
}
|
||||
task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
|
||||
task.tts_inp.set_lang(lang);
|
||||
task.tts_inp.data.stream = stream;
|
||||
task.tts_inp.data.out_type = response_format == "pcm" || pcm16
|
||||
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
|
||||
: MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
||||
// note: -1 is clamped to 0 by the sampler, it will disable the penalty
|
||||
task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512;
|
||||
task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG
|
||||
if (task.tts_inp.data.top_k > 0) {
|
||||
task.params.sampling.top_k = task.tts_inp.data.top_k;
|
||||
}
|
||||
if (task.tts_inp.data.top_p > 0) {
|
||||
task.params.sampling.top_p = task.tts_inp.data.top_p;
|
||||
}
|
||||
if (task.tts_inp.data.temp >= 0) {
|
||||
task.params.sampling.temp = task.tts_inp.data.temp;
|
||||
}
|
||||
|
||||
// speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
|
||||
const unsigned char * speaker_ref_data = nullptr;
|
||||
size_t speaker_ref_len = 0;
|
||||
std::string speaker_ref_b64_decoded;
|
||||
|
||||
auto speaker_ref_file = req.files.find("speaker_ref");
|
||||
if (speaker_ref_file != req.files.end()) {
|
||||
speaker_ref_data = speaker_ref_file->second.data.data();
|
||||
speaker_ref_len = speaker_ref_file->second.data.size();
|
||||
} else {
|
||||
std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
|
||||
if (!speaker_ref_b64.empty()) {
|
||||
try {
|
||||
speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
|
||||
} catch (const std::exception &) {
|
||||
res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
|
||||
speaker_ref_len = speaker_ref_b64_decoded.size();
|
||||
}
|
||||
}
|
||||
|
||||
if (speaker_ref_len > 0) {
|
||||
if (voice_ref) {
|
||||
res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
if (!mtmd_support_audio(ctx_server.mctx)) {
|
||||
res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
|
||||
if (!wrapper.bitmap) {
|
||||
res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
|
||||
return res;
|
||||
}
|
||||
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
|
||||
} else if (voice_ref) {
|
||||
task.tts_inp.set_speaker_ref(std::move(voice_ref));
|
||||
} else if (info.n_voices > 0) {
|
||||
// without a voice prompt the model produces near silence: the first voice is the default
|
||||
const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0);
|
||||
SRV_WRN("no voice given, speaking with \"%s\"\n", first);
|
||||
task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first)));
|
||||
} else {
|
||||
SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
|
||||
}
|
||||
|
||||
auto & rd = res->rd;
|
||||
task.id = rd.get_new_id();
|
||||
rd.post_task(std::move(task));
|
||||
|
||||
// raw LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586)
|
||||
const std::string pcm_type = "audio/pcm;rate=" + std::to_string(info.sample_rate);
|
||||
const std::string content_type = pcm16 ? pcm_type + ";encoding=signed-int;bits=16"
|
||||
: response_format == "pcm" ? pcm_type + ";encoding=float;bits=32"
|
||||
: "audio/wav";
|
||||
|
||||
// the pipeline outputs float32 samples
|
||||
auto take_audio = [pcm16](server_task_result_tts * tts_res) {
|
||||
const std::string & audio = tts_res->audio;
|
||||
return pcm16 ? pcm_f32_to_s16((const float *) audio.data(), audio.size() / sizeof(float)) : std::move(tts_res->audio);
|
||||
};
|
||||
|
||||
if (!stream) {
|
||||
auto result = rd.next(req.should_stop);
|
||||
if (!result) {
|
||||
GGML_ASSERT(req.should_stop());
|
||||
return res; // connection is closed
|
||||
}
|
||||
if (result->is_error()) {
|
||||
res->error(result->to_json());
|
||||
return res;
|
||||
}
|
||||
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
|
||||
GGML_ASSERT(tts_res != nullptr);
|
||||
res->status = 200;
|
||||
res->content_type = content_type;
|
||||
res->data = take_audio(tts_res);
|
||||
return res;
|
||||
} else {
|
||||
auto first_result = rd.next(req.should_stop);
|
||||
if (!first_result) {
|
||||
GGML_ASSERT(req.should_stop());
|
||||
return res; // connection is closed
|
||||
}
|
||||
if (first_result->is_error()) {
|
||||
res->error(first_result->to_json());
|
||||
return res;
|
||||
}
|
||||
auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
|
||||
GGML_ASSERT(first_tts_res != nullptr);
|
||||
|
||||
res->status = 200;
|
||||
res->content_type = content_type;
|
||||
res->data = take_audio(first_tts_res);
|
||||
bool is_done = first_tts_res->final;
|
||||
|
||||
res->set_next([res_this = res.get(), is_done, take_audio](std::string & output) mutable -> bool {
|
||||
// flush buffered audio before is_done - the first result can already be final
|
||||
if (!res_this->data.empty()) {
|
||||
output = std::move(res_this->data);
|
||||
res_this->data.clear();
|
||||
return true;
|
||||
}
|
||||
if (is_done) {
|
||||
return false;
|
||||
}
|
||||
if (res_this->should_stop()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
server_response_reader & rd = res_this->rd;
|
||||
if (!rd.has_next()) {
|
||||
return false;
|
||||
}
|
||||
auto result = rd.next([&res_this]() { return res_this->should_stop(); });
|
||||
if (!result || result->is_error()) {
|
||||
return false;
|
||||
}
|
||||
auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
|
||||
GGML_ASSERT(tts_res != nullptr);
|
||||
output = take_audio(tts_res);
|
||||
is_done = tts_res->final;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) {
|
||||
auto res = create_response();
|
||||
std::vector<raw_buffer> files;
|
||||
|
||||
Reference in new issue
Block a user