diff --git a/tests/test-server-component.cpp b/tests/test-server-component.cpp index 2c76e8904..c4ed1cacc 100644 --- a/tests/test-server-component.cpp +++ b/tests/test-server-component.cpp @@ -1,5 +1,6 @@ #include "testing.h" +#include "../tools/server/server-chat.h" #include "../tools/server/server-common.h" #include "../tools/server/server-decision.h" #include "../tools/server/server-schema.h" @@ -7,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -665,6 +667,79 @@ static void test_glob_match(testing & t) { }); } +// +// OpenAI speech API +// + +static void test_speech_convert(testing & t) { + const json base = {{"model", "tts"}, {"input", "Hello"}, {"voice", "alba"}}; + + auto with = [&](const char * key, const json & value) { + json body = base; + body[key] = value; + return body; + }; + auto rejected = [&](const char * key, const json & value) { + try { + convert_speech_to_tts(with(key, value)); + } catch (const std::invalid_argument &) { + return true; + } + return false; + }; + + // defaults + json out = convert_speech_to_tts(base); + t.assert_equal("input kept", std::string("Hello"), out.at("input").get()); + t.assert_equal("voice kept", std::string("alba"), out.at("voice").get()); + t.assert_equal("default format", std::string("wav"), out.at("response_format").get()); + t.assert_true ("no stream", !out.contains("stream")); + + // response_format + out = convert_speech_to_tts(with("response_format", "wav")); + t.assert_equal("wav", std::string("wav"), out.at("response_format").get()); + out = convert_speech_to_tts(with("response_format", "pcm")); + t.assert_equal("pcm is 16-bit", std::string("pcm16"), out.at("response_format").get()); + for (const char * format : {"mp3", "opus", "aac", "flac", "pcm16"}) { + t.assert_true(std::string("rejects ") + format, rejected("response_format", format)); + } + + // custom voice + out = convert_speech_to_tts(with("voice", json{{"id", "voice_1234"}})); + t.assert_equal("voice object", std::string("voice_1234"), out.at("voice").get()); + + // stream_format + out = convert_speech_to_tts(with("stream_format", "audio")); + t.assert_true ("audio streams", out.at("stream").get()); + t.assert_true ("rejects sse", rejected("stream_format", "sse")); + + // speed + t.assert_true ("speed 1.0", !rejected("speed", 1.0)); + t.assert_true ("rejects speed 1.5", rejected("speed", 1.5)); + + // instructions are ignored + t.assert_true ("instructions", !rejected("instructions", "whisper")); +} + +static void test_pcm_f32_to_s16(testing & t) { + auto sample = [](const std::string & bytes, size_t i) { + int16_t v; + memcpy(&v, bytes.data() + i * sizeof(v), sizeof(v)); + return (int) v; + }; + + const std::vector samples = {0.0f, 1.0f, -1.0f, 0.5f, 2.0f, -2.0f}; + const std::string bytes = pcm_f32_to_s16(samples.data(), samples.size()); + t.assert_equal("size", (size_t) 12, bytes.size()); + t.assert_equal("zero", 0, sample(bytes, 0)); + t.assert_equal("max", 32767, sample(bytes, 1)); + t.assert_equal("min", -32767, sample(bytes, 2)); + t.assert_equal("rounds", 16384, sample(bytes, 3)); + t.assert_equal("clamps above", 32767, sample(bytes, 4)); + t.assert_equal("clamps below", -32767, sample(bytes, 5)); + t.assert_true ("empty", pcm_f32_to_s16(nullptr, 0).empty()); +} + int main(int argc, char ** argv) { testing t; @@ -690,6 +765,8 @@ int main(int argc, char ** argv) { t.test("decision_depth", test_decision_depth); t.test("glob_match", test_glob_match); t.test("dry_sequence_breakers", test_dry_sequence_breakers); + t.test("speech_convert", test_speech_convert); + t.test("pcm_f32_to_s16", test_pcm_f32_to_s16); return t.summary(); } diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 5e77eccd9..73e7e1b27 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -72,7 +72,7 @@ static void write_wav16_header(std::vector & buf, uint32_t data_sz, int32_ static void append_wav16_pcm(std::vector & buf, const float * pcm, size_t n) { for (size_t i = 0; i < n; i++) { - int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f); + int16_t s = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f); put_bytes(buf, &s, 2); } } diff --git a/tools/server/README.md b/tools/server/README.md index 712205dd5..defc09c6b 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -872,7 +872,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info `n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token. -`response_format`: `wav` (default) or `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`). +`response_format`: `wav` (default), `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`) or `pcm16` (same, as signed 16-bit samples, e.g. `audio/pcm;rate=24000;encoding=signed-int;bits=16`). `stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF. @@ -1652,6 +1652,33 @@ Tools go into the system prompt with the chat template, and the tool calls are p See [examples/realtime_client.py](../../examples/realtime_client.py) for a client with the `openai` library. +### POST `/v1/audio/speech`: OpenAI-compatible Speech API + +Generates speech from text, for models that can generate audio. The request is converted to a [`/tts`](#post-tts-generate-speech-audio-from-text) request, so the `/tts` options such as `lang`, `seed` or `speaker_ref_b64` can also be used. + +*Options:* + +`input`: The text to speak. + +`voice`: One of the `voices` listed by `GET /props`, as a name or as `{"id": "..."}`. OpenAI voice names such as `alloy` are not mapped: a model with predefined voices rejects them. + +`response_format`: `wav` or `pcm` (raw signed 16-bit little-endian mono samples, no header). Defaults to `wav`, unlike the `mp3` of the OpenAI API. Other formats are rejected. The sample rate is the one of the model and is not resampled to the 24 kHz of the OpenAI API, the response `Content-Type` carries it. + +`stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted. + +`speed`: Only `1.0` is accepted. + +`instructions`: Ignored. + +*Examples:* + +```shell +curl http://127.0.0.1:8080/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model": "tts", "input": "Hello, this is a test."}' \ + -o output.wav +``` + ### POST `/v1/responses`: OpenAI-compatible Responses API *Options:* diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index a7431800b..8cf883a3a 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -737,6 +737,42 @@ json convert_transcriptions_to_chatcmpl( return chatcmpl_body; } +json convert_speech_to_tts(const json & inp_body) { + json tts_body = inp_body; // copy all fields + + // a custom voice is given as {"id": "..."} + if (inp_body.contains("voice") && inp_body.at("voice").is_object()) { + tts_body["voice"] = json_value(inp_body.at("voice"), "id", std::string()); + } + + // the default of the OpenAI API is mp3, we have no encoder for it + const std::string response_format = json_value(inp_body, "response_format", std::string("wav")); + if (response_format == "pcm") { + tts_body["response_format"] = "pcm16"; // "pcm" is float32 in /tts + } else if (response_format == "wav") { + tts_body["response_format"] = "wav"; + } else { + throw std::invalid_argument("Only 'wav' and 'pcm' response_format are supported for speech"); + } + + const std::string stream_format = json_value(inp_body, "stream_format", std::string()); + if (stream_format == "audio") { + tts_body["stream"] = true; + } else if (!stream_format.empty()) { + throw std::invalid_argument("Only 'audio' stream_format is supported for speech"); + } + + if (json_value(inp_body, "speed", 1.0) != 1.0) { + throw std::invalid_argument("'speed' is not supported for speech"); + } + + if (!json_value(inp_body, "instructions", std::string()).empty()) { + SRV_WRN("%s", "'instructions' is not supported for speech and is ignored\n"); + } + + return tts_body; +} + llama_tokens server_chat_get_asr_prompt(const llama_model * model) { const char * src = llama_model_chat_template(model, "asr"); if (src == nullptr) { diff --git a/tools/server/server-chat.h b/tools/server/server-chat.h index b4b6f1ec6..cba4a073c 100644 --- a/tools/server/server-chat.h +++ b/tools/server/server-chat.h @@ -24,6 +24,9 @@ json convert_transcriptions_to_chatcmpl( const std::map & in_files, std::vector & out_files); +// convert OpenAI speech API format to the /tts format +json convert_speech_to_tts(const json & body); + // prompt of the "asr" chat template, live ASR models (e.g. LFM3-Audio live ASR) transcribe with it // empty if the model has no such template, throws on a template error llama_tokens server_chat_get_asr_prompt(const llama_model * model); diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 05fd04c29..5bf2ec12a 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -9,6 +9,8 @@ #include "server-common.h" +#include +#include #include #include #include @@ -200,6 +202,19 @@ const char * get_media_marker() { return marker.c_str(); } +// +// audio utils +// + +std::string pcm_f32_to_s16(const float * samples, size_t n) { + std::string bytes(n * sizeof(int16_t), '\0'); + for (size_t i = 0; i < n; i++) { + const int16_t v = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, samples[i])) * 32767.0f); + memcpy(&bytes[2 * i], &v, sizeof(v)); + } + return bytes; +} + // // model output modalities // diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 01bf12e61..ca870465a 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -110,6 +110,13 @@ std::string gen_tool_call_id(); // get a random marker; note: each time the server restarts, the marker will be different const char * get_media_marker(); +// +// audio utils +// + +// float samples to signed 16-bit PCM bytes, clamped to [-1, 1] +std::string pcm_f32_to_s16(const float * samples, size_t n); + // // model output modalities // diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index be44c3e2e..5a44bb7b3 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5744,6 +5744,25 @@ json server_routes::get_model_info() const { return get_res_model_info(*meta); } +// body of /tts and /v1/audio/speech; multipart form fields arrive as strings, the numeric and boolean params get their types back +static json parse_tts_body(const server_http_req & req) { + json body = json::parse(req.body); + for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) { + if (body.contains(k) && body[k].is_string()) { + body[k] = std::stoll(body[k].get()); + } + } + for (const char * k : { "top_p", "temperature", "repeat_penalty", "speed" }) { + if (body.contains(k) && body[k].is_string()) { + body[k] = std::stof(body[k].get()); + } + } + if (body.contains("stream") && body["stream"].is_string()) { + body["stream"] = body["stream"] == "true"; + } + return body; +} + void server_routes::init_routes() { // IMPORTANT: all lambda functions must start with create_response() // this is to ensure that the server_res_generator can handle sleeping case correctly @@ -6504,199 +6523,14 @@ void server_routes::init_routes() { }; this->post_tts = [this](const server_http_req & req) { - auto res = create_response(); - res->set_req(&req); // will also set spipe if needed + return handle_tts_impl(req, parse_tts_body(req)); + }; - if (!ctx_server.has_cap_tts()) { - res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED)); - return res; - } - - const auto info = mtmd_gen_audio_get_info(ctx_server.mctx); - - json body = json::parse(req.body); - - // multipart form fields arrive as strings, correct the types of the numeric and boolean params - for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) { - if (body.contains(k) && body[k].is_string()) { - body[k] = std::stoll(body[k].get()); - } - } - for (const char * k : { "top_p", "temperature", "repeat_penalty" }) { - if (body.contains(k) && body[k].is_string()) { - body[k] = std::stof(body[k].get()); - } - } - if (body.contains("stream") && body["stream"].is_string()) { - body["stream"] = body["stream"] == "true"; - } - - std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string())); - if (prompt.empty()) { - res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - - const std::string response_format = json_value(body, "response_format", std::string("wav")); - - server_task task(SERVER_TASK_TYPE_TTS); - server_schema::eval_tts_schema(params, task, body); - const bool stream = task.params.stream; - // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it - const std::string lang = json_value(body, "lang", std::string()); - const std::string voice = json_value(body, "voice", std::string()); - const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice); - // a model with predefined voices only speaks with one of them - mtmd::bitmap_ptr voice_ref; - if (!voice.empty() && info.n_voices > 0) { - voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str())); - if (!voice_ref) { - res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - } - task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt); - task.tts_inp.set_lang(lang); - task.tts_inp.data.stream = stream; - task.tts_inp.data.out_type = response_format == "pcm" - ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM - : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; - // note: -1 is clamped to 0 by the sampler, it will disable the penalty - task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512; - task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG - if (task.tts_inp.data.top_k > 0) { - task.params.sampling.top_k = task.tts_inp.data.top_k; - } - if (task.tts_inp.data.top_p > 0) { - task.params.sampling.top_p = task.tts_inp.data.top_p; - } - if (task.tts_inp.data.temp >= 0) { - task.params.sampling.temp = task.tts_inp.data.temp; - } - - // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64") - const unsigned char * speaker_ref_data = nullptr; - size_t speaker_ref_len = 0; - std::string speaker_ref_b64_decoded; - - auto speaker_ref_file = req.files.find("speaker_ref"); - if (speaker_ref_file != req.files.end()) { - speaker_ref_data = speaker_ref_file->second.data.data(); - speaker_ref_len = speaker_ref_file->second.data.size(); - } else { - std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); - if (!speaker_ref_b64.empty()) { - try { - speaker_ref_b64_decoded = base64::decode(speaker_ref_b64); - } catch (const std::exception &) { - res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data(); - speaker_ref_len = speaker_ref_b64_decoded.size(); - } - } - - if (speaker_ref_len > 0) { - if (voice_ref) { - res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - if (!mtmd_support_audio(ctx_server.mctx)) { - res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); - if (!wrapper.bitmap) { - res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); - return res; - } - task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); - } else if (voice_ref) { - task.tts_inp.set_speaker_ref(std::move(voice_ref)); - } else if (info.n_voices > 0) { - // without a voice prompt the model produces near silence: the first voice is the default - const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0); - SRV_WRN("no voice given, speaking with \"%s\"\n", first); - task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first))); - } else { - SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n"); - } - - auto & rd = res->rd; - task.id = rd.get_new_id(); - rd.post_task(std::move(task)); - - // raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586) - const std::string content_type = response_format == "pcm" - ? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32" - : "audio/wav"; - - if (!stream) { - auto result = rd.next(req.should_stop); - if (!result) { - GGML_ASSERT(req.should_stop()); - return res; // connection is closed - } - if (result->is_error()) { - res->error(result->to_json()); - return res; - } - auto * tts_res = dynamic_cast(result.get()); - GGML_ASSERT(tts_res != nullptr); - res->status = 200; - res->content_type = content_type; - res->data = std::move(tts_res->audio); - return res; - } else { - auto first_result = rd.next(req.should_stop); - if (!first_result) { - GGML_ASSERT(req.should_stop()); - return res; // connection is closed - } - if (first_result->is_error()) { - res->error(first_result->to_json()); - return res; - } - auto * first_tts_res = dynamic_cast(first_result.get()); - GGML_ASSERT(first_tts_res != nullptr); - - res->status = 200; - res->content_type = content_type; - res->data = std::move(first_tts_res->audio); - bool is_done = first_tts_res->final; - - res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool { - // flush buffered audio before is_done - the first result can already be final - if (!res_this->data.empty()) { - output = std::move(res_this->data); - res_this->data.clear(); - return true; - } - if (is_done) { - return false; - } - if (res_this->should_stop()) { - return false; - } - - server_response_reader & rd = res_this->rd; - if (!rd.has_next()) { - return false; - } - auto result = rd.next([&res_this]() { return res_this->should_stop(); }); - if (!result || result->is_error()) { - return false; - } - auto * tts_res = dynamic_cast(result.get()); - GGML_ASSERT(tts_res != nullptr); - output = std::move(tts_res->audio); - is_done = tts_res->final; - return true; - }); - } - - return res; + this->post_speech_oai = [this](const server_http_req & req) { + json body = convert_speech_to_tts(parse_tts_body(req)); + SRV_DBG("%s\n", "Request converted: OpenAI Speech -> TTS"); + SRV_DBG("converted request: %s\n", body.dump().c_str()); + return handle_tts_impl(req, std::move(body)); }; this->post_systemone = [this](const server_http_req & req) { @@ -7215,6 +7049,193 @@ std::unique_ptr server_routes::handle_rerank_impl(const se return res; } +std::unique_ptr server_routes::handle_tts_impl(const server_http_req & req, json body) { + auto res = create_response(); + res->set_req(&req); // will also set spipe if needed + + if (!ctx_server.has_cap_tts()) { + res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + + const auto info = mtmd_gen_audio_get_info(ctx_server.mctx); + + std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string())); + if (prompt.empty()) { + res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + + const std::string response_format = json_value(body, "response_format", std::string("wav")); + const bool pcm16 = response_format == "pcm16"; + + server_task task(SERVER_TASK_TYPE_TTS); + server_schema::eval_tts_schema(params, task, body); + const bool stream = task.params.stream; + // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it + const std::string lang = json_value(body, "lang", std::string()); + const std::string voice = json_value(body, "voice", std::string()); + const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice); + // a model with predefined voices only speaks with one of them + mtmd::bitmap_ptr voice_ref; + if (!voice.empty() && info.n_voices > 0) { + voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str())); + if (!voice_ref) { + res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + } + task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt); + task.tts_inp.set_lang(lang); + task.tts_inp.data.stream = stream; + task.tts_inp.data.out_type = response_format == "pcm" || pcm16 + ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM + : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + // note: -1 is clamped to 0 by the sampler, it will disable the penalty + task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512; + task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG + if (task.tts_inp.data.top_k > 0) { + task.params.sampling.top_k = task.tts_inp.data.top_k; + } + if (task.tts_inp.data.top_p > 0) { + task.params.sampling.top_p = task.tts_inp.data.top_p; + } + if (task.tts_inp.data.temp >= 0) { + task.params.sampling.temp = task.tts_inp.data.temp; + } + + // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64") + const unsigned char * speaker_ref_data = nullptr; + size_t speaker_ref_len = 0; + std::string speaker_ref_b64_decoded; + + auto speaker_ref_file = req.files.find("speaker_ref"); + if (speaker_ref_file != req.files.end()) { + speaker_ref_data = speaker_ref_file->second.data.data(); + speaker_ref_len = speaker_ref_file->second.data.size(); + } else { + std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); + if (!speaker_ref_b64.empty()) { + try { + speaker_ref_b64_decoded = base64::decode(speaker_ref_b64); + } catch (const std::exception &) { + res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data(); + speaker_ref_len = speaker_ref_b64_decoded.size(); + } + } + + if (speaker_ref_len > 0) { + if (voice_ref) { + res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + if (!mtmd_support_audio(ctx_server.mctx)) { + res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); + if (!wrapper.bitmap) { + res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); + } else if (voice_ref) { + task.tts_inp.set_speaker_ref(std::move(voice_ref)); + } else if (info.n_voices > 0) { + // without a voice prompt the model produces near silence: the first voice is the default + const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0); + SRV_WRN("no voice given, speaking with \"%s\"\n", first); + task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first))); + } else { + SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n"); + } + + auto & rd = res->rd; + task.id = rd.get_new_id(); + rd.post_task(std::move(task)); + + // raw LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586) + const std::string pcm_type = "audio/pcm;rate=" + std::to_string(info.sample_rate); + const std::string content_type = pcm16 ? pcm_type + ";encoding=signed-int;bits=16" + : response_format == "pcm" ? pcm_type + ";encoding=float;bits=32" + : "audio/wav"; + + // the pipeline outputs float32 samples + auto take_audio = [pcm16](server_task_result_tts * tts_res) { + const std::string & audio = tts_res->audio; + return pcm16 ? pcm_f32_to_s16((const float *) audio.data(), audio.size() / sizeof(float)) : std::move(tts_res->audio); + }; + + if (!stream) { + auto result = rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + auto * tts_res = dynamic_cast(result.get()); + GGML_ASSERT(tts_res != nullptr); + res->status = 200; + res->content_type = content_type; + res->data = take_audio(tts_res); + return res; + } else { + auto first_result = rd.next(req.should_stop); + if (!first_result) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } + if (first_result->is_error()) { + res->error(first_result->to_json()); + return res; + } + auto * first_tts_res = dynamic_cast(first_result.get()); + GGML_ASSERT(first_tts_res != nullptr); + + res->status = 200; + res->content_type = content_type; + res->data = take_audio(first_tts_res); + bool is_done = first_tts_res->final; + + res->set_next([res_this = res.get(), is_done, take_audio](std::string & output) mutable -> bool { + // flush buffered audio before is_done - the first result can already be final + if (!res_this->data.empty()) { + output = std::move(res_this->data); + res_this->data.clear(); + return true; + } + if (is_done) { + return false; + } + if (res_this->should_stop()) { + return false; + } + + server_response_reader & rd = res_this->rd; + if (!rd.has_next()) { + return false; + } + auto result = rd.next([&res_this]() { return res_this->should_stop(); }); + if (!result || result->is_error()) { + return false; + } + auto * tts_res = dynamic_cast(result.get()); + GGML_ASSERT(tts_res != nullptr); + output = take_audio(tts_res); + is_done = tts_res->final; + return true; + }); + } + + return res; +} + std::unique_ptr server_routes::handle_count_tokens(const server_http_req & req, task_response_type res_type) { auto res = create_response(); std::vector files; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 05c538ea1..4b145cb8a 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -134,6 +134,7 @@ struct server_routes { server_http_context::handler_t post_embeddings_oai; server_http_context::handler_t post_rerank; server_http_context::handler_t post_tts; + server_http_context::handler_t post_speech_oai; server_http_context::handler_t post_systemone; server_http_context::handler_t get_lora_adapters; server_http_context::handler_t post_lora_adapters; @@ -154,6 +155,7 @@ private: std::unique_ptr handle_slots_erase(const server_http_req &, int id_slot); std::unique_ptr handle_embeddings_impl(const server_http_req & req, task_response_type res_type); std::unique_ptr handle_rerank_impl(const server_http_req & req); + std::unique_ptr handle_tts_impl(const server_http_req & req, json body); std::unique_ptr handle_count_tokens(const server_http_req & req, task_response_type res_type); // using unique_ptr to allow late initialization of const diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp index b608b87ab..cabcedb78 100644 --- a/tools/server/server-realtime.cpp +++ b/tools/server/server-realtime.cpp @@ -714,12 +714,7 @@ struct realtime_conversation { // static std::string encode_pcm16(const std::vector & samples) { - std::string bytes(samples.size() * sizeof(int16_t), '\0'); - for (size_t i = 0; i < samples.size(); i++) { - const int16_t v = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, samples[i])) * 32767.0f); - memcpy(&bytes[2 * i], &v, sizeof(v)); - } - return base64::encode(bytes); + return base64::encode(pcm_f32_to_s16(samples.data(), samples.size())); } // the message item of the response is added with its first delta diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 8b1d42650..4e838cb10 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -186,6 +186,8 @@ static void server_register_routes(server_http_context & ctx_http, server_routes ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); ctx_http.post("/tts", ex_wrapper(routes.post_tts)); + ctx_http.post("/v1/audio/speech", ex_wrapper(routes.post_speech_oai)); + ctx_http.post("/audio/speech", ex_wrapper(routes.post_speech_oai)); ctx_http.ws ("/v1/realtime", routes.ws_realtime); ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API ctx_http.post("/infill", ex_wrapper(routes.post_infill)); @@ -504,6 +506,7 @@ static int server_run_router(common_params & params, int argc, char ** argv, con routes.post_responses_oai = models_routes->proxy_post; routes.post_transcriptions_oai = models_routes->proxy_post; routes.post_tts = models_routes->proxy_post; + routes.post_speech_oai = models_routes->proxy_post; routes.ws_realtime = models_routes->proxy_ws; routes.post_anthropic_messages = models_routes->proxy_post; routes.post_anthropic_count_tokens = models_routes->proxy_post; diff --git a/tools/server/tests/unit/test_compat_oai_speech.py b/tools/server/tests/unit/test_compat_oai_speech.py new file mode 100644 index 000000000..888942f02 --- /dev/null +++ b/tools/server/tests/unit/test_compat_oai_speech.py @@ -0,0 +1,137 @@ +import pytest +import requests +import struct +from utils import * + +server: ServerProcess + +BODY = { + "model": "tts", + "input": "Hello there.", + "voice": "female", + "seed": 42, + "n_predict": 8, +} + + +@pytest.fixture(autouse=True) +def create_server(): + global server + server = ServerPreset.tinylfm3omni() + + +def post_audio(path: str, **extra) -> requests.Response: + return requests.post(server.make_url(path), json={**BODY, **extra}, timeout=DEFAULT_REQUEST_TIMEOUT) + + +def assert_wav(res: requests.Response, data_size: int | None = None): + assert res.status_code == 200 + assert res.headers["Content-Type"] == "audio/wav" + assert res.content[:4] == b"RIFF" and res.content[8:12] == b"WAVE" + assert struct.unpack(" pos + 8 + + +@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) +def test_speech_default_format_is_wav(path: str): + server.start() + assert_wav(post_audio(path)) + + +def test_speech_wav_same_as_tts(): + server.start() + res = post_audio("/v1/audio/speech", response_format="wav") + assert_wav(res) + assert res.content == post_audio("/tts").content + + +def test_speech_pcm_is_signed_16_bit(): + server.start() + res = post_audio("/v1/audio/speech", response_format="pcm") + assert res.status_code == 200 + assert res.headers["Content-Type"] == "audio/pcm;rate=24000;encoding=signed-int;bits=16" + + # /tts keeps the float samples + ref = post_audio("/tts", response_format="pcm") + assert ref.headers["Content-Type"] == "audio/pcm;rate=24000;encoding=float;bits=32" + + samples = struct.unpack(f"<{len(res.content) // 2}h", res.content) + floats = struct.unpack(f"<{len(ref.content) // 4}f", ref.content) + assert len(samples) == len(floats) > 0 + assert any(v != 0 for v in samples) + for s, f in zip(samples, floats): + assert abs(s - max(-1.0, min(1.0, f)) * 32767) <= 1 + + +def test_speech_stream_pcm(): + server.start() + res = post_audio("/v1/audio/speech", response_format="pcm", stream_format="audio") + assert res.status_code == 200 + assert res.headers.get("Transfer-Encoding") == "chunked" + assert res.content == post_audio("/v1/audio/speech", response_format="pcm").content + + +def test_speech_stream_wav_has_unknown_size(): + server.start() + res = post_audio("/v1/audio/speech", stream_format="audio") + assert_wav(res, data_size=0xFFFFFFFF) + + +def test_speech_voice(): + server.start() + female = post_audio("/v1/audio/speech", voice="female") + male = post_audio("/v1/audio/speech", voice="male") + assert_wav(female) + assert_wav(male) + assert female.content != male.content + assert post_audio("/v1/audio/speech", voice={"id": "male"}).content == male.content + + +def test_speech_multipart(): + server.start() + fields = {k: (None, str(v)) for k, v in BODY.items()} + res = requests.post(server.make_url("/v1/audio/speech"), files=fields, timeout=DEFAULT_REQUEST_TIMEOUT) + assert_wav(res) + assert res.content == post_audio("/v1/audio/speech").content + + +@pytest.mark.parametrize("extra", [ + {"speed": 1.0}, + {"instructions": "Speak slowly."}, +]) +def test_speech_accepted_params(extra: dict): + server.start() + assert_wav(post_audio("/v1/audio/speech", **extra)) + + +@pytest.mark.parametrize("extra,message", [ + ({"response_format": "mp3"}, "response_format"), + ({"response_format": "opus"}, "response_format"), + ({"stream_format": "sse"}, "stream_format"), + ({"speed": 1.5}, "speed"), + ({"voice": "alloy"}, "unknown voice"), + ({"input": ""}, "input"), +]) +def test_speech_invalid_request(extra: dict, message: str): + server.start() + res = post_audio("/v1/audio/speech", **extra) + assert res.status_code == 400 + assert message in res.json()["error"]["message"] + + +def test_speech_no_tts_model(): + global server + server = ServerPreset.tinyllama2() + server.start() + assert post_audio("/v1/audio/speech").status_code == 501 # ERROR_TYPE_NOT_SUPPORTED + + +def test_speech_router(): + global server + model = server.model_hf_repo + server = ServerPreset.router() + server.start() + assert_wav(post_audio("/v1/audio/speech", model=model)) diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index c1534fd72..af69fafe0 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -695,6 +695,20 @@ class ServerPreset: server.seed = 42 return server + @staticmethod + def tinylfm3omni() -> ServerProcess: + server = ServerProcess() + server.offline = True # will be downloaded by load_all() + # mmproj is already provided by HF registry API + server.model_hf_file = None + server.model_hf_repo = "ggml-org/tinyLFM3-Omni-for-testing-gguf:Q2_K" + server.model_alias = "tinylfm3omni" + server.n_ctx = 1024 + server.n_batch = 512 + server.n_slots = 1 + server.seed = 42 + return server + @staticmethod def router() -> ServerProcess: server = ServerProcess()