diff --git a/common/common.cpp b/common/common.cpp index b0ddfcb91..6ef3b785e 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1235,6 +1235,9 @@ common_gguf_info common_get_gguf_info(const std::string & fname) { info.n_ctx_train = gguf_get_val_u32(gguf_ctx.get(), ctx_id); } + // same key as llama_model_chat_template(model, nullptr) + info.has_chat_tmpl = gguf_find_key(gguf_ctx.get(), "tokenizer.chat_template") >= 0; + return info; } diff --git a/common/common.h b/common/common.h index b089abd7e..70d6a671f 100644 --- a/common/common.h +++ b/common/common.h @@ -985,6 +985,7 @@ common_decision_type common_get_decision_type(const struct llama_model * model); struct common_gguf_info { common_decision_type decision_type = COMMON_DECISION_TYPE_UNKNOWN; // UNKNOWN if the file is missing, unreadable, or invalid uint32_t n_ctx_train = 0; // 0 if unknown + bool has_chat_tmpl = false; // has the default tokenizer.chat_template }; common_gguf_info common_get_gguf_info(const std::string & fname); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index df61f82f0..e4750e52e 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -4278,8 +4278,9 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params struct clip_cap clip_get_cap(const char * fname) { clip_cap res; clip_model_loader loader(fname, /* skip_tensors= */ true); - res.has_vision = loader.has_vision; - res.has_audio = loader.has_audio; + res.has_vision = loader.has_vision; + res.has_audio = loader.has_audio; + res.has_gen_audio = loader.has_gen_audio; return res; } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 19db47608..cdbaeae0d 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -148,5 +148,6 @@ std::map clip_get_mem_usage(const struct clip_ctx * struct clip_cap { bool has_vision; bool has_audio; + bool has_gen_audio; }; struct clip_cap clip_get_cap(const char * fname); diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 5fb7ea9a0..47493d906 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -50,29 +50,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri return LLAMA_TOKEN_NULL; } +static void put_bytes(std::vector & buf, const void * p, size_t n) { + const char * c = (const char *) p; + buf.insert(buf.end(), c, c + n); +} + +// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe +static void write_wav16_header(std::vector & buf, uint32_t data_sz, int32_t rate) { + const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz; + const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2; + const uint16_t fmt = 1, ch = 1, align = 2, bits = 16; + const uint32_t rate32 = (uint32_t) rate; + put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4); + put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4); + put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4); + put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2); + put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4); +} + +static void append_wav16_pcm(std::vector & buf, const float * pcm, size_t n) { + for (size_t i = 0; i < n; i++) { + int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f); + put_bytes(buf, &s, 2); + } +} + static bool write_wav16(std::vector & buf, const std::vector & pcm, int32_t rate) { // RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) { return false; } - const uint32_t data_sz = (uint32_t) (pcm.size() * 2); - const uint32_t riff_sz = 36 + data_sz; - const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2; - const uint16_t fmt = 1, ch = 1, align = 2, bits = 16; - const uint32_t rate32 = (uint32_t) rate; - auto put = [&](const void * p, size_t n) { - const char * c = (const char *) p; - buf.insert(buf.end(), c, c + n); - }; - put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4); - put("fmt ", 4); put(&fmt_sz, 4); - put(&fmt, 2); put(&ch, 2); put(&rate32, 4); - put(&byte_rate, 4); put(&align, 2); put(&bits, 2); - put("data", 4); put(&data_sz, 4); - for (float v : pcm) { - int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f); - put(&s, 2); - } + write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate); + append_wav16_pcm(buf, pcm.data(), pcm.size()); return true; } @@ -92,6 +101,8 @@ public: // set out_stop on end-of-speech, h_state_out must be null if no frame is generated virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0; virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0; + // forces any buffered codes through code2wav now, regardless of window_frames + virtual int32_t flush() = 0; protected: llama_context * lctx; @@ -121,6 +132,9 @@ public: prompt_batch.reset(); n_prompt = 0; prompt_pos = 0; + stream = false; + pcm_sent = 0; + wav_header_sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { @@ -208,6 +222,7 @@ public: top_p = inp->top_p > 0 ? inp->top_p : def.top_p; seed = inp->seed; out_type = inp->out_type; + stream = inp->stream; // the prompt above holds the whole text stream up to tts_eos, so every generated // frame adds tts_pad on top of the codes embedding @@ -301,31 +316,60 @@ public: } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { - if (!flush_gen_wav()) { - return 1; + *out_sample_rate = info.sample_rate; + + if (!stream) { + // one-shot call: force out whatever's left, regardless of window_frames + if (!flush_gen_wav()) { + return 1; + } + if (out_n_samples) { + *out_n_samples = (int64_t) audio_pcm.size(); + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) audio_pcm.data(); + *out_data_len = audio_pcm.size() * sizeof(float); + return 0; + } + out_buf.clear(); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; } - *out_sample_rate = info.sample_rate; + // streaming: only return audio produced since the previous call + const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { - *out_n_samples = (int64_t) audio_pcm.size(); + *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { - *out_data = (const char *) audio_pcm.data(); - *out_data_len = audio_pcm.size() * sizeof(float); + *out_data = (const char *) (audio_pcm.data() + pcm_sent); + *out_data_len = n_new * sizeof(float); + pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); - if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { - LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); - return 1; + if (!wav_header_sent) { + write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); + wav_header_sent = true; } + append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); + pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } + int32_t flush() override { + return flush_gen_wav() ? 0 : 1; + } + private: bool ensure_cache() { if (specials_ok) { @@ -369,7 +413,7 @@ private: LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); return false; } - const std::string marker = mtmd_default_marker(); + const std::string marker = mtmd_get_marker(mctx); mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; mtmd_input_chunks * chunks = mtmd_input_chunks_init(); const mtmd_bitmap * bptr = bitmap; @@ -455,6 +499,9 @@ private: std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; + bool stream = false; + size_t pcm_sent = 0; // samples already returned by get_output() + bool wav_header_sent = false; }; // settings that only live in the reference's per-pack yaml, not in the checkpoint @@ -506,6 +553,9 @@ public: chunk_idx = 0; n_voice_pos = 0; chunk_budget = 0; + stream = false; + pcm_sent = 0; + wav_header_sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { @@ -575,6 +625,7 @@ public: seed = inp->seed; out_type = inp->out_type; + stream = inp->stream; return 0; } @@ -662,31 +713,60 @@ public: } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { - if (!flush_gen_wav()) { - return 1; + *out_sample_rate = info.sample_rate; + + if (!stream) { + // one-shot call: force out whatever's left, regardless of window_frames + if (!flush_gen_wav()) { + return 1; + } + if (out_n_samples) { + *out_n_samples = (int64_t) audio_pcm.size(); + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) audio_pcm.data(); + *out_data_len = audio_pcm.size() * sizeof(float); + return 0; + } + out_buf.clear(); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; } - *out_sample_rate = info.sample_rate; + // streaming: only return audio produced since the previous call + const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { - *out_n_samples = (int64_t) audio_pcm.size(); + *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { - *out_data = (const char *) audio_pcm.data(); - *out_data_len = audio_pcm.size() * sizeof(float); + *out_data = (const char *) (audio_pcm.data() + pcm_sent); + *out_data_len = n_new * sizeof(float); + pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); - if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { - LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); - return 1; + if (!wav_header_sent) { + write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); + wav_header_sent = true; } + append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); + pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } + int32_t flush() override { + return flush_gen_wav() ? 0 : 1; + } + private: bool ensure_cache() { if (specials_ok) { @@ -907,7 +987,7 @@ private: LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n"); return false; } - const std::string marker = mtmd_default_marker(); + const std::string marker = mtmd_get_marker(mctx); mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; mtmd_input_chunks * chunks = mtmd_input_chunks_init(); const mtmd_bitmap * bptr = bitmap; @@ -989,6 +1069,9 @@ private: std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; + bool stream = false; + size_t pcm_sent = 0; // samples already returned by get_output() + bool wav_header_sent = false; }; static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { @@ -1017,11 +1100,20 @@ void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) { } void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { - if (ctx->pipeline) { + if (ctx && ctx->pipeline) { ctx->pipeline->reset(); } } +struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) { + mtmd_helper_gen_audio_inp inp{}; + inp.top_k = 50; + inp.top_p = 1.0f; + inp.seed = UINT32_MAX; // random + inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + return inp; +} + int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) { if (!ctx->pipeline) { LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n"); @@ -1058,3 +1150,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * } return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples); } + +int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) { + if (!ctx->pipeline) { + return 1; + } + return ctx->pipeline->flush(); +} diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 7436230f0..acbfefcba 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -204,6 +204,7 @@ enum mtmd_helper_gen_audio_outtype { MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono }; struct mtmd_helper_gen_audio_inp { + bool stream; // if true, output() must be called after each step_gen() llama_seq_id seq_id; const char * prompt; @@ -219,6 +220,8 @@ struct mtmd_helper_gen_audio_inp { enum mtmd_helper_gen_audio_outtype out_type; }; +MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void); + MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( struct llama_context * lctx, struct mtmd_context * mctx); @@ -250,6 +253,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen( // out_data valid until next get_output() or reset() call // out_n_samples (optional, can be NULL) receives the number of generated PCM samples +// if inp->stream is true: returns only audio produced since the previous call, and +// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet MTMD_API int32_t mtmd_helper_gen_audio_get_output( mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate, @@ -257,6 +262,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output( size_t * out_data_len, int64_t * out_n_samples); +// forces any buffered codes through code2wav now, regardless of window_frames; +// call once when generation has ended, before the last get_output() in stream mode +MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx); + #ifdef __cplusplus } // extern "C" #endif @@ -283,8 +292,41 @@ struct mtmd_helper_gen_audio_deleter { }; using gen_audio_ptr = std::unique_ptr; struct gen_audio { + + // sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp + struct inp { + mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default(); + std::string prompt_str; + std::string lang_str; + mtmd::bitmap_ptr speaker_ref_ptr; + + inp() = default; + inp(inp &&) = default; + inp & operator=(inp &&) = default; + inp(const inp &) = delete; + inp & operator=(const inp &) = delete; + + void set_prompt (std::string p) { prompt_str = std::move(p); } + void set_lang (std::string l) { lang_str = std::move(l); } + void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); } + + // pointers are only valid as long as *this is alive + const mtmd_helper_gen_audio_inp * get() { + data.prompt = prompt_str.c_str(); + data.prompt_len = prompt_str.size(); + data.lang = lang_str.empty() ? nullptr : lang_str.c_str(); + data.speaker_ref = speaker_ref_ptr.get(); + return &data; + } + }; + gen_audio_ptr ctx; - gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {} + void init(struct llama_context * lctx, struct mtmd_context * mctx) { + ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx)); + } + bool valid() const { + return ctx.get() != nullptr; + } void reset() { mtmd_helper_gen_audio_reset(ctx.get()); } @@ -300,6 +342,9 @@ struct gen_audio { int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) { return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples); } + int32_t flush() { + return mtmd_helper_gen_audio_flush(ctx.get()); + } }; } // namespace mtmd_helper diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index b3ac1979e..ffd35b2bb 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -2846,10 +2846,11 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) { mtmd_caps cap; cap.inp_audio = tmp.has_audio; cap.inp_vision = tmp.has_vision; + cap.gen_audio = tmp.has_gen_audio; return cap; } catch (const std::exception & e) { LOG_ERR("%s: failed to get capabilities from file '%s': %s\n", __func__, fname, e.what()); - return mtmd_caps{ false, false }; + return mtmd_caps{ false, false, false }; } } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index df24c5656..c740f1e9f 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -360,6 +360,7 @@ MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); struct mtmd_caps { bool inp_vision; bool inp_audio; + bool gen_audio; }; MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); diff --git a/tools/server/README.md b/tools/server/README.md index 2977fcd4d..2886bca15 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -802,6 +802,50 @@ curl http://127.0.0.1:8012/v1/rerank \ }' | jq ``` +### POST `/tts`: Generate speech audio from text + +Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info, see [tts/README.md](../tts/README.md) + +*Options:* + +`input`: The text to speak (alias: `prompt`). + +`lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional. + +`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. + +`top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply. + +`repeat_penalty`: Repetition penalty applied to the backbone sampler over the whole generation (default `1.05`). Without this, the backbone can loop and re-generate the same utterance. + +`seed`: RNG seed for both the backbone sampler and the codec/vocoder (`-1` = random, the default). Set to a fixed value for reproducible output. + +`n_predict`: Max number of audio frames to generate. Defaults to `512`; generation normally stops earlier once the model emits an end-of-speech token. + +`response_format`: `wav` (default) or `pcm` (raw `float32` little-endian mono samples, no header; the response `Content-Type` carries the sample rate, e.g. `audio/pcm;rate=24000;encoding=float;bits=32`). + +`stream`: If `true`, the response is streamed as audio becomes available instead of waiting for the full generation to finish. WAV streaming writes an RFC-noncompliant header with an unknown (`0xFFFFFFFF`) size field, since the final length isn't known up front; most players and decoders (ffmpeg, VLC, ...) handle this by reading until EOF. + +Note: it's highly recommended to always provide a speaker reference voice; otherwise, the model's performance may be degraded. + +*Examples:* + +```shell +curl -X POST http://127.0.0.1:9931/tts \ + -H "Content-Type: application/json" \ + -d '{"input": "Hello, this is a test."}' \ + -o output.wav +``` + +With a speaker reference uploaded as a file (`multipart/form-data`), instead of base64-encoding it into the JSON body: + +```shell +curl -X POST http://127.0.0.1:9931/tts \ + -F "input=Hello, this is a test." \ + -F "speaker_ref=@/path/to/speaker-reference.wav;type=audio/wav" \ + -o output.wav +``` + ### POST `/infill`: For code infilling. Takes a prefix and a suffix and returns the predicted completion as stream. @@ -1262,11 +1306,12 @@ Each object in `data` has an `architecture` object. It has two string arrays: - `input_modalities` lists what the model can read. It always has `text`, plus each media type that the model supports. - `output_modalities` lists what the model can produce. -One output value is special: +Some output values are special: | Value | Meaning | |---|---| | `decisions` | The model is a native decision model. Serve it with [`/v1/systemone`](#post-v1systemone-typesafe-compatible-system-one-api). | +| `audio` | The model can generate speech. Serve it with [`/tts`](#post-tts-generate-speech-audio-from-text). A TTS-only model has `["audio"]` without `text`. | A language model that classifies with prompts does not get `decisions`. Only native decision models do. @@ -2054,7 +2099,7 @@ Note: - If a model is not running, it will be added or updated according to the source 2. When the model is loaded, the info from `/v1/models` is forwarded to router's `/v1/models`. This includes metadata about the model and the runtime instance. -Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps: +Each object in `data` has the same `architecture` object as [`GET /v1/models`](#get-v1models-openai-compatible-model-info-api) of a direct server. The server computes both arrays offline. It does not load the model, download files, or run inference. `output_modalities` comes from the GGUF metadata, plus `audio` when the projector file can generate speech. `input_modalities` comes from the projector file. A native decision model shows `decisions` before its first load, after unload, and while it sleeps: ```json { diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 075687602..baaca58eb 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -202,7 +202,8 @@ const char * get_media_marker() { // model output modalities // -std::vector server_model_output_modalities(common_decision_type decision_type) { +std::vector server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts) { + std::vector out; switch (decision_type) { case COMMON_DECISION_TYPE_OPENJEV: case COMMON_DECISION_TYPE_LEV: @@ -216,13 +217,27 @@ std::vector server_model_output_modalities(common_decision_type dec case COMMON_DECISION_TYPE_LFM2_D1: case COMMON_DECISION_TYPE_LFM2_D1_OMNI: case COMMON_DECISION_TYPE_H2O_LIGHTNING: - return {"decisions"}; + out = {"decisions"}; + break; case COMMON_DECISION_TYPE_GRANITE_SWITCH: - return {"text", "decisions"}; + out = {"text", "decisions"}; + break; default: - // fallback when there is no decision type or the metadata is bad - return {"text"}; + // a tts-only model cannot generate text + if (has_cap_chat || !has_cap_tts) { + // fallback when there is no decision type or the metadata is bad + out = {"text"}; + } + break; } + if (has_cap_tts) { + out.push_back("audio"); + } + return out; +} + +std::vector server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts) { + return server_model_output_modalities(common_get_decision_type(model), has_cap_chat, has_cap_tts); } json server_model_architecture_json( diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 277752122..538e008e9 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -114,7 +114,10 @@ const char * get_media_marker(); // // output modalities for architecture.output_modalities in GET /models -std::vector server_model_output_modalities(common_decision_type decision_type); +// has_cap_chat: false for a tts-only model, has_cap_tts: the model can generate audio +std::vector server_model_output_modalities(common_decision_type decision_type, bool has_cap_chat, bool has_cap_tts); +// same, with the decision type read from a loaded model +std::vector server_model_output_modalities(const llama_model * model, bool has_cap_chat, bool has_cap_tts); // architecture object of GET /models; shared by the direct server and the router json server_model_architecture_json( diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 6d8f8e7be..b8e132d7b 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -20,6 +20,7 @@ #include "speculative.h" #include "mtmd.h" #include "mtmd-helper.h" +#include "base64.hpp" #include #include @@ -43,6 +44,14 @@ constexpr int HTTP_POLLING_SECONDS = 1; static common_speculative_output_limits server_output_limits(const common_params & params) { + if (!params.mmproj.path.empty()) { + const auto mcaps = mtmd_get_cap_from_file(params.mmproj.path.c_str()); + if (mcaps.gen_audio) { + // some TTS models decode with embeddings output for the whole batch + return { params.n_batch, params.n_batch }; + } + } + if (params.embedding || (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) { return { params.n_batch, 1 }; @@ -257,6 +266,27 @@ struct server_slot { mtmd_context * mctx = nullptr; mtmd::batch_ptr mbatch = nullptr; + struct tts_ctx { + mtmd_helper::gen_audio ctx; + std::vector h_state_prompt; // the first h_state after step_prompt done + const float * h_state; + llama_token sampled; + int32_t n_decoded; + bool stop; + bool is_supported() const { + return ctx.valid(); + } + void reset() { + ctx.reset(); + h_state_prompt.clear(); + h_state = nullptr; + sampled = LLAMA_TOKEN_NULL; + n_decoded = 0; + stop = false; + } + }; + tts_ctx tts; + // speculative decoding common_speculative * spec; @@ -421,6 +451,9 @@ struct server_slot { // clear multimodal state mbatch.reset(); + + tts.reset(); + audio_stream.reset(); audio_wait = false; inp_embd.clear(); @@ -622,6 +655,11 @@ struct server_slot { prompt_clear(); } + // mtmd_helper always re-eval the whole prompt + if (task->type == SERVER_TASK_TYPE_TTS) { + prompt_clear(); + } + callback_on_reset(*this); reset(); @@ -975,6 +1013,14 @@ public: mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default(); const llama_vocab * vocab = nullptr; + bool has_cap_tts() const { + return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE; + } + + bool has_cap_chat() const { + return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx); + } + server_decision_context decision; server_queue queue_tasks; @@ -1463,6 +1509,10 @@ private: slot.mctx = mctx; slot.prompt.tokens.has_mtmd = mctx != nullptr; + if (has_cap_tts()) { + slot.tts.ctx.init(ctx_tgt, mctx); + } + SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx); slot.callback_on_release = [this](int id_slot) { @@ -1944,6 +1994,20 @@ private: SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str()); + if (task.type == SERVER_TASK_TYPE_TTS) { + GGML_ASSERT(has_cap_tts()); // should already checked in route handler + if (!slot.tts.is_supported()) { + slot.tts.ctx.init(ctx_tgt, slot.mctx); + } + // mtmd_helper always re-eval the whole prompt + slot.prompt_clear(); + task.tts_inp.data.seq_id = slot.id; + if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) { + send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER); + return false; + } + } + // initialize samplers if (task.need_sampling()) { try { @@ -1961,6 +2025,9 @@ private: // TODO: getting pre sampling logits is not yet supported with backend sampling use_backend_sampling &= !need_pre_sample_logits; + // TODO: check verify if this actually works with TTS + use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS; + // TODO: tmp until backend sampling is fully implemented if (use_backend_sampling) { llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get())); @@ -1986,9 +2053,13 @@ private: slot.task = std::make_unique(std::move(task)); - slot.state = slot.task->is_child() - ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt - : SLOT_STATE_STARTED; + if (slot.task->type == SERVER_TASK_TYPE_TTS) { + slot.state = SLOT_STATE_PROCESSING_PROMPT; + } else { + slot.state = slot.task->is_child() + ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt + : SLOT_STATE_STARTED; + } // no prompt, the rows come from the audio stream, which audio controls reach from now on if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) { @@ -2268,6 +2339,18 @@ private: queue_results.send(std::move(res)); } + void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) { + auto res = std::make_unique(); + + res->id = slot.task->id; + res->index = slot.task->index; + res->sample_rate = sample_rate; + res->audio.assign(data, data_len); + res->final = final; + + queue_results.send(std::move(res)); + } + void send_final_response(server_slot & slot) { auto res = std::make_unique(); @@ -2872,6 +2955,7 @@ private: case SERVER_TASK_TYPE_TRANSCRIBE_RT: case SERVER_TASK_TYPE_EMBEDDING: case SERVER_TASK_TYPE_RERANK: + case SERVER_TASK_TYPE_TTS: case SERVER_TASK_TYPE_DECISION: { // special case: if input is provided via CLI, tokenize it first @@ -3397,6 +3481,14 @@ private: return; } + // note: TTS slots bypass the shared batch entirely + try { + process_tts_slots(); + } catch (const std::exception & e) { + SRV_ERR("process_tts_slots() failed: %s\n", e.what()); + abort_all_slots("process_tts_slots() failed: " + std::string(e.what())); + } + GGML_ASSERT(batch.slot_batched || batch.size() == 0); if (batch.slot_batched) { @@ -3459,10 +3551,89 @@ private: } } + void process_tts_slots() { + iterate(slots, [&](server_slot & slot) { + if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) { + return; + } + + llama_set_embeddings(ctx_tgt, true); + + if (slot.state == SLOT_STATE_PROCESSING_PROMPT) { + const int32_t ret = slot.tts.ctx.step_prompt(llama_n_batch(ctx_tgt)); + if (ret < 0) { + send_error(slot, "TTS prompt processing failed", ERROR_TYPE_SERVER); + slot.release(); + } else if (ret == 0) { + // done prompt, do sample and save h_state_prompt + slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1); + common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true); + const float * h_embd = llama_get_embeddings_ith(ctx_tgt, -1); + slot.tts.h_state_prompt.assign(h_embd, h_embd + llama_model_n_embd(model_tgt)); + slot.tts.h_state = slot.tts.h_state_prompt.data(); + slot.state = SLOT_STATE_GENERATING; + } + return; + } + + const int32_t n_predict = slot.n_predict_max > 0 ? slot.n_predict_max : 512; + if (slot.tts.stop || slot.tts.n_decoded >= n_predict) { + int32_t sample_rate = 0; + const char * data = nullptr; + size_t data_len = 0; + // generation truly ends here: force out any sub-window remainder still buffered + if (slot.tts.ctx.flush() != 0 || slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) { + send_error(slot, "failed to finalize TTS output", ERROR_TYPE_SERVER); + } else { + send_tts_result(slot, sample_rate, data, data_len, true); + } + slot.release(); + return; + } + + const float * h_state_next = nullptr; + bool stop = false; + if (slot.tts.ctx.step_gen(slot.tts.sampled, slot.tts.h_state, &h_state_next, &stop) != 0) { + send_error(slot, "TTS generation failed", ERROR_TYPE_SERVER); + slot.release(); + return; + } + if (h_state_next == nullptr) { + // end-of-speech without a new frame (e.g. pocket-tts eos head) + slot.tts.stop = true; + return; + } + slot.tts.h_state = h_state_next; + slot.tts.n_decoded++; + slot.tts.stop = stop; + + if (!stop) { + slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1); + common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true); + } + + if (slot.task->params.stream) { + int32_t sample_rate = 0; + const char * data = nullptr; + size_t data_len = 0; + if (slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) { + send_error(slot, "TTS streaming output failed", ERROR_TYPE_SERVER); + slot.release(); + } else if (data_len > 0) { + send_tts_result(slot, sample_rate, data, data_len, false); + } + } + }); + } + void pre_decode() { // apply context-shift if needed // TODO: simplify and improve iterate(slots, [&](server_slot & slot) { + if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) { + // TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch + return; + } if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) { if (!params_base.ctx_shift) { // this check is redundant (for good) @@ -3535,7 +3706,7 @@ private: // determine which slots are generating and drafting iterate(slots, [&](server_slot & slot) { - if (slot.state != SLOT_STATE_GENERATING) { + if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) { return; } @@ -3681,7 +3852,7 @@ private: return; // batch is full, skip remaining slots } - if (!slot.is_processing()) { + if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) { return; } @@ -4926,11 +5097,13 @@ server_context_meta server_context::get_meta() const { /* model_aliases */ impl->model_aliases, /* model_tags */ impl->model_tags, /* model_path */ impl->params_base.model.path, - /* model_output_modalities */ server_model_output_modalities(common_get_decision_type(impl->model_tgt)), + /* model_output_modalities */ server_model_output_modalities(impl->model_tgt, impl->has_cap_chat(), impl->has_cap_tts()), /* has_mtmd */ impl->mctx != nullptr, /* has_inp_image */ impl->chat_params.allow_image, /* has_inp_audio */ impl->chat_params.allow_audio, /* has_inp_video */ impl->chat_params.allow_video, + /* has_cap_chat */ impl->has_cap_chat(), + /* has_cap_tts */ impl->has_cap_tts(), /* has_audio_stream */ impl->mctx != nullptr && mtmd_support_audio_stream(impl->mctx), /* json_ui_settings */ impl->json_ui_settings, /* slot_n_ctx */ impl->n_ctx_slot(), @@ -5006,6 +5179,11 @@ std::unique_ptr server_routes::handle_completions_impl( res->set_req(&req); // will also set spipe if needed + if (!ctx_server.has_cap_chat()) { + res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + int32_t sse_ping_interval = params.sse_ping_interval; try { @@ -6175,6 +6353,171 @@ void server_routes::init_routes() { return res; }; + this->post_tts = [this](const server_http_req & req) { + auto res = create_response(); + res->set_req(&req); // will also set spipe if needed + + if (!ctx_server.has_cap_tts()) { + res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + + const auto info = mtmd_gen_audio_get_info(ctx_server.mctx); + + json body = json::parse(req.body); + + // multipart form fields arrive as strings, correct the types of the numeric and boolean params + for (const char * k : { "top_k", "n_predict", "max_tokens", "seed" }) { + if (body.contains(k) && body[k].is_string()) { + body[k] = std::stoll(body[k].get()); + } + } + for (const char * k : { "top_p", "repeat_penalty" }) { + if (body.contains(k) && body[k].is_string()) { + body[k] = std::stof(body[k].get()); + } + } + if (body.contains("stream") && body["stream"].is_string()) { + body["stream"] = body["stream"] == "true"; + } + + std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string())); + if (prompt.empty()) { + res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + + const std::string response_format = json_value(body, "response_format", std::string("wav")); + + server_task task(SERVER_TASK_TYPE_TTS); + server_schema::eval_tts_schema(params, task, body); + const bool stream = task.params.stream; + task.tts_inp.set_prompt(prompt); + task.tts_inp.set_lang(json_value(body, "lang", std::string())); + task.tts_inp.data.stream = stream; + task.tts_inp.data.out_type = response_format == "pcm" + ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM + : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + // note: -1 is clamped to 0 by the sampler, it will disable the penalty + task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512; + task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG + if (task.tts_inp.data.top_k > 0) { + task.params.sampling.top_k = task.tts_inp.data.top_k; + } + if (task.tts_inp.data.top_p > 0) { + task.params.sampling.top_p = task.tts_inp.data.top_p; + } + + // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64") + const unsigned char * speaker_ref_data = nullptr; + size_t speaker_ref_len = 0; + std::string speaker_ref_b64_decoded; + + auto speaker_ref_file = req.files.find("speaker_ref"); + if (speaker_ref_file != req.files.end()) { + speaker_ref_data = speaker_ref_file->second.data.data(); + speaker_ref_len = speaker_ref_file->second.data.size(); + } else { + std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); + if (!speaker_ref_b64.empty()) { + try { + speaker_ref_b64_decoded = base64::decode(speaker_ref_b64); + } catch (const std::exception &) { + res->error(format_error_response("\"speaker_ref_b64\" is not valid base64", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data(); + speaker_ref_len = speaker_ref_b64_decoded.size(); + } + } + + if (speaker_ref_len > 0) { + auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); + if (!wrapper.bitmap) { + res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); + } else { + SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n"); + } + + auto & rd = res->rd; + task.id = rd.get_new_id(); + rd.post_task(std::move(task)); + + // raw float32 LE mono samples; audio/L16 would imply 16-bit big-endian (RFC 2586) + const std::string content_type = response_format == "pcm" + ? "audio/pcm;rate=" + std::to_string(info.sample_rate) + ";encoding=float;bits=32" + : "audio/wav"; + + if (!stream) { + auto result = rd.next(req.should_stop); + if (!result) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } + if (result->is_error()) { + res->error(result->to_json()); + return res; + } + auto * tts_res = dynamic_cast(result.get()); + GGML_ASSERT(tts_res != nullptr); + res->status = 200; + res->content_type = content_type; + res->data = std::move(tts_res->audio); + return res; + } else { + auto first_result = rd.next(req.should_stop); + if (!first_result) { + GGML_ASSERT(req.should_stop()); + return res; // connection is closed + } + if (first_result->is_error()) { + res->error(first_result->to_json()); + return res; + } + auto * first_tts_res = dynamic_cast(first_result.get()); + GGML_ASSERT(first_tts_res != nullptr); + + res->status = 200; + res->content_type = content_type; + res->data = std::move(first_tts_res->audio); + bool is_done = first_tts_res->final; + + res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool { + // flush buffered audio before is_done - the first result can already be final + if (!res_this->data.empty()) { + output = std::move(res_this->data); + res_this->data.clear(); + return true; + } + if (is_done) { + return false; + } + if (res_this->should_stop()) { + return false; + } + + server_response_reader & rd = res_this->rd; + if (!rd.has_next()) { + return false; + } + auto result = rd.next([&res_this]() { return res_this->should_stop(); }); + if (!result || result->is_error()) { + return false; + } + auto * tts_res = dynamic_cast(result.get()); + GGML_ASSERT(tts_res != nullptr); + output = std::move(tts_res->audio); + is_done = tts_res->final; + return true; + }); + } + + return res; + }; + this->post_systemone = [this](const server_http_req & req) { auto res = create_response(); const auto & decision = ctx_server.decision; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index da75e5327..a579b0eca 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -24,6 +24,8 @@ struct server_context_meta { bool has_inp_image; bool has_inp_audio; bool has_inp_video; + bool has_cap_chat; + bool has_cap_tts; bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream() json json_ui_settings; int slot_n_ctx; @@ -155,6 +157,7 @@ struct server_routes { server_http_context::handler_t post_embeddings; server_http_context::handler_t post_embeddings_oai; server_http_context::handler_t post_rerank; + server_http_context::handler_t post_tts; server_http_context::handler_t post_systemone; server_http_context::handler_t get_lora_adapters; server_http_context::handler_t post_lora_adapters; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index c8e235129..d38549db5 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -521,27 +521,31 @@ void server_model_meta::update_caps(const common_params & base) { return; } - // read the output modalities and the trained context from the GGUF metadata - std::vector output_modalities = {"text"}; - if (!params.model.path.empty()) { - const common_gguf_info info = common_get_gguf_info(params.model.path); - output_modalities = server_model_output_modalities(info.decision_type); - n_ctx_train = info.n_ctx_train; - } - bool inp_image = false; bool inp_audio = false; + bool gen_audio = false; try { if (!params.no_mmproj && !params.mmproj.path.empty()) { mtmd_caps caps = mtmd_get_cap_from_file(params.mmproj.path.c_str()); inp_image = caps.inp_vision; inp_audio = caps.inp_audio; + gen_audio = caps.gen_audio; } } catch (const std::exception & e) { LOG_WRN("failed to read the multimodal capabilities of '%s': %s\n", name.c_str(), e.what()); // keep the output modalities from the GGUF metadata } + // read the output modalities and the trained context from the GGUF metadata + std::vector output_modalities = {"text"}; + if (!params.model.path.empty()) { + const common_gguf_info info = common_get_gguf_info(params.model.path); + // same rule as mtmd_helper_model_can_chat() + const bool can_chat = !gen_audio || info.has_chat_tmpl; + output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio); + n_ctx_train = info.n_ctx_train; + } + // offline discovery cannot see video; a loaded model reports it architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities); } diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 54e126aa0..1e2dce888 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -511,6 +511,58 @@ task_params eval_llama_cmpl_schema( return params; } +// +// TTS schema +// + +std::vector> make_tts_schema(server_task & task) { + std::vector> fields; + auto add = [&](field * f) { + fields.emplace_back(f); + }; + + add((new field_num("top_k", task.tts_inp.data.top_k)) + ->set_limits(0, INT32_MAX) + ->set_desc("Top-k for the acoustic code predictor, 0 to use the model default")); + + add((new field_num("top_p", task.tts_inp.data.top_p)) + ->set_limits(0.0f, 1.0f) + ->set_desc("Top-p for the acoustic code predictor, 0.0 to use the model default")); + + add((new field_num("seed", task.tts_inp.data.seed)) + ->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)")); + + add((new field_num("n_predict", task.params.n_predict)) + ->set_hard_limits(-1, INT32_MAX) + ->add_alias("max_tokens") + ->set_desc("Max number of audio frames to generate")); + + add((new field_num("repeat_penalty", task.params.sampling.penalty_repeat)) + ->set_desc("Repetition penalty applied to the backbone sampler over the whole generation")); + + add((new field_bool("stream", task.params.stream)) + ->set_desc("Stream the audio as it becomes available instead of waiting for the full generation")); + + return fields; +} + +void eval_tts_schema(const common_params & params_base, server_task & task, const json & data) { + // baseline defaults, individual requests can override them + // TODO @ngxson : change the default values based on model + task.params.sampling = params_base.sampling; + task.params.sampling.penalty_repeat = 1.05f; + task.params.n_predict = -1; + task.tts_inp.data.top_k = 0; + task.tts_inp.data.top_p = 0.0f; + task.tts_inp.data.seed = params_base.sampling.seed; + + field_eval_context ctx(task.params); + + for (const auto & f : make_tts_schema(task)) { + f->eval(ctx, data); + } +} + // // eval() implementations // diff --git a/tools/server/server-schema.h b/tools/server/server-schema.h index d0a81431b..542136a97 100644 --- a/tools/server/server-schema.h +++ b/tools/server/server-schema.h @@ -101,4 +101,12 @@ task_params eval_llama_cmpl_schema( const std::vector & logit_bias_eog, const json & data); +std::vector> make_tts_schema(server_task & task); + +// evaluates the /tts request params into task.params and task.tts_inp.data +void eval_tts_schema( + const common_params & params_base, + server_task & task, + const json & data); + } // namespace server_schema diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 2e513a33e..d7ac2a1dc 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1508,6 +1508,17 @@ json server_task_result_rerank::to_json() { }; } +// +// server_task_result_tts +// +json server_task_result_tts::to_json() { + return json { + {"sample_rate", sample_rate}, + {"n_bytes", audio.size()}, + {"final", final}, + }; +} + // // server_task_result_decision // diff --git a/tools/server/server-task.h b/tools/server/server-task.h index de39ce3b8..810cefbfe 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -10,6 +10,7 @@ // TODO: prevent including the whole server-common.h as we only use server_tokens #include "server-common.h" +#include "mtmd-helper.h" enum server_task_type { @@ -29,6 +30,7 @@ enum server_task_type { SERVER_TASK_TYPE_SLOT_ERASE, SERVER_TASK_TYPE_GET_LORA, SERVER_TASK_TYPE_SET_LORA, + SERVER_TASK_TYPE_TTS, }; // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common @@ -219,6 +221,9 @@ struct server_task { // used by SERVER_TASK_TYPE_SET_LORA std::map set_lora; // mapping adapter ID -> scale + // used by SERVER_TASK_TYPE_TTS + mtmd_helper::gen_audio::inp tts_inp; + server_task() = default; server_task(server_task_type type) : type(type) {} @@ -256,6 +261,7 @@ struct server_task { switch (type) { case SERVER_TASK_TYPE_COMPLETION: case SERVER_TASK_TYPE_INFILL: + case SERVER_TASK_TYPE_TTS: case SERVER_TASK_TYPE_TRANSCRIBE_RT: return true; default: @@ -526,6 +532,16 @@ struct server_task_result_embd : server_task_result { json to_json_oaicompat(); }; +struct server_task_result_tts : server_task_result { + std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type) + int32_t sample_rate = 0; + bool final = false; // true for the last chunk of a request + + virtual bool is_stop() override { return final; } + + virtual json to_json() override; +}; + struct server_task_result_rerank : server_task_result { float score = -1e6; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 41f29ded5..ce44405b8 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -247,6 +247,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child & routes.post_control = models_routes->proxy_post; routes.post_responses_oai = models_routes->proxy_post; routes.post_transcriptions_oai = models_routes->proxy_post; + routes.post_tts = models_routes->proxy_post; routes.ws_realtime = models_routes->proxy_ws; routes.post_anthropic_messages = models_routes->proxy_post; routes.post_anthropic_count_tokens = models_routes->proxy_post; @@ -293,6 +294,7 @@ int llama_server(common_params & params, int argc, char ** argv, server_child & ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai)); ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai)); + ctx_http.post("/tts", ex_wrapper(routes.post_tts)); ctx_http.ws ("/v1/realtime", routes.ws_realtime); ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API ctx_http.post("/infill", ex_wrapper(routes.post_infill)); diff --git a/tools/tts/README.md b/tools/tts/README.md index 1b08d5ef3..55b99932c 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -12,6 +12,8 @@ Simple usage: llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output out.wav ``` +Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded. + Common params: - Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. - `-n ` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 6fd193632..aea794e65 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -111,8 +111,9 @@ int main(int argc, char ** argv) { speaker_bitmap.reset(wrapper.bitmap); } - mtmd_helper::gen_audio gen(lctx, mctx.get()); - mtmd_helper_gen_audio_inp inp{}; + mtmd_helper::gen_audio gen; + gen.init(lctx, mctx.get()); + mtmd_helper_gen_audio_inp inp = mtmd_helper_gen_audio_inp_default(); inp.seq_id = 0; inp.prompt = params.prompt.c_str(); inp.prompt_len = params.prompt.size(); @@ -121,7 +122,6 @@ int main(int argc, char ** argv) { inp.top_k = params.sampling.top_k; inp.top_p = params.sampling.top_p; inp.seed = params.sampling.seed; - inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; // // stage 1: process prompt via backbone model, generate semantic representation