* feat: support PocketTTS voices with no voice cloning The kyutai/pocket-tts-without-voice-cloning weights ship predefined voices as the backbone KV state after the voice prompt. The converter turns each state back into the prompt embeddings and stores them in the mmproj (voice.<name>), the zeroed voice encoder is left out, and the "voice" field of /tts and --tts-voice select one of them. A speaker reference is rejected when the model has no voice encoder. Assisted-by: Claude Fable 5.1 * convert : pocket-tts voices never fail the conversion, log the drift of the state The voice states shipped by Kyutai are not always the ones the pack's weights give for the prompt: italian, german and spanish match within the bf16 rounding, english_2026-04 is 2% off and french 20% (its layer 0 state comes from other weights), while the reference loads them as is and they still sound like the same voice. The converter now re-prompts the backbone with the recovered embeddings, compares the whole state and logs the distance per voice, warning above 5%. A state that does not belong to the weights at all is skipped, so the packs of kyutai/pocket-tts convert too. Assisted-by: Claude Fable 5.1 * mtmd : list the predefined voices, expose them in GET /props mtmd_gen_audio_info gains n_voices and mtmd_gen_audio_voice_name() the names, with the voices worded as a feature of any audio generation model. The server lists them under "voices" in /props when the model ships some. Assisted-by: Claude Fable 5.1 * server : an unknown voice is a bad request A model with predefined voices answers 400 naming the voice instead of 500 "failed to process TTS prompt". A model without them ignores "voice" with a warning, as the other audio generation models do. Assisted-by: Claude Fable 5.1 * refactor api * fix conversion * small renaming * fix lint * tts : a model with predefined voices speaks with the first one by default Without a voice prompt such a model only produces near silence. When neither a voice nor a speaker reference is given, llama-tts and the server now take the first predefined voice and say so in the log, as the LFM3-Audio template does with its default voice. Assisted-by: Claude Fable 5.1 * mtmd : one comment per voice accessor, converter states the rule only The clip.h comment described clip_gen_get_voice but sat above all three functions; the converter comment quoted the drift measured per pack, which goes stale with the next upload of the weights. Assisted-by: Claude Fable 5.1 * clean up redundant code and comments * mtmd: match the LFM3-Audio voice name case-insensitively The tts chat template lowers the voice name, so Male speaks with the male voice; the voice lookup lowers it the same way and names the bitmap after the canonical voice. --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
179 lines
6.5 KiB
C++
179 lines
6.5 KiB
C++
#pragma once
|
|
|
|
#include "server-http.h"
|
|
#include "server-task.h"
|
|
#include "server-queue.h"
|
|
|
|
#include "json.h"
|
|
|
|
#include <cstddef>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <set>
|
|
|
|
struct server_context_impl; // private implementation
|
|
|
|
struct server_context_meta {
|
|
std::string build_info;
|
|
std::string model_name;
|
|
std::set<std::string> model_aliases;
|
|
std::set<std::string> model_tags;
|
|
std::string model_path;
|
|
std::vector<std::string> model_output_modalities; // output modalities for GET /models
|
|
bool has_mtmd;
|
|
bool has_inp_image;
|
|
bool has_inp_audio;
|
|
bool has_inp_video;
|
|
bool has_cap_chat;
|
|
bool has_cap_tts;
|
|
bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream()
|
|
std::vector<std::string> tts_voices; // predefined voices of the audio generation model
|
|
json json_ui_settings;
|
|
int slot_n_ctx;
|
|
enum llama_pooling_type pooling_type;
|
|
|
|
// chat params
|
|
server_chat_params & chat_params;
|
|
json chat_template_caps;
|
|
|
|
// tokens
|
|
std::string bos_token_str;
|
|
std::string eos_token_str;
|
|
llama_token fim_pre_token;
|
|
llama_token fim_sub_token;
|
|
llama_token fim_mid_token;
|
|
llama_token fim_pad_token;
|
|
llama_token fim_rep_token;
|
|
llama_token fim_sep_token;
|
|
|
|
// sampling
|
|
std::vector<llama_logit_bias> logit_bias_eog;
|
|
|
|
// model meta
|
|
enum llama_vocab_type model_vocab_type;
|
|
int32_t model_vocab_n_tokens;
|
|
int32_t model_n_ctx_train;
|
|
int32_t model_n_embd_inp;
|
|
uint64_t model_n_params;
|
|
uint64_t model_size;
|
|
std::string model_ftype;
|
|
};
|
|
|
|
using server_state_callback_t = std::function<void(server_state, json /* payload */)>;
|
|
|
|
struct server_context {
|
|
std::unique_ptr<server_context_impl> impl;
|
|
|
|
server_context();
|
|
~server_context();
|
|
|
|
// load the model and initialize llama_context
|
|
// returns true on success
|
|
bool load_model(common_params & params);
|
|
|
|
// this function will block main thread until termination
|
|
void start_loop();
|
|
|
|
// terminate main loop (will unblock start_loop)
|
|
void terminate();
|
|
|
|
// get the underlaying llama_context, can return nullptr if sleeping
|
|
// not thread-safe, should only be used from the main thread
|
|
llama_context * get_llama_context() const;
|
|
|
|
// get a new response reader, used by CLI application
|
|
server_response_reader get_response_reader();
|
|
|
|
// get server metadata (read-only), can only be called after load_model()
|
|
// not thread-safe, should only be used from the main thread
|
|
server_context_meta get_meta() const;
|
|
|
|
// note: must be set before load_model() is called
|
|
void set_state_callback(server_state_callback_t callback);
|
|
};
|
|
|
|
|
|
// forward declarations
|
|
struct server_res_generator;
|
|
|
|
struct server_routes {
|
|
server_routes(const common_params & params, server_context & ctx_server);
|
|
|
|
void init_routes();
|
|
|
|
// note: this is not thread-safe and can only when ctx_http.is_ready is false
|
|
void update_meta(const server_context & ctx_server) {
|
|
this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());
|
|
}
|
|
|
|
// handlers using lambda function, so that they can capture `this` without `std::bind`
|
|
// they won't be called until ctx_http.is_ready is set to true
|
|
server_http_context::handler_t get_health;
|
|
server_http_context::handler_t get_metrics;
|
|
server_http_context::handler_t get_slots;
|
|
server_http_context::handler_t post_slots;
|
|
server_http_context::handler_t get_props;
|
|
server_http_context::handler_t post_props;
|
|
server_http_context::handler_t post_infill;
|
|
server_http_context::handler_t post_completions;
|
|
server_http_context::handler_t post_completions_oai;
|
|
server_http_context::handler_t post_chat_completions;
|
|
server_http_context::handler_t post_chat_completions_tok;
|
|
server_http_context::handler_t post_control;
|
|
server_http_context::handler_t post_responses_oai;
|
|
server_http_context::handler_t post_responses_tok_oai;
|
|
server_http_context::handler_t post_transcriptions_oai;
|
|
server_http_context::ws_handler_t ws_realtime;
|
|
server_http_context::handler_t post_anthropic_messages;
|
|
server_http_context::handler_t post_anthropic_count_tokens;
|
|
server_http_context::handler_t post_apply_template;
|
|
server_http_context::handler_t get_models;
|
|
server_http_context::handler_t post_tokenize;
|
|
server_http_context::handler_t post_detokenize;
|
|
server_http_context::handler_t post_embeddings;
|
|
server_http_context::handler_t post_embeddings_oai;
|
|
server_http_context::handler_t post_rerank;
|
|
server_http_context::handler_t post_tts;
|
|
server_http_context::handler_t post_systemone;
|
|
server_http_context::handler_t get_lora_adapters;
|
|
server_http_context::handler_t post_lora_adapters;
|
|
|
|
// to be used in router mode
|
|
json get_model_info() const;
|
|
|
|
private:
|
|
std::unique_ptr<server_res_generator> handle_completions_impl(
|
|
const server_http_req & req,
|
|
server_task_type type,
|
|
const json & data,
|
|
const std::vector<raw_buffer> & files,
|
|
task_response_type res_type,
|
|
const common_chat_session & chat_session = {});
|
|
std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
|
|
std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
|
|
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
|
|
std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
|
|
std::unique_ptr<server_res_generator> handle_rerank_impl(const server_http_req & req);
|
|
std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
|
|
|
|
// using unique_ptr to allow late initialization of const
|
|
std::unique_ptr<const server_context_meta> meta;
|
|
|
|
const common_params & params;
|
|
server_context_impl & ctx_server;
|
|
|
|
server_queue & queue_tasks;
|
|
server_response & queue_results;
|
|
std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
|
|
|
|
// cached responses, to be used during sleep
|
|
std::mutex mutex_cache;
|
|
json cached_models = nullptr;
|
|
json cached_props = nullptr;
|
|
server_metrics cached_metrics;
|
|
// set when a scrape during sleep already reported the throughput buckets
|
|
bool should_reset_buckets = false;
|
|
// call right before sleep to update the cached responses
|
|
void update_cached_responses(bool is_sleeping);
|
|
};
|