diff --git a/conversion/pockettts.py b/conversion/pockettts.py index 1c99e58cf..1b5c84111 100644 --- a/conversion/pockettts.py +++ b/conversion/pockettts.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, Iterable, TYPE_CHECKING import torch +import torch.nn.functional as F if TYPE_CHECKING: from torch import Tensor @@ -45,6 +46,102 @@ def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]: return {name: tuple(part[name].shape) for name in part.keys()} +def _has_voice_encoder(dir_model: Path) -> bool: + # without voice cloning, the mimi encoder weights are all zero + part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors") + with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part: + nonzero = {bool(part[name].mmap_bytes().any()) for name in part.keys() if name.startswith("mimi.encoder")} + if len(nonzero) != 1: + raise ValueError(f"{dir_model}: the mimi encoder is partly zeroed out") + return nonzero.pop() + + +def _load_safetensors(path: Path, prefixes: tuple[str, ...]) -> dict[str, Tensor]: + dtypes = {"F32": torch.float32, "BF16": torch.bfloat16, "I64": torch.int64} + with gguf.utility.SafetensorsLocal(path) as part: + return {name: torch.from_numpy(part[name].mmap_bytes()).view(dtypes[part[name].dtype]).reshape(part[name].shape) + for name in part.keys() if name.startswith(prefixes)} + + +def _rope_pairs(x: Tensor, n_head: int, rope_theta: float) -> Tensor: + # ggml NORM mode rope, x: [n_pos, n_embd] + n_pos, n_embd = x.shape + head_dim = n_embd // n_head + angle = torch.arange(n_pos, dtype=x.dtype)[:, None, None] * \ + torch.pow(rope_theta, -torch.arange(0, head_dim, 2, dtype=x.dtype) / head_dim) + x = x.view(n_pos, n_head, head_dim // 2, 2) + return torch.stack([x[..., 0] * angle.cos() - x[..., 1] * angle.sin(), + x[..., 0] * angle.sin() + x[..., 1] * angle.cos()], -1).reshape(n_pos, n_embd) + + +def _voice_prompt_embd(weights: dict[str, Tensor], kv: list[Tensor], n_head: int, rope_theta: float) -> tuple[Tensor, float]: + # the voice state only has K/V of each layer (kv[il] is [k/v, pos, n_embd]): layer 0 V gives each frame up to its scale, layer 1 V gives the scale + # returns the frames without bos_before_voice, and the max relative K/V error when the frames are prompted again + w = {name: t.double() for name, t in weights.items()} + kv = [t.double() for t in kv] + k0, v0, v1 = kv[0][0], kv[0][1], kv[1][1] + n_pos, n_embd = v0.shape + head_dim = n_embd // n_head + + def p(il: int, name: str) -> Tensor: + return w[f"flow_lm.transformer.layers.{il}.{name}"] + + def norm(x: Tensor, il: int, name: str) -> Tensor: + return F.layer_norm(x, (n_embd,), p(il, f"{name}.weight"), p(il, f"{name}.bias"), 1e-5) + + def heads(t: Tensor) -> Tensor: + return t.reshape(n_pos, n_head, head_dim).transpose(0, 1) + + def rows(t: Tensor) -> Tensor: + return t.transpose(0, 1).reshape(n_pos, n_embd) + + def attn(il: int, h: Tensor, k: Tensor, v: Tensor) -> Tensor: + q = _rope_pairs(h @ p(il, "self_attn.in_proj.weight").chunk(3)[0].T, n_head, rope_theta) + return rows(F.scaled_dot_product_attention(heads(q), heads(k), heads(v), is_causal=True)) @ p(il, "self_attn.out_proj.weight").T + + def ffn(il: int, x: Tensor) -> Tensor: + return F.gelu(norm(x, il, "norm2") @ p(il, "linear1.weight").T) @ p(il, "linear2.weight").T + + wv = p(0, "self_attn.in_proj.weight").chunk(3)[2] + h = torch.linalg.solve(wv, v0.T).T # norm1(x) + z = (h - p(0, "norm1.bias")) / p(0, "norm1.weight") # (x - mean(x)) / std(x) + + # fit z on [speaker_proj, 1] to get the latent direction back + proj = w["flow_lm.speaker_proj_weight"] + fit = torch.linalg.lstsq(torch.cat([proj, torch.ones(n_embd, 1, dtype=proj.dtype)], 1), z[1:].T).solution + frames = (proj @ fit[:-1]).T + + a0 = attn(0, h, k0, v0)[1:] + wv1 = p(1, "self_attn.in_proj.weight").chunk(3)[2] + + def err(scale: Tensor) -> Tensor: # per frame relative error of layer 1 V + x = scale[:, None] * frames + a0 + x = x + ffn(0, x) + return (norm(x, 1, "norm1") @ wv1.T - v1[1:]).norm(dim=1) / v1[1:].norm(dim=1) + + grid = torch.logspace(-3, 3, 49, dtype=torch.float64) + best = torch.stack([err(s.expand(n_pos - 1)) for s in grid]).argmin(0) + lo, hi = grid[(best - 1).clamp(min=0)], grid[(best + 1).clamp(max=len(grid) - 1)] + for _ in range(40): # golden section search, all frames at once + m1, m2 = hi - 0.618 * (hi - lo), lo + 0.618 * (hi - lo) + left = err(m1) < err(m2) + lo, hi = torch.where(left, lo, m1), torch.where(left, m2, hi) + frames = (lo + hi)[:, None] / 2 * frames + + # prompt the frames again and compare K/V with the shipped state + x = torch.cat([w["flow_lm.bos_before_voice"].reshape(1, n_embd), frames]) + drift = 0.0 + for il, (k_ref, v_ref) in enumerate(kv): + h = norm(x, il, "norm1") + _, wk, wv = p(il, "self_attn.in_proj.weight").chunk(3) + k, v = _rope_pairs(h @ wk.T, n_head, rope_theta), h @ wv.T + for a, b in ((k, k_ref), (v, v_ref)): + drift = max(drift, ((a - b).norm(dim=1) / b.norm(dim=1)).median().item()) + x = x + attn(il, h, k, v) + x = x + ffn(il, x) + return frames, drift + + @ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model)) def _load_hparams(dir_model: Path) -> dict[str, Any]: logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes") @@ -194,22 +291,27 @@ class PocketTTSMmprojModel(MmprojModel): (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V), ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.has_audio_encoder = _has_voice_encoder(self.dir_model) + def set_gguf_parameters(self): self.gguf_writer.add_file_type(self.ftype) assert self.hparams_audio is not None # voice-prompt encoder: mimi encoder + speaker_proj - self.gguf_writer.add_clip_has_audio_encoder(True) - # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models - self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC) - self.gguf_writer.add_audio_projection_dim(self.n_embd_text) - self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"]) - self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"]) - self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"]) - self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"]) - self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) - # mimi convolves the waveform directly, it is passed around as a 1-row "mel" - self.gguf_writer.add_audio_num_mel_bins(1) + if self.has_audio_encoder: + self.gguf_writer.add_clip_has_audio_encoder(True) + # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC) + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"]) + self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"]) + self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"]) + self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + # mimi convolves the waveform directly, it is passed around as a 1-row "mel" + self.gguf_writer.add_audio_num_mel_bins(1) # generation: flow-matching decoder + mimi decoder # the SEANet and flow net hparams are constant across the family, clip.cpp holds them @@ -229,12 +331,57 @@ class PocketTTSMmprojModel(MmprojModel): # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"): return gguf.GGMLQuantizationType.F16 + # voices are read on the host + if new_name.startswith("voice."): + return gguf.GGMLQuantizationType.F32 return False + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # embeddings/.safetensors hold the K/V state after the voice prompt (see get_state_for_audio_prompt() in the reference), turn it back into prompt embeddings + paths = sorted((self.dir_model / "embeddings").glob("*.safetensors")) + if not paths: + return + weights = _load_safetensors(self.dir_model / "model.safetensors", + ("flow_lm.transformer.layers.", "flow_lm.speaker_proj_weight", "flow_lm.bos_before_voice")) + n_layer = self.global_config["num_hidden_layers"] + for path in paths: + state = _load_safetensors(path, ("transformer.layers.",)) + kv = [] + for il in range(n_layer): + prefix = f"transformer.layers.{il}.self_attn/" + if any(prefix + k not in state for k in ("cache", "offset")): + break + n_pos = int(state[prefix + "offset"].item()) + # older packs have no pad entry + if prefix + "pad" in state and int(state[prefix + "pad"].item()) != 0: + break + kv.append(state[prefix + "cache"][:, 0, :n_pos].flatten(2)) # [k/v, pos, n_embd] + if len(kv) != n_layer or any(t.shape != kv[0].shape for t in kv) or kv[0].shape[1] < 2 or \ + kv[0].shape[2] != self.global_config["hidden_size"]: + logger.warning(f"voice {path.stem}: skipped, not an unpadded voice state of this backbone") + continue + embd, drift = _voice_prompt_embd(weights, kv, self.global_config["num_attention_heads"], + self.global_config["rope_theta"]) + # some packs ship a state made with other weights, the reference still uses it: keep up to 50% error, warn above 5% + if drift > 0.5: + logger.warning(f"voice {path.stem}: skipped, its state does not belong to these weights (error {drift:.1e})") + continue + log = logger.warning if drift > 0.05 else logger.info + log(f"voice {path.stem}: {embd.shape[0]} frames, prompting them again gives the shipped state within {drift:.1e}") + yield (f"voice.{path.stem}", embd.float()) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: del bid # the block index of the mimi transformers is parsed here, not by the base class T = gguf.MODEL_TENSOR + if name.startswith("voice."): + yield (name, data_torch) + return + + if not self.has_audio_encoder and (name.startswith(("mimi.encoder", "mimi.downsample.")) + or name == "flow_lm.speaker_proj_weight"): + return # voice-prompt encoder + if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"): return # folded into the backbone embedding table if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."): diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 88aa20de1..2810a5cc6 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -888,6 +888,9 @@ struct clip_model { ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate std::vector gen_tfm_layers; // mimi decoder_transformer + // predefined voices, no rows if the voice is named in the prompt (lfm3a) + std::map> gen_voices; + // soprano: Vocos decoder, hidden states -> STFT coefficients clip_vocos vocos; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index d723432a7..9a043d7f9 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -3502,9 +3502,33 @@ struct clip_model_loader { layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight")); layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight")); } + + for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { + const char * name = gguf_get_tensor_name(ctx_gguf.get(), i); + if (strncmp(name, "voice.", 6) != 0) { + continue; + } + const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name); + if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_input_lin_w->ne[1]) { + throw std::runtime_error(string_format("%s: invalid voice tensor %s\n", __func__, name)); + } + auto & embd = model.gen_voices[name + 6]; + embd.resize(ggml_nelements(t)); + fin.seekg(tensor_offset.at(name), std::ios::beg); + if (!fin.read(reinterpret_cast(embd.data()), ggml_nbytes(t))) { + throw std::runtime_error(string_format("%s: failed to read %s\n", __func__, name)); + } + } + if (!model.gen_voices.empty()) { + LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size()); + } } break; case PROJECTOR_TYPE_LFM3A_GEN: { + // must match the "tts" chat template, see conversion/lfm3.py + // note for contributor: this is one-off fix, do not repeat this for future models + model.gen_voices = { {"female", {}}, {"male", {}} }; + // depthformer, the generic loop above loaded the a.gen.code.blk.* tensors for (int il = 0; il < hparams.n_layer; ++il) { model.layers[il].attn_gate_w = get_tensor(string_format(TN_ATTN_GATE, prefix, il, "weight")); @@ -5142,6 +5166,10 @@ void clip_gen_state_reset(clip_gen_state * state) { } } +const std::map> & clip_gen_get_voices(const clip_ctx * ctx) { + return ctx->model.gen_voices; +} + clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) { const auto & hparams = ctx->model.hparams; if (ctx->proj_type() == PROJECTOR_TYPE_LFM3A) { diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index cd3f434da..b2c9e0759 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -139,6 +139,8 @@ struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed void clip_gen_state_free(struct clip_gen_state * state); void clip_gen_state_reset(struct clip_gen_state * state); +const std::map> & clip_gen_get_voices(const struct clip_ctx * ctx); + bool clip_is_llava(const struct clip_ctx * ctx); // note for contributor: this clip_is_(model) pattern is deprecated // do NOT add new functions like this diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index d0342b9bd..bed8f3d8c 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -430,7 +430,7 @@ private: // runs the reference wav through the speaker encoder, returns one x-vector embedding row bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { - if (!mtmd_support_audio(mctx)) { + if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); return false; } @@ -998,7 +998,7 @@ private: // runs the reference wav through the mimi encoder, returns one row per 12.5Hz frame bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { - if (!mtmd_support_audio(mctx)) { + if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n"); return false; } @@ -1298,7 +1298,8 @@ public: reset(); seq_id = inp->seq_id; - if (inp->speaker_ref) { + // a voice is named in the prompt instead + if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) { LOG_WRN("mtmd_helper_gen_audio: the model has no voice conditioning, the speaker file is ignored\n"); } diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 99ae1738d..e562aae18 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -243,9 +243,8 @@ struct mtmd_helper_gen_audio_inp { int32_t n_past; enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto) - mtmd_bitmap * speaker_ref; // optional, can be NULL + mtmd_bitmap * speaker_ref; // optional, can be NULL: audio to clone, or a voice bitmap const char * lang; // optional, can be NULL - const char * voice; // optional, can be NULL, model-dependent int32_t top_k; float top_p; @@ -345,7 +344,6 @@ struct gen_audio { mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default(); std::string prompt_str; std::string lang_str; - std::string voice_str; mtmd::bitmap_ptr speaker_ref_ptr; inp() = default; @@ -356,7 +354,6 @@ struct gen_audio { void set_prompt (std::string p) { prompt_str = std::move(p); } void set_lang (std::string l) { lang_str = std::move(l); } - void set_voice (std::string v) { voice_str = std::move(v); } void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); } // pointers are only valid as long as *this is alive @@ -364,7 +361,6 @@ struct gen_audio { data.prompt = prompt_str.c_str(); data.prompt_len = prompt_str.size(); data.lang = lang_str.empty() ? nullptr : lang_str.c_str(); - data.voice = voice_str.empty() ? nullptr : voice_str.c_str(); data.speaker_ref = speaker_ref_ptr.get(); return &data; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 410881b51..d4ea23852 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -19,6 +19,7 @@ #endif #include +#include #include #include #include @@ -174,6 +175,7 @@ struct mtmd_bitmap { uint32_t ny = 0; std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio + bool is_voice = false; bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video bool is_video = false; // see mtmd_bitmap_set_is_video() @@ -199,6 +201,14 @@ struct mtmd_bitmap { } } + static mtmd_bitmap * init_voice(const std::string & name, const float * rows, uint32_t n_embd, uint32_t n_rows) { + auto * bitmap = new mtmd_bitmap(nullptr, n_embd, n_rows); + bitmap->is_voice = true; + bitmap->id = name; + bitmap->data.assign((const unsigned char *) rows, (const unsigned char *) (rows + (size_t) n_embd * n_rows)); + return bitmap; + } + const std::vector & get_ro_buf() const { return data; } @@ -351,6 +361,9 @@ struct mtmd_audio_tokens { // true if one of entries in batch_f32 is a placeholder bool is_placeholder() const { + if (track && track->batch_f32.entries.empty()) { + return track->embd.empty(); // a voice: its rows are not serialized + } for (const auto & entry : (track ? track->batch_f32 : batch_f32).entries) { if (entry.is_placeholder()) { return true; @@ -660,9 +673,8 @@ struct mtmd_context { ctx_v = res.ctx_v; ctx_a = res.ctx_a; ctx_gen_a = res.ctx_gen_a; - // soprano has no speaker encoder, its mmproj only holds the decoder - const bool gen_only = ctx_gen_a && clip_get_projector_type(ctx_gen_a) == PROJECTOR_TYPE_SOPRANO; - if (!ctx_v && !ctx_a && !gen_only) { + // soprano and pocket-tts without voice cloning have no encoder + if (!ctx_v && !ctx_a && !ctx_gen_a) { throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); } @@ -1535,6 +1547,10 @@ struct mtmd_tokenizer { GGML_ASSERT(!bitmaps.empty()); // note: only one type of media is supported per call, caller should enforce this + if (bitmaps[0]->is_voice) { + GGML_ASSERT(bitmaps.size() == 1); + return add_voice(*bitmaps[0]); + } const bool is_vision = !bitmaps[0]->is_audio; if (is_vision) { @@ -1869,6 +1885,30 @@ struct mtmd_tokenizer { return 0; } + int32_t add_voice(const mtmd_bitmap & bitmap) { + if (bitmap.ny == 0) { + LOG_ERR("%s: error: voice \"%s\" holds no embeddings\n", __func__, bitmap.id.c_str()); + return 2; + } + auto track = std::make_shared(); + track->n_tokens = bitmap.ny; + const auto & buf = bitmap.get_ro_buf(); + track->embd.assign((const float *) buf.data(), (const float *) (buf.data() + buf.size())); + + mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens); + audio_tokens->n_tokens = bitmap.ny; + audio_tokens->id = bitmap.id; + audio_tokens->track = std::move(track); + + cur.entries.emplace_back(mtmd_input_chunk{ + MTMD_INPUT_CHUNK_TYPE_AUDIO, + {}, // text tokens + nullptr, // image tokens + std::move(audio_tokens), + }); + return 0; + } + // a piece of a soundtrack goes in without the audio markers, it lies inside the span of its video int32_t add_soundtrack_piece(soundtrack & st, const mtmd_bitmap & bitmap) { if (!st.track) { @@ -2164,10 +2204,6 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk } return mtmd_encode_impl(ctx, chunk->tokens_image.get(), out_embd); } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { - if (!ctx->ctx_a) { - LOG_ERR("%s: model does not support audio input\n", __func__); - return 1; - } if (chunk->tokens_audio == nullptr) { LOG_ERR("%s: audio tokens are null\n", __func__); return 1; @@ -2176,22 +2212,27 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk LOG_ERR("%s: audio tokens batch is placeholder\n", __func__); return 1; } - int n_mmproj_embd = ctx->n_embd_out(); const auto & track = chunk->tokens_audio->track; + // a filled track (e.g. a voice) needs no audio encoder + if (!ctx->ctx_a && (!track || track->embd.empty())) { + LOG_ERR("%s: model does not support audio input\n", __func__); + return 1; + } if (track) { if (track->embd.empty()) { - track->embd.resize((size_t) track->n_tokens * n_mmproj_embd); + track->embd.resize((size_t) track->n_tokens * ctx->n_embd_out()); if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &track->batch_f32, track->embd)) { track->embd.clear(); return 1; } } + const size_t n_mmproj_embd = track->embd.size() / track->n_tokens; const size_t off = (size_t) chunk->tokens_audio->row_begin * n_mmproj_embd; out_embd.assign(track->embd.begin() + off, track->embd.begin() + off + (size_t) chunk->tokens_audio->n_tokens * n_mmproj_embd); return 0; } - out_embd.resize((size_t)chunk->tokens_audio->n_tokens * n_mmproj_embd); + out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out()); bool ok = clip_image_batch_encode( ctx->ctx_a, ctx->n_threads, @@ -2239,6 +2280,7 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { return info; } info.model_variant = clip_get_hparams(ctx->ctx_gen_a)->gen_model_variant.c_str(); + info.n_voices = (int32_t) clip_gen_get_voices(ctx->ctx_gen_a).size(); switch (clip_get_projector_type(ctx->ctx_gen_a)) { case PROJECTOR_TYPE_QWEN3TTS_GEN: info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS; @@ -2263,6 +2305,31 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { return info; } +const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) { + if (!ctx->ctx_gen_a || idx < 0) { + return nullptr; + } + const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a); + return idx < (int32_t) voices.size() ? std::next(voices.begin(), idx)->first.c_str() : nullptr; +} + +mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * name) { + if (!ctx->ctx_gen_a || !name) { + return nullptr; + } + // the name is matched case-insensitively, as the lfm3a "tts" chat template lowers it + const auto & voices = clip_gen_get_voices(ctx->ctx_gen_a); + const auto it = std::find_if(voices.begin(), voices.end(), [name](const auto & voice) { + return std::equal(voice.first.begin(), voice.first.end(), name, name + strlen(name), + [](unsigned char a, unsigned char b) { return std::tolower(a) == std::tolower(b); }); + }); + if (it == voices.end()) { + return nullptr; + } + const uint32_t n_embd = clip_n_mmproj_embd(ctx->ctx_gen_a); + return mtmd_bitmap::init_voice(it->first, it->second.data(), n_embd, it->second.size() / n_embd); +} + mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { mtmd_gen_inp inp{}; inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; @@ -2804,6 +2871,10 @@ bool mtmd_bitmap_is_audio(const mtmd_bitmap * bitmap) { return bitmap->is_audio; } +bool mtmd_bitmap_is_voice(const mtmd_bitmap * bitmap) { + return bitmap->is_voice; +} + const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap) { return bitmap->id.c_str(); } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 3e03142a4..4f69a644d 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -191,6 +191,8 @@ MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); // if bitmap is audio: // length of data must be n_samples * sizeof(float) // the data is in float format (PCM F32) +// if bitmap is voice (see mtmd_gen_audio_get_voice()): +// nx = n_embd, ny = number of rows, data is F32 rows (can be empty), id is the voice name // // if data == nullptr: // the bitmap is considered "empty", and will be treated as a placeholder for counting tokens @@ -203,6 +205,7 @@ MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitma MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap); MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap); +MTMD_API bool mtmd_bitmap_is_voice (const mtmd_bitmap * bitmap); MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap); // bitmap ID is optional, but useful for KV cache tracking // these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data() @@ -468,10 +471,18 @@ struct mtmd_gen_audio_info { enum mtmd_gen_audio_type type; int32_t sample_rate; // in Hz, for example 24000 for qwen3tts const char * model_variant; // name of the weight variant, can be nullptr if not applicable + int32_t n_voices; // predefined voices, see mtmd_gen_audio_voice_name() }; MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +// name of the idx-th predefined voice, nullptr when idx is out of range +MTMD_API const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx); + +// predefined voice with this name as a voice bitmap, to pass as speaker reference +// returns nullptr if there is no such voice, the caller must free it +MTMD_API mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * name); + enum mtmd_gen_process_type { MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.) diff --git a/tools/server/README.md b/tools/server/README.md index 66a34ff84..afa448b8a 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -857,9 +857,9 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info `lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional. -`voice`: Voice for the utterance (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional. +`voice`: Voice for the utterance, one of the `voices` listed by `GET /props` (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional: a model with predefined voices speaks with the first of them when neither `voice` nor `speaker_ref` is given. It rejects any other name, and a predefined voice cannot be combined with `speaker_ref`. -`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. +`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning). `top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply. @@ -1017,6 +1017,7 @@ By default, it is read-only. To make POST request to change global properties, y - `default_generation_settings` - the default generation settings for the `/completion` endpoint, which has the same fields as the `generation_settings` response object from the `/completion` endpoint. - `total_slots` - the total number of slots for process requests (defined by `--parallel` option) +- `voices` - the predefined voices of an audio generation model, to pass as `voice` to `/tts`. Only present when the model has some (e.g. LFM3-Audio, Pocket TTS without voice cloning) - `model_path` - the path to model file (same with `-m` argument) - `chat_template` - the model's original Jinja2 prompt template - `chat_template_caps` - capabilities of the chat template (see `common/jinja/caps.h` for more info), `reasoning_effort_levels` lists the `reasoning_effort` values the template renders distinctly, from lowest to highest diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 9377e3c17..48f1e93a2 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5225,6 +5225,13 @@ server_context_meta server_context::get_meta() const { const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt)); + std::vector tts_voices; + if (impl->mctx) { + for (int32_t i = 0; i < mtmd_gen_audio_get_info(impl->mctx).n_voices; i++) { + tts_voices.push_back(mtmd_gen_audio_voice_name(impl->mctx, i)); + } + } + return server_context_meta { /* build_info */ std::string(llama_build_info()), /* model_name */ impl->model_name, @@ -5239,6 +5246,7 @@ server_context_meta server_context::get_meta() const { /* has_cap_chat */ impl->has_cap_chat(), /* has_cap_tts */ impl->has_cap_tts(), /* has_audio_stream */ impl->mtmd_cap.audio_stream, + /* tts_voices */ std::move(tts_voices), /* json_ui_settings */ impl->json_ui_settings, /* slot_n_ctx */ impl->n_ctx_slot(), /* pooling_type */ llama_pooling_type(impl->ctx_tgt), @@ -5697,6 +5705,9 @@ static json get_res_props(const server_context_meta & meta, const common_params if (!params.server_connect_code.empty()) { props["connect_code"] = params.server_connect_code; } + if (!meta.tts_voices.empty()) { + props["voices"] = meta.tts_voices; + } if (params.use_jinja) { if (!tmpl_tools.empty()) { props["chat_template_tool_use"] = tmpl_tools; @@ -6475,9 +6486,17 @@ void server_routes::init_routes() { const std::string lang = json_value(body, "lang", std::string()); const std::string voice = json_value(body, "voice", std::string()); const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice); + // a model with predefined voices only speaks with one of them + mtmd::bitmap_ptr voice_ref; + if (!voice.empty() && info.n_voices > 0) { + voice_ref.reset(mtmd_gen_audio_get_voice(ctx_server.mctx, voice.c_str())); + if (!voice_ref) { + res->error(format_error_response("unknown voice \"" + voice + "\", see \"voices\" in GET /props", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + } task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt); task.tts_inp.set_lang(lang); - task.tts_inp.set_voice(voice); task.tts_inp.data.stream = stream; task.tts_inp.data.out_type = response_format == "pcm" ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM @@ -6516,12 +6535,27 @@ void server_routes::init_routes() { } if (speaker_ref_len > 0) { + if (voice_ref) { + res->error(format_error_response("\"voice\" and \"speaker_ref\" cannot be used together", ERROR_TYPE_INVALID_REQUEST)); + return res; + } + if (!mtmd_support_audio(ctx_server.mctx)) { + res->error(format_error_response("this model has no voice encoder, \"speaker_ref\" cannot be used", ERROR_TYPE_INVALID_REQUEST)); + return res; + } auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); if (!wrapper.bitmap) { res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); return res; } task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); + } else if (voice_ref) { + task.tts_inp.set_speaker_ref(std::move(voice_ref)); + } else if (info.n_voices > 0) { + // without a voice prompt the model produces near silence: the first voice is the default + const char * first = mtmd_gen_audio_voice_name(ctx_server.mctx, 0); + SRV_WRN("no voice given, speaking with \"%s\"\n", first); + task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(mtmd_gen_audio_get_voice(ctx_server.mctx, first))); } else { SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n"); } diff --git a/tools/server/server-context.h b/tools/server/server-context.h index d283c9121..05c538ea1 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -27,6 +27,7 @@ struct server_context_meta { bool has_cap_chat; bool has_cap_tts; bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream() + std::vector tts_voices; // predefined voices of the audio generation model json json_ui_settings; int slot_n_ctx; enum llama_pooling_type pooling_type; diff --git a/tools/tts/README.md b/tools/tts/README.md index 1793b7992..9bec17624 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -39,6 +39,7 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \ Available params: - `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). It is required, the model produces almost no audio without it +- `--tts-voice` selects one of the predefined voices shipped with the weights (e.g. `giovanni`), when the GGUF holds them. The converter reads them from `embeddings/*.safetensors` next to the weights, of both repositories, and logs how closely each one is reproduced (some packs ship voice states computed with other weights, the converter warns above 5% and skips a voice only when it does not belong to the model at all) - Note: `lang` is not used, the language is a property of the weights Example usage: @@ -51,6 +52,16 @@ llama-tts -m pocket-tts.gguf \ --output out.wav ``` +The weights [without voice cloning](https://huggingface.co/kyutai/pocket-tts-without-voice-cloning) ship a set of predefined voices (`embeddings/*.safetensors`) next to each language; the converter stores them in the mmproj, as the voice prompt embeddings, and leaves out the zeroed voice encoder. With such a model `--tts-speaker-file` is rejected and a voice is selected with `--tts-voice` (the first one when it is not given): + +```sh +llama-tts -m pocket-tts.gguf \ + -mm mmproj-pocket-tts.gguf \ + -p "Ciao a tutti" \ + --tts-voice giovanni \ + --output out.wav +``` + **Note for GGUF conversion:** The [upstream repository](https://huggingface.co/kyutai/pocket-tts) holds one complete model per language under `languages/`, next to a set of shared files at the root. Convert one of the `languages/` directories, **not** the root directory: diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 8949373cb..8578f8869 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -112,6 +112,27 @@ int main(int argc, char ** argv) { speaker_bitmap.reset(wrapper.bitmap); } + // a model with predefined voices needs one, the first by default; other models name the voice in the prompt + if (mtmd_gen_audio_get_info(mctx.get()).n_voices > 0 && (!params.tts_voice.empty() || !speaker_bitmap)) { + if (speaker_bitmap) { + LOG_ERR("--tts-voice and --tts-speaker-file cannot be used together\n"); + return 1; + } + if (params.tts_voice.empty()) { + params.tts_voice = mtmd_gen_audio_voice_name(mctx.get(), 0); + LOG_WRN("no --tts-voice given, speaking with \"%s\"\n", params.tts_voice.c_str()); + } + speaker_bitmap.reset(mtmd_gen_audio_get_voice(mctx.get(), params.tts_voice.c_str())); + if (!speaker_bitmap) { + LOG_ERR("unknown voice \"%s\", the model has:", params.tts_voice.c_str()); + for (int32_t i = 0; const char * name = mtmd_gen_audio_voice_name(mctx.get(), i); i++) { + LOG(" %s", name); + } + LOG("\n"); + return 1; + } + } + // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it auto tmpls = common_chat_templates_init(model, ""); std::string prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice); @@ -127,7 +148,6 @@ int main(int argc, char ** argv) { inp.prompt_len = prompt.size(); inp.speaker_ref = speaker_bitmap.get(); inp.lang = params.tts_lang.c_str(); - inp.voice = params.tts_voice.c_str(); inp.top_k = params.sampling.top_k; inp.top_p = params.sampling.top_p; inp.seed = params.sampling.seed;