diff --git a/conversion/drama3.py b/conversion/drama3.py index 68bbb556b..9b490900a 100644 --- a/conversion/drama3.py +++ b/conversion/drama3.py @@ -42,6 +42,115 @@ def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]: return alpha, 1.0 / (alpha + 1e-9) +# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2 +_CODEC_ENC_STRIDES = (2, 4, 8, 8) + +# codec encoder transformers, both 1024 wide with 16 heads of 64 +# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module +_N_ENC_TFM_LAYERS = 4 +_N_PRE_TFM_LAYERS = 8 +_CODEC_HEAD_DIM = 64 + + +def _neox_rows(w: Tensor, head_dim: int) -> Tensor: + # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result + order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)]) + return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape) + + +def codec_encoder_tensors(sd: dict[str, Tensor], num_codebooks: int) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + + def get(name: str) -> Tensor: + return sd[name].float() + + def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str: + return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid) + + def fmt(tensor: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str: + return tn(tensor, bid) + suffix + + def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1): + # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp + w = fold_conv(sd, prefix) + n_out, n_in, k = w.shape + w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in) + yield (tn(tensor, bid, xid) + ".weight", w) + yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias")) + + def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None): + a, b = fold_snake(get(name)) + yield (tn(tensor, bid, xid) + ".alpha", a) + yield (tn(tensor, bid, xid) + ".beta", b) + + def tfm_layers(prefix: str, n_layer: int, offset: int): + for il in range(n_layer): + p = f"{prefix}layers.{il}." + bid = offset + il + wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0) + yield (fmt(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM)) + yield (fmt(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM)) + yield (fmt(T.A_ENC_ATTN_V, bid), wv) + yield (fmt(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight")) + yield (fmt(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight")) + yield (fmt(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma")) + yield (fmt(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight")) + yield (fmt(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight")) + yield (fmt(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight")) + yield (fmt(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight")) + yield (fmt(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma")) + + # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv + e = "encoder.block." + yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0") + for bid, stride in enumerate(_CODEC_ENC_STRIDES): + p = f"{e}{bid + 1}.block." + for xid in range(3): + r = f"{p}{xid}.block." + yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid) + yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid) + yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid) + yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid) + yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid) + yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride) + # the last block ends with a windowed transformer + yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0) + yield (fmt(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight")) + yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha") + yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6") + + # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ + q = "quantizer." + for bid in range(2): + p = f"{q}downsample.{bid}." + yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2) + # depthwise: one [C] row per tap + yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous()) + yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias")) + for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)): + yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight")) + yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias")) + yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma")) + yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS) + yield (fmt(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight")) + + # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents() + def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]: + w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"]) + cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1) + return w, get(prefix + ".in_proj.bias"), cb + + w, b, cb = vq(q + "semantic_quantizer.quantizers.0") + yield (fmt(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w) + yield (fmt(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b) + yield (fmt(T.A_CODEC_VQ_FIRST_CB), cb) + n_rest = num_codebooks - 1 + rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)] + yield (fmt(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest])) + yield (fmt(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest])) + yield (fmt(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest])) + + @ModelBase.register("FishOmniModel") @ModelBase.example("fishaudio/drama-3") class Drama3TextModel(Qwen3_5MoeTextModel): @@ -59,7 +168,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel): @ModelBase.example("fishaudio/drama-3") class Drama3MmprojModel(MmprojModel): has_vision_encoder = False - has_audio_encoder = False + has_audio_encoder = True # codec encoder, for voice cloning gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN _LAYER_MAP = { @@ -103,6 +212,16 @@ class Drama3MmprojModel(MmprojModel): self.gguf_writer.add_gen_audio_head_count_kv(cfg["n_local_heads"]) self.gguf_writer.add_gen_audio_attention_layernorm_eps(cfg["norm_eps"]) # other tower and codec hparams are hardcoded in tools/mtmd + # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC) + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_audio_num_mel_bins(1) + self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS) + self.gguf_writer.add_audio_embedding_length(1024) + self.gguf_writer.add_audio_feed_forward_length(3072) + self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: @@ -112,6 +231,12 @@ class Drama3MmprojModel(MmprojModel): return None def tensor_force_quant(self, name, new_name, bid, n_dims): + # the VQ picks the nearest codebook entry, small errors in the encoder change the codes + if new_name.startswith("a.codec."): + return gguf.GGMLQuantizationType.F32 + # codec transformers: F16 gives codes closer to the reference than BF16 + if new_name.startswith("a.blk.") and n_dims == 2: + return gguf.GGMLQuantizationType.F16 # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path if new_name.endswith(".weight") and ( new_name in ("a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight") @@ -130,7 +255,7 @@ class Drama3MmprojModel(MmprojModel): T = gguf.MODEL_TENSOR cfg = self.hparams - if name.startswith("a.gen.wav."): + if name.startswith("a."): yield (name, data_torch) return @@ -177,7 +302,7 @@ class Drama3MmprojModel(MmprojModel): raise ValueError(f"unexpected tensor {name}") def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - # only the DAC decoder, the encoder is for voice cloning (not supported yet) + # Fish DAC decoder, then the shared codec encoder for voice cloning sd = load_codec(self.dir_model) T = gguf.MODEL_TENSOR @@ -262,3 +387,5 @@ class Drama3MmprojModel(MmprojModel): yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6")) yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get(d + "6.conv.bias")) + + yield from codec_encoder_tensors(sd, self.hparams["num_codebooks"]) diff --git a/conversion/fish_s2.py b/conversion/fish_s2.py index 4c5aeff13..563b52b12 100644 --- a/conversion/fish_s2.py +++ b/conversion/fish_s2.py @@ -3,13 +3,11 @@ from __future__ import annotations from pathlib import Path from typing import Any, Callable, Iterable, TYPE_CHECKING -import torch - if TYPE_CHECKING: from torch import Tensor from .base import ModelBase, gguf -from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec +from .drama3 import Drama3MmprojModel, _neox_rows from .qwen import Qwen3Model # Fish Audio S2 Pro (dual-AR) @@ -19,15 +17,6 @@ from .qwen import Qwen3Model # residual codebook size of codec.pth, the fast AR head is cut to it _N_RESIDUAL_CODES = 1024 -# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2 -_CODEC_ENC_STRIDES = (2, 4, 8, 8) - -# codec encoder transformers, both 1024 wide with 16 heads of 64 -# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module -_N_ENC_TFM_LAYERS = 4 -_N_PRE_TFM_LAYERS = 8 -_CODEC_HEAD_DIM = 64 - def _load_hparams(dir_model: Path) -> dict[str, Any]: hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) @@ -48,12 +37,6 @@ def _load_hparams(dir_model: Path) -> dict[str, Any]: return hparams -def _neox_rows(w: Tensor, head_dim: int) -> Tensor: - # checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result - order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)]) - return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape) - - @ModelBase.register("fish_qwen3_omni") @ModelBase.example("fishaudio/s2-pro") class FishS2TextModel(Qwen3Model): @@ -88,7 +71,6 @@ class FishS2TextModel(Qwen3Model): @ModelBase.register("fish_qwen3_omni") @ModelBase.example("fishaudio/s2-pro") class FishS2MmprojModel(Drama3MmprojModel): - has_audio_encoder = True # codec encoder, for voice cloning gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN _FAST_AR_MAP = { @@ -109,28 +91,6 @@ class FishS2MmprojModel(Drama3MmprojModel): cfg["num_hidden_layers"] = cfg["n_layer"] return cfg - def set_gguf_parameters(self): - super().set_gguf_parameters() - # codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd - self.gguf_writer.add_clip_has_audio_encoder(True) - self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC) - self.gguf_writer.add_audio_projection_dim(self.n_embd_text) - self.gguf_writer.add_audio_num_mel_bins(1) - self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS) - self.gguf_writer.add_audio_embedding_length(1024) - self.gguf_writer.add_audio_feed_forward_length(3072) - self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM) - self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) - - def tensor_force_quant(self, name, new_name, bid, n_dims): - # the VQ picks the nearest codebook entry, small errors in the encoder change the codes - if new_name.startswith("a.codec."): - return gguf.GGMLQuantizationType.F32 - # codec transformers: F16 gives codes closer to the reference than BF16 - if new_name.startswith("a.blk.") and n_dims == 2: - return gguf.GGMLQuantizationType.F16 - return super().tensor_force_quant(name, new_name, bid, n_dims) - @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name = item[0] @@ -182,95 +142,3 @@ class FishS2MmprojModel(Drama3MmprojModel): yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v) return yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch) - - def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - yield from super().generate_extra_tensors() - - sd = load_codec(self.dir_model) - T = gguf.MODEL_TENSOR - - def get(name: str) -> Tensor: - return sd[name].float() - - def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str: - return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid) - - def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1): - # taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp - w = fold_conv(sd, prefix) - n_out, n_in, k = w.shape - w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in) - yield (tn(tensor, bid, xid) + ".weight", w) - yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias")) - - def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None): - a, b = fold_snake(get(name)) - yield (tn(tensor, bid, xid) + ".alpha", a) - yield (tn(tensor, bid, xid) + ".beta", b) - - def tfm_layers(prefix: str, n_layer: int, offset: int): - for il in range(n_layer): - p = f"{prefix}layers.{il}." - bid = offset + il - wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0) - yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM)) - yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM)) - yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv) - yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight")) - yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight")) - yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma")) - yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight")) - yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight")) - yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight")) - yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight")) - yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma")) - - # encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv - e = "encoder.block." - yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0") - for bid, stride in enumerate(_CODEC_ENC_STRIDES): - p = f"{e}{bid + 1}.block." - for xid in range(3): - r = f"{p}{xid}.block." - yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid) - yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid) - yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid) - yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid) - yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid) - yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride) - # the last block ends with a windowed transformer - yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0) - yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight")) - yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha") - yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6") - - # quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ - q = "quantizer." - for bid in range(2): - p = f"{q}downsample.{bid}." - yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2) - # depthwise: one [C] row per tap - yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous()) - yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias")) - for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)): - yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight")) - yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias")) - yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma")) - yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS) - yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight")) - - # nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents() - def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]: - w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"]) - cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1) - return w, get(prefix + ".in_proj.bias"), cb - - w, b, cb = vq(q + "semantic_quantizer.quantizers.0") - yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w) - yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b) - yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb) - n_rest = self.hparams["num_codebooks"] - 1 - rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)] - yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest])) - yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest])) - yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest])) diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md index 48c6d72e5..e009566c0 100644 --- a/docs/multimodal/tts.md +++ b/docs/multimodal/tts.md @@ -175,7 +175,7 @@ python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatte Available params: - Speaker tags `<|speaker:0|>` to `<|speaker:15|>` in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>` - The language is the one of the text, `--tts-lang` is not used -- Note: `--tts-speaker-file` is not supported yet (voice cloning) +- Voice cloning: `--tts-speaker-file` with the reference clip and `--tts-speaker-text` with its transcript (a transcript without a speaker tag is speaker 0). The clip is encoded by the codec encoder half of `codec.pth` and its codes are put in the prompt as one reference, as the reference does; the clip is truncated to the first 30 s. It needs an mmproj converted with the codec encoder - Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference (main token: temperature 1.0, top-p 0.9, repetition-aware redraw over the last 16 frames; other codes: temperature 1.0, top-k 30, top-p 0.9) - Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token - The codebook tower keeps the whole sequence in its KV cache, the prompt and the generated frames are limited to 4096 positions @@ -192,7 +192,7 @@ llama-tts -m drama3.gguf \ **Note for GGUF conversion:** -The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and the decoder half of `codec.pth`, it is best kept in BF16: +The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16; the codec encoder is always stored in F16/F32: ```sh python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf diff --git a/tools/mtmd/models/drama3-gen.cpp b/tools/mtmd/models/drama3-gen.cpp index e77fcf540..41d8c22b1 100644 --- a/tools/mtmd/models/drama3-gen.cpp +++ b/tools/mtmd/models/drama3-gen.cpp @@ -130,9 +130,19 @@ ggml_cgraph * mtmd_graph_drama3_gen::build_tower() { ggml_set_name(inp_pos, "inp_pos"); ggml_set_input(inp_pos); - ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads); - ggml_set_name(inp_rand, "inp_rand"); - ggml_set_input(inp_rand); + // teacher forcing (reference rows of a voice clone): codes 1.. of the frame are given, nothing is sampled + const bool forced = n_frames == 2; + ggml_tensor * inp_rand = nullptr; + ggml_tensor * inp_force = nullptr; + if (forced) { + inp_force = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_heads); + ggml_set_name(inp_force, "inp_force"); + ggml_set_input(inp_force); + } else { + inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_heads); + ggml_set_name(inp_rand, "inp_rand"); + ggml_set_input(inp_rand); + } ggml_tensor * mask = ggml_view_1d(ctx0, inp_scale, 1, 0); ggml_tensor * extra_on = ggml_view_1d(ctx0, inp_scale, 1, sizeof(float)); @@ -152,6 +162,11 @@ ggml_cgraph * mtmd_graph_drama3_gen::build_tower() { continue; } const int ih = il - n_shared; + if (forced) { + code = ggml_view_1d(ctx0, inp_force, 1, (size_t) ih * sizeof(int32_t)); + out_codes = cg.cache_set(out_codes, ih + 1, code); + continue; + } ggml_tensor * head = ggml_view_2d(ctx0, model.gen_code_head_w, model.gen_code_head_w->ne[0], model.gen_code_head_w->ne[1], model.gen_code_head_w->nb[1], (size_t) ih * model.gen_code_head_w->nb[2]); ggml_tensor * logits = ggml_mul_mat(ctx0, head, x); diff --git a/tools/mtmd/models/fish-s2-spkenc.cpp b/tools/mtmd/models/fish-s2-spkenc.cpp index a9c51d9cd..fe2eb28ef 100644 --- a/tools/mtmd/models/fish-s2-spkenc.cpp +++ b/tools/mtmd/models/fish-s2-spkenc.cpp @@ -2,6 +2,7 @@ #include #include +#include // reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode() // activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer @@ -117,17 +118,35 @@ ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() { return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]); }; - // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen const int64_t n_cb = model.gen_code_embd_w->ne[2]; - ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w); - ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx), - ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx)); + std::vector idx(n_cb); + idx[0] = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w); for (int64_t g = 1; g < n_cb; g++) { ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]); - idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1)); - out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx)); + idx[g] = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1)); + } + // codes of the clip, read back as the gen-side reference + for (int64_t g = 0; g < n_cb; g++) { + ggml_set_name(idx[g], ("out_ref_code_" + std::to_string(g)).c_str()); + ggml_set_output(idx[g]); + ggml_build_forward_expand(gf, idx[g]); + } + + ggml_tensor * out = nullptr; + if (hparams.spkenc_drama3_rows) { + // drama3 backbone row: (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2), see mtmd_graph_drama3_gen::build_tower() + out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx[0]), + ggml_get_rows(ctx0, page(model.gen_code_embd_w, 1), idx[1])); + out = ggml_scale(ctx0, out, 1.0f / sqrtf(2.0f)); + } else { + // backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen + out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx[0]), + ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx[0])); + for (int64_t g = 1; g < n_cb; g++) { + out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx[g])); + } + out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1))); } - out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1))); cb(out, "spk_rows", -1); ggml_build_forward_expand(gf, out); diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 64f7bf70d..fb75270ad 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -625,7 +625,7 @@ struct mtmd_graph_drama3_gen : mtmd_graph { mtmd_subctx_gen_process gen_process; mtmd_subctx_gen_state * gen_state; // nullptr on buffer reserve - int n_frames; // GEN_WAV only + int n_frames; // GEN_WAV: frames to decode; GEN_CODE: 2 when the caller gives the codes of the frame (teacher forcing) int top_k; float top_p; float temp; diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 32f1d39d4..3c33eff3a 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -2739,6 +2739,9 @@ public: seq_id = 0; pos = 0; prompt_tokens.clear(); + ref_codes.clear(); + ref_embd.clear(); + ref_begin = 0; prompt_pos = 0; frame.clear(); main_logits.clear(); @@ -2758,8 +2761,16 @@ public: seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; - if (inp->speaker_ref) { - LOG_WRN("mtmd_helper_gen_audio: voice cloning is not supported yet, the speaker file is ignored\n"); + + // voice cloning: backbone rows from the codec encoder, the codes of the clip go to the gen state + std::vector ref_rows; + if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref) && !mtmd_support_audio(mctx)) { + LOG_WRN("mtmd_helper_gen_audio: mmproj has no speaker encoder (re-convert it for voice cloning), the speaker file is ignored\n"); + } else if (inp->speaker_ref) { + if (!encode_speaker(inp->speaker_ref, ref_rows) || !init_gen_state(inp->seed) || + mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) { + return 1; + } } sem_begin = find_special_token(vocab, "<|semantic:0|>"); @@ -2770,7 +2781,12 @@ public: return 1; } - if (!tokenize_segments(prompt_segments(std::string(inp->prompt, inp->prompt_len)), prompt_tokens)) { + const std::string text(inp->prompt, inp->prompt_len); + if (ref_rows.empty()) { + if (!tokenize_segments(prompt_segments(text), prompt_tokens)) { + return 1; + } + } else if (!build_reference_prompt(text, inp->speaker_ref_text ? inp->speaker_ref_text : "", std::move(ref_rows))) { return 1; } if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) { @@ -2782,7 +2798,7 @@ public: return 1; } rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); - return init_gen_state(inp->seed) ? 0 : 1; + return gen_state || init_gen_state(inp->seed) ? 0 : 1; } // run the tower on every prompt row to fill its KV @@ -2792,27 +2808,41 @@ public: if (prompt_pos >= n_prompt) { return 0; } - // small batches: every row also outputs logits - const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 }); - llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); - if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) { - return -1; + // small batches of rows of one kind (text tokens or reference frames): every row also outputs logits + const bool speech = is_speech_row(prompt_pos); + int n = std::min({ n_batch, n_prompt - prompt_pos, 64 }); + for (int i = 1; i < n; i++) { + if (is_speech_row(prompt_pos + i) != speech) { + n = i; + break; + } } - for (int i = 0; i < n; i++) { - llama_batch_ext_set_output_logits(batch.get(), i, true); + int ret = 0; + if (speech) { + ret = decode_embd_rows(ref_embd.data() + (size_t) (prompt_pos - ref_begin) * n_embd, n); + } else { + llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); + if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) { + return -1; + } + for (int i = 0; i < n; i++) { + llama_batch_ext_set_output_logits(batch.get(), i, true); + } + ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); } - if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { + if (ret != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } for (int i = 0; i < n; i++) { + const int row = prompt_pos + i; const float * h = llama_get_embeddings_ith(lctx, i); - if (prompt_pos + i == n_prompt - 1) { + if (row == n_prompt - 1) { h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row save_main_logits(i); break; } - if (!run_tower(-1, h)) { + if (!run_prompt_tower(row, h)) { return -1; } } @@ -2845,17 +2875,7 @@ public: } codes_buf.insert(codes_buf.end(), frame.begin(), frame.end()); - // text/speech stream: all mrope sections use the same position - const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE || - llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE; - decode_embd_batch batch(next_embd, 1, mrope ? 4 : 1, n_embd); - if (mrope) { - batch.set_position_mrope_1d(pos, seq_id); - } else { - batch.set_position_normal(pos, seq_id); - } - batch.logits[0] = 1; - if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { + if (decode_embd_rows(next_embd, 1) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return 1; } @@ -2938,9 +2958,24 @@ protected: return true; } + // offset of the leading speaker tag <|speaker:N|> after optional whitespace, npos without one + static size_t speaker_tag_pos(const std::string & text) { + const size_t lead = text.find_first_not_of(" \t\n\r\f\v"); + static const std::string open = "<|speaker:"; + if (lead == std::string::npos || text.compare(lead, open.size(), open) != 0) { + return std::string::npos; + } + size_t i = lead + open.size(); + const size_t digits = i; + while (i < text.size() && text[i] >= '0' && text[i] <= '9') { + i++; + } + return i > digits && text.compare(i, 2, "|>") == 0 ? lead : std::string::npos; + } + // reference-bracket-v1 prompt without references virtual std::vector prompt_segments(std::string text) const { - if (text.rfind("<|speaker:", 0) != 0) { + if (speaker_tag_pos(text) == std::string::npos) { text = "<|speaker:0|> " + text; } return { @@ -2952,6 +2987,86 @@ protected: // prompt rows but the last + generated frames virtual size_t max_rows() const { return n_tower_rows; } + // n speech rows from pos, every row outputs logits + // text/speech stream: all mrope sections use the same position + int32_t decode_embd_rows(const float * embd, int n) { + const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE || + llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE; + decode_embd_batch batch(embd, n, mrope ? 4 : 1, n_embd); + if (mrope) { + batch.set_position_mrope_1d(pos, seq_id); + } else { + batch.set_position_normal(pos, seq_id); + } + for (int i = 0; i < n; i++) { + batch.logits[i] = 1; + } + return llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)); + } + + bool is_speech_row(int row) const { + return row >= ref_begin && (size_t) (row - ref_begin) < ref_embd.size() / n_embd; + } + + // reference-bracket-v1 with one reference, see vLLM-Omni fish_omni/prompt.py + // system: instruction, "\n\n[Reference 1]\nText: \nSpeech: " then one row per frame of the clip + bool build_reference_prompt(std::string text, std::string ref_text, std::vector && ref_rows) { + size_t n_codes = 0; + const int32_t * codes = mtmd_gen_state_get_ref_codes(gen_state.get(), &n_codes); + const size_t n_frames = ref_rows.size() / n_embd; + if (!codes || n_frames == 0 || n_codes != n_frames * n_cb || ref_rows.size() != n_frames * n_embd) { + LOG_ERR("mtmd_helper_gen_audio: the speaker encoder gave no codes for the clip\n"); + return false; + } + // the speaker tag follows "Text: " directly (default speaker 0), an untagged target reuses it + std::string tag = "<|speaker:0|>"; + const size_t lead = speaker_tag_pos(ref_text); + if (lead == std::string::npos) { + ref_text = tag + " " + ref_text; + } else { + ref_text = ref_text.substr(lead); + tag = ref_text.substr(0, ref_text.find("|>") + 2); + } + if (speaker_tag_pos(text) == std::string::npos) { + text = tag + " " + text; + } + + std::vector head, tail; + if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:", + "\n\n[Reference 1]\nText: " + ref_text + "\nSpeech: " }, head) || + !tokenize_segments({ "<|im_end|>\n", "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>" }, tail)) { + return false; + } + prompt_tokens.assign(head.begin(), head.end()); + prompt_tokens.insert(prompt_tokens.end(), n_frames, LLAMA_TOKEN_NULL); + prompt_tokens.insert(prompt_tokens.end(), tail.begin(), tail.end()); + ref_codes.assign(codes, codes + n_codes); + ref_embd = std::move(ref_rows); + ref_begin = (int) head.size(); + return true; + } + + // tower row of a prompt row: the input frame is the codes of this row (reference frame) or none, + // the codes of the next row are forced when it is a reference frame + bool run_prompt_tower(int row, const float * h) { + const bool cur = is_speech_row(row); + const bool next = is_speech_row(row + 1); + const auto ref_frame = [&](int r) { return ref_codes.begin() + (size_t) (r - ref_begin) * n_cb; }; + if (next) { + // [input frame, -1 for a text row | forced next frame] + frame.assign(n_cb, -1); + if (cur) { + std::copy(ref_frame(row), ref_frame(row + 1), frame.begin()); + } + frame.insert(frame.end(), ref_frame(row + 1), ref_frame(row + 2)); + } else if (cur) { + frame.assign(ref_frame(row), ref_frame(row + 1)); + } + const bool ok = run_tower(next ? frame[n_cb] : -1, h) != nullptr; + frame.clear(); + return ok; + } + // copy right after our own decode, other sequences may decode on lctx before the next step_gen // layout: eos, then semantic codes 0..n_cb_vocab-1 void save_main_logits(int32_t i) { @@ -3072,7 +3187,10 @@ protected: llama_seq_id seq_id = 0; llama_pos pos = 0; - std::vector prompt_tokens; + std::vector prompt_tokens; // one per prompt row, LLAMA_TOKEN_NULL for a reference frame + std::vector ref_codes; // codes of the reference frames, n_cb per frame + std::vector ref_embd; // backbone rows of the reference frames + int ref_begin = 0; // prompt row of the first reference frame int prompt_pos = 0; std::mt19937 rng; diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp index e26824d0f..2f4f40591 100644 --- a/tools/mtmd/mtmd-model.cpp +++ b/tools/mtmd/mtmd-model.cpp @@ -1171,6 +1171,10 @@ struct mtmd_model_loader { hparams.wav_tfm_n_head_kv = 16; hparams.wav_tfm_eps = 1e-5f; hparams.wav_tfm_rope_theta = 10000.0f; + // drama3 shares the codec, only the backbone rows of a frame differ + std::string gen_proj; + get_string(KEY_GEN_AUDIO_PROJ_TYPE, gen_proj, false); + hparams.spkenc_drama3_rows = projector_type_from_string(gen_proj) == PROJECTOR_TYPE_DRAMA3_GEN; } break; case PROJECTOR_TYPE_CHATTERBOX: { diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h index 893578b03..10c3154e8 100644 --- a/tools/mtmd/mtmd-model.h +++ b/tools/mtmd/mtmd-model.h @@ -239,6 +239,9 @@ struct mtmd_hparams { int32_t wav_tfm_swa = 0; // pre_transformer's KV cache size, in frames int32_t wav_frame_stack = 0; // lfm3a codec: decoder positions per frame + // fish_s2_spkenc in a drama3_gen mmproj: backbone rows as drama3_gen builds them + bool spkenc_drama3_rows = false; + // mimo-v2.5: LLM-side connector (input_local_transformer) int32_t audio_local_n_layer = 0; int32_t audio_local_group_size = 0; diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp index f754a913b..aea0c1b92 100644 --- a/tools/mtmd/mtmd-subcontext.cpp +++ b/tools/mtmd/mtmd-subcontext.cpp @@ -2008,12 +2008,16 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { } else { // code0 < 0: next row is text (prompt), the tower only fills its KV // codes: input frame codes for a speech row, empty for a text row + // then optionally the codes of the next frame (reference rows): teacher forcing, code0 must be its code 0 const bool next_speech = params->code0 >= 0; - const bool cur_speech = params->codes && (int64_t) params->codes->size() == n_cb; - if (params->code0 >= n_vocab) { - LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) n_vocab); + const size_t n_given = params->codes ? params->codes->size() : 0; + const bool forced = n_given == (size_t) (2 * n_cb); + if (params->code0 >= n_vocab || (n_given != 0 && n_given != (size_t) n_cb && !forced) || + (forced && params->code0 != (*params->codes)[n_cb])) { + LOG_ERR("%s: invalid tower input (code0 %d, %zu codes)\n", __func__, params->code0, n_given); return false; } + const bool cur_speech = n_given != 0 && (*params->codes)[0] >= 0; std::vector code0 = { std::max(params->code0, 0) }; set_input_i32("inp_code0", code0); @@ -2030,12 +2034,20 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { std::vector inp_pos = { (int32_t) pos }; set_input_i32("inp_pos", inp_pos); - std::uniform_real_distribution dist(0.0f, 1.0f); - std::vector r(model.gen_code_head_w->ne[2]); - for (auto & v : r) { - v = dist(params->gen_state->rng); + if (forced) { + std::vector force(model.gen_code_head_w->ne[2]); + for (size_t ih = 0; ih < force.size(); ih++) { + force[ih] = std::clamp((*params->codes)[n_cb + 1 + ih], 0, (int32_t) n_vocab - 1); + } + set_input_i32("inp_force", force); + } else { + std::uniform_real_distribution dist(0.0f, 1.0f); + std::vector r(model.gen_code_head_w->ne[2]); + for (auto & v : r) { + v = dist(params->gen_state->rng); + } + set_input_f32("inp_rand", r); } - set_input_f32("inp_rand", r); } } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: @@ -2596,10 +2608,8 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { // for audio gen models // - if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { - GGML_ASSERT(params->out_codes && params->out_code_embd); - - // frame-major codes, the layout of GEN_WAV's input + // reference clip codes, one out_ref_code_ tensor per codebook, read back frame-major (the layout of GEN_WAV's input) + auto read_ref_codes = [&gf](std::vector & out) -> size_t { std::vector> groups; for (int g = 0; ; g++) { ggml_tensor * t = ggml_graph_get_tensor(gf, ("out_ref_code_" + std::to_string(g)).c_str()); @@ -2611,12 +2621,23 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) { } GGML_ASSERT(!groups.empty()); const size_t n_frames = groups[0].size(); - params->out_codes->resize(n_frames * groups.size()); + out.resize(n_frames * groups.size()); for (size_t f = 0; f < n_frames; f++) { for (size_t g = 0; g < groups.size(); g++) { - (*params->out_codes)[f * groups.size() + g] = groups[g][f]; + out[f * groups.size() + g] = groups[g][f]; } } + return n_frames; + }; + + if (model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC && params->out_codes) { + read_ref_codes(*params->out_codes); + return true; + } + + if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) { + GGML_ASSERT(params->out_codes && params->out_code_embd); + const size_t n_frames = read_ref_codes(*params->out_codes); ggml_tensor * rows = ggml_graph_get_tensor(gf, "out_ref_rows"); GGML_ASSERT(rows && rows->type == GGML_TYPE_F32 && (size_t) rows->ne[1] == n_frames); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 511e01f93..666f48eeb 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -2617,8 +2617,9 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd); static int32_t voxcpm2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd); -static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector & out_embd) { - mtmd_image_f32_batch batch = batch_f32.clone(); +// copy of batch_f32, the mel is computed from source when there is one +static bool mtmd_audio_batch_prepare(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, mtmd_image_f32_batch & batch) { + batch = batch_f32.clone(); if (source) { if (!ctx->audio_preproc->compute(*source)) { LOG_ERR("%s: unable to compute the mel\n", __func__); @@ -2627,7 +2628,38 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat GGML_ASSERT(idx < source->entries.size()); batch.entries[0] = source->entries[idx]; } - return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd); + return true; +} + +static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_batch & batch_f32, mtmd_audio_preproc_out * source, uint32_t idx, std::vector & out_embd) { + mtmd_image_f32_batch batch; + return mtmd_audio_batch_prepare(ctx, batch_f32, source, idx, batch) && ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd); +} + +// fish_s2_spkenc: one backbone row per reference frame, the codes go to ctx->gen_ref, see mtmd_gen_state_set_ref() +static int32_t mtmd_fish_s2_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd) { + mtmd_image_f32_batch batch; + if (!mtmd_audio_batch_prepare(ctx, audio_tokens->batch_f32, audio_tokens->source.get(), audio_tokens->source_idx, batch)) { + return 1; + } + std::vector codes; + out_embd.resize((size_t) audio_tokens->n_tokens * ctx->n_embd_out()); + + mtmd_subctx_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.out_embd = &out_embd; + params.out_codes = &codes; + if (!ctx->ctx_a->encode(¶ms)) { + return 1; + } + const size_t n_cb = ctx->ctx_a->model.gen_code_embd_w->ne[2]; + if (codes.size() != (size_t) audio_tokens->n_tokens * n_cb) { + LOG_ERR("%s: %zu reference codes for %u frames\n", __func__, codes.size(), audio_tokens->n_tokens); + return 1; + } + ctx->gen_ref.tokens = std::move(codes); + return 0; } // qwen3tts: the speaker row, then one talker input row per reference frame @@ -2732,6 +2764,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_VOXCPM2_SPKENC) { return voxcpm2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd); } + if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_FISH_S2_SPKENC) { + return mtmd_fish_s2_ref_encode(ctx, chunk->tokens_audio.get(), out_embd); + } if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) { return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd); } @@ -4214,6 +4249,13 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) { return 0; } +const int32_t * mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes) { + if (n_codes) { + *n_codes = state ? state->ref.tokens.size() : 0; + } + return state && !state->ref.tokens.empty() ? state->ref.tokens.data() : nullptr; +} + static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { mtmd_subcontext * sctx = ctx->ctx_gen_a.get(); if (!sctx) { diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 49fc8136e..cf2d14ce2 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -593,6 +593,10 @@ MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state); // returns non-zero if the last encode produced no such data MTMD_API int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state); +// codes of the reference held by the state (frame-major, e.g. drama3: 10 per frame), NULL if it has none +// valid until the next mtmd_gen_state_set_ref() / mtmd_gen_state_free() on this state +MTMD_API const int32_t * mtmd_gen_state_get_ref_codes(const mtmd_gen_state * state, size_t * n_codes); + // defaults tuned for the loaded pipeline, callers override only what they care about MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx);