Files
llama.cpp/conversion/fish_s2.py
T
FeitengandXuan Son Nguyen 34ff2e047c mtmd: drama3 voice cloning (#32586)
* mtmd: drama3 voice cloning

The Fish S2 codec encoder (fish_s2_spkenc) now also serves drama3:
- conversion: drama3 writes the codec encoder and the audio encoder KV, shared with fish_s2
- fish_s2_spkenc: outputs the codes of the clip (gen-side reference) and, in a drama3_gen mmproj,
  builds the drama3 backbone rows (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2)
- drama3_gen tower: teacher forcing when the caller gives the codes of the next frame
- drama3 pipeline: reference-bracket-v1 prompt with one reference (vLLM-Omni fish_omni), the
  reference frames go through the backbone as codec rows and through the tower with their codes
- mtmd_gen_state_get_ref_codes(): read-only access to the reference codes of a gen state

Claude-Session: https://claude.ai/code/session_01PpX3DmaxNSmJbxGaxnUBrX

* fix correctness

* clean up

* rm redundant code

* fish_s2: rm unused import

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
2026-11-20 18:34:52 +01:00

145 lines
6.2 KiB
Python

from __future__ import annotations
from pathlib import Path
from typing import Any, Callable, Iterable, TYPE_CHECKING
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, gguf
from .drama3 import Drama3MmprojModel, _neox_rows
from .qwen import Qwen3Model
# Fish Audio S2 Pro (dual-AR)
# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame
# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3)
# residual codebook size of codec.pth, the fast AR head is cut to it
_N_RESIDUAL_CODES = 1024
def _load_hparams(dir_model: Path) -> dict[str, Any]:
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
cfg = hparams["text_config"]
hparams["text_config"] = {
"hidden_size": cfg["dim"],
"num_hidden_layers": cfg["n_layer"],
"num_attention_heads": cfg["n_head"],
"num_key_value_heads": cfg["n_local_heads"],
"head_dim": cfg["head_dim"],
"intermediate_size": cfg["intermediate_size"],
"rms_norm_eps": cfg["norm_eps"],
"rope_theta": cfg["rope_base"],
"max_position_embeddings": cfg["max_seq_len"],
"vocab_size": cfg["vocab_size"],
"tie_word_embeddings": cfg["tie_word_embeddings"],
}
return hparams
@ModelBase.register("fish_qwen3_omni")
@ModelBase.example("fishaudio/s2-pro")
class FishS2TextModel(Qwen3Model):
model_arch = gguf.MODEL_ARCH.QWEN3
def __init__(self, dir_model: Path, *args, **kwargs):
kwargs.pop("hparams", None)
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# the rest is mapped by tensor_mapping.py
name = name.removeprefix("text_model.")
T = gguf.MODEL_TENSOR
head_dim = self.hparams["head_dim"]
if name.endswith(".attention.wqkv.weight"):
n_q = self.hparams["num_attention_heads"] * head_dim
n_kv = self.hparams["num_key_value_heads"] * head_dim
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim))
yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim))
yield (self.format_tensor_name(T.ATTN_V, bid), v)
return
if name.endswith(".attention.q_norm.weight"):
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim))
return
if name.endswith(".attention.k_norm.weight"):
yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim))
return
yield from super().modify_tensors(data_torch, name, bid)
@ModelBase.register("fish_qwen3_omni")
@ModelBase.example("fishaudio/s2-pro")
class FishS2MmprojModel(Drama3MmprojModel):
gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
_FAST_AR_MAP = {
"attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
"attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
"ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
"feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
"feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
"feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
}
def __init__(self, dir_model: Path, *args, **kwargs):
kwargs.pop("hparams", None)
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
def get_audio_config(self) -> dict[str, Any] | None:
cfg = dict(self.global_config["audio_decoder_config"])
cfg["num_hidden_layers"] = cfg["n_layer"]
return cfg
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name = item[0]
if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight":
return item
return None
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
T = gguf.MODEL_TENSOR
cfg = self.hparams
# already named by generate_extra_tensors()
if name.startswith("a."):
yield (name, data_torch)
return
if name == "text_model.model.embeddings.weight":
# semantic band of the backbone embeddings
begin = self.global_config["semantic_start_token_id"]
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
return
if name == "audio_decoder.codebook_embeddings.weight":
# backbone input of a speech row, one table per codebook
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1]))
return
if name == "audio_decoder.embeddings.weight":
yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch)
return
if name == "audio_decoder.output.weight":
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES])
return
if name == "audio_decoder.norm.weight":
yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
return
assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.")
key = name.removeprefix(f"audio_decoder.layers.{bid}.")
if key == "attention.wqkv.weight":
head_dim = cfg["head_dim"]
n_q = cfg["n_head"] * head_dim
n_kv = cfg["n_local_heads"] * head_dim
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim))
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim))
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
return
yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)