* mtmd: drama3 voice cloning The Fish S2 codec encoder (fish_s2_spkenc) now also serves drama3: - conversion: drama3 writes the codec encoder and the audio encoder KV, shared with fish_s2 - fish_s2_spkenc: outputs the codes of the clip (gen-side reference) and, in a drama3_gen mmproj, builds the drama3 backbone rows (E_lm[semantic] + E_cb[1][code 1]) / sqrt(2) - drama3_gen tower: teacher forcing when the caller gives the codes of the next frame - drama3 pipeline: reference-bracket-v1 prompt with one reference (vLLM-Omni fish_omni), the reference frames go through the backbone as codec rows and through the tower with their codes - mtmd_gen_state_get_ref_codes(): read-only access to the reference codes of a gen state Claude-Session: https://claude.ai/code/session_01PpX3DmaxNSmJbxGaxnUBrX * fix correctness * clean up * rm redundant code * fish_s2: rm unused import --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
145 lines
6.2 KiB
Python
145 lines
6.2 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from torch import Tensor
|
|
|
|
from .base import ModelBase, gguf
|
|
from .drama3 import Drama3MmprojModel, _neox_rows
|
|
from .qwen import Qwen3Model
|
|
|
|
# Fish Audio S2 Pro (dual-AR)
|
|
# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame
|
|
# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3)
|
|
|
|
# residual codebook size of codec.pth, the fast AR head is cut to it
|
|
_N_RESIDUAL_CODES = 1024
|
|
|
|
|
|
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
|
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
|
cfg = hparams["text_config"]
|
|
hparams["text_config"] = {
|
|
"hidden_size": cfg["dim"],
|
|
"num_hidden_layers": cfg["n_layer"],
|
|
"num_attention_heads": cfg["n_head"],
|
|
"num_key_value_heads": cfg["n_local_heads"],
|
|
"head_dim": cfg["head_dim"],
|
|
"intermediate_size": cfg["intermediate_size"],
|
|
"rms_norm_eps": cfg["norm_eps"],
|
|
"rope_theta": cfg["rope_base"],
|
|
"max_position_embeddings": cfg["max_seq_len"],
|
|
"vocab_size": cfg["vocab_size"],
|
|
"tie_word_embeddings": cfg["tie_word_embeddings"],
|
|
}
|
|
return hparams
|
|
|
|
|
|
@ModelBase.register("fish_qwen3_omni")
|
|
@ModelBase.example("fishaudio/s2-pro")
|
|
class FishS2TextModel(Qwen3Model):
|
|
model_arch = gguf.MODEL_ARCH.QWEN3
|
|
|
|
def __init__(self, dir_model: Path, *args, **kwargs):
|
|
kwargs.pop("hparams", None)
|
|
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
# the rest is mapped by tensor_mapping.py
|
|
name = name.removeprefix("text_model.")
|
|
T = gguf.MODEL_TENSOR
|
|
head_dim = self.hparams["head_dim"]
|
|
if name.endswith(".attention.wqkv.weight"):
|
|
n_q = self.hparams["num_attention_heads"] * head_dim
|
|
n_kv = self.hparams["num_key_value_heads"] * head_dim
|
|
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
|
yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim))
|
|
yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim))
|
|
yield (self.format_tensor_name(T.ATTN_V, bid), v)
|
|
return
|
|
if name.endswith(".attention.q_norm.weight"):
|
|
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim))
|
|
return
|
|
if name.endswith(".attention.k_norm.weight"):
|
|
yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim))
|
|
return
|
|
yield from super().modify_tensors(data_torch, name, bid)
|
|
|
|
|
|
@ModelBase.register("fish_qwen3_omni")
|
|
@ModelBase.example("fishaudio/s2-pro")
|
|
class FishS2MmprojModel(Drama3MmprojModel):
|
|
gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
|
|
|
|
_FAST_AR_MAP = {
|
|
"attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
|
|
"attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
|
|
"ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
|
|
"feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
|
|
"feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
|
|
"feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
|
|
}
|
|
|
|
def __init__(self, dir_model: Path, *args, **kwargs):
|
|
kwargs.pop("hparams", None)
|
|
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
|
|
|
def get_audio_config(self) -> dict[str, Any] | None:
|
|
cfg = dict(self.global_config["audio_decoder_config"])
|
|
cfg["num_hidden_layers"] = cfg["n_layer"]
|
|
return cfg
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
|
name = item[0]
|
|
if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight":
|
|
return item
|
|
return None
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
T = gguf.MODEL_TENSOR
|
|
cfg = self.hparams
|
|
|
|
# already named by generate_extra_tensors()
|
|
if name.startswith("a."):
|
|
yield (name, data_torch)
|
|
return
|
|
|
|
if name == "text_model.model.embeddings.weight":
|
|
# semantic band of the backbone embeddings
|
|
begin = self.global_config["semantic_start_token_id"]
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
|
|
return
|
|
|
|
if name == "audio_decoder.codebook_embeddings.weight":
|
|
# backbone input of a speech row, one table per codebook
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1]))
|
|
return
|
|
|
|
if name == "audio_decoder.embeddings.weight":
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch)
|
|
return
|
|
|
|
if name == "audio_decoder.output.weight":
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES])
|
|
return
|
|
|
|
if name == "audio_decoder.norm.weight":
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
|
|
return
|
|
|
|
assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.")
|
|
key = name.removeprefix(f"audio_decoder.layers.{bid}.")
|
|
if key == "attention.wqkv.weight":
|
|
head_dim = cfg["head_dim"]
|
|
n_q = cfg["n_head"] * head_dim
|
|
n_kv = cfg["n_local_heads"] * head_dim
|
|
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim))
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim))
|
|
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
|
|
return
|
|
yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
|