Files
llama.cpp/conversion/zonos3.py
T
Xuan-Son Nguyen 7ae3c6b231 mtmd: add zonos3-mini TTS model (#32020)
* zonos3 init

* add audio gen pipeline

* fix string_format

* add default voice as gguf metadata

* better ref

* support voice clone

* add delivery params

* optimize speed

* nits

* nits 2

* add text normalization

* nits
2026-11-15 19:44:44 +01:00

576 lines
30 KiB
Python

from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Callable, Iterable, TYPE_CHECKING
import numpy as np
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger
LM_DIR = Path("lm") / "zonos3_mini"
# codec decoder: quantizers and semantic head from the generator, the rest from the head checkpoint
CODEC_GENERATOR = Path("autoencoder") / "generator.pt"
CODEC_VARIANT = "keep2_plain"
CODEC_HEAD = Path("autoencoder") / "heads" / CODEC_VARIANT / "step_0050000.pt"
CODEC_STAGES = [0, 2, 4, 6, 8] # transformer stages of the decoder
DEFAULT_VOICE = "Savannah_en-American_F_1"
# speaker encoder (ownmect_s2), MECT_B2 blocks: (cnn depth, out channels, (stride f, stride t), expansion)
SPK_CKPT = Path("speaker_encoders") / "ownmect_s2" / "ckpt_0037.pth"
SPK_STAGES = [
[(3, 32, (1, 1), 4)] * 2,
[(3, 64, (2, 1), 2)] + [(3, 64, (1, 2), 2)] * 3,
[(3, 128, (2, 1), 2)] + [(3, 128, (1, 2), 2)] * 4,
[(3, 256, (2, 1), 1)] + [(2, 256, (1, 2), 1)] * 2,
]
VOICE_MAX_FRAMES = 900 # reference_max_frames of the reference
# GDN geometry, hard-coded in the reference (zonos3.reference.lm)
GDN_HEADS, GDN_HEAD_DIM, GDN_KERNEL = 4, 128, 4
# the text table keeps the tokenizer ids shifted by this offset, ids 2 and 3 are BOS and EOS
TEXT_OFFSET = 4
TEXT_BOS, TEXT_EOS = 2, 3
def _round_up(value: float, multiple: int) -> int:
return multiple * ((int(value) + multiple - 1) // multiple)
def _load_params(dir_model: Path) -> dict[str, Any] | None:
path = dir_model / LM_DIR / "params.json"
if not path.is_file():
return None
with open(path, encoding="utf-8") as f:
params = json.load(f).get("model", {})
return params if params.get("model_type") == "zonos2" else None
def _is_zonos3_checkpoint(dir_model: Path) -> bool:
return _load_params(dir_model) is not None
@ModelBase.register_hparams_loader(_is_zonos3_checkpoint)
def _load_zonos3_hparams(dir_model: Path) -> dict[str, Any]:
logger.info("gguf: detected ZONOS3 checkpoint")
params = _load_params(dir_model)
assert params is not None
for key, want in (("attention_pattern", "gdn3_global1"), ("residual_type", "full_attnres"),
("moe_router_type", "linear"), ("moe_router_use_eda", False), ("use_xsa", False),
("moe_end_from_layer", 0), ("speaker_enabled", True)):
if params.get(key) != want:
raise NotImplementedError(f"ZONOS3: unsupported {key} = {params.get(key)!r}, expected {want!r}")
text_config = dict(params, hidden_size=params["dim"])
return {"architectures": ["Zonos3Model"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}}
def _index_lm_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]:
path = model.dir_model / LM_DIR / "model.pth"
logger.info(f"gguf: indexing model part '{LM_DIR / 'model.pth'}'")
state = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
tensors: dict[str, Callable[[], Tensor]] = {}
for name, data in state.items():
gen = (lambda data=data: LazyTorchTensor.from_eager(data)) if model.lazy else (lambda data=data: data)
titem = model.filter_tensors((name, gen))
model._source_tensor_names[name] = titem[0] if titem else None
if titem:
tensors[titem[0]] = titem[1]
return tensors
@ModelBase.register("Zonos3Model")
class Zonos3TextModel(TextModel):
model_arch = gguf.MODEL_ARCH.ZONOS3
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hp = self.hparams
self.n_embd = hp["dim"]
self.n_codebooks = hp["n_codebooks"]
self.audio_vocab = hp["codebook_size"] + 2 # codes, EOA, pad
self.audio_pad_id = hp["audio_pad_id"]
self.text_rows = hp["text_vocab"] + 1 # + the padding row, used by the generated rows
self.cond_counts = [n + 1 for n in hp["condition_bucket_counts"]] # + the unknown bucket
self.gdn_period = int(hp["attention_pattern"][len("gdn"):].split("_")[0]) + 1
self.n_ff = _round_up(hp["ffn_dim_multiplier"] * hp["dim"], hp["multiple_of"])
self.n_ff_exp = _round_up(hp["moe_ffn_dim_multiplier"] * hp["dim"], hp["moe_multiple_of"])
self.n_latent = int(hp["dim"] * hp["moe_alpha"])
self._embd: dict[str, Tensor] = {}
self._router: dict[int, dict[str, Tensor]] = {}
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
return _index_lm_tensors(self)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
# audio heads and speaker projection go to the mmproj
if name.startswith(("multi_output.", "speaker_projection.")):
return None
return item
def set_vocab(self):
with open(self.dir_model / LM_DIR / "tokenizer" / "superbpe8192.json", encoding="utf-8") as f:
tok = json.load(f)
model = tok["model"]
if model["type"] != "BPE" or model.get("byte_fallback") or tok.get("normalizer") is not None:
raise NotImplementedError("ZONOS3: unexpected tokenizer type")
vocab: dict[str, int] = model["vocab"]
n_bpe = len(vocab)
if n_bpe + TEXT_OFFSET != self.hparams["text_vocab"]:
raise ValueError(f"ZONOS3: tokenizer has {n_bpe} tokens, text_vocab is {self.hparams['text_vocab']}")
added = {t["id"] for t in tok.get("added_tokens", [])}
tokens: list[str] = ["<|unused_0|>", "<|unused_1|>", "<|bos|>", "<|eos|>"]
toktypes: list[int] = [gguf.TokenType.CONTROL] * TEXT_OFFSET
inv = {i: t for t, i in vocab.items()}
for i in range(n_bpe):
tokens.append(inv[i])
toktypes.append(gguf.TokenType.CONTROL if i in added else gguf.TokenType.NORMAL)
# padding row of the text table (the text column of every generated row)
tokens.append("<|text_pad|>")
toktypes.append(gguf.TokenType.CONTROL)
# condition rows, the last bucket of each table is "unknown"
for cond, n in zip(self.hparams["condition_names"], self.cond_counts):
tokens += [f"<|{cond}_{i}|>" for i in range(n - 1)] + [f"<|{cond}_unk|>"]
toktypes += [gguf.TokenType.CONTROL] * n
merges = [" ".join(m) if isinstance(m, list) else m for m in model["merges"]]
self.gguf_writer.add_tokenizer_model("gpt2")
self.gguf_writer.add_tokenizer_pre("zonos3")
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_types(toktypes)
self.gguf_writer.add_token_merges(merges)
self.gguf_writer.add_bos_token_id(TEXT_BOS)
self.gguf_writer.add_eos_token_id(TEXT_EOS)
self.gguf_writer.add_pad_token_id(TEXT_OFFSET + 1) # <|pad|> of the tokenizer
# the reference wraps the text as [BOS] + ids + [EOS]
self.gguf_writer.add_add_bos_token(True)
self.gguf_writer.add_add_eos_token(True)
# the reference normalizes the text with NeMo, the template covers the English numbers
template_path = Path(__file__).parent.parent / "models" / "templates" / "Zyphra-ZONOS3-Mini-tts.jinja"
with open(template_path, "r", encoding="utf-8") as f:
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
def set_gguf_parameters(self):
hp = self.hparams
n_layer = self.block_count
self.gguf_writer.add_file_type(self.ftype)
self.gguf_writer.add_block_count(n_layer)
self.gguf_writer.add_context_length(hp["max_seqlen"])
self.gguf_writer.add_embedding_length(self.n_embd)
self.gguf_writer.add_feed_forward_length(self.n_ff)
self.gguf_writer.add_head_count(hp["n_heads"])
self.gguf_writer.add_head_count_kv(hp["n_kv_heads"])
self.gguf_writer.add_key_length(hp["head_dim"])
self.gguf_writer.add_value_length(hp["head_dim"])
self.gguf_writer.add_rope_dimension_count(hp["head_dim"])
self.gguf_writer.add_rope_freq_base(hp["rope_theta"])
self.gguf_writer.add_layer_norm_rms_eps(hp["norm_eps"])
self.gguf_writer.add_full_attention_interval(self.gdn_period)
# gated delta net
self.gguf_writer.add_ssm_conv_kernel(GDN_KERNEL)
self.gguf_writer.add_ssm_inner_size(GDN_HEADS * GDN_HEAD_DIM)
self.gguf_writer.add_ssm_state_size(GDN_HEAD_DIM)
self.gguf_writer.add_ssm_time_step_rank(GDN_HEADS)
self.gguf_writer.add_ssm_group_count(GDN_HEADS)
# latent MoE, the gating (softmax over the selected logits) is hard-coded in libllama
self.gguf_writer.add_leading_dense_block_count(hp["moe_start_from_layer"])
self.gguf_writer.add_expert_count(hp["moe_n_experts"])
self.gguf_writer.add_expert_used_count(hp["moe_router_topk"])
self.gguf_writer.add_expert_feed_forward_length(self.n_ff_exp)
self.gguf_writer.add_moe_latent_size(self.n_latent)
def tensor_force_quant(self, name, new_name, bid, n_dims):
# router logits pick the experts, keep them exact
if new_name.endswith(("ffn_gate_inp.weight", "exp_probs_b.bias")):
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)
def _token_embd(self) -> Tensor:
n_cb = self.n_codebooks
pad = sum(self._embd[f"multi_embedder.embedders.{cb}.weight"][self.audio_pad_id].float() for cb in range(n_cb))
text = self._embd[f"multi_embedder.embedders.{n_cb}.weight"].float()
if text.shape[0] != self.text_rows:
raise ValueError(f"ZONOS3: text table has {text.shape[0]} rows, expected {self.text_rows}")
conds = [self._embd[f"condition_embeddings.{i}.weight"].float() for i in range(len(self.cond_counts))]
for t, n in zip(conds, self.cond_counts):
assert t.shape[0] == n
# a text row is [pad x n_codebooks, text_id], condition rows replace the summed row
return torch.cat([text + pad] + conds, dim=0)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
T = gguf.MODEL_TENSOR
hp = self.hparams
n_layer = self.block_count
if name.startswith(("multi_embedder.", "condition_embeddings.")):
self._embd[name] = data_torch
if len(self._embd) == self.n_codebooks + 1 + len(self.cond_counts):
yield (self.format_tensor_name(T.TOKEN_EMBD), self._token_embd())
self._embd.clear()
return
if name == "block_attnres_queries":
# one query per routing point: 2*il before the mixer, 2*il+1 before the FFN, the last one before out_norm
assert data_torch.shape[0] == 2 * n_layer + 1
for il in range(n_layer):
yield (self.format_tensor_name(T.ATTN_RES_SCORE, il), data_torch[2 * il])
yield (self.format_tensor_name(T.FFN_RES_SCORE, il), data_torch[2 * il + 1])
yield (self.format_tensor_name(T.OUTPUT_RES_SCORE), data_torch[2 * n_layer])
return
if name == "out_norm.weight":
yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch)
return
assert bid is not None, name
key = name.split(".", 2)[2] # strip "layers.{bid}."
if key == "attention_norm.weight":
yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch)
return
if key == "ffn_norm.weight":
yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch)
return
# gated delta net
if key == "attention.in_proj.weight":
# split the fused [qkv | z | a | b] weight into the qwen35 GDN tensors
# attn_gate holds z, per-head output gate in the attention layers
inner = GDN_HEADS * GDN_HEAD_DIM
qkv, z, a, b = data_torch.split([3 * inner, inner, GDN_HEADS, GDN_HEADS], dim=0)
yield (self.format_tensor_name(T.ATTN_QKV, bid), qkv)
yield (self.format_tensor_name(T.ATTN_GATE, bid), z)
yield (self.format_tensor_name(T.SSM_ALPHA, bid), a)
yield (self.format_tensor_name(T.SSM_BETA, bid), b)
return
if key == "attention.conv.weight":
yield (self.format_tensor_name(T.SSM_CONV1D, bid), data_torch.squeeze(1))
return
if key == "attention.A_log":
yield (self.format_tensor_name(T.SSM_A, bid, suffix=""), -torch.exp(data_torch.float()))
return
if key == "attention.dt_bias":
yield (self.format_tensor_name(T.SSM_DT, bid, suffix=".bias"), data_torch)
return
if key == "attention.out_weight":
yield (self.format_tensor_name(T.SSM_NORM, bid), data_torch)
return
if key == "attention.wo.weight" and (bid + 1) % self.gdn_period != 0:
yield (self.format_tensor_name(T.SSM_OUT, bid), data_torch)
return
# global attention
if key == "attention.wq.weight":
yield (self.format_tensor_name(T.ATTN_Q, bid), data_torch)
return
if key == "attention.wkv.weight":
yield (self.format_tensor_name(T.ATTN_K, bid), data_torch[0])
yield (self.format_tensor_name(T.ATTN_V, bid), data_torch[1])
return
if key == "attention.wo.weight":
yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
return
if key == "attention.gater.weight":
# one sigmoid gate per head, same tensor name as the GDN z gate
yield (self.format_tensor_name(T.ATTN_GATE, bid), data_torch)
return
if key == "attention.temp":
# Q = rms_norm(q) * |temp|, the per-head scale is stored as the Q norm weight
temp = data_torch.float().abs().reshape(hp["n_heads"], 1).expand(hp["n_heads"], hp["head_dim"])
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), temp.contiguous())
return
# dense FFN (layer 0): out = w_out(w_in[0](x) * silu(w_in[1](x)))
if key == "feed_forward.w_in.weight":
yield (self.format_tensor_name(T.FFN_UP, bid), data_torch[0])
yield (self.format_tensor_name(T.FFN_GATE, bid), data_torch[1])
return
if key == "feed_forward.w_out.weight":
yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
return
# latent MoE
if key in ("feed_forward.router.router_linear.weight", "feed_forward.router.rmsnorm_eda.weight"):
# logits = router_linear(rms_norm(x) * w), the norm weight is folded into the router columns
r = self._router.setdefault(bid, {})
r[key] = data_torch
if len(r) == 2:
w = r["feed_forward.router.router_linear.weight"].float() * r["feed_forward.router.rmsnorm_eda.weight"].float()[None, :]
yield (self.format_tensor_name(T.FFN_GATE_INP, bid), w)
del self._router[bid]
return
if key == "feed_forward.router.balancing_biases":
# the reference selects on logits - bias
yield (self.format_tensor_name(T.FFN_EXP_PROBS_B, bid, suffix=".bias"), -data_torch.float())
return
if key == "feed_forward.latent_down.weight":
yield (self.format_tensor_name(T.MOE_LATENT_DOWN, bid), data_torch)
return
if key == "feed_forward.latent_up.weight":
yield (self.format_tensor_name(T.MOE_LATENT_UP, bid), data_torch)
return
if key == "feed_forward.experts.gate_up_proj":
gate, up = data_torch.chunk(2, dim=1)
yield (self.format_tensor_name(T.FFN_GATE_EXP, bid), gate)
yield (self.format_tensor_name(T.FFN_UP_EXP, bid), up)
return
if key == "feed_forward.experts.down_proj":
yield (self.format_tensor_name(T.FFN_DOWN_EXP, bid), data_torch)
return
raise ValueError(f"ZONOS3: unexpected tensor {name}")
@ModelBase.register("Zonos3Model")
class Zonos3MmprojModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hp = self.global_config["text_config"]
self.n_codebooks = hp["n_codebooks"]
self.text_vocab = hp["text_vocab"]
self._embd: dict[str, Tensor] = {}
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
return _index_lm_tensors(self)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, _ = item
if name.startswith(("multi_output.", "multi_embedder.", "speaker_projection.")):
return item
return None
def set_gguf_parameters(self):
self.gguf_writer.add_file_type(self.ftype)
# speaker encoder for voice cloning, the front-end hparams are hard-coded in tools/mtmd
self.gguf_writer.add_clip_has_audio_encoder(True)
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.ZONOS3_SPKENC)
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_audio_num_mel_bins(80)
self.gguf_writer.add_audio_block_count(0)
self.gguf_writer.add_audio_embedding_length(192)
self.gguf_writer.add_audio_head_count(1)
self.gguf_writer.add_audio_feed_forward_length(192)
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.ZONOS3_GEN)
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_gen_audio_embedding_length(self.n_embd_text)
self.gguf_writer.add_gen_audio_feed_forward_length(0)
self.gguf_writer.add_gen_audio_block_count(0)
self.gguf_writer.add_gen_audio_head_count(0)
self.gguf_writer.add_gen_audio_attention_layernorm_eps(self.global_config["text_config"]["norm_eps"])
self.gguf_writer.add_gen_audio_model_variant(CODEC_VARIANT)
if any(self.dir_model.glob(f"gguf/voices/*/{DEFAULT_VOICE}.codes.npy")):
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
# softcap and the codec hparams are hard-coded in tools/mtmd
def tensor_force_quant(self, name, new_name, bid, n_dims):
# read on the host, or used by get_rows / ggml_ssm_conv / the conv ops of the speaker encoder
if new_name.startswith(("voice.", "a.spk.")) or new_name.endswith((".table", ".dw.weight", ".mixer.weight", ".resample.weight")):
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)
@staticmethod
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
# torch weight norm, over every dim but 0
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
return (g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))).squeeze(-1)
def _speaker_encoder(self) -> Iterable[tuple[str, Tensor]]:
# same layout as the reference ggml runtime (tools/convert_gguf.py speaker-encoder)
sd = torch.load(self.dir_model / SPK_CKPT, map_location="cpu", weights_only=True)["modules"]["spk_model"]
sd = {k[len("model."):]: v.double() for k, v in sd.items() if not k.endswith("num_batches_tracked")}
p = "a.spk"
def bn(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
scale = sd[src + ".weight"] / torch.sqrt(sd[src + ".running_var"] + 1e-5)
yield (f"{p}.{dst}.scale", scale)
yield (f"{p}.{dst}.shift", sd[src + ".bias"] - sd[src + ".running_mean"] * scale)
def put(dst: str, t: Tensor) -> tuple[str, Tensor]:
return (f"{p}.{dst}", t)
fe = "feature_extractor."
yield put("fe.head.conv.weight", sd[fe + "head.0.weight"])
yield put("fe.head.conv.bias", sd[fe + "head.0.bias"])
yield from bn("fe.head.bn", fe + "head.1")
c_in, f_in = 32, 80
for si, stage in enumerate(SPK_STAGES):
for k, (depth, out_ch, (s_f, s_t), expansion) in enumerate(stage):
src, dst = f"{fe}mect_block{si + 1}.{k}.", f"fe.b{si + 1}.{k}."
for r in range(depth):
cs, cd = f"{src}cnn_block.{r}.", f"{dst}cnn.{r}."
# grouped 3x3 convs, one depthwise kernel per input slot of a group
for conv in ("conv1", "conv2"):
w = sd[cs + conv + ".weight"]
for j in range(w.shape[1]):
yield put(f"{cd}{conv}.w{j}", w[:, j:j + 1])
for i in (1, 2):
yield put(f"{cd}pw{i}.weight", sd[f"{cs}conv_pw{i}.weight"])
yield put(f"{cd}pw{i}.bias", sd[f"{cs}conv_pw{i}.bias"])
yield from bn(f"{cd}bn{i}", f"{cs}bn{i}")
if cs + "shortcut.0.weight" in sd:
yield put(cd + "sc.weight", sd[cs + "shortcut.0.weight"])
yield from bn(cd + "sc.bn", cs + "shortcut.1")
if expansion > 1:
yield put(dst + "reduce.weight", sd[src + "reduce_conv.0.weight"])
yield put(dst + "reduce.bias", sd[src + "reduce_conv.0.bias"])
yield from bn(dst + "reduce.bn", src + "reduce_conv.1")
yield put(dst + "rproj.weight", sd[src + "reduce_project.0.weight"].squeeze(-1))
yield put(dst + "rproj.bias", sd[src + "reduce_project.0.bias"])
yield put(dst + "rproj.ln.weight", sd[src + "reduce_project.1.weight"])
yield put(dst + "rproj.ln.bias", sd[src + "reduce_project.1.bias"])
yield put(dst + "pos.dw.weight", sd[src + "pos_conv.pos_conv.0.weight"].squeeze(1))
yield put(dst + "pos.dw.bias", sd[src + "pos_conv.pos_conv.0.bias"])
yield from bn(dst + "pos.bn", src + "pos_conv.pos_conv.1")
yield put(dst + "pos.pw.weight", sd[src + "pos_conv.pos_conv.3.weight"].squeeze(-1))
yield put(dst + "pos.pw.bias", sd[src + "pos_conv.pos_conv.3.bias"])
tr = src + "transformer."
for a, b in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj"), ("o", "out_proj")):
yield put(f"{dst}attn.{a}.weight", sd[f"{tr}attention.{b}.weight"])
yield put(f"{dst}attn.{a}.bias", sd[f"{tr}attention.{b}.bias"])
yield put(dst + "ln1.weight", sd[tr + "layer_norm.weight"])
yield put(dst + "ln1.bias", sd[tr + "layer_norm.bias"])
moe = tr + "feed_forward.intermediate_dense.token_moe."
yield put(dst + "moe.gate.weight", sd[moe + "gate.weight"])
yield put(dst + "moe.experts.weight", sd[moe + "experts.weight"])
yield put(dst + "ffn_out.weight", sd[tr + "feed_forward.output_dense.weight"])
yield put(dst + "ffn_out.bias", sd[tr + "feed_forward.output_dense.bias"])
yield put(dst + "ln2.weight", sd[tr + "final_layer_norm.weight"])
yield put(dst + "ln2.bias", sd[tr + "final_layer_norm.bias"])
yield put(dst + "eproj.weight", sd[src + "expansion_project.0.weight"].squeeze(-1))
yield put(dst + "eproj.bias", sd[src + "expansion_project.0.bias"])
c_in, f_in = c_in * s_f, f_in // s_f
# the stage outputs are mixed by softmax weights
yield put("fe.dense_w", torch.softmax(sd[fe + "dense3.inputs_weights"], dim=1)[0, :, :, 0])
yield put("pool.l1.weight", sd["encoding_layers.linear1.weight"].squeeze(-1))
yield put("pool.l1.bias", sd["encoding_layers.linear1.bias"])
yield put("pool.l2.weight", sd["encoding_layers.linear2.weight"].squeeze(-1))
yield put("pool.l2.bias", sd["encoding_layers.linear2.bias"])
yield from bn("bn", "bn")
yield put("linear.weight", sd["linear.weight"])
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
yield from self._speaker_encoder()
logger.info(f"gguf: loading codec decoder from '{CODEC_GENERATOR}' and '{CODEC_HEAD}'")
gen = torch.load(self.dir_model / CODEC_GENERATOR, map_location="cpu", mmap=True, weights_only=True)["generator"]
head = torch.load(self.dir_model / CODEC_HEAD, map_location="cpu", mmap=True, weights_only=True)["trainable"]
p = "a.gen.wav"
def convnext(dst: str, src: dict[str, Tensor], prefix: str) -> Iterable[tuple[str, Tensor]]:
yield (f"{dst}.dw.weight", src[f"{prefix}.dwconv.weight"].squeeze(1))
yield (f"{dst}.dw.bias", src[f"{prefix}.dwconv.bias"])
yield (f"{dst}.norm.weight", src[f"{prefix}.norm.weight"])
yield (f"{dst}.norm.bias", src[f"{prefix}.norm.bias"])
yield (f"{dst}.pw1.weight", src[f"{prefix}.pwconv1.weight"])
yield (f"{dst}.pw1.bias", src[f"{prefix}.pwconv1.bias"])
yield (f"{dst}.pw2.weight", src[f"{prefix}.pwconv2.weight"])
yield (f"{dst}.pw2.bias", src[f"{prefix}.pwconv2.bias"])
# RVQ codebooks folded through their out_proj, the biases are added once
def rvq(prefix: str) -> tuple[Tensor, Tensor]:
return gen[prefix + ".codebook.weight"].float() @ self._wn(gen, prefix + ".out_proj").T, gen[prefix + ".out_proj.bias"].float()
t, b = rvq("quantizer.semantic_quantizer.quantizers.0")
yield (f"{p}.sem.table", t)
yield (f"{p}.sem.bias", b)
res = [rvq(f"quantizer.quantizer.quantizers.{i}") for i in range(self.n_codebooks - 1)]
yield (f"{p}.res.table", torch.cat([t for t, _ in res], dim=0))
# one bias per residual codebook, a frame decoded before all its codes are known leaves the missing ones out
yield (f"{p}.res.bias", torch.stack([b for _, b in res]))
n_sem = sum(1 for k in gen if k.startswith("semantic_head.") and k.endswith(".dwconv.weight"))
for i in range(n_sem):
yield from convnext(f"{p}.sem.{i}", gen, f"semantic_head.{i}")
yield (f"{p}.sem.out.weight", self._wn(gen, f"semantic_head.{n_sem}"))
yield (f"{p}.sem.out.bias", gen[f"semantic_head.{n_sem}.bias"])
names = {"norm1.weight": "norm1.weight", "norm1.bias": "norm1.bias", "qkv.weight": "qkv.weight",
"attention_out.weight": "wo.weight", "attention_scale": "attn_scale", "norm2.weight": "norm2.weight",
"norm2.bias": "norm2.bias", "ffn.0.weight": "ffn1.weight", "ffn.3.weight": "ffn2.weight", "ffn_scale": "ffn_scale"}
for si, s in enumerate(CODEC_STAGES):
yield (f"{p}.stage.{si}.in.weight", head[f"stages.{s}.input_proj.weight"])
if f"stages.{s}.output_proj.weight" in head:
yield (f"{p}.stage.{si}.out.weight", head[f"stages.{s}.output_proj.weight"])
n_layer = sum(1 for k in head if k.startswith(f"stages.{s}.layers.") and k.endswith(".qkv.weight"))
for il in range(n_layer):
for src, dst in names.items():
yield (f"{p}.stage.{si}.blk.{il}.{dst}", head[f"stages.{s}.layers.{il}.{src}"])
n_head = sum(1 for k in head if k.startswith("head.blocks.") and k.endswith(".dwconv.weight"))
for i in range(n_head):
yield from convnext(f"{p}.head.{i}", head, f"head.blocks.{i}")
yield (f"{p}.head.norm.weight", head["head.norm.weight"])
yield (f"{p}.head.norm.bias", head["head.norm.bias"])
# both iSTFT branches in one projection
yield (f"{p}.head.out.weight", torch.cat([head["head.outs.0.weight"], head["head.outs.1.weight"]], dim=0))
yield (f"{p}.head.out.bias", torch.cat([head["head.outs.0.bias"], head["head.outs.1.bias"]], dim=0))
# read on the host
yield (f"{p}.mixer.weight", head["head.mixer.weight"].squeeze(0))
yield (f"{p}.resample.weight", gen["to_external.kernel"].squeeze(1))
# "reference" speaker mode, the default of the reference: the voice is a prefix of delayed code rows
def _voices(self, tables: list[Tensor]) -> Iterable[tuple[str, Tensor]]:
n_cb, pad = self.n_codebooks, self.global_config["text_config"]["audio_pad_id"]
for path in sorted((self.dir_model / "gguf" / "voices").glob("*/*.codes.npy")):
codes = torch.from_numpy(np.load(path, allow_pickle=False).astype(np.int64))
if codes.ndim != 2 or codes.shape[1] != n_cb or codes.min() < 0 or codes.max() >= pad - 1:
logger.warning(f"voice {path.name}: skipped, expected [frames, {n_cb}] codes")
continue
codes = codes[:VOICE_MAX_FRAMES]
n_frames = codes.shape[0]
# codebook cb of frame t is in row t + cb, the other cells are pad
rows = torch.full((n_frames + n_cb - 1, n_cb), pad, dtype=torch.int64)
for cb in range(n_cb):
rows[cb:cb + n_frames, cb] = codes[:, cb]
embd = sum(tables[cb][rows[:, cb]] for cb in range(n_cb))
yield (f"voice.{path.name[:-len('.codes.npy')]}", embd)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
T = gguf.MODEL_TENSOR
if name.startswith(("a.gen.wav.", "a.spk.")):
yield (name, data_torch)
return
# the speaker encoder ends with the speaker projection of the backbone
if name.startswith("speaker_projection."):
yield ("a.spk.proj." + name.split(".")[-1], data_torch)
return
if name == "multi_output.weight":
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch)
return
assert name.startswith("multi_embedder.")
self._embd[name] = data_torch
if len(self._embd) < self.n_codebooks + 1:
return
# every row built by the mmproj has text id = text_vocab, so that row is folded into codebook 0
tables = [self._embd[f"multi_embedder.embedders.{cb}.weight"].float() for cb in range(self.n_codebooks)]
tables[0] = tables[0] + self._embd[f"multi_embedder.embedders.{self.n_codebooks}.weight"][self.text_vocab].float()
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), torch.cat(tables, dim=0))
yield from self._voices([LazyTorchTensor.to_eager(t) for t in tables])
self._embd.clear()