Files
llama.cpp/conversion/moshi.py
T
Xuan-Son NguyenandPascal c622f623fc mtmd: support kyutai/hibiki realtime full-duplex (#32491)
* mtmd: support hibiki realtime duplex

* fix and clean up

* fix long session quality

* fix ci

* tests : add the moshi fusion baseline for Metal

The Metal fusion check found 5 RMS_NORM+MUL fusions in the new moshi
arch and no baseline for them.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
2026-11-21 14:25:19 +01:00

379 lines
19 KiB
Python

from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Callable, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, MmprojModel, SentencePieceTokenTypes, TextModel, gguf, logger
from .pockettts import mimi_seanet_tensors, mimi_tfm_tensors
# Moshi family (Kyutai): a backbone over parallel streams (text + generated audio codes + input audio codes),
# one position per 80 ms frame. Only Hibiki (simultaneous speech translation) is supported for now.
# The checkpoint is not in the HF layout: a moshi-style config.json and two safetensors (LM + mimi).
#
# pipeline stage mapping:
# transformer + text_emb/text_linear --> normal libllama text model (arch moshi)
# mimi encoder + RVQ + emb[dep_q:] --> mtmd audio stream (projector moshi), rows of the input audio
# depformer + emb[:dep_q] --> MTMD_GEN_PROCESS_TYPE_GEN_CODE, codes of a frame + their backbone embedding
# RVQ decode + mimi decoder --> MTMD_GEN_PROCESS_TYPE_GEN_WAV
#
# Tricks:
# - the initial text token (text_card) is appended to the vocab as <|text_start|>, the head has no logit for it;
# it only opens the stream, the embeddings of the initial codes of the generated audio are folded into it
# - the condition of Hibiki ("very_good" description, added to every row) is folded into the text embeddings,
# every row has exactly one text embedding
# - Hibiki ignores an early EOS: the reference gives it the embedding of PAD, done here too
# - low-rank depformer embeddings are folded into full tables
_MIMI_PREFIX = "mimi."
_COND_VALUE = "very_good"
def _read_config(dir_model: Path) -> dict[str, Any] | None:
path = dir_model / "config.json"
if not path.is_file():
return None
with open(path, "r", encoding="utf-8") as f:
config = json.load(f)
return config if isinstance(config, dict) and "moshi_name" in config else None
def _tensor_shape(path: Path, name: str) -> tuple[int, ...]:
with gguf.utility.SafetensorsLocal(path) as part:
return tuple(part[name].shape)
@ModelBase.register_hparams_loader(lambda dir_model: _read_config(dir_model) is not None)
def _load_hparams(dir_model: Path) -> dict[str, Any]:
config = _read_config(dir_model)
assert config is not None
model_type = config.get("model_type", "moshi")
if model_type != "hibiki":
raise NotImplementedError(f"moshi checkpoint of type {model_type!r} is not supported yet")
logger.info(f"gguf: detected a moshi checkpoint ({model_type})")
# mtmd has the layout of Hibiki: the acoustic codes lag by 2 frames, the depformer has no positional embedding,
# its steps past the last weight set reuse it, and the initial code is the extra entry after the mimi codebook
n_q = config["dep_q"]
schedule = config.get("depformer_weights_per_step_schedule") or list(range(n_q))
# text, then the generated and the input codes
if config["delays"] != [0] + ([0] + [2] * (n_q - 1)) * 2:
raise NotImplementedError(f"unsupported delays {config['delays']}")
if schedule != [min(i, max(schedule)) for i in range(n_q)] or config.get("depformer_pos_emb", "none") != "none":
raise NotImplementedError("unsupported depformer layout")
if config["card"] != 2048 or config["n_q"] != 2 * n_q:
raise NotImplementedError("unsupported audio codebooks")
lm_path = dir_model / config["moshi_name"]
# the gating hidden size is derived from dim_feedforward, read it back from the tensors
n_ff = _tensor_shape(lm_path, "transformer.layers.0.gating.linear_out.weight")[1]
n_ff_dep = _tensor_shape(lm_path, "depformer.layers.0.gating.0.linear_out.weight")[1]
return {
"architectures": ["MoshiModel"],
"model_type": "moshi",
"moshi_config": config,
"num_hidden_layers": config["num_layers"],
"hidden_size": config["dim"],
"intermediate_size": n_ff,
"num_attention_heads": config["num_heads"],
"num_key_value_heads": config["num_heads"],
"max_position_embeddings": config["context"],
"sliding_window": config["context"],
"rope_theta": config["max_period"],
"rms_norm_eps": 1e-8, # rms_norm_f32
"vocab_size": config["text_card"] + 1,
"audio_config": {
# mimi encoder transformer, constant across the family
"num_hidden_layers": 8,
"hidden_size": 512,
"intermediate_size": 2048,
"num_attention_heads": 8,
},
"depformer_config": {
"num_hidden_layers": config["depformer_num_layers"],
"hidden_size": config["depformer_dim"],
"intermediate_size": n_ff_dep,
"num_attention_heads": config["depformer_num_heads"],
},
}
# the LM and mimi weights are in two files named by the config, mimi tensors get the "mimi." prefix
def _moshi_config(model: ModelBase) -> dict[str, Any]:
config = model.hparams.get("moshi_config")
if config is None and isinstance(model, MmprojModel):
config = model.global_config["moshi_config"] # the mmproj moves the hparams aside
assert config is not None
return config
def _index_moshi_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]:
config = _moshi_config(model)
tensors = model.index_local_tensors(Path(config["moshi_name"]).stem)
for name, gen in model.index_local_tensors(Path(config["mimi_name"]).stem).items():
tensors[_MIMI_PREFIX + name] = gen
return tensors
def _low_rank_embd(model: ModelBase, prefix: str) -> Tensor:
embd = model.model_tensors[prefix + ".weight"]().float()
proj = model.model_tensors.get(prefix + ".low_rank.weight")
return embd @ proj().float().T if proj is not None else embd
def _condition(model: ModelBase, value: str) -> Tensor:
# LUT conditioner: output_proj(embed[index of value])
lut = _moshi_config(model)["conditioners"]["description"]["lut"]
prefix = "condition_provider.conditioners.description."
embed = model.model_tensors[prefix + "embed.weight"]().float()
proj = model.model_tensors[prefix + "output_proj.weight"]().float()
return embed[lut["possible_values"].index(value)] @ proj.T
@ModelBase.register("MoshiModel")
@ModelBase.example("kyutai/hibiki-1b-pytorch-bf16", "kyutai/hibiki-2b-pytorch-bf16")
class MoshiModel(TextModel):
model_arch = gguf.MODEL_ARCH.MOSHI
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
assert remote_hf_model_id is None
return _index_moshi_tensors(self)
def set_vocab(self):
from sentencepiece import SentencePieceProcessor
config = _moshi_config(self)
sp = SentencePieceProcessor()
sp.LoadFromFile(str(self.dir_model / config["tokenizer_name"]))
assert sp.vocab_size() == config["text_card"]
tokens: list[bytes] = []
scores: list[float] = []
toktypes: list[int] = []
for i in range(sp.vocab_size()):
toktype = SentencePieceTokenTypes.NORMAL
if sp.IsControl(i) or sp.IsUnknown(i):
# <unk> (0) is the end-of-padding marker of the text stream, it has no text
toktype = SentencePieceTokenTypes.CONTROL
elif sp.IsUnused(i):
toktype = SentencePieceTokenTypes.UNUSED
elif sp.IsByte(i):
toktype = SentencePieceTokenTypes.BYTE
tokens.append(sp.IdToPiece(i).encode("utf-8"))
scores.append(sp.GetScore(i))
toktypes.append(toktype)
# the initial token of the text stream
tokens.append(b"<|text_start|>")
scores.append(-1000.0)
toktypes.append(SentencePieceTokenTypes.CONTROL)
self.gguf_writer.add_tokenizer_model("llama")
self.gguf_writer.add_tokenizer_pre("default")
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_scores(scores)
self.gguf_writer.add_token_types(toktypes)
self.gguf_writer.add_unk_token_id(sp.unk_id())
self.gguf_writer.add_bos_token_id(sp.bos_id())
self.gguf_writer.add_eos_token_id(sp.eos_id())
self.gguf_writer.add_pad_token_id(sp.pad_id())
self.gguf_writer.add_add_bos_token(False)
self.gguf_writer.add_add_eos_token(False)
def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
# sampling of the text tokens, the audio codes are sampled by mtmd
gen = _moshi_config(self).get("lm_gen_config", {})
self.gguf_writer.add_sampling_temp(gen.get("temp_text", 0.7))
self.gguf_writer.add_sampling_top_k(gen.get("top_k_text", 25))
self.gguf_writer.add_sampling_top_p(1.0)
self.gguf_writer.add_sampling_min_p(0.0)
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
T = gguf.MODEL_TENSOR
if name == "text_emb.weight":
config = _moshi_config(self)
embd = data_torch.float().clone()
embd[2] = embd[3] # EOS -> PAD, see get_moshi() of the reference
embd += _condition(self, _COND_VALUE)
assert embd.shape[0] == config["text_card"] + 1
# <|text_start|> only opens the stream, with the initial codes of the generated audio
for k in range(config["dep_q"]):
embd[-1] += self.model_tensors[f"emb.{k}.weight"]()[config["card"]].float()
yield (self.format_tensor_name(T.TOKEN_EMBD), embd)
return
if name == "text_linear.weight":
yield (self.format_tensor_name(T.OUTPUT), data_torch)
return
if name == "out_norm.alpha":
yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch.reshape(-1))
return
if not name.startswith("transformer.layers."):
return # mimi, depformer and the audio embeddings go to the mmproj
assert bid is not None
key = name.split(f"layers.{bid}.", 1)[1]
if key == "self_attn.in_proj_weight":
q, k, v = data_torch.chunk(3, dim=0)
yield (self.format_tensor_name(T.ATTN_Q, bid), q)
yield (self.format_tensor_name(T.ATTN_K, bid), k)
yield (self.format_tensor_name(T.ATTN_V, bid), v)
elif key == "self_attn.out_proj.weight":
yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
elif key == "norm1.alpha":
yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch.reshape(-1))
elif key == "norm2.alpha":
yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch.reshape(-1))
elif key == "gating.linear_in.weight":
# silu(first half) * second half
gate, up = data_torch.chunk(2, dim=0)
yield (self.format_tensor_name(T.FFN_GATE, bid), gate)
yield (self.format_tensor_name(T.FFN_UP, bid), up)
elif key == "gating.linear_out.weight":
yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
else:
raise ValueError(f"unexpected tensor {name}")
@ModelBase.register("MoshiModel")
@ModelBase.example("kyutai/hibiki-1b-pytorch-bf16", "kyutai/hibiki-2b-pytorch-bf16")
class MoshiMmprojModel(MmprojModel):
has_audio_encoder = True
has_vision_encoder = False
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
assert remote_hf_model_id is None
return _index_moshi_tensors(self)
def _n_codebooks(self) -> int:
return _moshi_config(self)["dep_q"]
def set_gguf_parameters(self):
self.gguf_writer.add_file_type(self.ftype)
assert self.hparams_audio is not None
dep = self.global_config["depformer_config"]
# input audio: mimi encoder + RVQ + embeddings of the input codes
self.gguf_writer.add_clip_has_audio_encoder(True)
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.MOSHI)
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
# mimi convolves the waveform directly, it is passed around as a 1-row "mel"
self.gguf_writer.add_audio_num_mel_bins(1)
# generation: depformer + mimi decoder, the mimi hparams are constant and held by mtmd
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.MOSHI_GEN)
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_gen_audio_embedding_length(dep["hidden_size"])
self.gguf_writer.add_gen_audio_feed_forward_length(dep["intermediate_size"])
self.gguf_writer.add_gen_audio_block_count(dep["num_hidden_layers"])
self.gguf_writer.add_gen_audio_head_count(dep["num_attention_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-8)
self.gguf_writer.add_gen_audio_model_variant(_moshi_config(self)["model_type"])
def tensor_force_quant(self, name, new_name, bid, n_dims):
del name, bid, n_dims
# conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"):
return gguf.GGMLQuantizationType.F16
# nearest-code search and RVQ decode stay exact
if ".codebook" in new_name or new_name.startswith("a.codec.vq."):
return gguf.GGMLQuantizationType.F32
return False
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
del bid
T = gguf.MODEL_TENSOR
if name.startswith(("a.", "mm.a.")):
yield (name, data_torch) # already named by generate_extra_tensors()
return
if not name.startswith(_MIMI_PREFIX):
return # the LM tensors are stacked in generate_extra_tensors()
if name == "mimi.downsample.conv.conv.conv.weight":
yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), data_torch)
elif name == "mimi.upsample.convtr.convtr.convtr.weight":
yield (self.format_tensor_name(T.A_GEN_WAV_UPSAMPLE), data_torch)
elif "_transformer.transformer.layers." in name:
# pocket-tts names it in_proj.weight
yield from mimi_tfm_tensors(self, name.replace(".in_proj_weight", ".in_proj.weight"), data_torch)
elif name.startswith(("mimi.encoder.model.", "mimi.decoder.model.")):
yield from mimi_seanet_tensors(self, name, data_torch, 4)
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
T = gguf.MODEL_TENSOR
config = _moshi_config(self)
n_q = self._n_codebooks()
def get(name: str) -> Tensor:
return self.model_tensors[name]()
def stack(names: Iterable[str]) -> Tensor:
return torch.stack([get(n) for n in names], dim=0)
# mimi RVQ: one semantic codebook, then n_q - 1 acoustic ones
def codebooks(prefix: str, n: int) -> Tensor:
books = []
for i in range(n):
embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum").float()
cluster_usage = get(f"{prefix}.vq.layers.{i}._codebook.cluster_usage").float()
books.append(embedding_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1))
return torch.stack(books, dim=0) if n > 1 else books[0]
q = "mimi.quantizer."
cb_first = codebooks(q + "rvq_first", 1)
cb_rest = codebooks(q + "rvq_rest", n_q - 1)
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN), get(q + "rvq_first.input_proj.weight").squeeze(-1))
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb_first)
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN), get(q + "rvq_rest.input_proj.weight").squeeze(-1))
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), cb_rest)
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT), get(q + "rvq_first.output_proj.weight").squeeze(-1))
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), cb_first)
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_OUT), get(q + "rvq_rest.output_proj.weight").squeeze(-1))
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB), cb_rest)
# backbone embeddings of the audio codes: the generated stream first, then the input stream
assert config["n_q"] == 2 * n_q
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), stack(f"emb.{k}.weight" for k in range(n_q)))
yield (self.format_tensor_name(T.A_MM_CODE_EMBD), stack(f"emb.{k}.weight" for k in range(n_q, 2 * n_q)))
# depformer: per-step weights are stacked on the last ggml dim, steps past the last weight set reuse it
dep = self.global_config["depformer_config"]
n_dim = dep["hidden_size"]
n_in = sum(1 for name in self.model_tensors if name.startswith("depformer_in."))
yield (self.format_tensor_name(T.A_GEN_CODE_PROJ_IN), stack(f"depformer_in.{i}.weight" for i in range(n_in)))
yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), _low_rank_embd(self, "depformer_text_emb"))
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD),
torch.stack([_low_rank_embd(self, f"depformer_emb.{i}") for i in range(n_q - 1)], dim=0))
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), stack(f"linears.{k}.weight" for k in range(n_q)))
for il in range(dep["num_hidden_layers"]):
p = f"depformer.layers.{il}."
n_w = sum(1 for name in self.model_tensors if name.startswith(p + "gating.") and name.endswith(".linear_in.weight"))
assert n_w == n_in, "depformer_in and the per-step weights use the same schedule"
qkv = get(p + "self_attn.in_proj_weight").reshape(n_w, 3, n_dim, n_dim)
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, il), qkv[:, 0])
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, il), qkv[:, 1])
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, il), qkv[:, 2])
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_OUT, il), get(p + "self_attn.out_proj.weight").reshape(n_w, n_dim, n_dim))
gate_up = stack(f"{p}gating.{i}.linear_in.weight" for i in range(n_w))
gate, up = gate_up.chunk(2, dim=1)
yield (self.format_tensor_name(T.A_GEN_CODE_FFN_GATE, il), gate)
yield (self.format_tensor_name(T.A_GEN_CODE_FFN_UP, il), up)
yield (self.format_tensor_name(T.A_GEN_CODE_FFN_DOWN, il), stack(f"{p}gating.{i}.linear_out.weight" for i in range(n_w)))
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_NORM, il), get(p + "norm1.alpha").reshape(-1))
yield (self.format_tensor_name(T.A_GEN_CODE_FFN_NORM, il), get(p + "norm2.alpha").reshape(-1))