Files
llama.cpp/conversion/voxcpm.py
T
Xuan-Son Nguyen e4b620af19 mtmd: support voxcpm2 (#32443)
* mtmd: support voxcpm

* speed up a bit

* various fixes
2026-11-19 02:39:05 +01:00

315 lines
15 KiB
Python

from __future__ import annotations
import json
import math
import re
from pathlib import Path
from typing import Any, Callable, Iterable, TYPE_CHECKING
import torch
if TYPE_CHECKING:
from torch import Tensor
from .base import ModelBase, MmprojModel, TextModel, LazyTorchTensor, gguf, logger
from .drama3 import fold_snake
from .minicpm import MiniCPMModel
# sample rate of the AudioVAE decoder output, picks the row of the sample-rate conditioning
VAE_OUT_SAMPLE_RATE = 48000
def _load_config(dir_model: Path) -> dict[str, Any] | None:
path = dir_model / "config.json"
if not path.is_file():
return None
with open(path, encoding="utf-8") as f:
config = json.load(f)
return config if config.get("architecture") == "voxcpm2" else None
def _is_voxcpm2_checkpoint(dir_model: Path) -> bool:
return _load_config(dir_model) is not None
@ModelBase.register_hparams_loader(_is_voxcpm2_checkpoint)
def _load_voxcpm2_hparams(dir_model: Path) -> dict[str, Any]:
logger.info("gguf: detected VoxCPM2 checkpoint")
config = _load_config(dir_model)
assert config is not None
if config["lm_config"].get("use_mup"):
raise NotImplementedError("VoxCPM2: use_mup = true is not supported")
if config["dit_config"].get("mean_mode"):
raise NotImplementedError("VoxCPM2: dit mean_mode = true is not supported")
if not config.get("residual_lm_no_rope"):
raise NotImplementedError("VoxCPM2: residual LM with RoPE is not supported")
for key in ("hidden_dim", "ffn_dim", "num_heads", "num_layers", "kv_channels"):
if config["encoder_config"][key] != config["dit_config"][key]:
raise NotImplementedError(f"VoxCPM2: LocEnc and LocDiT differ in {key}")
return {**config, "architectures": ["VoxCPM2Model"], "text_config": config["lm_config"], "audio_config": config["dit_config"]}
@ModelBase.register("VoxCPM2Model")
@ModelBase.example("openbmb/VoxCPM2")
class VoxCPM2TextModel(MiniCPMModel):
model_arch = gguf.MODEL_ARCH.MINICPM
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
name, gen = item
# the text-semantic LM is a plain MiniCPM, the rest goes to the mmproj
if not name.startswith("base_lm."):
return None
return super().filter_tensors(("model." + name.removeprefix("base_lm."), gen))
def set_gguf_parameters(self):
TextModel.set_gguf_parameters(self)
# use_mup = false: no embedding, residual or logit scaling
self.gguf_writer.add_embedding_scale(1.0)
self.gguf_writer.add_residual_scale(1.0)
self.gguf_writer.add_logit_scale(1.0)
def set_vocab(self):
# the checkpoint has no tokenizer.model and its tokenizer class is remote code, so read tokenizer.json
# SPM-style BPE: a merged token scores minus its merge rank, so the SPM tokenizer merges in the BPE order
with open(self.dir_model / "tokenizer.json", encoding="utf-8") as f:
tok = json.load(f)
with open(self.dir_model / "tokenizer_config.json", encoding="utf-8") as f:
tok_cfg = json.load(f)
model = tok["model"]
if model["type"] != "BPE" or not model.get("byte_fallback"):
raise NotImplementedError("VoxCPM2: unexpected tokenizer type")
vocab: dict[str, int] = model["vocab"]
vocab_size = self.hparams["vocab_size"]
tokens = [f"[PAD{i}]" for i in range(vocab_size)]
scores = [-1e9] * vocab_size
toktypes = [gguf.TokenType.UNUSED] * vocab_size
for text, i in vocab.items():
tokens[i] = text
toktypes[i] = gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-Fa-f]{2}>", text) else gguf.TokenType.NORMAL
for rank, merge in enumerate(model["merges"]):
a, b = merge.split(" ") if isinstance(merge, str) else merge
i = vocab[a + b]
scores[i] = max(scores[i], -float(rank))
# the special tokens (e.g. <|audio_start|>) are only listed in tokenizer_config.json
specials = {int(i): t["content"] for i, t in tok_cfg.get("added_tokens_decoder", {}).items() if t.get("special")}
specials.update({t["id"]: t["content"] for t in tok["added_tokens"] if t.get("special")})
for i, text in specials.items():
tokens[i] = text
toktypes[i] = gguf.TokenType.CONTROL
self.gguf_writer.add_tokenizer_model("llama")
self.gguf_writer.add_tokenizer_pre("default")
self.gguf_writer.add_token_list(tokens)
self.gguf_writer.add_token_scores(scores)
self.gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
special_vocab.add_to_gguf(self.gguf_writer)
@ModelBase.register("VoxCPM2Model")
@ModelBase.example("openbmb/VoxCPM2")
class VoxCPM2MmprojModel(MmprojModel):
has_vision_encoder = False
has_audio_encoder = True # AudioVAE encoder, for voice cloning
_LAYER_MAP = {
"input_layernorm.weight": "ln1",
"self_attn.q_proj.weight": "attn_q",
"self_attn.k_proj.weight": "attn_k",
"self_attn.v_proj.weight": "attn_v",
"self_attn.o_proj.weight": "attn_out",
"post_attention_layernorm.weight": "ln2",
"mlp.gate_proj.weight": "ffn_gate",
"mlp.up_proj.weight": "ffn_up",
"mlp.down_proj.weight": "ffn_down",
}
# libllama names, mtmd loads the residual LM as a text model
_RES_LAYER_MAP = {
"input_layernorm.weight": "attn_norm",
"self_attn.q_proj.weight": "attn_q",
"self_attn.k_proj.weight": "attn_k",
"self_attn.v_proj.weight": "attn_v",
"self_attn.o_proj.weight": "attn_output",
"post_attention_layernorm.weight": "ffn_norm",
"mlp.gate_proj.weight": "ffn_gate",
"mlp.up_proj.weight": "ffn_up",
"mlp.down_proj.weight": "ffn_down",
}
_PROJ_MAP = {
"fsq_layer.in_proj": "a.gen.fsq.in",
"fsq_layer.out_proj": "a.gen.fsq.out",
"enc_to_lm_proj": "a.gen.enc_to_lm",
"lm_to_dit_proj": "a.gen.lm_to_dit",
"res_to_dit_proj": "a.gen.res_to_dit",
"fusion_concat_proj": "a.gen.fusion",
"stop_proj": "a.gen.stop_proj",
"stop_head": "a.gen.stop_head",
"feat_encoder.in_proj": "a.gen.enc.in_proj",
"feat_decoder.estimator.in_proj": "a.gen.dit.in_proj",
"feat_decoder.estimator.cond_proj": "a.gen.dit.cond_proj",
"feat_decoder.estimator.out_proj": "a.gen.dit.out_proj",
"feat_decoder.estimator.time_mlp.linear_1": "a.gen.dit.time_mlp.0",
"feat_decoder.estimator.time_mlp.linear_2": "a.gen.dit.time_mlp.1",
}
def set_gguf_parameters(self):
lm = self.global_config["lm_config"]
dit = self.global_config["dit_config"]
vae = self.global_config["audio_vae_config"]
self.gguf_writer.add_file_type(self.ftype)
# LocEnc + LocDiT + residual LM + AudioVAE decoder, LocEnc has the hparams of LocDiT
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_GEN)
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_gen_audio_embedding_length(dit["hidden_dim"])
self.gguf_writer.add_gen_audio_feed_forward_length(dit["ffn_dim"])
self.gguf_writer.add_gen_audio_block_count(dit["num_layers"])
self.gguf_writer.add_gen_audio_head_count(dit["num_heads"])
self.gguf_writer.add_gen_audio_head_count_kv(lm["num_key_value_heads"])
self.gguf_writer.add_gen_audio_attention_layernorm_eps(lm["rms_norm_eps"])
# AudioVAE encoder: raw waveform -> latents, LocEnc in the gen model then makes the backbone rows
self.gguf_writer.add_clip_has_audio_encoder(True)
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_SPKENC)
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
self.gguf_writer.add_audio_num_mel_bins(1)
self.gguf_writer.add_audio_block_count(0)
self.gguf_writer.add_audio_embedding_length(vae["encoder_dim"])
self.gguf_writer.add_audio_feed_forward_length(vae["encoder_dim"])
self.gguf_writer.add_audio_head_count(1)
self.gguf_writer.add_audio_attention_layernorm_eps(lm["rms_norm_eps"])
# the ODE solver, the RoPE factors and the VAE layout are hard-coded in tools/mtmd
def tensor_force_quant(self, name, new_name, bid, n_dims):
# the AudioVAE convs and the 1D tensors stay F32
if new_name.startswith(("a.vae.", "a.gen.vae.")) or n_dims < 2:
return gguf.GGMLQuantizationType.F32
return super().tensor_force_quant(name, new_name, bid, n_dims)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
if item[0].startswith("base_lm."):
return None
return item
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# already named by generate_extra_tensors()
if name.startswith("a."):
yield (name, data_torch)
return
# dt is always 0 (mean_mode = false), delta_time_mlp gives a constant that is folded into the time_mlp bias
if name.startswith("feat_decoder.estimator.delta_time_mlp."):
return
if name == "feat_decoder.estimator.time_mlp.linear_2.bias":
data_torch = LazyTorchTensor.to_eager(data_torch).float() + self._dtime_const()
if name == "feat_encoder.special_token":
yield ("a.gen.enc.special_token", data_torch.reshape(-1))
return
for src, dst in (("residual_lm.", "a.gen.res"), ("feat_encoder.encoder.", "a.gen.enc"),
("feat_decoder.estimator.decoder.", "a.gen.code")):
if not name.startswith(src):
continue
key = name.removeprefix(src)
if key == "norm.weight":
yield (f"{dst}.output_norm.weight" if dst == "a.gen.res" else f"{dst}.post_ln.weight", data_torch)
return
assert bid is not None, name
key = key.removeprefix(f"layers.{bid}.")
layer_map = self._RES_LAYER_MAP if dst == "a.gen.res" else self._LAYER_MAP
yield (f"{dst}.blk.{bid}.{layer_map[key]}.weight", data_torch)
return
base, _, suffix = name.rpartition(".")
yield (f"{self._PROJ_MAP[base]}.{suffix}", data_torch)
# delta_time_mlp(SinusoidalPosEmb(0)), see VoxCPMLocDiTV2.forward()
def _dtime_const(self) -> Tensor:
d = {k: LazyTorchTensor.to_eager(self.model_tensors[f"feat_decoder.estimator.delta_time_mlp.{k}"]()).float()
for k in ("linear_1.weight", "linear_1.bias", "linear_2.weight", "linear_2.bias")}
dim = d["linear_1.weight"].shape[1]
emb = torch.cat([torch.zeros(dim // 2), torch.ones(dim // 2)]) # sin(0), cos(0)
h = torch.nn.functional.silu(d["linear_1.weight"] @ emb + d["linear_1.bias"])
return d["linear_2.weight"] @ h + d["linear_2.bias"]
@staticmethod
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
# torch weight norm, over every dim but 0
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
# LongRoPE factors of LocEnc and LocDiT, picked as in MiniCPMLongRoPE._set_cos_sin_cache()
lm = self.global_config["lm_config"]
rope = lm["rope_scaling"]
if rope.get("type") != "longrope":
raise NotImplementedError(f"VoxCPM2: unexpected rope_scaling {rope.get('type')}")
if lm["max_position_embeddings"] > rope["original_max_position_embeddings"]:
raise NotImplementedError("VoxCPM2: the long RoPE factors also scale cos/sin, not supported")
yield ("a.gen.rope_factors", torch.tensor(rope["short_factor"], dtype=torch.float32))
path = self.dir_model / "audiovae.pth"
logger.info(f"gguf: loading AudioVAE from '{path}'")
sd = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
sd = sd.get("state_dict", sd)
vae = self.global_config["audio_vae_config"]
def conv(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
yield (f"{dst}.weight", self._wn(sd, src))
yield (f"{dst}.bias", sd[src + ".bias"].float())
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
alpha, beta = fold_snake(sd[src + ".alpha"])
yield (f"{dst}.alpha", alpha)
yield (f"{dst}.beta", beta)
def res_units(dst: str, src: str, first: int) -> Iterable[tuple[str, Tensor]]:
# dilation 1, 3, 9: Snake, depthwise conv k = 7, Snake, conv k = 1
for r in range(3):
s = f"{src}.{first + r}.block"
d = f"{dst}.res.{r}"
yield from snake(f"{d}.snake1", f"{s}.0")
yield from conv(f"{d}.conv1", f"{s}.1")
yield from snake(f"{d}.snake2", f"{s}.2")
yield from conv(f"{d}.conv2", f"{s}.3")
# encoder: conv, then per stride (3 residual units, Snake, strided conv), then fc_mu
p = "a.vae.enc"
yield from conv(f"{p}.conv_in", "encoder.block.0")
for b in range(len(vae["encoder_rates"])):
s = f"encoder.block.{b + 1}.block"
yield from res_units(f"{p}.blk.{b}", s, 0)
yield from snake(f"{p}.blk.{b}.snake", f"{s}.3")
yield from conv(f"{p}.blk.{b}.down", f"{s}.4")
yield from conv(f"{p}.fc_mu", "encoder.fc_mu")
# decoder: depthwise conv, conv k = 1, then per stride (sample-rate scale and bias, Snake, transposed conv, 3 residual units), then Snake, conv
bounds = vae["sr_bin_boundaries"]
sr_idx = sum(1 for b in bounds if b < VAE_OUT_SAMPLE_RATE) # torch.bucketize
if vae["out_sample_rate"] != VAE_OUT_SAMPLE_RATE:
raise NotImplementedError(f"VoxCPM2: unexpected out_sample_rate {vae['out_sample_rate']}")
p = "a.gen.vae"
yield from conv(f"{p}.conv_in_dw", "decoder.model.0")
yield from conv(f"{p}.conv_in", "decoder.model.1")
n_up = len(vae["decoder_rates"])
for b in range(n_up):
c = f"decoder.sr_cond_model.{b + 2}"
yield (f"{p}.blk.{b}.sr_scale", sd[c + ".scale_embed.weight"][sr_idx].float())
yield (f"{p}.blk.{b}.sr_bias", sd[c + ".bias_embed.weight"][sr_idx].float())
s = f"decoder.model.{b + 2}.block"
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
yield from conv(f"{p}.blk.{b}.up", f"{s}.1")
yield from res_units(f"{p}.blk.{b}", s, 2)
yield from snake(f"{p}.post_snake", f"decoder.model.{n_up + 2}")
yield from conv(f"{p}.conv_out", f"decoder.model.{n_up + 3}")
assert math.prod(vae["encoder_rates"]) * VAE_OUT_SAMPLE_RATE == math.prod(vae["decoder_rates"]) * vae["sample_rate"]