315 lines
15 KiB
Python
315 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
|
|
|
import torch
|
|
|
|
if TYPE_CHECKING:
|
|
from torch import Tensor
|
|
|
|
from .base import ModelBase, MmprojModel, TextModel, LazyTorchTensor, gguf, logger
|
|
from .drama3 import fold_snake
|
|
from .minicpm import MiniCPMModel
|
|
|
|
# sample rate of the AudioVAE decoder output, picks the row of the sample-rate conditioning
|
|
VAE_OUT_SAMPLE_RATE = 48000
|
|
|
|
|
|
def _load_config(dir_model: Path) -> dict[str, Any] | None:
|
|
path = dir_model / "config.json"
|
|
if not path.is_file():
|
|
return None
|
|
with open(path, encoding="utf-8") as f:
|
|
config = json.load(f)
|
|
return config if config.get("architecture") == "voxcpm2" else None
|
|
|
|
|
|
def _is_voxcpm2_checkpoint(dir_model: Path) -> bool:
|
|
return _load_config(dir_model) is not None
|
|
|
|
|
|
@ModelBase.register_hparams_loader(_is_voxcpm2_checkpoint)
|
|
def _load_voxcpm2_hparams(dir_model: Path) -> dict[str, Any]:
|
|
logger.info("gguf: detected VoxCPM2 checkpoint")
|
|
config = _load_config(dir_model)
|
|
assert config is not None
|
|
if config["lm_config"].get("use_mup"):
|
|
raise NotImplementedError("VoxCPM2: use_mup = true is not supported")
|
|
if config["dit_config"].get("mean_mode"):
|
|
raise NotImplementedError("VoxCPM2: dit mean_mode = true is not supported")
|
|
if not config.get("residual_lm_no_rope"):
|
|
raise NotImplementedError("VoxCPM2: residual LM with RoPE is not supported")
|
|
for key in ("hidden_dim", "ffn_dim", "num_heads", "num_layers", "kv_channels"):
|
|
if config["encoder_config"][key] != config["dit_config"][key]:
|
|
raise NotImplementedError(f"VoxCPM2: LocEnc and LocDiT differ in {key}")
|
|
return {**config, "architectures": ["VoxCPM2Model"], "text_config": config["lm_config"], "audio_config": config["dit_config"]}
|
|
|
|
|
|
@ModelBase.register("VoxCPM2Model")
|
|
@ModelBase.example("openbmb/VoxCPM2")
|
|
class VoxCPM2TextModel(MiniCPMModel):
|
|
model_arch = gguf.MODEL_ARCH.MINICPM
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
|
name, gen = item
|
|
# the text-semantic LM is a plain MiniCPM, the rest goes to the mmproj
|
|
if not name.startswith("base_lm."):
|
|
return None
|
|
return super().filter_tensors(("model." + name.removeprefix("base_lm."), gen))
|
|
|
|
def set_gguf_parameters(self):
|
|
TextModel.set_gguf_parameters(self)
|
|
# use_mup = false: no embedding, residual or logit scaling
|
|
self.gguf_writer.add_embedding_scale(1.0)
|
|
self.gguf_writer.add_residual_scale(1.0)
|
|
self.gguf_writer.add_logit_scale(1.0)
|
|
|
|
def set_vocab(self):
|
|
# the checkpoint has no tokenizer.model and its tokenizer class is remote code, so read tokenizer.json
|
|
# SPM-style BPE: a merged token scores minus its merge rank, so the SPM tokenizer merges in the BPE order
|
|
with open(self.dir_model / "tokenizer.json", encoding="utf-8") as f:
|
|
tok = json.load(f)
|
|
with open(self.dir_model / "tokenizer_config.json", encoding="utf-8") as f:
|
|
tok_cfg = json.load(f)
|
|
model = tok["model"]
|
|
if model["type"] != "BPE" or not model.get("byte_fallback"):
|
|
raise NotImplementedError("VoxCPM2: unexpected tokenizer type")
|
|
|
|
vocab: dict[str, int] = model["vocab"]
|
|
vocab_size = self.hparams["vocab_size"]
|
|
tokens = [f"[PAD{i}]" for i in range(vocab_size)]
|
|
scores = [-1e9] * vocab_size
|
|
toktypes = [gguf.TokenType.UNUSED] * vocab_size
|
|
for text, i in vocab.items():
|
|
tokens[i] = text
|
|
toktypes[i] = gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-Fa-f]{2}>", text) else gguf.TokenType.NORMAL
|
|
for rank, merge in enumerate(model["merges"]):
|
|
a, b = merge.split(" ") if isinstance(merge, str) else merge
|
|
i = vocab[a + b]
|
|
scores[i] = max(scores[i], -float(rank))
|
|
|
|
# the special tokens (e.g. <|audio_start|>) are only listed in tokenizer_config.json
|
|
specials = {int(i): t["content"] for i, t in tok_cfg.get("added_tokens_decoder", {}).items() if t.get("special")}
|
|
specials.update({t["id"]: t["content"] for t in tok["added_tokens"] if t.get("special")})
|
|
for i, text in specials.items():
|
|
tokens[i] = text
|
|
toktypes[i] = gguf.TokenType.CONTROL
|
|
|
|
self.gguf_writer.add_tokenizer_model("llama")
|
|
self.gguf_writer.add_tokenizer_pre("default")
|
|
self.gguf_writer.add_token_list(tokens)
|
|
self.gguf_writer.add_token_scores(scores)
|
|
self.gguf_writer.add_token_types(toktypes)
|
|
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
|
|
special_vocab.add_to_gguf(self.gguf_writer)
|
|
|
|
|
|
@ModelBase.register("VoxCPM2Model")
|
|
@ModelBase.example("openbmb/VoxCPM2")
|
|
class VoxCPM2MmprojModel(MmprojModel):
|
|
has_vision_encoder = False
|
|
has_audio_encoder = True # AudioVAE encoder, for voice cloning
|
|
|
|
_LAYER_MAP = {
|
|
"input_layernorm.weight": "ln1",
|
|
"self_attn.q_proj.weight": "attn_q",
|
|
"self_attn.k_proj.weight": "attn_k",
|
|
"self_attn.v_proj.weight": "attn_v",
|
|
"self_attn.o_proj.weight": "attn_out",
|
|
"post_attention_layernorm.weight": "ln2",
|
|
"mlp.gate_proj.weight": "ffn_gate",
|
|
"mlp.up_proj.weight": "ffn_up",
|
|
"mlp.down_proj.weight": "ffn_down",
|
|
}
|
|
|
|
# libllama names, mtmd loads the residual LM as a text model
|
|
_RES_LAYER_MAP = {
|
|
"input_layernorm.weight": "attn_norm",
|
|
"self_attn.q_proj.weight": "attn_q",
|
|
"self_attn.k_proj.weight": "attn_k",
|
|
"self_attn.v_proj.weight": "attn_v",
|
|
"self_attn.o_proj.weight": "attn_output",
|
|
"post_attention_layernorm.weight": "ffn_norm",
|
|
"mlp.gate_proj.weight": "ffn_gate",
|
|
"mlp.up_proj.weight": "ffn_up",
|
|
"mlp.down_proj.weight": "ffn_down",
|
|
}
|
|
|
|
_PROJ_MAP = {
|
|
"fsq_layer.in_proj": "a.gen.fsq.in",
|
|
"fsq_layer.out_proj": "a.gen.fsq.out",
|
|
"enc_to_lm_proj": "a.gen.enc_to_lm",
|
|
"lm_to_dit_proj": "a.gen.lm_to_dit",
|
|
"res_to_dit_proj": "a.gen.res_to_dit",
|
|
"fusion_concat_proj": "a.gen.fusion",
|
|
"stop_proj": "a.gen.stop_proj",
|
|
"stop_head": "a.gen.stop_head",
|
|
"feat_encoder.in_proj": "a.gen.enc.in_proj",
|
|
"feat_decoder.estimator.in_proj": "a.gen.dit.in_proj",
|
|
"feat_decoder.estimator.cond_proj": "a.gen.dit.cond_proj",
|
|
"feat_decoder.estimator.out_proj": "a.gen.dit.out_proj",
|
|
"feat_decoder.estimator.time_mlp.linear_1": "a.gen.dit.time_mlp.0",
|
|
"feat_decoder.estimator.time_mlp.linear_2": "a.gen.dit.time_mlp.1",
|
|
}
|
|
|
|
def set_gguf_parameters(self):
|
|
lm = self.global_config["lm_config"]
|
|
dit = self.global_config["dit_config"]
|
|
vae = self.global_config["audio_vae_config"]
|
|
self.gguf_writer.add_file_type(self.ftype)
|
|
|
|
# LocEnc + LocDiT + residual LM + AudioVAE decoder, LocEnc has the hparams of LocDiT
|
|
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
|
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_GEN)
|
|
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
|
self.gguf_writer.add_gen_audio_embedding_length(dit["hidden_dim"])
|
|
self.gguf_writer.add_gen_audio_feed_forward_length(dit["ffn_dim"])
|
|
self.gguf_writer.add_gen_audio_block_count(dit["num_layers"])
|
|
self.gguf_writer.add_gen_audio_head_count(dit["num_heads"])
|
|
self.gguf_writer.add_gen_audio_head_count_kv(lm["num_key_value_heads"])
|
|
self.gguf_writer.add_gen_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
|
|
|
# AudioVAE encoder: raw waveform -> latents, LocEnc in the gen model then makes the backbone rows
|
|
self.gguf_writer.add_clip_has_audio_encoder(True)
|
|
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_SPKENC)
|
|
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
|
self.gguf_writer.add_audio_num_mel_bins(1)
|
|
self.gguf_writer.add_audio_block_count(0)
|
|
self.gguf_writer.add_audio_embedding_length(vae["encoder_dim"])
|
|
self.gguf_writer.add_audio_feed_forward_length(vae["encoder_dim"])
|
|
self.gguf_writer.add_audio_head_count(1)
|
|
self.gguf_writer.add_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
|
# the ODE solver, the RoPE factors and the VAE layout are hard-coded in tools/mtmd
|
|
|
|
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
|
# the AudioVAE convs and the 1D tensors stay F32
|
|
if new_name.startswith(("a.vae.", "a.gen.vae.")) or n_dims < 2:
|
|
return gguf.GGMLQuantizationType.F32
|
|
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
|
if item[0].startswith("base_lm."):
|
|
return None
|
|
return item
|
|
|
|
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
|
# already named by generate_extra_tensors()
|
|
if name.startswith("a."):
|
|
yield (name, data_torch)
|
|
return
|
|
|
|
# dt is always 0 (mean_mode = false), delta_time_mlp gives a constant that is folded into the time_mlp bias
|
|
if name.startswith("feat_decoder.estimator.delta_time_mlp."):
|
|
return
|
|
if name == "feat_decoder.estimator.time_mlp.linear_2.bias":
|
|
data_torch = LazyTorchTensor.to_eager(data_torch).float() + self._dtime_const()
|
|
|
|
if name == "feat_encoder.special_token":
|
|
yield ("a.gen.enc.special_token", data_torch.reshape(-1))
|
|
return
|
|
|
|
for src, dst in (("residual_lm.", "a.gen.res"), ("feat_encoder.encoder.", "a.gen.enc"),
|
|
("feat_decoder.estimator.decoder.", "a.gen.code")):
|
|
if not name.startswith(src):
|
|
continue
|
|
key = name.removeprefix(src)
|
|
if key == "norm.weight":
|
|
yield (f"{dst}.output_norm.weight" if dst == "a.gen.res" else f"{dst}.post_ln.weight", data_torch)
|
|
return
|
|
assert bid is not None, name
|
|
key = key.removeprefix(f"layers.{bid}.")
|
|
layer_map = self._RES_LAYER_MAP if dst == "a.gen.res" else self._LAYER_MAP
|
|
yield (f"{dst}.blk.{bid}.{layer_map[key]}.weight", data_torch)
|
|
return
|
|
|
|
base, _, suffix = name.rpartition(".")
|
|
yield (f"{self._PROJ_MAP[base]}.{suffix}", data_torch)
|
|
|
|
# delta_time_mlp(SinusoidalPosEmb(0)), see VoxCPMLocDiTV2.forward()
|
|
def _dtime_const(self) -> Tensor:
|
|
d = {k: LazyTorchTensor.to_eager(self.model_tensors[f"feat_decoder.estimator.delta_time_mlp.{k}"]()).float()
|
|
for k in ("linear_1.weight", "linear_1.bias", "linear_2.weight", "linear_2.bias")}
|
|
dim = d["linear_1.weight"].shape[1]
|
|
emb = torch.cat([torch.zeros(dim // 2), torch.ones(dim // 2)]) # sin(0), cos(0)
|
|
h = torch.nn.functional.silu(d["linear_1.weight"] @ emb + d["linear_1.bias"])
|
|
return d["linear_2.weight"] @ h + d["linear_2.bias"]
|
|
|
|
@staticmethod
|
|
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
|
# torch weight norm, over every dim but 0
|
|
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
|
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
|
|
|
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
|
# LongRoPE factors of LocEnc and LocDiT, picked as in MiniCPMLongRoPE._set_cos_sin_cache()
|
|
lm = self.global_config["lm_config"]
|
|
rope = lm["rope_scaling"]
|
|
if rope.get("type") != "longrope":
|
|
raise NotImplementedError(f"VoxCPM2: unexpected rope_scaling {rope.get('type')}")
|
|
if lm["max_position_embeddings"] > rope["original_max_position_embeddings"]:
|
|
raise NotImplementedError("VoxCPM2: the long RoPE factors also scale cos/sin, not supported")
|
|
yield ("a.gen.rope_factors", torch.tensor(rope["short_factor"], dtype=torch.float32))
|
|
|
|
path = self.dir_model / "audiovae.pth"
|
|
logger.info(f"gguf: loading AudioVAE from '{path}'")
|
|
sd = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
|
sd = sd.get("state_dict", sd)
|
|
vae = self.global_config["audio_vae_config"]
|
|
|
|
def conv(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
|
yield (f"{dst}.weight", self._wn(sd, src))
|
|
yield (f"{dst}.bias", sd[src + ".bias"].float())
|
|
|
|
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
|
alpha, beta = fold_snake(sd[src + ".alpha"])
|
|
yield (f"{dst}.alpha", alpha)
|
|
yield (f"{dst}.beta", beta)
|
|
|
|
def res_units(dst: str, src: str, first: int) -> Iterable[tuple[str, Tensor]]:
|
|
# dilation 1, 3, 9: Snake, depthwise conv k = 7, Snake, conv k = 1
|
|
for r in range(3):
|
|
s = f"{src}.{first + r}.block"
|
|
d = f"{dst}.res.{r}"
|
|
yield from snake(f"{d}.snake1", f"{s}.0")
|
|
yield from conv(f"{d}.conv1", f"{s}.1")
|
|
yield from snake(f"{d}.snake2", f"{s}.2")
|
|
yield from conv(f"{d}.conv2", f"{s}.3")
|
|
|
|
# encoder: conv, then per stride (3 residual units, Snake, strided conv), then fc_mu
|
|
p = "a.vae.enc"
|
|
yield from conv(f"{p}.conv_in", "encoder.block.0")
|
|
for b in range(len(vae["encoder_rates"])):
|
|
s = f"encoder.block.{b + 1}.block"
|
|
yield from res_units(f"{p}.blk.{b}", s, 0)
|
|
yield from snake(f"{p}.blk.{b}.snake", f"{s}.3")
|
|
yield from conv(f"{p}.blk.{b}.down", f"{s}.4")
|
|
yield from conv(f"{p}.fc_mu", "encoder.fc_mu")
|
|
|
|
# decoder: depthwise conv, conv k = 1, then per stride (sample-rate scale and bias, Snake, transposed conv, 3 residual units), then Snake, conv
|
|
bounds = vae["sr_bin_boundaries"]
|
|
sr_idx = sum(1 for b in bounds if b < VAE_OUT_SAMPLE_RATE) # torch.bucketize
|
|
if vae["out_sample_rate"] != VAE_OUT_SAMPLE_RATE:
|
|
raise NotImplementedError(f"VoxCPM2: unexpected out_sample_rate {vae['out_sample_rate']}")
|
|
p = "a.gen.vae"
|
|
yield from conv(f"{p}.conv_in_dw", "decoder.model.0")
|
|
yield from conv(f"{p}.conv_in", "decoder.model.1")
|
|
n_up = len(vae["decoder_rates"])
|
|
for b in range(n_up):
|
|
c = f"decoder.sr_cond_model.{b + 2}"
|
|
yield (f"{p}.blk.{b}.sr_scale", sd[c + ".scale_embed.weight"][sr_idx].float())
|
|
yield (f"{p}.blk.{b}.sr_bias", sd[c + ".bias_embed.weight"][sr_idx].float())
|
|
s = f"decoder.model.{b + 2}.block"
|
|
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
|
yield from conv(f"{p}.blk.{b}.up", f"{s}.1")
|
|
yield from res_units(f"{p}.blk.{b}", s, 2)
|
|
yield from snake(f"{p}.post_snake", f"decoder.model.{n_up + 2}")
|
|
yield from conv(f"{p}.conv_out", f"decoder.model.{n_up + 3}")
|
|
|
|
assert math.prod(vae["encoder_rates"]) * VAE_OUT_SAMPLE_RATE == math.prod(vae["decoder_rates"]) * vae["sample_rate"]
|