From 0f3a71be15af836d277c9f918adfafb45732677e Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 2 Sep 2026 12:46:16 +0200 Subject: [PATCH] mtmd: Fix Qwen3-tts-0.6b (#28231) * mtmd: load the qwen3-tts code predictor proj_in as optional The talker and the code predictor share the hidden size on the 0.6B checkpoints, so the reference builds no small_to_mtp_projection and the conversion emits no tensor for it. The graph already falls back to identity when the weight is missing, the loader now agrees. * mtmd: keep the qwen3-tts code predictor ffn_down in F32 The code predictor carries a massive activation: its layer 2 FFN intermediate peaks around 1.5e5, well past the 65504 ceiling of F16. mul_mat casts its input to the weight type, so an F16 ffn_down turns that peak into inf, the residual follows, and the next rms_norm yields NaN. Reference forward in float32 gives 145109 against 145396 measured in the graph. --- conversion/qwen3tts.py | 4 ++++ tools/mtmd/clip.cpp | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index 1f6b9a1b0..2c35799f7 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -276,6 +276,10 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): # ConvTranspose1d kernels: only F16/F32 are implemented, no BF16 if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name): return gguf.GGMLQuantizationType.F32 + # the code predictor FFN intermediate peaks around 1.5e5, above the F16 range, and mul_mat + # casts its input to the weight type + if new_name.startswith("a.gen.code.blk.") and new_name.endswith(".ffn_down.weight"): + return gguf.GGMLQuantizationType.F32 return super().tensor_force_quant(name, new_name, bid, n_dims) @classmethod diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 90de19575..46f0437a7 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -2988,9 +2988,9 @@ struct clip_model_loader { } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: { - // code_predictor - model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight")); - model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias")); + // code_predictor, proj_in is absent when the talker and the predictor share the hidden size + model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"), false); + model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"), false); model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight")); model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight")); model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));