From 92192d31e457d9e7ad68da84608f040ad72e42bb Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 16 Nov 2026 20:22:23 +0100 Subject: [PATCH] mtmd: support YuE2-3B (#32369) * mtmd: support YuE2-3B * improve speed * add CFG * fix streaming * add cfg control from api * check ctx size * fix n_predict * add flow_steps config * nits * fix batch api * nits * adapt * mtmd: yue2 samples from its own saved logits The context logits belong to the last decode of any slot, so with several server slots the score and the unguided codes were sampled from another sequence. Keep the logits of each own decode in save_logits(), which also does the guidance mix. * mtmd: let the yue2 snake chain fuse The beta reshape landed between sqr and mul, so the mul, sin, sqr, mul, add chain was not consecutive and the backends never fused it. Expand both reshapes first, like qwen3tts does. --------- Co-authored-by: Pascal --- conversion/__init__.py | 2 + conversion/yue2.py | 234 +++++++++++ docs/multimodal/tts.md | 48 +++ gguf-py/gguf/constants.py | 1 + tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/README-dev.md | 12 + tools/mtmd/models/models.h | 27 ++ tools/mtmd/models/yue2-gen.cpp | 89 ++++ tools/mtmd/mtmd-graph.cpp | 11 + tools/mtmd/mtmd-helper-gen.cpp | 695 +++++++++++++++++++++++++++++++- tools/mtmd/mtmd-helper.h | 23 +- tools/mtmd/mtmd-impl.h | 2 + tools/mtmd/mtmd-model.cpp | 77 +++- tools/mtmd/mtmd-model.h | 41 ++ tools/mtmd/mtmd-subcontext.cpp | 5 +- tools/mtmd/mtmd-subcontext.h | 2 + tools/mtmd/mtmd.cpp | 330 +++++++++++++++ tools/mtmd/mtmd.h | 14 +- tools/server/README.md | 10 +- tools/server/server-context.cpp | 17 +- tools/server/server-schema.cpp | 7 + tools/tts/README.md | 2 +- tools/tts/tts.cpp | 6 +- 23 files changed, 1633 insertions(+), 23 deletions(-) create mode 100644 conversion/yue2.py create mode 100644 tools/mtmd/models/yue2-gen.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 05f55fccf..497797c06 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -284,6 +284,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", "Zonos3Model": "zonos3", + "YuE2ForCausalLM": "yue2", "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", @@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Exaone4_5_ForConditionalGeneration": "exaone", "FishOmniModel": "drama3", "Zonos3Model": "zonos3", + "YuE2ForCausalLM": "yue2", "Gemma3ForConditionalGeneration": "gemma", "Gemma3nForConditionalGeneration": "gemma", "Gemma4ForConditionalGeneration": "gemma", diff --git a/conversion/yue2.py b/conversion/yue2.py new file mode 100644 index 000000000..2242af8eb --- /dev/null +++ b/conversion/yue2.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import base64 +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, MmprojModel, gguf, logger +from .qwen import QwenModel, Qwen3Model + +# token ids of the yue2_infer protocol (protocol.py) +EOD = 151643 +ABC_START, ABC_END = 151847, 151848 +MUSIC_START, MUSIC_END = 151851, 151852 +CODEC_OFFSET, CODEC_SIZE = 151853, 32768 +LATENT_START = 184621 + +# the VAE is a separate repo (m-a-p/YuE2-Vae), downloaded into this subdir of the model dir +VAE_DIR = "vae" + +INSTRUCTIONS = { + "off": "Generate music with codec tokens from the given conditions.", + "melody": "Generate a melody-only ABC transcription without chord symbols, then generate music with codec tokens from the given conditions.", + "full": "Generate a chord-annotated ABC transcription, then generate music with codec tokens from the given conditions.", +} + +# the text is "[Tags]\n