diff --git a/conversion/__init__.py b/conversion/__init__.py index 05f55fccf..497797c06 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -284,6 +284,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", "Zonos3Model": "zonos3", + "YuE2ForCausalLM": "yue2", "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", @@ -361,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Exaone4_5_ForConditionalGeneration": "exaone", "FishOmniModel": "drama3", "Zonos3Model": "zonos3", + "YuE2ForCausalLM": "yue2", "Gemma3ForConditionalGeneration": "gemma", "Gemma3nForConditionalGeneration": "gemma", "Gemma4ForConditionalGeneration": "gemma", diff --git a/conversion/yue2.py b/conversion/yue2.py new file mode 100644 index 000000000..2242af8eb --- /dev/null +++ b/conversion/yue2.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import base64 +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, MmprojModel, gguf, logger +from .qwen import QwenModel, Qwen3Model + +# token ids of the yue2_infer protocol (protocol.py) +EOD = 151643 +ABC_START, ABC_END = 151847, 151848 +MUSIC_START, MUSIC_END = 151851, 151852 +CODEC_OFFSET, CODEC_SIZE = 151853, 32768 +LATENT_START = 184621 + +# the VAE is a separate repo (m-a-p/YuE2-Vae), downloaded into this subdir of the model dir +VAE_DIR = "vae" + +INSTRUCTIONS = { + "off": "Generate music with codec tokens from the given conditions.", + "melody": "Generate a melody-only ABC transcription without chord symbols, then generate music with codec tokens from the given conditions.", + "full": "Generate a chord-annotated ABC transcription, then generate music with codec tokens from the given conditions.", +} + +# the text is "[Tags]\n