* mtmd: support YuE2-3B * improve speed * add CFG * fix streaming * add cfg control from api * check ctx size * fix n_predict * add flow_steps config * nits * fix batch api * nits * adapt * mtmd: yue2 samples from its own saved logits The context logits belong to the last decode of any slot, so with several server slots the score and the unguided codes were sampled from another sequence. Keep the logits of each own decode in save_logits(), which also does the guidance mix. * mtmd: let the yue2 snake chain fuse The beta reshape landed between sqr and mul, so the mul, sin, sqr, mul, add chain was not consecutive and the backends never fused it. Expand both reshapes first, like qwen3tts does. --------- Co-authored-by: Pascal <admin@serveurperso.com>
1274 lines
53 KiB
C++
1274 lines
53 KiB
C++
#pragma once
|
|
|
|
#include "ggml.h"
|
|
#include "ggml-cpp.h"
|
|
#include "gguf.h"
|
|
#include "mtmd.h"
|
|
|
|
#include <array>
|
|
#include <climits>
|
|
#include <cmath>
|
|
#include <cstdarg>
|
|
#include <cinttypes>
|
|
#include <string>
|
|
#include <map>
|
|
#include <sstream>
|
|
#include <vector>
|
|
#include <memory>
|
|
#include <fstream>
|
|
#include <random>
|
|
|
|
#ifdef _WIN32
|
|
#ifndef NOMINMAX
|
|
#define NOMINMAX
|
|
#endif
|
|
#include <windows.h>
|
|
#endif
|
|
|
|
// !!! Internal header, to be used by mtmd only !!!
|
|
|
|
#define MTMD_INTERNAL_HEADER
|
|
|
|
#define KEY_FTYPE "general.file_type"
|
|
#define KEY_NAME "general.name"
|
|
#define KEY_DESCRIPTION "general.description"
|
|
#define KEY_PROJ_TYPE "clip.projector_type"
|
|
#define KEY_HAS_AUDIO_ENC "clip.has_audio_encoder"
|
|
#define KEY_HAS_VISION_ENC "clip.has_vision_encoder"
|
|
#define KEY_HAS_GEN_AUDIO_ENC "clip.has_gen_audio_encoder"
|
|
#define KEY_USE_GELU "clip.use_gelu"
|
|
#define KEY_USE_SILU "clip.use_silu"
|
|
|
|
#define KEY_N_EMBD "clip.%s.embedding_length"
|
|
#define KEY_N_FF "clip.%s.feed_forward_length"
|
|
#define KEY_N_BLOCK "clip.%s.block_count"
|
|
#define KEY_PROJ_DIM "clip.%s.projection_dim"
|
|
#define KEY_N_HEAD "clip.%s.attention.head_count"
|
|
#define KEY_N_HEAD_KV "clip.%s.attention.head_count_kv"
|
|
#define KEY_N_EMBD_HEAD "clip.%s.attention.head_dim"
|
|
#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon"
|
|
#define KEY_FEATURE_LAYERS "clip.%s.feature_layer"
|
|
|
|
// vision-specific
|
|
#define KEY_VISION_PROJ_TYPE "clip.vision.projector_type" // for models with mixed modalities
|
|
#define KEY_IMAGE_SIZE "clip.vision.image_size"
|
|
#define KEY_IMAGE_MIN_PIXELS "clip.vision.image_min_pixels"
|
|
#define KEY_IMAGE_MAX_PIXELS "clip.vision.image_max_pixels"
|
|
#define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles"
|
|
#define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles"
|
|
#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size"
|
|
#define KEY_PREPROC_TILE_SIZE "clip.vision.preproc_tile_size"
|
|
#define KEY_PREPROC_MAX_PIXELS_TOLERANCE "clip.vision.preproc_max_pixels_tolerance"
|
|
#define KEY_PATCH_SIZE "clip.vision.patch_size"
|
|
#define KEY_IMAGE_MEAN "clip.vision.image_mean"
|
|
#define KEY_IMAGE_STD "clip.vision.image_std"
|
|
#define KEY_IMAGE_RESIZE_ALGO "clip.vision.image_resize_algo"
|
|
#define KEY_PROJ_SCALE_FACTOR "clip.vision.projector.scale_factor"
|
|
#define KEY_PROJ_SAMPLE_QUERY_SIDE "clip.vision.projector.query_side"
|
|
#define KEY_PROJ_SAMPLE_WINDOW_SIDE "clip.vision.projector.window_side"
|
|
#define KEY_PROJ_SPATIAL_OFFSETS "clip.vision.projector.spatial_offsets"
|
|
#define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size"
|
|
#define KEY_SWIGLU_CLAMP "clip.vision.swiglu_clamp"
|
|
|
|
#define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type"
|
|
#define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints"
|
|
#define KEY_MAX_SLICE_NUMS "clip.vision.max_slice_nums"
|
|
#define KEY_WIN_ATTN_PATTERN "clip.vision.n_wa_pattern"
|
|
#define KEY_WIN_ATTN_LAYER_INDEXES "clip.vision.wa_layer_indexes"
|
|
#define KEY_WA_PATTERN_MODE "clip.vision.wa_pattern_mode"
|
|
#define KEY_ATTN_WINDOW_SIZE "clip.vision.window_size"
|
|
#define KEY_MINICPMV_VERSION "clip.minicpmv_version"
|
|
#define KEY_MINICPMV_QUERY_NUM "clip.minicpmv_query_num"
|
|
#define KEY_SAM_N_HEAD "clip.vision.sam.head_count"
|
|
#define KEY_SAM_N_BLOCK "clip.vision.sam.block_count"
|
|
#define KEY_SAM_N_EMBD "clip.vision.sam.embedding_length"
|
|
#define KEY_VISION_N_EXPERT_USED "clip.vision.expert_used_count"
|
|
// audio-specific
|
|
#define KEY_AUDIO_PROJ_TYPE "clip.audio.projector_type" // for models with mixed modalities
|
|
#define KEY_A_NUM_MEL_BINS "clip.audio.num_mel_bins"
|
|
#define KEY_A_PROJ_STACK_FACTOR "clip.audio.projector.stack_factor"
|
|
#define KEY_A_CHUNK_SIZE "clip.audio.chunk_size"
|
|
#define KEY_A_CONV_KERNEL_SIZE "clip.audio.conv_kernel_size"
|
|
#define KEY_A_HIDDEN_SIZES "clip.audio.hidden_sizes"
|
|
#define KEY_A_PATCH_SIZES "clip.audio.patch_sizes"
|
|
#define KEY_A_DEPTHS "clip.audio.depths"
|
|
#define KEY_A_MAX_POS_EMB "clip.audio.max_pos_emb"
|
|
#define KEY_A_PROJ_WINDOW_SIZE "clip.audio.projector.window_size"
|
|
#define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate"
|
|
#define KEY_A_PROJ_HEAD_COUNT "clip.audio.projector.head_count"
|
|
#define KEY_A_RVQ_NUM_QUANTIZERS "clip.audio.rvq.num_quantizers" // mimo-audio-tokenizer
|
|
#define KEY_A_RVQ_CODEBOOK_SIZE "clip.audio.rvq.codebook_size" // mimo-audio-tokenizer: per-quantizer bin count
|
|
#define KEY_A_WA_PATTERN_MODE "clip.audio.wa_pattern_mode" // mimo-audio-tokenizer, per-layer -1 (full) / 0 (windowed)
|
|
#define KEY_A_ATTN_WINDOW_SIZE "clip.audio.window_size" // mimo-audio-tokenizer: sliding-window radius
|
|
#define KEY_A_LOCAL_BLOCK_COUNT "clip.audio.local_block_count" // mimo-v2.5: input_local_transformer layer count
|
|
#define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size
|
|
// audio generation (gen-audio)-specific
|
|
#define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities
|
|
// name of the weight variant, for settings that are not in the checkpoint
|
|
#define KEY_GEN_AUDIO_VARIANT "clip.gen.audio.model_variant"
|
|
#define KEY_GEN_AUDIO_DEFAULT_VOICE "clip.gen.audio.default_voice"
|
|
// sampling defaults of the generated audio, like general.sampling.* for text
|
|
#define KEY_GEN_AUDIO_TEMP "clip.gen.audio.sampling.temp"
|
|
#define KEY_GEN_AUDIO_TOP_K "clip.gen.audio.sampling.top_k"
|
|
#define KEY_GEN_AUDIO_PENALTY_REPEAT "clip.gen.audio.sampling.penalty_repeat"
|
|
#define KEY_GEN_AUDIO_PENALTY_LAST_N "clip.gen.audio.sampling.penalty_last_n"
|
|
#define KEY_AUDIO_SUBSMPL_FACTOR "clip.audio.subsampling_factor"
|
|
|
|
// nemotron transcribe
|
|
#define KEY_A_ROPE_DIMS "clip.audio.rope_dims"
|
|
#define KEY_A_SPK_N_BLOCK "clip.audio.spk.block_count"
|
|
#define KEY_A_SPK_N_HEAD "clip.audio.spk.head_count"
|
|
|
|
//
|
|
// tensor name constants
|
|
//
|
|
|
|
#define TN_POS_EMBD "%s.position_embd.weight"
|
|
#define TN_CLASS_EMBD "v.class_embd"
|
|
#define TN_CLASS_POS_EMBD "v.class_position_embd.weight"
|
|
#define TN_PATCH_EMBD "v.patch_embd.weight" // not rename tensor with ".0" postfix for backward compat
|
|
#define TN_PATCH_EMBD_1 "v.patch_embd.weight.1"
|
|
#define TN_PATCH_BIAS "v.patch_embd.bias"
|
|
#define TN_ROPE_FREQS "v.rope_freqs.weight"
|
|
#define TN_NORM_EMBD "v.norm_embd.%s"
|
|
#define TN_PATCH_NORM "v.patch_norm.%d.%s"
|
|
#define TN_ATTN_QKV "%s.blk.%d.attn_qkv.%s"
|
|
#define TN_ATTN_K "%s.blk.%d.attn_k.%s"
|
|
#define TN_ATTN_Q "%s.blk.%d.attn_q.%s"
|
|
#define TN_ATTN_V "%s.blk.%d.attn_v.%s"
|
|
#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s"
|
|
#define TN_ATTN_SINKS "%s.blk.%d.attn_sinks"
|
|
#define TN_ATTN_K_NORM "%s.blk.%d.attn_k_norm.%s"
|
|
#define TN_ATTN_Q_NORM "%s.blk.%d.attn_q_norm.%s"
|
|
#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s"
|
|
#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s"
|
|
#define TN_FFN_UP "%s.blk.%d.ffn_up.%s"
|
|
#define TN_FFN_GATE_INP "%s.blk.%d.ffn_gate_inp.%s" // MoE router (dots3note)
|
|
#define TN_FFN_GATE_EXPS "%s.blk.%d.ffn_gate_exps.%s"
|
|
#define TN_FFN_UP_EXPS "%s.blk.%d.ffn_up_exps.%s"
|
|
#define TN_FFN_DOWN_EXPS "%s.blk.%d.ffn_down_exps.%s"
|
|
#define TN_FFN_EXP_PROBS_B "%s.blk.%d.exp_probs_b.%s"
|
|
#define TN_LN_1 "%s.blk.%d.ln1.%s" // layer norm
|
|
#define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm
|
|
#define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale
|
|
#define TN_LS_2 "%s.blk.%d.ls2.%s" // layer scale
|
|
#define TN_LS_OUT "%s.blk.%d.out_scale.%s" // layer out scale (gemma4)
|
|
#define TN_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s" // post-attn norm (gemma4)
|
|
#define TN_FFN_POST_NORM "%s.blk.%d.ffn_post_norm.%s" // post-FFN norm (gemma4)
|
|
#define TN_LN_PRE "%s.pre_ln.%s"
|
|
#define TN_LN_POST "%s.post_ln.%s"
|
|
#define TN_LLAVA_PROJ "mm.%d.%s"
|
|
#define TN_MM_UP "mm.up.%s"
|
|
#define TN_MM_GATE "mm.gate.%s"
|
|
#define TN_MM_DOWN "mm.down.%s"
|
|
#define TN_MM_POST_NORM "mm.post_norm.%s"
|
|
#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"
|
|
#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"
|
|
#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s"
|
|
#define TN_IMAGE_NEWLINE "v.image_newline"
|
|
#define TN_IMAGE_SEPERATOR "v.view_seperator"
|
|
#define TN_TOK_EMBD_SEP "v.tok_embd_sep"
|
|
#define TN_MM_INP_NORM "mm.input_norm.weight"
|
|
#define TN_MM_INP_NORM_B "mm.input_norm.bias"
|
|
#define TN_MM_INP_PROJ "mm.input_projection.weight" // gemma3
|
|
#define TN_MM_SOFT_EMB_N "mm.soft_emb_norm.weight" // gemma3
|
|
#define TN_MM_PROJECTOR "mm.model.fc.%s" // idefics3, deepseekocr
|
|
#define TN_MM_PATCH_MERGER "mm.patch_merger.%s" // mistral small 3.1, glm4v
|
|
#define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP
|
|
#define TN_MM_MERGER_FC2 "mm.merger.fc2.%s"
|
|
#define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral
|
|
#define TN_TOK_IMG_START "v.token_embd.img_start" // deepseek4v
|
|
#define TN_TOK_IMG_END "v.token_embd.img_end" // deepseek4v
|
|
#define TN_TOK_IMG_PAD "v.token_embd.img_pad" // deepseek4v
|
|
#define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model)
|
|
#define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model)
|
|
#define TN_DEEPSTACK_NORM "v.deepstack.%d.norm.%s" // qwen3vl deepstack
|
|
#define TN_DEEPSTACK_FC1 "v.deepstack.%d.fc1.%s" // qwen3vl deepstack
|
|
#define TN_DEEPSTACK_FC2 "v.deepstack.%d.fc2.%s" // qwen3vl deepstack
|
|
|
|
// lfm3vl
|
|
#define TN_LFM3VL_EMBEDDING_NORM "%s.embedding_norm.%s"
|
|
#define TN_LFM3VL_CONV2D_IN_PROJ "%s.blk.%d.conv2d_in_proj.%s"
|
|
#define TN_LFM3VL_CONV2D_SPATIAL "%s.blk.%d.conv2d_spatial.%s"
|
|
#define TN_LFM3VL_CONV2D_OUT_PROJ "%s.blk.%d.conv2d_out_proj.%s"
|
|
|
|
// mimicpmv
|
|
#define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"
|
|
#define TN_MINICPMV_QUERY "resampler.query"
|
|
#define TN_MINICPMV_PROJ "resampler.proj.weight"
|
|
#define TN_MINICPMV_KV_PROJ "resampler.kv.weight"
|
|
#define TN_MINICPMV_ATTN "resampler.attn.%s.%s"
|
|
#define TN_MINICPMV_LN "resampler.ln_%s.%s"
|
|
|
|
// MiniCPM-V 4.6 ViT merger (window attention + MLP downsample),
|
|
// matching the upstream `vit_merger` module name in transformers.
|
|
#define TN_VIT_MERGER_LN1 "v.vit_merger.ln1.%s"
|
|
#define TN_VIT_MERGER_ATTN_Q "v.vit_merger.attn_q.%s"
|
|
#define TN_VIT_MERGER_ATTN_K "v.vit_merger.attn_k.%s"
|
|
#define TN_VIT_MERGER_ATTN_V "v.vit_merger.attn_v.%s"
|
|
#define TN_VIT_MERGER_ATTN_O "v.vit_merger.attn_out.%s"
|
|
#define TN_VIT_MERGER_DS_LN "v.vit_merger.ds_ln.%s"
|
|
#define TN_VIT_MERGER_DS_UP "v.vit_merger.ds_ffn_up.%s"
|
|
#define TN_VIT_MERGER_DS_DOWN "v.vit_merger.ds_ffn_down.%s"
|
|
|
|
#define TN_GLM_ADAPER_CONV "adapter.conv.%s"
|
|
#define TN_GLM_ADAPTER_LINEAR "adapter.linear.linear.%s"
|
|
#define TN_GLM_ADAPTER_NORM_1 "adapter.linear.norm1.%s"
|
|
#define TN_GLM_ADAPTER_D_H_2_4H "adapter.linear.dense_h_to_4h.%s"
|
|
#define TN_GLM_ADAPTER_GATE "adapter.linear.gate.%s"
|
|
#define TN_GLM_ADAPTER_D_4H_2_H "adapter.linear.dense_4h_to_h.%s"
|
|
|
|
// ultravox
|
|
#define TN_CONV1D "a.conv1d.%d.%s"
|
|
#define TN_CONV2D "a.conv2d.%d.%s"
|
|
#define TN_CONV_OUT "a.conv_out.%s"
|
|
#define TN_MM_AUDIO_MLP "mm.a.mlp.%d.%s"
|
|
#define TN_MM_AUDIO_FC "mm.a.fc.%s" // fully connected layer
|
|
#define TN_MM_NORM_PRE "mm.a.norm_pre.%s"
|
|
#define TN_MM_NORM_MID "mm.a.norm_mid.%s"
|
|
|
|
// mimo-audio-tokenizer
|
|
#define TN_A_DOWNSAMPLE_CONV "a.downsample.conv.%s"
|
|
#define TN_A_DOWNSAMPLE_NORM "a.downsample.norm.%s"
|
|
#define TN_A_RVQ_CODEBOOK "a.rvq.codebook.%s"
|
|
// mimo-v2.5: text-side RVQ code embedding ("text codebook")
|
|
#define TN_MM_A_CODE_EMBD "mm.a.code_embd.%s"
|
|
// mimo-v2.5: LLM-side connector (input_local_transformer)
|
|
#define TN_MM_A_LOCAL_ATTN_Q "mm.a.local_blk.%d.attn_q.%s"
|
|
#define TN_MM_A_LOCAL_ATTN_K "mm.a.local_blk.%d.attn_k.%s"
|
|
#define TN_MM_A_LOCAL_ATTN_V "mm.a.local_blk.%d.attn_v.%s"
|
|
#define TN_MM_A_LOCAL_ATTN_OUT "mm.a.local_blk.%d.attn_out.%s"
|
|
#define TN_MM_A_LOCAL_FFN_GATE "mm.a.local_blk.%d.ffn_gate.%s"
|
|
#define TN_MM_A_LOCAL_FFN_UP "mm.a.local_blk.%d.ffn_up.%s"
|
|
#define TN_MM_A_LOCAL_FFN_DOWN "mm.a.local_blk.%d.ffn_down.%s"
|
|
#define TN_MM_A_LOCAL_LN1 "mm.a.local_blk.%d.ln1.%s"
|
|
#define TN_MM_A_LOCAL_LN2 "mm.a.local_blk.%d.ln2.%s"
|
|
#define TN_MM_A_LOCAL_NORM "mm.a.local_norm.%s"
|
|
|
|
// qwen3tts speaker encoder (ECAPA-TDNN)
|
|
#define TN_A_SE_CONV1 "a.blk.%d.se_conv1.%s"
|
|
#define TN_A_SE_CONV2 "a.blk.%d.se_conv2.%s"
|
|
#define TN_A_CONV_RES2 "a.blk.%d.res2.%d.%s"
|
|
#define TN_A_ASP_ATTN "a.asp_attn.%s"
|
|
#define TN_A_ASP_TDNN "a.asp_tdnn.%s"
|
|
|
|
// qwen3tts code_predictor
|
|
#define TN_A_GEN_CODE_PROJ_IN "a.gen.code.proj_in.%s"
|
|
#define TN_A_GEN_CODE_EMBD "a.gen.code.embd.%s"
|
|
#define TN_A_GEN_CODE_HEAD "a.gen.code.head.%s"
|
|
#define TN_A_GEN_CODE_OUT_EMBD "a.gen.code.out_embd.%s"
|
|
#define TN_A_GEN_CODE_NORM "a.gen.code.output_norm.%s"
|
|
|
|
// lfm3a depthformer (MaskGIT), the codec decoder uses the generic layer names with prefix "a.gen.wav.tfm"
|
|
#define TN_A_GEN_CODE_HEAD_NORM "a.gen.code.head_norm.%s"
|
|
#define TN_A_GEN_CODE_MASK_EMBD "a.gen.code.mask_embd"
|
|
#define TN_ATTN_GATE "%s.blk.%d.attn_gate.%s"
|
|
|
|
// drama3 tower, the rest of the block uses the generic names with prefix "a.gen.code"
|
|
#define TN_A_GEN_CODE_ENORM "a.gen.code.blk.%d.enorm.%s"
|
|
#define TN_A_GEN_CODE_HNORM "a.gen.code.blk.%d.hnorm.%s"
|
|
#define TN_A_GEN_CODE_EH_PROJ "a.gen.code.blk.%d.eh_proj.%s"
|
|
#define TN_A_GEN_CODE_BLK_EMBD "a.gen.code.blk.%d.embd.%s"
|
|
#define TN_A_GEN_CODE_BLK_OUT_NORM "a.gen.code.blk.%d.out_norm.%s"
|
|
|
|
// fish_s2: fast AR input embedding, shared by all codebooks
|
|
#define TN_A_GEN_CODE_TOK_EMBD "a.gen.code.tok_embd.%s"
|
|
|
|
// fish_s2 codec encoder, the transformer layers use the generic a.blk.* names
|
|
#define TN_A_CODEC_ENC_ENTRY "a.codec.enc.entry.%s"
|
|
#define TN_A_CODEC_ENC_SNAKE "a.codec.enc.blk.%d.snake.%s"
|
|
#define TN_A_CODEC_ENC_CONV "a.codec.enc.blk.%d.conv.%s"
|
|
#define TN_A_CODEC_ENC_RES_ACT1 "a.codec.enc.blk.%d.res.%d.act1.%s"
|
|
#define TN_A_CODEC_ENC_RES_CONV1 "a.codec.enc.blk.%d.res.%d.conv1.%s"
|
|
#define TN_A_CODEC_ENC_RES_ACT2 "a.codec.enc.blk.%d.res.%d.act2.%s"
|
|
#define TN_A_CODEC_ENC_RES_CONV2 "a.codec.enc.blk.%d.res.%d.conv2.%s"
|
|
#define TN_A_CODEC_ENC_POST_SNAKE "a.codec.enc.post_snake.%s"
|
|
#define TN_A_CODEC_ENC_POST_CONV "a.codec.enc.post_conv.%s"
|
|
#define TN_A_CODEC_DOWN_CONV "a.codec.down.blk.%d.conv.%s"
|
|
#define TN_A_CODEC_DOWN_DWCONV "a.codec.down.blk.%d.dwconv.%s"
|
|
#define TN_A_CODEC_DOWN_NORM "a.codec.down.blk.%d.norm.%s"
|
|
#define TN_A_CODEC_DOWN_PW1 "a.codec.down.blk.%d.pw1.%s"
|
|
#define TN_A_CODEC_DOWN_PW2 "a.codec.down.blk.%d.pw2.%s"
|
|
#define TN_A_CODEC_DOWN_GAMMA "a.codec.down.blk.%d.gamma"
|
|
#define TN_A_CODEC_PRE_NORM "a.codec.pre_norm.%s"
|
|
#define TN_A_CODEC_VQ_FIRST_IN "a.codec.vq.first.in_proj.%s"
|
|
#define TN_A_CODEC_VQ_FIRST_CB "a.codec.vq.first.codebook.%s"
|
|
#define TN_A_CODEC_VQ_REST_IN "a.codec.vq.rest.in_proj.%s"
|
|
#define TN_A_CODEC_VQ_REST_CB "a.codec.vq.rest.codebook.%s"
|
|
|
|
// qwen3tts code2wav (RVQ codes -> raw PCM)
|
|
// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm"
|
|
#define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s"
|
|
#define TN_A_GEN_WAV_QUANT_FIRST_OUT "a.gen.wav.quant.first.out_proj.%s"
|
|
#define TN_A_GEN_WAV_QUANT_FIRST_CB "a.gen.wav.quant.first.codebook.%s"
|
|
#define TN_A_GEN_WAV_QUANT_REST_IN "a.gen.wav.quant.rest.in_proj.%s"
|
|
#define TN_A_GEN_WAV_QUANT_REST_OUT "a.gen.wav.quant.rest.out_proj.%s"
|
|
#define TN_A_GEN_WAV_QUANT_REST_CB "a.gen.wav.quant.rest.codebook.%s"
|
|
#define TN_A_GEN_WAV_PRE_CONV "a.gen.wav.pre_conv.%s"
|
|
#define TN_A_GEN_WAV_TFM_IN_PROJ "a.gen.wav.tfm.in_proj.%s"
|
|
#define TN_A_GEN_WAV_TFM_OUT_PROJ "a.gen.wav.tfm.out_proj.%s"
|
|
#define TN_A_GEN_WAV_TFM_OUT_NORM "a.gen.wav.tfm.output_norm.%s"
|
|
#define TN_A_GEN_WAV_UP_CONV "a.gen.wav.up.blk.%d.conv.%s"
|
|
#define TN_A_GEN_WAV_UP_DWCONV "a.gen.wav.up.blk.%d.dwconv.%s"
|
|
#define TN_A_GEN_WAV_UP_NORM "a.gen.wav.up.blk.%d.norm.%s"
|
|
#define TN_A_GEN_WAV_UP_PW1 "a.gen.wav.up.blk.%d.pw1.%s"
|
|
#define TN_A_GEN_WAV_UP_PW2 "a.gen.wav.up.blk.%d.pw2.%s"
|
|
#define TN_A_GEN_WAV_UP_GAMMA "a.gen.wav.up.blk.%d.gamma"
|
|
#define TN_A_GEN_WAV_DAC_ENTRY "a.gen.wav.dac.entry.%s"
|
|
#define TN_A_GEN_WAV_DAC_SNAKE "a.gen.wav.dac.blk.%d.snake.%s"
|
|
#define TN_A_GEN_WAV_DAC_CONV "a.gen.wav.dac.blk.%d.conv.%s"
|
|
#define TN_A_GEN_WAV_DAC_RES_ACT1 "a.gen.wav.dac.blk.%d.res.%d.act1.%s"
|
|
#define TN_A_GEN_WAV_DAC_RES_CONV1 "a.gen.wav.dac.blk.%d.res.%d.conv1.%s"
|
|
#define TN_A_GEN_WAV_DAC_RES_ACT2 "a.gen.wav.dac.blk.%d.res.%d.act2.%s"
|
|
#define TN_A_GEN_WAV_DAC_RES_CONV2 "a.gen.wav.dac.blk.%d.res.%d.conv2.%s"
|
|
#define TN_A_GEN_WAV_DAC_POST_SNAKE "a.gen.wav.dac.post_snake.%s"
|
|
#define TN_A_GEN_WAV_DAC_POST_CONV "a.gen.wav.dac.post_conv.%s"
|
|
|
|
// pocket-tts
|
|
#define TN_A_SEANET_CONV_IN "a.seanet.conv_in.%s"
|
|
#define TN_A_SEANET_CONV_OUT "a.seanet.conv_out.%s"
|
|
#define TN_A_SEANET_RES_CONV1 "a.seanet.blk.%d.res_conv1.%s"
|
|
#define TN_A_SEANET_RES_CONV2 "a.seanet.blk.%d.res_conv2.%s"
|
|
#define TN_A_SEANET_SCALE_CONV "a.seanet.blk.%d.scale_conv.%s"
|
|
#define TN_A_SPEAKER_PROJ "a.speaker_proj.%s"
|
|
#define TN_A_DOWNSAMPLE_CONV "a.downsample.conv.%s"
|
|
#define TN_A_GEN_FLOW_INPUT_PROJ "a.gen.flow.input_proj.%s"
|
|
#define TN_A_GEN_FLOW_COND_EMBD "a.gen.flow.cond_embd.%s"
|
|
#define TN_A_GEN_FLOW_TIME_FREQS "a.gen.flow.time.%d.freqs"
|
|
#define TN_A_GEN_FLOW_TIME_UP "a.gen.flow.time.%d.up.%s"
|
|
#define TN_A_GEN_FLOW_TIME_DOWN "a.gen.flow.time.%d.down.%s"
|
|
#define TN_A_GEN_FLOW_TIME_NORM "a.gen.flow.time.%d.norm"
|
|
#define TN_A_GEN_FLOW_BLK_NORM "a.gen.flow.blk.%d.norm.%s"
|
|
#define TN_A_GEN_FLOW_BLK_UP "a.gen.flow.blk.%d.up.%s"
|
|
#define TN_A_GEN_FLOW_BLK_DOWN "a.gen.flow.blk.%d.down.%s"
|
|
#define TN_A_GEN_FLOW_BLK_ADA "a.gen.flow.blk.%d.ada.%s"
|
|
#define TN_A_GEN_FLOW_FINAL_ADA "a.gen.flow.final.ada.%s"
|
|
#define TN_A_GEN_FLOW_FINAL_PROJ "a.gen.flow.final.proj.%s"
|
|
#define TN_A_GEN_OUT_EOS "a.gen.out_eos.%s"
|
|
#define TN_A_GEN_INPUT_LINEAR "a.gen.input_linear.%s"
|
|
#define TN_A_GEN_EMB_MEAN "a.gen.emb_mean"
|
|
#define TN_A_GEN_EMB_STD "a.gen.emb_std"
|
|
#define TN_A_GEN_WAV_QUANT_OUT "a.gen.wav.quant_out.%s"
|
|
#define TN_A_GEN_WAV_UPSAMPLE "a.gen.wav.upsample.%s"
|
|
#define TN_A_GEN_WAV_SEANET_CONV_IN "a.gen.wav.seanet.conv_in.%s"
|
|
#define TN_A_GEN_WAV_SEANET_CONV_OUT "a.gen.wav.seanet.conv_out.%s"
|
|
#define TN_A_GEN_WAV_SEANET_RES_CONV1 "a.gen.wav.seanet.blk.%d.res_conv1.%s"
|
|
#define TN_A_GEN_WAV_SEANET_RES_CONV2 "a.gen.wav.seanet.blk.%d.res_conv2.%s"
|
|
#define TN_A_GEN_WAV_SEANET_SCALE_CONV "a.gen.wav.seanet.blk.%d.scale_conv.%s"
|
|
|
|
// soprano: Vocos decoder, the ConvNeXt blocks use TN_A_GEN_WAV_UP_*
|
|
#define TN_A_GEN_WAV_INPUT "a.gen.wav.input.%s"
|
|
#define TN_A_GEN_WAV_NORM "a.gen.wav.norm.%s"
|
|
#define TN_A_GEN_WAV_OUTPUT_NORM "a.gen.wav.output_norm.%s"
|
|
#define TN_A_GEN_WAV_OUTPUT "a.gen.wav.output.%s"
|
|
|
|
// cogvlm
|
|
#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
|
|
#define TN_MM_H_TO_4H "mm.up.%s"
|
|
#define TN_MM_GATE "mm.gate.%s"
|
|
#define TN_MM_4H_TO_H "mm.down.%s"
|
|
#define TN_TOK_BOI "v.boi"
|
|
#define TN_TOK_EOI "v.eoi"
|
|
|
|
// hunyuanvl (shared GGUF tensor names)
|
|
#define TN_MM_PRE_NORM "mm.pre_norm.%s"
|
|
#define TN_MM_IMG_BEGIN "mm.image_begin" // note: legacy name, new models should use v.token_embd.*
|
|
#define TN_MM_IMG_END "mm.image_end" // note: legacy name, new models should use v.token_embd.*
|
|
|
|
// deepseek-ocr
|
|
#define TN_SAM_POS_EMBD "v.sam.pos_embd.%s"
|
|
#define TN_SAM_PATCH_EMBD "v.sam.patch_embd.%s"
|
|
#define TN_SAM_PRE_NORM "v.sam.blk.%d.pre_ln.%s"
|
|
#define TN_SAM_POST_NORM "v.sam.blk.%d.post_ln.%s"
|
|
#define TN_SAM_ATTN_POS_H "v.sam.blk.%d.attn.pos_h.%s"
|
|
#define TN_SAM_ATTN_POS_W "v.sam.blk.%d.attn.pos_w.%s"
|
|
#define TN_SAM_ATTN_QKV "v.sam.blk.%d.attn.qkv.%s"
|
|
#define TN_SAM_ATTN_OUT "v.sam.blk.%d.attn.out.%s"
|
|
#define TN_SAM_FFN_UP "v.sam.blk.%d.mlp.lin1.%s"
|
|
#define TN_SAM_FFN_DOWN "v.sam.blk.%d.mlp.lin2.%s"
|
|
#define TN_SAM_NECK "v.sam.neck.%d.%s"
|
|
#define TN_SAM_NET "v.sam.net_%d.%s"
|
|
// deepseek-ocr-2
|
|
#define TN_RESMPL_QUERY "v.resample_query_%d.%s"
|
|
// (conformer) lfm2
|
|
#define TN_PRE_ENCODE_OUT "a.pre_encode.out.%s"
|
|
#define TN_FFN_NORM "%s.blk.%d.ffn_norm.%s"
|
|
#define TN_FFN_NORM_1 "%s.blk.%d.ffn_norm_1.%s"
|
|
#define TN_FFN_UP_1 "%s.blk.%d.ffn_up_1.%s"
|
|
#define TN_FFN_DOWN_1 "%s.blk.%d.ffn_down_1.%s"
|
|
#define TN_POS_BIAS_U "%s.blk.%d.pos_bias_u"
|
|
#define TN_POS_BIAS_V "%s.blk.%d.pos_bias_v"
|
|
#define TN_NORM_CONV "%s.blk.%d.norm_conv.%s"
|
|
#define TN_LINEAR_POS "%s.blk.%d.linear_pos.%s"
|
|
#define TN_CONV_DW "%s.blk.%d.conv_dw.%s"
|
|
#define TN_CONV_NORM "%s.blk.%d.conv_norm.%s"
|
|
#define TN_CONV_PW1 "%s.blk.%d.conv_pw1.%s"
|
|
#define TN_CONV_PW2 "%s.blk.%d.conv_pw2.%s"
|
|
#define TN_INP_PROJ "a.input_projection.%s"
|
|
#define TN_CTC_OUT "a.enc_ctc_out.%s"
|
|
#define TN_CTC_OUT_MID "a.enc_ctc_out_mid.%s"
|
|
#define TN_ATTN_REL_POS_EMB "%s.blk.%d.attn_rel_pos_emb"
|
|
// qformer projector
|
|
#define TN_QF_PROJ_QUERY "%s.proj_query"
|
|
#define TN_QF_PROJ_NORM "%s.proj_norm.%s"
|
|
#define TN_QF_PROJ_LINEAR "%s.proj_linear.%s"
|
|
#define TN_QF_SELF_ATTN_Q "%s.proj_blk.%d.self_attn_q.%s"
|
|
#define TN_QF_SELF_ATTN_K "%s.proj_blk.%d.self_attn_k.%s"
|
|
#define TN_QF_SELF_ATTN_V "%s.proj_blk.%d.self_attn_v.%s"
|
|
#define TN_QF_SELF_ATTN_O "%s.proj_blk.%d.self_attn_out.%s"
|
|
#define TN_QF_SELF_ATTN_N "%s.proj_blk.%d.self_attn_norm.%s"
|
|
#define TN_QF_CROSS_ATTN_Q "%s.proj_blk.%d.cross_attn_q.%s"
|
|
#define TN_QF_CROSS_ATTN_K "%s.proj_blk.%d.cross_attn_k.%s"
|
|
#define TN_QF_CROSS_ATTN_V "%s.proj_blk.%d.cross_attn_v.%s"
|
|
#define TN_QF_CROSS_ATTN_O "%s.proj_blk.%d.cross_attn_out.%s"
|
|
#define TN_QF_CROSS_ATTN_N "%s.proj_blk.%d.cross_attn_norm.%s"
|
|
#define TN_QF_FFN_UP "%s.proj_blk.%d.ffn_up.%s"
|
|
#define TN_QF_FFN_DOWN "%s.proj_blk.%d.ffn_down.%s"
|
|
#define TN_QF_FFN_NORM "%s.proj_blk.%d.ffn_norm.%s"
|
|
// multi-projector qformer (bid => projector ID)
|
|
#define TN_MULTI_PROJ_IMG_POS "v.proj_blk.%d.img_pos"
|
|
#define TN_MULTI_PROJ_QUERY "%s.proj_blk.%d.query"
|
|
#define TN_MULTI_PROJ_LINEAR "%s.proj_blk.%d.linear.%s"
|
|
#define TN_MULTI_PROJ_NORM "%s.proj_blk.%d.norm.%s"
|
|
#define TN_MULTI_PROJ_POST_NORM "%s.proj_blk.%d.post_norm.%s"
|
|
|
|
// gemma4 audio conformer
|
|
#define TN_A_MM_INP_PROJ "mm.a.input_projection.%s"
|
|
#define TN_A_MM_SOFT_EMB_N "mm.a.soft_emb_norm.%s"
|
|
#define TN_A_INP_PROJ "a.input_projection.%s"
|
|
#define TN_A_CONV1D "a.conv1d.%d.%s"
|
|
#define TN_A_CONV1D_NORM "a.conv1d.%d.norm.%s"
|
|
#define TN_A_PATCH "a.patch.%d.%s"
|
|
#define TN_A_PATCH_NORM "a.patch.%d.norm.%s"
|
|
|
|
// vibevoice-asr: causal ConvNeXt encoders, first index = encoder (0 = acoustic, 1 = semantic)
|
|
#define TN_A_CNX_CONV "a.cnx.%d.conv.%d.%s"
|
|
#define TN_A_CNX_NORM "a.cnx.%d.blk.%d.norm.%s"
|
|
#define TN_A_CNX_DW "a.cnx.%d.blk.%d.dw.%s"
|
|
#define TN_A_CNX_LS1 "a.cnx.%d.blk.%d.ls1.%s"
|
|
#define TN_A_CNX_FFN_NORM "a.cnx.%d.blk.%d.ffn_norm.%s"
|
|
#define TN_A_CNX_FFN_UP "a.cnx.%d.blk.%d.ffn_up.%s"
|
|
#define TN_A_CNX_FFN_DOWN "a.cnx.%d.blk.%d.ffn_down.%s"
|
|
#define TN_A_CNX_LS2 "a.cnx.%d.blk.%d.ls2.%s"
|
|
#define TN_MM_AUDIO_MLP_NORM "mm.a.mlp_norm.%d.%s"
|
|
#define TN_A_OUT_PROJ "a.pre_encode.out.%s"
|
|
#define TN_A_ATTN_PRE_NORM "%s.blk.%d.attn_pre_norm.%s"
|
|
#define TN_A_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s"
|
|
#define TN_A_ATTN_K_REL "%s.blk.%d.attn_k_rel.%s"
|
|
#define TN_A_PER_DIM_SCALE "%s.blk.%d.per_dim_scale.%s"
|
|
#define TN_A_PER_DIM_K_SCALE "%s.blk.%d.per_dim_k_scale.%s"
|
|
#define TN_A_FFN_POST_NORM "%s.blk.%d.ffn_post_norm.%s"
|
|
#define TN_A_FFN_POST_NORM_1 "%s.blk.%d.ffn_post_norm_1.%s"
|
|
|
|
// mobilenetv5 (gemma3n) definitions
|
|
#define TN_MNV5_STEM_CONV "v.conv_stem.conv.weight"
|
|
#define TN_MNV5_STEM_BIAS "v.conv_stem.conv.bias"
|
|
#define TN_MNV5_STEM_BN "v.conv_stem.bn.weight"
|
|
|
|
// Stage 0 Block (Edge Residual)
|
|
#define TN_MNV5_BLK_S0_EXP_W "v.blk.%d.%d.conv_exp.weight"
|
|
#define TN_MNV5_BLK_S0_BN1_W "v.blk.%d.%d.bn1.weight"
|
|
#define TN_MNV5_BLK_S0_PWL_W "v.blk.%d.%d.conv_pwl.weight"
|
|
#define TN_MNV5_BLK_S0_BN2_W "v.blk.%d.%d.bn2.weight"
|
|
|
|
// Stage 1+ Block (Universal Inverted Residual)
|
|
#define TN_MNV5_BLK_DW_START_W "v.blk.%d.%d.dw_start.conv.weight"
|
|
#define TN_MNV5_BLK_DW_START_BN "v.blk.%d.%d.dw_start.bn.weight"
|
|
#define TN_MNV5_BLK_DW_MID_W "v.blk.%d.%d.dw_mid.conv.weight"
|
|
#define TN_MNV5_BLK_DW_MID_BN "v.blk.%d.%d.dw_mid.bn.weight"
|
|
#define TN_MNV5_BLK_PW_EXP_W "v.blk.%d.%d.pw_exp.conv.weight"
|
|
#define TN_MNV5_BLK_PW_EXP_BN "v.blk.%d.%d.pw_exp.bn.weight"
|
|
#define TN_MNV5_BLK_PW_PROJ_W "v.blk.%d.%d.pw_proj.conv.weight"
|
|
#define TN_MNV5_BLK_PW_PROJ_BN "v.blk.%d.%d.pw_proj.bn.weight"
|
|
#define TN_MNV5_BLK_LAYER_SCALE "v.blk.%d.%d.layer_scale.gamma"
|
|
|
|
// Attention Components
|
|
#define TN_MNV5_ATTN_Q_W "v.blk.%d.%d.attn.query.proj.weight"
|
|
#define TN_MNV5_ATTN_K_W "v.blk.%d.%d.attn.key.proj.weight"
|
|
#define TN_MNV5_ATTN_V_W "v.blk.%d.%d.attn.value.proj.weight"
|
|
#define TN_MNV5_ATTN_O_W "v.blk.%d.%d.attn.output.proj.weight"
|
|
#define TN_MNV5_ATTN_K_DW "v.blk.%d.%d.attn.key.down_conv.weight"
|
|
#define TN_MNV5_ATTN_K_NORM "v.blk.%d.%d.attn.key.norm.weight"
|
|
#define TN_MNV5_ATTN_V_DW "v.blk.%d.%d.attn.value.down_conv.weight"
|
|
#define TN_MNV5_ATTN_V_NORM "v.blk.%d.%d.attn.value.norm.weight"
|
|
#define TN_MNV5_ATTN_NORM "v.blk.%d.%d.norm.weight" // Block norm used in attn blocks
|
|
|
|
// MSFA
|
|
#define TN_MNV5_MSFA_FFN_EXP_W "v.msfa.ffn.pw_exp.conv.weight"
|
|
#define TN_MNV5_MSFA_FFN_EXP_BN "v.msfa.ffn.pw_exp.bn.weight"
|
|
#define TN_MNV5_MSFA_FFN_PROJ_W "v.msfa.ffn.pw_proj.conv.weight"
|
|
#define TN_MNV5_MSFA_FFN_PROJ_BN "v.msfa.ffn.pw_proj.bn.weight"
|
|
#define TN_MNV5_MSFA_NORM "v.msfa.norm.weight"
|
|
|
|
// gemma4
|
|
#define TN_STD_BIAS "v.std_bias"
|
|
#define TN_STD_SCALE "v.std_scale"
|
|
|
|
// yasa2
|
|
#define TN_YASA_PATCH_LN_W "v.patch_ln.weight"
|
|
#define TN_YASA_PATCH_LN_B "v.patch_ln.bias"
|
|
#define TN_YASA_BACKBONE_LN_W "v.backbone_ln.weight"
|
|
#define TN_YASA_BACKBONE_LN_B "v.backbone_ln.bias"
|
|
#define TN_YASA_POS_EMBD "v.vision_pos_embed"
|
|
#define TN_YASA_STAGE_DOWN_LN "v.stage.%d.down.ln.%s"
|
|
#define TN_YASA_STAGE_DOWN_CONV "v.stage.%d.down.conv.%s"
|
|
#define TN_YASA_STAGE_BLK "v.stage.%d.blk.%d.%s.%s"
|
|
|
|
// parakeet
|
|
#define TN_MEL_FILTERS "a.mel_filters"
|
|
#define TN_WINDOW "a.window"
|
|
#define TN_CONV_NORM_MEAN "%s.blk.%d.conv_norm_mean"
|
|
#define TN_CONV_NORM_VAR "%s.blk.%d.conv_norm_var"
|
|
|
|
// nemotron transcribe
|
|
#define TN_A_SPK_INP_PROJ "a.spk.input_projection.%s"
|
|
#define TN_A_SPK_HEAD "a.spk.head.%s.%s" // proj, upsample, fc1, fc2
|
|
#define TN_A_SPK_SIL_EMB "a.spk.head.sil_emb"
|
|
#define TN_A_FUSION_NORM "a.fusion.%s.%s" // asr_norm, spk_norm
|
|
#define TN_A_FUSION_KERNEL "a.fusion.spk_kernel"
|
|
|
|
// align x to upper multiple of n
|
|
#define MTMD_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))
|
|
|
|
enum projector_type {
|
|
PROJECTOR_TYPE_MLP,
|
|
PROJECTOR_TYPE_MLP_NORM,
|
|
PROJECTOR_TYPE_LDP,
|
|
PROJECTOR_TYPE_LDPV2,
|
|
PROJECTOR_TYPE_MINICPMV,
|
|
PROJECTOR_TYPE_GLM_EDGE,
|
|
PROJECTOR_TYPE_QWEN2VL,
|
|
PROJECTOR_TYPE_QWEN3VL,
|
|
PROJECTOR_TYPE_LING3VL,
|
|
PROJECTOR_TYPE_STEP3VL,
|
|
PROJECTOR_TYPE_GEMMA3,
|
|
PROJECTOR_TYPE_GEMMA3NV,
|
|
PROJECTOR_TYPE_GEMMA3NA,
|
|
PROJECTOR_TYPE_GEMMA4V,
|
|
PROJECTOR_TYPE_GEMMA4A,
|
|
PROJECTOR_TYPE_GEMMA4UV,
|
|
PROJECTOR_TYPE_GEMMA4UA,
|
|
PROJECTOR_TYPE_PHI4,
|
|
PROJECTOR_TYPE_IDEFICS3,
|
|
PROJECTOR_TYPE_PIXTRAL,
|
|
PROJECTOR_TYPE_QWEN25VL,
|
|
PROJECTOR_TYPE_ULTRAVOX,
|
|
PROJECTOR_TYPE_INTERNVL,
|
|
PROJECTOR_TYPE_LLAMA4,
|
|
PROJECTOR_TYPE_QWEN2A,
|
|
PROJECTOR_TYPE_QWEN3A,
|
|
PROJECTOR_TYPE_GLMA,
|
|
PROJECTOR_TYPE_HIGGS_AUDIO,
|
|
PROJECTOR_TYPE_VIBEVOICE_ASR,
|
|
PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on mtmd_subcontext
|
|
PROJECTOR_TYPE_VOXTRAL,
|
|
PROJECTOR_TYPE_VOXTRAL_RT,
|
|
PROJECTOR_TYPE_MERALION,
|
|
PROJECTOR_TYPE_MUSIC_FLAMINGO,
|
|
PROJECTOR_TYPE_LFM2,
|
|
PROJECTOR_TYPE_LFM3VL,
|
|
PROJECTOR_TYPE_D1OMNI_V,
|
|
PROJECTOR_TYPE_KIMIVL,
|
|
PROJECTOR_TYPE_PADDLEOCR,
|
|
PROJECTOR_TYPE_LIGHTONOCR,
|
|
PROJECTOR_TYPE_COGVLM,
|
|
PROJECTOR_TYPE_JANUS_PRO,
|
|
PROJECTOR_TYPE_DOTS_OCR,
|
|
PROJECTOR_TYPE_DOTS3NOTE_V,
|
|
PROJECTOR_TYPE_DOTS3NOTE_A,
|
|
PROJECTOR_TYPE_DEEPSEEKOCR,
|
|
PROJECTOR_TYPE_DEEPSEEKOCR2,
|
|
PROJECTOR_TYPE_DEEPSEEK4V,
|
|
PROJECTOR_TYPE_DEEPSEEK41V,
|
|
PROJECTOR_TYPE_LFM2A,
|
|
PROJECTOR_TYPE_D1OMNI_A,
|
|
PROJECTOR_TYPE_LFM3A,
|
|
PROJECTOR_TYPE_GLM4V,
|
|
PROJECTOR_TYPE_GLM5V,
|
|
PROJECTOR_TYPE_YOUTUVL,
|
|
PROJECTOR_TYPE_YASA2,
|
|
PROJECTOR_TYPE_KIMIK25,
|
|
PROJECTOR_TYPE_KIMIK31,
|
|
PROJECTOR_TYPE_NEMOTRON_V2_VL,
|
|
PROJECTOR_TYPE_NEMOTRON_OMNI,
|
|
PROJECTOR_TYPE_HUNYUANVL,
|
|
PROJECTOR_TYPE_PARAKEET,
|
|
PROJECTOR_TYPE_EXAONE4_5,
|
|
PROJECTOR_TYPE_MINICPMV4_6,
|
|
PROJECTOR_TYPE_MINICPMV4_7,
|
|
PROJECTOR_TYPE_MINICPM_OCR_1_0,
|
|
PROJECTOR_TYPE_GRANITE_SPEECH,
|
|
PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE,
|
|
PROJECTOR_TYPE_MIMOVL,
|
|
PROJECTOR_TYPE_MAGEVL,
|
|
PROJECTOR_TYPE_MINIMAX_M3,
|
|
PROJECTOR_TYPE_GRANITE4_VISION,
|
|
PROJECTOR_TYPE_MIMO_AUDIO,
|
|
PROJECTOR_TYPE_QWEN3TTS_SPKENC,
|
|
PROJECTOR_TYPE_QWEN3TTS_GEN,
|
|
PROJECTOR_TYPE_POCKETTTS_SPKENC,
|
|
PROJECTOR_TYPE_POCKETTTS_GEN,
|
|
PROJECTOR_TYPE_SOPRANO,
|
|
PROJECTOR_TYPE_LFM3A_GEN,
|
|
PROJECTOR_TYPE_DRAMA3_GEN,
|
|
PROJECTOR_TYPE_ZONOS3_GEN,
|
|
PROJECTOR_TYPE_ZONOS3_SPKENC,
|
|
PROJECTOR_TYPE_YUE2_GEN,
|
|
PROJECTOR_TYPE_FISH_S2_GEN,
|
|
PROJECTOR_TYPE_FISH_S2_SPKENC,
|
|
PROJECTOR_TYPE_MUSE_GLIMMER,
|
|
PROJECTOR_TYPE_SENSENOVA_U1,
|
|
PROJECTOR_TYPE_COHERE2V,
|
|
PROJECTOR_TYPE_MUSE_SPARK_AUDIO,
|
|
PROJECTOR_TYPE_CHATTERBOX,
|
|
PROJECTOR_TYPE_CHATTERBOX_SPKENC,
|
|
PROJECTOR_TYPE_INKLING2V,
|
|
PROJECTOR_TYPE_UNKNOWN,
|
|
};
|
|
|
|
static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
|
{ PROJECTOR_TYPE_MLP, "mlp" },
|
|
{ PROJECTOR_TYPE_LDP, "ldp" },
|
|
{ PROJECTOR_TYPE_LDPV2, "ldpv2"},
|
|
{ PROJECTOR_TYPE_MINICPMV, "resampler"},
|
|
{ PROJECTOR_TYPE_GLM_EDGE, "adapter"},
|
|
{ PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"},
|
|
{ PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"},
|
|
{ PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"},
|
|
{ PROJECTOR_TYPE_LING3VL, "ling3vl"},
|
|
{ PROJECTOR_TYPE_STEP3VL, "step3vl"},
|
|
{ PROJECTOR_TYPE_GEMMA3, "gemma3"},
|
|
{ PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"},
|
|
{ PROJECTOR_TYPE_GEMMA3NA, "gemma3na"},
|
|
{ PROJECTOR_TYPE_GEMMA4V, "gemma4v"},
|
|
{ PROJECTOR_TYPE_GEMMA4A, "gemma4a"},
|
|
{ PROJECTOR_TYPE_GEMMA4UV, "gemma4uv"},
|
|
{ PROJECTOR_TYPE_GEMMA4UA, "gemma4ua"},
|
|
{ PROJECTOR_TYPE_PHI4, "phi4"},
|
|
{ PROJECTOR_TYPE_IDEFICS3, "idefics3"},
|
|
{ PROJECTOR_TYPE_PIXTRAL, "pixtral"},
|
|
{ PROJECTOR_TYPE_ULTRAVOX, "ultravox"},
|
|
{ PROJECTOR_TYPE_INTERNVL, "internvl"},
|
|
{ PROJECTOR_TYPE_LLAMA4, "llama4"},
|
|
{ PROJECTOR_TYPE_QWEN2A, "qwen2a"},
|
|
{ PROJECTOR_TYPE_QWEN3A, "qwen3a"},
|
|
{ PROJECTOR_TYPE_GLMA, "glma"},
|
|
{ PROJECTOR_TYPE_HIGGS_AUDIO, "higgs_audio"},
|
|
{ PROJECTOR_TYPE_VIBEVOICE_ASR, "vibevoice_asr"},
|
|
{ PROJECTOR_TYPE_QWEN25O, "qwen2.5o"},
|
|
{ PROJECTOR_TYPE_VOXTRAL, "voxtral"},
|
|
{ PROJECTOR_TYPE_VOXTRAL_RT, "voxtral_rt"},
|
|
{ PROJECTOR_TYPE_MERALION, "meralion"},
|
|
{ PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"},
|
|
{ PROJECTOR_TYPE_LFM2, "lfm2"},
|
|
{ PROJECTOR_TYPE_LFM3VL, "lfm3vl"},
|
|
{ PROJECTOR_TYPE_D1OMNI_V, "d1omni_v"},
|
|
{ PROJECTOR_TYPE_KIMIVL, "kimivl"},
|
|
{ PROJECTOR_TYPE_PADDLEOCR, "paddleocr"},
|
|
{ PROJECTOR_TYPE_LIGHTONOCR, "lightonocr"},
|
|
{ PROJECTOR_TYPE_COGVLM, "cogvlm"},
|
|
{ PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
|
|
{ PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"},
|
|
{ PROJECTOR_TYPE_DOTS3NOTE_V, "dots3note_v"},
|
|
{ PROJECTOR_TYPE_DOTS3NOTE_A, "dots3note_a"},
|
|
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
|
|
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
|
|
{ PROJECTOR_TYPE_DEEPSEEK4V, "deepseek4v"},
|
|
{ PROJECTOR_TYPE_DEEPSEEK41V, "deepseek41v"},
|
|
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
|
|
{ PROJECTOR_TYPE_D1OMNI_A, "d1omni_a"},
|
|
{ PROJECTOR_TYPE_LFM3A, "lfm3a"},
|
|
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
|
|
{ PROJECTOR_TYPE_GLM5V, "glm5v"},
|
|
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
|
|
{ PROJECTOR_TYPE_YASA2, "yasa2"},
|
|
{ PROJECTOR_TYPE_KIMIK25, "kimik25"},
|
|
{ PROJECTOR_TYPE_KIMIK31, "kimik31"},
|
|
{ PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},
|
|
{ PROJECTOR_TYPE_NEMOTRON_OMNI, "nemotron_omni"},
|
|
{ PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"},
|
|
{ PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"},
|
|
{ PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"},
|
|
{ PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"},
|
|
{ PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"},
|
|
{ PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"},
|
|
{ PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE, "nemotron_transcribe"},
|
|
{ PROJECTOR_TYPE_MIMOVL, "mimovl"},
|
|
{ PROJECTOR_TYPE_MAGEVL, "magevl"},
|
|
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
|
|
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
|
|
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
|
|
{ PROJECTOR_TYPE_PARAKEET, "parakeet"},
|
|
{ PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"},
|
|
{ PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"},
|
|
{ PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"},
|
|
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
|
|
{ PROJECTOR_TYPE_SOPRANO, "soprano"},
|
|
{ PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"},
|
|
{ PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"},
|
|
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
|
|
{ PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"},
|
|
{ PROJECTOR_TYPE_YUE2_GEN, "yue2_gen"},
|
|
{ PROJECTOR_TYPE_FISH_S2_GEN, "fish_s2_gen"},
|
|
{ PROJECTOR_TYPE_FISH_S2_SPKENC, "fish_s2_spkenc"},
|
|
{ PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"},
|
|
{ PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"},
|
|
{ PROJECTOR_TYPE_COHERE2V, "cohere2v"},
|
|
{ PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"},
|
|
{ PROJECTOR_TYPE_CHATTERBOX, "chatterbox"},
|
|
{ PROJECTOR_TYPE_CHATTERBOX_SPKENC, "chatterbox_spkenc"},
|
|
{ PROJECTOR_TYPE_INKLING2V, "inkling2v"},
|
|
};
|
|
|
|
static projector_type projector_type_from_string(const std::string & str) {
|
|
for (const auto & pair : PROJECTOR_TYPE_NAMES) {
|
|
if (pair.second == str) {
|
|
return pair.first;
|
|
}
|
|
}
|
|
return PROJECTOR_TYPE_UNKNOWN;
|
|
}
|
|
|
|
struct mtmd_image_size {
|
|
int width;
|
|
int height;
|
|
bool operator==(const mtmd_image_size & other) const {
|
|
return width == other.width && height == other.height;
|
|
}
|
|
bool operator!=(const mtmd_image_size & other) const {
|
|
return !(*this == other);
|
|
}
|
|
int area() const {
|
|
// avoid overflow when computing area
|
|
GGML_ASSERT(width >= 0 && width <= 46000);
|
|
GGML_ASSERT(height >= 0 && height <= 46000);
|
|
return width * height;
|
|
}
|
|
};
|
|
|
|
// RGB uint8 image
|
|
struct mtmd_image_u8 {
|
|
mtmd_image_size get_size() const {
|
|
return { nx, ny };
|
|
}
|
|
|
|
void set_size(mtmd_image_size size, bool is_placeholder) {
|
|
nx = size.width;
|
|
ny = size.height;
|
|
if (is_placeholder) {
|
|
buf.clear();
|
|
} else {
|
|
buf.resize((size_t) nx * (size_t) ny * 3);
|
|
}
|
|
}
|
|
|
|
void cpy_buf(const std::vector<uint8_t> & new_buf) {
|
|
buf = new_buf;
|
|
}
|
|
|
|
const std::vector<uint8_t> & get_ro_buf() const {
|
|
if (is_placeholder()) {
|
|
throw std::runtime_error("this mtmd_image_u8 is a placeholder");
|
|
}
|
|
return buf;
|
|
}
|
|
|
|
// note to contributors: NEVER add a get_rw_buf(), it is a DANGEROUS pattern. always use get_pixel / set_pixel for buffer manipulation
|
|
|
|
bool is_placeholder() const {
|
|
return buf.empty();
|
|
}
|
|
|
|
std::array<uint8_t, 3> get_pixel(int x, int y) const {
|
|
if (is_placeholder()) {
|
|
// return a dummy value, so that legacy code can still process image without errors
|
|
return { 0, 0, 0 };
|
|
}
|
|
size_t idx = ((size_t) y * (size_t) nx + (size_t) x) * 3;
|
|
return { buf[idx], buf[idx + 1], buf[idx + 2] };
|
|
}
|
|
|
|
void set_pixel(int x, int y, const std::array<uint8_t, 3> & rgb) {
|
|
if (is_placeholder()) {
|
|
return; // no-op
|
|
}
|
|
size_t idx = ((size_t) y * (size_t) nx + (size_t) x) * 3;
|
|
buf[idx] = rgb[0];
|
|
buf[idx + 1] = rgb[1];
|
|
buf[idx + 2] = rgb[2];
|
|
}
|
|
|
|
size_t n_elements() const {
|
|
return n_pixels() * 3;
|
|
}
|
|
|
|
private:
|
|
std::vector<uint8_t> buf;
|
|
int nx = 0;
|
|
int ny = 0;
|
|
|
|
size_t n_pixels() const {
|
|
return (size_t) nx * (size_t) ny;
|
|
}
|
|
};
|
|
|
|
struct mtmd_serialization; // forward declaration
|
|
|
|
// separators appended after the image tokens of one entry, as rows of v.tok_embd_sep
|
|
enum mtmd_suffix_type : int32_t {
|
|
MTMD_SUFFIX_NONE = 0,
|
|
// MiniCPM-V 4.7 tiles
|
|
MTMD_SUFFIX_MINICPMV_OV, // </image>
|
|
MTMD_SUFFIX_MINICPMV_OV_SLICE, // </image><slice>
|
|
MTMD_SUFFIX_MINICPMV_SLICE, // </slice><slice>
|
|
MTMD_SUFFIX_MINICPMV_ROW_END, // </slice>\n<slice>
|
|
MTMD_SUFFIX_MINICPMV_LAST, // </slice>
|
|
MTMD_SUFFIX_COUNT,
|
|
};
|
|
|
|
// rows of v.tok_embd_sep for each suffix type
|
|
// MiniCPM-V 4.7 rows (set by the converter): 0 = </image>, 1 = <slice>, 2 = </slice>, 3 = \n
|
|
static inline const std::vector<int> & mtmd_suffix_rows(mtmd_suffix_type type) {
|
|
static const std::vector<int> none;
|
|
static const std::vector<int> minicpmv_ov = { 0 };
|
|
static const std::vector<int> minicpmv_ov_slice = { 0, 1 };
|
|
static const std::vector<int> minicpmv_slice = { 2, 1 };
|
|
static const std::vector<int> minicpmv_row_end = { 2, 3, 1 };
|
|
static const std::vector<int> minicpmv_last = { 2 };
|
|
switch (type) {
|
|
case MTMD_SUFFIX_NONE: return none;
|
|
case MTMD_SUFFIX_MINICPMV_OV: return minicpmv_ov;
|
|
case MTMD_SUFFIX_MINICPMV_OV_SLICE: return minicpmv_ov_slice;
|
|
case MTMD_SUFFIX_MINICPMV_SLICE: return minicpmv_slice;
|
|
case MTMD_SUFFIX_MINICPMV_ROW_END: return minicpmv_row_end;
|
|
case MTMD_SUFFIX_MINICPMV_LAST: return minicpmv_last;
|
|
default: GGML_ABORT("invalid suffix type");
|
|
}
|
|
}
|
|
|
|
// For images, buf.size() == nx*ny*3
|
|
// Memory layout: RGBRGBRGB...
|
|
// For seq, buf.size() == nx*ny*3*nt
|
|
// Memory layout: RGBRGB...RGBRGB... (nt times)
|
|
// For audio, only one channel is used, buf.size() == nx*ny
|
|
// nx will be n_frames and ny will be n_mel
|
|
struct mtmd_image_f32 {
|
|
// marks the global view in e.g., DeepSeek-OCR Models
|
|
bool add_viewsep = false;
|
|
// appends a learned newline (or EOI) token after the image
|
|
// no model uses it now (Granite4 Vision moved to anyres), kept for future models
|
|
bool add_newline = false;
|
|
// deepseek4v: number of leading IMAGE_PAD embeddings, aligns IMAGE_START to the LLM compressor ratio
|
|
// depends on the chunk position, set at tokenize time (see mtmd_tokenizer::add_media)
|
|
int32_t lead_pad = 0;
|
|
// audio (vibevoice): leading samples that only feed the causal convs, they produce no embedding
|
|
int32_t audio_n_ctx = 0;
|
|
// separators appended after the image tokens
|
|
mtmd_suffix_type suffix_type = MTMD_SUFFIX_NONE;
|
|
|
|
// tile grid of the image group this entry belongs to
|
|
// llava-next "anyres" (Granite4 Vision): the whole grid is encoded and assembled in a single graph
|
|
// MiniCPM-V 4.7: set on the overview entry, the decoder positions of all tiles are derived from it
|
|
struct anyres_info {
|
|
int grid_x = 0; // tiles per row, 0 means the image is not tiled
|
|
int grid_y = 0; // tiles per column
|
|
int orig_nx = 0; // size of the source image, used to drop the padding tokens
|
|
int orig_ny = 0;
|
|
|
|
bool is_tiled() const {
|
|
return grid_x > 0 && grid_y > 0;
|
|
}
|
|
};
|
|
anyres_info anyres;
|
|
|
|
mtmd_image_size get_size() const {
|
|
return { nx_, ny_ };
|
|
}
|
|
|
|
int nx() const { return nx_; }
|
|
int ny() const { return ny_; }
|
|
|
|
void set_size(mtmd_image_size size, bool is_placeholder, bool is_audio) {
|
|
nx_ = size.width;
|
|
ny_ = size.height;
|
|
if (is_placeholder) {
|
|
buf.clear();
|
|
} else {
|
|
if (is_audio) {
|
|
buf.resize((size_t) nx_ * (size_t) ny_);
|
|
} else {
|
|
buf.resize((size_t) nx_ * (size_t) ny_ * 3);
|
|
}
|
|
}
|
|
}
|
|
|
|
void cpy_buf(const std::vector<float> & new_buf) {
|
|
buf = new_buf;
|
|
}
|
|
|
|
void from_u8(const mtmd_image_u8 & img) {
|
|
auto size = img.get_size();
|
|
nx_ = size.width;
|
|
ny_ = size.height;
|
|
if (img.is_placeholder()) {
|
|
buf.clear();
|
|
return; // no-op
|
|
}
|
|
buf.resize(img.n_elements());
|
|
const auto & u8_buf = img.get_ro_buf();
|
|
for (size_t i = 0; i < img.n_elements(); ++i) {
|
|
buf[i] = (float) u8_buf[i] / 255.0f;
|
|
}
|
|
}
|
|
|
|
size_t n_elements() const {
|
|
return n_pixels() * 3;
|
|
}
|
|
|
|
void normalize(const float mean[3], const float std[3]) {
|
|
if (is_placeholder()) {
|
|
return; // no-op
|
|
}
|
|
for (size_t i = 0; i < n_pixels(); ++i) {
|
|
buf[i * 3 + 0] = (buf[i * 3 + 0] - mean[0]) / std[0];
|
|
buf[i * 3 + 1] = (buf[i * 3 + 1] - mean[1]) / std[1];
|
|
buf[i * 3 + 2] = (buf[i * 3 + 2] - mean[2]) / std[2];
|
|
}
|
|
}
|
|
|
|
const std::vector<float> & get_ro_buf() const {
|
|
if (is_placeholder()) {
|
|
throw std::runtime_error("this mtmd_image_f32 is a placeholder");
|
|
}
|
|
return buf;
|
|
}
|
|
|
|
// note to contributors: NEVER add a get_rw_buf(), it is a DANGEROUS pattern
|
|
|
|
bool is_placeholder() const {
|
|
return buf.empty();
|
|
}
|
|
|
|
void serialize(struct mtmd_serialization & ser) const;
|
|
void deserialize(struct mtmd_serialization & ser);
|
|
|
|
private:
|
|
std::vector<float> buf;
|
|
int nx_ = 0;
|
|
int ny_ = 0;
|
|
|
|
size_t n_pixels() const {
|
|
return (size_t) nx_ * (size_t) ny_;
|
|
}
|
|
};
|
|
|
|
// token area kept after removing the padding added by the anyres resize
|
|
// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L109
|
|
static inline void mtmd_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_h,
|
|
int & off_x, int & off_y, int & out_w, int & out_h) {
|
|
off_x = 0;
|
|
off_y = 0;
|
|
out_w = cur_w;
|
|
out_h = cur_h;
|
|
if ((float) orig_w / orig_h > (float) cur_w / cur_h) {
|
|
const int new_h = (int) std::floor((double) orig_h * cur_w / orig_w + 1e-7);
|
|
off_y = (cur_h - new_h) / 2;
|
|
out_h = cur_h - 2 * off_y;
|
|
} else {
|
|
const int new_w = (int) std::floor((double) orig_w * cur_h / orig_h + 1e-7);
|
|
off_x = (cur_w - new_w) / 2;
|
|
out_w = cur_w - 2 * off_x;
|
|
}
|
|
}
|
|
|
|
// deepseek41v: [START] ([IMAGE]*n_llm_w [NEWLINE])*n_llm_h [END], ref: num_image_tokens() in image_processor.py
|
|
static inline int dsv41_n_block_tokens(int n_llm_w, int n_llm_h) {
|
|
return n_llm_h * (n_llm_w + 1) + 2;
|
|
}
|
|
|
|
// deepseek4v: layout of the LLM token block built from the aligner grid
|
|
struct dsv4_block_layout {
|
|
int rows; // grid rows, padded to an even count
|
|
int row_len; // grid width + 1 newline
|
|
int pad_last; // trailing pads
|
|
int n_out; // total block size, including lead pads and the start/end sentinels
|
|
};
|
|
static inline dsv4_block_layout dsv4_get_block_layout(int n_llm_w, int n_llm_h, int lead_pad) {
|
|
dsv4_block_layout bl;
|
|
bl.rows = n_llm_h + (n_llm_h % 2);
|
|
bl.row_len = n_llm_w + 1;
|
|
bl.pad_last = (bl.rows / 2 * bl.row_len) % 2 * 2;
|
|
bl.n_out = lead_pad + 1 + bl.rows * bl.row_len + bl.pad_last + 1;
|
|
return bl;
|
|
}
|
|
|
|
//
|
|
// logging
|
|
//
|
|
|
|
static void mtmd_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {
|
|
(void) level;
|
|
(void) user_data;
|
|
fputs(text, stderr);
|
|
fflush(stderr);
|
|
}
|
|
|
|
struct mtmd_logger_state {
|
|
ggml_log_callback log_callback;
|
|
void * log_callback_user_data;
|
|
};
|
|
|
|
extern struct mtmd_logger_state g_logger_state;
|
|
|
|
static void mtmd_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {
|
|
if (format == NULL) {
|
|
return;
|
|
}
|
|
va_list args_copy;
|
|
va_copy(args_copy, args);
|
|
char buffer[128];
|
|
int len = vsnprintf(buffer, 128, format, args);
|
|
if (len < 128) {
|
|
g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);
|
|
} else {
|
|
char * buffer2 = (char *) calloc(len + 1, sizeof(char));
|
|
vsnprintf(buffer2, len + 1, format, args_copy);
|
|
buffer2[len] = 0;
|
|
g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);
|
|
free(buffer2);
|
|
}
|
|
va_end(args_copy);
|
|
}
|
|
|
|
static void mtmd_log_internal(enum ggml_log_level level, const char * format, ...) {
|
|
va_list args;
|
|
va_start(args, format);
|
|
mtmd_log_internal_v(level, format, args);
|
|
va_end(args);
|
|
}
|
|
|
|
#define LOG_TRC(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
|
|
#define LOG_DBG(...) mtmd_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
|
|
#define LOG_INF(...) mtmd_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
|
|
#define LOG_WRN(...) mtmd_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
|
|
#define LOG_ERR(...) mtmd_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
|
|
#define LOG_CNT(...) mtmd_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__)
|
|
|
|
//
|
|
// cpp wrappers
|
|
//
|
|
|
|
struct mtmd_image_f32_batch {
|
|
std::vector<mtmd_image_f32> entries;
|
|
bool is_audio = false;
|
|
bool is_video = false;
|
|
|
|
mtmd_image_f32_batch clone() const {
|
|
mtmd_image_f32_batch new_batch{
|
|
/* entries */ {},
|
|
/* is_audio */ is_audio,
|
|
/* is_video */ is_video,
|
|
};
|
|
new_batch.entries.reserve(entries.size());
|
|
for (const auto & entry : entries) {
|
|
new_batch.entries.emplace_back(entry); // copy
|
|
}
|
|
return new_batch;
|
|
}
|
|
|
|
void serialize(struct mtmd_serialization & ser) const;
|
|
void deserialize(struct mtmd_serialization & ser);
|
|
};
|
|
|
|
//
|
|
// common utils
|
|
//
|
|
|
|
#ifdef _WIN32
|
|
static std::ifstream open_ifstream_binary(const std::string & fname) {
|
|
int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0);
|
|
if (!wlen) {
|
|
throw std::runtime_error("failed to convert filename to UTF-16: " + fname);
|
|
}
|
|
std::vector<wchar_t> wfname(wlen);
|
|
(void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen);
|
|
return std::ifstream(wfname.data(), std::ios::binary);
|
|
}
|
|
#else
|
|
static std::ifstream open_ifstream_binary(const std::string & fname) {
|
|
return std::ifstream(fname, std::ios::binary);
|
|
}
|
|
#endif
|
|
|
|
std::string mtmd_format(const char * fmt, ...);
|
|
|
|
// in test-mtmd-impl, we include woth common.h and this file, and these functions are duplicated
|
|
// this is a quick fix to avoid compilation errors
|
|
#ifndef DIRECTORY_SEPARATOR
|
|
static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {
|
|
if (search.empty()) {
|
|
return;
|
|
}
|
|
std::string builder;
|
|
builder.reserve(s.length());
|
|
size_t pos = 0;
|
|
size_t last_pos = 0;
|
|
while ((pos = s.find(search, last_pos)) != std::string::npos) {
|
|
builder.append(s, last_pos, pos - last_pos);
|
|
builder.append(replace);
|
|
last_pos = pos + search.length();
|
|
}
|
|
builder.append(s, last_pos, std::string::npos);
|
|
s = std::move(builder);
|
|
}
|
|
|
|
// split string by a `std::string delim` instead of `char delim`
|
|
static std::vector<std::string> string_split_str(std::string s, const std::string & delimiter) {
|
|
std::vector<std::string> tokens;
|
|
size_t pos = 0;
|
|
std::string token;
|
|
while ((pos = s.find(delimiter)) != std::string::npos) {
|
|
token = s.substr(0, pos);
|
|
tokens.push_back(token);
|
|
s.erase(0, pos + delimiter.length());
|
|
}
|
|
tokens.push_back(s);
|
|
return tokens;
|
|
}
|
|
|
|
// remove when moving to c++20
|
|
inline bool string_starts_with(std::string_view str, std::string_view prefix) {
|
|
return str.size() >= prefix.size() &&
|
|
str.compare(0, prefix.size(), prefix) == 0;
|
|
}
|
|
|
|
// remove when moving to c++20
|
|
inline bool string_ends_with(std::string_view str, std::string_view suffix) {
|
|
return str.size() >= suffix.size() &&
|
|
str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;
|
|
}
|
|
#endif
|
|
|
|
//
|
|
// gguf utils
|
|
//
|
|
|
|
static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {
|
|
switch (type) {
|
|
case GGUF_TYPE_UINT8: return std::to_string(((const uint8_t *)data)[i]);
|
|
case GGUF_TYPE_INT8: return std::to_string(((const int8_t *)data)[i]);
|
|
case GGUF_TYPE_UINT16: return std::to_string(((const uint16_t *)data)[i]);
|
|
case GGUF_TYPE_INT16: return std::to_string(((const int16_t *)data)[i]);
|
|
case GGUF_TYPE_UINT32: return std::to_string(((const uint32_t *)data)[i]);
|
|
case GGUF_TYPE_INT32: return std::to_string(((const int32_t *)data)[i]);
|
|
case GGUF_TYPE_UINT64: return std::to_string(((const uint64_t *)data)[i]);
|
|
case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);
|
|
case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);
|
|
case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);
|
|
case GGUF_TYPE_BOOL: return ((const int8_t *)data)[i] != 0 ? "true" : "false";
|
|
default: return mtmd_format("unknown type %d", type);
|
|
}
|
|
}
|
|
|
|
static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {
|
|
const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);
|
|
|
|
switch (type) {
|
|
case GGUF_TYPE_STRING:
|
|
return gguf_get_val_str(ctx_gguf, i);
|
|
case GGUF_TYPE_ARRAY:
|
|
{
|
|
const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);
|
|
int arr_n = gguf_get_arr_n(ctx_gguf, i);
|
|
const void * data = arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx_gguf, i);
|
|
std::stringstream ss;
|
|
ss << "[";
|
|
for (int j = 0; j < arr_n; j++) {
|
|
if (arr_type == GGUF_TYPE_STRING) {
|
|
std::string val = gguf_get_arr_str(ctx_gguf, i, j);
|
|
// escape quotes
|
|
string_replace_all(val, "\\", "\\\\");
|
|
string_replace_all(val, "\"", "\\\"");
|
|
ss << '"' << val << '"';
|
|
} else if (arr_type == GGUF_TYPE_ARRAY) {
|
|
ss << "???";
|
|
} else {
|
|
ss << gguf_data_to_str(arr_type, data, j);
|
|
}
|
|
if (j < arr_n - 1) {
|
|
ss << ", ";
|
|
}
|
|
}
|
|
ss << "]";
|
|
return ss.str();
|
|
}
|
|
default:
|
|
return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);
|
|
}
|
|
}
|
|
|
|
//
|
|
// debugging
|
|
//
|
|
|
|
static void print_tensor_shape(ggml_tensor * t) {
|
|
printf("%s.shape = [", t->name);
|
|
for (int i = 0; i < ggml_n_dims(t); ++i) {
|
|
printf("%" PRId64, t->ne[i]);
|
|
if (i < ggml_n_dims(t) - 1) {
|
|
printf(", ");
|
|
}
|
|
}
|
|
printf("]\n");
|
|
}
|
|
|
|
static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) {
|
|
ggml_type type = t->type;
|
|
int64_t * ne = t->ne;
|
|
size_t * nb = t->nb;
|
|
for (int64_t i3 = 0; i3 < ne[3]; i3++) {
|
|
printf("%s.data: [\n", t->name);
|
|
for (int64_t i2 = 0; i2 < ne[2]; i2++) {
|
|
if (i2 == n && ne[2] > 2*n) {
|
|
printf(" ..., \n");
|
|
i2 = ne[2] - n;
|
|
}
|
|
printf(" [\n");
|
|
for (int64_t i1 = 0; i1 < ne[1]; i1++) {
|
|
if (i1 == n && ne[1] > 2*n) {
|
|
printf(" ..., \n");
|
|
i1 = ne[1] - n;
|
|
}
|
|
printf(" [");
|
|
for (int64_t i0 = 0; i0 < ne[0]; i0++) {
|
|
if (i0 == n && ne[0] > 2*n) {
|
|
printf("..., ");
|
|
i0 = ne[0] - n;
|
|
}
|
|
size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0];
|
|
float v;
|
|
if (type == GGML_TYPE_F16) {
|
|
v = ggml_fp16_to_fp32(*(ggml_fp16_t *) &data[i]);
|
|
} else if (type == GGML_TYPE_F32) {
|
|
v = *(float *) &data[i];
|
|
} else if (type == GGML_TYPE_I32) {
|
|
v = (float) *(int32_t *) &data[i];
|
|
} else if (type == GGML_TYPE_I16) {
|
|
v = (float) *(int16_t *) &data[i];
|
|
} else if (type == GGML_TYPE_I8) {
|
|
v = (float) *(int8_t *) &data[i];
|
|
} else {
|
|
GGML_ABORT("fatal error");
|
|
}
|
|
printf("%8.4f", v);
|
|
if (i0 < ne[0] - 1) printf(", ");
|
|
}
|
|
printf("],\n");
|
|
}
|
|
printf(" ],\n");
|
|
}
|
|
printf(" ]\n");
|
|
}
|
|
}
|
|
|