Files
llama.cpp/tools/mtmd/clip.h
T
Xuan-Son Nguyen a2654ec431 mtmd: move log-mel computation to cgraph (#31808)
* mtmd: move log-mel computation to cgraph

* rm stray file

* also include source in mtmd_audio_track

* rm stray file

* rm mtmd_audio_stft_op

* add test
2026-11-07 23:53:11 +01:00

201 lines
7.6 KiB
C++

#pragma once
#include "ggml.h"
#include "mtmd.h"
#include <stddef.h>
#include <stdint.h>
#include <map>
// !!! Internal header, to be used by mtmd only !!!
#define MTMD_INTERNAL_HEADER
struct clip_ctx;
struct clip_enc_state;
struct clip_gen_state;
struct clip_image_size {
int width;
int height;
bool operator==(const clip_image_size & other) const {
return width == other.width && height == other.height;
}
bool operator!=(const clip_image_size & other) const {
return !(*this == other);
}
int area() const {
// avoid overflow when computing area
GGML_ASSERT(width >= 0 && width <= 46000);
GGML_ASSERT(height >= 0 && height <= 46000);
return width * height;
}
};
struct clip_image_f32;
struct clip_image_f32_batch;
enum clip_modality {
CLIP_MODALITY_VISION,
CLIP_MODALITY_AUDIO,
CLIP_MODALITY_GEN_AUDIO,
};
enum clip_flash_attn_type {
CLIP_FLASH_ATTN_TYPE_AUTO = -1,
CLIP_FLASH_ATTN_TYPE_DISABLED = 0,
CLIP_FLASH_ATTN_TYPE_ENABLED = 1,
};
struct clip_context_params {
bool use_gpu;
ggml_backend_dev_t device;
enum clip_flash_attn_type flash_attn_type;
int image_min_tokens;
int image_max_tokens;
bool warmup;
ggml_backend_sched_eval_callback cb_eval;
void * cb_eval_user_data;
bool no_alloc;
mtmd_progress_callback progress_callback;
void * progress_callback_user_data;
int n_threads;
};
struct clip_init_result {
struct clip_ctx * ctx_v; // vision context
struct clip_ctx * ctx_a; // audio context
struct clip_ctx * ctx_gen_a; // audio generation context
};
struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params);
// max number of output tokens per image, -1 if not dynamic size
int clip_get_image_max_tokens(const struct clip_ctx * ctx);
void clip_free(struct clip_ctx * ctx);
// TODO: should be enum, not string
const char * clip_patch_merge_type(const struct clip_ctx * ctx);
int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img);
// for M-RoPE, this will be the number of token positions in X and Y directions
// for other models, X will be the total number of tokens and Y will be 1
int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img);
int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img);
// this should be equal to the embedding dimension of the text model
int clip_n_mmproj_embd(const struct clip_ctx * ctx);
ggml_backend_sched_t clip_get_sched(const struct clip_ctx * ctx);
// compute the log-mel of n_frames frames, the waveform must already be padded and window must hold one whole frame
// out has the layout [n_frames, n_mel]; like the encoders, must not run at the same time as another pass on config.sched
struct mtmd_audio_mel_config;
bool clip_audio_log_mel(const mtmd_audio_mel_config & config, const float * samples, const float * window, int64_t n_frames, float * out);
// TODO: remove clip_image_encode() and always use batched version
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec);
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
enum clip_gen_process_type {
CLIP_GEN_PROCESS_GEN_UNKNOWN,
CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes
CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio
CLIP_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio
CLIP_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input
CLIP_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens
};
// gen-side voice reference (chatterbox): speech tokens, mel-rate features and the speaker vector
struct clip_gen_ref {
std::vector<int32_t> tokens;
std::vector<float> feat;
std::vector<float> spk;
};
struct clip_encode_params {
int n_threads = 1;
const clip_image_f32_batch * imgs = nullptr;
std::vector<float> * out_embd = nullptr;
// for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV)
clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN;
// GEN_CODE: out_embd receives the embd to feed back to the backbone
int32_t code0 = 0; // semantic code sampled by the backbone
int32_t top_k = 50;
float top_p = 1.0f;
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
bool * out_is_eos = nullptr;
// TOKENIZE: speech embedding rows of the produced codes
std::vector<float> * out_code_embd = nullptr;
// GEN_WAV
const std::vector<int32_t> * codes = nullptr; // this frame's 16 RVQ codes
const std::vector<float> * feats = nullptr; // continuous counterpart of codes
std::vector<float> * out_audio = nullptr; // decoded PCM samples, F32
const clip_gen_ref * ref = nullptr; // TTS reference conditioning of the voice
// TTS_VOCODE internal stage inputs
const std::vector<float> * mel_in = nullptr;
const std::vector<float> * sstft_in = nullptr;
int vocode_n_mel = 0;
int vocode_n_stft = 0;
std::vector<float> * out_spec = nullptr;
// audio gen: state of one sequence, clip_encode() updates it in place
struct clip_gen_state * gen_state = nullptr;
// streaming audio input, updated in place by clip_encode()
struct clip_enc_state * enc_state = nullptr;
};
bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params);
// persistent state of a causal audio encoder, for streaming input (one per stream)
// it holds the sliding window of past keys/values, so each clip_encode() call only takes the new frames
// n_frames_max is the maximum number of encoder frames per call
struct clip_enc_state * clip_enc_state_init(struct clip_ctx * ctx, int32_t n_frames_max);
void clip_enc_state_free(struct clip_enc_state * state);
// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls
// seed UINT32_MAX means random; reset() clears the slots and keeps the RNG stream
struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed);
void clip_gen_state_free(struct clip_gen_state * state);
void clip_gen_state_reset(struct clip_gen_state * state);
// read a chatterbox tensor by name as F32, returns the element count (0 if not found), out may be null to query the size
size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * out, size_t n_max);
// a predefined voice: backbone rows, plus the gen-side reference when the gen stack needs one
struct clip_gen_voice {
std::vector<float> rows;
clip_gen_ref ref;
};
const std::map<std::string, clip_gen_voice> & clip_gen_get_voices(const struct clip_ctx * ctx);
bool clip_is_llava(const struct clip_ctx * ctx);
// note for contributor: this clip_is_(model) pattern is deprecated
// do NOT add new functions like this
bool clip_has_vision_encoder(const struct clip_ctx * ctx);
bool clip_has_audio_encoder(const struct clip_ctx * ctx);
bool clip_support_batch(const struct clip_ctx * ctx);
int clip_model_n_temporal_merge(const struct clip_ctx * ctx); // TODO @ngxson : remove, refactor this
std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
struct clip_cap {
bool has_vision;
bool has_audio;
bool has_gen_audio;
};
struct clip_cap clip_get_cap(const char * fname);