model: add Clover support (#31306)
* model : add AR-only Clover MoE support Add GGUF conversion, model loading, and causal hybrid attention for Clover, reusing shared attention, recurrent-state, MoE, and LoRA helpers. * Update tests/test-llama-archs.cpp Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * convert : filter Clover tensors using filter_tensors * clover : use the standard ssm_a tensor name --------- Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
1 parent
1b6935b227
commit
e4b6c8ca6a
11 files changed
+560
-6
No files matched your search
+8
-4
@@ -350,6 +350,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_clef(params);
|
||||
case LLM_ARCH_VLLM_DECISION2:
|
||||
return new llama_model_vllm_decision2(params);
|
||||
case LLM_ARCH_CLOVER_MOE:
|
||||
return new llama_model_clover_moe(params);
|
||||
case LLM_ARCH_QWEN4EXP:
|
||||
return new llama_model_qwen4exp(params);
|
||||
case LLM_ARCH_MISTRAL3:
|
||||
@@ -644,7 +646,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
|
||||
// if a model has fused tensors they need to be separated into "segments", see the comments on ggml_backend_meta_split_state struct
|
||||
auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> {
|
||||
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
|
||||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
|
||||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE ||
|
||||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
|
||||
|
||||
// fused full attention layers with Q gate tensors that need n_embd doubled:
|
||||
@@ -803,7 +805,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
|
||||
GGML_ASSERT(segments.size() == 1);
|
||||
// some models have Q gate tensors, for those cases the granularity needs to be doubled:
|
||||
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
|
||||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
|
||||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE ||
|
||||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
|
||||
return {std::lcm(2*n_embd_q, blck_size_perf)};
|
||||
}
|
||||
@@ -837,7 +839,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
|
||||
if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {
|
||||
// fused full attention layers need Q gate tensors handled like above:
|
||||
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
|
||||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
|
||||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE ||
|
||||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
|
||||
return {std::lcm(2*n_embd_q, blck_size_perf), granularity_kv};
|
||||
}
|
||||
@@ -2601,6 +2603,7 @@ void llama_model::print_info() const {
|
||||
arch == LLM_ARCH_QWEN3NEXT ||
|
||||
arch == LLM_ARCH_QWEN35 ||
|
||||
arch == LLM_ARCH_QWEN35MOE ||
|
||||
arch == LLM_ARCH_CLOVER_MOE ||
|
||||
arch == LLM_ARCH_NEMOTRON_H ||
|
||||
arch == LLM_ARCH_NEMOTRON_H_MOE) {
|
||||
LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);
|
||||
@@ -3190,7 +3193,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
|
||||
filter_recr = [&](uint32_t il) {
|
||||
return hparams.is_recr(il) && hparams.n_ff(il) == 0;
|
||||
};
|
||||
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) {
|
||||
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) {
|
||||
filter_attn = [&](uint32_t il) {
|
||||
return il < hparams.n_layer() && !hparams.is_recr(il);
|
||||
};
|
||||
@@ -3685,6 +3688,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_AFMOE:
|
||||
case LLM_ARCH_LAGUNA:
|
||||
case LLM_ARCH_QWEN3NEXT:
|
||||
case LLM_ARCH_CLOVER_MOE:
|
||||
case LLM_ARCH_MIMO2:
|
||||
case LLM_ARCH_STEP35:
|
||||
case LLM_ARCH_SPARK2_5:
|
||||
|
||||
Reference in new issue
Block a user