model: support vllm decision 2 model (#30378)
* models: support pplx-decider * vllm d2 * update * clarify naming rule * stricter validation * fix ci * rm redundant filter_tensors
This commit is contained in:
1 parent
ccdbe29053
commit
097cf33d94
23 files changed
+549
-19
No files matched your search
@@ -346,6 +346,11 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
|
||||
quantize &= name.find("attn_v_b.weight") == std::string::npos;
|
||||
}
|
||||
|
||||
// the decision head of vllm-decision2 is small and computed in F32
|
||||
if (arch == LLM_ARCH_VLLM_DECISION2) {
|
||||
quantize &= name.compare(0, 9, "decision.") != 0;
|
||||
}
|
||||
|
||||
// do not quantize RWKV's small yet 2D weights
|
||||
quantize &= name.find("time_mix_first.weight") == std::string::npos;
|
||||
quantize &= name.find("time_mix_w0.weight") == std::string::npos;
|
||||
|
||||
Reference in new issue
Block a user