model: add Muse Spark Open (text, vision and audio) (#30616)
* model: add Muse Spark Open (text, vision and audio) Muse Spark Open is a latent space MoE: the residual stream stays at 8192 while the 256 experts live in 4096, with a projection down and up around them, top-8 sigmoid routing with a correction bias. Attention interleaves three sliding layers at a 2048 window with rope for one global layer without rope, with a sigmoid output gate and a weightless QK-norm. Residuals are norm preserving, their gate is folded at conversion time. Vision reuses the Muse Glimmer tower. Audio is a causal convolutional pyramid with no attention and no spectrogram, four stages folding 16, 6, 5 and 4 samples into the channel dim, one embedding per 80 ms of 24 kHz. The quantizer keeps the latent projections and the attention gate at Q8_0, a little over 1% of the weights, since the whole residual stream passes through them and the low-bit types turned the output into a repeating loop. The model saver writes the post norm epsilon, and the arch is registered in test-llama-archs. Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pedro Cuenca <pedro@huggingface.co> * Muse Spark: drop the redundant forward expand * Muse Spark: drop the redundant forward expand and tighten the comments * Update src/llama-quant.cpp Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * Update src/llama-graph.cpp Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * Update conversion/muse_spark.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * Update conversion/muse_spark.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * Update conversion/muse_spark.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * Update conversion/muse_spark.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pedro Cuenca <pedro@huggingface.co> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
36 files changed
+1290
-9
No files matched your search
@@ -31,9 +31,11 @@ enum class tensor_category {
|
||||
ATTENTION_QKV,
|
||||
ATTENTION_KV_B,
|
||||
ATTENTION_OUTPUT,
|
||||
ATTENTION_GATE,
|
||||
FFN_UP,
|
||||
FFN_GATE,
|
||||
FFN_DOWN,
|
||||
FFN_LATENT,
|
||||
OUTPUT,
|
||||
OTHER
|
||||
};
|
||||
@@ -147,6 +149,12 @@ static tensor_category tensor_get_category(const std::string & tensor_name) {
|
||||
if (tensor_name.find("ffn_gate") != std::string::npos) {
|
||||
return tensor_category::FFN_GATE;
|
||||
}
|
||||
if (tensor_name.find("ffn_latent") != std::string::npos) {
|
||||
return tensor_category::FFN_LATENT;
|
||||
}
|
||||
if (tensor_name.find("attn_gate.weight") != std::string::npos) {
|
||||
return tensor_category::ATTENTION_GATE;
|
||||
}
|
||||
if (tensor_name.find("ffn_down") != std::string::npos) {
|
||||
return tensor_category::FFN_DOWN;
|
||||
}
|
||||
@@ -490,6 +498,19 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type
|
||||
}
|
||||
}
|
||||
|
||||
// bumping to q8_0 is preferred since these tensors are not really big
|
||||
// TODO: nemotron 3 super shares ffn_latent and afmoe, bailingmoe3, dots3note, hy-v4, kimi-k3 and laguna share attn_gate.
|
||||
// Widening the guard is likely a win there too, but it changes their files, so it wants a perplexity comparison per architecture first.
|
||||
if (arch == LLM_ARCH_MUSE_SPARK &&
|
||||
(category == tensor_category::FFN_LATENT || category == tensor_category::ATTENTION_GATE)) {
|
||||
auto bits_per_weight = [](ggml_type t) {
|
||||
return 8.0 * ggml_type_size(t) / ggml_blck_size(t);
|
||||
};
|
||||
if (ggml_is_quantized(new_type) && bits_per_weight(new_type) < bits_per_weight(GGML_TYPE_Q8_0)) {
|
||||
new_type = GGML_TYPE_Q8_0;
|
||||
}
|
||||
}
|
||||
|
||||
// for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings
|
||||
// with the quantization of the output tensor
|
||||
if (category == tensor_category::OUTPUT || (qs.has_tied_embeddings && category == tensor_category::TOKEN_EMBD)) {
|
||||
|
||||
Reference in new issue
Block a user