Support DFLASH2 for Recursive Ripple-65B-Preview (#31904)
* fixed support for DFlash2 for Ripple 65B * spec: Add NoPE handling to DFlash (cherry picked from commit 89ea8e3e77e3359d17902a2db136022d5d419d88) * spec: Fix NoPE not being picked up by DFlash (cherry picked from commit 43393b95fda2ce7e7d243e2650bf7b1cadf1c732) --------- Co-authored-by: James Pickering <james_pic@hotmail.com>
This commit is contained in:
1 parent
4e5116f476
commit
de32b268c4
6 files changed
+24
-15
No files matched your search
@@ -2923,8 +2923,11 @@ uint32_t llama_context::graph_max_nodes() const {
|
||||
res = std::max<uint32_t>(1024u, 32u * model.n_tensors());
|
||||
} else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) {
|
||||
// DFlash2's convolutions and selector are shape work rather than matmuls,
|
||||
// so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft
|
||||
res = std::max<uint32_t>(1024u, 12u*model.n_tensors());
|
||||
// so they cost ~8.6 nodes per tensor against ~5.9 for a plain DFlash draft.
|
||||
// The selector lattice scales with the draft block size; 12u was calibrated
|
||||
// on block-8 drafters, so scale the budget for larger blocks (e.g. Ripple's 16).
|
||||
const uint32_t block_scale = std::max<uint32_t>(1u, model.hparams.dflash_block_size/8u);
|
||||
res = std::max<uint32_t>(1024u, 12u*block_scale*model.n_tensors());
|
||||
} else {
|
||||
res = std::max<uint32_t>(1024u, 8u*model.n_tensors());
|
||||
for (const auto & lora : model.loras) {
|
||||
|
||||
Reference in new issue
Block a user