Lumpiasty/klaster
0
0
Fork 0
Code Issues Pull Requests 2 Packages Projects Releases Wiki Activity

set bonsai model ctx to 64k

Browse Source
This commit is contained in:
Lumpiasty
2026-04-18 15:48:30 +02:00
parent d701485823
commit 480fb1c6d6
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/llama/configs/config.yaml
+3
View File
@@ -5,6 +5,7 @@ logToStdout: "both" # proxy and upstream
macros: macros:
base_args: "--no-warmup --port ${PORT}" base_args: "--no-warmup --port ${PORT}"
common_args: "--fit-target 1536 --no-warmup --port ${PORT}" common_args: "--fit-target 1536 --no-warmup --port ${PORT}"
ctx_64k: "--ctx-size 65536"
ctx_128k: "--ctx-size 131072" ctx_128k: "--ctx-size 131072"
ctx_256k: "--ctx-size 262144" ctx_256k: "--ctx-size 262144"
gemma_sampling: "--prio 2 --temp 1.0 --repeat-penalty 1.0 --min-p 0.00 --top-k 64 --top-p 0.95" gemma_sampling: "--prio 2 --temp 1.0 --repeat-penalty 1.0 --min-p 0.00 --top-k 64 --top-p 0.95"
@@ -283,10 +284,12 @@ models:
${ctx_128k} ${ctx_128k}
${gemma4_sampling} ${gemma4_sampling}
${common_args} ${common_args}
"prism-ml/Bonsai-8B-gguf": "prism-ml/Bonsai-8B-gguf":
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf prism-ml/Bonsai-8B-gguf -hf prism-ml/Bonsai-8B-gguf
${ctx_64k}
--temp 0.5 --top-p 0.9 --top-k 20 --min-p 0.00 --temp 0.5 --top-p 0.9 --top-k 20 --min-p 0.00
--repeat-penalty 1.0 -ctk q8_0 -ctv q8_0 --repeat-penalty 1.0 -ctk q8_0 -ctv q8_0
${common_args} ${common_args}