set bonsai model ctx to 64k

2026-04-18 15:48:30 +02:00
parent d701485823
commit 480fb1c6d6
1 changed files with 3 additions and 0 deletions
@@ -5,6 +5,7 @@ logToStdout: "both" # proxy and upstream
 macros:
  base_args: "--no-warmup --port ${PORT}"
  common_args: "--fit-target 1536 --no-warmup --port ${PORT}"
+  ctx_64k: "--ctx-size 65536"
  ctx_128k: "--ctx-size 131072"
  ctx_256k: "--ctx-size 262144"
  gemma_sampling: "--prio 2 --temp 1.0 --repeat-penalty 1.0 --min-p 0.00 --top-k 64 --top-p 0.95"
@@ -283,10 +284,12 @@ models:
        ${ctx_128k}
        ${gemma4_sampling}
        ${common_args}
+
  "prism-ml/Bonsai-8B-gguf":
    cmd: |
      /app/llama-server
        -hf prism-ml/Bonsai-8B-gguf
+        ${ctx_64k}
        --temp 0.5 --top-p 0.9 --top-k 20 --min-p 0.00
        --repeat-penalty 1.0 -ctk q8_0 -ctv q8_0
        ${common_args}