diff --git a/apps/llama/configs/config.yaml b/apps/llama/configs/config.yaml index d7b0861..ecc1cee 100644 --- a/apps/llama/configs/config.yaml +++ b/apps/llama/configs/config.yaml @@ -5,6 +5,7 @@ logToStdout: "both" # proxy and upstream macros: base_args: "--no-warmup --port ${PORT}" common_args: "--fit-target 1536 --no-warmup --port ${PORT}" + ctx_64k: "--ctx-size 65536" ctx_128k: "--ctx-size 131072" ctx_256k: "--ctx-size 262144" gemma_sampling: "--prio 2 --temp 1.0 --repeat-penalty 1.0 --min-p 0.00 --top-k 64 --top-p 0.95" @@ -283,10 +284,12 @@ models: ${ctx_128k} ${gemma4_sampling} ${common_args} + "prism-ml/Bonsai-8B-gguf": cmd: | /app/llama-server -hf prism-ml/Bonsai-8B-gguf + ${ctx_64k} --temp 0.5 --top-p 0.9 --top-k 20 --min-p 0.00 --repeat-penalty 1.0 -ctk q8_0 -ctv q8_0 ${common_args}