Lumpiasty/klaster
0
0
Fork 0
Code Issues Pull Requests 4 Packages Projects Releases Wiki Activity

update qwen3.5 4b ctx size to 128k and kv quant to q4_0

Browse Source
This commit is contained in:
Lumpiasty
2026-03-30 21:01:29 +02:00
parent 08db022d0d
commit fd6a7362a6
1 changed files with 7 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
apps/llama/configs/config.yaml
View File

@@ -6,10 +6,11 @@ macros:
base_args: "--no-warmup --port ${PORT}" base_args: "--no-warmup --port ${PORT}"
common_args: "--fit-target 1536 --no-warmup --port ${PORT}" common_args: "--fit-target 1536 --no-warmup --port ${PORT}"
gemma3_ctx_128k: "--ctx-size 131072" gemma3_ctx_128k: "--ctx-size 131072"
qwen35_ctx_128k: "--ctx-size 131072"
qwen35_ctx_256k: "--ctx-size 262144" qwen35_ctx_256k: "--ctx-size 262144"
gemma_sampling: "--prio 2 --temp 1.0 --repeat-penalty 1.0 --min-p 0.00 --top-k 64 --top-p 0.95" gemma_sampling: "--prio 2 --temp 1.0 --repeat-penalty 1.0 --min-p 0.00 --top-k 64 --top-p 0.95"
qwen35_sampling: "--temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00 -ctk q8_0 -ctv q8_0" qwen35_sampling: "--temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0"
qwen35_35b_args: "--temp 1.0 --min-p 0.00 --top-p 0.95 --top-k 20 -ctk q8_0 -ctv q8_0" qwen35_35b_args: "--temp 1.0 --min-p 0.00 --top-p 0.95 --top-k 20 -ctk q4_0 -ctv q4_0"
qwen35_35b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf" qwen35_35b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf"
qwen35_4b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-4B-GGUF_mmproj-F16.gguf" qwen35_4b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-4B-GGUF_mmproj-F16.gguf"
glm47_flash_args: "--temp 0.7 --top-p 1.0 --min-p 0.01 --repeat-penalty 1.0" glm47_flash_args: "--temp 0.7 --top-p 1.0 --min-p 0.01 --repeat-penalty 1.0"
@@ -163,7 +164,7 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M -hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
${qwen35_ctx_256k} ${qwen35_ctx_128k}
${qwen35_sampling} ${qwen35_sampling}
${common_args} ${common_args}
${thinking_on} ${thinking_on}
@@ -172,7 +173,7 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M -hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
${qwen35_ctx_256k} ${qwen35_ctx_128k}
${qwen35_sampling} ${qwen35_sampling}
${common_args} ${common_args}
${thinking_off} ${thinking_off}
@@ -182,7 +183,7 @@ models:
/app/llama-server /app/llama-server
-hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M -hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
${qwen35_4b_heretic_mmproj} ${qwen35_4b_heretic_mmproj}
${qwen35_ctx_256k} ${qwen35_ctx_128k}
${qwen35_sampling} ${qwen35_sampling}
${common_args} ${common_args}
${thinking_on} ${thinking_on}
@@ -192,7 +193,7 @@ models:
/app/llama-server /app/llama-server
-hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M -hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
${qwen35_4b_heretic_mmproj} ${qwen35_4b_heretic_mmproj}
${qwen35_ctx_256k} ${qwen35_ctx_128k}
${qwen35_sampling} ${qwen35_sampling}
${common_args} ${common_args}
${thinking_off} ${thinking_off}