Lumpiasty/klaster
0
0
Fork 0
Code Issues Pull Requests 4 Packages Projects Releases Wiki Activity

automatically fit models by llama.cpp

Browse Source
This commit is contained in:
Lumpiasty
2026-02-26 01:37:55 +01:00
parent ec038d7154
commit 2c83eb26b3
1 changed files with 0 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
apps/llama/configs/config.yaml
View File

@@ -8,7 +8,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF:Q4_K_M -hf unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF:Q4_K_M
--n-gpu-layers 37
--ctx-size 16384 --ctx-size 16384
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
@@ -18,7 +17,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-8B-GGUF:Q4_K_M -hf unsloth/Qwen3-8B-GGUF:Q4_K_M
--n-gpu-layers 37
--ctx-size 16384 --ctx-size 16384
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
@@ -28,7 +26,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-8B-GGUF:Q4_K_M -hf unsloth/Qwen3-8B-GGUF:Q4_K_M
--n-gpu-layers 37
--ctx-size 16384 --ctx-size 16384
--jinja --jinja
--chat-template-file /config/qwen_nothink_chat_template.jinja --chat-template-file /config/qwen_nothink_chat_template.jinja
@@ -41,7 +38,6 @@ models:
/app/llama-server /app/llama-server
-hf unsloth/gemma-3n-E4B-it-GGUF:UD-Q4_K_XL -hf unsloth/gemma-3n-E4B-it-GGUF:UD-Q4_K_XL
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99
--seed 3407 --seed 3407
--prio 2 --prio 2
--temp 1.0 --temp 1.0
@@ -58,7 +54,6 @@ models:
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
@@ -74,7 +69,6 @@ models:
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
@@ -91,7 +85,6 @@ models:
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q2_K_L -hf unsloth/gemma-3-12b-it-GGUF:Q2_K_L
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
@@ -107,7 +100,6 @@ models:
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
@@ -123,7 +115,6 @@ models:
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
@@ -139,7 +130,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 16384 --ctx-size 16384
--predict 8192 --predict 8192
--temp 0.6 --temp 0.6
@@ -155,7 +145,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 262144 --ctx-size 262144
--predict 81920 --predict 81920
--temp 0.6 --temp 0.6
@@ -174,7 +163,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 16384 --ctx-size 16384
--predict 8192 --predict 8192
--temp 0.7 --temp 0.7
@@ -190,7 +178,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 262144 --ctx-size 262144
--predict 81920 --predict 81920
--temp 0.7 --temp 0.7
@@ -209,7 +196,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:IQ1_S -hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:IQ1_S
--n-gpu-layers 99
--ctx-size 16384 --ctx-size 16384
--predict 8192 --predict 8192
--temp 0.7 --temp 0.7
@@ -225,7 +211,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:Q2_K_L -hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:Q2_K_L
--n-gpu-layers 99
--ctx-size 16384 --ctx-size 16384
--predict 8192 --predict 8192
--temp 0.7 --temp 0.7
@@ -241,7 +226,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M -hf unsloth/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M
--n-gpu-layers 37
--ctx-size 16384 --ctx-size 16384
--predict 8192 --predict 8192
--temp 0.7 --temp 0.7
@@ -257,7 +241,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-2B-Instruct-GGUF:Q8_0 -hf Qwen/Qwen3-VL-2B-Instruct-GGUF:Q8_0
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -277,7 +260,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-4B-Instruct-GGUF:Q8_0 -hf Qwen/Qwen3-VL-4B-Instruct-GGUF:Q8_0
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -297,7 +279,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-8B-Instruct-GGUF:Q4_K_M -hf Qwen/Qwen3-VL-8B-Instruct-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -317,7 +298,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-2B-Instruct-GGUF:Q8_0 -hf Qwen/Qwen3-VL-2B-Instruct-GGUF:Q8_0
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -337,7 +317,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-4B-Instruct-GGUF:Q8_0 -hf Qwen/Qwen3-VL-4B-Instruct-GGUF:Q8_0
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -357,7 +336,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-8B-Instruct-GGUF:Q4_K_M -hf Qwen/Qwen3-VL-8B-Instruct-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -377,7 +355,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-2B-Thinking-GGUF:Q8_0 -hf Qwen/Qwen3-VL-2B-Thinking-GGUF:Q8_0
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -396,7 +373,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-4B-Thinking-GGUF:Q4_K_M -hf Qwen/Qwen3-VL-4B-Thinking-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -415,7 +391,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf Qwen/Qwen3-VL-8B-Thinking-GGUF:Q4_K_M -hf Qwen/Qwen3-VL-8B-Thinking-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -434,7 +409,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf noctrex/Huihui-Qwen3-VL-8B-Instruct-abliterated-GGUF:Q6_K -hf noctrex/Huihui-Qwen3-VL-8B-Instruct-abliterated-GGUF:Q6_K
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto
@@ -454,7 +428,6 @@ models:
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf noctrex/Huihui-Qwen3-VL-8B-Thinking-abliterated-GGUF:Q6_K -hf noctrex/Huihui-Qwen3-VL-8B-Thinking-abliterated-GGUF:Q6_K
--n-gpu-layers 99
--ctx-size 12288 --ctx-size 12288
--predict 4096 --predict 4096
--flash-attn auto --flash-attn auto