Lumpiasty/klaster
0
0
Fork 0
Code Issues Pull Requests 4 Packages Projects Releases Wiki Activity

add qwen3-vl, fix librechat taking over settings and clean up llama config

Browse Source
This commit is contained in:
Lumpiasty
2025-11-15 19:09:13 +01:00
parent e3325670de
commit 0b677d0faf
2 changed files with 234 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
apps/librechat/release.yaml
View File

@@ -57,7 +57,9 @@ spec:
"Qwen3-4B-Thinking-2507-long-ctx", "Qwen3-4B-Thinking-2507-long-ctx",
"Qwen2.5-VL-7B-Instruct-GGUF", "Qwen2.5-VL-7B-Instruct-GGUF",
"Qwen2.5-VL-32B-Instruct-GGUF-IQ1_S", "Qwen2.5-VL-32B-Instruct-GGUF-IQ1_S",
"Qwen2.5-VL-32B-Instruct-GGUF-Q2_K_L" "Qwen2.5-VL-32B-Instruct-GGUF-Q2_K_L",
"Qwen3-VL-4B-Instruct-GGUF",
"Qwen3-VL-4B-Instruct-GGUF-unslothish"
] ]
titleConvo: true titleConvo: true
titleModel: "gemma3-4b-novision" titleModel: "gemma3-4b-novision"
@@ -65,6 +67,16 @@ spec:
summaryModel: "gemma3-4b-novision" summaryModel: "gemma3-4b-novision"
forcePrompt: false forcePrompt: false
modelDisplayLabel: "Llama.cpp" modelDisplayLabel: "Llama.cpp"
# ✨ IMPORTANT: let llama-swap/llama-server own all these
dropParams:
- "temperature"
- "top_p"
- "top_k"
- "presence_penalty"
- "frequency_penalty"
- "stop"
- "max_tokens"
imageVolume: imageVolume:
enabled: true enabled: true
size: 10G size: 10G

370
apps/llama/configs/config.yaml
View File

@@ -5,212 +5,284 @@ models:
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF:Q4_K_M -hf unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF:Q4_K_M
-ngl 37 -c 16384 --n-gpu-layers 37
--no-warmup --ctx-size 16384
--port ${PORT} --no-warmup
--port ${PORT}
"Qwen3-8B-GGUF": "Qwen3-8B-GGUF":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-8B-GGUF:Q4_K_M -hf unsloth/Qwen3-8B-GGUF:Q4_K_M
-ngl 37 -c 16384 --n-gpu-layers 37
--no-warmup --ctx-size 16384
--port ${PORT} --no-warmup
--port ${PORT}
"Qwen3-8B-GGUF-no-thinking": "Qwen3-8B-GGUF-no-thinking":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-8B-GGUF:Q4_K_M -hf unsloth/Qwen3-8B-GGUF:Q4_K_M
-ngl 37 -c 16384 --n-gpu-layers 37
--jinja --chat-template-file /config/qwen_nothink_chat_template.jinja --ctx-size 16384
--no-warmup --jinja
--port ${PORT} --chat-template-file /config/qwen_nothink_chat_template.jinja
--no-warmup
--port ${PORT}
"gemma3n-e4b": "gemma3n-e4b":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3n-E4B-it-GGUF:UD-Q4_K_XL -hf unsloth/gemma-3n-E4B-it-GGUF:UD-Q4_K_XL
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99 --n-gpu-layers 99
--seed 3407 --seed 3407
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
--min-p 0.00 --min-p 0.00
--top-k 64 --top-k 64
--top-p 0.95 --top-p 0.95
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-12b": "gemma3-12b":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99 --n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
--min-p 0.00 --min-p 0.00
--top-k 64 --top-k 64
--top-p 0.95 --top-p 0.95
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-12b-novision": "gemma3-12b-novision":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99 --n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
--min-p 0.00 --min-p 0.00
--top-k 64 --top-k 64
--top-p 0.95 --top-p 0.95
--no-mmproj --no-mmproj
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-12b-q2": "gemma3-12b-q2":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q2_K_L -hf unsloth/gemma-3-12b-it-GGUF:Q2_K_L
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99 --n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
--min-p 0.00 --min-p 0.00
--top-k 64 --top-k 64
--top-p 0.95 --top-p 0.95
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-4b": "gemma3-4b":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99 --n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
--min-p 0.00 --min-p 0.00
--top-k 64 --top-k 64
--top-p 0.95 --top-p 0.95
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-4b-novision": "gemma3-4b-novision":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M
--ctx-size 16384 --ctx-size 16384
--n-gpu-layers 99 --n-gpu-layers 99
--prio 2 --prio 2
--temp 1.0 --temp 1.0
--repeat-penalty 1.0 --repeat-penalty 1.0
--min-p 0.00 --min-p 0.00
--top-k 64 --top-k 64
--top-p 0.95 --top-p 0.95
--no-mmproj --no-mmproj
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"Qwen3-4B-Thinking-2507": "Qwen3-4B-Thinking-2507":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M
-ngl 99 -c 16384 --predict 8192 --n-gpu-layers 99
--temp 0.6 --ctx-size 16384
--min-p 0.00 --predict 8192
--top-p 0.95 --temp 0.6
--top-k 20 --min-p 0.00
--repeat-penalty 1.0 --top-p 0.95
--no-warmup --top-k 20
--port ${PORT} --repeat-penalty 1.0
--no-warmup
--port ${PORT}
"Qwen3-4B-Thinking-2507-long-ctx": "Qwen3-4B-Thinking-2507-long-ctx":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Thinking-2507-GGUF:Q4_K_M
-ngl 99 -c 262144 --predict 81920 --n-gpu-layers 99
--temp 0.6 --ctx-size 262144
--min-p 0.00 --predict 81920
--top-p 0.95 --temp 0.6
--top-k 20 --min-p 0.00
--repeat-penalty 1.0 --top-p 0.95
--no-warmup --top-k 20
--flash-attn --repeat-penalty 1.0
--cache-type-k q8_0 --cache-type-v q8_0 --no-warmup
--port ${PORT} --flash-attn auto
--cache-type-k q8_0
--cache-type-v q8_0
--port ${PORT}
"Qwen3-4B-Instruct-2507": "Qwen3-4B-Instruct-2507":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M
-ngl 99 -c 16384 --predict 8192 --n-gpu-layers 99
--temp 0.7 --ctx-size 16384
--min-p 0.00 --predict 8192
--top-p 0.8 --temp 0.7
--top-k 20 --min-p 0.00
--repeat-penalty 1.0 --top-p 0.8
--no-warmup --top-k 20
--port ${PORT} --repeat-penalty 1.0
--no-warmup
--port ${PORT}
"Qwen3-4B-Instruct-2507-long-ctx": "Qwen3-4B-Instruct-2507-long-ctx":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M -hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M
-ngl 99 -c 262144 --predict 81920 --n-gpu-layers 99
--temp 0.7 --ctx-size 262144
--min-p 0.00 --predict 81920
--top-p 0.8 --temp 0.7
--top-k 20 --min-p 0.00
--repeat-penalty 1.0 --top-p 0.8
--no-warmup --top-k 20
--flash-attn --repeat-penalty 1.0
--cache-type-k q8_0 --cache-type-v q8_0 --no-warmup
--port ${PORT} --flash-attn auto
--cache-type-k q8_0
--cache-type-v q8_0
--port ${PORT}
"Qwen2.5-VL-32B-Instruct-GGUF-IQ1_S": "Qwen2.5-VL-32B-Instruct-GGUF-IQ1_S":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:IQ1_S -hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:IQ1_S
-ngl 99 -c 16384 --predict 8192 --n-gpu-layers 99
--temp 0.7 --ctx-size 16384
--min-p 0.00 --predict 8192
--top-p 0.8 --temp 0.7
--top-k 20 --min-p 0.00
--repeat-penalty 1.0 --top-p 0.8
--no-warmup --top-k 20
--port ${PORT} --repeat-penalty 1.0
--no-warmup
--port ${PORT}
"Qwen2.5-VL-32B-Instruct-GGUF-Q2_K_L": "Qwen2.5-VL-32B-Instruct-GGUF-Q2_K_L":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:Q2_K_L -hf unsloth/Qwen2.5-VL-32B-Instruct-GGUF:Q2_K_L
-ngl 99 -c 16384 --predict 8192 --n-gpu-layers 99
--temp 0.7 --ctx-size 16384
--min-p 0.00 --predict 8192
--top-p 0.8 --temp 0.7
--top-k 20 --min-p 0.00
--repeat-penalty 1.0 --top-p 0.8
--no-warmup --top-k 20
--port ${PORT} --repeat-penalty 1.0
--no-warmup
--port ${PORT}
"Qwen2.5-VL-7B-Instruct-GGUF": "Qwen2.5-VL-7B-Instruct-GGUF":
ttl: 600 ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M -hf unsloth/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M
-ngl 37 -c 16384 --predict 8192 --n-gpu-layers 37
--temp 0.7 --ctx-size 16384
--min-p 0.00 --predict 8192
--top-p 0.8 --temp 0.7
--top-k 20 --min-p 0.00
--repeat-penalty 1.0 --top-p 0.8
--no-warmup --top-k 20
--port ${PORT} --repeat-penalty 1.0
--no-warmup
--port ${PORT}
"Qwen3-VL-4B-Instruct-GGUF":
ttl: 600
cmd: |
/app/llama-server
-hf unsloth/Qwen3-VL-4B-Instruct-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 12288
--predict 4096
--flash-attn auto
--jinja
--top-p 0.95
--top-k 20
--temp 1.0
--min-p 0.05
--repeat-penalty 1.0
--presence-penalty 0.0
--no-warmup
--port ${PORT}
"Qwen3-VL-4B-Instruct-GGUF-unslothish":
ttl: 600
cmd: |
/app/llama-server
-hf unsloth/Qwen3-VL-4B-Instruct-GGUF:Q4_K_M
--n-gpu-layers 99
--ctx-size 12288
--predict 4096
--flash-attn auto
--jinja
--top-p 0.8
--top-k 20
--temp 0.7
--min-p 0.0
--presence-penalty 0.7
--no-warmup
--port ${PORT}