Lumpiasty/klaster
0
0
Fork 0
Code Issues Pull Requests 4 Packages Projects Releases Wiki Activity

llama automatic unloading and longer start timeout

Browse Source
This commit is contained in:
Lumpiasty
2025-07-29 02:31:39 +02:00
parent 071e87ee44
commit c4628523bc
1 changed files with 11 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
apps/llama/configs/config.yaml
View File

@@ -1,5 +1,8 @@
healthCheckTimeout: 600
models: models:
"DeepSeek-R1-0528-Qwen3-8B-GGUF": "DeepSeek-R1-0528-Qwen3-8B-GGUF":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF:Q4_K_M -hf unsloth/DeepSeek-R1-0528-Qwen3-8B-GGUF:Q4_K_M
@@ -7,6 +10,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"Qwen3-8B-GGUF": "Qwen3-8B-GGUF":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-8B-GGUF:Q4_K_M -hf unsloth/Qwen3-8B-GGUF:Q4_K_M
@@ -14,6 +18,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"Qwen3-8B-GGUF-no-thinking": "Qwen3-8B-GGUF-no-thinking":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/Qwen3-8B-GGUF:Q4_K_M -hf unsloth/Qwen3-8B-GGUF:Q4_K_M
@@ -22,6 +27,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3n-e3b": "gemma3n-e3b":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3n-E4B-it-GGUF:UD-Q4_K_XL -hf unsloth/gemma-3n-E4B-it-GGUF:UD-Q4_K_XL
@@ -37,6 +43,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-12b": "gemma3-12b":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
@@ -51,6 +58,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-12b-novision": "gemma3-12b-novision":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-12b-it-GGUF:Q4_K_M
@@ -66,6 +74,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-12b-q2": "gemma3-12b-q2":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-12b-it-GGUF:Q2_K_L -hf unsloth/gemma-3-12b-it-GGUF:Q2_K_L
@@ -80,6 +89,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-4b": "gemma3-4b":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M
@@ -94,6 +104,7 @@ models:
--no-warmup --no-warmup
--port ${PORT} --port ${PORT}
"gemma3-4b-novision": "gemma3-4b-novision":
ttl: 600
cmd: | cmd: |
/app/llama-server /app/llama-server
-hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M -hf unsloth/gemma-3-4b-it-GGUF:Q4_K_M