5 Commits

3 changed files with 25 additions and 8 deletions

View File

@@ -5,15 +5,15 @@ logToStdout: "both" # proxy and upstream
hooks:
on_startup:
preload:
- "Qwen3-VL-2B-Instruct-GGUF:Q4_K_M"
- "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
groups:
qwen-vl-always:
always:
persistent: true
exclusive: false
swap: false
members:
- "Qwen3-VL-2B-Instruct-GGUF:Q4_K_M"
- "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
models:
"DeepSeek-R1-0528-Qwen3-8B-GGUF":
@@ -505,6 +505,7 @@ models:
/app/llama-server
-hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf
--mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf
--ctx-size 16384
--temp 1.0
--min-p 0.00
@@ -519,6 +520,7 @@ models:
/app/llama-server
-hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf
--mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf
--ctx-size 16384
--temp 1.0
--min-p 0.00
@@ -529,12 +531,13 @@ models:
--chat-template-kwargs "{\"enable_thinking\": false}"
"Qwen3-VL-2B-Instruct-GGUF:Q4_K_M":
ttl: 0
ttl: 600
cmd: |
/app/llama-server
-hf unsloth/Qwen3-VL-2B-Instruct-GGUF:Q4_K_M
--ctx-size 16384
--predict 4096
--ctx-size 6144
--cache-type-k q8_0
--cache-type-v q8_0
--temp 0.7
--top-p 0.8
--top-k 20
@@ -557,3 +560,17 @@ models:
--repeat-penalty 1.0
--no-warmup
--port ${PORT}
"Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL":
ttl: 0
cmd: |
/app/llama-server
-hf unsloth/Qwen3.5-0.8B-GGUF:Q4_K_XL
--ctx-size 16384
--temp 0.6
--top-p 0.95
--top-k 20
--min-p 0.00
--no-warmup
--port ${PORT}
--chat-template-kwargs "{\"enable_thinking\": false}"

View File

@@ -17,7 +17,7 @@ spec:
containers:
- name: llama-swap
# TODO: make renovate update the image tag
image: ghcr.io/mostlygeek/llama-swap:v195-vulkan-b8148
image: ghcr.io/mostlygeek/llama-swap:v197-vulkan-b8202
imagePullPolicy: IfNotPresent
command:
- /app/llama-swap

View File

@@ -15,7 +15,7 @@ spec:
- name: renovate
# Update this to the latest available and then enable Renovate on
# the manifest
image: renovate/renovate:43.55.1-full
image: renovate/renovate:43.56.1-full
envFrom:
- secretRef:
name: renovate-gitea-token