445 lines
15 KiB
YAML
445 lines
15 KiB
YAML
# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json
|
|
healthCheckTimeout: 600
|
|
logToStdout: "both" # proxy and upstream
|
|
|
|
macros:
|
|
base_args: "--no-warmup --port ${PORT} --mlock --no-mmap"
|
|
common_args: "--fit-target 256 --no-warmup --port ${PORT} --no-mmap -tb 12 -t 6"
|
|
cpu_args: "--no-warmup --port ${PORT} -ngl 0"
|
|
ctx_64k: "--ctx-size 65536"
|
|
ctx_128k: "--ctx-size 131072"
|
|
ctx_256k: "--ctx-size 131072"
|
|
qwen35_think_args: "--temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0 --presence_penalty 1.5 --reasoning on"
|
|
qwen35_nothink_args: "--temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0 --presence_penalty 1.5 --reasoning off"
|
|
qwen35_35b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf"
|
|
qwen35_4b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-4B-GGUF_mmproj-F16.gguf"
|
|
gemma4_sampling: "--temp 1.0 --top-p 0.95 --top-k 64 -ctk q4_0 -ctv q4_0"
|
|
gemma4_nothink_sampling: "--temp 1.0 --top-p 0.95 --top-k 64 -ctk q4_0 -ctv q4_0 --reasoning off"
|
|
|
|
hooks:
|
|
on_startup:
|
|
preload:
|
|
- "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
|
|
- "parakeet-tdt_ctc-1.1b"
|
|
|
|
# matrix replaces groups (they are mutually exclusive).
|
|
# The small 0.8B model runs alongside any LLM.
|
|
# FLUX runs alone — it needs all available VRAM and will evict the 0.8B first.
|
|
matrix:
|
|
vars:
|
|
q8: "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
|
|
stt: "parakeet-tdt_ctc-1.1b"
|
|
flux: "flux2-klein-4b:Q4_K_M"
|
|
coder: "Qwen3-Coder-Next-GGUF:Q4_K_M"
|
|
q35t: "Qwen3.5-35B-A3B-GGUF:Q4_K_M"
|
|
q35nt: "Qwen3.5-35B-A3B-GGUF-nothink:Q4_K_M"
|
|
q35ht: "Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M"
|
|
q35hnt: "Qwen3.5-35B-A3B-heretic-GGUF-nothink:Q4_K_M"
|
|
q4t: "Qwen3.5-4B-GGUF:Q4_K_M"
|
|
q4nt: "Qwen3.5-4B-GGUF-nothink:Q4_K_M"
|
|
q4ht: "Qwen3.5-4B-heretic-GGUF:Q4_K_M"
|
|
q4hnt: "Qwen3.5-4B-heretic-GGUF-nothink:Q4_K_M"
|
|
g26xl: "gemma-4-26B-A4B-it-qat:UD-Q4_K_XL"
|
|
g26xlnt: "gemma-4-26B-A4B-it-qat-nothink:UD-Q4_K_XL"
|
|
g26mtp: "gemma-4-26B-A4B-it-qat-mtp:UD-Q4_K_XL"
|
|
g26mtpnt: "gemma-4-26B-A4B-it-qat-mtp-nothink:UD-Q4_K_XL"
|
|
g26ht: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL"
|
|
g26hnt: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-nothink:UD-Q4_K_XL"
|
|
g26hmtp: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp:UD-Q4_K_XL"
|
|
g26hmnt: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp-nothink:UD-Q4_K_XL"
|
|
ge4qat: "unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL"
|
|
ge4qatnt: "unsloth/gemma-4-E4B-it-qat-GGUF-nothink:UD-Q4_K_XL"
|
|
ge2qat: "unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL"
|
|
ge2qatnt: "unsloth/gemma-4-E2B-it-qat-GGUF-nothink:UD-Q4_K_XL"
|
|
ge4mtp: "unsloth/gemma-4-E4B-it-qat-GGUF-mtp:UD-Q4_K_XL"
|
|
ge4mtpnt: "unsloth/gemma-4-E4B-it-qat-GGUF-mtp-nothink:UD-Q4_K_XL"
|
|
ge4ht: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M"
|
|
ge4hnt: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-nothink:Q4_K_M"
|
|
ge4hmtp: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp:Q4_K_M"
|
|
ge4hmnt: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp-nothink:Q4_K_M"
|
|
q36t: "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL"
|
|
q36nt: "unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL"
|
|
haut: "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M"
|
|
haunt: "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-nothink:Q4_K_M"
|
|
mtpt: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M"
|
|
mtpnt: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF-nothink:Q4_K_M"
|
|
|
|
evict_costs:
|
|
flux: 10 # large files, slow to reload
|
|
|
|
sets:
|
|
# any LLM can run alongside the small always-on model + STT + TTS (all CPU, no VRAM cost)
|
|
with_q8: "(coder | q35t | q35nt | q35ht | q35hnt | q4t | q4nt | q4ht | q4hnt | g26xl | g26xlnt | g26mtp | g26mtpnt | g26ht | g26hnt | g26hmtp | g26hmnt | ge4qat | ge4qatnt | ge2qat | ge2qatnt | ge4mtp | ge4mtpnt | ge4ht | ge4hnt | ge4hmtp | ge4hmnt | q36t | q36nt | haut | haunt | mtpt | mtpnt) & q8 & stt"
|
|
# FLUX runs alone — evicts everything including q8, but keeps STT for voice during image gen
|
|
image_gen: "flux & stt"
|
|
|
|
|
|
models:
|
|
"Qwen3-Coder-Next-GGUF:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3-Coder-Next-GGUF:Q4_K_M
|
|
--ctx-size 65536
|
|
--predict 8192
|
|
--temp 1.0
|
|
--min-p 0.01
|
|
--top-p 0.95
|
|
--top-k 40
|
|
--repeat-penalty 1.0
|
|
-ctk q4_0 -ctv q4_0
|
|
${common_args}
|
|
|
|
"Qwen3.5-35B-A3B-GGUF:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
|
|
${ctx_256k}
|
|
${qwen35_think_args}
|
|
${common_args}
|
|
|
|
"Qwen3.5-35B-A3B-GGUF-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
|
|
${ctx_256k}
|
|
${qwen35_nothink_args}
|
|
${common_args}
|
|
|
|
# The "heretic" version does not provide the mmproj
|
|
# so providing url to the one from the non-heretic version.
|
|
"Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
|
|
${qwen35_35b_heretic_mmproj}
|
|
${ctx_256k}
|
|
${qwen35_think_args}
|
|
${common_args}
|
|
|
|
"Qwen3.5-35B-A3B-heretic-GGUF-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
|
|
${qwen35_35b_heretic_mmproj}
|
|
${ctx_256k}
|
|
${qwen35_nothink_args}
|
|
${common_args}
|
|
|
|
"Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.5-0.8B-GGUF:Q4_K_XL
|
|
--ctx-size 4096
|
|
${qwen35_nothink_args}
|
|
${base_args}
|
|
|
|
"Qwen3.5-4B-GGUF:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
|
|
${ctx_128k}
|
|
${qwen35_think_args}
|
|
${common_args}
|
|
|
|
"Qwen3.5-4B-GGUF-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
|
|
${ctx_128k}
|
|
${qwen35_nothink_args}
|
|
${common_args}
|
|
|
|
"Qwen3.5-4B-heretic-GGUF:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
|
|
${qwen35_4b_heretic_mmproj}
|
|
${ctx_128k}
|
|
${qwen35_think_args}
|
|
${common_args}
|
|
|
|
"Qwen3.5-4B-heretic-GGUF-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
|
|
${qwen35_4b_heretic_mmproj}
|
|
${ctx_128k}
|
|
${qwen35_nothink_args}
|
|
${common_args}
|
|
|
|
"gemma-4-26B-A4B-it-qat:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
${ctx_256k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"gemma-4-26B-A4B-it-qat-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
${ctx_256k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"gemma-4-26B-A4B-it-qat-mtp:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_256k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"gemma-4-26B-A4B-it-qat-mtp-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_256k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
|
|
${ctx_256k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
|
|
${ctx_256k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
# The heretic QAT repo does not ship an MTP drafter,
|
|
# so borrow the one from the non-heretic unsloth QAT repo.
|
|
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
|
|
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_256k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
|
|
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_256k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
${ctx_128k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"unsloth/gemma-4-E4B-it-qat-GGUF-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
${ctx_128k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL \
|
|
${ctx_128k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"unsloth/gemma-4-E2B-it-qat-GGUF-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL \
|
|
${ctx_128k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"unsloth/gemma-4-E4B-it-qat-GGUF-mtp:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_128k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"unsloth/gemma-4-E4B-it-qat-GGUF-mtp-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
|
|
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_128k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
|
|
${ctx_128k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
|
|
${ctx_128k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
|
|
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_128k}
|
|
${gemma4_sampling}
|
|
${common_args}
|
|
|
|
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
|
|
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
|
|
--spec-type draft-mtp
|
|
--spec-draft-n-max 1
|
|
--swa-full
|
|
--kv-unified
|
|
--parallel 1
|
|
${ctx_128k}
|
|
${gemma4_nothink_sampling}
|
|
${common_args}
|
|
|
|
"unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL
|
|
${ctx_256k}
|
|
${qwen35_think_args}
|
|
${common_args}
|
|
|
|
"unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL
|
|
${ctx_256k}
|
|
${qwen35_nothink_args}
|
|
${common_args}
|
|
|
|
"HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M
|
|
${ctx_256k}
|
|
${qwen35_think_args}
|
|
${common_args}
|
|
|
|
"HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M
|
|
${ctx_256k}
|
|
${qwen35_nothink_args}
|
|
${common_args}
|
|
|
|
"unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M
|
|
${ctx_256k}
|
|
${qwen35_think_args}
|
|
--spec-type draft-mtp --spec-draft-n-max 1
|
|
--parallel 1
|
|
${common_args}
|
|
|
|
"unsloth/Qwen3.6-35B-A3B-MTP-GGUF-nothink:Q4_K_M":
|
|
cmd: |
|
|
llama-server
|
|
-hf unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M
|
|
${ctx_256k}
|
|
${qwen35_nothink_args}
|
|
--spec-type draft-mtp --spec-draft-n-max 1
|
|
--parallel 1
|
|
${common_args}
|
|
|
|
# STT via parakeet-server (parakeet.cpp OpenAI-compatible server, CPU, always loaded)
|
|
# Model downloaded on first start and cached under /root/.cache/parakeet.cpp/models
|
|
# parakeet-proxy.py sits in front to convert any audio format to WAV via ffmpeg,
|
|
# since parakeet-server only accepts real WAV but browsers send Ogg/Opus.
|
|
"parakeet-tdt_ctc-1.1b":
|
|
checkEndpoint: none
|
|
cmd: |
|
|
env PROXY_PORT=${PORT} FFMPEG_BIN=/root/.cache/ffmpeg/ffmpeg python3 /config/parakeet-proxy.py
|
|
|
|
|
|
# Image generation via stable-diffusion.cpp (sd-server)
|
|
# Models must be pre-downloaded to /root/.cache/sd/
|
|
# FLUX.2-klein-4B: fast unified text-to-image and image editing model (Apache 2.0)
|
|
# Download: uv run --with huggingface_hub hf download unsloth/FLUX.2-klein-4B-GGUF flux-2-klein-4b-Q4_K_M.gguf --local-dir /root/.cache/sd
|
|
# Download VAE: uv run --with huggingface_hub hf download Comfy-Org/flux2-klein-4B split_files/vae/flux2-vae.safetensors --local-dir /root/.cache/sd/flux2-klein && cp /root/.cache/sd/flux2-klein/split_files/vae/flux2-vae.safetensors /root/.cache/sd/
|
|
# Download LLM: uv run --with huggingface_hub hf download ponpoke/flux2-klein-4b-uncensored-text-encoder flux2-klein-4b-uncensored-q4_k_m.gguf --local-dir /root/.cache/sd
|
|
"flux2-klein-4b:Q4_K_M":
|
|
checkEndpoint: "/"
|
|
cmd: |
|
|
sd-server
|
|
--listen-port ${PORT}
|
|
--diffusion-model /root/.cache/sd/flux-2-klein-4b-Q4_K_M.gguf
|
|
--vae /root/.cache/sd/flux2-vae.safetensors
|
|
--llm /root/.cache/sd/flux2-klein-4b-uncensored-q4_k_m.gguf
|
|
--cfg-scale 1.0
|
|
--sampling-method euler
|
|
--steps 4
|
|
--diffusion-fa
|
|
--offload-to-cpu
|