# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json
healthCheckTimeout: 600
logToStdout: "both" # proxy and upstream

macros:
  base_args: "--no-warmup --port ${PORT} --mlock --no-mmap"
  common_args: "--fit-target 256 --no-warmup --port ${PORT} --no-mmap -tb 12 -t 6"
  cpu_args: "--no-warmup --port ${PORT} -ngl 0"
  ctx_64k: "--ctx-size 65536"
  ctx_128k: "--ctx-size 131072"
  ctx_256k: "--ctx-size 131072"
  qwen35_think_args:   "--temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0 --presence_penalty 1.5 --reasoning on"
  qwen35_nothink_args: "--temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0 --presence_penalty 1.5 --reasoning off"
  qwen35_35b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf"
  qwen35_4b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-4B-GGUF_mmproj-F16.gguf"
  gemma4_sampling: "--temp 1.0 --top-p 0.95 --top-k 64 -ctk q4_0 -ctv q4_0"

hooks:
  on_startup:
    preload:
      - "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
      - "whisper-small"

# matrix replaces groups (they are mutually exclusive).
# The small 0.8B model runs alongside any LLM.
# FLUX runs alone — it needs all available VRAM and will evict the 0.8B first.
matrix:
  vars:
    q8: "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
    stt: "whisper-small"
    flux: "flux2-klein-4b:Q4_K_M"
    coder: "Qwen3-Coder-Next-GGUF:Q4_K_M"
    q35t: "Qwen3.5-35B-A3B-GGUF:Q4_K_M"
    q35nt: "Qwen3.5-35B-A3B-GGUF-nothink:Q4_K_M"
    q35ht: "Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M"
    q35hnt: "Qwen3.5-35B-A3B-heretic-GGUF-nothink:Q4_K_M"
    q4t: "Qwen3.5-4B-GGUF:Q4_K_M"
    q4nt: "Qwen3.5-4B-GGUF-nothink:Q4_K_M"
    q4ht: "Qwen3.5-4B-heretic-GGUF:Q4_K_M"
    q4hnt: "Qwen3.5-4B-heretic-GGUF-nothink:Q4_K_M"
    g26xl: "gemma-4-26B-A4B-it:UD-Q4_K_XL"
    g26q2: "gemma-4-26B-A4B-it:UD-Q2_K_XL"
    ge4xl: "unsloth/gemma-4-E4B-it-GGUF:UD-Q4_K_XL"
    ge2xl: "unsloth/gemma-4-E2B-it-GGUF:UD-Q4_K_XL"
    q36t: "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL"
    q36nt: "unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL"
    haut: "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M"
    haunt: "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-nothink:Q4_K_M"
    mtpt: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M"
    mtpnt: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF-nothink:Q4_K_M"

  evict_costs:
    flux: 10  # large files, slow to reload

  sets:
    # any LLM can run alongside the small always-on model + STT + TTS (all CPU, no VRAM cost)
    with_q8: "(coder | q35t | q35nt | q35ht | q35hnt | q4t | q4nt | q4ht | q4hnt | g26xl | g26q2 | ge4xl | ge2xl | q36t | q36nt | haut | haunt | mtpt | mtpnt) & q8 & stt"
    # FLUX runs alone — evicts everything including q8, but keeps STT for voice during image gen
    image_gen: "flux & stt"

models:
  "Qwen3-Coder-Next-GGUF:Q4_K_M":
    cmd: |
      llama-server
        -hf unsloth/Qwen3-Coder-Next-GGUF:Q4_K_M
        --ctx-size 65536
        --predict 8192
        --temp 1.0
        --min-p 0.01
        --top-p 0.95
        --top-k 40
        --repeat-penalty 1.0
        -ctk q4_0 -ctv q4_0
        ${common_args}

  "Qwen3.5-35B-A3B-GGUF:Q4_K_M":
    cmd: |
      llama-server
        -hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
        ${ctx_256k}
        ${qwen35_think_args}
        ${common_args}

  "Qwen3.5-35B-A3B-GGUF-nothink:Q4_K_M":
    cmd: |
      llama-server
        -hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
        ${ctx_256k}
        ${qwen35_nothink_args}
        ${common_args}

  # The "heretic" version does not provide the mmproj
  # so providing url to the one from the non-heretic version.
  "Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M":
    cmd: |
      llama-server
        -hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
        ${qwen35_35b_heretic_mmproj}
        ${ctx_256k}
        ${qwen35_think_args}
        ${common_args}

  "Qwen3.5-35B-A3B-heretic-GGUF-nothink:Q4_K_M":
    cmd: |
      llama-server
        -hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
        ${qwen35_35b_heretic_mmproj}
        ${ctx_256k}
        ${qwen35_nothink_args}
        ${common_args}

  "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL":
    cmd: |
      llama-server
        -hf unsloth/Qwen3.5-0.8B-GGUF:Q4_K_XL
        --ctx-size 4096
        ${qwen35_nothink_args}
        ${base_args}

  "Qwen3.5-4B-GGUF:Q4_K_M":
    cmd: |
      llama-server
        -hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
        ${ctx_128k}
        ${qwen35_think_args}
        ${common_args}

  "Qwen3.5-4B-GGUF-nothink:Q4_K_M":
    cmd: |
      llama-server
        -hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
        ${ctx_128k}
        ${qwen35_nothink_args}
        ${common_args}

  "Qwen3.5-4B-heretic-GGUF:Q4_K_M":
    cmd: |
      llama-server
        -hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
        ${qwen35_4b_heretic_mmproj}
        ${ctx_128k}
        ${qwen35_think_args}
        ${common_args}

  "Qwen3.5-4B-heretic-GGUF-nothink:Q4_K_M":
    cmd: |
      llama-server
        -hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
        ${qwen35_4b_heretic_mmproj}
        ${ctx_128k}
        ${qwen35_nothink_args}
        ${common_args}

  "gemma-4-26B-A4B-it:UD-Q4_K_XL":
    cmd: |
      llama-server
        -hf unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q4_K_XL \
        ${ctx_256k}
        ${gemma4_sampling}
        ${common_args}

  "gemma-4-26B-A4B-it:UD-Q2_K_XL":
    cmd: |
      llama-server
        -hf unsloth/gemma-4-26B-A4B-it-GGUF:UD-Q2_K_XL \
        ${ctx_256k}
        ${gemma4_sampling}
        ${common_args}

  "unsloth/gemma-4-E4B-it-GGUF:UD-Q4_K_XL":
    cmd: |
      llama-server
        -hf unsloth/gemma-4-E4B-it-GGUF:UD-Q4_K_XL \
        ${ctx_128k}
        ${gemma4_sampling}
        ${common_args}

  "unsloth/gemma-4-E2B-it-GGUF:UD-Q4_K_XL":
    cmd: |
      llama-server
        -hf unsloth/gemma-4-E2B-it-GGUF:UD-Q4_K_XL \
        ${ctx_128k}
        ${gemma4_sampling}
        ${common_args}

  "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL":
    cmd: |
      llama-server
      -hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL
      ${ctx_256k}
      ${qwen35_think_args}
      ${common_args}

  "unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL":
    cmd: |
      llama-server
      -hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL
      ${ctx_256k}
      ${qwen35_nothink_args}
      ${common_args}

  "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M":
    cmd: |
      llama-server
      -hf HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M
      ${ctx_256k}
      ${qwen35_think_args}
      ${common_args}

  "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-nothink:Q4_K_M":
    cmd: |
      llama-server
      -hf HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M
      ${ctx_256k}
      ${qwen35_nothink_args}
      ${common_args}

  "unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M":
    cmd: |
      llama-server
      -hf unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M
      ${ctx_256k}
      ${qwen35_think_args}
      --spec-type draft-mtp --spec-draft-n-max 1
      --parallel 1
      ${common_args}

  "unsloth/Qwen3.6-35B-A3B-MTP-GGUF-nothink:Q4_K_M":
    cmd: |
      llama-server
      -hf unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M
      ${ctx_256k}
      ${qwen35_nothink_args}
      --spec-type draft-mtp --spec-draft-n-max 1
      --parallel 1
      ${common_args}

  # STT via whisper.cpp (Vulkan GPU on RX 580, always loaded, ~600MB VRAM)
  # Model auto-downloaded by init container, see deployment.yaml
  # Note: Vulkan whisper on AMD GPUs has known quality issues on some cards;
  # if transcriptions come out as garbage/gibberish, add --no-gpu to fall back.
  "whisper-small":
    checkEndpoint: none
    cmd: |
      whisper-server
        --port ${PORT}
        -m /root/.cache/whisper/ggml-small.bin
        --request-path /v1/audio
        --inference-path /transcriptions
        --convert
        --threads 6


  # Image generation via stable-diffusion.cpp (sd-server)
  # Models must be pre-downloaded to /root/.cache/sd/
  # FLUX.2-klein-4B: fast unified text-to-image and image editing model (Apache 2.0)
  # Download: uv run --with huggingface_hub hf download unsloth/FLUX.2-klein-4B-GGUF flux-2-klein-4b-Q4_K_M.gguf --local-dir /root/.cache/sd
  # Download VAE: uv run --with huggingface_hub hf download Comfy-Org/flux2-klein-4B split_files/vae/flux2-vae.safetensors --local-dir /root/.cache/sd/flux2-klein && cp /root/.cache/sd/flux2-klein/split_files/vae/flux2-vae.safetensors /root/.cache/sd/
  # Download LLM: uv run --with huggingface_hub hf download ponpoke/flux2-klein-4b-uncensored-text-encoder flux2-klein-4b-uncensored-q4_k_m.gguf --local-dir /root/.cache/sd
  "flux2-klein-4b:Q4_K_M":
    checkEndpoint: "/"
    cmd: |
      sd-server
        --listen-port ${PORT}
        --diffusion-model /root/.cache/sd/flux-2-klein-4b-Q4_K_M.gguf
        --vae /root/.cache/sd/flux2-vae.safetensors
        --llm /root/.cache/sd/flux2-klein-4b-uncensored-q4_k_m.gguf
        --cfg-scale 1.0
        --sampling-method euler
        --steps 4
        --diffusion-fa
        --offload-to-cpu