Lumpiasty/klaster
0
0
Fork 0
Code Issues Pull Requests 3 Packages Projects Releases Wiki Activity

use supervisord instead of gitops llama.cpp
ci/woodpecker/push/flux-reconcile-source Pipeline was successful
Details

Browse Source
This commit is contained in:
Lumpiasty
2026-07-12 23:32:47 +02:00
parent 2d980550eb
commit ea888c57fb
4 changed files with 4 additions and 733 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/llama/configs/config.yaml
-453
View File
@@ -1,453 +0,0 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json
healthCheckTimeout: 600
logToStdout: "both" # proxy and upstream
macros:
base_args: "--no-warmup --port ${PORT} --mlock --no-mmap"
common_args: "--fit-target 256 --no-warmup --port ${PORT} --no-mmap -tb 12 -t 6"
cpu_args: "--no-warmup --port ${PORT} -ngl 0"
ctx_64k: "--ctx-size 65536"
ctx_128k: "--ctx-size 131072"
ctx_256k: "--ctx-size 131072"
qwen35_think_args: "--temp 1.0 --top-p 0.95 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0 --presence_penalty 1.5 --reasoning on"
qwen35_nothink_args: "--temp 0.7 --top-p 0.80 --top-k 20 --min-p 0.00 -ctk q4_0 -ctv q4_0 --presence_penalty 1.5 --reasoning off"
qwen35_35b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-35B-A3B-GGUF_mmproj-F16.gguf"
qwen35_4b_heretic_mmproj: "--mmproj-url https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/mmproj-F16.gguf --mmproj /root/.cache/llama.cpp/unsloth_Qwen3.5-4B-GGUF_mmproj-F16.gguf"
gemma4_sampling: "--temp 1.0 --top-p 0.95 --top-k 64 -ctk q4_0 -ctv q4_0"
gemma4_nothink_sampling: "--temp 1.0 --top-p 0.95 --top-k 64 -ctk q4_0 -ctv q4_0 --reasoning off"
hooks:
on_startup:
preload:
- "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
- "parakeet-tdt_ctc-1.1b"
# matrix replaces groups (they are mutually exclusive).
# The small 0.8B model runs alongside any LLM.
# FLUX runs alone — it needs all available VRAM and will evict the 0.8B first.
matrix:
vars:
q8: "Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL"
stt: "parakeet-tdt_ctc-1.1b"
flux: "flux2-klein-4b:Q4_K_M"
coder: "Qwen3-Coder-Next-GGUF:Q4_K_M"
q35t: "Qwen3.5-35B-A3B-GGUF:Q4_K_M"
q35nt: "Qwen3.5-35B-A3B-GGUF-nothink:Q4_K_M"
q35ht: "Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M"
q35hnt: "Qwen3.5-35B-A3B-heretic-GGUF-nothink:Q4_K_M"
q4t: "Qwen3.5-4B-GGUF:Q4_K_M"
q4nt: "Qwen3.5-4B-GGUF-nothink:Q4_K_M"
q4ht: "Qwen3.5-4B-heretic-GGUF:Q4_K_M"
q4hnt: "Qwen3.5-4B-heretic-GGUF-nothink:Q4_K_M"
g26xl: "gemma-4-26B-A4B-it-qat:UD-Q4_K_XL"
g26xlnt: "gemma-4-26B-A4B-it-qat-nothink:UD-Q4_K_XL"
g26mtp: "gemma-4-26B-A4B-it-qat-mtp:UD-Q4_K_XL"
g26mtpnt: "gemma-4-26B-A4B-it-qat-mtp-nothink:UD-Q4_K_XL"
g26ht: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL"
g26hnt: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-nothink:UD-Q4_K_XL"
g26hmtp: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp:UD-Q4_K_XL"
g26hmnt: "SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp-nothink:UD-Q4_K_XL"
ge4qat: "unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL"
ge4qatnt: "unsloth/gemma-4-E4B-it-qat-GGUF-nothink:UD-Q4_K_XL"
ge2qat: "unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL"
ge2qatnt: "unsloth/gemma-4-E2B-it-qat-GGUF-nothink:UD-Q4_K_XL"
ge4mtp: "unsloth/gemma-4-E4B-it-qat-GGUF-mtp:UD-Q4_K_XL"
ge4mtpnt: "unsloth/gemma-4-E4B-it-qat-GGUF-mtp-nothink:UD-Q4_K_XL"
ge4ht: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M"
ge4hnt: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-nothink:Q4_K_M"
ge4hmtp: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp:Q4_K_M"
ge4hmnt: "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp-nothink:Q4_K_M"
q36t: "unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL"
q36nt: "unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL"
haut: "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M"
haunt: "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-nothink:Q4_K_M"
mtpt: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M"
mtpnt: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF-nothink:Q4_K_M"
qwy9: "empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF:Q4_K_M"
evict_costs:
flux: 10 # large files, slow to reload
sets:
# any LLM can run alongside the small always-on model + STT + TTS (all CPU, no VRAM cost)
with_q8: "(coder | q35t | q35nt | q35ht | q35hnt | q4t | q4nt | q4ht | q4hnt | g26xl | g26xlnt | g26mtp | g26mtpnt | g26ht | g26hnt | g26hmtp | g26hmnt | ge4qat | ge4qatnt | ge2qat | ge2qatnt | ge4mtp | ge4mtpnt | ge4ht | ge4hnt | ge4hmtp | ge4hmnt | q36t | q36nt | haut | haunt | mtpt | mtpnt | qwy9) & q8 & stt"
# FLUX runs alone — evicts everything including q8, but keeps STT for voice during image gen
image_gen: "flux & stt"
models:
"Qwen3-Coder-Next-GGUF:Q4_K_M":
cmd: |
llama-server
-hf unsloth/Qwen3-Coder-Next-GGUF:Q4_K_M
--ctx-size 65536
--predict 8192
--temp 1.0
--min-p 0.01
--top-p 0.95
--top-k 40
--repeat-penalty 1.0
-ctk q4_0 -ctv q4_0
${common_args}
"Qwen3.5-35B-A3B-GGUF:Q4_K_M":
cmd: |
llama-server
-hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
${ctx_256k}
${qwen35_think_args}
${common_args}
"Qwen3.5-35B-A3B-GGUF-nothink:Q4_K_M":
cmd: |
llama-server
-hf unsloth/Qwen3.5-35B-A3B-GGUF:Q4_K_M
${ctx_256k}
${qwen35_nothink_args}
${common_args}
# The "heretic" version does not provide the mmproj
# so providing url to the one from the non-heretic version.
"Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M":
cmd: |
llama-server
-hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
${qwen35_35b_heretic_mmproj}
${ctx_256k}
${qwen35_think_args}
${common_args}
"Qwen3.5-35B-A3B-heretic-GGUF-nothink:Q4_K_M":
cmd: |
llama-server
-hf mradermacher/Qwen3.5-35B-A3B-heretic-GGUF:Q4_K_M
${qwen35_35b_heretic_mmproj}
${ctx_256k}
${qwen35_nothink_args}
${common_args}
"Qwen3.5-0.8B-GGUF-nothink:Q4_K_XL":
cmd: |
llama-server
-hf unsloth/Qwen3.5-0.8B-GGUF:Q4_K_XL
--ctx-size 4096
${qwen35_nothink_args}
${base_args}
"Qwen3.5-4B-GGUF:Q4_K_M":
cmd: |
llama-server
-hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
${ctx_128k}
${qwen35_think_args}
${common_args}
"Qwen3.5-4B-GGUF-nothink:Q4_K_M":
cmd: |
llama-server
-hf unsloth/Qwen3.5-4B-GGUF:Q4_K_M
${ctx_128k}
${qwen35_nothink_args}
${common_args}
"Qwen3.5-4B-heretic-GGUF:Q4_K_M":
cmd: |
llama-server
-hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
${qwen35_4b_heretic_mmproj}
${ctx_128k}
${qwen35_think_args}
${common_args}
"Qwen3.5-4B-heretic-GGUF-nothink:Q4_K_M":
cmd: |
llama-server
-hf mradermacher/Qwen3.5-4B-heretic-GGUF:Q4_K_M
${qwen35_4b_heretic_mmproj}
${ctx_128k}
${qwen35_nothink_args}
${common_args}
"gemma-4-26B-A4B-it-qat:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
${ctx_256k}
${gemma4_sampling}
${common_args}
"gemma-4-26B-A4B-it-qat-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
${ctx_256k}
${gemma4_nothink_sampling}
${common_args}
"gemma-4-26B-A4B-it-qat-mtp:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_256k}
${gemma4_sampling}
${common_args}
"gemma-4-26B-A4B-it-qat-mtp-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:UD-Q4_K_XL \
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_256k}
${gemma4_nothink_sampling}
${common_args}
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL":
cmd: |
llama-server
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
${ctx_256k}
${gemma4_sampling}
${common_args}
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
${ctx_256k}
${gemma4_nothink_sampling}
${common_args}
# The heretic QAT repo does not ship an MTP drafter,
# so borrow the one from the non-heretic unsloth QAT repo.
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp:UD-Q4_K_XL":
cmd: |
llama-server
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_256k}
${gemma4_sampling}
${common_args}
"SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF-mtp-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf SC117/gemma-4-26B-A4B-it-qat-heretic-GGUF:UD-Q4_K_XL \
--spec-draft-hf unsloth/gemma-4-26B-A4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_256k}
${gemma4_nothink_sampling}
${common_args}
"unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
${ctx_128k}
${gemma4_sampling}
${common_args}
"unsloth/gemma-4-E4B-it-qat-GGUF-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
${ctx_128k}
${gemma4_nothink_sampling}
${common_args}
"unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL \
${ctx_128k}
${gemma4_sampling}
${common_args}
"unsloth/gemma-4-E2B-it-qat-GGUF-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-E2B-it-qat-GGUF:UD-Q4_K_XL \
${ctx_128k}
${gemma4_nothink_sampling}
${common_args}
"unsloth/gemma-4-E4B-it-qat-GGUF-mtp:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_128k}
${gemma4_sampling}
${common_args}
"unsloth/gemma-4-E4B-it-qat-GGUF-mtp-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/gemma-4-E4B-it-qat-GGUF:UD-Q4_K_XL \
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_128k}
${gemma4_nothink_sampling}
${common_args}
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M":
cmd: |
llama-server
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
${ctx_128k}
${gemma4_sampling}
${common_args}
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-nothink:Q4_K_M":
cmd: |
llama-server
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
${ctx_128k}
${gemma4_nothink_sampling}
${common_args}
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp:Q4_K_M":
cmd: |
llama-server
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_128k}
${gemma4_sampling}
${common_args}
"llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF-mtp-nothink:Q4_K_M":
cmd: |
llama-server
-hf llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic-GGUF:Q4_K_M \
--spec-draft-hf unsloth/gemma-4-E4B-it-qat-GGUF:Q8_0-MTP \
--spec-type draft-mtp
--spec-draft-n-max 1
--swa-full
--kv-unified
--parallel 1
${ctx_128k}
${gemma4_nothink_sampling}
${common_args}
"unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL
${ctx_256k}
${qwen35_think_args}
${common_args}
"unsloth/Qwen3.6-35B-A3B-GGUF-nothink:UD-Q4_K_XL":
cmd: |
llama-server
-hf unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL
${ctx_256k}
${qwen35_nothink_args}
${common_args}
"HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M":
cmd: |
llama-server
-hf HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M
${ctx_256k}
${qwen35_think_args}
${common_args}
"HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-nothink:Q4_K_M":
cmd: |
llama-server
-hf HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q4_K_M
${ctx_256k}
${qwen35_nothink_args}
${common_args}
"unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M":
cmd: |
llama-server
-hf unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M
${ctx_256k}
${qwen35_think_args}
--spec-type draft-mtp --spec-draft-n-max 1
--parallel 1
${common_args}
"unsloth/Qwen3.6-35B-A3B-MTP-GGUF-nothink:Q4_K_M":
cmd: |
llama-server
-hf unsloth/Qwen3.6-35B-A3B-MTP-GGUF:Q4_K_M
${ctx_256k}
${qwen35_nothink_args}
--spec-type draft-mtp --spec-draft-n-max 1
--parallel 1
${common_args}
"empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF:Q4_K_M":
cmd: |
llama-server
-hf empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF:Q4_K_M
${ctx_128k}
--temp 0.6 --top-p 0.95 --top-k 20 -ctk q4_0 -ctv q4_0
${common_args}
# STT via parakeet-server (parakeet.cpp OpenAI-compatible server, CPU, always loaded)
# Model downloaded on first start and cached under /root/.cache/parakeet.cpp/models
# parakeet-proxy.py sits in front to convert any audio format to WAV via ffmpeg,
# since parakeet-server only accepts real WAV but browsers send Ogg/Opus.
"parakeet-tdt_ctc-1.1b":
checkEndpoint: none
cmd: |
env PROXY_PORT=${PORT} FFMPEG_BIN=/root/.cache/ffmpeg/ffmpeg python3 /config/parakeet-proxy.py
# Image generation via stable-diffusion.cpp (sd-server)
# Models must be pre-downloaded to /root/.cache/sd/
# FLUX.2-klein-4B: fast unified text-to-image and image editing model (Apache 2.0)
# Download: uv run --with huggingface_hub hf download unsloth/FLUX.2-klein-4B-GGUF flux-2-klein-4b-Q4_K_M.gguf --local-dir /root/.cache/sd
# Download VAE: uv run --with huggingface_hub hf download Comfy-Org/flux2-klein-4B split_files/vae/flux2-vae.safetensors --local-dir /root/.cache/sd/flux2-klein && cp /root/.cache/sd/flux2-klein/split_files/vae/flux2-vae.safetensors /root/.cache/sd/
# Download LLM: uv run --with huggingface_hub hf download ponpoke/flux2-klein-4b-uncensored-text-encoder flux2-klein-4b-uncensored-q4_k_m.gguf --local-dir /root/.cache/sd
"flux2-klein-4b:Q4_K_M":
checkEndpoint: "/"
cmd: |
sd-server
--listen-port ${PORT}
--diffusion-model /root/.cache/sd/flux-2-klein-4b-Q4_K_M.gguf
--vae /root/.cache/sd/flux2-vae.safetensors
--llm /root/.cache/sd/flux2-klein-4b-uncensored-q4_k_m.gguf
--cfg-scale 1.0
--sampling-method euler
--steps 4
--diffusion-fa
--offload-to-cpu
apps/llama/configs/parakeet-proxy.py
-227
View File
@@ -1,227 +0,0 @@
#!/usr/bin/env python3
"""
Thin reverse proxy for parakeet-server.
Accepts POST /v1/audio/transcriptions with any audio format,
converts the audio to 16 kHz mono WAV via ffmpeg, then forwards
the converted file to the real parakeet-server running on PARAKEET_PORT.
Also proxies GET /health straight through.
Usage:
PROXY_PORT=<port> PARAKEET_PORT=<upstream> python3 parakeet-proxy.py
"""
import http.server
import io
import os
import subprocess
import sys
import tempfile
import urllib.request
import urllib.error
PROXY_PORT = int(os.environ.get("PROXY_PORT", "8080"))
PARAKEET_PORT = PROXY_PORT + 1
FFMPEG = os.environ.get("FFMPEG_BIN", "ffmpeg")
MODEL = os.environ.get("PARAKEET_MODEL", "tdt_ctc-1.1b-q4_k.gguf")
CACHE_DIR = os.environ.get("PARAKEET_CACHE_DIR", "/root/.cache/parakeet.cpp/models")
def convert_to_wav(data: bytes) -> bytes:
"""Convert any audio bytes to 16 kHz mono PCM WAV via ffmpeg."""
with tempfile.NamedTemporaryFile(suffix=".input", delete=False) as inf:
inf.write(data)
inf_path = inf.name
out_path = inf_path + ".wav"
try:
subprocess.run(
[
FFMPEG, "-y",
"-i", inf_path,
"-ar", "16000",
"-ac", "1",
"-f", "wav",
out_path,
],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
with open(out_path, "rb") as f:
return f.read()
finally:
os.unlink(inf_path)
if os.path.exists(out_path):
os.unlink(out_path)
def parse_multipart(content_type: str, body: bytes):
"""
Parse a multipart/form-data body.
Returns a dict of field_name -> (filename_or_None, content_type, data).
"""
import email
from email import policy as email_policy
# email.parser needs the full MIME headers to parse multipart
raw = b"Content-Type: " + content_type.encode() + b"\r\n\r\n" + body
msg = email.message_from_bytes(raw, policy=email_policy.compat32)
parts = {}
for part in msg.get_payload():
cd = part.get("Content-Disposition", "")
name = None
filename = None
for item in cd.split(";"):
item = item.strip()
if item.startswith('name='):
name = item[5:].strip('"')
elif item.startswith('filename='):
filename = item[9:].strip('"')
if name is not None:
parts[name] = (filename, part.get_content_type(), part.get_payload(decode=True))
return parts
def build_multipart(fields: dict) -> tuple[bytes, str]:
"""
Build a multipart/form-data body from fields dict:
field_name -> (filename_or_None, content_type, data_bytes)
Returns (body_bytes, content_type_header_value).
"""
boundary = b"----ParakeetProxyBoundary0xDEADBEEF"
body = b""
for name, (filename, ct, data) in fields.items():
body += b"--" + boundary + b"\r\n"
if filename:
body += (
f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\r\n'
).encode()
else:
body += f'Content-Disposition: form-data; name="{name}"\r\n'.encode()
body += f"Content-Type: {ct}\r\n\r\n".encode()
body += data + b"\r\n"
body += b"--" + boundary + b"--\r\n"
return body, f"multipart/form-data; boundary={boundary.decode()}"
class ProxyHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
print(f"[parakeet-proxy] {self.address_string()} - {fmt % args}", flush=True)
def do_GET(self):
if self.path == "/health":
self._forward_get("/health")
else:
self.send_response(404)
self.end_headers()
def do_POST(self):
if self.path.rstrip("/") == "/v1/audio/transcriptions":
self._handle_transcription()
else:
self.send_response(404)
self.end_headers()
def _forward_get(self, path):
try:
url = f"http://127.0.0.1:{PARAKEET_PORT}{path}"
with urllib.request.urlopen(url, timeout=5) as resp:
body = resp.read()
self.send_response(resp.status)
self.send_header("Content-Type", resp.headers.get("Content-Type", "application/json"))
self.end_headers()
self.wfile.write(body)
except Exception as e:
self.send_response(502)
self.end_headers()
self.wfile.write(str(e).encode())
def _handle_transcription(self):
length = int(self.headers.get("Content-Length", 0))
body = self.rfile.read(length)
ct = self.headers.get("Content-Type", "")
try:
fields = parse_multipart(ct, body)
except Exception as e:
self._error(400, f"failed to parse multipart: {e}")
return
if "file" not in fields:
self._error(400, "missing required field 'file'")
return
filename, file_ct, audio_data = fields["file"]
# Convert to WAV regardless of what we received
try:
wav_data = convert_to_wav(audio_data)
except subprocess.CalledProcessError:
self._error(400, "ffmpeg could not decode audio")
return
except Exception as e:
self._error(500, f"conversion error: {e}")
return
# Rebuild multipart with converted WAV, preserve other fields
new_fields = {}
for name, (fn, fct, data) in fields.items():
if name == "file":
new_fields[name] = ("recording.wav", "audio/wav", wav_data)
else:
new_fields[name] = (fn, fct, data)
new_body, new_ct = build_multipart(new_fields)
# Forward to parakeet-server
try:
url = f"http://127.0.0.1:{PARAKEET_PORT}/v1/audio/transcriptions"
req = urllib.request.Request(
url,
data=new_body,
headers={"Content-Type": new_ct},
method="POST",
)
with urllib.request.urlopen(req, timeout=300) as resp:
resp_body = resp.read()
self.send_response(resp.status)
self.send_header("Content-Type", resp.headers.get("Content-Type", "application/json"))
self.end_headers()
self.wfile.write(resp_body)
except urllib.error.HTTPError as e:
resp_body = e.read()
self.send_response(e.code)
self.send_header("Content-Type", e.headers.get("Content-Type", "application/json"))
self.end_headers()
self.wfile.write(resp_body)
except Exception as e:
self._error(502, f"upstream error: {e}")
def _error(self, code: int, msg: str):
body = f'{{"error":{{"message":"{msg}","type":"proxy_error"}}}}'.encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(body)
if __name__ == "__main__":
proc = subprocess.Popen([
"parakeet-server",
"--host", "127.0.0.1",
"--port", str(PARAKEET_PORT),
"--model", MODEL,
"--cache-dir", CACHE_DIR,
])
print(f"[parakeet-proxy] started parakeet-server pid={proc.pid} on :{PARAKEET_PORT}", flush=True)
server = http.server.HTTPServer(("0.0.0.0", PROXY_PORT), ProxyHandler)
print(f"[parakeet-proxy] listening on :{PROXY_PORT}", flush=True)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
finally:
proc.terminate()
proc.wait()
apps/llama/deployment.yaml
+4 -47
View File
@@ -2,7 +2,7 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: llama-swap
name: supervisord
namespace: llama
spec:
replicas: 1
@@ -16,55 +16,17 @@ spec:
labels:
app: llama-swap
spec:
initContainers:
- name: download-whisper
image: gitea.lumpiasty.xyz/lumpiasty/llama-swap:unified-vulkan-parakeet-2026-07-06
command:
- sh
- -c
- |
mkdir -p /root/.cache/whisper
if [ ! -f /root/.cache/whisper/ggml-small.bin ]; then
echo "Downloading whisper-small model..."
curl -L -o /root/.cache/whisper/ggml-small.bin \
https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin
else
echo "whisper-small model already present, skipping download"
fi
if [ ! -f /root/.cache/ffmpeg/ffmpeg ]; then
echo "Downloading static ffmpeg..."
mkdir -p /root/.cache/ffmpeg
apt-get update -qq && apt-get install -y --no-install-recommends xz-utils
curl -L -o /root/.cache/ffmpeg/ffmpeg.tar.xz \
https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
tar -xJf /root/.cache/ffmpeg/ffmpeg.tar.xz -C /root/.cache/ffmpeg --wildcards '*/ffmpeg' --strip-components=2
rm /root/.cache/ffmpeg/ffmpeg.tar.xz
chmod +x /root/.cache/ffmpeg/ffmpeg
else
echo "ffmpeg already present, skipping download"
fi
volumeMounts:
- name: models
mountPath: /root/.cache
containers:
- name: llama-swap
image: gitea.lumpiasty.xyz/lumpiasty/llama-swap:unified-vulkan-parakeet-2026-07-06
- name: supervisord
image: gitea.lumpiasty.xyz/lumpiasty/supervisord:latest@sha256:e9c5d44065722a9811a45f0fd2ff0f72647225d6761e5950b9df5c21dec211eb
imagePullPolicy: IfNotPresent
command:
- llama-swap
args:
- --config=/config/config.yaml
- --watch-config
env:
- name: RADV_EXPERIMENTAL
value: transfer_queue
ports:
- containerPort: 8080
name: http
protocol: TCP
volumeMounts:
- name: models
mountPath: /root/.cache
mountPath: /workspace
- name: models
mountPath: /usr/local/bin/ffmpeg
subPath: ffmpeg/ffmpeg
@@ -72,8 +34,6 @@ spec:
name: kfd
- mountPath: /dev/dri
name: dri
- mountPath: /config
name: config
securityContext:
privileged: true
volumes:
@@ -88,9 +48,6 @@ spec:
hostPath:
path: /dev/dri
type: Directory
- name: config
configMap:
name: llama-swap
---
apiVersion: v1
kind: Service
apps/llama/kustomization.yaml
-6
View File
@@ -8,9 +8,3 @@ resources:
- pvc-ssd.yaml
- deployment.yaml
- kokoro.yaml
configMapGenerator:
- name: llama-swap
namespace: llama
files:
- config.yaml=configs/config.yaml
- parakeet-proxy.py=configs/parakeet-proxy.py