* server: add realtime transcription API * drop transcription_sessions * proxy ws * add docs * server: let the API key middleware guard the websocket httplib runs the pre-routing handler before the websocket upgrade, so the API key middleware answers 401 at handshake like any other route. This covers the router proxy, whose children run keyless, and accepts the same key forms as HTTP (Bearer, raw key, X-Api-Key). * server: keep realtime slots out of the prompt loop A realtime slot waiting for audio is GENERATING with nothing to decode, so it fell through the prompt loop and took slot_batched. A request on a higher slot then failed can_batch_with on the task type and starved for the whole session. Realtime slots have no prompt: the loop sets them up once and always leaves them. * server: start the realtime session once its slot is live The audio stream was created in the prompt loop, which skips a slot that cannot batch with the one generating, and session.created went out before the task was posted: early appends failed and the reader quit, leaving the socket open and the slot held until the HTTP timeout. The slot now sets up its stream at launch and signals it with the begin result, session.created waits for it, and a reader that stops closes the socket so the session ends with it. * server: name the committed item in input_audio_buffer.committed Per the Realtime spec, item_id is the item made from the committed audio and previous_item_id the one committed before it, null for the first. committed now comes before the transcript that completes that item, and both go out with the item switch under one lock, so a delta cannot land between them. * server: use the GA transcription session shape The session object follows the GA Realtime schema: type transcription, with the input format, transcription model, turn detection and noise reduction under audio.input. The rate stays the model sample rate, which is what the client must send for now. --------- Co-authored-by: Pascal <admin@serveurperso.com>
199 lines
7.1 KiB
C++
199 lines
7.1 KiB
C++
#pragma once
|
|
|
|
#include "server-http.h"
|
|
#include "server-task.h"
|
|
#include "server-queue.h"
|
|
|
|
#include "json.h"
|
|
|
|
#include <cstddef>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <set>
|
|
|
|
struct server_context_impl; // private implementation
|
|
|
|
struct server_context_meta {
|
|
std::string build_info;
|
|
std::string model_name;
|
|
std::set<std::string> model_aliases;
|
|
std::set<std::string> model_tags;
|
|
std::string model_path;
|
|
std::vector<std::string> model_output_modalities; // output modalities for GET /models
|
|
bool has_mtmd;
|
|
bool has_inp_image;
|
|
bool has_inp_audio;
|
|
bool has_inp_video;
|
|
bool has_audio_stream; // realtime transcription, see mtmd_support_audio_stream()
|
|
json json_ui_settings;
|
|
int slot_n_ctx;
|
|
enum llama_pooling_type pooling_type;
|
|
|
|
// chat params
|
|
server_chat_params & chat_params;
|
|
std::map<std::string, bool> chat_template_caps;
|
|
|
|
// tokens
|
|
std::string bos_token_str;
|
|
std::string eos_token_str;
|
|
llama_token fim_pre_token;
|
|
llama_token fim_sub_token;
|
|
llama_token fim_mid_token;
|
|
llama_token fim_pad_token;
|
|
llama_token fim_rep_token;
|
|
llama_token fim_sep_token;
|
|
|
|
// sampling
|
|
std::vector<llama_logit_bias> logit_bias_eog;
|
|
|
|
// model meta
|
|
enum llama_vocab_type model_vocab_type;
|
|
int32_t model_vocab_n_tokens;
|
|
int32_t model_n_ctx_train;
|
|
int32_t model_n_embd_inp;
|
|
uint64_t model_n_params;
|
|
uint64_t model_size;
|
|
std::string model_ftype;
|
|
};
|
|
|
|
enum server_state {
|
|
SERVER_STATE_DOWNLOADING,
|
|
SERVER_STATE_LOADING,
|
|
SERVER_STATE_READY,
|
|
SERVER_STATE_SLEEPING,
|
|
};
|
|
|
|
static std::string server_state_to_str(server_state state) {
|
|
switch (state) {
|
|
case SERVER_STATE_DOWNLOADING: return "downloading";
|
|
case SERVER_STATE_LOADING: return "loading";
|
|
case SERVER_STATE_READY: return "ready";
|
|
case SERVER_STATE_SLEEPING: return "sleeping";
|
|
default: GGML_ASSERT(false && "invalid server_state");
|
|
}
|
|
}
|
|
|
|
static server_state server_state_from_str(const std::string & str) {
|
|
if (str == "downloading") return SERVER_STATE_DOWNLOADING;
|
|
if (str == "loading") return SERVER_STATE_LOADING;
|
|
if (str == "ready") return SERVER_STATE_READY;
|
|
if (str == "sleeping") return SERVER_STATE_SLEEPING;
|
|
GGML_ASSERT(false && "invalid server_state string");
|
|
}
|
|
|
|
using server_state_callback_t = std::function<void(server_state, json /* payload */)>;
|
|
|
|
struct server_context {
|
|
std::unique_ptr<server_context_impl> impl;
|
|
|
|
server_context();
|
|
~server_context();
|
|
|
|
// load the model and initialize llama_context
|
|
// returns true on success
|
|
bool load_model(common_params & params);
|
|
|
|
// this function will block main thread until termination
|
|
void start_loop();
|
|
|
|
// terminate main loop (will unblock start_loop)
|
|
void terminate();
|
|
|
|
// get the underlaying llama_context, can return nullptr if sleeping
|
|
// not thread-safe, should only be used from the main thread
|
|
llama_context * get_llama_context() const;
|
|
|
|
// get a new response reader, used by CLI application
|
|
server_response_reader get_response_reader();
|
|
|
|
// get server metadata (read-only), can only be called after load_model()
|
|
// not thread-safe, should only be used from the main thread
|
|
server_context_meta get_meta() const;
|
|
|
|
// note: must be set before load_model() is called
|
|
void set_state_callback(server_state_callback_t callback);
|
|
};
|
|
|
|
|
|
// forward declarations
|
|
struct server_res_generator;
|
|
|
|
struct server_routes {
|
|
server_routes(const common_params & params, server_context & ctx_server);
|
|
|
|
void init_routes();
|
|
|
|
// note: this is not thread-safe and can only when ctx_http.is_ready is false
|
|
void update_meta(const server_context & ctx_server) {
|
|
this->meta = std::make_unique<server_context_meta>(ctx_server.get_meta());
|
|
}
|
|
|
|
// handlers using lambda function, so that they can capture `this` without `std::bind`
|
|
// they won't be called until ctx_http.is_ready is set to true
|
|
server_http_context::handler_t get_health;
|
|
server_http_context::handler_t get_metrics;
|
|
server_http_context::handler_t get_slots;
|
|
server_http_context::handler_t post_slots;
|
|
server_http_context::handler_t get_props;
|
|
server_http_context::handler_t post_props;
|
|
server_http_context::handler_t post_infill;
|
|
server_http_context::handler_t post_completions;
|
|
server_http_context::handler_t post_completions_oai;
|
|
server_http_context::handler_t post_chat_completions;
|
|
server_http_context::handler_t post_chat_completions_tok;
|
|
server_http_context::handler_t post_control;
|
|
server_http_context::handler_t post_responses_oai;
|
|
server_http_context::handler_t post_responses_tok_oai;
|
|
server_http_context::handler_t post_transcriptions_oai;
|
|
server_http_context::ws_handler_t ws_realtime;
|
|
server_http_context::handler_t post_anthropic_messages;
|
|
server_http_context::handler_t post_anthropic_count_tokens;
|
|
server_http_context::handler_t post_apply_template;
|
|
server_http_context::handler_t get_models;
|
|
server_http_context::handler_t post_tokenize;
|
|
server_http_context::handler_t post_detokenize;
|
|
server_http_context::handler_t post_embeddings;
|
|
server_http_context::handler_t post_embeddings_oai;
|
|
server_http_context::handler_t post_rerank;
|
|
server_http_context::handler_t post_systemone;
|
|
server_http_context::handler_t get_lora_adapters;
|
|
server_http_context::handler_t post_lora_adapters;
|
|
|
|
// to be used in router mode
|
|
json get_model_info() const;
|
|
|
|
private:
|
|
std::unique_ptr<server_res_generator> handle_completions_impl(
|
|
const server_http_req & req,
|
|
server_task_type type,
|
|
const json & data,
|
|
const std::vector<raw_buffer> & files,
|
|
task_response_type res_type,
|
|
const common_chat_session & chat_session = {});
|
|
std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);
|
|
std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
|
|
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
|
|
std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
|
|
std::unique_ptr<server_res_generator> handle_count_tokens(const server_http_req & req, task_response_type res_type);
|
|
|
|
// using unique_ptr to allow late initialization of const
|
|
std::unique_ptr<const server_context_meta> meta;
|
|
|
|
const common_params & params;
|
|
server_context_impl & ctx_server;
|
|
|
|
server_queue & queue_tasks;
|
|
server_response & queue_results;
|
|
std::unique_ptr<server_res_generator> create_response(bool bypass_sleep = false);
|
|
|
|
// cached responses, to be used during sleep
|
|
std::mutex mutex_cache;
|
|
json cached_models = nullptr;
|
|
json cached_props = nullptr;
|
|
server_metrics cached_metrics;
|
|
// set when a scrape during sleep already reported the throughput buckets
|
|
bool should_reset_buckets = false;
|
|
// call right before sleep to update the cached responses
|
|
void update_cached_responses(bool is_sleeping);
|
|
};
|