#include "server-task.h" #include "build-info.h" #include "server-chat.h" #include "chat.h" #include "common.h" #include "json-schema-to-grammar.h" #include "llama.h" #include "sampling.h" #include "speculative.h" #include "server-common.h" #include #include #include #include #include #include #include #if defined(__linux__) #include #include #endif // // task_params // json task_params::format_logit_bias(const std::vector & logit_bias) const { json data = json::array(); for (const auto & lb : logit_bias) { data.push_back(json{ {"bias", lb.bias}, {"token", lb.token}, }); } return data; } json task_params::to_json(bool only_metrics) const { std::vector samplers; samplers.reserve(sampling.samplers.size()); for (const auto & sampler : sampling.samplers) { samplers.emplace_back(common_sampler_type_to_str(sampler)); } json lora = json::array(); for (auto & it : this->lora) { lora.push_back({{"id", it.first}, {"scale", it.second}}); } if (only_metrics) { return json { {"seed", sampling.seed}, {"temperature", sampling.temp}, {"dynatemp_range", sampling.dynatemp_range}, {"dynatemp_exponent", sampling.dynatemp_exponent}, {"top_k", sampling.top_k}, {"top_p", sampling.top_p}, {"min_p", sampling.min_p}, {"top_n_sigma", sampling.top_n_sigma}, {"xtc_probability", sampling.xtc_probability}, {"xtc_threshold", sampling.xtc_threshold}, {"typical_p", sampling.typ_p}, {"repeat_last_n", sampling.penalty_last_n}, {"repeat_penalty", sampling.penalty_repeat}, {"presence_penalty", sampling.penalty_present}, {"frequency_penalty", sampling.penalty_freq}, {"dry_multiplier", sampling.dry_multiplier}, {"dry_base", sampling.dry_base}, {"dry_allowed_length", sampling.dry_allowed_length}, {"dry_penalty_last_n", sampling.dry_penalty_last_n}, {"mirostat", sampling.mirostat}, {"mirostat_tau", sampling.mirostat_tau}, {"mirostat_eta", sampling.mirostat_eta}, {"adaptive_target", sampling.adaptive_target}, {"adaptive_decay", sampling.adaptive_decay}, {"max_tokens", n_predict}, {"n_predict", n_predict}, // TODO: deduplicate? {"n_keep", n_keep}, {"n_discard", n_discard}, {"ignore_eos", sampling.ignore_eos}, {"stream", stream}, {"n_probs", sampling.n_probs}, {"min_keep", sampling.min_keep}, {"chat_format", common_chat_format_name(chat_format)}, {"reasoning_format", common_reasoning_format_name(reasoning_format)}, {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY}, {"generation_prompt", sampling.generation_prompt}, {"samplers", samplers}, {"speculative.types", common_speculative_type_name_str(speculative.types)}, {"timings_per_token", timings_per_token}, {"post_sampling_probs", post_sampling_probs}, {"backend_sampling", sampling.backend_sampling}, {"lora", lora}, }; } auto grammar_triggers = json::array(); for (const auto & trigger : sampling.grammar_triggers) { server_grammar_trigger ct(trigger); grammar_triggers.push_back(ct.to_json()); } return json { {"seed", sampling.seed}, {"temperature", sampling.temp}, {"dynatemp_range", sampling.dynatemp_range}, {"dynatemp_exponent", sampling.dynatemp_exponent}, {"top_k", sampling.top_k}, {"top_p", sampling.top_p}, {"min_p", sampling.min_p}, {"top_n_sigma", sampling.top_n_sigma}, {"xtc_probability", sampling.xtc_probability}, {"xtc_threshold", sampling.xtc_threshold}, {"typical_p", sampling.typ_p}, {"repeat_last_n", sampling.penalty_last_n}, {"repeat_penalty", sampling.penalty_repeat}, {"presence_penalty", sampling.penalty_present}, {"frequency_penalty", sampling.penalty_freq}, {"dry_multiplier", sampling.dry_multiplier}, {"dry_base", sampling.dry_base}, {"dry_allowed_length", sampling.dry_allowed_length}, {"dry_penalty_last_n", sampling.dry_penalty_last_n}, {"dry_sequence_breakers", sampling.dry_sequence_breakers}, {"mirostat", sampling.mirostat}, {"mirostat_tau", sampling.mirostat_tau}, {"mirostat_eta", sampling.mirostat_eta}, {"adaptive_target", sampling.adaptive_target}, {"adaptive_decay", sampling.adaptive_decay}, {"stop", antiprompt}, {"max_tokens", n_predict}, {"n_predict", n_predict}, // TODO: deduplicate? {"n_keep", n_keep}, {"n_discard", n_discard}, {"ignore_eos", sampling.ignore_eos}, {"stream", stream}, {"logit_bias", format_logit_bias(sampling.logit_bias)}, {"n_probs", sampling.n_probs}, {"min_keep", sampling.min_keep}, {"grammar", common_grammar_value(sampling.grammar)}, {"grammar_lazy", sampling.grammar_lazy}, {"grammar_triggers", grammar_triggers}, {"preserved_tokens", sampling.preserved_tokens}, {"chat_format", common_chat_format_name(chat_format)}, {"reasoning_format", common_reasoning_format_name(reasoning_format)}, {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY}, {"generation_prompt", sampling.generation_prompt}, {"samplers", samplers}, {"speculative.types", common_speculative_type_name_str(speculative.types)}, {"timings_per_token", timings_per_token}, {"post_sampling_probs", post_sampling_probs}, {"backend_sampling", sampling.backend_sampling}, {"lora", lora}, }; } // // task_result_state // task_result_state::task_result_state(common_chat_session session) : chat_session(std::move(session)) , chat_msg(chat_session.msg()) , oai_resp_id("resp_" + random_string()) , oai_resp_reasoning_id("rs_" + random_string()) , oai_resp_message_id("msg_" + random_string()) { } common_chat_msg task_result_state::update_chat_msg( const common_chat_input & added, bool is_partial, std::vector & diffs, bool filter_tool_calls) { auto msg_prv_copy = chat_msg; auto new_msg = is_partial ? chat_session.feed(added) : chat_session.finish(added); if (!new_msg.empty()) { new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id); chat_msg = new_msg; auto all_diffs = common_chat_msg_diff::compute_diffs(msg_prv_copy, chat_msg); if (!filter_tool_calls) { diffs = std::move(all_diffs); } else { for (auto & d : all_diffs) { // If this is a new type of delta, flush all currently pending tool call names for (size_t i = 0; i < chat_msg.tool_calls.size(); ++i) { if (sent_tool_call_names.count(i) || chat_msg.tool_calls[i].name.empty()) { continue; } if (d.tool_call_index != i || !d.tool_call_delta.arguments.empty()) { common_chat_msg_diff header; header.tool_call_index = i; header.tool_call_delta.id = chat_msg.tool_calls[i].id; header.tool_call_delta.name = chat_msg.tool_calls[i].name; diffs.push_back(std::move(header)); sent_tool_call_names.insert(i); } } if (d.tool_call_index == std::string::npos) { diffs.push_back(std::move(d)); } else { size_t i = d.tool_call_index; if (sent_tool_call_names.count(i)) { if (!d.tool_call_delta.arguments.empty()) { d.tool_call_delta.name = ""; d.tool_call_delta.id = ""; diffs.push_back(std::move(d)); } } else { // Not sent yet. if (!d.tool_call_delta.arguments.empty() || !is_partial) { d.tool_call_delta.name = chat_msg.tool_calls[i].name; d.tool_call_delta.id = chat_msg.tool_calls[i].id; diffs.push_back(std::move(d)); sent_tool_call_names.insert(i); } else { // Suppress } } } } // Final check at EOF if (!is_partial) { for (size_t i = 0; i < chat_msg.tool_calls.size(); ++i) { if (!sent_tool_call_names.count(i) && !chat_msg.tool_calls[i].name.empty()) { common_chat_msg_diff header; header.tool_call_index = i; header.tool_call_delta.id = chat_msg.tool_calls[i].id; header.tool_call_delta.name = chat_msg.tool_calls[i].name; diffs.push_back(std::move(header)); sent_tool_call_names.insert(i); } } } } } return chat_msg; } // // result_prompt_progress // json result_prompt_progress::to_json() const { return json { {"total", total}, {"cache", cache}, {"processed", processed}, {"time_ms", time_ms}, }; } static inline std::string stop_type_to_str(stop_type type) { switch (type) { case STOP_TYPE_EOS: return "eos"; case STOP_TYPE_WORD: return "word"; case STOP_TYPE_LIMIT: return "limit"; default: return "none"; } } // // completion_token_output // json completion_token_output::to_json(bool post_sampling_probs) const { json probs_for_token = json::array(); for (const auto & p : probs) { std::string txt(p.txt); txt.resize(validate_utf8(txt)); probs_for_token.push_back(json { {"id", p.tok}, {"token", txt}, {"bytes", str_to_bytes(p.txt)}, { post_sampling_probs ? "prob" : "logprob", post_sampling_probs ? p.prob : logarithm(p.prob) }, }); } return probs_for_token; } json completion_token_output::probs_vector_to_json(const std::vector & probs, bool post_sampling_probs) { json out = json::array(); for (const auto & p : probs) { std::string txt(p.text_to_send); txt.resize(validate_utf8(txt)); out.push_back(json { {"id", p.tok}, {"token", txt}, {"bytes", str_to_bytes(p.text_to_send)}, { post_sampling_probs ? "prob" : "logprob", post_sampling_probs ? p.prob : logarithm(p.prob) }, { post_sampling_probs ? "top_probs" : "top_logprobs", p.to_json(post_sampling_probs) }, }); } return out; } float completion_token_output::logarithm(float x) { // the JSON library converts -inf to null, so we need to prevent that return x == 0.0f ? std::numeric_limits::lowest() : std::log(x); } std::vector completion_token_output::str_to_bytes(const std::string & str) { std::vector bytes; for (unsigned char c : str) { bytes.push_back(c); } return bytes; } // // server_task_result_cmpl_final // json server_task_result_cmpl_final::to_json() { GGML_ASSERT(is_updated && "update() must be called before to_json()"); switch (res_type) { case TASK_RESPONSE_TYPE_NONE: return to_json_non_oaicompat(); case TASK_RESPONSE_TYPE_OAI_CMPL: return to_json_oaicompat(); case TASK_RESPONSE_TYPE_OAI_CHAT: return stream ? to_json_oaicompat_chat_stream() : to_json_oaicompat_chat(); case TASK_RESPONSE_TYPE_OAI_RESP: return stream ? to_json_oaicompat_resp_stream() : to_json_oaicompat_resp(); case TASK_RESPONSE_TYPE_OAI_ASR: return to_json_oaicompat_asr(); case TASK_RESPONSE_TYPE_ANTHROPIC: return stream ? to_json_anthropic_stream() : to_json_anthropic(); default: GGML_ASSERT(false && "Invalid task_response_type"); } } json server_task_result_cmpl_final::to_json_non_oaicompat() { json res = json { {"index", index}, {"content", content.text}, {"tokens", tokens}, {"id_slot", id_slot}, {"stop", true}, {"model", oaicompat_model}, {"tokens_predicted", n_decoded}, {"tokens_evaluated", n_prompt_tokens}, {"generation_settings", generation_params.to_json()}, {"prompt", prompt}, {"has_new_line", has_new_line}, {"truncated", truncated}, {"stop_type", stop_type_to_str(stop)}, {"stopping_word", stopping_word}, {"tokens_cached", n_tokens_cached}, {"timings", stats.to_json()}, }; if (!stream && !probs_output.empty()) { res["completion_probabilities"] = completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs); } return response_fields.empty() ? res : json_get_nested_values(response_fields, res); } json server_task_result_cmpl_final::usage_json_oaicompat() { return json { {"completion_tokens", n_decoded}, {"prompt_tokens", n_prompt_tokens}, {"total_tokens", n_decoded + n_prompt_tokens}, {"prompt_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }}, }; } json server_task_result_cmpl_final::to_json_oaicompat() { std::time_t t = std::time(0); json logprobs = json(nullptr); // OAI default to null if (!stream && probs_output.size() > 0) { logprobs = json{ {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)}, }; } json finish_reason = "length"; if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { finish_reason = "stop"; } json res = json { {"choices", json::array({ json{ {"text", content.text}, {"index", index}, {"logprobs", logprobs}, {"finish_reason", finish_reason}, } })}, {"created", t}, {"model", oaicompat_model}, {"system_fingerprint", std::string(llama_build_info())}, {"object", "text_completion"}, {"usage", usage_json_oaicompat()}, {"id", oaicompat_cmpl_id} }; // extra fields for debugging purposes if (verbose) { res["__verbose"] = to_json_non_oaicompat(); } if (stats.is_set()) { res["timings"] = stats.to_json(); } return res; } json server_task_result_cmpl_final::to_json_oaicompat_chat() { std::string finish_reason = "length"; common_chat_msg msg; if (!oaicompat_msg.empty()) { msg = oaicompat_msg; } else { msg.role = "assistant"; msg.content = content.text; } if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls"; } json choice { {"finish_reason", finish_reason}, {"index", index}, {"message", msg.to_json_oaicompat()}, }; if (!stream && probs_output.size() > 0) { choice["logprobs"] = json{ {"content", completion_token_output::probs_vector_to_json(probs_output, post_sampling_probs)}, }; } std::time_t t = std::time(0); json res = json { {"choices", json::array({choice})}, {"created", t}, {"model", oaicompat_model}, {"system_fingerprint", std::string(llama_build_info())}, {"object", "chat.completion"}, {"usage", usage_json_oaicompat()}, {"id", oaicompat_cmpl_id} }; // extra fields for debugging purposes if (verbose) { res["__verbose"] = to_json_non_oaicompat(); } if (stats.is_set()) { res["timings"] = stats.to_json(); } return res; } json server_task_result_cmpl_final::to_json_oaicompat_chat_stream() { std::time_t t = std::time(0); std::string finish_reason = "length"; if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { finish_reason = oaicompat_msg.tool_calls.empty() ? "stop" : "tool_calls"; } json deltas = json::array(); for (const auto & diff : oaicompat_msg_diffs) { deltas.push_back({ {"choices", json::array({ json { {"finish_reason", nullptr}, {"index", index}, {"delta", server_chat_msg_diff_to_json_oaicompat(diff)}, }, })}, {"created", t}, {"id", oaicompat_cmpl_id}, {"model", oaicompat_model}, {"system_fingerprint", std::string(llama_build_info())}, {"object", "chat.completion.chunk"}, }); } deltas.push_back({ {"choices", json::array({ json { {"finish_reason", finish_reason}, {"index", index}, {"delta", json::object()}, }, })}, {"created", t}, {"id", oaicompat_cmpl_id}, {"model", oaicompat_model}, {"system_fingerprint", std::string(llama_build_info())}, {"object", "chat.completion.chunk"}, }); if (include_usage) { // OpenAI API spec for chat.completion.chunks specifies an empty `choices` array for the last chunk when including usage // https://platform.openai.com/docs/api-reference/chat_streaming/streaming#chat_streaming/streaming-choices deltas.push_back({ {"choices", json::array()}, {"created", t}, {"id", oaicompat_cmpl_id}, {"model", oaicompat_model}, {"system_fingerprint", std::string(llama_build_info())}, {"object", "chat.completion.chunk"}, {"usage", usage_json_oaicompat()}, }); } if (stats.is_set()) { deltas.back()["timings"] = stats.to_json(); } // extra fields for debugging purposes if (verbose && !deltas.empty()) { deltas.front()["__verbose"] = to_json_non_oaicompat(); } return deltas; } json server_task_result_cmpl_final::to_json_oaicompat_resp() { common_chat_msg msg; if (!oaicompat_msg.empty()) { msg = oaicompat_msg; } else { msg.role = "assistant"; msg.content = content.text; } std::vector output; if (msg.reasoning_content != "") { output.push_back(json { {"id", "rs_" + random_string()}, {"summary", json::array()}, {"type", "reasoning"}, {"content", json::array({ json { {"text", msg.reasoning_content}, {"type", "reasoning_text"}, }})}, {"encrypted_content", ""}, {"status", "completed"}, }); } if (msg.content != "") { output.push_back(json { {"content", json::array({ json { {"type", "output_text"}, {"annotations", json::array()}, {"logprobs", json::array()}, {"text", msg.content}, }})}, {"id", "msg_" + random_string()}, {"role", msg.role}, {"status", "completed"}, {"type", "message"}, }); } for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) { output.push_back(json { {"id", "fc_" + tool_call.id}, {"type", "function_call"}, {"status", "completed"}, {"arguments", tool_call.arguments}, {"call_id", "call_" + tool_call.id}, {"name", tool_call.name}, }); } std::time_t t = std::time(0); json res = { {"completed_at", t}, {"created_at", t}, {"id", oai_resp_id}, {"model", oaicompat_model}, {"object", "response"}, {"output", output}, {"status", "completed"}, {"usage", json { {"input_tokens", n_prompt_tokens}, {"output_tokens", n_decoded}, {"total_tokens", n_decoded + n_prompt_tokens}, {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }}, }}, }; return res; } json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() { std::vector server_sent_events; std::vector output; size_t output_index = 0; if (oaicompat_msg.reasoning_content != "") { const size_t item_output_index = output_index++; const json output_item = json { {"id", oai_resp_reasoning_id}, {"summary", json::array()}, {"type", "reasoning"}, {"content", json::array({ json { {"text", oaicompat_msg.reasoning_content}, {"type", "reasoning_text"}, }})}, {"encrypted_content", ""}, }; server_sent_events.push_back(json { {"event", "response.output_item.done"}, {"data", json { {"type", "response.output_item.done"}, {"output_index", item_output_index}, {"item", output_item} }} }); output.push_back(output_item); } if (oaicompat_msg.content != "") { const size_t item_output_index = output_index++; server_sent_events.push_back(json { {"event", "response.output_text.done"}, {"data", json { {"type", "response.output_text.done"}, {"item_id", oai_resp_message_id}, {"output_index", item_output_index}, {"content_index", 0}, {"text", oaicompat_msg.content} }} }); const json content_part = { {"type", "output_text"}, {"annotations", json::array()}, {"logprobs", json::array()}, {"text", oaicompat_msg.content} }; server_sent_events.push_back(json { {"event", "response.content_part.done"}, {"data", json { {"type", "response.content_part.done"}, {"item_id", oai_resp_message_id}, {"output_index", item_output_index}, {"content_index", 0}, {"part", content_part} }} }); const json output_item = { {"type", "message"}, {"status", "completed"}, {"id", oai_resp_message_id}, {"content", json::array({content_part})}, {"role", "assistant"} }; server_sent_events.push_back(json { {"event", "response.output_item.done"}, {"data", json { {"type", "response.output_item.done"}, {"output_index", item_output_index}, {"item", output_item} }} }); output.push_back(output_item); } for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) { const size_t item_output_index = output_index++; const json output_item = { {"id", "fc_" + tool_call.id}, {"type", "function_call"}, {"status", "completed"}, {"arguments", tool_call.arguments}, {"call_id", "call_" + tool_call.id}, {"name", tool_call.name} }; server_sent_events.push_back(json { {"event", "response.output_item.done"}, {"data", json { {"type", "response.output_item.done"}, {"output_index", item_output_index}, {"item", output_item} }} }); output.push_back(output_item); } std::time_t t = std::time(0); server_sent_events.push_back(json { {"event", "response.completed"}, {"data", json { {"type", "response.completed"}, {"response", json { {"id", oai_resp_id}, {"object", "response"}, {"created_at", t}, {"status", "completed"}, {"model", oaicompat_model}, {"output", output}, {"usage", json { {"input_tokens", n_prompt_tokens}, {"output_tokens", n_decoded}, {"total_tokens", n_decoded + n_prompt_tokens}, {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }}, }} }}, }} }); if (stats.is_set()) { server_sent_events.back().at("data")["timings"] = stats.to_json(); } return server_sent_events; } json server_task_result_cmpl_final::to_json_oaicompat_asr() { json event = json { {"type", "transcript.text.done"}, {"text", oaicompat_msg.content}, {"usage", json { {"type", "tokens"}, {"input_tokens", n_prompt_tokens}, {"output_tokens", n_decoded}, {"total_tokens", n_decoded + n_prompt_tokens}, {"input_tokens_details", json { {"cached_tokens", n_prompt_tokens_cache} }}, }}, }; return event; } json server_task_result_cmpl_final::to_json_anthropic() { std::string stop_reason = "max_tokens"; if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use"; } json content_blocks = json::array(); common_chat_msg msg; if (!oaicompat_msg.empty()) { msg = oaicompat_msg; } else { msg.role = "assistant"; msg.content = content.text; } // thinking block comes first (Anthropic extended thinking format) if (!msg.reasoning_content.empty()) { content_blocks.push_back({ {"type", "thinking"}, {"thinking", msg.reasoning_content}, {"signature", ""} // empty signature for local models (no cryptographic verification) }); } if (!msg.content.empty()) { content_blocks.push_back({ {"type", "text"}, {"text", msg.content} }); } for (const auto & tool_call : msg.tool_calls) { json tool_use_block = { {"type", "tool_use"}, {"id", tool_call.id}, {"name", tool_call.name} }; try { tool_use_block["input"] = json::parse(tool_call.arguments); } catch (const std::exception &) { tool_use_block["input"] = json::object(); } content_blocks.push_back(tool_use_block); } json res = { {"id", oaicompat_cmpl_id}, {"type", "message"}, {"role", "assistant"}, {"content", content_blocks}, {"model", oaicompat_model}, {"stop_reason", stop_reason}, {"stop_sequence", stopping_word.empty() ? nullptr : json(stopping_word)}, {"usage", { {"cache_read_input_tokens", n_prompt_tokens_cache}, {"input_tokens", n_prompt_tokens - n_prompt_tokens_cache}, {"output_tokens", n_decoded} }} }; return res; } json server_task_result_cmpl_final::to_json_anthropic_stream() { json events = json::array(); std::string stop_reason = "max_tokens"; if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) { stop_reason = oaicompat_msg.tool_calls.empty() ? "end_turn" : "tool_use"; } bool has_thinking = !oaicompat_msg.reasoning_content.empty(); bool has_text = !oaicompat_msg.content.empty(); size_t num_tool_calls = oaicompat_msg.tool_calls.size(); // content block indices: thinking (0) -> text (0 or 1) -> tool_use (n+) size_t thinking_block_index = 0; size_t text_block_index = has_thinking ? 1 : 0; bool thinking_block_started = false; bool text_block_started = false; std::unordered_set tool_calls_started; for (const auto & diff : oaicompat_msg_diffs) { // handle thinking/reasoning content if (!diff.reasoning_content_delta.empty()) { if (!thinking_block_started) { events.push_back({ {"event", "content_block_start"}, {"data", { {"type", "content_block_start"}, {"index", thinking_block_index}, {"content_block", { {"type", "thinking"}, {"thinking", ""} }} }} }); thinking_block_started = true; } events.push_back({ {"event", "content_block_delta"}, {"data", { {"type", "content_block_delta"}, {"index", thinking_block_index}, {"delta", { {"type", "thinking_delta"}, {"thinking", diff.reasoning_content_delta} }} }} }); } // handle regular text content if (!diff.content_delta.empty()) { if (!text_block_started) { events.push_back({ {"event", "content_block_start"}, {"data", { {"type", "content_block_start"}, {"index", text_block_index}, {"content_block", { {"type", "text"}, {"text", ""} }} }} }); text_block_started = true; } events.push_back({ {"event", "content_block_delta"}, {"data", { {"type", "content_block_delta"}, {"index", text_block_index}, {"delta", { {"type", "text_delta"}, {"text", diff.content_delta} }} }} }); } // handle tool calls if (diff.tool_call_index != std::string::npos) { size_t content_block_index = (has_thinking ? 1 : 0) + (has_text ? 1 : 0) + diff.tool_call_index; if (tool_calls_started.find(diff.tool_call_index) == tool_calls_started.end()) { const auto & full_tool_call = oaicompat_msg.tool_calls[diff.tool_call_index]; events.push_back({ {"event", "content_block_start"}, {"data", { {"type", "content_block_start"}, {"index", content_block_index}, {"content_block", { {"type", "tool_use"}, {"id", full_tool_call.id}, {"name", full_tool_call.name} }} }} }); tool_calls_started.insert(diff.tool_call_index); } if (!diff.tool_call_delta.arguments.empty()) { events.push_back({ {"event", "content_block_delta"}, {"data", { {"type", "content_block_delta"}, {"index", content_block_index}, {"delta", { {"type", "input_json_delta"}, {"partial_json", diff.tool_call_delta.arguments} }} }} }); } } } // close content blocks in order if (has_thinking) { // Anthropic API requires a signature_delta before closing thinking blocks // We use an empty signature since we can't generate a cryptographic signature for local models events.push_back({ {"event", "content_block_delta"}, {"data", { {"type", "content_block_delta"}, {"index", thinking_block_index}, {"delta", { {"type", "signature_delta"}, {"signature", ""} }} }} }); events.push_back({ {"event", "content_block_stop"}, {"data", { {"type", "content_block_stop"}, {"index", thinking_block_index} }} }); } if (has_text) { events.push_back({ {"event", "content_block_stop"}, {"data", { {"type", "content_block_stop"}, {"index", text_block_index} }} }); } for (size_t i = 0; i < num_tool_calls; i++) { size_t content_block_index = (has_thinking ? 1 : 0) + (has_text ? 1 : 0) + i; events.push_back({ {"event", "content_block_stop"}, {"data", { {"type", "content_block_stop"}, {"index", content_block_index} }} }); } events.push_back({ {"event", "message_delta"}, {"data", { {"type", "message_delta"}, {"delta", { {"stop_reason", stop_reason}, {"stop_sequence", stopping_word.empty() ? nullptr : json(stopping_word)} }}, {"usage", { {"output_tokens", n_decoded} }} }} }); events.push_back({ {"event", "message_stop"}, {"data", { {"type", "message_stop"} }} }); return events; } // // server_task_result_cmpl_partial // void server_task_result_cmpl_partial::update(task_result_state & state) { is_updated = true; if (is_begin) { return; // begin marker only flushes headers, skip parsing } state.update_chat_msg(content, true, oaicompat_msg_diffs); // Copy current state for use in to_json_*() (reflects state BEFORE this chunk) thinking_block_started = state.thinking_block_started; text_block_started = state.text_block_started; oai_resp_created = state.oai_resp_created; oai_resp_id = state.oai_resp_id; oai_resp_reasoning_id = state.oai_resp_reasoning_id; oai_resp_message_id = state.oai_resp_message_id; oai_resp_fc_id = state.oai_resp_fc_id; // track if the accumulated message has any reasoning content anthropic_has_reasoning = !state.chat_msg.reasoning_content.empty(); if (res_type == TASK_RESPONSE_TYPE_OAI_RESP && !state.oai_resp_created && (is_progress || n_decoded == 1)) { state.oai_resp_created = true; } // Pre-compute state updates based on diffs (for next chunk) for (const common_chat_msg_diff & diff : oaicompat_msg_diffs) { if (!diff.reasoning_content_delta.empty() && !state.thinking_block_started) { state.thinking_block_started = true; } if (!diff.content_delta.empty() && !state.text_block_started) { state.text_block_started = true; } if (!diff.tool_call_delta.name.empty()) { state.oai_resp_fc_id = diff.tool_call_delta.id; } } } json server_task_result_cmpl_partial::to_json() { GGML_ASSERT(is_updated && "update() must be called before to_json()"); if (is_begin) { return nullptr; // simply signal to HTTP handler to send the headers and status code } switch (res_type) { case TASK_RESPONSE_TYPE_NONE: return to_json_non_oaicompat(); case TASK_RESPONSE_TYPE_OAI_CMPL: return to_json_oaicompat(); case TASK_RESPONSE_TYPE_OAI_CHAT: return to_json_oaicompat_chat(); case TASK_RESPONSE_TYPE_OAI_RESP: return to_json_oaicompat_resp(); case TASK_RESPONSE_TYPE_OAI_ASR: return to_json_oaicompat_asr(); case TASK_RESPONSE_TYPE_ANTHROPIC: return to_json_anthropic(); default: GGML_ASSERT(false && "Invalid task_response_type"); } } json server_task_result_cmpl_partial::to_json_non_oaicompat() { // non-OAI-compat JSON json res = json { {"index", index}, {"content", content.text}, {"tokens", tokens}, {"stop", false}, {"id_slot", id_slot}, {"tokens_predicted", n_decoded}, {"tokens_evaluated", n_prompt_tokens}, }; // populate the timings object when needed (usually for the last response or with timings_per_token enabled) if (stats.is_set()) { res["timings"] = stats.to_json(); } if (is_progress) { res["prompt_progress"] = progress.to_json(); } if (!prob_output.probs.empty()) { res["completion_probabilities"] = completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs); } return res; } json server_task_result_cmpl_partial::to_json_oaicompat() { std::time_t t = std::time(0); json logprobs = json(nullptr); // OAI default to null if (prob_output.probs.size() > 0) { logprobs = json{ {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)}, }; } json res = json { {"choices", json::array({ json{ {"text", content.text}, {"index", index}, {"logprobs", logprobs}, {"finish_reason", nullptr}, } })}, {"created", t}, {"model", oaicompat_model}, {"system_fingerprint", std::string(llama_build_info())}, {"object", "text_completion"}, {"id", oaicompat_cmpl_id} }; // extra fields for debugging purposes if (verbose) { res["__verbose"] = to_json_non_oaicompat(); } if (stats.is_set()) { res["timings"] = stats.to_json(); } if (is_progress) { res["prompt_progress"] = progress.to_json(); } return res; } json server_task_result_cmpl_partial::to_json_oaicompat_chat() { bool first = n_decoded == 1; std::time_t t = std::time(0); json choices; std::vector deltas; auto add_delta = [&](const json & delta) { deltas.push_back({ {"choices", json::array({ json { {"finish_reason", nullptr}, {"index", index}, {"delta", delta}, }, })}, {"created", t}, {"id", oaicompat_cmpl_id}, {"model", oaicompat_model}, {"system_fingerprint", std::string(llama_build_info())}, {"object", "chat.completion.chunk"}, }); }; // We have to send an initial update to conform to openai behavior if (first || is_progress) { add_delta({ {"role", "assistant"}, {"content", nullptr}, }); } for (const auto & diff : oaicompat_msg_diffs) { add_delta(server_chat_msg_diff_to_json_oaicompat(diff)); } if (!deltas.empty()) { auto & last_json = deltas[deltas.size() - 1]; GGML_ASSERT(last_json.at("choices").size() >= 1); if (prob_output.probs.size() > 0) { last_json.at("choices").at(0)["logprobs"] = json { {"content", completion_token_output::probs_vector_to_json({prob_output}, post_sampling_probs)}, }; } if (stats.is_set()) { last_json["timings"] = stats.to_json(); } if (is_progress) { last_json["prompt_progress"] = progress.to_json(); } } return deltas; } json server_task_result_cmpl_partial::to_json_oaicompat_resp() { std::vector events; if (!oai_resp_created) { events.push_back(json { {"event", "response.created"}, {"data", json { {"type", "response.created"}, {"response", json { {"id", oai_resp_id}, {"object", "response"}, {"status", "in_progress"}, }}, }}, }); events.push_back(json { {"event", "response.in_progress"}, {"data", json { {"type", "response.in_progress"}, {"response", json { {"id", oai_resp_id}, {"object", "response"}, {"status", "in_progress"}, }}, }}, }); } else if (is_progress) { events.push_back(json { {"event", "response.in_progress"}, {"data", json { {"type", "response.in_progress"}, {"response", json { {"id", oai_resp_id}, {"object", "response"}, {"status", "in_progress"}, }}, }}, }); } for (const common_chat_msg_diff & diff : oaicompat_msg_diffs) { if (!diff.reasoning_content_delta.empty()) { if (!thinking_block_started) { events.push_back(json { {"event", "response.output_item.added"}, {"data", json { {"type", "response.output_item.added"}, {"output_index", 0}, {"item", json { {"id", oai_resp_reasoning_id}, {"summary", json::array()}, {"type", "reasoning"}, {"content", json::array()}, {"encrypted_content", ""}, {"status", "in_progress"}, }}, }}, }); thinking_block_started = true; } events.push_back(json { {"event", "response.reasoning_text.delta"}, {"data", json { {"type", "response.reasoning_text.delta"}, {"output_index", 0}, {"content_index", 0}, {"delta", diff.reasoning_content_delta}, {"item_id", oai_resp_reasoning_id}, }}, }); } if (!diff.content_delta.empty()) { if (!text_block_started) { events.push_back(json { {"event", "response.output_item.added"}, {"data", json { {"type", "response.output_item.added"}, {"output_index", anthropic_has_reasoning ? 1 : 0}, {"item", json { {"content", json::array()}, {"id", oai_resp_message_id}, {"role", "assistant"}, {"status", "in_progress"}, {"type", "message"}, }}, }}, }); events.push_back(json { {"event", "response.content_part.added"}, {"data", json { {"type", "response.content_part.added"}, {"item_id", oai_resp_message_id}, {"output_index", anthropic_has_reasoning ? 1 : 0}, {"content_index", 0}, {"part", json { {"type", "output_text"}, {"text", ""}, }}, }}, }); text_block_started = true; } events.push_back(json { {"event", "response.output_text.delta"}, {"data", json { {"type", "response.output_text.delta"}, {"item_id", oai_resp_message_id}, {"output_index", anthropic_has_reasoning ? 1 : 0}, {"content_index", 0}, {"delta", diff.content_delta}, }}, }); } if (!diff.tool_call_delta.name.empty()) { events.push_back(json { {"event", "response.output_item.added"}, {"data", json { {"type", "response.output_item.added"}, {"output_index", (anthropic_has_reasoning ? 1 : 0) + (text_block_started ? 1 : 0) + diff.tool_call_index}, {"item", json { {"id", "fc_" + diff.tool_call_delta.id}, {"arguments", ""}, {"call_id", "call_" + diff.tool_call_delta.id}, {"name", diff.tool_call_delta.name}, {"type", "function_call"}, {"status", "in_progress"}, }}, }}, }); oai_resp_fc_id = diff.tool_call_delta.id; } if (!diff.tool_call_delta.arguments.empty()) { events.push_back(json { {"event", "response.function_call_arguments.delta"}, {"data", json { {"type", "response.function_call_arguments.delta"}, {"delta", diff.tool_call_delta.arguments}, {"item_id", "fc_" + oai_resp_fc_id}, {"output_index", (anthropic_has_reasoning ? 1 : 0) + (text_block_started ? 1 : 0) + diff.tool_call_index}, }}, }); } } if (!events.empty()) { json & data = events.back().at("data"); if (stats.is_set()) { data["timings"] = stats.to_json(); } if (is_progress) { data["prompt_progress"] = progress.to_json(); } } return events; } json server_task_result_cmpl_partial::to_json_oaicompat_asr() { json event = json { {"type", "transcript.text.delta"}, {"delta", content.text}, }; return event; } json server_task_result_cmpl_partial::to_json_anthropic() { json events = json::array(); bool first = (n_decoded == 1); // use member variables to track block state across streaming calls // (anthropic_thinking_block_started, anthropic_text_block_started) if (first) { events.push_back({ {"event", "message_start"}, {"data", { {"type", "message_start"}, {"message", { {"id", oaicompat_cmpl_id}, {"type", "message"}, {"role", "assistant"}, {"content", json::array()}, {"model", oaicompat_model}, {"stop_reason", nullptr}, {"stop_sequence", nullptr}, {"usage", { {"cache_read_input_tokens", n_prompt_tokens_cache}, {"input_tokens", n_prompt_tokens - n_prompt_tokens_cache}, {"output_tokens", 0} }} }} }} }); } // content block indices: thinking (0) -> text (0 or 1) -> tool_use (n+) size_t thinking_block_index = 0; // use anthropic_has_reasoning (set in update()) to know if ANY reasoning was generated size_t text_block_index = anthropic_has_reasoning ? 1 : 0; // use local copies of streaming state (copied from task_result_state in update()) // these reflect the state BEFORE this chunk was processed bool thinking_started = thinking_block_started; bool text_started = text_block_started; for (const auto & diff : oaicompat_msg_diffs) { // handle thinking/reasoning content if (!diff.reasoning_content_delta.empty()) { if (!thinking_started) { events.push_back({ {"event", "content_block_start"}, {"data", { {"type", "content_block_start"}, {"index", thinking_block_index}, {"content_block", { {"type", "thinking"}, {"thinking", ""} }} }} }); thinking_started = true; } events.push_back({ {"event", "content_block_delta"}, {"data", { {"type", "content_block_delta"}, {"index", thinking_block_index}, {"delta", { {"type", "thinking_delta"}, {"thinking", diff.reasoning_content_delta} }} }} }); } // handle regular text content if (!diff.content_delta.empty()) { if (!text_started) { events.push_back({ {"event", "content_block_start"}, {"data", { {"type", "content_block_start"}, {"index", text_block_index}, {"content_block", { {"type", "text"}, {"text", ""} }} }} }); text_started = true; } events.push_back({ {"event", "content_block_delta"}, {"data", { {"type", "content_block_delta"}, {"index", text_block_index}, {"delta", { {"type", "text_delta"}, {"text", diff.content_delta} }} }} }); } // handle tool calls if (diff.tool_call_index != std::string::npos) { // use anthropic_has_reasoning for thinking block count (persists across calls) size_t content_block_index = (anthropic_has_reasoning ? 1 : 0) + (text_started ? 1 : 0) + diff.tool_call_index; if (!diff.tool_call_delta.name.empty()) { events.push_back({ {"event", "content_block_start"}, {"data", { {"type", "content_block_start"}, {"index", content_block_index}, {"content_block", { {"type", "tool_use"}, {"id", diff.tool_call_delta.id}, {"name", diff.tool_call_delta.name} }} }} }); } if (!diff.tool_call_delta.arguments.empty()) { events.push_back({ {"event", "content_block_delta"}, {"data", { {"type", "content_block_delta"}, {"index", content_block_index}, {"delta", { {"type", "input_json_delta"}, {"partial_json", diff.tool_call_delta.arguments} }} }} }); } } } return events; } // // server_task_result_embd // json server_task_result_embd::to_json() { return res_type == TASK_RESPONSE_TYPE_OAI_EMBD ? to_json_oaicompat() : to_json_non_oaicompat(); } json server_task_result_embd::to_json_non_oaicompat() { return json { {"index", index}, {"embedding", embedding}, }; } json server_task_result_embd::to_json_oaicompat() { return json { {"index", index}, {"embedding", embedding[0]}, {"tokens_evaluated", n_tokens}, }; } // // server_task_result_rerank // json server_task_result_rerank::to_json() { return json { {"index", index}, {"score", score}, {"tokens_evaluated", n_tokens}, }; } // // server_task_result_tts // json server_task_result_tts::to_json() { return json { {"sample_rate", sample_rate}, {"n_bytes", audio.size()}, {"final", final}, }; } // // server_task_result_decision // json server_task_result_decision::to_json() { return json { {"index", index}, {"scores", scores}, {"tokens_evaluated", n_tokens}, }; } // // server_task_result_error // json server_task_result_error::to_json() { json res = format_error_response(err_msg, err_type); if (err_type == ERROR_TYPE_EXCEED_CONTEXT_SIZE) { res["n_prompt_tokens"] = n_prompt_tokens; res["n_ctx"] = n_ctx; } return res; } // // server_task_result_metrics // json server_task_result_slots::to_json() { return slots_data; } json server_task_result_metrics::to_json() { // not used, /metrics renders prometheus text via to_metrics() return json{}; } // metrics definition: https://prometheus.io/docs/practices/naming/#metric-names std::string server_task_result_metrics::to_metrics() { const std::vector counters = { { "prompt_tokens_total", "Number of prompt tokens processed, excluding cached tokens", (double) metrics.prompt.count }, { "prompt_tokens_cached_total", "Number of prompt tokens reused from the cache", (double) metrics.n_prompt_cached }, { "prompt_seconds_total", "Total time spent processing prompts", metrics.prompt.time / 1.e6 }, { "tokens_predicted_total", "Number of generation tokens processed", (double) metrics.predict.count }, { "tokens_predicted_seconds_total", "Total time spent generating tokens", metrics.predict.time / 1.e6 }, { "n_decode_total", "Total number of llama_decode() calls, excluding speculative decoding and multimodal decoding", (double) metrics.n_decode }, { "n_tokens_max", "Largest observed sequence length (prompt + generation)", (double) metrics.n_tokens_max }, { "spec_decode_num_draft_tokens_total", "Speculative: Total draft tokens generated", (double) metrics.n_draft_tokens }, { "spec_decode_num_accepted_tokens_total", "Speculative: Total draft tokens accepted by the target model", (double) metrics.n_draft_accepted }, { "spec_decode_num_drafts_total", "Speculative: Total speculative decoding verification steps", (double) metrics.n_draft_verif_steps }, }; const std::vector gauges = { { "prompt_tokens_seconds", "Average prompt throughput in tokens/s", metrics.prompt_bucket.n_per_second() }, { "predicted_tokens_seconds", "Average generation throughput in tokens/s", metrics.predict_bucket.n_per_second() }, { "requests_processing", "Number of requests processing", (double) n_processing_slots }, { "requests_deferred", "Number of requests deferred", (double) n_tasks_deferred }, { "n_busy_slots_per_decode", "Average number of busy slots per llama_decode() call", (double) metrics.n_busy_slots / std::max((double) metrics.n_decode, 1.0) }, }; std::stringstream prometheus; auto add_items = [&prometheus](const char * type, const std::vector & items) { for (const auto & item : items) { prometheus << "# HELP llamacpp:" << item.name << " " << item.description << "\n" << "# TYPE llamacpp:" << item.name << " " << type << "\n" << "llamacpp:" << item.name << " " << item.value << "\n"; } }; add_items("counter", counters); add_items("gauge", gauges); // labeled counter: one time series per draft position if (!metrics.n_accepted_per_pos.empty()) { prometheus << "# HELP llamacpp:spec_decode_num_accepted_tokens_per_pos_total" " Accepted tokens per draft position\n" << "# TYPE llamacpp:spec_decode_num_accepted_tokens_per_pos_total counter\n"; for (size_t i = 0; i < metrics.n_accepted_per_pos.size(); i++) { prometheus << "llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\"" << i << "\"} " << metrics.n_accepted_per_pos[i] << "\n"; } } return prometheus.str(); } // // server_task_result_slot_save_load // json server_task_result_slot_save_load::to_json() { if (is_save) { return json { { "id_slot", id_slot }, { "filename", filename }, { "n_saved", n_tokens }, { "n_written", n_bytes }, { "timings", { { "save_ms", t_ms } }}, }; } return json { { "id_slot", id_slot }, { "filename", filename }, { "n_restored", n_tokens }, { "n_read", n_bytes }, { "timings", { { "restore_ms", t_ms } }}, }; } // // server_task_result_slot_erase // json server_task_result_slot_erase::to_json() { return json { { "id_slot", id_slot }, { "n_erased", n_erased }, }; } // // server_task_result_get_lora // json server_task_result_get_lora::to_json() { json result = json::array(); for (size_t i = 0; i < loras.size(); ++i) { auto & lora = loras[i]; json entry = { {"id", i}, {"path", lora.info.path}, {"scale", lora.info.scale}, {"task_name", lora.info.task_name}, {"prompt_prefix", lora.info.prompt_prefix}, }; if (!lora.alora_invocation_tokens.empty()) { entry["alora_invocation_string"] = lora.alora_invocation_string; entry["alora_invocation_tokens"] = lora.alora_invocation_tokens; } result.push_back(std::move(entry)); } return result; } // // server_task_result_apply_lora // json server_task_result_apply_lora::to_json() { return json {{ "success", true }}; } // // server_prompt_cache // size_t server_prompt_cache::size() const { size_t res = 0; for (const auto & state : states) { res += state.size(); } return res; } size_t server_prompt_cache::n_tokens() const { size_t res = 0; for (const auto & state : states) { res += state.prompt.n_tokens(); } return res; } server_prompt_cache_state * server_prompt_cache::alloc(const server_prompt & prompt, size_t state_size_tgt, size_t state_size_dft) { // first check if the current state is contained fully in the cache for (auto it = states.begin(); it != states.end(); ++it) { const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens); if (cur_lcp_len == (int) prompt.tokens.size()) { SRV_TRC("%s", " - prompt is already in the cache, skipping\n"); return nullptr; } } // calculate checkpoints size to see if it will fit with the prompt size_t checkpoints_size = 0; for (const auto & ckpt : prompt.checkpoints) { checkpoints_size += ckpt.size(); } const size_t state_size_new = state_size_tgt + state_size_dft + checkpoints_size; // skip over-limit entries to avoid disturbing the cache if (limit_size > 0 && state_size_new > limit_size) { SRV_WRN(" - prompt state size %.3f MiB exceeds cache size limit %.3f MiB, skipping\n", state_size_new / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0)); return nullptr; } // remove any cached prompts that are fully contained in the current prompt for (auto it = states.begin(); it != states.end();) { const int len = it->prompt.tokens.get_common_prefix(prompt.tokens); if (len == (int) it->prompt.tokens.size()) { SRV_TRC(" - removing obsolete cached prompt with length %d\n", len); it = states.erase(it); } else { ++it; } } if (limit_size > 0) { // make room before allocating the new vectors to avoid breaching the limit while (!states.empty() && size() + state_size_new > limit_size) { SRV_WRN(" - making room for prompt cache entry, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } std::vector state_data_tgt; std::vector state_data_dft; // check if we can allocate enough memory for the new state try { state_data_tgt.resize(state_size_tgt); state_data_dft.resize(state_size_dft); } catch (const std::bad_alloc & e) { SRV_ERR("failed to allocate memory for prompt cache state: %s\n", e.what()); limit_size = std::max(1, 0.4*size()); SRV_WRN(" - cache size limit reduced to %.3f MiB\n", limit_size / (1024.0 * 1024.0)); update(); return nullptr; } states.push_back({ /*.prompt =*/ { /*.tokens =*/ prompt.tokens.clone(), /*.checkpoints =*/ prompt.checkpoints, }, /*.data =*/ { /*.main =*/ std::move(state_data_tgt), /*.drft =*/ std::move(state_data_dft), }, }); return &states.back(); } bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot) { const int lcp_best = prompt.tokens.get_common_prefix(tokens_new); float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins float f_sim_best = float(lcp_best) / tokens_new.size(); SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); auto it_best = states.end(); // find the most similar cached prompt, that would also preserve the most context for (auto it = states.begin(); it != states.end(); ++it) { const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new); const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size(); const float f_sim_cur = float(lcp_cur) / tokens_new.size(); SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur); // don't trash large prompts if (f_keep_cur < 0.25f) { continue; } if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { f_keep_best = f_keep_cur; f_sim_best = f_sim_cur; it_best = it; } } if (it_best != states.end()) { SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); { auto & data = it_best->data.main; const size_t size = data.size(); const size_t n = llama_state_seq_set_data_ext(ctx_tgt, data.data(), size, id_slot, 0); if (n != size) { SRV_ERR("failed to restore state with size %zu\n", size); return false; } data.clear(); data.shrink_to_fit(); } { auto & data = it_best->data.drft; if (!data.empty()) { GGML_ASSERT(ctx_dft); const size_t size = data.size(); const size_t n = llama_state_seq_set_data_ext(ctx_dft, data.data(), size, id_slot, 0); if (n != size) { SRV_WRN("failed to restore state with size %zu\n", size); return false; } data.clear(); data.shrink_to_fit(); } } prompt = std::move(it_best->prompt); states.erase(it_best); } return true; } void server_prompt_cache::update() { if (limit_size > 0) { while (!states.empty() && size() > limit_size) { SRV_WRN(" - cache size limit reached, removing oldest entry (size = %.3f MiB)\n", states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } // average size per token const float size_per_token = std::max(1.0f, float(size()) / (std::max(1, n_tokens()))); // dynamically increase the token limit if it can fit in the memory limit const size_t limit_tokens_cur = limit_size > 0 ? std::max(limit_tokens, limit_size/size_per_token) : limit_tokens; if (limit_tokens > 0) { while (!states.empty() && n_tokens() > limit_tokens_cur) { SRV_WRN(" - cache token limit (%zu, est: %zu) reached, removing oldest entry (size = %.3f MiB)\n", limit_tokens, limit_tokens_cur, states.front().size() / (1024.0 * 1024.0)); states.pop_front(); } } SRV_TRC(" - cache state: %zu prompts, %.3f MiB (limits: %.3f MiB, %zu tokens, %zu est)\n", states.size(), size() / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0), limit_tokens, limit_tokens_cur); for (const auto & state : states) { SRV_TRC(" - prompt %p: %7d tokens, checkpoints: %2zu, %9.3f MiB\n", (const void *)&state, state.prompt.n_tokens(), state.prompt.checkpoints.size(), state.size() / (1024.0 * 1024.0)); } } // // server_prompt_disk_cache // // file = header + payload; the payload is a list of sections, each padded to 8 bytes: // tokens, checkpoint table, state size, state, the state of each checkpoint // hash_payload is FNV-1a over the 64-bit words of the payload, so a truncated or corrupt file is never used static constexpr uint32_t DISK_CACHE_MAGIC = 0x4443504c; // "LPCD" static constexpr uint32_t DISK_CACHE_VERSION = 1; static constexpr const char * DISK_CACHE_EXT = ".lpcd"; // keep this much space free on the file system static constexpr uint64_t DISK_CACHE_MIN_FREE = 1024ull*1024ull*1024ull; struct disk_cache_header { uint32_t magic; uint32_t version; uint64_t key_hash; uint64_t n_tokens; uint64_t n_ckpt; uint64_t size_payload; uint64_t hash_payload; }; struct disk_cache_ckpt { int64_t n_tokens; llama_pos pos_min; llama_pos pos_max; uint64_t size; }; static_assert(sizeof(disk_cache_header) == 48 && sizeof(disk_cache_ckpt) == 24, "disk cache layout"); static uint64_t disk_cache_pad(uint64_t n) { return (8 - n % 8) % 8; } static uint64_t disk_cache_hash_bytes(const void * data, size_t n) { uint64_t h = 0xcbf29ce484222325ull; for (size_t i = 0; i < n; i++) { h = (h ^ ((const uint8_t *) data)[i]) * 0x100000001b3ull; } return h; } static uint64_t disk_cache_payload_size(uint64_t n_tokens, const std::vector & ckpts, uint64_t n_state) { const auto padded = [](uint64_t n) { return n + disk_cache_pad(n); }; uint64_t res = padded(n_tokens*sizeof(llama_token)) + padded(ckpts.size()*sizeof(disk_cache_ckpt)) + sizeof(uint64_t) + padded(n_state); for (const auto & c : ckpts) { res += padded(c.size); } return res; } // sequential section IO with the running payload hash struct disk_cache_io { FILE * f = nullptr; uint64_t hash = 0xcbf29ce484222325ull; bool ok = true; ~disk_cache_io() { if (f) { fclose(f); } } void add_hash(const uint8_t * p, size_t size, const uint8_t * pad) { const size_t n_full = size - size % 8; for (size_t i = 0; i < n_full; i += 8) { uint64_t w; memcpy(&w, p + i, 8); hash = (hash ^ w) * 0x100000001b3ull; } if (size % 8) { uint64_t w; memcpy(&w, p + n_full, size % 8); memcpy((uint8_t *) &w + size % 8, pad, disk_cache_pad(size)); hash = (hash ^ w) * 0x100000001b3ull; } } void write(const void * src, size_t size) { static const uint8_t zero[8] = {}; const size_t n_pad = disk_cache_pad(size); ok = ok && fwrite(src, 1, size, f) == size && fwrite(zero, 1, n_pad, f) == n_pad; add_hash((const uint8_t *) src, size, zero); } void read(void * dst, size_t size) { uint8_t pad[8]; const size_t n_pad = disk_cache_pad(size); ok = ok && fread(dst, 1, size, f) == size && fread(pad, 1, n_pad, f) == n_pad; if (ok) { add_hash((const uint8_t *) dst, size, pad); } } // the data is in our buffers, the cached pages are waste (see llama_file::drop_cache) void drop_cache() { #if defined(__linux__) posix_fadvise(fileno(f), 0, 0, POSIX_FADV_DONTNEED); #endif } }; // open a file and read everything up to the state data: 1 = entry of this key, 0 = other key or version, -1 = bad file static int disk_cache_open(const std::string & path, uint64_t key_hash, disk_cache_io & io, disk_cache_header & hdr, llama_tokens & tokens, std::vector & ckpts, uint64_t & n_state) { std::error_code ec; const uint64_t size = std::filesystem::file_size(path, ec); io.f = ec ? nullptr : fopen(path.c_str(), "rb"); if (io.f == nullptr || fread(&hdr, 1, sizeof(hdr), io.f) != sizeof(hdr) || hdr.magic != DISK_CACHE_MAGIC) { return -1; } if (hdr.version != DISK_CACHE_VERSION) { return 0; } if (size != sizeof(hdr) + hdr.size_payload || hdr.n_tokens > size/sizeof(llama_token) || hdr.n_ckpt > size/sizeof(disk_cache_ckpt)) { return -1; } if (hdr.key_hash != key_hash) { return 0; } tokens.resize(hdr.n_tokens); ckpts.resize(hdr.n_ckpt); io.read(tokens.data(), tokens.size()*sizeof(llama_token)); io.read(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt)); io.read(&n_state, sizeof(n_state)); if (!io.ok || n_state > size) { return -1; } for (const auto & c : ckpts) { if (c.size > size) { return -1; } } return disk_cache_payload_size(hdr.n_tokens, ckpts, n_state) == hdr.size_payload ? 1 : -1; } static void disk_cache_touch(const std::string & path) { std::error_code ec; std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec); } server_prompt_disk_cache::server_prompt_disk_cache(const std::string & dir, int32_t limit_size_mib, const std::string & key) : dir(dir) { key_hash = disk_cache_hash_bytes(key.data(), key.size()); limit_size = 1024ull*1024ull*std::max(0, limit_size_mib); size_t n_other = 0; size_t n_bad = 0; try { std::filesystem::create_directories(dir); const auto now = std::filesystem::file_time_type::clock::now(); for (const auto & de : std::filesystem::directory_iterator(dir)) { const std::string path = de.path().string(); if (!de.is_regular_file()) { continue; } // left by a writer that was killed if (path.find(std::string(DISK_CACHE_EXT) + ".tmp") != std::string::npos) { if (now - de.last_write_time() > std::chrono::hours(1)) { std::filesystem::remove(de.path()); } continue; } if (de.path().extension() != DISK_CACHE_EXT) { continue; } disk_cache_io io; disk_cache_header hdr; server_prompt_disk_entry e; std::vector ckpts; uint64_t n_state = 0; const int res = disk_cache_open(path, key_hash, io, hdr, e.tokens, ckpts, n_state); if (res < 0) { SRV_WRN("disk cache: '%s' is truncated or corrupt, removing it\n", path.c_str()); std::filesystem::remove(de.path()); n_bad++; continue; } if (res == 0) { n_other++; continue; } e.path = path; for (const auto & c : ckpts) { e.ckpt_n_tokens.push_back(c.n_tokens); } entries.push_back(std::move(e)); } } catch (const std::exception & err) { SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what()); } SRV_INF("disk cache: '%s', %zu prompts of this model, %zu other files, %zu bad files removed, limit %.0f MiB\n", dir.c_str(), entries.size(), n_other, n_bad, limit_size / (1024.0 * 1024.0)); } size_t server_prompt_disk_cache::n_reuse(const llama_tokens & tokens, const std::vector & ckpt_n_tokens, const llama_tokens & tokens_new, bool can_trim) { size_t lcp = 0; while (lcp < tokens.size() && lcp < tokens_new.size() && tokens[lcp] == tokens_new[lcp]) { lcp++; } if (lcp == tokens.size() || can_trim) { return lcp; } size_t res = 0; for (const int64_t n : ckpt_n_tokens) { if (n <= (int64_t) lcp) { res = std::max(res, (size_t) n); } } return res; } int server_prompt_disk_cache::find(const llama_tokens & tokens_new, size_t n_have, size_t n_min, bool can_trim) const { int best = -1; size_t n_best = 0; for (size_t i = 0; i < entries.size(); i++) { const size_t n = n_reuse(entries[i].tokens, entries[i].ckpt_n_tokens, tokens_new, can_trim); if (n >= n_have + n_min && n > n_best) { best = (int) i; n_best = n; } } return best; } bool server_prompt_disk_cache::has(const llama_tokens & tokens) { for (const auto & e : entries) { std::error_code ec; if (e.tokens == tokens && std::filesystem::exists(e.path, ec)) { disk_cache_touch(e.path); return true; } } return false; } void server_prompt_disk_cache::remove(int i) { std::error_code ec; std::filesystem::remove(entries[i].path, ec); entries.erase(entries.begin() + i); } bool server_prompt_disk_cache::load(int i, server_prompt & prompt, std::vector & data, bool has_mtmd) { const std::string path = entries[i].path; disk_cache_io io; disk_cache_header hdr; llama_tokens tokens; std::vector ckpts; uint64_t n_state = 0; std::list checkpoints; bool ok = disk_cache_open(path, key_hash, io, hdr, tokens, ckpts, n_state) == 1 && tokens == entries[i].tokens; if (ok) { try { data.resize(n_state); io.read(data.data(), data.size()); for (const auto & c : ckpts) { auto & cur = checkpoints.emplace_back(); cur.id_task = -1; // not created by a task: may fail to load, then the prompt is re-processed cur.n_tokens = c.n_tokens; cur.pos_min = c.pos_min; cur.pos_max = c.pos_max; cur.data_tgt.resize(c.size); io.read(cur.data_tgt.data(), cur.data_tgt.size()); } } catch (const std::bad_alloc &) { SRV_WRN("disk cache: out of memory reading '%s'\n", path.c_str()); data.clear(); return false; } ok = io.ok && io.hash == hdr.hash_payload; io.drop_cache(); } if (!ok) { SRV_WRN("disk cache: '%s' is missing, truncated or corrupt, removing it\n", path.c_str()); data.clear(); remove(i); return false; } disk_cache_touch(path); prompt.tokens = server_tokens(tokens, has_mtmd); prompt.checkpoints = std::move(checkpoints); return true; } bool server_prompt_disk_cache::save(const server_prompt & prompt, const std::vector & data) { const llama_tokens & tokens = prompt.tokens.get_tokens(); std::vector ckpts; for (const auto & c : prompt.checkpoints) { ckpts.push_back({ c.n_tokens, c.pos_min, c.pos_max, c.data_tgt.size() }); } disk_cache_header hdr = { /*.magic =*/ DISK_CACHE_MAGIC, /*.version =*/ DISK_CACHE_VERSION, /*.key_hash =*/ key_hash, /*.n_tokens =*/ tokens.size(), /*.n_ckpt =*/ ckpts.size(), /*.size_payload =*/ disk_cache_payload_size(tokens.size(), ckpts, data.size()), /*.hash_payload =*/ 0, }; const uint64_t size_file = sizeof(hdr) + hdr.size_payload; char name[64]; snprintf(name, sizeof(name), "%016llx-%016llx", (unsigned long long) key_hash, (unsigned long long) disk_cache_hash_bytes(tokens.data(), tokens.size()*sizeof(llama_token))); const std::string path = (std::filesystem::path(dir) / name).string() + DISK_CACHE_EXT; const std::string path_tmp = path + ".tmp"; // entries contained in the new one are obsolete: removed once it is written, not counted for the limit const auto is_obsolete = [&](const server_prompt_disk_entry & e) { return e.path != path && e.tokens.size() <= tokens.size() && std::equal(e.tokens.begin(), e.tokens.end(), tokens.begin()); }; std::vector obsolete; for (const auto & e : entries) { if (is_obsolete(e)) { obsolete.push_back(e.path); } } if (limit_size > 0 && size_file > limit_size) { SRV_WRN("disk cache: prompt state of %.1f MiB exceeds the limit of %.1f MiB, not saved\n", size_file / (1024.0 * 1024.0), limit_size / (1024.0 * 1024.0)); return false; } try { // evict the least recently used files of every model in dir struct file_info { std::filesystem::path path; uint64_t size; std::filesystem::file_time_type time; }; std::vector files; uint64_t size_total = 0; for (const auto & de : std::filesystem::directory_iterator(dir)) { if (de.is_regular_file() && de.path().extension() == DISK_CACHE_EXT && de.path().string() != path && std::find(obsolete.begin(), obsolete.end(), de.path().string()) == obsolete.end()) { files.push_back({ de.path(), de.file_size(), de.last_write_time() }); size_total += files.back().size; } } std::sort(files.begin(), files.end(), [](const file_info & a, const file_info & b) { return a.time < b.time; }); for (size_t i = 0; limit_size > 0 && size_total + size_file > limit_size && i < files.size(); i++) { SRV_INF("disk cache: size limit, removing '%s' (%.1f MiB)\n", files[i].path.string().c_str(), files[i].size / (1024.0 * 1024.0)); std::filesystem::remove(files[i].path); size_total -= files[i].size; for (size_t j = 0; j < entries.size(); j++) { if (entries[j].path == files[i].path.string()) { entries.erase(entries.begin() + j); break; } } } const uint64_t avail = std::filesystem::space(dir).available; if (avail < size_file + DISK_CACHE_MIN_FREE) { SRV_WRN("disk cache: only %.1f MiB free in '%s', prompt state of %.1f MiB not saved\n", avail / (1024.0 * 1024.0), dir.c_str(), size_file / (1024.0 * 1024.0)); return false; } } catch (const std::exception & err) { SRV_WRN("disk cache: error reading '%s': %s\n", dir.c_str(), err.what()); return false; } bool ok; { disk_cache_io io; io.f = fopen(path_tmp.c_str(), "wb"); ok = io.f != nullptr && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr); if (ok) { const uint64_t n_state = data.size(); io.write(tokens.data(), tokens.size()*sizeof(llama_token)); io.write(ckpts.data(), ckpts.size()*sizeof(disk_cache_ckpt)); io.write(&n_state, sizeof(n_state)); io.write(data.data(), data.size()); for (const auto & c : prompt.checkpoints) { io.write(c.data_tgt.data(), c.data_tgt.size()); } hdr.hash_payload = io.hash; ok = io.ok && fseek(io.f, 0, SEEK_SET) == 0 && fwrite(&hdr, 1, sizeof(hdr), io.f) == sizeof(hdr) && fflush(io.f) == 0; #if defined(__linux__) // write back now, so the pages can be dropped and the rename never exposes missing data ok = ok && fdatasync(fileno(io.f)) == 0; io.drop_cache(); #endif } if (io.f != nullptr && fclose(io.f) != 0) { ok = false; } io.f = nullptr; if (!ok) { SRV_WRN("disk cache: failed to write '%s': %s, not saved\n", path_tmp.c_str(), strerror(errno)); } } std::error_code ec; if (ok) { std::filesystem::rename(path_tmp, path, ec); ok = !ec; if (!ok) { SRV_WRN("disk cache: failed to rename '%s': %s, not saved\n", path_tmp.c_str(), ec.message().c_str()); } } if (!ok) { std::filesystem::remove(path_tmp, ec); return false; } for (size_t i = 0; i < entries.size(); ) { const auto & e = entries[i]; if (e.path == path) { entries.erase(entries.begin() + i); } else if (is_obsolete(e)) { SRV_INF("disk cache: removing '%s', a prefix of the new prompt\n", e.path.c_str()); remove((int) i); } else { i++; } } server_prompt_disk_entry e; e.path = path; e.tokens = tokens; for (const auto & c : ckpts) { e.ckpt_n_tokens.push_back(c.n_tokens); } entries.push_back(std::move(e)); return true; }