quantize: cap working memory size to avoid loading big tensors onto RAM (#27795)

2026-08-27 18:31:13 +02:00
parent cb300598d5
commit 732707dff2
5 changed files with 112 additions and 79 deletions
@@ -445,6 +445,7 @@ extern "C" {
        const struct llama_model_kv_override * kv_overrides;        // pointer to kv overrides
        const struct llama_model_tensor_override * tt_overrides;    // pointer to tensor overrides
        const int32_t * prune_layers;                               // pointer to layer indices to prune
+        size_t max_buf_size;                                        // max bytes of tensor rows kept in memory at once, 0 = default (8 GiB)
    } llama_model_quantize_params;

    typedef struct llama_logit_bias {