* model : allow loading external PLE files
Assisted-by: Codex
* convert : add separate PLE exports
Assisted-by: Codex
* tests : fix PLE export test lint
Assisted-by: Codex
* model : make --ple an alias of --shard, allow multiple shard locations
Reuse the multi-shard loading path for external PLE files instead of a
separate option.
* --ple is an alias of --shard: both take a comma separated list and may
be given more than once, and the shards may live anywhere
* metadata comes from the main file; numbered shards are still
auto-discovered and strictly validated
* explicit files outside the discovered set are appended as extra shards,
exempt from split.no and split.count checks
* -isnm/--ignore-shard-num-mismatch loads only the given files, in order,
with no auto-discovery and no split metadata validation
* ple_path, path_ple and llama_model_loader::add_ple_file are removed;
common_model_load() uses llama_model_load_from_splits for two or more
paths
* common/fit.{h,cpp} take the shard path vector, so --fit and the router
memory measurement load every shard
Assisted-by: GPT-6-Astra
* rm -isnm
* make sure to dedup paths
* rm test_convert_ple
* address comments
* allow ple shard to override tensor
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
111 lines
5.3 KiB
C++
111 lines
5.3 KiB
C++
#pragma once
|
|
|
|
#include "ggml.h"
|
|
#include "llama.h"
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
enum common_params_fit_status {
|
|
COMMON_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit
|
|
COMMON_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit
|
|
COMMON_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path
|
|
};
|
|
|
|
// a second model that shares the devices of the main model, e.g. a draft model
|
|
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
|
|
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
|
|
struct common_fit_extra_model {
|
|
std::vector<std::string> paths_model;
|
|
llama_model_params * mparams;
|
|
llama_context_params * cparams;
|
|
bool shares_model;
|
|
};
|
|
|
|
// fits mparams and cparams to free device memory (assumes system memory is unlimited)
|
|
// - returns true if the parameters could be successfully modified to fit device memory
|
|
// - this function is NOT thread safe because it modifies the global llama logger state
|
|
// - only parameters that have the same value as in llama_default_model_params are modified
|
|
// with the exception of the context size which is modified if and only if equal to 0
|
|
common_params_fit_status common_fit_params(
|
|
const std::vector<std::string> & paths_model,
|
|
llama_model_params * mparams,
|
|
llama_context_params * cparams,
|
|
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
|
|
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
|
|
size_t * margins, // margins of memory to leave per device in bytes
|
|
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
|
|
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
|
|
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log
|
|
|
|
// print estimated memory to stdout
|
|
void common_fit_print(
|
|
const std::vector<std::string> & paths_model,
|
|
llama_model_params * mparams,
|
|
llama_context_params * cparams);
|
|
|
|
void common_memory_breakdown_print(const llama_context * ctx);
|
|
|
|
// one row of the memory breakdown: a device, the host or another buffer type
|
|
struct common_memory_breakdown_row {
|
|
enum kind_t { DEVICE, HOST, BUFFER_TYPE };
|
|
|
|
kind_t kind = DEVICE;
|
|
std::string name; // device or buffer type name, "Host" for the host row
|
|
std::string description; // device description, empty otherwise
|
|
int64_t total = 0; // device memory in bytes, 0 unless kind == DEVICE
|
|
int64_t free = 0; // free device memory in bytes, 0 unless kind == DEVICE
|
|
size_t model = 0; // bytes held by this context, by kind of buffer
|
|
size_t context = 0;
|
|
size_t compute = 0;
|
|
};
|
|
|
|
// memory a model needs at the min and max context --fit may pick
|
|
struct common_memory_reports {
|
|
uint32_t n_ctx_train = 0;
|
|
uint32_t n_ctx_max = 0; // the largest context that fits the free device memory right now
|
|
uint32_t n_ctx_min = 0; // --fit-ctx (the smallest context --fit may pick), with the placement set in params
|
|
std::vector<common_memory_breakdown_row> info_max; // breakdown at n_ctx_max
|
|
std::vector<common_memory_breakdown_row> info_min; // breakdown at n_ctx_min
|
|
};
|
|
|
|
// rows of common_memory_breakdown_print: model devices first, then host
|
|
std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llama_context * ctx);
|
|
|
|
// load model + context with no_alloc and return the memory breakdown, params are used as-is
|
|
// n_ctx_train and n_ctx are optional outputs; throws on failure
|
|
std::vector<common_memory_breakdown_row> common_fit_measure(
|
|
const std::vector<std::string> & paths_model,
|
|
const llama_model_params * mparams,
|
|
const llama_context_params * cparams,
|
|
uint32_t * n_ctx_train,
|
|
uint32_t * n_ctx,
|
|
ggml_log_level log_level);
|
|
|
|
struct common_params;
|
|
|
|
// measure min and max without allocating, params may be modified as with --fit; throws on failure
|
|
common_memory_reports common_measure_memory(common_params & params);
|
|
|
|
struct common_device_memory_data {
|
|
int64_t total;
|
|
int64_t free;
|
|
size_t model;
|
|
size_t context;
|
|
size_t compute;
|
|
};
|
|
|
|
using common_device_memory_data_vec = std::vector<common_device_memory_data>;
|
|
|
|
// Load a model + context with no_alloc and return the per-device memory breakdown.
|
|
common_device_memory_data_vec common_get_device_memory_data(
|
|
const std::vector<std::string> & paths_model,
|
|
const llama_model_params * mparams,
|
|
const llama_context_params * cparams,
|
|
std::vector<ggml_backend_dev_t> & devs,
|
|
uint32_t & hp_ngl,
|
|
uint32_t & hp_ngl_skip,
|
|
uint32_t & hp_n_ctx_train,
|
|
uint32_t & hp_n_expert,
|
|
ggml_log_level log_level);
|