tests : speed up tensor data generation and model saving (#31145)
* tests : speed up tensor data generation and model saving Refactor save_models to create only the model, not a llama_context, which is not needed for saving. Replace the slow std::normal_distribution draw with a splitmix64 + box-muller normal generator. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) * tests : use a shared pool of normal draws for tensor data Generate a pool of normal draws once with std::mt19937 and std::normal_distribution, then have every tensor copy a cyclic slice of it instead of running a per-tensor box-muller. The name-keyed slice selection keeps the draw deterministic. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) * tests : log the size of saved models Report the produced file size in bytes and MB after each save. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) * tests : simplify the model save log Put the model name, target and size on one line per saved model. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB)
This commit is contained in:
1 parent
922f53ed33
commit
8542e94350
1 file changed
+55
-18
+55
-18
@@ -8,6 +8,7 @@
|
||||
#include "llama-cpp.h"
|
||||
#include "../src/llama-ext.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cinttypes>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
@@ -54,12 +55,34 @@ struct tensor_data_params {
|
||||
// fills the values of one tensor, the name carries whatever the property needs
|
||||
typedef void (*tensor_filler)(std::vector<float> & dst, const char * name, const tensor_data_params & params);
|
||||
|
||||
// a pool of normal draws, generated once and reused for every model
|
||||
static std::vector<float> g_normal_pool;
|
||||
static const size_t NORMAL_POOL_SIZE = 16 * 1024 * 1024;
|
||||
|
||||
// fill the pool once; each tensor copies a slice of it
|
||||
static void init_normal_pool(const size_t seed, const float stdev) {
|
||||
g_normal_pool.resize(NORMAL_POOL_SIZE);
|
||||
std::mt19937 gen(seed);
|
||||
std::normal_distribution<float> dist(0.0f, stdev);
|
||||
for (float & v : g_normal_pool) {
|
||||
v = dist(gen);
|
||||
}
|
||||
}
|
||||
|
||||
// a normal draw keyed on a name, so that tensors which share a draw ask for the same key
|
||||
static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params & params) {
|
||||
std::mt19937 gen(params.seed ^ std::hash<std::string>{}(key));
|
||||
std::normal_distribution<float> dis(0.0f, params.stdev);
|
||||
for (float & v : dst) {
|
||||
v = dis(gen);
|
||||
static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params &) {
|
||||
const size_t n = dst.size();
|
||||
const size_t pool_size = g_normal_pool.size();
|
||||
const size_t start = std::hash<std::string>{}(key) % pool_size;
|
||||
|
||||
// copy a cyclic slice of the pool, so a tensor larger than the pool still fills
|
||||
size_t i = 0;
|
||||
size_t pos = start;
|
||||
while (i < n) {
|
||||
const size_t chunk = std::min(pool_size - pos, n - i);
|
||||
std::memcpy(dst.data() + i, g_normal_pool.data() + pos, chunk * sizeof(float));
|
||||
i += chunk;
|
||||
pos = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -682,11 +705,11 @@ static std::string get_tmp_gguf_path() {
|
||||
return (std::filesystem::temp_directory_path() / ("test-llama-archs-" + std::to_string(++id) + ".gguf")).string();
|
||||
}
|
||||
|
||||
static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
|
||||
static llama_model_ptr get_model(
|
||||
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
|
||||
const std::vector<ggml_backend_dev_t> & devs,
|
||||
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false,
|
||||
const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr, const size_t moe_cache_size = 0) {
|
||||
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER,
|
||||
const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr) {
|
||||
GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));
|
||||
llama_model_params model_params = llama_model_default_params();
|
||||
model_params.progress_callback = silent_model_load_progress;
|
||||
@@ -696,6 +719,23 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
|
||||
model_params.split_mode = split_mode;
|
||||
model_params.tensor_buft_overrides = tensor_buft_overrides;
|
||||
|
||||
tensor_data_params tensor_params = { seed, stdev };
|
||||
llama_model_ptr model(gguf_ctx != nullptr ?
|
||||
llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
|
||||
llama_model_load_from_file_ptr(file, model_params));
|
||||
if (!model) {
|
||||
throw std::runtime_error("failed to create llama model");
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
|
||||
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
|
||||
const std::vector<ggml_backend_dev_t> & devs,
|
||||
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false,
|
||||
const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr, const size_t moe_cache_size = 0) {
|
||||
llama_model_ptr model = get_model(gguf_ctx, file, seed, stdev, devs, split_mode, tensor_buft_overrides);
|
||||
|
||||
llama_context_params ctx_params = llama_context_default_params();
|
||||
ctx_params.n_ctx = 0;
|
||||
ctx_params.n_threads = 4;
|
||||
@@ -709,13 +749,6 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
|
||||
ctx_params.n_ubatch = 2;
|
||||
}
|
||||
|
||||
tensor_data_params tensor_params = { seed, stdev };
|
||||
llama_model_ptr model(gguf_ctx != nullptr ?
|
||||
llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
|
||||
llama_model_load_from_file_ptr(file, model_params));
|
||||
if (!model) {
|
||||
throw std::runtime_error("failed to create llama model");
|
||||
}
|
||||
llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));
|
||||
if (!lctx) {
|
||||
throw std::runtime_error("failed to create llama context");
|
||||
@@ -1027,10 +1060,12 @@ static int save_models(const std::string & arch_filter, const size_t seed, const
|
||||
continue;
|
||||
}
|
||||
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
|
||||
auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {});
|
||||
llama_model_ptr model = get_model(gguf_ctx.get(), nullptr, seed, stdev, {});
|
||||
const std::string path = dir + "/" + llama_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
|
||||
LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llama_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
|
||||
llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());
|
||||
LOG_INF("%s: '%s %s' -> %s ...", __func__, llama_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
|
||||
llama_model_save_to_file(model.get(), path.c_str());
|
||||
const std::uintmax_t size = std::filesystem::file_size(path);
|
||||
LOG_CNT(" (%.2f MB)\n", (double) size / (1024.0 * 1024.0));
|
||||
}
|
||||
}
|
||||
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
|
||||
@@ -1419,6 +1454,8 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev);
|
||||
|
||||
init_normal_pool(seed, stdev);
|
||||
|
||||
try {
|
||||
if (!out.empty()) {
|
||||
return save_models(arch_filter, seed, stdev, verbosity, out);
|
||||
|
||||
Reference in new issue
Block a user