diff --git a/common/common.cpp b/common/common.cpp index a2f162b6f..b69eacc76 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1033,47 +1033,6 @@ std::filesystem::path fs_get_cache_file(const std::string & filename) { return cache_directory / std::filesystem::u8path(filename); } -std::vector fs_list(const std::string & path, bool include_directories) { - std::vector files; - if (path.empty()) return files; - - std::filesystem::path dir(path); - if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) { - return files; - } - - for (const auto & entry : std::filesystem::directory_iterator(dir)) { - try { - // Only include regular files (skip directories) - const auto & p = entry.path(); - if (std::filesystem::is_regular_file(p)) { - common_file_info info; - info.path = p.string(); - info.name = p.filename().string(); - info.is_dir = false; - try { - info.size = static_cast(std::filesystem::file_size(p)); - } catch (const std::filesystem::filesystem_error &) { - info.size = 0; - } - files.push_back(std::move(info)); - } else if (include_directories && std::filesystem::is_directory(p)) { - common_file_info info; - info.path = p.string(); - info.name = p.filename().string(); - info.size = 0; // Directories have no size - info.is_dir = true; - files.push_back(std::move(info)); - } - } catch (const std::filesystem::filesystem_error &) { - // skip entries we cannot inspect - continue; - } - } - - return files; -} - // // TTY utils // diff --git a/common/common.h b/common/common.h index 4308b0703..6f51675c3 100644 --- a/common/common.h +++ b/common/common.h @@ -930,14 +930,6 @@ std::filesystem::path fs_get_cache_directory(); std::filesystem::path fs_get_cache_file(const std::string & filename); std::filesystem::path fs_get_config_directory(); -struct common_file_info { - std::string path; - std::string name; - size_t size = 0; // in bytes - bool is_dir = false; -}; -std::vector fs_list(const std::string & path, bool include_directories); - void fs_write_atomic(const std::filesystem::path & path, const std::string & data); // diff --git a/common/preset.cpp b/common/preset.cpp index 7ca24ec38..14d51809d 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -385,63 +385,75 @@ static bool is_draft_file(const std::string & fname) { } common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const { - if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { + const std::filesystem::path dir = std::filesystem::u8path(models_dir); + if (!std::filesystem::exists(dir) || !std::filesystem::is_directory(dir)) { throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str())); } std::vector models; - auto scan_subdir = [&models](const std::string & subdir_path, const std::string & name) { - auto files = fs_list(subdir_path, false); - common_file_info model_file; - common_file_info first_shard_file; - common_file_info mmproj_file; - common_file_info draft_file; - for (const auto & file : files) { - if (string_ends_with(file.name, ".gguf")) { - if (is_mmproj_file(file.name)) { - mmproj_file = file; - } else if (is_draft_file(file.name)) { - if (draft_file.path.empty()) { - draft_file = file; // first sidecar found wins - } - } else if (file.name.find("-00001-of-") != std::string::npos) { - first_shard_file = file; - } else { - model_file = file; + auto scan_subdir = [&models](const std::filesystem::path & subdir_path, const std::string & name) { + std::filesystem::path model_file; + std::filesystem::path first_shard_file; + std::filesystem::path mmproj_file; + std::filesystem::path draft_file; + std::error_code ec; + for (const auto & entry : std::filesystem::directory_iterator(subdir_path)) { + if (!entry.is_regular_file(ec)) { + continue; + } + const std::string fname = fs_path_to_utf8(entry.path().filename()); + if (!string_ends_with(fname, ".gguf")) { + continue; + } + if (is_mmproj_file(fname)) { + mmproj_file = entry.path(); + } else if (is_draft_file(fname)) { + if (draft_file.empty()) { + draft_file = entry.path(); // first sidecar found wins } + } else if (fname.find("-00001-of-") != std::string::npos) { + first_shard_file = entry.path(); + } else { + model_file = entry.path(); } } // single file model - local_model model{ - /* name */ name, - /* path */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path, - /* path_mmproj */ mmproj_file.path, // can be empty - /* path_draft */ draft_file.path // can be empty - }; - if (!model.path.empty()) { - models.push_back(model); + const std::filesystem::path & path = first_shard_file.empty() ? model_file : first_shard_file; + if (!path.empty()) { + models.push_back({ + /* name */ name, + /* path */ fs_path_to_utf8(path), + /* path_mmproj */ fs_path_to_utf8(mmproj_file), // can be empty + /* path_draft */ fs_path_to_utf8(draft_file) // can be empty + }); } }; - auto files = fs_list(models_dir, true); - for (const auto & file : files) { - if (file.is_dir) { - scan_subdir(file.path, file.name); - } else if (string_ends_with(file.name, ".gguf")) { - if (is_mmproj_file(file.name) || is_draft_file(file.name)) { - continue; // companion file, cannot be loaded as a model on its own - } - // single file model - std::string name = file.name; - string_replace_all(name, ".gguf", ""); - local_model model{ - /* name */ name, - /* path */ file.path, - /* path_mmproj */ "", - /* path_draft */ "" - }; - models.push_back(model); + for (const auto & entry : std::filesystem::directory_iterator(dir)) { + std::error_code ec; + if (entry.is_directory(ec)) { + scan_subdir(entry.path(), fs_path_to_utf8(entry.path().filename())); + continue; } + if (!entry.is_regular_file(ec)) { + continue; + } + const std::string fname = fs_path_to_utf8(entry.path().filename()); + if (!string_ends_with(fname, ".gguf")) { + continue; + } + if (is_mmproj_file(fname) || is_draft_file(fname)) { + continue; // companion file, cannot be loaded as a model on its own + } + // single file model + std::string name = fname; + string_replace_all(name, ".gguf", ""); + models.push_back({ + /* name */ name, + /* path */ fs_path_to_utf8(entry.path()), + /* path_mmproj */ "", + /* path_draft */ "" + }); } // convert local models to presets