server: harden the file_glob_search directory walk (#26626)
* server: don't walk Windows junctions in file_glob_search std::filesystem reports a junction as a plain directory, so the symlink guard misses it and a junction pointing back at an ancestor is walked until the path length gives out read the reparse tag and treat a symlink and a mount point as links, leaving any other reparse point walkable so cloud placeholders and dedup stubs still get searched look junk directory names up case insensitively on Windows, where NTFS makes Build the same directory as build test that a junk directory stays selectable while its contents stay out of search results * server: report a directory the walk could not read a directory that fails to open or to iterate was skipped in silence, so a caller got a listing that looked complete while a whole subtree was missing: a path over the platform limit, a volume going away, a name the filesystem rejects skip_permission_denied never reaches this path, so an error here is an incomplete answer rather than a deliberate omission, and it now sets the truncated flag * server: simplify the file_glob_search listing plumbing return a small result struct instead of two out params and a caller path that only fed an error string, taking list_entries from six parameters down to three scope the error code to the directory being read, act on the status code the entry lookups already returned, and treat an unreadable link state as a link so the walk never descends on a guess check the deadline when a directory is popped, not only per entry, so a tree of empty directories cannot outlive the budget read the path parameter once, and reject an invalid limit the way an invalid type is already rejected, instead of silently falling back normalize the resolved path, so a "." or ".." a caller typed reaches neither git nor the client, and return the generic path form with '/' separators on every platform, so the base sent to clients no longer needs a local fixup * ui: expire cached picker searches the cache grew for the lifetime of the component: entries went stale after the TTL but were never removed, so every distinct query typed in a session stayed in memory drop expired entries when a new result is stored * server: address review from @ngxson trim comments to one line each, and drop two that restate the code rename junk_lookup_name to get_effective_name, and move it and the link check to private static members next to junk_dir_names merge the Windows and Linux link checks into one is_link, so symlinks are checked everywhere and junctions only add to it on Windows * server: convert tool paths as UTF-8 on Windows a narrow path uses the active code page there, so a file name came back mangled and a path with an accent could not be opened at all convert explicitly at every crossing between a std::string, which always carries UTF-8 here, and fs::path read the home directory through the wide environment, since the narrow one returns the profile path in the active code page too the walker no longer normalizes separators by hand, since paths now come back in generic form * server: fold the platform branch inside console_output_to_utf8 match the shape of the other helpers, one definition with the #if inside, instead of two definitions wrapped in #if and #else inline the single caller helper and trim the comment
This commit is contained in:
+154
-93
@@ -30,20 +30,14 @@ namespace fs = std::filesystem;
|
|||||||
// internal helpers
|
// internal helpers
|
||||||
//
|
//
|
||||||
|
|
||||||
#if defined(_WIN32)
|
// a child process writes in the OEM code page, so accented output would reach
|
||||||
// A chunk can end in the middle of a multi-byte sequence, so the incomplete
|
// the JSON layer as invalid bytes. run() spawns without a console, so the
|
||||||
// tail is dropped before validating what precedes it.
|
// console code page never applies
|
||||||
static bool is_utf8_text(const std::string & text) {
|
|
||||||
return is_valid_utf8(text.substr(0, validate_utf8(text)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// A child process writes its output in the OEM code page, which is not UTF-8
|
|
||||||
// on a western Windows install, so accented text reaches the JSON layer as
|
|
||||||
// invalid bytes and is replaced there. Text that already decodes as UTF-8 is
|
|
||||||
// returned untouched, so a child that emits UTF-8 is never decoded twice.
|
|
||||||
// run() spawns without a console, so the console code page does not apply.
|
|
||||||
static std::string console_output_to_utf8(const std::string & text) {
|
static std::string console_output_to_utf8(const std::string & text) {
|
||||||
if (text.empty() || is_utf8_text(text)) {
|
#if defined(_WIN32)
|
||||||
|
// a chunk can end mid sequence, so the incomplete tail is dropped first
|
||||||
|
if (text.empty() || is_valid_utf8(text.substr(0, validate_utf8(text)))) {
|
||||||
|
// never decode twice a child that already emits UTF-8
|
||||||
return text;
|
return text;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,12 +58,10 @@ static std::string console_output_to_utf8(const std::string & text) {
|
|||||||
std::string utf8(utf8_len, '\0');
|
std::string utf8(utf8_len, '\0');
|
||||||
WideCharToMultiByte(CP_UTF8, 0, wide.data(), wide_len, utf8.data(), utf8_len, nullptr, nullptr);
|
WideCharToMultiByte(CP_UTF8, 0, wide.data(), wide_len, utf8.data(), utf8_len, nullptr, nullptr);
|
||||||
return utf8;
|
return utf8;
|
||||||
}
|
|
||||||
#else
|
#else
|
||||||
static std::string console_output_to_utf8(const std::string & text) {
|
|
||||||
return text;
|
return text;
|
||||||
}
|
|
||||||
#endif
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
json server_tool::to_json() const {
|
json server_tool::to_json() const {
|
||||||
return {
|
return {
|
||||||
@@ -94,14 +86,30 @@ enum class list_kind {
|
|||||||
all, // both
|
all, // both
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// a narrow path uses the active code page on Windows, so every crossing between
|
||||||
|
// a std::string (always UTF-8 here) and fs::path is converted explicitly
|
||||||
|
static fs::path path_from_utf8(const std::string & s) {
|
||||||
|
return fs::u8path(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
// '/' separators on every platform: Windows accepts them, the web UI needs them
|
||||||
|
static std::string path_to_utf8(const fs::path & p) {
|
||||||
|
const auto s = p.generic_u8string();
|
||||||
|
return std::string(s.begin(), s.end());
|
||||||
|
}
|
||||||
|
|
||||||
// home directory, read once at first use (getenv is not thread safe against setenv)
|
// home directory, read once at first use (getenv is not thread safe against setenv)
|
||||||
static const std::string & home_dir() {
|
static const std::string & home_dir() {
|
||||||
static const std::string home = [] {
|
static const std::string home = [] {
|
||||||
const char * h = getenv("HOME");
|
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
if (h == nullptr) h = getenv("USERPROFILE");
|
// the narrow getenv would return the profile path in the active code page
|
||||||
#endif
|
const wchar_t * w = _wgetenv(L"HOME");
|
||||||
|
if (w == nullptr) w = _wgetenv(L"USERPROFILE");
|
||||||
|
return w ? path_to_utf8(fs::path(w)) : std::string();
|
||||||
|
#else
|
||||||
|
const char * h = getenv("HOME");
|
||||||
return h ? std::string(h) : std::string();
|
return h ? std::string(h) : std::string();
|
||||||
|
#endif
|
||||||
}();
|
}();
|
||||||
return home;
|
return home;
|
||||||
}
|
}
|
||||||
@@ -140,11 +148,14 @@ public:
|
|||||||
std::string rel; // '/'-separated, relative to `base`
|
std::string rel; // '/'-separated, relative to `base`
|
||||||
bool is_dir = false;
|
bool is_dir = false;
|
||||||
};
|
};
|
||||||
// entries relative to `base`; sets `err` if `base` isn't a directory
|
struct list_result {
|
||||||
|
std::vector<list_entry> entries;
|
||||||
|
std::string err; // set when `base` is not a directory
|
||||||
|
bool truncated = false; // set when the walk could not see everything
|
||||||
|
};
|
||||||
|
// entries relative to `base`, which must already be resolved (absolute)
|
||||||
// max_depth == 0 means unlimited, 1 means direct children of `base` only
|
// max_depth == 0 means unlimited, 1 means direct children of `base` only
|
||||||
// `base` must already be resolved (absolute); `caller_path` is the path the
|
virtual list_result list_entries(const std::string & base, int max_depth, list_kind kind) const = 0;
|
||||||
// caller passed, used only for error messages
|
|
||||||
virtual std::vector<list_entry> list_entries(const std::string & base, const std::string & caller_path, int max_depth, list_kind kind, std::string & err, bool & truncated) const = 0;
|
|
||||||
// on_chunk, if set, is called with each chunk of output as it is read (before truncation cuts in);
|
// on_chunk, if set, is called with each chunk of output as it is read (before truncation cuts in);
|
||||||
// returning false terminates the process early (e.g. the client disconnected)
|
// returning false terminates the process early (e.g. the client disconnected)
|
||||||
virtual exec_result run(
|
virtual exec_result run(
|
||||||
@@ -162,37 +173,47 @@ public:
|
|||||||
// expands a leading `~`, then resolves `path` against `cwd` (or the server
|
// expands a leading `~`, then resolves `path` against `cwd` (or the server
|
||||||
// working directory when `cwd` is unset); the result is always absolute
|
// working directory when `cwd` is unset); the result is always absolute
|
||||||
std::string resolve(const std::string & path) const override {
|
std::string resolve(const std::string & path) const override {
|
||||||
std::string p = expand_home(path);
|
const std::string p = expand_home(path);
|
||||||
if (fs::path(p).is_absolute()) {
|
|
||||||
return p;
|
fs::path full = path_from_utf8(p);
|
||||||
|
if (!full.is_absolute()) {
|
||||||
|
if (cwd.empty()) {
|
||||||
|
std::error_code ec;
|
||||||
|
const fs::path cur = fs::current_path(ec);
|
||||||
|
if (ec) return p;
|
||||||
|
full = cur / full;
|
||||||
|
} else {
|
||||||
|
full = path_from_utf8(cwd) / full;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (cwd.empty()) {
|
|
||||||
std::error_code ec;
|
// drop "." and ".." so they never reach git or the client
|
||||||
fs::path cur = fs::current_path(ec);
|
full = full.lexically_normal();
|
||||||
if (ec) return p;
|
// a trailing ".." normalizes to a path that ends with a separator
|
||||||
return (cur / p).string();
|
if (!full.has_filename() && full != full.root_path()) {
|
||||||
|
full = full.parent_path();
|
||||||
}
|
}
|
||||||
return (fs::path(cwd) / p).string();
|
return path_to_utf8(full);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool is_directory(const std::string & path) const override {
|
bool is_directory(const std::string & path) const override {
|
||||||
std::error_code ec;
|
std::error_code ec;
|
||||||
return fs::is_directory(resolve(path), ec) && !ec;
|
return fs::is_directory(path_from_utf8(resolve(path)), ec) && !ec;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool is_regular_file(const std::string & path) const override {
|
bool is_regular_file(const std::string & path) const override {
|
||||||
std::error_code ec;
|
std::error_code ec;
|
||||||
return fs::is_regular_file(resolve(path), ec) && !ec;
|
return fs::is_regular_file(path_from_utf8(resolve(path)), ec) && !ec;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_size(const std::string & path, uintmax_t & out_size) const override {
|
bool file_size(const std::string & path, uintmax_t & out_size) const override {
|
||||||
std::error_code ec;
|
std::error_code ec;
|
||||||
out_size = fs::file_size(resolve(path), ec);
|
out_size = fs::file_size(path_from_utf8(resolve(path)), ec);
|
||||||
return !ec;
|
return !ec;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool read_file(const std::string & path, std::string & out) const override {
|
bool read_file(const std::string & path, std::string & out) const override {
|
||||||
std::ifstream f(resolve(path), std::ios::binary);
|
std::ifstream f(path_from_utf8(resolve(path)), std::ios::binary);
|
||||||
if (!f) return false;
|
if (!f) return false;
|
||||||
std::ostringstream ss;
|
std::ostringstream ss;
|
||||||
ss << f.rdbuf();
|
ss << f.rdbuf();
|
||||||
@@ -202,7 +223,7 @@ public:
|
|||||||
|
|
||||||
bool write_file(const std::string & path, const std::string & content) const override {
|
bool write_file(const std::string & path, const std::string & content) const override {
|
||||||
std::error_code ec;
|
std::error_code ec;
|
||||||
fs::path fpath(resolve(path));
|
fs::path fpath = path_from_utf8(resolve(path));
|
||||||
if (fpath.has_parent_path()) {
|
if (fpath.has_parent_path()) {
|
||||||
fs::create_directories(fpath.parent_path(), ec);
|
fs::create_directories(fpath.parent_path(), ec);
|
||||||
if (ec) return false;
|
if (ec) return false;
|
||||||
@@ -213,13 +234,13 @@ public:
|
|||||||
return (bool) f;
|
return (bool) f;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<list_entry> list_entries(const std::string & base, const std::string & caller_path, int max_depth, list_kind kind, std::string & err, bool & truncated) const override {
|
list_result list_entries(const std::string & base, int max_depth, list_kind kind) const override {
|
||||||
err.clear();
|
list_result out;
|
||||||
truncated = false;
|
|
||||||
std::error_code ec;
|
std::error_code ec;
|
||||||
if (!fs::is_directory(base, ec) || ec) {
|
if (!fs::is_directory(base, ec) || ec) {
|
||||||
err = "path does not exist or is not a directory: " + caller_path;
|
out.err = "path does not exist or is not a directory";
|
||||||
return {};
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
|
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
|
||||||
@@ -231,7 +252,6 @@ public:
|
|||||||
SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
|
SERVER_TOOL_GIT_LS_FILES_MAX_OUTPUT, SERVER_TOOL_LIST_ENTRIES_TIMEOUT);
|
||||||
|
|
||||||
if (res.exit_code == 0 && !res.timed_out) {
|
if (res.exit_code == 0 && !res.timed_out) {
|
||||||
std::vector<list_entry> result;
|
|
||||||
std::istringstream iss(res.output);
|
std::istringstream iss(res.output);
|
||||||
std::string line;
|
std::string line;
|
||||||
while (std::getline(iss, line)) {
|
while (std::getline(iss, line)) {
|
||||||
@@ -239,15 +259,16 @@ public:
|
|||||||
if (line.empty()) continue;
|
if (line.empty()) continue;
|
||||||
std::replace(line.begin(), line.end(), '\\', '/');
|
std::replace(line.begin(), line.end(), '\\', '/');
|
||||||
if (max_depth > 0 && entry_depth(line) > max_depth) continue;
|
if (max_depth > 0 && entry_depth(line) > max_depth) continue;
|
||||||
if (is_regular_file((fs::path(base) / line).string())) {
|
if (is_regular_file(path_to_utf8(path_from_utf8(base) / path_from_utf8(line)))) {
|
||||||
result.push_back({line, false});
|
out.entries.push_back({line, false});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return result;
|
return out;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return list_entries_fallback(base, max_depth, kind, deadline, truncated);
|
out.entries = list_entries_fallback(base, max_depth, kind, deadline, out.truncated);
|
||||||
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
exec_result run(
|
exec_result run(
|
||||||
@@ -326,6 +347,42 @@ public:
|
|||||||
private:
|
private:
|
||||||
std::string cwd;
|
std::string cwd;
|
||||||
|
|
||||||
|
// a link can point back to an ancestor and loop forever, so it is never walked
|
||||||
|
static bool is_link(const fs::directory_entry & entry) {
|
||||||
|
std::error_code ec;
|
||||||
|
if (entry.is_symlink(ec) || ec) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
#if defined(_WIN32)
|
||||||
|
// a junction looks like a plain directory to std::filesystem, so read the reparse tag
|
||||||
|
WIN32_FIND_DATAW data;
|
||||||
|
const HANDLE h = FindFirstFileW(entry.path().c_str(), &data);
|
||||||
|
if (h == INVALID_HANDLE_VALUE) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
FindClose(h);
|
||||||
|
if ((data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// other reparse points (cloud placeholder, dedup stub) are real directories
|
||||||
|
return data.dwReserved0 == IO_REPARSE_TAG_SYMLINK || data.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT;
|
||||||
|
#else
|
||||||
|
return false;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// NTFS is case insensitive, so Build and build are the same directory
|
||||||
|
static std::string get_effective_name(const std::string & fname) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
std::string lowered = fname;
|
||||||
|
std::transform(lowered.begin(), lowered.end(), lowered.begin(),
|
||||||
|
[](unsigned char c) { return (char) std::tolower(c); });
|
||||||
|
return lowered;
|
||||||
|
#else
|
||||||
|
return fname;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
static const std::unordered_set<std::string> & junk_dir_names() {
|
static const std::unordered_set<std::string> & junk_dir_names() {
|
||||||
static const std::unordered_set<std::string> names = {
|
static const std::unordered_set<std::string> names = {
|
||||||
".git", ".svn", ".hg", "node_modules", "__pycache__",
|
".git", ".svn", ".hg", "node_modules", "__pycache__",
|
||||||
@@ -337,46 +394,53 @@ private:
|
|||||||
std::vector<list_entry> list_entries_fallback(const std::string & base, int max_depth, list_kind kind,
|
std::vector<list_entry> list_entries_fallback(const std::string & base, int max_depth, list_kind kind,
|
||||||
std::chrono::steady_clock::time_point deadline, bool & truncated) const {
|
std::chrono::steady_clock::time_point deadline, bool & truncated) const {
|
||||||
std::vector<list_entry> result;
|
std::vector<list_entry> result;
|
||||||
std::error_code ec;
|
|
||||||
|
|
||||||
std::vector<std::tuple<fs::path, fs::path, int>> stack;
|
std::vector<std::tuple<fs::path, fs::path, int>> stack;
|
||||||
stack.emplace_back(fs::path(base), fs::path(), 0);
|
stack.emplace_back(path_from_utf8(base), fs::path(), 0);
|
||||||
|
|
||||||
while (!stack.empty()) {
|
while (!stack.empty()) {
|
||||||
auto [dir, rel_dir, depth] = stack.back();
|
if (std::chrono::steady_clock::now() >= deadline) {
|
||||||
|
truncated = true;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto [dir, rel_dir, depth] = std::move(stack.back());
|
||||||
stack.pop_back();
|
stack.pop_back();
|
||||||
|
|
||||||
// the throwing increment would escape the tool on a directory that
|
std::error_code ec;
|
||||||
// goes away mid walk, so step the iterator explicitly
|
// step the iterator by hand: the throwing increment escapes on a directory that goes away
|
||||||
fs::directory_iterator it(dir, fs::directory_options::skip_permission_denied, ec);
|
fs::directory_iterator it(dir, fs::directory_options::skip_permission_denied, ec);
|
||||||
|
// permission errors are skipped above, so this is a subtree the caller never sees
|
||||||
|
if (ec) {
|
||||||
|
truncated = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
for (const fs::directory_iterator end; it != end; it.increment(ec)) {
|
for (const fs::directory_iterator end; it != end; it.increment(ec)) {
|
||||||
if (ec) break;
|
if (ec) {
|
||||||
|
truncated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
if (std::chrono::steady_clock::now() >= deadline) {
|
if (std::chrono::steady_clock::now() >= deadline) {
|
||||||
truncated = true;
|
truncated = true;
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
const fs::directory_entry & entry = *it;
|
const fs::directory_entry & entry = *it;
|
||||||
std::string fname = entry.path().filename().string();
|
const fs::path fname = entry.path().filename();
|
||||||
std::error_code tec;
|
std::error_code tec;
|
||||||
if (entry.is_directory(tec)) {
|
const bool is_dir = entry.is_directory(tec);
|
||||||
std::string rel = (rel_dir / fname).string();
|
if (tec) continue;
|
||||||
std::replace(rel.begin(), rel.end(), '\\', '/');
|
if (is_dir) {
|
||||||
if (kind == list_kind::dirs || kind == list_kind::all) {
|
if (kind == list_kind::dirs || kind == list_kind::all) {
|
||||||
result.push_back({rel, true});
|
result.push_back({path_to_utf8(rel_dir / fname), true});
|
||||||
}
|
}
|
||||||
// junk directories stay selectable but are never walked: they
|
// junk directories stay selectable but are never walked: they can be enormous
|
||||||
// hold nothing worth searching and can be enormous
|
if (junk_dir_names().count(get_effective_name(path_to_utf8(fname))) > 0) continue;
|
||||||
if (junk_dir_names().count(fname) > 0) continue;
|
if (!is_link(entry) && (max_depth == 0 || depth + 1 < max_depth)) {
|
||||||
// do not descend into symlinks: a link can point back to an
|
|
||||||
// ancestor and loop forever
|
|
||||||
if (!entry.is_symlink(tec) && (max_depth == 0 || depth + 1 < max_depth)) {
|
|
||||||
stack.emplace_back(entry.path(), rel_dir / fname, depth + 1);
|
stack.emplace_back(entry.path(), rel_dir / fname, depth + 1);
|
||||||
}
|
}
|
||||||
} else if (entry.is_regular_file(tec)) {
|
} else if (entry.is_regular_file(tec)) {
|
||||||
std::string rel = (rel_dir / fname).string();
|
|
||||||
std::replace(rel.begin(), rel.end(), '\\', '/');
|
|
||||||
if (kind == list_kind::files || kind == list_kind::all) {
|
if (kind == list_kind::files || kind == list_kind::all) {
|
||||||
result.push_back({rel, false});
|
result.push_back({path_to_utf8(rel_dir / fname), false});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -394,7 +458,7 @@ static std::unique_ptr<tools_io> make_tools_io(const json & params) {
|
|||||||
// no '/' in pattern -> match basename at any depth; else match full relative path
|
// no '/' in pattern -> match basename at any depth; else match full relative path
|
||||||
static bool path_glob_match(const std::string & pattern, const std::string & rel_path) {
|
static bool path_glob_match(const std::string & pattern, const std::string & rel_path) {
|
||||||
if (pattern.find('/') == std::string::npos) {
|
if (pattern.find('/') == std::string::npos) {
|
||||||
return glob_match(pattern, fs::path(rel_path).filename().string());
|
return glob_match(pattern, path_to_utf8(path_from_utf8(rel_path).filename()));
|
||||||
}
|
}
|
||||||
if (pattern == "**" || pattern.rfind("**/", 0) == 0 || pattern.rfind('/', 0) == 0) {
|
if (pattern == "**" || pattern.rfind("**/", 0) == 0 || pattern.rfind('/', 0) == 0) {
|
||||||
return glob_match(pattern, rel_path);
|
return glob_match(pattern, rel_path);
|
||||||
@@ -491,7 +555,7 @@ struct server_tool_read_file : server_tool {
|
|||||||
// file_glob_search: find files matching a glob pattern under a base directory
|
// file_glob_search: find files matching a glob pattern under a base directory
|
||||||
//
|
//
|
||||||
|
|
||||||
static constexpr size_t SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100;
|
static constexpr int SERVER_TOOL_FILE_SEARCH_MAX_RESULTS = 100;
|
||||||
static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_FILE = "file";
|
static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_FILE = "file";
|
||||||
static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_DIR = "dir";
|
static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_DIR = "dir";
|
||||||
static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_ALL = "all";
|
static constexpr const char * SERVER_TOOL_FILE_SEARCH_TYPE_ALL = "all";
|
||||||
@@ -525,7 +589,7 @@ struct server_tool_file_glob_search : server_tool {
|
|||||||
{"exclude", {{"type", "string"}, {"description", "Glob pattern for files to exclude"}}},
|
{"exclude", {{"type", "string"}, {"description", "Glob pattern for files to exclude"}}},
|
||||||
{"type", {{"type", "string"}, {"description", "Entry type to return: \"file\" (default), \"dir\" or \"all\""}}},
|
{"type", {{"type", "string"}, {"description", "Entry type to return: \"file\" (default), \"dir\" or \"all\""}}},
|
||||||
{"max_depth", {{"type", "integer"}, {"description", "Maximum depth to descend into subdirectories (default: 0 = unlimited; 1 = direct children only)"}}},
|
{"max_depth", {{"type", "integer"}, {"description", "Maximum depth to descend into subdirectories (default: 0 = unlimited; 1 = direct children only)"}}},
|
||||||
{"limit", {{"type", "integer"}, {"description", string_format("Maximum number of results to return (default %zu; values below 1 fall back to the default)", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS)}}},
|
{"limit", {{"type", "integer"}, {"description", string_format("Maximum number of results to return, capped at %d (default %d)", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS)}}},
|
||||||
}},
|
}},
|
||||||
{"required", json::array({"path"})},
|
{"required", json::array({"path"})},
|
||||||
}},
|
}},
|
||||||
@@ -536,17 +600,18 @@ struct server_tool_file_glob_search : server_tool {
|
|||||||
json invoke(json params, server_tool::stream *) const override {
|
json invoke(json params, server_tool::stream *) const override {
|
||||||
auto io = make_tools_io(params);
|
auto io = make_tools_io(params);
|
||||||
|
|
||||||
std::string base = io->resolve(params.at("path").get<std::string>());
|
const std::string path = params.at("path").get<std::string>();
|
||||||
// normalize to forward slashes so the web UI (which assumes '/') can
|
|
||||||
// join the relative entries into absolute paths on Windows too
|
std::string base = io->resolve(path);
|
||||||
std::replace(base.begin(), base.end(), '\\', '/');
|
|
||||||
std::string include = json_value(params, "include", std::string("**"));
|
std::string include = json_value(params, "include", std::string("**"));
|
||||||
std::string exclude = json_value(params, "exclude", std::string(""));
|
std::string exclude = json_value(params, "exclude", std::string(""));
|
||||||
std::string type = json_value(params, "type", std::string("file"));
|
std::string type = json_value(params, "type", std::string("file"));
|
||||||
int max_depth = std::max(0, json_value(params, "max_depth", 0));
|
int max_depth = std::max(0, json_value(params, "max_depth", 0));
|
||||||
int limit = json_value(params, "limit", (int) SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
|
const int limit_req = json_value(params, "limit", SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
|
||||||
if (limit < 1) limit = SERVER_TOOL_FILE_SEARCH_MAX_RESULTS;
|
if (limit_req < 1) {
|
||||||
limit = std::min(limit, (int) SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
|
return {{"error", "invalid limit: " + std::to_string(limit_req) + " (expected 1 or more)"}};
|
||||||
|
}
|
||||||
|
const int limit = std::min(limit_req, SERVER_TOOL_FILE_SEARCH_MAX_RESULTS);
|
||||||
|
|
||||||
list_kind kind;
|
list_kind kind;
|
||||||
if (type == SERVER_TOOL_FILE_SEARCH_TYPE_FILE) {
|
if (type == SERVER_TOOL_FILE_SEARCH_TYPE_FILE) {
|
||||||
@@ -559,15 +624,13 @@ struct server_tool_file_glob_search : server_tool {
|
|||||||
return {{"error", "invalid type: " + type + " (expected \"file\", \"dir\" or \"all\")"}};
|
return {{"error", "invalid type: " + type + " (expected \"file\", \"dir\" or \"all\")"}};
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string err;
|
const auto listing = io->list_entries(base, max_depth, kind);
|
||||||
bool truncated = false;
|
if (!listing.err.empty()) {
|
||||||
auto entries = io->list_entries(base, params.at("path").get<std::string>(), max_depth, kind, err, truncated);
|
return {{"error", listing.err + ": " + path}};
|
||||||
if (!err.empty()) {
|
|
||||||
return {{"error", err}};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<tools_io::list_entry> matches;
|
std::vector<tools_io::list_entry> matches;
|
||||||
for (const auto & entry : entries) {
|
for (const auto & entry : listing.entries) {
|
||||||
if (!path_glob_match(include, entry.rel)) continue;
|
if (!path_glob_match(include, entry.rel)) continue;
|
||||||
if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
|
if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
|
||||||
matches.push_back(entry);
|
matches.push_back(entry);
|
||||||
@@ -592,8 +655,8 @@ struct server_tool_file_glob_search : server_tool {
|
|||||||
"[%zu results limit reached (%zu total matches). Refine the glob pattern to narrow the search.]\n",
|
"[%zu results limit reached (%zu total matches). Refine the glob pattern to narrow the search.]\n",
|
||||||
shown, total);
|
shown, total);
|
||||||
}
|
}
|
||||||
if (truncated) {
|
if (listing.truncated) {
|
||||||
output_text << "[search timed out, results truncated]\n";
|
output_text << "[results truncated: time budget or unreadable directory]\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
// `base` is always absolute (resolve falls back to the server cwd), so
|
// `base` is always absolute (resolve falls back to the server cwd), so
|
||||||
@@ -688,16 +751,14 @@ struct server_tool_grep_search : server_tool {
|
|||||||
if (io->is_regular_file(abs_path)) {
|
if (io->is_regular_file(abs_path)) {
|
||||||
files.emplace_back(abs_path, path);
|
files.emplace_back(abs_path, path);
|
||||||
} else if (io->is_directory(abs_path)) {
|
} else if (io->is_directory(abs_path)) {
|
||||||
std::string err;
|
const auto listing = io->list_entries(abs_path, 0, list_kind::files);
|
||||||
bool truncated = false;
|
if (!listing.err.empty()) {
|
||||||
auto candidates = io->list_entries(abs_path, path, 0, list_kind::files, err, truncated);
|
return {{"error", listing.err + ": " + path}};
|
||||||
if (!err.empty()) {
|
|
||||||
return {{"error", err}};
|
|
||||||
}
|
}
|
||||||
for (const auto & entry : candidates) {
|
for (const auto & entry : listing.entries) {
|
||||||
if (!path_glob_match(include, entry.rel)) continue;
|
if (!path_glob_match(include, entry.rel)) continue;
|
||||||
if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
|
if (!exclude.empty() && path_glob_match(exclude, entry.rel)) continue;
|
||||||
files.emplace_back((fs::path(abs_path) / entry.rel).string(), entry.rel);
|
files.emplace_back(path_to_utf8(path_from_utf8(abs_path) / path_from_utf8(entry.rel)), entry.rel);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return {{"error", "path does not exist: " + path}};
|
return {{"error", "path does not exist: " + path}};
|
||||||
@@ -1306,7 +1367,7 @@ struct server_tool_get_info : server_tool {
|
|||||||
std::string cwd = json_value(params, "cwd", std::string());
|
std::string cwd = json_value(params, "cwd", std::string());
|
||||||
if (cwd.empty()) {
|
if (cwd.empty()) {
|
||||||
std::error_code ec;
|
std::error_code ec;
|
||||||
cwd = fs::current_path(ec).string();
|
cwd = path_to_utf8(fs::current_path(ec));
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -214,6 +214,27 @@ def test_tools_builtin_file_glob_search_max_depth_and_limit(tmp_path):
|
|||||||
assert "Total matches: 3" in res["plain_text_response"]
|
assert "Total matches: 3" in res["plain_text_response"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_tools_builtin_file_glob_search_junk_dirs(tmp_path):
|
||||||
|
global server
|
||||||
|
server.start()
|
||||||
|
|
||||||
|
(tmp_path / "build" / "nested").mkdir(parents=True)
|
||||||
|
(tmp_path / "build" / "artifact.txt").write_text("built")
|
||||||
|
(tmp_path / "src").mkdir()
|
||||||
|
(tmp_path / "src" / "main.cpp").write_text("int main() {}")
|
||||||
|
|
||||||
|
# a junk directory stays selectable as a working directory
|
||||||
|
res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "dir", "max_depth": 1})
|
||||||
|
assert "build" in [e["path"] for e in res["entries"]]
|
||||||
|
|
||||||
|
# but it is never walked, so nothing inside it shows up
|
||||||
|
res = call_tool("file_glob_search", {"path": str(tmp_path), "type": "all"})
|
||||||
|
paths = [e["path"] for e in res["entries"]]
|
||||||
|
assert "src/main.cpp" in paths
|
||||||
|
assert "build/artifact.txt" not in paths
|
||||||
|
assert "build/nested" not in paths
|
||||||
|
|
||||||
|
|
||||||
def test_tools_builtin_file_glob_search_rejects_invalid_type(tmp_path):
|
def test_tools_builtin_file_glob_search_rejects_invalid_type(tmp_path):
|
||||||
global server
|
global server
|
||||||
server.start()
|
server.start()
|
||||||
|
|||||||
@@ -87,8 +87,9 @@
|
|||||||
let searchSeq = 0;
|
let searchSeq = 0;
|
||||||
|
|
||||||
// Cache of the last file_glob_search result per (parent, include, max_depth),
|
// Cache of the last file_glob_search result per (parent, include, max_depth),
|
||||||
// so repeated queries in the same directory don't re-walk the tree. Entries
|
// so repeated queries in the same directory don't re-walk the tree. Entering
|
||||||
// expire after a short TTL.
|
// a directory hits it every time: the children listed for an exactly typed
|
||||||
|
// segment are what the next keystroke, the trailing slash, asks for again.
|
||||||
const SEARCH_CACHE_TTL_MS = 2000;
|
const SEARCH_CACHE_TTL_MS = 2000;
|
||||||
const searchCache = new SvelteMap<string, { results: GlobEntry[]; base: string; at: number }>();
|
const searchCache = new SvelteMap<string, { results: GlobEntry[]; base: string; at: number }>();
|
||||||
|
|
||||||
@@ -161,7 +162,11 @@
|
|||||||
if (typeof res.error === 'string') return { base: '', entries: [], error: res.error };
|
if (typeof res.error === 'string') return { base: '', entries: [], error: res.error };
|
||||||
const base = typeof res.base === 'string' ? res.base : '';
|
const base = typeof res.base === 'string' ? res.base : '';
|
||||||
const entries = Array.isArray(res.entries) ? (res.entries as GlobEntry[]) : [];
|
const entries = Array.isArray(res.entries) ? (res.entries as GlobEntry[]) : [];
|
||||||
searchCache.set(key, { results: entries, base, at: Date.now() });
|
const now = Date.now();
|
||||||
|
for (const [k, v] of searchCache) {
|
||||||
|
if (now - v.at >= SEARCH_CACHE_TTL_MS) searchCache.delete(k);
|
||||||
|
}
|
||||||
|
searchCache.set(key, { results: entries, base, at: now });
|
||||||
return { base, entries };
|
return { base, entries };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user