common : allow concurrent downloads across processes (#29312)

Signed-off-by: Adrien Gallouët <angt@huggingface.co>
This commit is contained in:
Adrien Gallouët authored and GitHub committed 2026-10-23 23:52:56 +02:00
1 parent 9145ddb955
commit 21ec307146
9 files changed
+380 -55

No files matched your search

+6 -1
View File
@@ -26,6 +26,7 @@
#include <fstream>
#include <iostream>
#include <iterator>
#include <random>
#include <regex>
#include <sstream>
#include <string>
@@ -917,9 +918,13 @@ std::string fs_path_to_utf8(const std::filesystem::path & path) {
}
void fs_write_atomic(const std::filesystem::path & path, const std::string & data) {
// a random suffix gives every writer its own temporary file, across processes and containers
std::random_device rd;
const uint64_t id = ((uint64_t) rd() << 32) | rd();
std::error_code ec;
std::filesystem::path path_tmp = path;
path_tmp += ".tmp";
path_tmp += "." + std::to_string(id) + ".tmp";
if (path.has_parent_path()) {
std::filesystem::create_directories(path.parent_path(), ec);