unicode : add opt-in NFC normalization for Unigram tokenizers (#30390)

* unicode : add opt-in NFC normalization for Unigram tokenizers

Note: used help from Codex, Sol 6.1 Medium

* Fixing initial bot comments + linter error (using existing out instead of print)

* removed test_nfc_tokenizer
This commit is contained in:
Sandro Hanea authored and GitHub committed 2026-10-14 08:33:24 +03:00
1 parent eca5458ce0
commit 10da8c463f
12 files changed
+4165 -7

No files matched your search

+22
View File
@@ -2,9 +2,31 @@
#include <cstdio>
#include <string>
#include <utility>
#include <vector>
int main() {
const std::vector<std::pair<std::vector<uint32_t>, std::vector<uint32_t>>> normalization = {
{{}, {}},
{{0x65, 0x301}, {0xE9}},
{{0xE9}, {0xE9}},
{{0x212B}, {0xC5}},
{{0x61, 0x315, 0x300}, {0xE0, 0x315}},
{{0x61, 0x305, 0x301}, {0x61, 0x305, 0x301}},
{{0x301, 0x327}, {0x327, 0x301}},
{{0x1100, 0x1161, 0x11A8}, {0xAC01}},
{{0xAC01}, {0xAC01}},
{{0xFB01, 0xFF21}, {0xFB01, 0xFF21}},
{{0x958}, {0x915, 0x93C}},
{{0x1D15E}, {0x1D157, 0x1D165}},
};
for (const auto & test : normalization) {
if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) {
fprintf(stderr, "unexpected NFC normalization\n");
return 1;
}
}
const std::vector<std::string> regex_exprs = {
"[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
};