unicode : add opt-in NFC normalization for Unigram tokenizers (#30390)
* unicode : add opt-in NFC normalization for Unigram tokenizers Note: used help from Codex, Sol 6.1 Medium * Fixing initial bot comments + linter error (using existing out instead of print) * removed test_nfc_tokenizer
This commit is contained in:
1 parent
eca5458ce0
commit
10da8c463f
12 files changed
+4165
-7
No files matched your search
@@ -2,9 +2,31 @@
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
int main() {
|
||||
const std::vector<std::pair<std::vector<uint32_t>, std::vector<uint32_t>>> normalization = {
|
||||
{{}, {}},
|
||||
{{0x65, 0x301}, {0xE9}},
|
||||
{{0xE9}, {0xE9}},
|
||||
{{0x212B}, {0xC5}},
|
||||
{{0x61, 0x315, 0x300}, {0xE0, 0x315}},
|
||||
{{0x61, 0x305, 0x301}, {0x61, 0x305, 0x301}},
|
||||
{{0x301, 0x327}, {0x327, 0x301}},
|
||||
{{0x1100, 0x1161, 0x11A8}, {0xAC01}},
|
||||
{{0xAC01}, {0xAC01}},
|
||||
{{0xFB01, 0xFF21}, {0xFB01, 0xFF21}},
|
||||
{{0x958}, {0x915, 0x93C}},
|
||||
{{0x1D15E}, {0x1D157, 0x1D165}},
|
||||
};
|
||||
for (const auto & test : normalization) {
|
||||
if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) {
|
||||
fprintf(stderr, "unexpected NFC normalization\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<std::string> regex_exprs = {
|
||||
"[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user