Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

vocab : refactor normalizer flags into options struct, add strip_accents (#24371)

Browse Source 
* vocab : refactor normalizer flags into options struct, add strip_accents

* Update src/llama-vocab.h

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

* Update src/llama-vocab.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
o7si
2026-06-11 10:36:50 +03:00
committed by GitHub
co-authored by Sigbjørn Skjæret
parent db94854ff5
commit 68f30663cf
7 changed files with 84 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-vocab.h
+7 -1
View File
@@ -76,6 +76,12 @@ struct llama_vocab {
llama_token_attr attr;
};
struct normalizer_options {
bool lowercase = true;
bool strip_accents = true;
// TODO: clean_text, handle_chinese_chars
};
llama_vocab();
~llama_vocab();
@@ -141,7 +147,7 @@ struct llama_vocab {
bool get_remove_extra_whitespaces () const;
bool get_escape_whitespaces () const;
bool get_treat_whitespace_as_suffix() const;
bool get_normalizer_lowercase () const;
const normalizer_options & get_normalizer_opts() const;
const std::vector<llama_token> & get_suppress_tokens() const;