grammar : degrade max repetition >= 2000 to unbounded (#26613)
This commit is contained in:
@@ -648,10 +648,12 @@ const char * llama_grammar_parser::parse_sequence(
|
|||||||
} else {
|
} else {
|
||||||
throw std::runtime_error(std::string("expecting ',' at ") + pos);
|
throw std::runtime_error(std::string("expecting ',' at ") + pos);
|
||||||
}
|
}
|
||||||
bool has_max = max_times != UINT64_MAX;
|
if (min_times > MAX_REPETITION_THRESHOLD) {
|
||||||
if (min_times > MAX_REPETITION_THRESHOLD || (has_max && max_times > MAX_REPETITION_THRESHOLD)) {
|
|
||||||
throw std::runtime_error(std::string("number of repetitions exceeds sane defaults, please reduce the number of repetitions"));
|
throw std::runtime_error(std::string("number of repetitions exceeds sane defaults, please reduce the number of repetitions"));
|
||||||
}
|
}
|
||||||
|
if (max_times != UINT64_MAX && max_times > MAX_REPETITION_THRESHOLD) {
|
||||||
|
max_times = UINT64_MAX;
|
||||||
|
}
|
||||||
handle_repetitions(min_times, max_times);
|
handle_repetitions(min_times, max_times);
|
||||||
} else {
|
} else {
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -153,6 +153,53 @@ int main()
|
|||||||
root ::= "a"{,10}"
|
root ::= "a"{,10}"
|
||||||
)""");
|
)""");
|
||||||
|
|
||||||
|
verify_failure(R"""(
|
||||||
|
root ::= "a"{5000}
|
||||||
|
)""");
|
||||||
|
|
||||||
|
verify_failure(R"""(
|
||||||
|
root ::= "a"{5000,}
|
||||||
|
)""");
|
||||||
|
|
||||||
|
verify_failure(R"""(
|
||||||
|
root ::= "a"{5000,6000}
|
||||||
|
)""");
|
||||||
|
|
||||||
|
verify_parsing(R"""(
|
||||||
|
root ::= "a"{0,5000}
|
||||||
|
)""", {
|
||||||
|
{"root", 0},
|
||||||
|
{"root_1", 1},
|
||||||
|
}, {
|
||||||
|
// root (index 0)
|
||||||
|
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||||
|
{LLAMA_GRETYPE_END, 0},
|
||||||
|
// root_1 (index 1)
|
||||||
|
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||||
|
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||||
|
{LLAMA_GRETYPE_ALT, 0},
|
||||||
|
{LLAMA_GRETYPE_END, 0},
|
||||||
|
});
|
||||||
|
|
||||||
|
verify_parsing(R"""(
|
||||||
|
root ::= "a"{3,5000}
|
||||||
|
)""", {
|
||||||
|
{"root", 0},
|
||||||
|
{"root_1", 1},
|
||||||
|
}, {
|
||||||
|
// root (index 0)
|
||||||
|
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||||
|
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||||
|
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||||
|
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||||
|
{LLAMA_GRETYPE_END, 0},
|
||||||
|
// root_1 (index 1)
|
||||||
|
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||||
|
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||||
|
{LLAMA_GRETYPE_ALT, 0},
|
||||||
|
{LLAMA_GRETYPE_END, 0},
|
||||||
|
});
|
||||||
|
|
||||||
verify_parsing(R"""(
|
verify_parsing(R"""(
|
||||||
root ::= "a"
|
root ::= "a"
|
||||||
)""", {
|
)""", {
|
||||||
|
|||||||
Reference in New Issue
Block a user