diff --git a/common/arg.cpp b/common/arg.cpp index 5dc7c8d3a..192f70607 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -912,6 +912,10 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); } + if (params.sampling.ignore_eos && !common_grammar_value(params.sampling.grammar).empty()) { + throw std::invalid_argument("error: --ignore-eos is not supported with --grammar or --json-schema\n"); + } + const bool skip_model_download = // server will call common_params_handle_models() later, so we skip it here ctx_arg.ex == LLAMA_EXAMPLE_SERVER || diff --git a/common/sampling.cpp b/common/sampling.cpp index 0fe1c7405..078bfd984 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -203,6 +203,13 @@ struct common_sampler * common_sampler_init( throw std::invalid_argument("penalty_present must be finite"); } const llama_vocab * vocab = llama_model_get_vocab(model); + if (!common_grammar_value(params.grammar).empty()) { + for (const auto & b : params.logit_bias) { + if (b.bias == -INFINITY && llama_vocab_is_eog(vocab, b.token)) { + throw std::invalid_argument("EOG tokens cannot be banned when a grammar is set"); + } + } + } llama_sampler_chain_params lparams = llama_sampler_chain_default_params(); lparams.no_perf = params.no_perf; diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index 843f20709..36624ba86 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -239,6 +239,17 @@ static void test(void) { argv = {"binary_name", "-lm", "hello"}; assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); + // --ignore-eos is not supported with a grammar + { + common_params grammar_params; + argv = {"binary_name", "-m", "model_file.gguf", "--grammar", "root ::= \"ok\""}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), grammar_params, LLAMA_EXAMPLE_COMMON)); + + common_params grammar_eos_params; + argv = {"binary_name", "-m", "model_file.gguf", "--grammar", "root ::= \"ok\"", "--ignore-eos"}; + assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), grammar_eos_params, LLAMA_EXAMPLE_COMMON)); + } + printf("test-arg-parser: test valid usage\n\n"); argv = {"binary_name", "-m", "model_file.gguf"}; diff --git a/tools/server/tests/unit/test_ignore_eos.py b/tools/server/tests/unit/test_ignore_eos.py index f40faf5a8..4876d4a16 100644 --- a/tools/server/tests/unit/test_ignore_eos.py +++ b/tools/server/tests/unit/test_ignore_eos.py @@ -41,3 +41,22 @@ def test_ignore_eos_false_no_logit_bias(): assert res.status_code == 200 logit_bias = res.body["generation_settings"]["logit_bias"] assert len(logit_bias) == 0 + + +@pytest.mark.parametrize("constraint", [ + {"grammar": 'root ::= "ok"'}, + {"json_schema": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}}, +]) +def test_ignore_eos_with_grammar_rejected(constraint): + """ignore_eos is not supported together with a grammar""" + global server + server.start() + res = server.make_request("POST", "/completion", data={ + "n_predict": 8, + "prompt": "Once upon a time", + "ignore_eos": True, + "temperature": 0.0, + **constraint, + }) + assert res.status_code == 400, res.body + assert res.body["error"]["type"] == "invalid_request_error"