Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

common: add json.h abstraction (#27511)

Browse Source 
* add common/json

* migrate common

* adapt jinja

* migrate server

* big wip

* migrate tests

* wip

* revert some excessive changes

* wip

* wip 2

* revert redundant changes

* fix server crash

* various fixes

* fix ci

* harden a bit

* clean up

* rm json-shim

* add some comments

* rm redundant decl
This commit is contained in:
Xuan-Son Nguyen
2026-08-22 16:28:28 +02:00
committed by GitHub
parent 2fb989b9e7
commit d9f918d2d0
45 changed files with 987 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/CMakeLists.txt
+2
View File
@@ -81,6 +81,8 @@ add_library(${TARGET}
imatrix-loader.cpp
imatrix-loader.h
json-schema-to-grammar.cpp
json.cpp
json.h
llguidance.cpp
log.cpp
log.h
common/arg.cpp
+3 -4
View File
@@ -5,6 +5,7 @@
#include "common.h"
#include "download.h"
#include "json-schema-to-grammar.h"
#include "json.h"
#include "llama.h"
#include "log.h"
#include "sampling.h"
@@ -21,9 +22,6 @@
#include <shellapi.h>
#endif
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include <algorithm>
#include <cinttypes>
#include <climits>
@@ -32,6 +30,7 @@
#include <filesystem>
#include <fstream>
#include <list>
#include <numeric>
#include <regex>
#include <set>
#include <string>
@@ -55,7 +54,7 @@
#define LLAMA_MAX_URL_LENGTH 2084 // Maximum URL Length in Chrome: 2083
using json = nlohmann::ordered_json;
using json = common_json;
using namespace common_arg_utils;
static std::initializer_list<enum llama_example> mmproj_examples = {
common/chat-auto-parser-generator.cpp
+2 -3
View File
@@ -5,13 +5,12 @@
#include "common.h"
#include "json-schema-to-grammar.h"
#include "log.h"
#include "nlohmann/json.hpp"
#include "peg-parser.h"
#include <stdexcept>
#include <string>
using json = nlohmann::ordered_json;
using json = common_json;
// Helper to iterate over tools/functions
static void foreach_function(const json & tools, const std::function<void(const json &)> & fn) {
@@ -391,7 +390,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
std::set<std::string> required;
if (params.contains("required")) {
params.at("required").get_to(required);
required = params.at("required").get<std::set<std::string>>();
}
auto schema_info = common_schema_info();
common/chat-auto-parser-helpers.cpp
-3
View File
@@ -4,14 +4,11 @@
#include "chat-peg-parser.h"
#include "chat.h"
#include "log.h"
#include "nlohmann/json.hpp"
#include "peg-parser.h"
#include <cctype>
#include <numeric>
using json = nlohmann::ordered_json;
std::string trim_whitespace(const std::string & str) {
size_t start = 0;
while (start < str.length() && std::isspace(static_cast<unsigned char>(str[start]))) {
common/chat-auto-parser.h
+2 -2
View File
@@ -4,7 +4,7 @@
#include "common.h"
#include "jinja/caps.h"
#include "peg-parser.h"
#include "nlohmann/json.hpp"
#include "json.h"
#include <chrono>
#include <optional>
@@ -12,7 +12,7 @@
#include <utility>
#include <vector>
using json = nlohmann::ordered_json;
using json = common_json;
class common_chat_peg_builder;
common/chat-diff-analyzer.cpp
+3 -3
View File
@@ -4,11 +4,11 @@
#include "chat.h"
#include "common.h"
#include "log.h"
#include "nlohmann/json.hpp"
#include "peg-parser.h"
#include <algorithm>
#include <cctype>
#include <numeric>
#include <ostream>
#include <sstream>
@@ -17,7 +17,7 @@
#define ANSI_ORANGE "\033[1m\x1b[38;5;214m"
#define ANSI_RED "\033[1m\x1b[38;5;196m"
using json = nlohmann::ordered_json;
using json = common_json;
namespace autoparser {
@@ -929,7 +929,7 @@ void analyze_tools::analyze_tool_call_format_json_native(const std::string & cle
int json_end = clean_haystack.find_last_of('}');
std::string cut = clean_haystack.substr(json_start, json_end - json_start + 1);
json call_struct = json::parse(cut);
auto register_field = [&](const std::string & prefix, const nlohmann::detail::iteration_proxy_value<json::iterator> & subel) {
auto register_field = [&](const std::string & prefix, const common_json_entry & subel) {
if (subel.value().is_string() && std::string(subel.value()).find("call0000") != std::string::npos) {
format.id_field = !prefix.empty() ? prefix + "." + subel.key() : subel.key();
} else if (subel.value().is_string() && std::string(subel.value()) == fun_name_needle) {
common/chat-peg-parser.cpp
+1 -3
View File
@@ -4,12 +4,10 @@
#include "ggml.h"
#include "peg-parser.h"
#include <nlohmann/json.hpp>
#include <cstdint>
#include <functional>
using ordered_json = nlohmann::ordered_json;
using ordered_json = common_json;
static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
int count = 0;
common/chat-peg-parser.h
+6 -6
View File
@@ -128,7 +128,7 @@ class common_chat_peg_builder : public common_peg_parser_builder {
// parameters_order: order in which JSON fields should be parsed
common_peg_parser standard_json_tools(const std::string & section_start,
const std::string & section_end,
const nlohmann::ordered_json & tools,
const common_json & tools,
bool parallel_tool_calls,
bool force_tool_calls,
const std::string & name_key = "",
@@ -143,13 +143,13 @@ class common_chat_peg_builder : public common_peg_parser_builder {
// Legacy-compatible helper for building XML/tagged style tool calls
// Used by tests and manual parsers
common_peg_parser standard_constructed_tools(const std::map<std::string, std::string> & markers,
const nlohmann::ordered_json & tools,
const common_json & tools,
bool parallel_tool_calls,
bool force_tool_calls);
// Helper for Python-style function call format: name(arg1="value1", arg2=123)
// Used by LFM2 and similar templates
common_peg_parser python_style_tool_calls(const nlohmann::ordered_json & tools,
common_peg_parser python_style_tool_calls(const common_json & tools,
bool parallel_tool_calls,
bool allow_json_literals);
@@ -158,19 +158,19 @@ class common_chat_peg_builder : public common_peg_parser_builder {
common_peg_parser python_or_json_value();
// Implementation helpers for standard_json_tools — one per JSON tool call layout mode
common_peg_parser build_json_tools_function_is_key(const nlohmann::ordered_json & tools,
common_peg_parser build_json_tools_function_is_key(const common_json & tools,
const std::string & args_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key);
common_peg_parser build_json_tools_nested_keys(const nlohmann::ordered_json & tools,
common_peg_parser build_json_tools_nested_keys(const common_json & tools,
const std::string & effective_name_key,
const std::string & effective_args_key,
const std::string & call_id_key,
const std::string & gen_call_id_key);
common_peg_parser build_json_tools_flat_keys(const nlohmann::ordered_json & tools,
common_peg_parser build_json_tools_flat_keys(const common_json & tools,
const std::string & effective_name_key,
const std::string & effective_args_key,
const std::string & call_id_key,
common/chat.cpp
+19 -19
View File
@@ -6,6 +6,7 @@
#include "common.h"
#include "ggml.h"
#include "json-schema-to-grammar.h"
#include "json.h"
#include "log.h"
#include "jinja/value.h"
@@ -13,14 +14,13 @@
#include "jinja/caps.h"
#include "peg-parser.h"
#include "nlohmann/json.hpp"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <exception>
#include <functional>
#include <iomanip>
#include <map>
#include <optional>
@@ -30,7 +30,7 @@
#include <utility>
#include <vector>
using json = nlohmann::ordered_json;
using json = common_json;
static std::string format_time(const std::chrono::system_clock::time_point & now, const std::string & format) {
auto time = std::chrono::system_clock::to_time_t(now);
@@ -48,7 +48,7 @@ static json safe_args_parse(const std::string & to_parse) {
}
try {
return json::parse(stripped);
} catch (json::exception & e) {
} catch (const common_json_error & e) {
return stripped;
}
}
@@ -488,17 +488,17 @@ struct messages_inp_normalizer {
json normalized = json::array();
for (const auto & msg : messages) {
json copy = msg;
auto it = copy.find("content");
if (it != copy.end()) {
if (only_typed && it->is_string()) {
*it = json::array({
if (copy.contains("content")) {
json & it = copy.at("content");
if (only_typed && it.is_string()) {
it = json::array({
json{
{"type", "text"},
{"text", it->get<std::string>()},
{"text", it.get<std::string>()},
}
});
} else if (only_string && it->is_array()) {
*it = concat_content_parts(*it);
} else if (only_string && it.is_array()) {
it = concat_content_parts(it);
}
}
normalized.push_back(std::move(copy));
@@ -608,7 +608,7 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
return result;
}
common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value) {
common_chat_continuation common_chat_continuation_parse(const common_json & value) {
if (value.is_boolean() && value.get<bool>()) {
return COMMON_CHAT_CONTINUATION_AUTO;
}
@@ -920,7 +920,7 @@ static void foreach_parameter(const json &
const auto & props = params.at("properties");
std::set<std::string> required;
if (params.contains("required") && params.at("required").is_array()) {
params.at("required").get_to(required);
required = params.at("required").get<std::set<std::string>>();
}
for (const auto & [name, prop] : props.items()) {
bool is_required = (required.find(name) != required.end());
@@ -937,7 +937,7 @@ static std::string common_chat_template_direct_apply_impl(
jinja::context ctx(tmpl.source());
// messages_override is already built for this template, do not touch its content parts
nlohmann::ordered_json inp = nlohmann::ordered_json{
json inp = json{
{"messages", messages_override.has_value()
? *messages_override
: messages_inp_normalizer(tmpl.original_caps()).normalize(inputs.messages)},
@@ -1058,7 +1058,7 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_
});
} else if (msg.at("content").is_array()) {
auto blocks = msg.at("content");
content.insert(content.end(), blocks.begin(), blocks.end());
content.insert(blocks);
}
}
@@ -2238,7 +2238,7 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
std::set<std::string> required;
if (params.contains("required")) {
params.at("required").get_to(required);
required = params.at("required").get<std::set<std::string>>();
}
auto schema_info = common_schema_info();
@@ -2860,7 +2860,7 @@ static common_chat_params common_chat_params_init_minimax_m3(const common_chat_t
std::set<std::string> required;
if (schema.contains("required")) {
schema.at("required").get_to(required);
required = schema.at("required").get<std::set<std::string>>();
}
std::vector<common_peg_parser> required_elements;
@@ -2972,10 +2972,10 @@ static void system_message_not_supported(json & messages) {
auto & second_msg = messages[1];
second_msg["content"] = first_msg.at("content").get<std::string>()
+ "\n" + second_msg.at("content").get<std::string>();
messages.erase(messages.begin());
messages.erase(0);
} else {
LOG_WRN("Removing system prompt due to template not supporting system role\n");
messages.erase(messages.begin());
messages.erase(0);
}
}
}
common/chat.h
+9 -10
View File
@@ -8,7 +8,7 @@
#include "jinja/runtime.h"
#include "jinja/caps.h"
#include "nlohmann/json_fwd.hpp"
#include "json.h"
#include <chrono>
#include <functional>
@@ -17,7 +17,6 @@
#include <vector>
using chat_template_caps = jinja::caps;
using json = nlohmann::ordered_json;
struct common_chat_templates;
@@ -87,7 +86,7 @@ struct common_chat_msg {
std::string tool_name;
std::string tool_call_id;
nlohmann::ordered_json to_json_oaicompat(bool concat_typed_text = false) const;
common_json to_json_oaicompat(bool concat_typed_text = false) const;
std::string render_content(const std::string & delimiter = "\n\n") const;
@@ -211,7 +210,7 @@ struct common_chat_msg_delimiters {
// split tokens into message spans. skips maps a start index to a length of a region to jump over without matching
common_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;
nlohmann::ordered_json to_json() const;
common_json to_json() const;
};
struct common_chat_tool {
@@ -350,16 +349,16 @@ common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::strin
bool common_chat_templates_support_enable_thinking(const common_chat_templates * chat_templates);
// Parses a JSON array of messages in OpenAI's chat completion API format.
std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const nlohmann::ordered_json & messages);
std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const common_json & messages);
std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const nlohmann::ordered_json & tools);
std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const common_json & tools);
common_chat_continuation common_chat_continuation_parse(const nlohmann::ordered_json & value);
common_chat_continuation common_chat_continuation_parse(const common_json & value);
// DEPRECATED: only used in tests
nlohmann::ordered_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);
common_json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text = false);
nlohmann::ordered_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & tools);
// get template caps, useful for reporting to server /props endpoint
std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
@@ -386,4 +385,4 @@ struct common_chat_prompt_preset {
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const nlohmann::ordered_json & delimiters);
common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);
common/download.cpp
+6 -10
View File
@@ -5,9 +5,7 @@
#include "log.h"
#include "download.h"
#include "hf-cache.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include "json.h"
#include <algorithm>
#include <filesystem>
@@ -44,8 +42,6 @@
#include <unistd.h>
#endif
using json = nlohmann::ordered_json;
//
// downloader
//
@@ -856,8 +852,8 @@ static std::string common_docker_get_token(const std::string & repo) {
throw std::runtime_error("Failed to get Docker registry token, HTTP code: " + std::to_string(res.first));
}
std::string response_str(res.second.begin(), res.second.end());
nlohmann::ordered_json response = nlohmann::ordered_json::parse(response_str);
std::string response_str(res.second.begin(), res.second.end());
common_json response = common_json::parse(response_str);
if (!response.contains("token")) {
throw std::runtime_error("Docker registry token response missing 'token' field");
@@ -919,9 +915,9 @@ std::string common_docker_resolve_model(const std::string & docker) {
throw std::runtime_error("Failed to get Docker manifest, HTTP code: " + std::to_string(manifest_res.first));
}
std::string manifest_str(manifest_res.second.begin(), manifest_res.second.end());
nlohmann::ordered_json manifest = nlohmann::ordered_json::parse(manifest_str);
std::string gguf_digest; // Find the GGUF layer
std::string manifest_str(manifest_res.second.begin(), manifest_res.second.end());
common_json manifest = common_json::parse(manifest_str);
std::string gguf_digest; // Find the GGUF layer
if (manifest.contains("layers")) {
for (const auto & layer : manifest["layers"]) {
if (layer.contains("mediaType")) {
common/hf-cache.cpp
+7 -11
View File
@@ -4,9 +4,7 @@
#include "common.h"
#include "log.h"
#include "http.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include "json.h"
#include <filesystem>
#include <fstream>
@@ -15,8 +13,6 @@
#include <string_view>
#include <stdexcept>
namespace nl = nlohmann;
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
@@ -195,8 +191,8 @@ static void safe_write_file(const fs::path & path, const std::string & data) {
}
}
static nl::json api_get(const std::string & url,
const std::string & token) {
static common_json api_get(const std::string & url,
const std::string & token) {
auto [cli, parts] = common_http_client(url);
httplib::Headers headers = {
@@ -214,10 +210,10 @@ static nl::json api_get(const std::string & url,
auto body = res->body;
if (res->status == 200) {
return nl::json::parse(res->body);
return common_json::parse(res->body);
}
try {
body = nl::json::parse(res->body)["error"].get<std::string>();
body = common_json::parse(res->body)["error"].get<std::string>();
} catch (...) { }
throw std::runtime_error("GET failed (" + std::to_string(res->status) + "): " + body);
@@ -280,7 +276,7 @@ static std::string get_repo_commit(const std::string & repo_id,
safe_write_file(refs_path / name, commit);
return commit;
} catch (const nl::json::exception & e) {
} catch (const common_json_error & e) {
LOG_ERR("%s: JSON error: %s\n", __func__, e.what());
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
@@ -358,7 +354,7 @@ hf_files get_repo_files(const std::string & repo_id,
files.push_back(file);
}
} catch (const nl::json::exception & e) {
} catch (const common_json_error & e) {
LOG_ERR("%s: JSON error: %s\n", __func__, e.what());
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
common/jinja/README.md
+1 -1
View File
@@ -7,7 +7,7 @@ The implementation can be found in the `common/jinja` directory.
## Key Features
- Input marking: security against special token injection
- Decoupled from `nlohmann::json`: this dependency is only used for JSON-to-internal type translation and is completely optional
- Decoupled from the JSON library: `common_json` is only used for JSON-to-internal type translation and is completely optional
- Minimal primitive types: int, float, bool, string, array, object, none, undefined
- Detailed logging: allow source tracing on error
- Clean architecture: workarounds are applied to input data before entering the runtime (see `common/chat.cpp`)
common/jinja/caps.cpp
+2 -2
View File
@@ -4,14 +4,14 @@
// note: the json dependency is only for defining input in a convenient way
// we can remove it in the future when we figure out a better way to define inputs using jinja::value
#include <nlohmann/json.hpp>
#include "json.h"
#include <functional>
#include <sstream>
#define FILENAME "jinja-caps"
using json = nlohmann::ordered_json;
using json = common_json;
namespace jinja {
common/jinja/value.cpp
+3 -3
View File
@@ -3,7 +3,7 @@
#include "value.h"
// for converting from JSON to jinja values
#include <nlohmann/json.hpp>
#include "json.h"
#include <sstream>
#include <string>
@@ -1355,7 +1355,7 @@ const func_builtins & value_undefined_t::get_builtins() const {
//////////////////////////////////
static value from_json(const nlohmann::ordered_json & j, bool mark_input) {
static value from_json(const common_json & j, bool mark_input) {
if (j.is_null()) {
return mk_val<value_none>();
} else if (j.is_boolean()) {
@@ -1452,7 +1452,7 @@ bool value_compare(const value & a, const value & b, value_compare_op op) {
}
template<>
void global_from_json(context & ctx, const nlohmann::ordered_json & json_obj, bool mark_input) {
void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
// printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
if (json_obj.is_null() || !json_obj.is_object()) {
throw std::runtime_error("global_from_json: input JSON value must be an object");
common/jinja/value.h
+1 -1
View File
@@ -86,7 +86,7 @@ struct context; // forward declaration
// marking input can be useful for tracking data provenance
// and preventing template injection attacks
//
// Note: T_JSON can be nlohmann::ordered_json
// Note: T_JSON can be common_json
template<typename T_JSON>
void global_from_json(context & ctx, const T_JSON & json_obj, bool mark_input);
common/json-schema-to-grammar.cpp
+12 -9
View File
@@ -1,9 +1,8 @@
#include "json-schema-to-grammar.h"
#include "common.h"
#include <nlohmann/json.hpp>
#include <algorithm>
#include <limits>
#include <map>
#include <regex>
#include <sstream>
@@ -12,7 +11,7 @@
#include <unordered_set>
#include <vector>
using json = nlohmann::ordered_json;
using json = common_json;
static std::string build_repetition(const std::string & item_rule, int min_items, int max_items, const std::string & separator_rule = "") {
auto has_max = max_items != std::numeric_limits<int>::max();
@@ -917,7 +916,11 @@ public:
return _add_rule(rule_name, _resolve_ref(schema["$ref"]));
}
if (schema.contains("oneOf") || schema.contains("anyOf")) {
std::vector<json> alt_schemas = schema.contains("oneOf") ? schema["oneOf"].get<std::vector<json>>() : schema["anyOf"].get<std::vector<json>>();
const json & alts = schema.contains("oneOf") ? schema.at("oneOf") : schema.at("anyOf");
std::vector<json> alt_schemas;
for (const auto & alt : alts) {
alt_schemas.push_back(alt);
}
return _add_rule(rule_name, _generate_union_rule(name, alt_schemas));
}
if (schema_type.is_array()) {
@@ -1111,7 +1114,7 @@ common_schema_info::~common_schema_info() = default;
common_schema_info::common_schema_info(common_schema_info &&) noexcept = default;
common_schema_info & common_schema_info::operator=(common_schema_info &&) noexcept = default;
void common_schema_info::resolve_refs(nlohmann::ordered_json & schema) {
void common_schema_info::resolve_refs(common_json & schema) {
impl_->resolve_refs(schema, "");
}
@@ -1119,7 +1122,7 @@ void common_schema_info::resolve_refs(nlohmann::ordered_json & schema) {
// Some models emit raw string values rather than JSON-encoded strings for string parameters.
// If any branch of the schema (via oneOf, anyOf, $ref, etc.) permits a string, this returns
// true, allowing callers to handle the value as a raw string for simplicity.
bool common_schema_info::resolves_to_string(const nlohmann::ordered_json & schema) {
bool common_schema_info::resolves_to_string(const common_json & schema) {
std::unordered_set<std::string> visited_refs;
std::function<bool(const json &)> check = [&](const json & s) -> bool {
@@ -1227,7 +1230,7 @@ bool common_schema_info::resolves_to_string(const nlohmann::ordered_json & schem
return check(schema);
}
std::string json_schema_to_grammar(const json & schema, bool force_gbnf) {
std::string json_schema_to_grammar(const common_json & schema, bool force_gbnf) {
#ifdef LLAMA_USE_LLGUIDANCE
if (!force_gbnf) {
return "%llguidance {}\nstart: %json " + schema.dump();
@@ -1248,10 +1251,10 @@ std::string build_grammar(const std::function<void(const common_grammar_builder
/* .add_rule = */ [&](const std::string & name, const std::string & rule) {
return converter._add_rule(name, rule);
},
/* .add_schema = */ [&](const std::string & name, const nlohmann::ordered_json & schema) {
/* .add_schema = */ [&](const std::string & name, const common_json & schema) {
return converter.visit(schema, name == "root" ? "" : name);
},
/* .resolve_refs = */ [&](nlohmann::ordered_json & schema) {
/* .resolve_refs = */ [&](common_json & schema) {
converter.resolve_refs(schema, "");
}
};
common/json-schema-to-grammar.h
+6 -6
View File
@@ -1,12 +1,12 @@
#pragma once
#include <nlohmann/json_fwd.hpp>
#include "json.h"
#include <functional>
#include <memory>
#include <string>
std::string json_schema_to_grammar(const nlohmann::ordered_json & schema,
std::string json_schema_to_grammar(const common_json & schema,
bool force_gbnf = false);
class common_schema_converter;
@@ -24,14 +24,14 @@ class common_schema_info {
common_schema_info(common_schema_info &&) noexcept;
common_schema_info & operator=(common_schema_info &&) noexcept;
void resolve_refs(nlohmann::ordered_json & schema);
bool resolves_to_string(const nlohmann::ordered_json & schema);
void resolve_refs(common_json & schema);
bool resolves_to_string(const common_json & schema);
};
struct common_grammar_builder {
std::function<std::string(const std::string &, const std::string &)> add_rule;
std::function<std::string(const std::string &, const nlohmann::ordered_json &)> add_schema;
std::function<void(nlohmann::ordered_json &)> resolve_refs;
std::function<std::string(const std::string &, const common_json &)> add_schema;
std::function<void(common_json &)> resolve_refs;
};
struct common_grammar_options {
common/json.cpp
+437
View File
@@ -0,0 +1,437 @@
#include "json.h"
#include "ggml.h"
#define JSON_ASSERT GGML_ASSERT
#include <nlohmann/json.hpp>
#include <iterator>
#include <new>
#include <set>
#include <unordered_map>
#include <vector>
using nlohmann::ordered_json;
// a common_json is the backing value, so any value of a tree can be used as a common_json
static_assert(sizeof(ordered_json) <= sizeof(common_json), "common_json storage is too small");
static_assert(alignof(ordered_json) <= alignof(common_json), "common_json alignment is too weak");
// runs fn and gives every error of the backing library as a common_json_error
template <typename F>
static decltype(auto) guard(F && fn) {
try {
return fn();
} catch (const ordered_json::exception & e) {
throw common_json_error(e.what());
}
}
static ordered_json & as_json(common_json * self) {
return *reinterpret_cast<ordered_json *>(self);
}
static const ordered_json & as_json(const common_json * self) {
return *reinterpret_cast<const ordered_json *>(self);
}
static common_json & as_common(ordered_json & json) {
return *reinterpret_cast<common_json *>(&json);
}
static const common_json & as_common(const ordered_json & json) {
return *reinterpret_cast<const common_json *>(&json);
}
static ordered_json to_json(const common_json_value & val) {
switch (val.type) {
case common_json_value::VAL_NULL: return nullptr;
case common_json_value::VAL_BOOL: return val.val_bool;
case common_json_value::VAL_INT: return val.val_int;
case common_json_value::VAL_UINT: return val.val_uint;
case common_json_value::VAL_DOUBLE: return val.val_double;
case common_json_value::VAL_STRING: return val.val_string;
case common_json_value::VAL_JSON:
// one owner means no one else can see this tree, so it is safe to move it out
// note: this makes a value single use, same as the json_ref of the backing library
if (val.val_json.use_count() == 1) {
return std::move(as_json(val.val_json.get()));
}
return as_json(val.val_json.get());
}
return nullptr;
}
common_json_value::common_json_value(const char * val) {
if (val) {
type = VAL_STRING;
val_string = val;
} else {
type = VAL_NULL;
}
}
common_json_value::common_json_value(const common_json & val) :
type(VAL_JSON), val_json(std::make_shared<common_json>(val)) {}
common_json_value::common_json_value(common_json && val) :
type(VAL_JSON), val_json(std::make_shared<common_json>(std::move(val))) {}
template <typename T>
common_json_value::common_json_value(const std::set<T> & vals) : type(VAL_JSON) {
common_json out = common_json::array();
for (const auto & val : vals) {
out.push_back(val);
}
val_json = std::make_shared<common_json>(std::move(out));
}
// a set value is usable only for the types below
#define COMMON_JSON_SET(...) template common_json_value::common_json_value(const std::set<__VA_ARGS__> &);
COMMON_JSON_SET(int)
COMMON_JSON_SET(std::string)
#undef COMMON_JSON_SET
template <typename T>
common_json_value::common_json_value(const std::map<std::string, T> & vals) : type(VAL_JSON) {
common_json out = common_json::object();
for (const auto & val : vals) {
out.set({ val.first, val.second });
}
val_json = std::make_shared<common_json>(std::move(out));
}
// a map value is usable only for the types below
#define COMMON_JSON_MAP(...) template common_json_value::common_json_value(const std::map<std::string, __VA_ARGS__> &);
COMMON_JSON_MAP(bool)
COMMON_JSON_MAP(std::string)
#undef COMMON_JSON_MAP
template <typename T>
common_json_value::common_json_value(const std::unordered_map<std::string, T> & vals) : type(VAL_JSON) {
common_json out = common_json::object();
for (const auto & val : vals) {
out.set({ val.first, val.second });
}
val_json = std::make_shared<common_json>(std::move(out));
}
// an unordered map value is usable only for the types below
#define COMMON_JSON_UMAP(...) template common_json_value::common_json_value(const std::unordered_map<std::string, __VA_ARGS__> &);
COMMON_JSON_UMAP(size_t)
#undef COMMON_JSON_UMAP
template <typename T>
common_json_value::common_json_value(const std::vector<T> & vals) : type(VAL_JSON) {
common_json out = common_json::array();
for (const auto & val : vals) {
out.push_back(val);
}
val_json = std::make_shared<common_json>(std::move(out));
}
// a vector value is usable only for the types below
// note: std::vector<bool> is not here, its proxy reference does not convert
#define COMMON_JSON_VEC(...) template common_json_value::common_json_value(const std::vector<__VA_ARGS__> &);
COMMON_JSON_VEC(int)
COMMON_JSON_VEC(unsigned char)
COMMON_JSON_VEC(unsigned int)
COMMON_JSON_VEC(long)
COMMON_JSON_VEC(unsigned long)
COMMON_JSON_VEC(long long)
COMMON_JSON_VEC(unsigned long long)
COMMON_JSON_VEC(float)
COMMON_JSON_VEC(double)
COMMON_JSON_VEC(std::string)
COMMON_JSON_VEC(std::vector<float>)
COMMON_JSON_VEC(common_json)
#undef COMMON_JSON_VEC
common_json_value::common_json_value(std::initializer_list<common_json_item> items) :
type(VAL_JSON), val_json(std::make_shared<common_json>(items)) {}
// null, same as the backing library
// operator[] turns it into an object, push_back() into an array
common_json::common_json() {
new (storage) ordered_json();
}
common_json::common_json(const common_json & other) {
new (storage) ordered_json(as_json(&other));
}
common_json::common_json(common_json && other) noexcept {
new (storage) ordered_json(std::move(as_json(&other)));
}
common_json::common_json(std::initializer_list<common_json_item> items) {
new (storage) ordered_json(ordered_json::object());
for (const auto & item : items) {
set(item);
}
}
common_json::common_json(const common_json_value & val) {
new (storage) ordered_json(to_json(val));
}
common_json::common_json(std::nullptr_t) {
new (storage) ordered_json(nullptr);
}
common_json & common_json::operator=(common_json other) noexcept {
as_json(this).swap(as_json(&other));
return *this;
}
common_json::~common_json() {
as_json(this).~basic_json();
}
common_json common_json::parse(const std::string & text) {
try {
// the assignment moves the parsed tree in, it does not copy
common_json out;
as_json(&out) = ordered_json::parse(text);
return out;
} catch (const std::exception & e) {
throw common_json_error(e.what());
}
}
common_json common_json::parse_no_throw(const std::string & text) {
common_json out;
as_json(&out) = ordered_json::parse(text, nullptr, false);
return out;
}
bool common_json::is_discarded() const {
return as_json(this).is_discarded();
}
common_json common_json::array() {
common_json out;
as_json(&out) = ordered_json::array();
return out;
}
common_json common_json::array(std::initializer_list<common_json_value> vals) {
common_json out;
ordered_json & arr = as_json(&out);
arr = ordered_json::array();
for (const auto & val : vals) {
arr.push_back(to_json(val));
}
return out;
}
common_json common_json::object() {
common_json out;
as_json(&out) = ordered_json::object();
return out;
}
common_json common_json::object(std::initializer_list<common_json_item> items) {
return common_json(items);
}
common_json common_json::make(const common_json_value & val) {
return common_json(val);
}
bool common_json::is_null() const { return as_json(this).is_null(); }
bool common_json::is_object() const { return as_json(this).is_object(); }
bool common_json::is_array() const { return as_json(this).is_array(); }
bool common_json::is_string() const { return as_json(this).is_string(); }
bool common_json::is_boolean() const { return as_json(this).is_boolean(); }
bool common_json::is_number() const { return as_json(this).is_number(); }
bool common_json::is_number_integer() const { return as_json(this).is_number_integer(); }
bool common_json::is_number_float() const { return as_json(this).is_number_float(); }
bool common_json::empty() const { return as_json(this).empty(); }
size_t common_json::size() const { return as_json(this).size(); }
bool common_json::contains(const std::string & key) const {
return as_json(this).contains(key);
}
bool common_json::operator==(const common_json_value & val) const {
// compare a tree in place, to_json() would copy it
if (val.type == common_json_value::VAL_JSON) {
return as_json(this) == as_json(val.val_json.get());
}
return as_json(this) == to_json(val);
}
bool common_json::operator!=(const common_json_value & val) const {
return !(*this == val);
}
common_json & common_json::at(const std::string & key) { return guard([&]() -> common_json & { return as_common(as_json(this).at(key)); }); }
const common_json & common_json::at(const std::string & key) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(key)); }); }
common_json & common_json::at(size_t idx) { return guard([&]() -> common_json & { return as_common(as_json(this).at(idx)); }); }
const common_json & common_json::at(size_t idx) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(idx)); }); }
common_json & common_json::operator[](const std::string & key) { return guard([&]() -> common_json & { return as_common(as_json(this)[key]); }); }
const common_json & common_json::operator[](const std::string & key) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(key)); }); }
common_json & common_json::operator[](size_t idx) { return guard([&]() -> common_json & { return as_common(as_json(this)[idx]); }); }
const common_json & common_json::operator[](size_t idx) const { return guard([&]() -> const common_json & { return as_common(as_json(this).at(idx)); }); }
common_json & common_json::front() { return as_common(as_json(this).front()); }
const common_json & common_json::front() const { return as_common(as_json(this).front()); }
common_json & common_json::back() { return as_common(as_json(this).back()); }
const common_json & common_json::back() const { return as_common(as_json(this).back()); }
void common_json::clear() {
as_json(this).clear();
}
void common_json::erase(const std::string & key) {
guard([&] { as_json(this).erase(key); });
}
void common_json::erase(size_t idx) {
guard([&] { as_json(this).erase(idx); });
}
void common_json::assign(const common_json_value & val) {
as_json(this) = to_json(val);
}
void common_json::set(const common_json_item & item) {
guard([&] { as_json(this)[item.key] = to_json(item.val); });
}
void common_json::push_back(const common_json_value & val) {
guard([&] { as_json(this).push_back(to_json(val)); });
}
void common_json::push_back(std::initializer_list<common_json_item> items) {
common_json val(items);
guard([&] { as_json(this).push_back(std::move(as_json(&val))); });
}
size_t common_json::count(const std::string & key) const {
return as_json(this).count(key);
}
void common_json::insert(const common_json & vals) {
guard([&] {
ordered_json & self = as_json(this);
self.insert(self.end(), as_json(&vals).begin(), as_json(&vals).end());
});
}
std::string common_json::dump(int indent) const {
return guard([&] { return as_json(this).dump(indent); });
}
std::string common_json::dump_safe(int indent) const {
return as_json(this).dump(indent, ' ', false, ordered_json::error_handler_t::replace);
}
// an array is indexed directly, an object needs a walk from the start
common_json & common_json::iterator::operator*() const {
return guard([&]() -> common_json & {
ordered_json & j = as_json(node);
if (j.is_object()) {
return as_common(std::next(j.begin(), idx).value());
}
if (j.is_array()) {
return as_common(j[idx]);
}
// a plain value gives itself once, same as the backing library
return *node;
});
}
std::string common_json::iterator::key() const {
return guard([&] { return std::next(as_json(node).begin(), idx).key(); });
}
common_json::iterator common_json::begin() const {
return iterator(const_cast<common_json *>(this), 0);
}
common_json::iterator common_json::end() const {
return iterator(const_cast<common_json *>(this), size());
}
// the keys follow the backing library: the index for an array, "" for a plain value
common_json::items_view::entry common_json::items_view::iterator::operator*() const {
return guard([&]() -> entry {
ordered_json & j = as_json(node);
if (j.is_object()) {
auto it = std::next(j.begin(), idx);
return { it.key(), as_common(it.value()) };
}
if (j.is_array()) {
return { std::to_string(idx), as_common(j[idx]) };
}
return { std::string(), *node };
});
}
common_json::items_view common_json::items() const {
return items_view(const_cast<common_json *>(this), size());
}
template <typename T> T common_json::get() const {
return guard([&] { return as_json(this).get<T>(); });
}
// the backing library cannot build a common_json, so this one is just a copy
template <> common_json common_json::get<common_json>() const {
return *this;
}
// get<T>() is usable only for the types below
#define COMMON_JSON_GET(...) template __VA_ARGS__ common_json::get<__VA_ARGS__>() const;
COMMON_JSON_GET(bool)
COMMON_JSON_GET(int)
COMMON_JSON_GET(unsigned int)
COMMON_JSON_GET(long)
COMMON_JSON_GET(unsigned long)
COMMON_JSON_GET(long long)
COMMON_JSON_GET(unsigned long long)
COMMON_JSON_GET(float)
COMMON_JSON_GET(double)
COMMON_JSON_GET(std::string)
COMMON_JSON_GET(std::vector<float>)
COMMON_JSON_GET(std::vector<std::string>)
COMMON_JSON_GET(std::set<std::string>)
COMMON_JSON_GET(std::vector<int>)
COMMON_JSON_GET(std::vector<size_t>)
COMMON_JSON_GET(std::unordered_map<std::string, size_t>)
#undef COMMON_JSON_GET
common/json.h
+354
View File
@@ -0,0 +1,354 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <iterator>
#include <map>
#include <memory>
#include <set>
#include <stdexcept>
#include <string>
#include <string_view>
#include <type_traits>
#include <unordered_map>
#include <utility>
#include <vector>
// common_json, a thin wrapper around vendor json library
// the underlay library is pimpl, we are using nlohmann::json for now
//
// many features of the library are deliberately left out, to keep this interface small and generic and to keep compile time down
//
// some main differences compared to nlohmann::json :
// - object keys keep the order in which they are added
// - errors are always throw as common_json_error
// - obj.push_back({key, val}) is intentionally unsupported to avoid confusion with push_back on a vector; write it as obj[key] = val for clarity
// - a braced pair in value position does not build, e.g. {"key", {"a", "b"}}; write array({"a", "b"}) where nlohmann made an array
//
// in doubt, search the code base for an existing usage example; do not add anything to this header unless absolutely necessary
class common_json;
// common_json_value holds a list of these, and each of them holds a value, so one must come first
struct common_json_item;
struct common_json_error : std::runtime_error {
using std::runtime_error::runtime_error;
};
// one value, tagged so that this header stays free of the backing library
// note: a value that holds a tree is single use, the second use gives null
struct common_json_value {
enum value_type {
VAL_NULL,
VAL_BOOL,
VAL_INT,
VAL_UINT,
VAL_DOUBLE,
VAL_STRING,
VAL_JSON,
};
value_type type = VAL_NULL;
union {
bool val_bool;
int64_t val_int;
uint64_t val_uint = 0;
double val_double;
};
std::string val_string;
std::shared_ptr<common_json> val_json;
common_json_value(std::nullptr_t = nullptr) : type(VAL_NULL) {}
common_json_value(bool val) : type(VAL_BOOL), val_bool(val) {}
common_json_value(std::string val) : type(VAL_STRING), val_string(std::move(val)) {}
// without this a string_view lands on the common_json ctor below and recurses
common_json_value(std::string_view val) : type(VAL_STRING), val_string(val) {}
common_json_value(const char * val);
common_json_value(const common_json & val);
common_json_value(common_json && val);
// only for the types instantiated in json.cpp, the rest fails at link time
template <typename T> common_json_value(const std::vector<T> & vals);
// a set becomes an array, in the set's own order
template <typename T> common_json_value(const std::set<T> & vals);
// a map becomes an object, keyed in the map's own order
template <typename T> common_json_value(const std::map<std::string, T> & vals);
template <typename T> common_json_value(const std::unordered_map<std::string, T> & vals);
// nested object, e.g. {"fn", {{"name", "x"}}}
// note: a nested pair {"a", "b"} does not build, use common_json::array({"a", "b"}) for an array
common_json_value(std::initializer_list<common_json_item> items);
template <typename T, typename std::enable_if<std::is_integral<T>::value && !std::is_same<T, bool>::value, int>::type = 0>
common_json_value(T val) : type(std::is_signed<T>::value ? VAL_INT : VAL_UINT) {
if (std::is_signed<T>::value) {
val_int = (int64_t) val;
} else {
val_uint = (uint64_t) val;
}
}
template <typename T, typename std::enable_if<std::is_floating_point<T>::value, int>::type = 0>
common_json_value(T val) : type(VAL_DOUBLE), val_double((double) val) {}
};
struct common_json_item {
std::string key;
common_json_value val;
template <typename T>
common_json_item(std::string key, T && val) :
key(std::move(key)), val(std::forward<T>(val)) {}
// a braced list cannot deduce T, so it needs its own overload
common_json_item(std::string key, std::initializer_list<common_json_item> items) :
key(std::move(key)), val(items) {}
};
// the types common_json_value holds on its own
// anything else reaches its common_json ctor and recurses forever
template <typename T> struct common_json_is_value : std::integral_constant<bool,
std::is_arithmetic<T>::value ||
std::is_same<T, std::nullptr_t>::value ||
std::is_same<T, std::string>::value ||
std::is_same<T, std::string_view>::value ||
std::is_same<T, char *>::value ||
std::is_same<T, const char *>::value ||
std::is_same<T, common_json>::value> {};
template <typename T, typename A>
struct common_json_is_value<std::vector<T, A>> : std::true_type {};
template <typename T, typename C, typename A>
struct common_json_is_value<std::set<T, C, A>> : std::true_type {};
template <typename V, typename C, typename A>
struct common_json_is_value<std::map<std::string, V, C, A>> : std::true_type {};
template <typename V, typename H, typename E, typename A>
struct common_json_is_value<std::unordered_map<std::string, V, H, E, A>> : std::true_type {};
class common_json {
public:
common_json();
common_json(const common_json & other);
common_json(common_json && other) noexcept;
common_json(std::initializer_list<common_json_item> items);
common_json(const common_json_value & val);
// direct, a value would need two conversions in a row
common_json(std::nullptr_t);
// one step, so that "abc" or a vector can go straight into a common_json
template <typename T, typename std::enable_if<!std::is_same<typename std::decay<T>::type, common_json>::value &&
!std::is_same<typename std::decay<T>::type, common_json_value>::value, int>::type = 0>
common_json(T && val) : common_json(common_json_value(std::forward<T>(val))) {
static_assert(common_json_is_value<typename std::decay<T>::type>::value,
"no common_json_value ctor holds this type, add one instead of letting it recurse");
}
// by value, same as the backing library
// the right side is copied before the left side can invalidate it, e.g. msg["a"] = msg.at("b")
common_json & operator=(common_json other) noexcept;
~common_json();
// throws common_json_error if the text is not valid JSON
static common_json parse(const std::string & text);
// gives a discarded value instead of throwing, check it with is_discarded()
static common_json parse_no_throw(const std::string & text);
bool is_discarded() const;
static common_json array();
static common_json array(std::initializer_list<common_json_value> vals);
static common_json object();
static common_json object(std::initializer_list<common_json_item> items);
// holds a single value, e.g. make("abc").dump() gives "\"abc\""
static common_json make(const common_json_value & val);
bool is_null() const;
bool is_object() const;
bool is_array() const;
bool is_string() const;
bool is_boolean() const;
bool is_number() const;
bool is_number_integer() const;
bool is_number_float() const;
bool empty() const;
size_t size() const;
bool contains(const std::string & key) const;
bool operator==(const common_json_value & val) const;
bool operator!=(const common_json_value & val) const;
// at() throws common_json_error if the key is missing, operator[] adds a null value instead
// note: a const operator[] cannot add, it throws like at()
common_json & at(const std::string & key);
const common_json & at(const std::string & key) const;
common_json & at(size_t idx);
const common_json & at(size_t idx) const;
common_json & operator[](const std::string & key);
const common_json & operator[](const std::string & key) const;
common_json & operator[](const char * key) { return (*this)[std::string(key)]; }
const common_json & operator[](const char * key) const { return (*this)[std::string(key)]; }
common_json & operator[](int idx) { return (*this)[to_idx(idx)]; }
const common_json & operator[](int idx) const { return (*this)[to_idx(idx)]; }
common_json & operator[](size_t idx);
const common_json & operator[](size_t idx) const;
common_json & front();
const common_json & front() const;
common_json & back();
const common_json & back() const;
void clear();
void erase(const std::string & key);
void erase(size_t idx);
// only for the types instantiated in json.cpp, the rest fails at link time
template <typename T> T get() const;
// implicit get<T>() for plain values, so they can be assigned to their C++ type directly
// note: kept to this short list on purpose, a wider one makes j["key"] ambiguous
// note: a numeric one would make "str = json;" ambiguous, a number converts to char too
operator std::string() const { return get<std::string>(); }
template <typename T>
T value(const std::string & key, T def) const {
return contains(key) ? at(key).get<T>() : def;
}
std::string value(const std::string & key, const char * def) const {
return contains(key) ? at(key).get<std::string>() : std::string(def);
}
// a JSON default needs no get<T>(), it is already the right type
common_json value(const std::string & key, const common_json & def) const {
return contains(key) ? at(key) : def;
}
void assign(const common_json_value & val);
void set(const common_json_item & item);
void push_back(const common_json_value & val);
// appends one object, e.g. push_back({{"a", 1}})
void push_back(std::initializer_list<common_json_item> items);
// 1 if the key is there, 0 if not
size_t count(const std::string & key) const;
// appends every value of another array; inserting an array into itself throws
void insert(const common_json & vals);
// a common_json goes through the copy assignment above, everything else becomes a value
template <typename T, typename std::enable_if<!std::is_same<typename std::decay<T>::type, common_json>::value, int>::type = 0>
common_json & operator=(T && val) {
assign(common_json_value(std::forward<T>(val)));
return *this;
}
std::string dump(int indent = -1) const;
// same as dump(), but bad UTF-8 gets replaced instead of throwing
std::string dump_safe(int indent = -1) const;
// walks an array by index, or an object in insertion order
// a plain value gives itself once, same as the backing library
class iterator {
public:
using iterator_category = std::forward_iterator_tag;
using value_type = common_json;
using difference_type = std::ptrdiff_t;
using pointer = common_json *;
using reference = common_json &;
iterator(common_json * node, size_t idx) : node(node), idx(idx) {}
common_json & operator*() const;
common_json & value() const { return **this; }
std::string key() const;
iterator & operator++() {
idx++;
return *this;
}
bool operator!=(const iterator & other) const { return idx != other.idx; }
bool operator==(const iterator & other) const { return idx == other.idx; }
private:
common_json * node;
size_t idx;
};
iterator begin() const;
iterator end() const;
// allows: for (const auto & [key, val] : obj.items())
class items_view {
public:
// the members are public, so an entry also works with structured bindings
struct entry {
std::string k;
common_json & v;
const std::string & key() const { return k; }
common_json & value() const { return v; }
};
items_view(common_json * node, size_t n) : node(node), n(n) {}
class iterator {
public:
iterator(common_json * node, size_t idx) : node(node), idx(idx) {}
entry operator*() const;
iterator & operator++() {
idx++;
return *this;
}
bool operator!=(const iterator & other) const { return idx != other.idx; }
private:
common_json * node;
size_t idx;
};
iterator begin() const { return iterator(node, 0); }
iterator end() const { return iterator(node, n); }
private:
common_json * node;
size_t n;
};
items_view items() const;
private:
// a negative index must not turn into a huge size_t
static size_t to_idx(int idx) {
if (idx < 0) {
throw common_json_error("negative array index");
}
return (size_t) idx;
}
// the backing value is built here, json.cpp checks that it fits
// it cannot be a pointer: a value inside a tree would then not be a common_json
// at() could then only give back a copy instead of a real reference
alignas(8) unsigned char storage[32];
};
using common_json_entry = common_json::items_view::entry;
common/peg-parser.cpp
+15 -16
View File
@@ -10,7 +10,6 @@
#include <initializer_list>
#include <map>
#include <memory>
#include <nlohmann/json.hpp>
#include <regex>
#include <set>
#include <stdexcept>
@@ -1120,8 +1119,8 @@ common_peg_parser common_peg_parser_builder::chars(const std::string & classes,
return wrap(arena_.add_parser(common_peg_chars_parser{classes, ranges, negated, min, max}));
}
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<nlohmann::ordered_json>(schema), raw}));
common_peg_parser common_peg_parser_builder::schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw) {
return wrap(arena_.add_parser(common_peg_schema_parser{p.id(), name, std::make_shared<common_json>(schema), raw}));
}
common_peg_parser common_peg_parser_builder::rule(const std::string & name, const common_peg_parser & p, bool trigger) {
@@ -1805,8 +1804,8 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo
}
}
static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & variant) {
using json = nlohmann::json;
static common_json serialize_parser_variant(const common_peg_parser_variant & variant) {
using json = common_json;
return std::visit([](const auto & p) -> json {
using T = std::decay_t<decltype(p)>;
@@ -1860,7 +1859,7 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
{"type", "schema"},
{"child", p.child},
{"name", p.name},
{"schema", p.schema ? *p.schema : nullptr},
{"schema", p.schema ? *p.schema : json(nullptr)},
{"raw", p.raw}
};
} else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {
@@ -1888,19 +1887,19 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant &
}, variant);
}
nlohmann::json common_peg_arena::to_json() const {
auto parsers = nlohmann::json::array();
common_json common_peg_arena::to_json() const {
auto parsers = common_json::array();
for (const auto & parser : parsers_) {
parsers.push_back(serialize_parser_variant(parser));
}
return nlohmann::json{
return common_json{
{"parsers", parsers},
{"rules", rules_},
{"root", root_}
};
}
static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json & j) {
static common_peg_parser_variant deserialize_parser_variant(const common_json & j) {
if (!j.contains("type") || !j["type"].is_string()) {
throw std::runtime_error("Parser variant JSON missing or invalid 'type' field");
}
@@ -1969,9 +1968,9 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
}
common_peg_chars_parser parser;
parser.pattern = j["pattern"];
parser.negated = j["negated"];
parser.min_count = j["min_count"];
parser.max_count = j["max_count"];
parser.negated = j["negated"].get<bool>();
parser.min_count = j["min_count"].get<int>();
parser.max_count = j["max_count"].get<int>();
for (const auto & range_json : j["ranges"]) {
if (!range_json.contains("start") || !range_json.contains("end")) {
throw std::runtime_error("char_range missing 'start' or 'end' field");
@@ -2007,7 +2006,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
parser.child = j["child"].get<common_peg_parser_id>();
parser.name = j["name"];
if (!j["schema"].is_null()) {
parser.schema = std::make_shared<nlohmann::ordered_json>(j["schema"]);
parser.schema = std::make_shared<common_json>(j["schema"]);
}
parser.raw = j["raw"].get<bool>();
return parser;
@@ -2069,7 +2068,7 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json
throw std::runtime_error("Unknown parser type: " + type);
}
common_peg_arena common_peg_arena::from_json(const nlohmann::json & j) {
common_peg_arena common_peg_arena::from_json(const common_json & j) {
if (!j.contains("parsers") || !j["parsers"].is_array()) {
throw std::runtime_error("JSON missing or invalid 'parsers' array");
}
@@ -2109,7 +2108,7 @@ std::string common_peg_arena::save() const {
}
void common_peg_arena::load(const std::string & data) {
*this = from_json(nlohmann::json::parse(data));
*this = from_json(common_json::parse(data));
}
common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {
common/peg-parser.h
+5 -5
View File
@@ -1,6 +1,6 @@
#pragma once
#include <nlohmann/json_fwd.hpp>
#include "json.h"
#include <memory>
#include <set>
@@ -245,7 +245,7 @@ struct common_peg_until_parser {
struct common_peg_schema_parser {
common_peg_parser_id child;
std::string name;
std::shared_ptr<nlohmann::ordered_json> schema;
std::shared_ptr<common_json> schema;
// Indicates if the GBNF should accept a raw string that matches the schema.
bool raw;
@@ -332,8 +332,8 @@ class common_peg_arena {
std::string dump(common_peg_parser_id id) const;
nlohmann::json to_json() const;
static common_peg_arena from_json(const nlohmann::json & j);
common_json to_json() const;
static common_peg_arena from_json(const common_json & j);
std::string save() const;
void load(const std::string & data);
@@ -490,7 +490,7 @@ class common_peg_parser_builder {
// Wraps a parser with JSON schema metadata for grammar generation.
// Used internally to convert JSON schemas to GBNF grammar rules.
common_peg_parser schema(const common_peg_parser & p, const std::string & name, const nlohmann::ordered_json & schema, bool raw = false);
common_peg_parser schema(const common_peg_parser & p, const std::string & name, const common_json & schema, bool raw = false);
// Creates a named rule, stores it in the grammar, and returns a ref.
// If trigger=true, marks this rule as an entry point for lazy grammar generation.