diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index d60a99d25..8bd8ab1c8 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -89,21 +89,22 @@ value statement::execute(context & ctx) const { } value identifier::execute_impl(context & ctx) const { - auto it = ctx.get_val(val); - auto builtins = global_builtins(); - if (!it->is_undefined()) { + auto it = ctx.find_val(key); + if (it && !it->is_undefined()) { if (ctx.is_get_stats) { value_t::stats_t::mark_used(it); } JJ_DEBUG("Identifier '%s' found, type = %s", val.c_str(), it->type().c_str()); return it; - } else if (builtins.find(val) != builtins.end()) { - JJ_DEBUG("Identifier '%s' found in builtins", val.c_str()); - return mk_val(val, builtins.at(val)); - } else { - JJ_DEBUG("Identifier '%s' not found, returning undefined", val.c_str()); - return mk_val(val); } + const auto & builtins = global_builtins(); + auto builtin = builtins.find(val); + if (builtin != builtins.end()) { + JJ_DEBUG("Identifier '%s' found in builtins", val.c_str()); + return mk_val(val, builtin->second); + } + JJ_DEBUG("Identifier '%s' not found, returning undefined", val.c_str()); + return mk_val(val); } value object_literal::execute_impl(context & ctx) const { @@ -693,7 +694,7 @@ value set_statement::execute_impl(context & ctx) const { } auto obj_ptr = cast_val(object); JJ_DEBUG("Setting object property '%s' with value type %s", prop_name.c_str(), rhs->type().c_str()); - obj_ptr->insert(prop_name, rhs); + obj_ptr->insert(cached_val(ctx, member->static_property), rhs); } else { throw std::runtime_error("Invalid LHS inside assignment expression: " + assignee->type()); @@ -813,6 +814,19 @@ value call_statement::execute_impl(context & ctx) const { return callee_func->invoke(args); } +member_expression::member_expression(statement_ptr && object, statement_ptr && property, bool computed) : + object(std::move(object)), + property(std::move(property)), + computed(computed) { + chk_type(this->object); + chk_type(this->property); + if (!computed) { + if (const auto * id = cast_stmt(this->property)) { + static_property = id->key; + } + } +} + value member_expression::execute_impl(context & ctx) const { value object = this->object->execute(ctx); @@ -821,14 +835,13 @@ value member_expression::execute_impl(context & ctx) const { // syntax: obj[expr] JJ_DEBUG("Member expression, computing property type %s", this->property->type().c_str()); - int64_t arr_size = 0; - if (is_val(object)) { - arr_size = object->as_array().size(); - } else if (is_val(object)) { - arr_size = object->as_string().length(); - } - if (is_stmt(this->property)) { + int64_t arr_size = 0; + if (is_val(object)) { + arr_size = static_cast(object->as_array().size()); + } else if (is_val(object)) { + arr_size = static_cast(cast_val(object)->val_str.length()); + } auto s = cast_stmt(this->property); value step_val = s->step_expr ? s->step_expr->execute(ctx) : mk_val(1); value start_val = s->start_expr ? s->start_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val(arr_size - 1) : mk_val(0)); @@ -850,7 +863,7 @@ value member_expression::execute_impl(context & ctx) const { } } else if (is_stmt(this->property)) { // syntax: obj.index - property = mk_val(cast_stmt(this->property)->val); + property = this->property->execute(ctx); if (property->as_int() < 0) { throw std::runtime_error("Static member property cannot be negative"); } @@ -859,8 +872,8 @@ value member_expression::execute_impl(context & ctx) const { if (!is_stmt(this->property)) { throw std::runtime_error("Static member property must be an identifier"); } - property = mk_val(cast_stmt(this->property)->val); - std::string prop = property->as_string().str(); + property = cached_val(ctx, static_property); + const auto & prop = cast_stmt(this->property)->val; JJ_DEBUG("Member expression, object type %s, static property '%s'", object->type().c_str(), prop.c_str()); // behavior of jinja2: obj having prop as a built-in function AND 'prop', as an object key, @@ -868,51 +881,56 @@ value member_expression::execute_impl(context & ctx) const { // while obj['prop'] returns the property value. // example: {"obj": {"items": 123}} -> obj.items is the built-in function, obj['items'] is 123 - value val = try_builtin_func(ctx, prop, object, true); - if (!is_val(val)) { + value val = find_builtin_func(ctx, prop, object); + if (val) { return val; } // else, fallthrough to normal property access below } JJ_DEBUG("Member expression on object type %s, property type %s", object->type().c_str(), property->type().c_str()); - value val = mk_val("object_property"); + value val; if (property->is_undefined()) { JJ_DEBUG("%s", "Member expression property is undefined, returning undefined"); - return val; + return mk_val("object_property"); } ensure_key_type_allowed(property); if (is_val(object)) { JJ_DEBUG("%s", "Accessing property on undefined object, returning undefined"); - return val; + return mk_val("object_property"); } else if (is_val(object)) { - auto key = property->as_string().str(); - val = object->at(property, val); - if (is_val(val)) { - val = try_builtin_func(ctx, key, object, true); + val = object->find(property); + if (!val || is_val(val)) { + val = try_builtin_func(ctx, property->as_string().str(), object, true); } - JJ_DEBUG("Accessed property '%s' value, got type: %s", key.c_str(), val->type().c_str()); + JJ_DEBUG("Accessed property '%s' value, got type: %s", property->as_string().str().c_str(), val->type().c_str()); } else if (is_val(object) || is_val(object)) { if (is_val(property) || is_val(property)) { int64_t index = property->as_int(); JJ_DEBUG("Accessing %s index %d", object->type().c_str(), (int)index); if (is_val(object)) { - auto & arr = object->as_array(); + const auto * arr = cast_val(object); + const auto count = arr->as_array().size(); if (index < 0) { - index += static_cast(arr.size()); - } - if (index >= 0 && index < static_cast(arr.size())) { - val = arr[index]; + // Avoid negating INT64_MIN. + const uint64_t distance = static_cast(-(index + 1)) + 1; + if (distance <= count) { + val = arr->as_array().at(static_cast(count - distance)); + } + } else if (static_cast(index) < count) { + val = arr->as_array().at(static_cast(index)); } } else { // value_string - auto str = object->as_string().str(); - if (index >= 0 && index < static_cast(str.size())) { - val = mk_val(std::string(1, str[index])); + if (index >= 0) { + const auto byte = cast_val(object)->val_str.byte_at(static_cast(index)); + if (byte) { + val = mk_val(std::string(1, *byte)); + } } } @@ -930,7 +948,10 @@ value member_expression::execute_impl(context & ctx) const { val = try_builtin_func(ctx, key, object, true); } - if (ctx.is_get_stats && val && object && property) { + if (!val) { + val = mk_val("object_property"); + } + if (ctx.is_get_stats && object && property) { value_t::stats_t::mark_used(val); value_t::stats_t::mark_used(object); value_t::stats_t::mark_used(property); diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 43e108551..97db5de33 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -84,9 +84,14 @@ struct context { // inherit variables (for example, when entering a new scope) env(mk_val(*parent.env)) {} + // returns nullptr if not found + value find_val(const value & key) { + return env->find(key); + } + value get_val(const std::string & name) { - value default_val = mk_val(name); - return env->at(name, default_val); + auto result = find_val(mk_val(name)); + return result ? result : mk_val(name); } void set_val(const std::string & name, const value & val) { @@ -105,6 +110,12 @@ private: value_object env; }; +// values cached in the AST are shared across renders, but stats mutate values, so stats mode gets a fresh copy +template +value cached_val(const context & ctx, const value & cached) { + return ctx.is_get_stats ? mk_val(*cast_val(cached)) : cached; +} + // utils for visiting AST nodes static std::vector stmts_to_ptr(const statements & stmts) { std::vector children; @@ -328,11 +339,8 @@ struct member_expression : public expression { statement_ptr property; bool computed; // true if obj[expr] and false if obj.prop - member_expression(statement_ptr && object, statement_ptr && property, bool computed) - : object(std::move(object)), property(std::move(property)), computed(computed) { - chk_type(this->object); - chk_type(this->property); - } + value static_property; // set if property is an identifier, see cached_val() + member_expression(statement_ptr && object, statement_ptr && property, bool computed); std::string type() const override { return "MemberExpression"; } value execute_impl(context & ctx) const override; void visit(context & ctx) const override { @@ -367,7 +375,8 @@ struct call_expression : public expression { */ struct identifier : public expression { std::string val; - explicit identifier(const std::string & val) : val(val) {} + const value key; // lookup key, safe to share across renders as lookups do not touch stats + explicit identifier(const std::string & val) : val(val), key(mk_val(val)) {} std::string type() const override { return "Identifier"; } value execute_impl(context & ctx) const override; }; @@ -376,28 +385,31 @@ struct identifier : public expression { struct integer_literal : public expression { int64_t val; - explicit integer_literal(int64_t val) : val(val) {} + const value constant; // see cached_val() + explicit integer_literal(int64_t val) : val(val), constant(mk_val(val)) {} std::string type() const override { return "IntegerLiteral"; } - value execute_impl(context &) const override { - return mk_val(val); + value execute_impl(context & ctx) const override { + return cached_val(ctx, constant); } }; struct float_literal : public expression { double val; - explicit float_literal(double val) : val(val) {} + const value constant; // see cached_val() + explicit float_literal(double val) : val(val), constant(mk_val(val)) {} std::string type() const override { return "FloatLiteral"; } - value execute_impl(context &) const override { - return mk_val(val); + value execute_impl(context & ctx) const override { + return cached_val(ctx, constant); } }; struct string_literal : public expression { std::string val; - explicit string_literal(const std::string & val) : val(val) {} + const value constant; // see cached_val() + explicit string_literal(const std::string & val) : val(val), constant(mk_val(val)) {} std::string type() const override { return "StringLiteral"; } - value execute_impl(context &) const override { - return mk_val(val); + value execute_impl(context & ctx) const override { + return cached_val(ctx, constant); } }; @@ -748,7 +760,7 @@ static void gather_string_parts_recursive(const value & val, value_string & part } else if (is_val(val) && depth >= JINJA_RECURSIVE_LIMIT) { throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep"); } else if (is_val(val)) { - auto items = cast_val(val)->as_array(); + const auto & items = cast_val(val)->as_array(); for (const auto & item : items) { gather_string_parts_recursive(item, parts, depth + 1); } diff --git a/common/jinja/string.cpp b/common/jinja/string.cpp index bde679e4e..22189fa6f 100644 --- a/common/jinja/string.cpp +++ b/common/jinja/string.cpp @@ -43,14 +43,59 @@ void string::mark_input() { } std::string string::str() const { - if (parts.size() == 1) { - return parts[0].val; - } - std::ostringstream oss; + std::string result; + result.reserve(length()); for (const auto & part : parts) { - oss << part.val; + result += part.val; } - return oss.str(); + return result; +} + +bool string::empty() const { + for (const auto & part : parts) { + if (!part.val.empty()) { + return false; + } + } + return true; +} + +std::optional string::byte_at(size_t index) const { + for (const auto & part : parts) { + if (index < part.val.size()) { + return part.val[index]; + } + index -= part.val.size(); + } + return {}; +} + +bool string::equals(const string & other) const { + size_t part = 0, offset = 0; + for (const auto & left : parts) { + std::string_view text(left.val); + while (!text.empty()) { + while (part < other.parts.size() && offset == other.parts[part].val.size()) { + ++part; + offset = 0; + } + if (part == other.parts.size()) { + return false; + } + const auto & right = other.parts[part].val; + const auto count = std::min(text.size(), right.size() - offset); + if (text.substr(0, count) != std::string_view(right).substr(offset, count)) { + return false; + } + text.remove_prefix(count); + offset += count; + } + } + while (part < other.parts.size() && offset == other.parts[part].val.size()) { + ++part; + offset = 0; + } + return part == other.parts.size(); } size_t string::length() const { diff --git a/common/jinja/string.h b/common/jinja/string.h index 669afb8f1..119ad11c2 100644 --- a/common/jinja/string.h +++ b/common/jinja/string.h @@ -39,6 +39,9 @@ struct string { std::string str() const; size_t length() const; + bool empty() const; + bool equals(const string & other) const; + std::optional byte_at(size_t index) const; void hash_update(hasher & hash) const noexcept; bool all_parts_are_input() const; bool is_uppercase() const; diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index 333241d7a..561a4b86a 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -276,7 +276,7 @@ static value tojson(const func_args & args) { if (args.ctx.is_get_stats) { // mark as used (recursively) for stats auto val_input = args.get_pos(0); - value_t::stats_t::mark_used(const_cast(val_input), true); + value_t::stats_t::mark_used(val_input, true); } if (is_val(val_indent)) { indent = static_cast(val_indent->as_int()); @@ -507,7 +507,12 @@ const func_builtins & global_builtins() { }}, {"test_is_divisibleby", [](const func_args & args) -> value { args.ensure_vals(); - bool res = args.get_pos(0)->val_int % args.get_pos(1)->val_int == 0; + const int64_t a = args.get_pos(0)->as_int(); + const int64_t b = args.get_pos(1)->as_int(); + if (b == 0) { + throw std::runtime_error("divisibleby: division by zero"); + } + bool res = b == -1 || a % b == 0; // avoid INT64_MIN % -1 overflow return mk_val(res); }}, {"test_is_string", test_type_fn}, @@ -519,11 +524,11 @@ const func_builtins & global_builtins() { {"test_is_mapping", test_type_fn}, {"test_is_lower", [](const func_args & args) -> value { args.ensure_vals(); - return mk_val(args.get_pos(0)->val_str.is_lowercase()); + return mk_val(cast_val(args.get_pos(0))->val_str.is_lowercase()); }}, {"test_is_upper", [](const func_args & args) -> value { args.ensure_vals(); - return mk_val(args.get_pos(0)->val_str.is_uppercase()); + return mk_val(cast_val(args.get_pos(0))->val_str.is_uppercase()); }}, {"test_is_none", test_type_fn}, {"test_is_defined", [](const func_args & args) -> value { @@ -571,7 +576,7 @@ const func_builtins & global_builtins() { {"test_is_test", [](const func_args & args) -> value { args.ensure_vals(); auto & builtins = global_builtins(); - std::string test_name = args.get_pos(0)->val_str.str(); + std::string test_name = cast_val(args.get_pos(0))->val_str.str(); auto it = builtins.find("test_is_" + test_name); bool res = it != builtins.end(); return mk_val(res); @@ -613,18 +618,26 @@ const func_builtins & global_builtins() { return builtins; } -value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) { +value find_builtin_func(context & ctx, const std::string & name, value & input) { JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str()); if (ctx.is_get_stats) { value_t::stats_t::mark_used(input); input->stats.ops.insert(name); } - auto builtins = input->get_builtins(); + const auto & builtins = input->get_builtins(); auto it = builtins.find(name); if (it != builtins.end()) { JJ_DEBUG("Binding built-in '%s'", name.c_str()); return mk_val(name, it->second, input); } + return nullptr; +} + +value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) { + value func = find_builtin_func(ctx, name, input); + if (func) { + return func; + } if (undef_on_missing) { return mk_val(name); } @@ -831,7 +844,7 @@ const func_builtins & value_string_t::get_builtins() const { --maxsplit; } auto res = mk_val(str); - res->val_str.mark_input_based_on(args.get_pos(0)->val_str); + res->val_str.mark_input_based_on(cast_val(args.get_pos(0))->val_str); result->push_back(std::move(res)); return result; }}, @@ -858,7 +871,7 @@ const func_builtins & value_string_t::get_builtins() const { --maxsplit; } auto res = mk_val(str); - res->val_str.mark_input_based_on(args.get_pos(0)->val_str); + res->val_str.mark_input_based_on(cast_val(args.get_pos(0))->val_str); result->push_back(std::move(res)); result->reverse(); return result; @@ -891,7 +904,7 @@ const func_builtins & value_string_t::get_builtins() const { } } auto res = mk_val(str); - res->val_str.mark_input_based_on(args.get_pos(0)->val_str); + res->val_str.mark_input_based_on(cast_val(args.get_pos(0))->val_str); return res; }}, {"format", [](const func_args & args) -> value { @@ -1191,7 +1204,7 @@ const func_builtins & value_array_t::get_builtins() const { if (args.ctx.is_get_stats) { // mark as used (recursively) for stats auto val_input = args.get_pos(0); - value_t::stats_t::mark_used(const_cast(val_input), true); + value_t::stats_t::mark_used(val_input, true); } return mk_val(args.get_pos(0)->as_string()); }}, @@ -1439,7 +1452,7 @@ const func_builtins & value_object_t::get_builtins() const { if (args.ctx.is_get_stats) { // mark as used (recursively) for stats auto val_input = args.get_pos(0); - value_t::stats_t::mark_used(const_cast(val_input), true); + value_t::stats_t::mark_used(val_input, true); } return mk_val(args.get_pos(0)->as_string()); }}, @@ -1833,18 +1846,18 @@ std::string value_to_string_repr(const value & val) { } // stats utility -void value_t::stats_t::mark_used(value & val, bool deep, int depth) { +void value_t::stats_t::mark_used(const value & val, bool deep, int depth) { val->stats.used = true; if (deep && (is_val(val) || is_val(val)) && depth >= JINJA_RECURSIVE_LIMIT) { throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep"); } if (deep) { if (is_val(val)) { - for (auto & item : val->val_arr) { + for (auto & item : val->as_array()) { mark_used(item, deep, depth + 1); } } else if (is_val(val)) { - for (auto & pair : val->val_obj) { + for (auto & pair : val->as_ordered_object()) { mark_used(pair.first, deep, depth + 1); mark_used(pair.second, deep, depth + 1); } diff --git a/common/jinja/value.h b/common/jinja/value.h index 8fdff884e..bdc35ed39 100644 --- a/common/jinja/value.h +++ b/common/jinja/value.h @@ -21,6 +21,20 @@ namespace jinja { struct value_t; using value = std::shared_ptr; +enum class value_kind { + INTEGER, + REAL, + STRING, + BOOLEAN, + ARRAY, + TUPLE, + OBJECT, + NONE, + UNDEFINED, + FUNCTION, + KEYWORD +}; +bool has_value_kind(const value_t * val, value_kind kind); // Helper to check the type of a value template @@ -34,12 +48,12 @@ struct extract_pointee> { template bool is_val(const value & ptr) { using PointeeType = typename extract_pointee::type; - return dynamic_cast(ptr.get()) != nullptr; + return has_value_kind(ptr.get(), PointeeType::kind_id); } template bool is_val(const value_t * ptr) { using PointeeType = typename extract_pointee::type; - return dynamic_cast(ptr) != nullptr; + return has_value_kind(ptr, PointeeType::kind_id); } template std::shared_ptr::type> mk_val(Args&&... args) { @@ -49,12 +63,12 @@ std::shared_ptr::type> mk_val(Args&&... args) { template const typename extract_pointee::type * cast_val(const value & ptr) { using PointeeType = typename extract_pointee::type; - return dynamic_cast(ptr.get()); + return is_val(ptr) ? static_cast(ptr.get()) : nullptr; } template typename extract_pointee::type * cast_val(value & ptr) { using PointeeType = typename extract_pointee::type; - return dynamic_cast(ptr.get()); + return is_val(ptr) ? static_cast(ptr.get()) : nullptr; } // End Helper @@ -105,14 +119,7 @@ enum value_compare_op { eq, ge, gt, lt, ne }; bool value_compare(const value & a, const value & b, value_compare_op op); struct value_t { - int64_t val_int; - double val_flt; - string val_str; - - std::vector val_arr; - std::vector> val_obj; - - func_handler val_func; + const value_kind kind; // only used if ctx.is_get_stats = true struct stats_t { @@ -120,10 +127,10 @@ struct value_t { // ops can be builtin calls or operators: "array_access", "object_access" std::set ops; // utility to recursively mark value and its children as used - static void mark_used(value & val, bool deep = false, int depth = 0); + static void mark_used(const value & val, bool deep = false, int depth = 0); } stats; - value_t() = default; + explicit value_t(value_kind kind) : kind(kind) {} value_t(const value_t &) = default; virtual ~value_t() = default; @@ -141,6 +148,10 @@ struct value_t { virtual bool is_undefined() const { return false; } virtual const func_builtins & get_builtins() const { throw_type_error("has no builtins"); } + virtual value find(const value & key) { + value missing; + return at(key, missing); + } virtual bool has_key(const value &) { throw_type_error("is not an object value"); } virtual void insert(const value & /* key */, const value & /* val */) { throw_type_error("is not an object value"); } virtual value & at(const value & /* key */, value & /* default_val */) { throw_type_error("is not an object value"); } @@ -172,11 +183,19 @@ protected: virtual bool nonequal(const value_t & other) const { return !equivalent(other); } }; +inline bool has_value_kind(const value_t * val, value_kind kind) { + return val && (val->kind == kind || + (kind == value_kind::ARRAY && (val->kind == value_kind::TUPLE))); +} + // // utils // const func_builtins & global_builtins(); +// returns nullptr if input has no built-in named `name` +value find_builtin_func(context & ctx, const std::string & name, value & input); +// returns value_undefined(name) if not found and undef_on_missing is set, throws otherwise value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false); value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args); @@ -215,7 +234,10 @@ struct value_equality { // struct value_int_t : public value_t { - value_int_t(int64_t v) { + static constexpr auto kind_id = value_kind::INTEGER; + int64_t val_int; + double val_flt; + value_int_t(int64_t v) : value_t(kind_id) { val_int = v; val_flt = static_cast(v); if (static_cast(val_flt) != v) { @@ -239,21 +261,22 @@ struct value_int_t : public value_t { } protected: virtual bool equivalent(const value_t & other) const override { - return other.is_numeric() && val_int == other.val_int && val_flt == other.val_flt; + return other.is_numeric() && val_int == other.as_int() && val_flt == other.as_float(); } virtual bool nonequal(const value_t & other) const override { - return !(typeid(*this) == typeid(other) && val_int == other.val_int); + return !(kind == other.kind && val_int == other.as_int()); } }; using value_int = std::shared_ptr; struct value_float_t : public value_t { - value val; - value_float_t(double v) { + static constexpr auto kind_id = value_kind::REAL; + int64_t val_int; + double val_flt; + value_float_t(double v) : value_t(kind_id) { val_flt = v; val_int = std::isfinite(v) ? static_cast(v) : 0; - val = mk_val(val_int); } virtual std::string type() const override { return "Float"; } virtual double as_float() const override { return val_flt; } @@ -272,7 +295,7 @@ struct value_float_t : public value_t { virtual bool is_hashable() const override { return true; } virtual hasher unique_hash() const noexcept override { if (static_cast(val_int) == val_flt) { - return val->unique_hash(); + return value_int_t(val_int).unique_hash(); } else { return hasher(typeid(*this)) .update(&val_int, sizeof(val_int)) @@ -281,19 +304,22 @@ struct value_float_t : public value_t { } protected: virtual bool equivalent(const value_t & other) const override { - return other.is_numeric() && val_int == other.val_int && val_flt == other.val_flt; + return other.is_numeric() && val_int == other.as_int() && val_flt == other.as_float(); } virtual bool nonequal(const value_t & other) const override { - return !(typeid(*this) == typeid(other) && val_flt == other.val_flt); + return !(kind == other.kind && val_flt == other.as_float()); } }; using value_float = std::shared_ptr; struct value_string_t : public value_t { - value_string_t() { val_str = string(); } - value_string_t(const std::string & v) { val_str = string(v); } - value_string_t(const string & v) { val_str = v; } + static constexpr auto kind_id = value_kind::STRING; + string val_str; + value_string_t() : value_t(kind_id) {} + value_string_t(const std::string & v) : value_t(kind_id), val_str(v) {} + value_string_t(const string & v) : value_t(kind_id), val_str(v) {} + value_string_t(string && v) : value_t(kind_id), val_str(std::move(v)) {} virtual std::string type() const override { return "String"; } virtual string as_string() const override { return val_str; } virtual std::string as_repr() const override { @@ -304,7 +330,7 @@ struct value_string_t : public value_t { return ss.str(); } virtual bool as_bool() const override { - return val_str.length() > 0; + return !val_str.empty(); } virtual const func_builtins & get_builtins() const override; virtual bool is_hashable() const override { return true; } @@ -320,51 +346,50 @@ struct value_string_t : public value_t { } protected: virtual bool equivalent(const value_t & other) const override { - return typeid(*this) == typeid(other) && val_str.str() == other.val_str.str(); + return kind == other.kind && val_str.equals(static_cast(other).val_str); } }; using value_string = std::shared_ptr; struct value_bool_t : public value_t { - value val; - value_bool_t(bool v) { + static constexpr auto kind_id = value_kind::BOOLEAN; + int64_t val_int; + double val_flt; + value_bool_t(bool v) : value_t(kind_id) { val_int = static_cast(v); val_flt = static_cast(v); - val = mk_val(val_int); } virtual std::string type() const override { return "Boolean"; } virtual int64_t as_int() const override { return val_int; } + virtual double as_float() const override { return val_flt; } virtual bool as_bool() const override { return val_int; } virtual string as_string() const override { return std::string(val_int ? "True" : "False"); } virtual const func_builtins & get_builtins() const override; virtual bool is_numeric() const override { return true; } virtual bool is_hashable() const override { return true; } virtual hasher unique_hash() const noexcept override { - return val->unique_hash(); + return value_int_t(val_int).unique_hash(); } protected: virtual bool equivalent(const value_t & other) const override { - return other.is_numeric() && val_int == other.val_int && val_flt == other.val_flt; + return other.is_numeric() && val_int == other.as_int() && val_flt == other.as_float(); } virtual bool nonequal(const value_t & other) const override { - return !(typeid(*this) == typeid(other) && val_int == other.val_int); + return !(kind == other.kind && val_int == other.as_int()); } }; using value_bool = std::shared_ptr; struct value_array_t : public value_t { - value_array_t() = default; - value_array_t(value & v) { - val_arr = v->val_arr; - } - value_array_t(std::vector && arr) { - val_arr = arr; - } - value_array_t(const std::vector & arr) { - val_arr = arr; - } + static constexpr auto kind_id = value_kind::ARRAY; + std::vector val_arr; + + value_array_t() : value_t(kind_id) {} + value_array_t(value & v) : value_t(kind_id), val_arr(v->as_array()) {} + value_array_t(std::vector && arr) : value_t(kind_id), val_arr(std::move(arr)) {} + value_array_t(const std::vector & arr) : value_t(kind_id), val_arr(arr) {} void reverse() { if (is_immutable()) { throw std::runtime_error("Attempting to modify immutable type"); @@ -462,24 +487,27 @@ struct value_array_t : public value_t { return hash; } protected: + explicit value_array_t(value_kind kind) : value_t(kind) {} + virtual bool equivalent(const value_t & other) const override { - return typeid(*this) == typeid(other) && is_hashable() && other.is_hashable() && std::equal(val_arr.begin(), val_arr.end(), other.val_arr.begin(), other.val_arr.end(), value_equivalence()); + return kind == other.kind && is_hashable() && other.is_hashable() && std::equal(val_arr.begin(), val_arr.end(), other.as_array().begin(), other.as_array().end(), value_equivalence()); } }; using value_array = std::shared_ptr; struct value_tuple_t : public value_array_t { - value_tuple_t(value & v) { - val_arr = v->val_arr; + static constexpr auto kind_id = value_kind::TUPLE; + value_tuple_t(value & v) : value_array_t(kind_id) { + val_arr = v->as_array(); } - value_tuple_t(std::vector && arr) { + value_tuple_t(std::vector && arr) : value_array_t(kind_id) { + val_arr = std::move(arr); + } + value_tuple_t(const std::vector & arr) : value_array_t(kind_id) { val_arr = arr; } - value_tuple_t(const std::vector & arr) { - val_arr = arr; - } - value_tuple_t(const std::pair & pair) { + value_tuple_t(const std::pair & pair) : value_array_t(kind_id) { val_arr.push_back(pair.first); val_arr.push_back(pair.second); } @@ -490,21 +518,23 @@ using value_tuple = std::shared_ptr; struct value_object_t : public value_t { - std::unordered_map unordered; + static constexpr auto kind_id = value_kind::OBJECT; + std::vector> val_obj; + std::unordered_map key_index; // key -> index in val_obj bool has_builtins = true; // context and loop objects do not have builtins - value_object_t() = default; - value_object_t(value & v) { - val_obj = v->val_obj; - for (const auto & pair : val_obj) { - unordered[pair.first] = pair.second; + value_object_t() : value_t(kind_id) {} + value_object_t(value & v) : value_t(kind_id) { + val_obj = v->as_ordered_object(); + for (size_t i = 0; i < val_obj.size(); i++) { + key_index[val_obj[i].first] = i; } } - value_object_t(const std::map & obj) { + value_object_t(const std::map & obj) : value_t(kind_id) { for (const auto & pair : obj) { insert(pair.first, pair.second); } } - value_object_t(const std::vector> & obj) { + value_object_t(const std::vector> & obj) : value_t(kind_id) { for (const auto & pair : obj) { insert(pair.first, pair.second); } @@ -527,45 +557,40 @@ struct value_object_t : public value_t { return ss.str(); } virtual bool as_bool() const override { - return !unordered.empty(); + return !val_obj.empty(); } virtual bool has_key(const value & key) override { if (!key->is_immutable() || !key->is_hashable()) { throw std::runtime_error("Object key of unhashable type: " + key->type()); } - return unordered.find(key) != unordered.end(); + return key_index.find(key) != key_index.end(); } virtual void insert(const value & key, const value & val) override { - bool replaced = false; if (is_immutable()) { throw std::runtime_error("Attempting to modify immutable type"); } - if (has_key(key)) { - // if key exists, replace value in ordered list instead of appending - for (auto & pair : val_obj) { - if (*(pair.first) == *key) { - pair.second = val; - replaced = true; - break; - } - } + if (!key->is_immutable() || !key->is_hashable()) { + throw std::runtime_error("Object key of unhashable type: " + key->type()); } - unordered[key] = val; - if (!replaced) { + const auto entry = key_index.try_emplace(key, val_obj.size()); + if (entry.second) { val_obj.push_back({key, val}); + } else { + val_obj[entry.first->second].second = val; } } virtual value & at(const value & key, value & default_val) override { - if (!has_key(key)) { - return default_val; + if (!key->is_immutable() || !key->is_hashable()) { + throw std::runtime_error("Object key of unhashable type: " + key->type()); } - return unordered.at(key); + const auto found = key_index.find(key); + return found == key_index.end() ? default_val : val_obj[found->second].second; } virtual value & at(const value & key) override { if (!has_key(key)) { throw std::runtime_error("Key '" + key->as_string().str() + "' not found in value of type " + type()); } - return unordered.at(key); + return val_obj[key_index.at(key)].second; } virtual value & at(const std::string & key, value & default_val) override { value key_val = mk_val(key); @@ -599,7 +624,7 @@ struct value_object_t : public value_t { } protected: virtual bool equivalent(const value_t & other) const override { - return typeid(*this) == typeid(other) && is_hashable() && other.is_hashable() && std::equal(val_obj.begin(), val_obj.end(), other.val_obj.begin(), other.val_obj.end(), value_equivalence()); + return kind == other.kind && is_hashable() && other.is_hashable() && std::equal(val_obj.begin(), val_obj.end(), other.as_ordered_object().begin(), other.as_ordered_object().end(), value_equivalence()); } }; using value_object = std::shared_ptr; @@ -609,6 +634,8 @@ using value_object = std::shared_ptr; // struct value_none_t : public value_t { + static constexpr auto kind_id = value_kind::NONE; + value_none_t() : value_t(kind_id) {} virtual std::string type() const override { return "None"; } virtual bool is_none() const override { return true; } virtual bool as_bool() const override { return false; } @@ -621,14 +648,15 @@ struct value_none_t : public value_t { } protected: virtual bool equivalent(const value_t & other) const override { - return typeid(*this) == typeid(other); + return kind == other.kind; } }; using value_none = std::shared_ptr; struct value_undefined_t : public value_t { + static constexpr auto kind_id = value_kind::UNDEFINED; std::string hint; // for debugging, to indicate where undefined came from - value_undefined_t(const std::string & h = "") : hint(h) {} + value_undefined_t(const std::string & h = "") : value_t(kind_id), hint(h) {} virtual std::string type() const override { return hint.empty() ? "Undefined" : "Undefined (hint: '" + hint + "')"; } virtual bool is_undefined() const override { return true; } virtual bool as_bool() const override { return false; } @@ -705,12 +733,14 @@ private: }; struct value_func_t : public value_t { + static constexpr auto kind_id = value_kind::FUNCTION; + func_handler val_func; std::string name; value arg0; // bound "this" argument, if any - value_func_t(const std::string & name, const func_handler & func) : name(name) { + value_func_t(const std::string & name, const func_handler & func) : value_t(kind_id), name(name) { val_func = func; } - value_func_t(const std::string & name, const func_handler & func, const value & arg_this) : name(name), arg0(arg_this) { + value_func_t(const std::string & name, const func_handler & func, const value & arg_this) : value_t(kind_id), name(name), arg0(arg_this) { val_func = func; } virtual value invoke(const func_args & args) const override { @@ -735,18 +765,22 @@ protected: // Note: this is unused for now, we don't support function as object keys // compare function pointers // (val_func == other.val_func does not work as std::function::operator== is only used for nullptr check) + if (kind != other.kind) { + return false; + } const auto target_this = this->val_func.target(); - const auto target_other = other.val_func.target(); - return typeid(*this) == typeid(other) && target_this == target_other; + const auto target_other = static_cast(other).val_func.target(); + return target_this == target_other; } }; using value_func = std::shared_ptr; // special value for kwarg struct value_kwarg_t : public value_t { + static constexpr auto kind_id = value_kind::KEYWORD; std::string key; value val; - value_kwarg_t(const std::string & k, const value & v) : key(k), val(v) {} + value_kwarg_t(const std::string & k, const value & v) : value_t(kind_id), key(k), val(v) {} virtual std::string type() const override { return "KwArg"; } virtual std::string as_repr() const override { return type(); } virtual bool is_hashable() const override { return true; } @@ -760,7 +794,7 @@ struct value_kwarg_t : public value_t { protected: virtual bool equivalent(const value_t & other) const override { const value_kwarg_t & other_val = static_cast(other); - return typeid(*this) == typeid(other) && key == other_val.key && val == other_val.val; + return kind == other.kind && key == other_val.key && val == other_val.val; } }; using value_kwarg = std::shared_ptr; diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index c79b0d24b..d52f1af9c 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "chat-tts-norm.h" #include "json.h" @@ -42,6 +43,8 @@ static void test_string_parts(testing & t); static void test_fuzzing(testing & t); static void test_tts_norm_patterns(testing & t); +static void test_shared_render(testing & t); + static bool g_python_mode = false; int main(int argc, char *argv[]) { @@ -79,6 +82,7 @@ int main(int argc, char *argv[]) { t.test("object methods", test_object_methods); t.test("tts norm patterns", test_tts_norm_patterns); if (!g_python_mode) { + t.test("shared render", test_shared_render); t.test("hasher", test_hasher); t.test("stats", test_stats); t.test("caps", test_caps); @@ -89,6 +93,45 @@ int main(int argc, char *argv[]) { return t.summary(); } +static void test_shared_render(testing & t) { + t.test("shared AST with stats and input marking", [](testing & t) { + const std::string source = "{% macro emit(x) %}{{ x }}{% endmacro %}{% for i in range(2) %}{{ emit(prefix) }}{{ i }}{% endfor %}{% filter upper %}{{ body }}{% endfilter %}|{{ 'literal' is string }}|{{ obj.value }}"; + const auto program = jinja::parse_from_tokens(jinja::lexer().tokenize(source)); + std::vector> workers; + for (int worker = 0; worker < 8; ++worker) { + workers.push_back(std::async(std::launch::async, [&, worker] { + const auto prefix = "p" + std::to_string(worker); + for (int i = 0; i < 25; ++i) { + jinja::context ctx(source); + ctx.is_get_stats = i % 2 != 0; + const json input = { + { "prefix", prefix }, + { "body", "hi" }, + { "obj", { { "value", "V" } } } + }; + jinja::global_from_json(ctx, input, true); + const auto result = jinja::runtime::gather_string_parts(jinja::runtime(ctx).execute(program)); + if (result->as_string().str() != prefix + "0" + prefix + "1HI|True|V") { + return false; + } + if (ctx.get_val("obj")->at("value")->stats.used != ctx.is_get_stats) { + return false; + } + if (result->val_str.parts.empty() || !result->val_str.parts.front().is_input || + !result->val_str.parts.back().is_input) { + return false; + } + } + return true; + })); + } + for (auto & worker : workers) { + t.assert_true("request-local state", worker.get()); + } + }); + +} + static void test_whitespace_control(testing & t) { test_template(t, "trim_blocks removes newline after tag", "{% if true %}\n" @@ -502,6 +545,13 @@ static void test_expressions(testing & t) { "c" ); + test_template(t, "array index boundaries", + "{% for items in arrays %}{% for i in indices %}{{ items[i] | default('_') }}|{% endfor %};{% endfor %}", + {{"arrays", json::array({json::array({"a", "b", "c"}), json::array()})}, + {"indices", json::array({INT64_MIN, -4, -3, -1, 0, 2, 3, INT64_MAX})}}, + "_|_|a|c|a|c|_|_|;_|_|_|_|_|_|_|_|;" + ); + test_template(t, "array bool access", "{{ items[true] }}", {{"items", json::array({"a", "b", "c"})}}, @@ -575,6 +625,14 @@ static void test_expressions(testing & t) { "('c', 'b', 'a')" ); + test_template(t, "multipart string comparison and access", + "{% set left = a + 'bc' %}{% set right = 'ab' + c %}{{ left == right }}|" + "{{ left[0] }}{{ left[1] }}{{ left[2] }}|{{ right[1:3] }}|" + "{{ left[3] is undefined }}|{{ empty[0] is undefined }}", + {{"a", "a"}, {"c", "c"}, {"empty", ""}}, + "True|abc|bc|True|True" + ); + test_template(t, "string slice negative step", "{{ 'abcdef'[::-2] }}", json::object(), @@ -633,6 +691,31 @@ static void test_expressions(testing & t) { "{{ " + std::string(1000, '(') + "1" + std::string(1000, ')') + " }}", "template is nested more than 128 levels deep" ); + + test_template_error(t, "undefined hint: missing object key (attr)", + "{% set m = {'a': 1} %}{{ m.missing_key() }}", + "Callee is not a function: got Undefined (hint: 'missing_key')" + ); + + test_template_error(t, "undefined hint: missing object key (subscript)", + "{% set m = {'a': 1} %}{{ m['missing_key']() }}", + "Callee is not a function: got Undefined (hint: 'missing_key')" + ); + + test_template_error(t, "undefined hint: unknown array attribute", + "{% set a = [1] %}{{ a.missing_key() }}", + "Callee is not a function: got Undefined (hint: 'missing_key')" + ); + + test_template_error(t, "undefined hint: unknown string attribute", + "{% set s = 'x' %}{{ s.missing_key() }}", + "Callee is not a function: got Undefined (hint: 'missing_key')" + ); + + test_template_error(t, "undefined hint: unknown int attribute", + "{% set n = 1 %}{{ n.missing_key() }}", + "Callee is not a function: got Undefined (hint: 'missing_key')" + ); } static void test_set_statement(testing & t) { @@ -1440,6 +1523,17 @@ static void test_tests(testing & t) { "yes" ); + test_template(t, "is divisibleby -1 with INT64_MIN", + "{{ x is divisibleby(-1) }}", + {{"x", INT64_MIN}}, + "True" + ); + + test_template_error(t, "is divisibleby zero", + "{{ 10 is divisibleby(0) }}", + "divisibleby: division by zero" + ); + test_template(t, "is eq", "{{ 'yes' if 3 is eq(3) }}", json::object(), @@ -2730,6 +2824,27 @@ static void test_string_parts(testing & t) { return runtime.gather_string_parts(runtime.execute(ast))->as_string(); }; + t.test("comparison across part boundaries", [](testing & t) { + jinja::string left, right; + left.parts = {{true, "a"}, {false, "bc"}}; + right.parts = {{false, "ab"}, {true, "c"}}; + t.assert_true("equal text with different partitions", left.equals(right) && right.equals(left)); + + right.parts = {{false, ""}, {false, "ab"}, {true, ""}, {true, "c"}, {false, ""}}; + t.assert_true("empty parts do not change equality", left.equals(right) && right.equals(left)); + t.assert_true("different bytes are unequal", !left.equals(jinja::string("abd"))); + t.assert_true("prefixes are unequal", !left.equals(jinja::string("ab")) && !jinja::string("ab").equals(left)); + + jinja::string empty; + empty.parts = {{true, ""}, {false, ""}}; + t.assert_true("empty partitions equal an empty string", empty.equals(jinja::string()) && jinja::string().equals(empty)); + + auto object = jinja::mk_val(); + object->insert(jinja::mk_val(left), jinja::mk_val(7)); + const auto found = object->find(jinja::mk_val(right)); + t.assert_true("object keys ignore partition boundaries", found && found->as_int() == 7); + }); + t.test("merge joins only the neighbours with the same type", [](testing & t) { // "AB" comes from the input and merges, "-" comes from the template and must not jinja::string res = render("{{ val.a }}{{ val.b }}-{{ val.c }}",