From b4c0e00a4fcc0287d0bb32fd7cb0d8b286af73ed Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 16 Nov 2026 00:40:44 +0100 Subject: [PATCH] jinja: add 4 missing features (#32352) * jinja: add 4 missing features * apply suggestion Co-authored-by: Aldehir Rojas * fix error msg --------- Co-authored-by: Aldehir Rojas --- common/jinja/lexer.cpp | 35 --------------------------------- common/jinja/lexer.h | 3 +-- common/jinja/parser.cpp | 3 ++- common/jinja/runtime.cpp | 34 +++++++++++++++++++++++++++----- common/jinja/value.cpp | 26 ++++++++++++++++++++++--- tests/test-jinja.cpp | 42 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 46 deletions(-) diff --git a/common/jinja/lexer.cpp b/common/jinja/lexer.cpp index bf2dfacda..800763153 100644 --- a/common/jinja/lexer.cpp +++ b/common/jinja/lexer.cpp @@ -263,41 +263,6 @@ lexer_result lexer::tokenize(const std::string & source) { char ch = src[pos]; - bool is_closing_block = ch == '-' && next_pos_is( {'%', '}'} ); - - // Check for unary operators - if (!is_closing_block && (ch == '-' || ch == '+')) { - start_pos = pos; - token::type last_token_type = tokens.empty() ? token::eof : tokens.back().t; - if (last_token_type == token::text || last_token_type == token::eof) { - throw lexer_exception(std::string("unexpected character: ") + ch, source, pos); - } - switch (last_token_type) { - case token::identifier: - case token::numeric_literal: - case token::string_literal: - case token::close_paren: - case token::close_square_bracket: - // Part of a binary operator - // a - 1, 1 - 1, true - 1, "apple" - 1, (1) - 1, a[1] - 1 - // Continue parsing normally - break; - default: { - // Is part of a unary operator - // (-1), [-1], (1 + -1), not -1, -apple - ++pos; // Consume the operator - - // Check for numbers following the unary operator - std::string num = consume_numeric(); - std::string value = std::string(1, ch) + num; - token::type t = num.empty() ? token::unary_operator : token::numeric_literal; - // JJ_DEBUG("consumed unary operator or numeric literal: '%s'", value.c_str()); - tokens.push_back({t, value, start_pos}); - continue; - } - } - } - // Try to match one of the tokens in the mapping table bool matched = false; for (const auto & [seq, typ] : ordered_mapping_table) { diff --git a/common/jinja/lexer.h b/common/jinja/lexer.h index 3f4ebb62a..3b203c6f1 100644 --- a/common/jinja/lexer.h +++ b/common/jinja/lexer.h @@ -39,7 +39,6 @@ struct token { multiplicative_binary_operator, // * / % power_operator, // ** comparison_binary_operator, // < > <= >= == != - unary_operator, // ! - + comment, // {# ... #} }; type t; @@ -74,7 +73,6 @@ static std::string type_to_string(token::type t) { case token::multiplicative_binary_operator: return "multiplicative_binary_operator"; case token::power_operator: return "power_operator"; case token::comparison_binary_operator: return "comparison_binary_operator"; - case token::unary_operator: return "unary_operator"; case token::comment: return "comment"; default: return "unknown"; } @@ -141,6 +139,7 @@ struct lexer { {"~", token::additive_binary_operator}, {"**", token::power_operator}, {"*", token::multiplicative_binary_operator}, + {"//", token::multiplicative_binary_operator}, {"/", token::multiplicative_binary_operator}, {"%", token::multiplicative_binary_operator}, // Assignment operator diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp index 0841828c4..4807b6995 100644 --- a/common/jinja/parser.cpp +++ b/common/jinja/parser.cpp @@ -482,7 +482,8 @@ private: } statement_ptr parse_unary_expression() { - if (is(token::unary_operator)) { + // a sign where an operand is expected is unary: -x, 1 + -x, x if c else -1 + if (is(token::additive_binary_operator) && peek().value != "~") { depth_guard guard(*this); size_t start_pos = current; auto op = next(); diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index 8bd8ab1c8..aae67cab0 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -196,6 +196,25 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val } else if (op.value == "/") { JJ_DEBUG("Division operation: %f / %f", a, b); return mk_val(a / b); + } else if (op.value == "//") { + // python semantics: round toward -inf + if (is_val(left_val) && is_val(right_val)) { + const int64_t x = left_val->as_int(); + const int64_t y = right_val->as_int(); + if (y == 0) { + throw std::runtime_error("integer division by zero"); + } + if (y == -1) { + // INT64_MIN / -1 is UB, wrap around instead + return mk_val((int64_t) (0 - (uint64_t) x)); + } + int64_t q = x / y; + if (x % y != 0 && (x < 0) != (y < 0)) { + q -= 1; + } + return mk_val(q); + } + return mk_val(std::floor(a / b)); } else if (op.value == "%") { double rem = std::fmod(a, b); JJ_DEBUG("Modulo operation: %f %% %f = %f", a, b, rem); @@ -274,12 +293,17 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val // String membership if (is_val(left_val) && is_val(right_val)) { // case: "a" in "abc" - bool member = test_is_in(); - if (op.value == "in") { - return mk_val(member); - } else if (op.value == "not in") { - return mk_val(!member); + if (op.value == "in" || op.value == "not in") { + bool member = test_is_in(); + return mk_val(op.value == "in" ? member : !member); } + // case: "a" < "b"; UTF-8 byte order is the same as codepoint order, like python + const std::string a = left_val->as_string().str(); + const std::string b = right_val->as_string().str(); + if (op.value == "<") { return mk_val(a < b); } + if (op.value == ">") { return mk_val(a > b); } + if (op.value == "<=") { return mk_val(a <= b); } + if (op.value == ">=") { return mk_val(a >= b); } } // Value key in object diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index 561a4b86a..335461136 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -1101,8 +1101,28 @@ const func_builtins & value_bool_t::get_builtins() const { return builtins; } -[[noreturn]] static value array_sum_not_implemented(const func_args &) { - throw not_implemented_exception("Array sum builtin not implemented"); +static value array_sum(const func_args & args) { + args.ensure_kwargs({"attribute", "start"}); + args.ensure_count(1, 3); + args.ensure_vals(); + value undef = mk_val(); + value attribute = args.get_kwarg_or_pos("attribute", 1); + value start = args.get_kwarg_or_pos("start", 2); + // same as the + operator: compute in double, int result if all operands are int + bool is_float = is_val(start); + double sum = start->is_undefined() ? 0.0 : start->as_float(); + for (const auto & item : args.get_pos(0)->as_array()) { + value v = attribute->is_undefined() ? item : get_attribute(item, attribute, undef); + if (v->is_undefined() && !attribute->is_undefined()) { + throw raised_exception("sum: " + item->type() + " has no attribute '" + attribute->as_string().str() + "'"); + } + if (!is_val(v) && !is_val(v)) { + throw raised_exception("sum: unsupported operand type " + v->type()); + } + is_float = is_float || is_val(v); + sum += v->as_float(); + } + return is_float ? value(mk_val(sum)) : value(mk_val(static_cast(sum))); } const func_builtins & value_array_t::get_builtins() const { @@ -1171,7 +1191,7 @@ const func_builtins & value_array_t::get_builtins() const { {"select", select_or_reject}, {"rejectattr", select_or_reject}, {"reject", select_or_reject}, - {"sum", array_sum_not_implemented}, + {"sum", array_sum}, {"join", [](const func_args & args) -> value { args.ensure_kwargs({"d", "attribute"}); args.ensure_count(1, 3); diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index 3f86d1535..06064c494 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -662,6 +662,42 @@ static void test_expressions(testing & t) { "20" ); + test_template(t, "floor division", + "{{ 7 // 2 }},{{ -7 // 2 }},{{ 7 // -2 }},{{ 7.5 // 2 }},{{ a // b }}", + {{"a", 10}, {"b", 3}}, + "3,-4,-4,3.0,3" + ); + + test_template(t, "string comparison", + "{{ 'b' > 'a' }},{{ 'VBD' > 'VB' }},{{ 'a' <= 'a' }},{{ 'Z' < 'a' }},{{ 'ab' >= 'b' }}", + json::object(), + "True,True,True,True,False" + ); + + test_template(t, "unary minus after keyword", + "{{ 1 if false else -1 }},{{ not -1 }},{{ x and -2 }},{{ -3 if x }}", + {{"x", true}}, + "-1,False,-2,-3" + ); + + test_template(t, "unary minus after elif", + "{% if false %}a{% elif -1 < 0 %}b{% endif %}", + json::object(), + "b" + ); + + test_template(t, "unary and binary sign", + "{{ -x }},{{ - 1 }},{{ --x }},{{ 1 - -x }},{{ 1 -1 }},{{ x -1 }},{{ [x, -1][-1] }},{{ +x }},{{ -x|abs }}", + {{"x", 3}}, + "-3,-1,3,4,0,2,-1,3,3" + ); + + test_template(t, "binary minus after keyword attribute", + "{{ a.else - 1 }}", + {{"a", {{"else", 3}}}}, + "2" + ); + test_template(t, "string concat ~", "{{ 'hello' ~ ' ' ~ 'world' }}", json::object(), @@ -2195,6 +2231,12 @@ static void test_array_methods(testing & t) { "6" ); + test_template(t, "array|sum", + "{{ [1, 2.5, 3]|sum }},{{ [1, 2]|sum(start=10) }},{{ items|sum(attribute='w') }},{{ []|sum }}", + {{"items", json::array({{{"w", 1}}, {{"w", 2}}})}}, + "6.5,13,3,0" + ); + test_template(t, "array|min", "{{ [tool_calls_count, tool_sep_count]|min }}", {{"tool_calls_count", 2}, {"tool_sep_count", 1}},