diff --git a/common/jinja/lexer.h b/common/jinja/lexer.h index 439c85764..3f4ebb62a 100644 --- a/common/jinja/lexer.h +++ b/common/jinja/lexer.h @@ -37,6 +37,7 @@ struct token { call_operator, // () additive_binary_operator, // + - ~ multiplicative_binary_operator, // * / % + power_operator, // ** comparison_binary_operator, // < > <= >= == != unary_operator, // ! - + comment, // {# ... #} @@ -71,6 +72,7 @@ static std::string type_to_string(token::type t) { case token::call_operator: return "call_operator"; case token::additive_binary_operator: return "additive_binary_operator"; case token::multiplicative_binary_operator: return "multiplicative_binary_operator"; + case token::power_operator: return "power_operator"; case token::comparison_binary_operator: return "comparison_binary_operator"; case token::unary_operator: return "unary_operator"; case token::comment: return "comment"; @@ -137,6 +139,7 @@ struct lexer { {"+", token::additive_binary_operator}, {"-", token::additive_binary_operator}, {"~", token::additive_binary_operator}, + {"**", token::power_operator}, {"*", token::multiplicative_binary_operator}, {"/", token::multiplicative_binary_operator}, {"%", token::multiplicative_binary_operator}, diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp index 7e99a1ac8..0841828c4 100644 --- a/common/jinja/parser.cpp +++ b/common/jinja/parser.cpp @@ -178,7 +178,7 @@ private: // bool has_caller_args = false; if (is(token::open_paren)) { // Optional caller arguments, e.g. {% call(user) dump_users(...) %} - caller_args = parse_args(); + caller_args = parse_args(false); // has_caller_args = true; } auto callee = parse_primary_expression(); @@ -283,7 +283,7 @@ private: statement_ptr parse_macro_statement(size_t start_pos) { auto name = parse_primary_expression(); - auto args = parse_args(); + auto args = parse_args(false); expect(token::close_statement, "Expected %}"); statements body; // Keep going until we hit {% endmacro @@ -397,11 +397,13 @@ private: statement_ptr parse_comparison_expression() { // NOTE: membership has same precedence as comparison - // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana'))) - auto left = parse_additive_expression(); + // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in 'apple' and 'apple' == 'b' and 'b' in 'banana') + statements operands; + std::vector ops; + operands.push_back(parse_additive_expression()); + size_t pos = current; while (true) { token op; - size_t start_pos = current; if (is_identifier("not") && peek(1).t == token::identifier && peek(1).value == "in") { op = {token::identifier, "not in", tokens[current].pos}; ++current; // consume 'not' @@ -411,9 +413,17 @@ private: } else if (is(token::comparison_binary_operator)) { op = next(); } else break; - left = mk_stmt(start_pos, op, std::move(left), parse_additive_expression()); + ops.push_back(std::move(op)); + operands.push_back(parse_additive_expression()); } - return left; + if (ops.empty()) { + return std::move(operands[0]); + } + if (ops.size() == 1) { + return mk_stmt(pos, ops[0], std::move(operands[0]), std::move(operands[1])); + } + // chained comparison, e.g. 1 <= x <= 100 + return mk_stmt(pos, std::move(operands), std::move(ops)); } statement_ptr parse_additive_expression() { @@ -500,17 +510,22 @@ private: return parse_call_expression(std::move(member)); // foo.x()() } - statements parse_args() { + statements parse_args(bool allow_spread = true) { // comma-separated arguments list expect(token::open_paren, "Expected ("); statements args; while (!is(token::close_paren)) { statement_ptr arg; - // unpacking: *expr - if (peek().t == token::multiplicative_binary_operator && peek().value == "*") { + // unpacking: *expr or **expr + const auto & t = peek(); + if (t.t == token::power_operator || (t.t == token::multiplicative_binary_operator && t.value == "*")) { + if (!allow_spread) { + throw std::runtime_error("Unexpected token: " + t.value + " of type " + std::to_string(t.t)); + } size_t start_pos = current; - ++current; // consume * - arg = mk_stmt(start_pos, parse_expression()); + ++current; // consume * or ** + bool is_kwargs = t.t == token::power_operator; + arg = mk_stmt(start_pos, parse_expression(), is_kwargs); } else { arg = parse_expression(); if (is(token::equals)) { diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index d32c79fa5..d60a99d25 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -117,20 +117,8 @@ value object_literal::execute_impl(context & ctx) const { return obj; } -value binary_expression::execute_impl(context & ctx) const { - value left_val = left->execute(ctx); - - // Logical operators - if (op.value == "and") { - JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str()); - return left_val->as_bool() ? right->execute(ctx) : std::move(left_val); - } else if (op.value == "or") { - JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str()); - return left_val->as_bool() ? std::move(left_val) : right->execute(ctx); - } - +static value exec_binary_op(context & ctx, const token & op, value left_val, value right_val) { // Equality operators - value right_val = right->execute(ctx); JJ_DEBUG("Executing binary expression %s '%s' %s", left_val->type().c_str(), op.value.c_str(), right_val->type().c_str()); if (op.value == "==") { return mk_val(*left_val == *right_val); @@ -307,6 +295,67 @@ value binary_expression::execute_impl(context & ctx) const { throw std::runtime_error("Unknown operator \"" + op.value + "\" between " + left_val->type() + " and " + right_val->type()); } +value binary_expression::execute_impl(context & ctx) const { + value left_val = left->execute(ctx); + + // Logical operators + if (op.value == "and") { + JJ_DEBUG("Executing logical test: %s AND %s", left->type().c_str(), right->type().c_str()); + return left_val->as_bool() ? right->execute(ctx) : std::move(left_val); + } else if (op.value == "or") { + JJ_DEBUG("Executing logical test: %s OR %s", left->type().c_str(), right->type().c_str()); + return left_val->as_bool() ? std::move(left_val) : right->execute(ctx); + } + + return exec_binary_op(ctx, op, std::move(left_val), right->execute(ctx)); +} + +value comparison_chain_expression::execute_impl(context & ctx) const { + value left_val = operands[0]->execute(ctx); + value res; + for (size_t i = 0; i < ops.size(); ++i) { + value right_val = operands[i + 1]->execute(ctx); + res = exec_binary_op(ctx, ops[i], left_val, right_val); + if (!res->as_bool()) { + break; // same as "and", the next operands are not executed + } + left_val = std::move(right_val); + } + return res; +} + +static void exec_call_args(context & ctx, const statements & arg_stmts, func_args & args) { + for (const auto & arg_stmt : arg_stmts) { + if (!is_stmt(arg_stmt)) { + auto arg_val = arg_stmt->execute(ctx); + JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); + args.push_back(arg_val); + continue; + } + auto spread = cast_stmt(arg_stmt); + value val = spread->argument->execute(ctx); + JJ_DEBUG(" Unpacked argument type: %s", val->type().c_str()); + if (spread->is_kwargs) { + if (!is_val(val)) { + throw std::runtime_error("Argument after ** must be an object, got " + val->type()); + } + for (const auto & pair : val->as_ordered_object()) { + if (!is_val(pair.first)) { + throw std::runtime_error("Keyword argument key must be a string, got " + pair.first->type()); + } + args.push_back(mk_val(pair.first->as_string().str(), pair.second)); + } + } else { + if (!is_val(val)) { + throw std::runtime_error("Argument after * must be an array, got " + val->type()); + } + for (const auto & item : val->as_array()) { + args.push_back(item); + } + } + } +} + static value apply_filter(context & ctx, const statement_ptr & filter, value input) { JJ_DEBUG("Applying filter to %s", input->type().c_str()); @@ -324,9 +373,7 @@ static value apply_filter(context & ctx, const statement_ptr & filter, value inp JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str()); func_args args(ctx); - for (const auto & arg_expr : call->args) { - args.push_back(arg_expr->execute(ctx)); - } + exec_call_args(ctx, call->args, args); return apply_filter_by_name(ctx, filter_id, input, args); @@ -369,9 +416,7 @@ value test_expression::execute_impl(context & ctx) const { test_id = cast_stmt(call->callee)->val; JJ_DEBUG("Applying test '%s' with arguments to %s", test_id.c_str(), input->type().c_str()); - for (const auto & arg_expr : call->args) { - args.push_back(arg_expr->execute(ctx)); - } + exec_call_args(ctx, call->args, args); } else { throw std::runtime_error("Invalid test expression"); @@ -762,12 +807,7 @@ value call_statement::execute_impl(context & ctx) const { call_ctx.set_val("caller", mk_val("caller", func)); func_args args(call_ctx); - - for (const auto & arg_expr : call_expr->args) { - auto arg_val = arg_expr->execute(ctx); - JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); - args.push_back(arg_val); - } + exec_call_args(ctx, call_expr->args, args); JJ_DEBUG("Calling macro '%s' with %zu arguments", callee_func->name.c_str(), args.count()); return callee_func->invoke(args); @@ -907,11 +947,7 @@ value member_expression::execute_impl(context & ctx) const { value call_expression::execute_impl(context & ctx) const { // gather arguments func_args args(ctx); - for (auto & arg_stmt : this->args) { - auto arg_val = arg_stmt->execute(ctx); - JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); - args.push_back(arg_val); - } + exec_call_args(ctx, this->args, args); // execute callee value callee_val = callee->execute(ctx); if (!is_val(callee_val)) { diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 101e927c4..43e108551 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -471,6 +471,28 @@ struct binary_expression : public expression { } }; +/** + * Two or more comparisons in a row, e.g. `a < b <= c` + * It is the same as `a < b and b <= c`, but each operand is executed only one time. + */ +struct comparison_chain_expression : public expression { + statements operands; + std::vector ops; + + comparison_chain_expression(statements && operands, std::vector && ops) + : operands(std::move(operands)), ops(std::move(ops)) { + assert(this->operands.size() == this->ops.size() + 1); + for (const auto & operand : this->operands) chk_type(operand); + } + std::string type() const override { return "ComparisonChainExpression"; } + value execute_impl(context & ctx) const override; + void visit(context & ctx) const override { + ctx.visitor(false, this, { + {"operands", stmts_to_ptr(operands)} + }); + } +}; + /** * An operation with two sides, separated by the | operator. * Operator precedence: https://github.com/pallets/jinja/issues/379#issuecomment-168076202 @@ -633,7 +655,9 @@ struct keyword_argument_expression : public expression { struct spread_expression : public expression { statement_ptr argument; - explicit spread_expression(statement_ptr && argument) : argument(std::move(argument)) { + bool is_kwargs; + spread_expression(statement_ptr && argument, bool is_kwargs) + : argument(std::move(argument)), is_kwargs(is_kwargs) { chk_type(this->argument); } std::string type() const override { return "SpreadExpression"; } diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index 29de01ea9..c79b0d24b 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -172,6 +172,18 @@ static void test_conditionals(testing & t) { "big" ); + test_template(t, "chained comparison", + "{{ 1 <= x <= 100 }},{{ 1 <= y <= 100 }},{{ 1 < 2 < 3 < 2 }},{{ 1 == 1 != 2 }},{{ 'a' in 'apple' == 'apple' }}", + {{"x", 42}, {"y", 500}}, + "True,False,False,True,True" + ); + + test_template(t, "chained comparison short circuit", + "{{ 2 < 1 < y }}", + json::object(), + "False" + ); + test_template(t, "object comparison", "{% if {0: 1, none: 2, 1.0: 3, '0': 4, true: 5} == {false: 1, none: 2, 1: 5, '0': 4} %}equal{% endif %}", json::object(), @@ -1321,6 +1333,18 @@ static void test_macros(testing & t) { "Hello, John Smith,Hi, Jane Doe" ); + test_template(t, "macro with unpacked args", + "{% macro add(a, b, c=0) %}{{ a + b + c }}{% endmacro %}{{ add(*[1, 2]) }},{{ add(1, **{'b': 2, 'c': 3}) }},{{ add(*xs, **kw) }}", + {{"xs", json::array({1, 2})}, {"kw", {{"c", 5}}}}, + "3,6,8" + ); + + test_template(t, "filter with unpacked args", + "{{ 'a-b' | replace(*['-', '+']) }}", + json::object(), + "a+b" + ); + test_template(t, "macro with caller", "\ {%- macro nest_dict(o, i, ff='') %}\n\ @@ -2429,6 +2453,12 @@ static void test_object_methods(testing & t) { {{"obj", {{"a", 3}, {"b", 1}}}}, "{\"a\": 3, \"b\": 1, \"c\": 2}" ); + + test_template(t, "dict from dict and unpacked kwargs", + "{% set o = dict(obj, **{k: 2}) %}{{ o|tojson }}", + {{"obj", {{"a", 3}, {"b", 1}}}, {"k", "c"}}, + "{\"a\": 3, \"b\": 1, \"c\": 2}" + ); } static void test_hasher(testing & t) {