From d5ce26807480769121961f7adadd69f1c3677f56 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 8 Nov 2015 13:36:09 -0800 Subject: [PATCH] Fix handling of changing precedence within lexical rules. A precedence annotation wrapping a sequence of characters now only affects how tightly those characters bind to *each other*, not how tightly they bind to the preceding character. This bug surfaced because a generated lexer was failing to recognize a '\n' character as a token, instead treating it as ubiquitous whitespace. It made this error because, even though anonymous ubiquitous tokens have the lowest precedence, the character immediately *after* the '\n' was part of a normal token, which had *normal* precedence (0). Advancing into that following token was incorrectly prioritized above accepting the line-break token. --- spec/compiler/build_tables/lex_item_spec.cc | 103 ++++++++---- spec/compiler/helpers/rule_helpers.cc | 7 + spec/compiler/helpers/rule_helpers.h | 1 + spec/fixtures/parsers/c.c | 157 ++++++------------ src/compiler/build_tables/build_lex_table.cc | 19 ++- src/compiler/build_tables/lex_item.cc | 14 +- src/compiler/build_tables/lex_item.h | 2 +- .../build_tables/lex_item_transitions.cc | 36 ++-- .../prepare_grammar/extract_tokens.cc | 2 +- .../prepare_grammar/flatten_grammar.cc | 16 +- src/compiler/prepare_grammar/is_token.cc | 2 +- src/compiler/rules/metadata.cc | 16 +- src/compiler/rules/metadata.h | 3 +- 13 files changed, 194 insertions(+), 184 deletions(-) diff --git a/spec/compiler/build_tables/lex_item_spec.cc b/spec/compiler/build_tables/lex_item_spec.cc index faa3aeb0..954b8d29 100644 --- a/spec/compiler/build_tables/lex_item_spec.cc +++ b/spec/compiler/build_tables/lex_item_spec.cc @@ -21,7 +21,7 @@ describe("LexItem", []() { it("returns false for rules not designated as token starts", [&]() { AssertThat(LexItem(sym, make_shared(str("a"), map({ - { START_TOKEN, 0 } + { PRECEDENCE, 5 } }))).is_token_start(), IsFalse()); AssertThat(LexItem(sym, str("a")).is_token_start(), IsFalse()); }); @@ -43,7 +43,7 @@ describe("LexItem", []() { it("indicates whether the item is done, its precedence, and whether it is a string", [&]() { LexItem item1(Symbol(0, true), character({ 'a', 'b', 'c' })); AssertThat(item1.completion_status().is_done, IsFalse()); - AssertThat(item1.completion_status().precedence, Equals(0)); + AssertThat(item1.completion_status().precedence, Equals(PrecedenceRange())); AssertThat(item1.completion_status().is_string, IsFalse()); LexItem item2(Symbol(0, true), choice({ @@ -52,12 +52,12 @@ describe("LexItem", []() { })); AssertThat(item2.completion_status().is_done, IsTrue()); - AssertThat(item2.completion_status().precedence, Equals(3)); + AssertThat(item2.completion_status().precedence, Equals(PrecedenceRange(3))); AssertThat(item2.completion_status().is_string, IsTrue()); LexItem item3(Symbol(0, true), repeat(character({ ' ', '\t' }))); AssertThat(item3.completion_status().is_done, IsTrue()); - AssertThat(item3.completion_status().precedence, Equals(0)); + AssertThat(item3.completion_status().precedence, Equals(PrecedenceRange())); AssertThat(item3.completion_status().is_string, IsFalse()); }); }); @@ -117,6 +117,7 @@ describe("LexItemSet::transitions()", [&]() { LexItemSet item_set({ LexItem(Symbol(1), seq({ prec(3, seq({ + character({ 'v' }), prec(4, seq({ character({ 'w' }), character({ 'x' }) })), @@ -125,46 +126,92 @@ describe("LexItemSet::transitions()", [&]() { })), }); + auto transitions = item_set.transitions(); + AssertThat( - item_set.transitions(), + transitions, Equals(LexItemSet::TransitionMap({ { - CharacterSet().include('w'), + CharacterSet().include('v'), { + // The outer precedence is now 'active', because we are within its + // contained rule. LexItemSet({ LexItem(Symbol(1), seq({ - prec(3, seq({ - prec(4, character({ 'x' })), + active_prec(3, seq({ + prec(4, seq({ + character({ 'w' }), + character({ 'x' }) })), character({ 'y' }) })), character({ 'z' }), })), }), - PrecedenceRange(4) + + // No precedence is applied upon entering a rule. + PrecedenceRange() } } }))); - LexItemSet item_set2({ - LexItem(Symbol(1), seq({ - prec(3, seq({ - prec(4, character({ 'x' })), - character({ 'y' }) })), - character({ 'z' }), - })), - }); + LexItemSet item_set2 = transitions[CharacterSet().include('v')].first; + transitions = item_set2.transitions(); AssertThat( - item_set2.transitions(), + transitions, + Equals(LexItemSet::TransitionMap({ + { + CharacterSet().include('w'), + { + // The inner precedence is now 'active' + LexItemSet({ + LexItem(Symbol(1), seq({ + active_prec(3, seq({ + active_prec(4, character({ 'x' })), + character({ 'y' }) })), + character({ 'z' }), + })), + }), + + // The outer precedence is applied. + PrecedenceRange(3) + } + } + }))); + + LexItemSet item_set3 = transitions[CharacterSet().include('w')].first; + transitions = item_set3.transitions(); + + AssertThat( + transitions, Equals(LexItemSet::TransitionMap({ { CharacterSet().include('x'), { LexItemSet({ LexItem(Symbol(1), seq({ - prec(3, character({ 'y' })), + active_prec(3, character({ 'y' })), character({ 'z' }), })), }), + + // The inner precedence is applied. + PrecedenceRange(4) + } + } + }))); + + LexItemSet item_set4 = transitions[CharacterSet().include('x')].first; + transitions = item_set4.transitions(); + + AssertThat( + transitions, + Equals(LexItemSet::TransitionMap({ + { + CharacterSet().include('y'), + { + LexItemSet({ + LexItem(Symbol(1), character({ 'z' })), + }), PrecedenceRange(3) } } @@ -261,7 +308,7 @@ describe("LexItemSet::transitions()", [&]() { it("handles repeats with precedence", [&]() { LexItemSet item_set({ - LexItem(Symbol(1), prec(-1, repeat1(character({ 'a' })))) + LexItem(Symbol(1), active_prec(-1, repeat1(character({ 'a' })))) }); AssertThat( @@ -271,8 +318,8 @@ describe("LexItemSet::transitions()", [&]() { CharacterSet().include('a'), { LexItemSet({ - LexItem(Symbol(1), prec(-1, repeat1(character({ 'a' })))), - LexItem(Symbol(1), prec(-1, blank())), + LexItem(Symbol(1), active_prec(-1, repeat1(character({ 'a' })))), + LexItem(Symbol(1), active_prec(-1, blank())), }), PrecedenceRange(-1) } @@ -283,11 +330,11 @@ describe("LexItemSet::transitions()", [&]() { it("handles choices between overlapping character sets", [&]() { LexItemSet item_set({ LexItem(Symbol(1), choice({ - prec(2, seq({ + active_prec(2, seq({ character({ 'a', 'b', 'c', 'd' }), character({ 'x' }), })), - prec(3, seq({ + active_prec(3, seq({ character({ 'c', 'd', 'e', 'f' }), character({ 'y' }), })), @@ -301,7 +348,7 @@ describe("LexItemSet::transitions()", [&]() { CharacterSet().include('a', 'b'), { LexItemSet({ - LexItem(Symbol(1), prec(2, character({ 'x' }))), + LexItem(Symbol(1), active_prec(2, character({ 'x' }))), }), PrecedenceRange(2) } @@ -310,8 +357,8 @@ describe("LexItemSet::transitions()", [&]() { CharacterSet().include('c', 'd'), { LexItemSet({ - LexItem(Symbol(1), prec(2, character({ 'x' }))), - LexItem(Symbol(1), prec(3, character({ 'y' }))), + LexItem(Symbol(1), active_prec(2, character({ 'x' }))), + LexItem(Symbol(1), active_prec(3, character({ 'y' }))), }), PrecedenceRange(2, 3) } @@ -320,7 +367,7 @@ describe("LexItemSet::transitions()", [&]() { CharacterSet().include('e', 'f'), { LexItemSet({ - LexItem(Symbol(1), prec(3, character({ 'y' }))), + LexItem(Symbol(1), active_prec(3, character({ 'y' }))), }), PrecedenceRange(3) } diff --git a/spec/compiler/helpers/rule_helpers.cc b/spec/compiler/helpers/rule_helpers.cc index 466ea9a8..31141d3e 100644 --- a/spec/compiler/helpers/rule_helpers.cc +++ b/spec/compiler/helpers/rule_helpers.cc @@ -39,6 +39,13 @@ namespace tree_sitter { return make_shared(rule, values); } + rule_ptr active_prec(int precedence, rule_ptr rule) { + return std::make_shared(rule, map({ + { rules::PRECEDENCE, precedence }, + { rules::IS_ACTIVE, true } + })); + } + bool operator==(const Variable &left, const Variable &right) { return left.name == right.name && left.rule->operator==(*right.rule) && left.type == right.type; diff --git a/spec/compiler/helpers/rule_helpers.h b/spec/compiler/helpers/rule_helpers.h index 7072dd59..7e6ba276 100644 --- a/spec/compiler/helpers/rule_helpers.h +++ b/spec/compiler/helpers/rule_helpers.h @@ -12,6 +12,7 @@ namespace tree_sitter { rule_ptr character(const std::set &, bool sign); rule_ptr i_sym(size_t index); rule_ptr i_token(size_t index); + rule_ptr active_prec(int precedence, rule_ptr); bool operator==(const Variable &left, const Variable &right); } diff --git a/spec/fixtures/parsers/c.c b/spec/fixtures/parsers/c.c index 63a4a4f9..9a745993 100644 --- a/spec/fixtures/parsers/c.c +++ b/spec/fixtures/parsers/c.c @@ -1803,7 +1803,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { if (lookahead == '/') ADVANCE(145); if (lookahead == '\\') - ADVANCE(149); + ADVANCE(153); if (!((lookahead == 0) || (lookahead == '\t') || (lookahead == '\n') || @@ -1811,7 +1811,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == ' ') || (lookahead == '/') || (lookahead == '\\'))) - ADVANCE(151); + ADVANCE(155); LEX_ERROR(); case 144: START_TOKEN(); @@ -1822,7 +1822,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { if (lookahead == '/') ADVANCE(145); if (lookahead == '\\') - ADVANCE(149); + ADVANCE(153); if (!((lookahead == 0) || (lookahead == '\t') || (lookahead == '\n') || @@ -1830,21 +1830,21 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == ' ') || (lookahead == '/') || (lookahead == '\\'))) - ADVANCE(151); + ADVANCE(155); ACCEPT_TOKEN(sym_preproc_arg); case 145: if (lookahead == '*') ADVANCE(146); if (lookahead == '/') - ADVANCE(154); + ADVANCE(151); if (lookahead == '\\') - ADVANCE(149); + ADVANCE(153); if (!((lookahead == 0) || (lookahead == '\n') || (lookahead == '*') || (lookahead == '/') || (lookahead == '\\'))) - ADVANCE(151); + ADVANCE(155); ACCEPT_TOKEN(sym_preproc_arg); case 146: if (lookahead == '\n') @@ -1852,7 +1852,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { if (lookahead == '*') ADVANCE(147); if (lookahead == '\\') - ADVANCE(152); + ADVANCE(149); if (!((lookahead == 0) || (lookahead == '\n') || (lookahead == '*') || @@ -1865,7 +1865,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { if (lookahead == '/') ADVANCE(148); if (lookahead == '\\') - ADVANCE(152); + ADVANCE(149); if (!((lookahead == 0) || (lookahead == '\n') || (lookahead == '/') || @@ -1873,75 +1873,77 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { ADVANCE(146); ACCEPT_TOKEN(sym_preproc_arg); case 148: - if (lookahead == '\\') - ADVANCE(149); ACCEPT_TOKEN(sym_comment); case 149: if (lookahead == '\n') ADVANCE(150); - if (lookahead == '\\') - ADVANCE(149); - if (!((lookahead == 0) || - (lookahead == '\n') || - (lookahead == '\\'))) - ADVANCE(151); - ACCEPT_TOKEN(sym_preproc_arg); - case 150: - if (lookahead == '\\') - ADVANCE(149); - if (!((lookahead == 0) || - (lookahead == '\n') || - (lookahead == '\\'))) - ADVANCE(151); - ACCEPT_TOKEN(sym_preproc_arg); - case 151: - if (lookahead == '\\') - ADVANCE(149); - if (!((lookahead == 0) || - (lookahead == '\n') || - (lookahead == '\\'))) - ADVANCE(151); - ACCEPT_TOKEN(sym_preproc_arg); - case 152: - if (lookahead == '\n') - ADVANCE(153); if (lookahead == '*') ADVANCE(147); if (lookahead == '\\') - ADVANCE(152); + ADVANCE(149); if (!((lookahead == 0) || (lookahead == '\n') || (lookahead == '*') || (lookahead == '\\'))) ADVANCE(146); ACCEPT_TOKEN(sym_preproc_arg); - case 153: + case 150: if (lookahead == '\n') ADVANCE(11); if (lookahead == '*') ADVANCE(147); if (lookahead == '\\') - ADVANCE(152); + ADVANCE(149); if (!((lookahead == 0) || (lookahead == '\n') || (lookahead == '*') || (lookahead == '\\'))) ADVANCE(146); ACCEPT_TOKEN(sym_preproc_arg); + case 151: + if (lookahead == '\\') + ADVANCE(152); + if (!((lookahead == 0) || + (lookahead == '\n') || + (lookahead == '\\'))) + ADVANCE(151); + ACCEPT_TOKEN(sym_comment); + case 152: + if (lookahead == '\\') + ADVANCE(152); + if (!((lookahead == 0) || + (lookahead == '\n') || + (lookahead == '\\'))) + ADVANCE(151); + ACCEPT_TOKEN(sym_comment); + case 153: + if (lookahead == '\n') + ADVANCE(154); + if (lookahead == '\\') + ADVANCE(153); + if (!((lookahead == 0) || + (lookahead == '\n') || + (lookahead == '\\'))) + ADVANCE(155); + ACCEPT_TOKEN(sym_preproc_arg); case 154: if (lookahead == '\\') + ADVANCE(153); + if (!((lookahead == 0) || + (lookahead == '\n') || + (lookahead == '\\'))) ADVANCE(155); - ACCEPT_TOKEN(sym_comment); + ACCEPT_TOKEN(sym_preproc_arg); case 155: if (lookahead == '\\') + ADVANCE(153); + if (!((lookahead == 0) || + (lookahead == '\n') || + (lookahead == '\\'))) ADVANCE(155); - ACCEPT_TOKEN(sym_comment); + ACCEPT_TOKEN(sym_preproc_arg); case 156: START_TOKEN(); - if (lookahead == '/') - ADVANCE(145); - if (lookahead == '\\') - ADVANCE(149); ACCEPT_TOKEN(anon_sym_LF); case 157: START_TOKEN(); @@ -1956,8 +1958,6 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { LEX_ERROR(); case 158: START_TOKEN(); - if (lookahead == '/') - ADVANCE(10); ACCEPT_TOKEN(anon_sym_LF); case 159: START_TOKEN(); @@ -2418,67 +2418,6 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { LEX_ERROR(); case 178: START_TOKEN(); - if (lookahead == 0) - ADVANCE(2); - if (lookahead == '\"') - ADVANCE(117); - if (lookahead == '#') - ADVANCE(3); - if (lookahead == '&') - ADVANCE(121); - if (lookahead == '(') - ADVANCE(97); - if (lookahead == ')') - ADVANCE(106); - if (lookahead == '*') - ADVANCE(98); - if (lookahead == '+') - ADVANCE(127); - if (lookahead == ',') - ADVANCE(134); - if (lookahead == '.') - ADVANCE(167); - if (lookahead == '/') - ADVANCE(10); - if ('0' <= lookahead && lookahead <= '9') - ADVANCE(122); - if (lookahead == ';') - ADVANCE(99); - if (lookahead == '=') - ADVANCE(161); - if (('A' <= lookahead && lookahead <= 'Z') || - (lookahead == 'b') || - (lookahead == 'd') || - ('f' <= lookahead && lookahead <= 'k') || - ('m' <= lookahead && lookahead <= 'q') || - ('w' <= lookahead && lookahead <= 'z')) - ADVANCE(15); - if (lookahead == '[') - ADVANCE(114); - if (lookahead == ']') - ADVANCE(125); - if (lookahead == 'a') - ADVANCE(16); - if (lookahead == 'c') - ADVANCE(20); - if (lookahead == 'e') - ADVANCE(25); - if (lookahead == 'l') - ADVANCE(31); - if (lookahead == 'r') - ADVANCE(35); - if (lookahead == 's') - ADVANCE(49); - if (lookahead == 't') - ADVANCE(68); - if (lookahead == 'u') - ADVANCE(75); - if (lookahead == 'v') - ADVANCE(83); - if (lookahead == '{') - ADVANCE(104); - if (lookahead == '}') - ADVANCE(109); ACCEPT_TOKEN(anon_sym_LF); case ts_lex_state_error: START_TOKEN(); diff --git a/src/compiler/build_tables/build_lex_table.cc b/src/compiler/build_tables/build_lex_table.cc index 53bcd3a8..1777700a 100644 --- a/src/compiler/build_tables/build_lex_table.cc +++ b/src/compiler/build_tables/build_lex_table.cc @@ -1,4 +1,5 @@ #include "compiler/build_tables/build_lex_table.h" +#include #include #include #include @@ -85,14 +86,14 @@ class LexTableBuilder { for (const rule_ptr &rule : rules) for (const rule_ptr &separator_rule : separator_rules) result.entries.insert(LexItem( - symbol, rules::Seq::build({ - rules::Metadata::build(separator_rule, - { - { rules::START_TOKEN, 1 }, - { rules::PRECEDENCE, -99999 }, - }), - rule, - }))); + symbol, rules::Metadata::build( + rules::Seq::build({ + rules::Metadata::build(separator_rule, {{rules::START_TOKEN, 1}}), + rules::Metadata::build(rule, {{rules::PRECEDENCE, 0}}), + }), { + {rules::PRECEDENCE, INT_MIN}, + {rules::IS_ACTIVE, true}, + }))); } return result; @@ -135,7 +136,7 @@ class LexTableBuilder { LexItem::CompletionStatus completion_status = item.completion_status(); if (completion_status.is_done) { auto current_action = lex_table.state(state_id).default_action; - auto action = LexAction::Accept(item.lhs, completion_status.precedence, + auto action = LexAction::Accept(item.lhs, completion_status.precedence.max, completion_status.is_string); if (conflict_manager.resolve(action, current_action)) lex_table.state(state_id).default_action = action; diff --git a/src/compiler/build_tables/lex_item.cc b/src/compiler/build_tables/lex_item.cc index 43c8fe72..d97e82e8 100644 --- a/src/compiler/build_tables/lex_item.cc +++ b/src/compiler/build_tables/lex_item.cc @@ -34,7 +34,7 @@ bool LexItem::is_token_start() const { } bool apply_to(const rules::Metadata *rule) { - return (rule->value_for(rules::START_TOKEN) > 0) || apply(rule->rule); + return (rule->value_for(rules::START_TOKEN).second) || apply(rule->rule); } bool apply_to(const rules::Choice *rule) { @@ -57,15 +57,15 @@ LexItem::CompletionStatus LexItem::completion_status() const { if (status.is_done) return status; } - return { false, 0, false }; + return { false, PrecedenceRange(), false }; } CompletionStatus apply_to(const rules::Metadata *rule) { CompletionStatus result = apply(rule->rule); if (result.is_done) { - if (!result.precedence && rule->value_for(rules::PRECEDENCE)) - result.precedence = rule->value_for(rules::PRECEDENCE); - if (rule->value_for(rules::IS_STRING)) + if (result.precedence.empty && rule->value_for(rules::PRECEDENCE).second) + result.precedence.add(rule->value_for(rules::PRECEDENCE).first); + if (rule->value_for(rules::IS_STRING).second) result.is_string = true; } return result; @@ -76,7 +76,7 @@ LexItem::CompletionStatus LexItem::completion_status() const { } CompletionStatus apply_to(const rules::Blank *rule) { - return { true, 0, false }; + return { true, PrecedenceRange(), false }; } CompletionStatus apply_to(const rules::Seq *rule) { @@ -84,7 +84,7 @@ LexItem::CompletionStatus LexItem::completion_status() const { if (left_status.is_done) return apply(rule->right); else - return { false, 0, false }; + return { false, PrecedenceRange(), false }; } }; diff --git a/src/compiler/build_tables/lex_item.h b/src/compiler/build_tables/lex_item.h index b547df4d..55daa80f 100644 --- a/src/compiler/build_tables/lex_item.h +++ b/src/compiler/build_tables/lex_item.h @@ -18,7 +18,7 @@ class LexItem { struct CompletionStatus { bool is_done; - int precedence; + PrecedenceRange precedence; bool is_string; }; diff --git a/src/compiler/build_tables/lex_item_transitions.cc b/src/compiler/build_tables/lex_item_transitions.cc index e3312af3..ec358cc9 100644 --- a/src/compiler/build_tables/lex_item_transitions.cc +++ b/src/compiler/build_tables/lex_item_transitions.cc @@ -73,18 +73,20 @@ class LexItemTransitions : public rules::RuleFn { { new_char_set, { new_item_set, new_precedence_range } }); } - PrecedenceRange merge_precedence(PrecedenceRange precedence) { - if (precedence.empty && !precedence_stack->empty()) - precedence.add(precedence_stack->back()); - return precedence; + map activate_precedence(map metadata) { + if (metadata.find(rules::PRECEDENCE) != metadata.end()) + metadata.insert({ rules::IS_ACTIVE, 1 }); + return metadata; } void apply_to(const CharacterSet *rule) { + PrecedenceRange precedence; + if (!precedence_stack->empty()) + precedence.add(precedence_stack->back()); + merge_transition(transitions, *rule, - LexItemSet({ - LexItem(item_lhs, rules::Blank::build()), - }), - PrecedenceRange()); + LexItemSet({ LexItem(item_lhs, rules::Blank::build()) }), + precedence); } void apply_to(const rules::Choice *rule) { @@ -100,7 +102,7 @@ class LexItemTransitions : public rules::RuleFn { transitions, pair.first, transform_item_set(pair.second.first, [&rule](rule_ptr item_rule) { return rules::Seq::build({ item_rule, rule->right }); - }), merge_precedence(pair.second.second)); + }), pair.second.second); } if (rule_can_be_blank(rule->left)) @@ -112,28 +114,32 @@ class LexItemTransitions : public rules::RuleFn { LexItemTransitions(&content_transitions, this).apply(rule->content); for (const auto &pair : content_transitions) { merge_transition(transitions, pair.first, pair.second.first, - merge_precedence(pair.second.second)); + pair.second.second); merge_transition( transitions, pair.first, transform_item_set(pair.second.first, [&rule](rule_ptr item_rule) { return rules::Seq::build({ item_rule, rule->copy() }); - }), merge_precedence(pair.second.second)); + }), pair.second.second); } } void apply_to(const rules::Metadata *rule) { LexItemSet::TransitionMap content_transitions; - precedence_stack->push_back(rule->value_for(rules::PRECEDENCE)); + auto precedence = rule->value_for(rules::PRECEDENCE); + bool has_active_precedence = precedence.second && rule->value_for(rules::IS_ACTIVE).second; + if (has_active_precedence) + precedence_stack->push_back(precedence.first); LexItemTransitions(&content_transitions, this).apply(rule->rule); for (const auto &pair : content_transitions) merge_transition( transitions, pair.first, - transform_item_set(pair.second.first, [&rule](rule_ptr item_rule) { - return rules::Metadata::build(item_rule, rule->value); + transform_item_set(pair.second.first, [this, &rule](rule_ptr item_rule) { + return rules::Metadata::build(item_rule, activate_precedence(rule->value)); }), pair.second.second); - precedence_stack->pop_back(); + if (has_active_precedence) + precedence_stack->pop_back(); } public: diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index ecc1295f..6b9f9211 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -79,7 +79,7 @@ class TokenExtractor : public rules::IdentityRuleFn { } rule_ptr apply_to(const rules::Metadata *rule) { - if (rule->value_for(rules::IS_TOKEN) > 0) + if (rule->value_for(rules::IS_TOKEN).second) return apply_to_token(rule->rule.get(), VariableTypeAuxiliary); else return rules::IdentityRuleFn::apply_to(rule); diff --git a/src/compiler/prepare_grammar/flatten_grammar.cc b/src/compiler/prepare_grammar/flatten_grammar.cc index d787e31f..5d5f6834 100644 --- a/src/compiler/prepare_grammar/flatten_grammar.cc +++ b/src/compiler/prepare_grammar/flatten_grammar.cc @@ -29,23 +29,23 @@ class FlattenRule : public rules::RuleFn { } void apply_to(const rules::Metadata *metadata) { - int precedence = metadata->value_for(rules::PRECEDENCE); - int associativity = metadata->value_for(rules::ASSOCIATIVITY); + auto precedence = metadata->value_for(rules::PRECEDENCE); + auto associativity = metadata->value_for(rules::ASSOCIATIVITY); - if (precedence != 0) - precedence_stack.push_back(precedence); - if (associativity != 0) + if (precedence.second) + precedence_stack.push_back(precedence.first); + if (associativity.second) associativity_stack.push_back( - static_cast(associativity)); + static_cast(associativity.first)); apply(metadata->rule); - if (precedence != 0) { + if (precedence.second) { precedence_stack.pop_back(); production.back().precedence = precedence_stack.back(); } - if (associativity != 0) { + if (associativity.second) { associativity_stack.pop_back(); production.back().associativity = associativity_stack.back(); } diff --git a/src/compiler/prepare_grammar/is_token.cc b/src/compiler/prepare_grammar/is_token.cc index eb1628e3..b7dfab1f 100644 --- a/src/compiler/prepare_grammar/is_token.cc +++ b/src/compiler/prepare_grammar/is_token.cc @@ -18,7 +18,7 @@ class IsToken : public rules::RuleFn { } bool apply_to(const rules::Metadata *rule) { - return rule->value_for(rules::IS_TOKEN); + return rule->value_for(rules::IS_TOKEN).second; } }; diff --git a/src/compiler/rules/metadata.cc b/src/compiler/rules/metadata.cc index 9b783e9f..36479900 100644 --- a/src/compiler/rules/metadata.cc +++ b/src/compiler/rules/metadata.cc @@ -10,6 +10,7 @@ namespace rules { using std::hash; using std::make_shared; using std::map; +using std::pair; Metadata::Metadata(rule_ptr rule, map values) : rule(rule), value(values) {} @@ -36,13 +37,20 @@ rule_ptr Metadata::copy() const { return make_shared(rule->copy(), value); } -int Metadata::value_for(MetadataKey key) const { - auto pair = value.find(key); - return (pair != value.end()) ? pair->second : 0; +pair Metadata::value_for(MetadataKey key) const { + auto entry = value.find(key); + if (entry == value.end()) + return {0, false}; + else + return {entry->second, true}; } std::string Metadata::to_string() const { - return "(metadata " + rule->to_string() + ")"; + auto precedence = value_for(rules::PRECEDENCE); + if (precedence.second && value_for(rules::IS_ACTIVE).second) + return "(metadata prec:" + std::to_string(precedence.first) + " " + rule->to_string() + ")"; + else + return "(metadata " + rule->to_string() + ")"; } void Metadata::accept(Visitor *visitor) const { diff --git a/src/compiler/rules/metadata.h b/src/compiler/rules/metadata.h index bcf23bd8..2b22dfea 100644 --- a/src/compiler/rules/metadata.h +++ b/src/compiler/rules/metadata.h @@ -21,6 +21,7 @@ enum MetadataKey { ASSOCIATIVITY, IS_TOKEN, IS_STRING, + IS_ACTIVE, }; class Metadata : public Rule { @@ -33,7 +34,7 @@ class Metadata : public Rule { rule_ptr copy() const; std::string to_string() const; void accept(Visitor *visitor) const; - int value_for(MetadataKey key) const; + std::pair value_for(MetadataKey key) const; const rule_ptr rule; const std::map value;