diff --git a/spec/compiler/build_tables/build_conflict_spec.cc b/spec/compiler/build_tables/build_conflict_spec.cc index 1f30ac12..8a0a4c3e 100644 --- a/spec/compiler/build_tables/build_conflict_spec.cc +++ b/spec/compiler/build_tables/build_conflict_spec.cc @@ -30,15 +30,15 @@ describe("build_conflict", []() { ParseAction::Reduce(Symbol(2), 1, 0), // reduced_rule ParseItemSet({ { - ParseItem(Symbol(0), blank(), 2), // in_progress_rule1 + ParseItem(Symbol(0), blank(), { Symbol(101) }), // in_progress_rule1 set({ Symbol(2, SymbolOptionToken) }) }, { - ParseItem(Symbol(1), blank(), 2), // in_progress_rule2 + ParseItem(Symbol(1), blank(), { Symbol(101) }), // in_progress_rule2 set({ Symbol(2, SymbolOptionToken) }) }, { - ParseItem(Symbol(3), blank(), 0), // other_rule1 + ParseItem(Symbol(3), blank(), {}), // other_rule1 set({ Symbol(2, SymbolOptionToken) }) }, }), @@ -58,11 +58,11 @@ describe("build_conflict", []() { ParseAction::Shift(2, set()), ParseItemSet({ { - ParseItem(Symbol(0), blank(), 2), // in_progress_rule1 + ParseItem(Symbol(0), blank(), { Symbol(101) }), // in_progress_rule1 set({ Symbol(2, SymbolOptionToken) }) }, { - ParseItem(Symbol(1), blank(), 2), // in_progress_rule2 + ParseItem(Symbol(1), blank(), { Symbol(101) }), // in_progress_rule2 set({ Symbol(2, SymbolOptionToken) }) }, }), diff --git a/spec/compiler/build_tables/item_set_closure_spec.cc b/spec/compiler/build_tables/item_set_closure_spec.cc index 0b81b66a..4716450b 100644 --- a/spec/compiler/build_tables/item_set_closure_spec.cc +++ b/spec/compiler/build_tables/item_set_closure_spec.cc @@ -20,18 +20,18 @@ describe("item_set_closure", []() { it("adds items at the beginnings of referenced rules", [&]() { ParseItemSet item_set = item_set_closure( - ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0), + ParseItem(Symbol(0), grammar.rule(Symbol(0)), {}), set({ Symbol(10, SymbolOptionToken) }), grammar ); AssertThat(item_set, Equals(ParseItemSet({ { - ParseItem(Symbol(1), grammar.rule(Symbol(1)), 0), + ParseItem(Symbol(1), grammar.rule(Symbol(1)), {}), set({ Symbol(11, SymbolOptionToken) }), }, { - ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0), + ParseItem(Symbol(0), grammar.rule(Symbol(0)), {}), set({ Symbol(10, SymbolOptionToken) }), }, }))); diff --git a/spec/compiler/build_tables/item_set_transitions_spec.cc b/spec/compiler/build_tables/item_set_transitions_spec.cc index 977054e4..78a78a16 100644 --- a/spec/compiler/build_tables/item_set_transitions_spec.cc +++ b/spec/compiler/build_tables/item_set_transitions_spec.cc @@ -50,7 +50,7 @@ describe("sym_transitions(ParseItemSet, SyntaxGrammar)", [&]() { it("computes the closure of the new item sets", [&]() { ParseItemSet set1({ { - ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), 3), + ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), { Symbol(101) }), set({ Symbol(23, SymbolOptionToken) }) }, }); @@ -60,11 +60,11 @@ describe("sym_transitions(ParseItemSet, SyntaxGrammar)", [&]() { Symbol(22, SymbolOptionToken), ParseItemSet({ { - ParseItem(Symbol(0), i_sym(1), 4), + ParseItem(Symbol(0), i_sym(1), { Symbol(101), Symbol(22) }), set({ Symbol(23, SymbolOptionToken) }), }, { - ParseItem(Symbol(1), i_token(21), 0), + ParseItem(Symbol(1), i_token(21), {}), set({ Symbol(23, SymbolOptionToken) }) }, }) diff --git a/src/compiler/build_tables/build_conflict.cc b/src/compiler/build_tables/build_conflict.cc index ae239696..9c92dc42 100644 --- a/src/compiler/build_tables/build_conflict.cc +++ b/src/compiler/build_tables/build_conflict.cc @@ -37,7 +37,7 @@ static string action_description(const ParseAction &action, case ParseActionTypeShift: { string result("shift ("); for (const auto &item : item_set) - if (item.first.consumed_symbol_count > 0) + if (!item.first.consumed_symbols.empty()) result += " " + symbol_name(item.first.lhs, grammar, lex_grammar); return result + " )"; } diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index 9ba6e3f8..61e7b103 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -43,7 +43,7 @@ class ParseTableBuilder { auto start_symbol = grammar.rules.empty() ? make_shared(0, rules::SymbolOptionToken) : make_shared(0); - ParseItem start_item(rules::START(), start_symbol, 0); + ParseItem start_item(rules::START(), start_symbol, {}); add_parse_state( item_set_closure(start_item, { rules::END_OF_INPUT() }, grammar)); @@ -104,7 +104,7 @@ class ParseTableBuilder { ParseAction action = (item.lhs == rules::START()) ? ParseAction::Accept() - : ParseAction::Reduce(item.lhs, item.consumed_symbol_count, + : ParseAction::Reduce(item.lhs, item.consumed_symbols.size(), item.precedence()); for (const auto &lookahead_sym : lookahead_symbols) @@ -169,7 +169,7 @@ class ParseTableBuilder { set result; for (const auto &pair : item_set) { const ParseItem &item = pair.first; - if (item.consumed_symbol_count > 0) + if (!item.consumed_symbols.empty()) result.insert(item.precedence()); } return result; diff --git a/src/compiler/build_tables/item_set_closure.cc b/src/compiler/build_tables/item_set_closure.cc index b84104b3..ca5a08c0 100644 --- a/src/compiler/build_tables/item_set_closure.cc +++ b/src/compiler/build_tables/item_set_closure.cc @@ -50,7 +50,7 @@ const ParseItemSet item_set_closure(const ParseItem &starting_item, next_lookahead_symbols.insert(lookahead_symbols.begin(), lookahead_symbols.end()); - items_to_process.push_back({ ParseItem(symbol, grammar.rule(symbol), 0), + items_to_process.push_back({ ParseItem(symbol, grammar.rule(symbol), {}), next_lookahead_symbols }); } } diff --git a/src/compiler/build_tables/item_set_transitions.cc b/src/compiler/build_tables/item_set_transitions.cc index 93dcba18..d4c5eefc 100644 --- a/src/compiler/build_tables/item_set_transitions.cc +++ b/src/compiler/build_tables/item_set_transitions.cc @@ -12,6 +12,7 @@ namespace build_tables { using std::map; using std::set; +using std::vector; using rules::CharacterSet; using rules::Symbol; @@ -22,8 +23,9 @@ map sym_transitions(const ParseItemSet &item_set, const ParseItem &item = pair.first; const set &lookahead_symbols = pair.second; for (auto &transition : sym_transitions(item.rule)) { - ParseItem new_item(item.lhs, transition.second, - item.consumed_symbol_count + 1); + vector consumed_symbols(item.consumed_symbols); + consumed_symbols.push_back(transition.first); + ParseItem new_item(item.lhs, transition.second, consumed_symbols); merge_sym_transition( &result, { transition.first, item_set_closure(new_item, lookahead_symbols, grammar) }, diff --git a/src/compiler/build_tables/parse_item.cc b/src/compiler/build_tables/parse_item.cc index 840eb1d4..ea886151 100644 --- a/src/compiler/build_tables/parse_item.cc +++ b/src/compiler/build_tables/parse_item.cc @@ -5,15 +5,16 @@ namespace tree_sitter { namespace build_tables { using std::string; +using std::vector; using std::ostream; ParseItem::ParseItem(const rules::Symbol &lhs, const rules::rule_ptr rule, - size_t consumed_symbol_count) - : Item(lhs, rule), consumed_symbol_count(consumed_symbol_count) {} + const vector &consumed_symbols) + : Item(lhs, rule), consumed_symbols(consumed_symbols) {} bool ParseItem::operator==(const ParseItem &other) const { return (lhs == other.lhs) && - (consumed_symbol_count == other.consumed_symbol_count) && + (consumed_symbols.size() == other.consumed_symbols.size()) && (rule == other.rule || rule->operator==(*other.rule)); } diff --git a/src/compiler/build_tables/parse_item.h b/src/compiler/build_tables/parse_item.h index f3ec4413..aa6216b3 100644 --- a/src/compiler/build_tables/parse_item.h +++ b/src/compiler/build_tables/parse_item.h @@ -3,6 +3,7 @@ #include #include +#include #include "compiler/build_tables/item.h" #include "compiler/rules/symbol.h" @@ -12,9 +13,9 @@ namespace build_tables { class ParseItem : public Item { public: ParseItem(const rules::Symbol &lhs, rules::rule_ptr rule, - const size_t consumed_symbol_count); + const std::vector &consumed_symbols); bool operator==(const ParseItem &other) const; - size_t consumed_symbol_count; + std::vector consumed_symbols; }; std::ostream &operator<<(std::ostream &stream, const ParseItem &item); @@ -31,7 +32,7 @@ struct hash { size_t operator()(const tree_sitter::build_tables::ParseItem &item) const { return hash()(item.lhs) ^ hash()(item.rule) ^ - hash()(item.consumed_symbol_count); + hash()(item.consumed_symbols.size()); } };