traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Store parse items' list of consumed symbols

Browse source 
Not just the consumed symbol count.
This commit is contained in:
Max Brunsfeld 2015-02-20 22:53:27 -08:00
parent 7a71d91fb6
commit ccb0d0d043
9 changed files with 28 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

10
spec/compiler/build_tables/build_conflict_spec.cc
View file

@ -30,15 +30,15 @@ describe("build_conflict", []() {
ParseAction::Reduce(Symbol(2), 1, 0), // reduced_rule
ParseItemSet({
{
ParseItem(Symbol(0), blank(), 2), // in_progress_rule1
ParseItem(Symbol(0), blank(), { Symbol(101) }), // in_progress_rule1
set<Symbol>({ Symbol(2, SymbolOptionToken) })
},
{
ParseItem(Symbol(1), blank(), 2), // in_progress_rule2
ParseItem(Symbol(1), blank(), { Symbol(101) }), // in_progress_rule2
set<Symbol>({ Symbol(2, SymbolOptionToken) })
},
{
ParseItem(Symbol(3), blank(), 0), // other_rule1
ParseItem(Symbol(3), blank(), {}), // other_rule1
set<Symbol>({ Symbol(2, SymbolOptionToken) })
},
}),
@ -58,11 +58,11 @@ describe("build_conflict", []() {
ParseAction::Shift(2, set<int>()),
ParseItemSet({
{
ParseItem(Symbol(0), blank(), 2), // in_progress_rule1
ParseItem(Symbol(0), blank(), { Symbol(101) }), // in_progress_rule1
set<Symbol>({ Symbol(2, SymbolOptionToken) })
},
{
ParseItem(Symbol(1), blank(), 2), // in_progress_rule2
ParseItem(Symbol(1), blank(), { Symbol(101) }), // in_progress_rule2
set<Symbol>({ Symbol(2, SymbolOptionToken) })
},
}),

6
spec/compiler/build_tables/item_set_closure_spec.cc
View file

@ -20,18 +20,18 @@ describe("item_set_closure", []() {
it("adds items at the beginnings of referenced rules", [&]() {
ParseItemSet item_set = item_set_closure(
ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0),
ParseItem(Symbol(0), grammar.rule(Symbol(0)), {}),
set<Symbol>({ Symbol(10, SymbolOptionToken) }),
grammar
);
AssertThat(item_set, Equals(ParseItemSet({
{
ParseItem(Symbol(1), grammar.rule(Symbol(1)), 0),
ParseItem(Symbol(1), grammar.rule(Symbol(1)), {}),
set<Symbol>({ Symbol(11, SymbolOptionToken) }),
},
{
ParseItem(Symbol(0), grammar.rule(Symbol(0)), 0),
ParseItem(Symbol(0), grammar.rule(Symbol(0)), {}),
set<Symbol>({ Symbol(10, SymbolOptionToken) }),
},
})));

6
spec/compiler/build_tables/item_set_transitions_spec.cc
View file

@ -50,7 +50,7 @@ describe("sym_transitions(ParseItemSet, SyntaxGrammar)", [&]() {
it("computes the closure of the new item sets", [&]() {
ParseItemSet set1({
{
ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), 3),
ParseItem(Symbol(0), seq({ i_token(22), i_sym(1) }), { Symbol(101) }),
set<Symbol>({ Symbol(23, SymbolOptionToken) })
},
});
@ -60,11 +60,11 @@ describe("sym_transitions(ParseItemSet, SyntaxGrammar)", [&]() {
Symbol(22, SymbolOptionToken),
ParseItemSet({
{
ParseItem(Symbol(0), i_sym(1), 4),
ParseItem(Symbol(0), i_sym(1), { Symbol(101), Symbol(22) }),
set<Symbol>({ Symbol(23, SymbolOptionToken) }),
},
{
ParseItem(Symbol(1), i_token(21), 0),
ParseItem(Symbol(1), i_token(21), {}),
set<Symbol>({ Symbol(23, SymbolOptionToken) })
},
})

2
src/compiler/build_tables/build_conflict.cc
View file

@ -37,7 +37,7 @@ static string action_description(const ParseAction &action,
case ParseActionTypeShift: {
string result("shift (");
for (const auto &item : item_set)
if (item.first.consumed_symbol_count > 0)
if (!item.first.consumed_symbols.empty())
result += " " + symbol_name(item.first.lhs, grammar, lex_grammar);
return result + " )";
}

6
src/compiler/build_tables/build_parse_table.cc
View file

@ -43,7 +43,7 @@ class ParseTableBuilder {
auto start_symbol = grammar.rules.empty()
? make_shared<Symbol>(0, rules::SymbolOptionToken)
: make_shared<Symbol>(0);
ParseItem start_item(rules::START(), start_symbol, 0);
ParseItem start_item(rules::START(), start_symbol, {});
add_parse_state(
item_set_closure(start_item, { rules::END_OF_INPUT() }, grammar));
@ -104,7 +104,7 @@ class ParseTableBuilder {
ParseAction action =
(item.lhs == rules::START())
? ParseAction::Accept()
: ParseAction::Reduce(item.lhs, item.consumed_symbol_count,
: ParseAction::Reduce(item.lhs, item.consumed_symbols.size(),
item.precedence());
for (const auto &lookahead_sym : lookahead_symbols)
@ -169,7 +169,7 @@ class ParseTableBuilder {
set<int> result;
for (const auto &pair : item_set) {
const ParseItem &item = pair.first;
if (item.consumed_symbol_count > 0)
if (!item.consumed_symbols.empty())
result.insert(item.precedence());
}
return result;

2
src/compiler/build_tables/item_set_closure.cc
View file

@ -50,7 +50,7 @@ const ParseItemSet item_set_closure(const ParseItem &starting_item,
next_lookahead_symbols.insert(lookahead_symbols.begin(),
lookahead_symbols.end());
items_to_process.push_back({ ParseItem(symbol, grammar.rule(symbol), 0),
items_to_process.push_back({ ParseItem(symbol, grammar.rule(symbol), {}),
next_lookahead_symbols });
}
}

6
src/compiler/build_tables/item_set_transitions.cc
View file

@ -12,6 +12,7 @@ namespace build_tables {
using std::map;
using std::set;
using std::vector;
using rules::CharacterSet;
using rules::Symbol;
@ -22,8 +23,9 @@ map<Symbol, ParseItemSet> sym_transitions(const ParseItemSet &item_set,
const ParseItem &item = pair.first;
const set<Symbol> &lookahead_symbols = pair.second;
for (auto &transition : sym_transitions(item.rule)) {
ParseItem new_item(item.lhs, transition.second,
item.consumed_symbol_count + 1);
vector<Symbol> consumed_symbols(item.consumed_symbols);
consumed_symbols.push_back(transition.first);
ParseItem new_item(item.lhs, transition.second, consumed_symbols);
merge_sym_transition<ParseItemSet>(
&result, { transition.first,
item_set_closure(new_item, lookahead_symbols, grammar) },

7
src/compiler/build_tables/parse_item.cc
View file

@ -5,15 +5,16 @@ namespace tree_sitter {
namespace build_tables {
using std::string;
using std::vector;
using std::ostream;
ParseItem::ParseItem(const rules::Symbol &lhs, const rules::rule_ptr rule,
size_t consumed_symbol_count)
: Item(lhs, rule), consumed_symbol_count(consumed_symbol_count) {}
const vector<rules::Symbol> &consumed_symbols)
: Item(lhs, rule), consumed_symbols(consumed_symbols) {}
bool ParseItem::operator==(const ParseItem &other) const {
return (lhs == other.lhs) &&
(consumed_symbol_count == other.consumed_symbol_count) &&
(consumed_symbols.size() == other.consumed_symbols.size()) &&
(rule == other.rule || rule->operator==(*other.rule));
}

7
src/compiler/build_tables/parse_item.h
View file

@ -3,6 +3,7 @@
#include <set>
#include <unordered_map>
#include <vector>
#include "compiler/build_tables/item.h"
#include "compiler/rules/symbol.h"
@ -12,9 +13,9 @@ namespace build_tables {
class ParseItem : public Item {
public:
ParseItem(const rules::Symbol &lhs, rules::rule_ptr rule,
const size_t consumed_symbol_count);
const std::vector<rules::Symbol> &consumed_symbols);
bool operator==(const ParseItem &other) const;
size_t consumed_symbol_count;
std::vector<rules::Symbol> consumed_symbols;
};
std::ostream &operator<<(std::ostream &stream, const ParseItem &item);
@ -31,7 +32,7 @@ struct hash<tree_sitter::build_tables::ParseItem> {
size_t operator()(const tree_sitter::build_tables::ParseItem &item) const {
return hash<tree_sitter::rules::Symbol>()(item.lhs) ^
hash<tree_sitter::rules::rule_ptr>()(item.rule) ^
hash<size_t>()(item.consumed_symbol_count);
hash<size_t>()(item.consumed_symbols.size());
}
};