traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cache state transition computations by grammar rule

Browse source 
This results in a huge speed increase
This commit is contained in:
Max Brunsfeld 2014-04-27 23:03:37 -07:00
parent 4eece88e18
commit 4dd1f9fbce
5 changed files with 8925 additions and 9053 deletions
Download patch file Download diff file Expand all files Collapse all files

17896
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

1
src/compiler/build_tables/build_tables.cc
View file

@ -194,6 +194,7 @@ namespace tree_sitter {
return conflict_manager.conflicts();
};
SymTransitions sym_transitions;
ParseTable parse_table;
LexTable lex_table;
};

67
src/compiler/build_tables/item_set_transitions.cc
View file

@ -19,6 +19,54 @@ namespace tree_sitter {
return result;
}
const ISymbol placeholder = ISymbol(-99);
static map<ISymbol, ParseItemSet> sym_transitions_for_rule(SymTransitions *self, const rules::rule_ptr &rule, const PreparedGrammar &grammar) {
auto pair = self->transitions_cache.find(rule);
if (pair != self->transitions_cache.end()) return pair->second;
map<ISymbol, ParseItemSet> result;
for (auto &transition : sym_transitions(rule)) {
ParseItem new_item(placeholder, transition.second, 10, placeholder);
result.insert({
transition.first,
item_set_closure(new_item, grammar)
});
}
self->transitions_cache.insert({ rule, result });
return result;
}
static map<ISymbol, ParseItemSet> sym_transitions_for_item(SymTransitions *self, const ParseItem &item, const PreparedGrammar &grammar) {
auto result = sym_transitions_for_rule(self, item.rule, grammar);
for (auto &pair : result) {
ParseItemSet new_items;
for (auto &i : pair.second) {
ParseItem new_item(i);
if (new_item.consumed_symbol_count > 0)
new_item.consumed_symbol_count = item.consumed_symbol_count + 1;
if (new_item.lookahead_sym == placeholder)
new_item.lookahead_sym = item.lookahead_sym;
if (new_item.lhs == placeholder)
new_item.lhs = item.lhs;
new_items.insert(new_item);
}
pair.second = new_items;
}
return result;
}
map<ISymbol, ParseItemSet>
SymTransitions::operator()(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
map<ISymbol, ParseItemSet> result;
for (const ParseItem &item : item_set)
merge_sym_transitions<ParseItemSet>(result,
sym_transitions_for_item(this, item, grammar),
[&](const ParseItemSet &l, const ParseItemSet &r) {
return merge_sets(l, r);
});
return result;
}
map<CharacterSet, LexItemSet>
char_transitions(const LexItemSet &item_set, const PreparedGrammar &grammar) {
map<CharacterSet, LexItemSet> result;
@ -37,24 +85,5 @@ namespace tree_sitter {
}
return result;
}
map<ISymbol, ParseItemSet>
sym_transitions(const ParseItemSet &item_set, const PreparedGrammar &grammar) {
map<ISymbol, ParseItemSet> result;
for (const ParseItem &item : item_set) {
map<ISymbol, ParseItemSet> item_transitions;
for (auto &transition : sym_transitions(item.rule)) {
ParseItem new_item(item.lhs, transition.second, item.consumed_symbol_count + 1, item.lookahead_sym);
item_transitions.insert({
transition.first,
item_set_closure(new_item, grammar)
});
}
merge_sym_transitions<ParseItemSet>(result, item_transitions, [&](const ParseItemSet &l, const ParseItemSet &r) {
return merge_sets(l, r);
});
}
return result;
}
}
}

10
src/compiler/build_tables/item_set_transitions.h
View file

@ -13,11 +13,15 @@ namespace tree_sitter {
}
namespace build_tables {
class SymTransitions {
public:
std::map<rules::ISymbol, ParseItemSet>
operator()(const ParseItemSet &item_set, const PreparedGrammar &grammar);
std::map<rules::rule_ptr, std::map<rules::ISymbol, ParseItemSet>> transitions_cache;
};
std::map<rules::CharacterSet, LexItemSet>
char_transitions(const LexItemSet &item_set, const PreparedGrammar &grammar);
std::map<rules::ISymbol, ParseItemSet>
sym_transitions(const ParseItemSet &item_set, const PreparedGrammar &grammar);
}
}

4
src/compiler/build_tables/parse_item.h
View file

@ -18,8 +18,8 @@ namespace tree_sitter {
bool operator==(const ParseItem &other) const;
int precedence() const;
const size_t consumed_symbol_count;
const rules::ISymbol lookahead_sym;
size_t consumed_symbol_count;
rules::ISymbol lookahead_sym;
};
std::ostream& operator<<(std::ostream &stream, const ParseItem &item);