traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make separate Item classes for parsing and lexing

Browse source
This commit is contained in:
Max Brunsfeld 2014-01-18 09:47:26 -08:00
parent d015d57a53
commit 289992344e
10 changed files with 132 additions and 117 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/compiler/build_tables/perform.cpp
View file

@ -17,50 +17,50 @@ namespace tree_sitter {
class TableBuilder {
const Grammar grammar;
const Grammar lex_grammar;
unordered_map<const ItemSet, size_t> parse_state_indices;
unordered_map<const ItemSet, size_t> lex_state_indices;
unordered_map<const ParseItemSet, size_t> parse_state_indices;
unordered_map<const LexItemSet, size_t> lex_state_indices;
ParseTable parse_table;
LexTable lex_table;
long parse_state_index_for_item_set(const ItemSet &item_set) const {
long parse_state_index_for_item_set(const ParseItemSet &item_set) const {
auto entry = parse_state_indices.find(item_set);
return (entry == parse_state_indices.end()) ? NOT_FOUND : entry->second;
}
long lex_state_index_for_item_set(const ItemSet &item_set) const {
long lex_state_index_for_item_set(const LexItemSet &item_set) const {
auto entry = lex_state_indices.find(item_set);
return (entry == lex_state_indices.end()) ? NOT_FOUND : entry->second;
}
void add_shift_actions(const ItemSet &item_set, size_t state_index) {
void add_shift_actions(const ParseItemSet &item_set, size_t state_index) {
auto x = sym_transitions(item_set, grammar);
for (auto transition : x) {
rules::Symbol symbol = *transition.first;
ItemSet item_set = *transition.second;
ParseItemSet item_set = *transition.second;
size_t new_state_index = add_parse_state(item_set);
parse_table.add_action(state_index, symbol.name, ParseAction::Shift(new_state_index));
}
}
void add_advance_actions(const ItemSet &item_set, size_t state_index) {
void add_advance_actions(const LexItemSet &item_set, size_t state_index) {
for (auto transition : char_transitions(item_set, grammar)) {
rules::Character rule = *transition.first;
ItemSet item_set = *transition.second;
LexItemSet item_set = *transition.second;
size_t new_state_index = add_lex_state(item_set);
lex_table.add_action(state_index, rule.value, LexAction::Advance(new_state_index));
}
}
void add_accept_token_actions(const ItemSet &item_set, size_t state_index) {
for (Item item : item_set) {
void add_accept_token_actions(const LexItemSet &item_set, size_t state_index) {
for (LexItem item : item_set) {
if (item.is_done()) {
lex_table.add_default_action(state_index, LexAction::Accept(item.rule_name));
}
}
}
void add_reduce_actions(const ItemSet &item_set, size_t state_index) {
for (Item item : item_set) {
void add_reduce_actions(const ParseItemSet &item_set, size_t state_index) {
for (ParseItem item : item_set) {
if (item.is_done()) {
if (item.rule_name == ParseTable::START) {
parse_table.add_action(state_index, ParseTable::END_OF_INPUT, ParseAction::Accept());
@ -71,7 +71,7 @@ namespace tree_sitter {
}
}
size_t add_lex_state(const ItemSet &item_set) {
size_t add_lex_state(const LexItemSet &item_set) {
auto state_index = lex_state_index_for_item_set(item_set);
if (state_index == NOT_FOUND) {
state_index = lex_table.add_state();
@ -82,20 +82,20 @@ namespace tree_sitter {
return state_index;
}
ItemSet lex_item_set_for_parse_item_set(const ItemSet &parse_item_set) {
ItemSet result;
LexItemSet lex_item_set_for_parse_item_set(const ParseItemSet &parse_item_set) {
LexItemSet result;
for (rules::Symbol symbol : next_terminals(parse_item_set, grammar))
result.insert(Item::at_beginning_of_token(symbol.name, lex_grammar));
result.insert(LexItem(symbol.name, lex_grammar.rule(symbol.name)));
return result;
}
size_t add_parse_state(const ItemSet &item_set) {
size_t add_parse_state(const ParseItemSet &item_set) {
auto state_index = parse_state_index_for_item_set(item_set);
if (state_index == NOT_FOUND) {
state_index = parse_table.add_state();
parse_state_indices[item_set] = state_index;
ItemSet lex_item_set = lex_item_set_for_parse_item_set(item_set);
LexItemSet lex_item_set = lex_item_set_for_parse_item_set(item_set);
parse_table.states[state_index].lex_state_index = add_lex_state(lex_item_set);
add_shift_actions(item_set, state_index);
add_reduce_actions(item_set, state_index);
@ -110,8 +110,8 @@ namespace tree_sitter {
lex_grammar(lex_grammar) {};
pair<ParseTable, LexTable> build() {
auto item = Item(ParseTable::START, rules::sym(grammar.start_rule_name), 0);
auto item_set = item_set_closure(ItemSet({ item }), grammar);
auto item = ParseItem(ParseTable::START, rules::sym(grammar.start_rule_name), 0);
ParseItemSet item_set = item_set_closure(ParseItemSet({ item }), grammar);
add_parse_state(item_set);
return pair<ParseTable, LexTable>(parse_table, lex_table);
}