traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Introduce 'ubiquitous_tokens' concept, for parsing comments and such

Browse source
This commit is contained in:
Max Brunsfeld 2014-05-06 12:54:04 -07:00
parent b010e1667e
commit 4700e33746
24 changed files with 18321 additions and 19057 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/compiler/build_tables/build_parse_table.cc
View file

@ -28,7 +28,7 @@ namespace tree_sitter {
unordered_map<const ParseItemSet, ParseStateId> parse_state_ids;
SymTransitions sym_transitions;
ParseTable parse_table;
ParseStateId add_parse_state(const ParseItemSet &item_set) {
auto pair = parse_state_ids.find(item_set);
if (pair == parse_state_ids.end()) {
@ -46,18 +46,24 @@ namespace tree_sitter {
for (const auto &transition : sym_transitions(item_set, grammar)) {
const Symbol &symbol = transition.first;
const ParseItemSet &item_set = transition.second;
auto current_actions = parse_table.states[state_id].actions;
auto current_action = current_actions.find(symbol);
auto &actions = parse_table.states[state_id].actions;
auto current_action = actions.find(symbol);
set<int> precedence_values = precedence_values_for_item_set(item_set);
if (current_action == current_actions.end() ||
if (current_action == actions.end() ||
conflict_manager.resolve_parse_action(symbol, current_action->second, ParseAction::Shift(0, precedence_values))) {
ParseStateId new_state_id = add_parse_state(item_set);
parse_table.add_action(state_id, symbol, ParseAction::Shift(new_state_id, precedence_values));
}
}
for (const Symbol &symbol : grammar.options.ubiquitous_tokens) {
auto &actions = parse_table.states[state_id].actions;
if (actions.find(symbol) == actions.end())
parse_table.add_action(state_id, symbol, ParseAction::Shift(state_id, { 0 }));
}
}
void add_reduce_actions(const ParseItemSet &item_set, ParseStateId state_id) {
for (const ParseItem &item : item_set) {
if (item.is_done()) {
@ -66,7 +72,7 @@ namespace tree_sitter {
ParseAction::Reduce(item.lhs, item.consumed_symbol_count, item.precedence());
auto current_actions = parse_table.states[state_id].actions;
auto current_action = current_actions.find(item.lookahead_sym);
if (current_action == current_actions.end() ||
conflict_manager.resolve_parse_action(item.lookahead_sym, current_action->second, action)) {
parse_table.add_action(state_id, item.lookahead_sym, action);
@ -74,7 +80,7 @@ namespace tree_sitter {
}
}
}
set<int> precedence_values_for_item_set(const ParseItemSet &item_set) {
set<int> result;
for (const auto &item : item_set)