traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow ubiquitous tokens to also be used in grammar rules

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-26 08:52:42 -07:00
parent a9dff20658
commit 9686c57e90
22 changed files with 49452 additions and 47887 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/compiler/build_tables/build_parse_table.cc
View file

@ -62,7 +62,7 @@ namespace tree_sitter {
for (const Symbol &symbol : grammar.ubiquitous_tokens) {
auto &actions = parse_table.states[state_id].actions;
if (actions.find(symbol) == actions.end())
parse_table.add_action(state_id, symbol, ParseAction::Shift(state_id, { 0 }));
parse_table.add_action(state_id, symbol, ParseAction::ShiftExtra());
}
}

14
src/compiler/generate_code/c_code.cc
View file

@ -51,7 +51,6 @@ namespace tree_sitter {
state_and_symbol_counts();
symbol_enum();
symbol_names_list();
ubiquitous_symbols_list();
hidden_symbols_list();
lex_function();
lex_states_list();
@ -104,16 +103,6 @@ namespace tree_sitter {
line();
}
void ubiquitous_symbols_list() {
line("UBIQUITOUS_SYMBOLS = {");
indent([&]() {
for (auto &symbol : syntax_grammar.ubiquitous_tokens)
line("[" + symbol_id(symbol) + "] = 1,");
});
line("};");
line();
}
void hidden_symbols_list() {
line("HIDDEN_SYMBOLS = {");
indent([&]() {
@ -293,6 +282,9 @@ namespace tree_sitter {
case ParseActionTypeShift:
add("SHIFT(" + to_string(action.state_index) + ")");
break;
case ParseActionTypeShiftExtra:
add("SHIFT_EXTRA()");
break;
case ParseActionTypeReduce:
add("REDUCE(" +
symbol_id(action.symbol) + ", " +

6
src/compiler/parse_table.cc
View file

@ -38,6 +38,10 @@ namespace tree_sitter {
return ParseAction(ParseActionTypeShift, state_index, Symbol(-1), 0, precedence_values);
}
ParseAction ParseAction::ShiftExtra() {
return ParseAction(ParseActionTypeShiftExtra, -1, Symbol(-1), 0, set<int>({}));
}
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count, int precedence) {
return ParseAction(ParseActionTypeReduce, -1, symbol, consumed_symbol_count, { precedence });
}
@ -58,6 +62,8 @@ namespace tree_sitter {
return stream << string("#<accept>");
case ParseActionTypeShift:
return stream << (string("#<shift ") + to_string(action.state_index) + ">");
case ParseActionTypeShiftExtra:
return stream << string("#<shift_extra");
case ParseActionTypeReduce:
return stream << (string("#<reduce sym") + to_string(action.symbol.index) + " " + to_string(action.consumed_symbol_count) + ">");
default:

2
src/compiler/parse_table.h
View file

@ -12,6 +12,7 @@ namespace tree_sitter {
typedef enum {
ParseActionTypeError,
ParseActionTypeShift,
ParseActionTypeShiftExtra,
ParseActionTypeReduce,
ParseActionTypeAccept,
} ParseActionType;
@ -27,6 +28,7 @@ namespace tree_sitter {
static ParseAction Accept();
static ParseAction Error();
static ParseAction Shift(size_t state_index, std::set<int> precedence_values);
static ParseAction ShiftExtra();
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count, int precedence);
bool operator==(const ParseAction &action) const;