traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle extra tokens properly during error recovery

Browse source
This commit is contained in:
Max Brunsfeld 2016-06-18 20:35:33 -07:00
parent 773e50f26b
commit 45f7cee0c8
3 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

3
include/tree_sitter/parser.h
View file

@ -175,6 +175,9 @@ enum {
.type = TSParseActionTypeRecover, .data = {.to_state = to_state_value } \
}
#define RECOVER_EXTRA() \
{ .type = TSParseActionTypeShift, .extra = true, }
#define SHIFT_EXTRA() \
{ \
{ .type = TSParseActionTypeShift, .extra = true } \

5
src/compiler/build_tables/build_parse_table.cc
View file

@ -109,6 +109,11 @@ class ParseTableBuilder {
add_out_of_context_parse_state(symbol);
}
for (const Symbol &symbol : grammar.extra_tokens) {
parse_table.error_state.actions[symbol].push_back(
ParseAction::ShiftExtra());
}
for (size_t i = 0; i < grammar.variables.size(); i++) {
Symbol symbol(i, false);
add_out_of_context_parse_state(symbol);

10
src/compiler/generate_code/c_code.cc
View file

@ -224,9 +224,13 @@ class CCodeGenerator {
for (const auto &entry : parse_table.error_state.actions) {
const rules::Symbol &symbol = entry.first;
if (!entry.second.empty()) {
ParseStateId state = entry.second[0].state_index;
line("[" + symbol_id(symbol) + "] = RECOVER(" + to_string(state) +
"),");
line("[" + symbol_id(symbol) + "] = ");
ParseAction action = entry.second[0];
if (action.extra) {
add("RECOVER_EXTRA(),");
} else {
add("RECOVER(" + to_string(action.state_index) + "),");
}
}
}
});