traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Discard tokens after error detection to find the best repair

Browse source 
* Use GLR stack-splitting to try all numbers of tokens to
  discard until a repair is found.
* Check the validity of repairs by looking at the child trees,
  rather than the statically-computed 'in-progress symbols' list
This commit is contained in:
Max Brunsfeld 2016-05-09 14:31:44 -07:00
parent 9d247e45b2
commit 22c550c9d6
21 changed files with 454 additions and 506 deletions
Download patch file Download diff file Expand all files Collapse all files

49
src/compiler/generate_code/c_code.cc
View file

@ -102,7 +102,6 @@ class CCodeGenerator {
add_lex_states_list();
add_recovery_parse_states_list();
add_parse_table();
add_in_progress_symbol_table();
add_parser_export();
return buffer;
@ -221,13 +220,14 @@ class CCodeGenerator {
}
void add_recovery_parse_states_list() {
line("static TSStateId ts_recovery_states[SYMBOL_COUNT] = {");
line("static TSParseAction ts_recovery_actions[SYMBOL_COUNT] = {");
indent([&]() {
for (const auto &entry : parse_table.error_state.actions) {
const rules::Symbol &symbol = entry.first;
if (!entry.second.empty()) {
ParseStateId state = entry.second[0].state_index;
line("[" + symbol_id(symbol) + "] = " + to_string(state) + ",");
line("[" + symbol_id(symbol) + "] = RECOVER(" + to_string(state) +
"),");
}
}
});
@ -266,29 +266,6 @@ class CCodeGenerator {
line();
}
void add_in_progress_symbol_table() {
add_in_progress_symbol_list_id({});
line("static unsigned short ts_in_progress_symbol_table[STATE_COUNT] = {");
indent([&]() {
size_t state_id = 0;
for (const ParseState &state : parse_table.states) {
if (!state.in_progress_symbols.empty()) {
line("[" + to_string(state_id) + "] = ");
add(to_string(
add_in_progress_symbol_list_id(state.in_progress_symbols)));
add(",");
}
state_id++;
}
});
line("};");
line();
add_in_progress_symbols_list();
line();
}
void add_parser_export() {
line("EXPORT_LANGUAGE(ts_language_" + name + ");");
line();
@ -408,26 +385,6 @@ class CCodeGenerator {
line("};");
}
void add_in_progress_symbols_list() {
line("static TSInProgressSymbolEntry ts_in_progress_symbols[] = {");
indent([&]() {
for (const auto &pair : in_progress_symbols) {
size_t index = pair.first;
line("[" + to_string(index) + "] = {.count = " +
to_string(pair.second.size()) + "},");
for (const rules::Symbol &symbol : pair.second) {
add(" ");
add("{" + symbol_id(symbol) + "}");
add(",");
}
}
});
line("};");
}
size_t add_parse_action_list_id(const vector<ParseAction> &actions) {
for (const auto &pair : parse_actions) {
if (pair.second == actions) {