traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Discard tokens after error detection to find the best repair

Browse source 
* Use GLR stack-splitting to try all numbers of tokens to
  discard until a repair is found.
* Check the validity of repairs by looking at the child trees,
  rather than the statically-computed 'in-progress symbols' list
This commit is contained in:
Max Brunsfeld 2016-05-09 14:31:44 -07:00
parent 9d247e45b2
commit 22c550c9d6
21 changed files with 454 additions and 506 deletions
Download patch file Download diff file Expand all files Collapse all files

48
src/runtime/language.c
View file

@ -1,32 +1,33 @@
#include "tree_sitter/parser.h"
#include "runtime/language.h"
#include "runtime/tree.h"
const TSParseAction *ts_language_actions(const TSLanguage *language,
TSStateId state, TSSymbol symbol,
size_t *count) {
size_t action_index = 0;
if (symbol != ts_builtin_sym_error) {
if (state == ts_parse_state_error)
state = language->recovery_states[symbol];
action_index =
(language->parse_table + (state * language->symbol_count))[symbol];
const TSParseAction *ts_language_actions(const TSLanguage *self, TSStateId state,
TSSymbol symbol, size_t *count) {
if (state == ts_parse_state_error) {
*count = 1;
return &self->recovery_actions[symbol];
}
*count = language->parse_actions[action_index].count;
const TSParseActionEntry *entry = language->parse_actions + action_index + 1;
size_t action_index = 0;
if (symbol != ts_builtin_sym_error)
action_index = self->parse_table[state * self->symbol_count + symbol];
*count = self->parse_actions[action_index].count;
const TSParseActionEntry *entry = self->parse_actions + action_index + 1;
return (const TSParseAction *)entry;
}
TSParseAction ts_language_last_action(const TSLanguage *language,
TSStateId state, TSSymbol sym) {
TSParseAction ts_language_last_action(const TSLanguage *self, TSStateId state,
TSSymbol sym) {
size_t count;
const TSParseAction *actions =
ts_language_actions(language, state, sym, &count);
const TSParseAction *actions = ts_language_actions(self, state, sym, &count);
return actions[count - 1];
}
bool ts_language_has_action(const TSLanguage *language, TSStateId state,
bool ts_language_has_action(const TSLanguage *self, TSStateId state,
TSSymbol symbol) {
TSParseAction action = ts_language_last_action(language, state, symbol);
TSParseAction action = ts_language_last_action(self, state, symbol);
return action.type != TSParseActionTypeError;
}
@ -50,16 +51,3 @@ const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol)
else
return language->symbol_names[symbol];
}
bool ts_language_symbol_is_in_progress(const TSLanguage *self, TSStateId state,
TSSymbol symbol) {
if (state == ts_parse_state_error)
return false;
unsigned index = self->in_progress_symbol_table[state];
unsigned short count = self->in_progress_symbols[index].count;
const TSInProgressSymbolEntry *entries = self->in_progress_symbols + index + 1;
for (size_t i = 0; i < count; i++)
if (entries[i].symbol == symbol)
return true;
return false;
}

9
src/runtime/language.h
View file

@ -6,6 +6,7 @@ extern "C" {
#endif
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
bool ts_language_symbol_is_in_progress(const TSLanguage *, TSStateId, TSSymbol);
@ -15,8 +16,12 @@ TSParseAction ts_language_last_action(const TSLanguage *, TSStateId, TSSymbol);
bool ts_language_has_action(const TSLanguage *, TSStateId, TSSymbol);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *language,
TSSymbol symbol);
TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *, TSSymbol);
static inline TSStateId ts_language_lex_state(const TSLanguage *self,
TSStateId state) {
return state == ts_parse_state_error ? 0 : self->lex_states[state];
}
#ifdef __cplusplus
}

392
src/runtime/parser.c
View file

@ -10,6 +10,7 @@
#include "runtime/array.h"
#include "runtime/language.h"
#include "runtime/alloc.h"
#include "runtime/reduce_action.h"
#define LOG(...) \
if (self->lexer.debugger.debug_fn) { \
@ -51,19 +52,12 @@ typedef struct {
size_t char_index;
} ReusableNode;
struct ErrorRepair {
TSSymbol symbol;
size_t in_progress_state_count;
size_t count_below_error;
};
typedef struct {
TSParser *parser;
TSSymbol lookahead_symbol;
ErrorRepairArray *repairs;
TreeArray *trees_above_error;
bool found_repair;
ErrorRepair best_repair;
TSStateId best_repair_state;
ReduceAction best_repair;
TSStateId best_repair_next_state;
size_t best_repair_skip_count;
} ErrorRepairSession;
@ -198,7 +192,7 @@ static bool ts_parser__can_reuse(TSParser *self, StackVersion version,
TSStateId top_state = ts_stack_top_state(self->stack, version);
if (tree->lex_state != TS_TREE_STATE_INDEPENDENT &&
tree->lex_state != self->language->lex_states[top_state])
tree->lex_state != ts_language_lex_state(self->language, top_state))
return false;
const TSParseAction action =
@ -253,9 +247,10 @@ static TSTree *ts_parser__get_lookahead(TSParser *self, StackVersion version,
ts_lexer_reset(&self->lexer, position);
TSStateId parse_state = ts_stack_top_state(self->stack, version);
TSStateId lex_state = self->language->lex_states[parse_state];
bool error_mode = parse_state == ts_parse_state_error;
TSStateId lex_state = error_mode ? 0 : self->language->lex_states[parse_state];
LOG("lex state:%d", lex_state);
return self->language->lex_fn(&self->lexer, lex_state, false);
return self->language->lex_fn(&self->lexer, lex_state, error_mode);
}
static bool ts_parser__select_tree(TSParser *self, TSTree *left, TSTree *right) {
@ -273,12 +268,23 @@ static bool ts_parser__select_tree(TSParser *self, TSTree *left, TSTree *right)
SYM_NAME(left->symbol), SYM_NAME(right->symbol));
return false;
}
return ts_tree_compare(right, left) < 0;
}
static void ts_parser__remove_version(TSParser *self, StackVersion version) {
LOG_ACTION("bail version:%d", version);
ts_stack_remove_version(self->stack, version);
int comparison = ts_tree_compare(left, right);
switch (comparison) {
case -1:
LOG_ACTION("select_earlier symbol:%s, over_symbol:%s",
SYM_NAME(left->symbol), SYM_NAME(right->symbol));
return false;
break;
case 1:
LOG_ACTION("select_earlier symbol:%s, over_symbol:%s",
SYM_NAME(right->symbol), SYM_NAME(left->symbol));
return true;
default:
LOG_ACTION("select_existing symbol:%s, over_symbol:%s",
SYM_NAME(left->symbol), SYM_NAME(right->symbol));
return false;
}
}
static bool ts_parser__push(TSParser *self, StackVersion version, TSTree *tree,
@ -291,7 +297,6 @@ static bool ts_parser__push(TSParser *self, StackVersion version, TSTree *tree,
static bool ts_parser__shift(TSParser *self, StackVersion version,
TSStateId state, TSTree *lookahead, bool extra) {
if (extra) {
LOG_ACTION("shift_extra");
TSSymbolMetadata metadata =
ts_language_symbol_metadata(self->language, lookahead->symbol);
if (metadata.structural && ts_stack_version_count(self->stack) > 1) {
@ -301,7 +306,6 @@ static bool ts_parser__shift(TSParser *self, StackVersion version,
}
lookahead->extra = true;
} else {
LOG_ACTION("shift state:%u", state);
ts_tree_retain(lookahead);
}
@ -337,13 +341,6 @@ static bool ts_parser__switch_children(TSParser *self, TSTree *tree,
static Reduction ts_parser__reduce(TSParser *self, StackVersion version,
TSSymbol symbol, unsigned count, bool extra,
bool fragile) {
if (extra) {
LOG_ACTION("reduce_extra");
} else {
LOG_ACTION("reduce sym:%s, child_count:%u, fragile:%s", SYM_NAME(symbol),
count, BOOL_STRING(fragile));
}
size_t initial_version_count = ts_stack_version_count(self->stack);
StackPopResult pop = ts_stack_pop_count(self->stack, version, count);
switch (pop.status) {
@ -406,7 +403,8 @@ static Reduction ts_parser__reduce(TSParser *self, StackVersion version,
new_state = state;
} else {
TSParseAction action = ts_language_last_action(language, state, symbol);
assert(action.type == TSParseActionTypeShift);
assert(action.type == TSParseActionTypeShift ||
action.type == TSParseActionTypeRecover);
new_state = action.data.to_state;
}
@ -425,57 +423,96 @@ error:
return (Reduction){ ReduceFailed };
}
static StackIterateAction ts_parser__error_repair_callback(void *payload,
TSStateId state,
size_t tree_count,
bool is_done,
bool is_pending) {
ErrorRepairSession *session = (ErrorRepairSession *)payload;
const TSParser *self = session->parser;
static bool ts_parser__is_valid_repair(
const TSParser *self, const TreeArray *trees_below,
const TreeArray *trees_above, TSStateId start_state, TSSymbol goal_symbol,
size_t goal_count_below, TSSymbol lookahead_symbol) {
const TSLanguage *language = self->language;
TSSymbol lookahead_symbol = session->lookahead_symbol;
StackIterateAction result = StackIterateNone;
TSStateId state = start_state;
size_t count_below = 0;
for (size_t i = 0; i < session->repairs->size; i++) {
ErrorRepair *repair = &session->repairs->contents[i];
TSSymbol symbol = repair->symbol;
for (size_t i = trees_below->size - 1; i + 1 > 0; i--) {
TSTree *tree = trees_below->contents[i];
TSParseAction action =
ts_language_last_action(language, state, tree->symbol);
if (action.type != TSParseActionTypeShift)
return false;
if (action.extra || tree->extra)
continue;
if (tree_count >= repair->count_below_error) {
size_t skip_count = tree_count - repair->count_below_error;
state = action.data.to_state;
count_below++;
if (session->found_repair && skip_count > session->best_repair_skip_count) {
array_erase(session->repairs, i--);
continue;
if (count_below == goal_count_below) {
for (size_t j = 0; j < trees_above->size; j++) {
TSTree *tree = trees_above->contents[j];
TSParseAction action =
ts_language_last_action(language, state, tree->symbol);
if (action.type != TSParseActionTypeShift)
return false;
if (action.extra || tree->extra)
continue;
state = action.data.to_state;
}
if (repair->in_progress_state_count > 0) {
TSParseAction action = ts_language_last_action(language, state, symbol);
if (action.type == TSParseActionTypeShift) {
TSStateId next_state = action.data.to_state;
if (ts_language_has_action(language, next_state, lookahead_symbol) &&
(!session->found_repair ||
repair->in_progress_state_count >
session->best_repair.in_progress_state_count)) {
result |= StackIteratePop;
session->found_repair = true;
session->best_repair = *repair;
session->best_repair_state = state;
session->best_repair_skip_count = skip_count;
session->best_repair_next_state = next_state;
array_erase(session->repairs, i--);
continue;
}
}
}
size_t action_count = 0;
const TSParseAction *actions =
ts_language_actions(language, state, lookahead_symbol, &action_count);
for (size_t k = 0; k < action_count; k++)
if (actions[k].type == TSParseActionTypeReduce &&
actions[k].data.symbol == goal_symbol)
return true;
}
if (ts_language_symbol_is_in_progress(self->language, state, symbol))
repair->in_progress_state_count++;
else
repair->in_progress_state_count = 0;
}
if (session->repairs->size == 0)
return false;
}
static StackIterateAction ts_parser__error_repair_callback(
void *payload, TSStateId state, TreeArray *trees, size_t tree_count,
bool is_done, bool is_pending) {
ErrorRepairSession *session = (ErrorRepairSession *)payload;
TSParser *self = session->parser;
const TSLanguage *language = self->language;
TSSymbol lookahead_symbol = session->lookahead_symbol;
ReduceActionSet *repairs = &self->reduce_actions;
TreeArray *trees_above_error = session->trees_above_error;
StackIterateAction result = StackIterateNone;
for (size_t i = 0; i < repairs->size; i++) {
ReduceAction *repair = &repairs->contents[i];
if (repair->count > tree_count)
continue;
size_t skip_count = tree_count - repair->count;
if (session->found_repair && skip_count >= session->best_repair_skip_count) {
array_erase(repairs, i--);
continue;
}
TSParseAction repair_symbol_action =
ts_language_last_action(language, state, repair->symbol);
if (repair_symbol_action.type != TSParseActionTypeShift)
continue;
TSStateId state_after_repair = repair_symbol_action.data.to_state;
if (!ts_language_has_action(language, state_after_repair, lookahead_symbol))
continue;
if (ts_parser__is_valid_repair(self, trees, trees_above_error, state,
repair->symbol, repair->count,
lookahead_symbol)) {
result |= StackIteratePop;
session->found_repair = true;
session->best_repair = *repair;
session->best_repair_skip_count = skip_count;
session->best_repair_next_state = state_after_repair;
array_erase(repairs, i--);
}
}
if (repairs->size == 0)
result |= StackIterateStop;
return result;
@ -486,26 +523,22 @@ static RepairResult ts_parser__repair_error(TSParser *self, StackSlice slice,
const TSParseAction *actions,
size_t action_count) {
size_t count_above_error = ts_tree_array_essential_count(&slice.trees);
LOG_ACTION("repair count_above_error:%lu", count_above_error);
ErrorRepairSession session = {
.parser = self,
.lookahead_symbol = lookahead->symbol,
.repairs = &self->error_repairs,
.found_repair = false,
.trees_above_error = &slice.trees,
};
array_clear(&self->error_repairs);
array_clear(&self->reduce_actions);
for (size_t i = 0; i < action_count; i++)
if (actions[i].type == TSParseActionTypeReduce &&
actions[i].data.child_count > count_above_error)
CHECK(array_push(
&self->error_repairs,
((ErrorRepair){
.symbol = actions[i].data.symbol,
.count_below_error = actions[i].data.child_count - count_above_error,
.in_progress_state_count = 0,
})));
CHECK(array_push(&self->reduce_actions,
((ReduceAction){
.symbol = actions[i].data.symbol,
.count = actions[i].data.child_count - count_above_error,
})));
StackPopResult pop = ts_stack_iterate(
self->stack, slice.version, ts_parser__error_repair_callback, &session);
@ -518,7 +551,7 @@ static RepairResult ts_parser__repair_error(TSParser *self, StackSlice slice,
return RepairNoneFound;
}
ErrorRepair repair = session.best_repair;
ReduceAction repair = session.best_repair;
TSStateId next_state = session.best_repair_next_state;
size_t skip_count = session.best_repair_skip_count;
TSSymbol symbol = repair.symbol;
@ -534,20 +567,18 @@ static RepairResult ts_parser__repair_error(TSParser *self, StackSlice slice,
ts_stack_remove_version(self->stack, other_slice.version);
}
LOG_ACTION(
"repair_found sym:%s, child_count:%lu, match_count:%lu, skipped:%lu",
SYM_NAME(symbol), repair.count_below_error + count_above_error,
repair.in_progress_state_count, skip_count);
LOG_ACTION("repair_found sym:%s, child_count:%lu, skipped:%lu",
SYM_NAME(symbol), repair.count + count_above_error, skip_count);
if (skip_count > 0) {
TreeArray skipped_children = array_new();
CHECK(array_grow(&skipped_children, skip_count));
for (size_t i = repair.count_below_error; i < children_below.size; i++)
for (size_t i = repair.count; i < children_below.size; i++)
array_push(&skipped_children, children_below.contents[i]);
TSTree *error = ts_tree_make_error_node(&skipped_children);
CHECK(error);
children_below.size = repair.count_below_error;
children_below.size = repair.count;
array_push(&children_below, error);
}
@ -581,7 +612,6 @@ static void ts_parser__start(TSParser *self, TSInput input,
}
static bool ts_parser__accept(TSParser *self, StackVersion version) {
LOG_ACTION("accept");
StackPopResult pop = ts_stack_pop_all(self->stack, version);
CHECK(pop.status);
CHECK(pop.slices.size);
@ -624,67 +654,91 @@ error:
return false;
}
static ParseActionResult ts_parser__handle_error(TSParser *self,
StackVersion version,
TSTree *invalid_tree) {
const TSLanguage *language = self->language;
TreeArray invalid_trees = array_new();
TSTree *next_token = self->language->lex_fn(&self->lexer, 0, true);
ts_tree_retain(invalid_tree);
CHECK(array_push(&invalid_trees, invalid_tree));
LOG_ACTION("handle_error %s", SYM_NAME(invalid_tree->symbol));
for (;;) {
if (next_token->symbol == ts_builtin_sym_end) {
LOG_ACTION("fail_to_recover");
ts_tree_release(next_token);
TSTree *error = ts_tree_make_error_node(&invalid_trees);
CHECK(error);
CHECK(ts_parser__push(self, version, error, 0));
TSTree *parent = ts_tree_make_leaf(
ts_builtin_sym_start, ts_length_zero(), ts_length_zero(),
ts_language_symbol_metadata(language, ts_builtin_sym_start));
CHECK(parent);
CHECK(ts_parser__push(self, version, parent, 0));
CHECK(ts_parser__accept(self, version));
return ParseActionRemoved;
static bool ts_parser__handle_error(TSParser *self, StackVersion version,
TSStateId state, TSTree *lookahead) {
bool has_shift_action = false;
array_clear(&self->reduce_actions);
for (TSSymbol symbol = 0; symbol < self->language->symbol_count; symbol++) {
size_t action_count;
const TSParseAction *actions =
ts_language_actions(self->language, state, symbol, &action_count);
for (size_t i = 0; i < action_count; i++) {
TSParseAction action = actions[i];
if (action.extra)
continue;
if (action.type == TSParseActionTypeShift ||
action.type == TSParseActionTypeRecover)
has_shift_action = true;
if (action.type == TSParseActionTypeReduce)
CHECK(ts_reduce_action_set_add(
&self->reduce_actions,
(ReduceAction){
.symbol = action.data.symbol, .count = action.data.child_count,
}));
}
TSLength position = self->lexer.current_position;
TSTree *following_token = language->lex_fn(&self->lexer, 0, true);
CHECK(following_token);
if (!ts_language_symbol_metadata(language, next_token->symbol).extra) {
TSParseAction action = ts_language_last_action(
language, ts_parse_state_error, next_token->symbol);
assert(action.type == TSParseActionTypeShift);
TSStateId next_state = action.data.to_state;
if (ts_language_has_action(language, next_state, following_token->symbol) &&
!ts_language_symbol_metadata(language, following_token->symbol).extra) {
LOG_ACTION("resume_without_context state:%d", next_state);
ts_tree_release(following_token);
ts_lexer_reset(&self->lexer, position);
ts_tree_steal_padding(*array_back(&invalid_trees), next_token);
TSTree *error = ts_tree_make_error_node(&invalid_trees);
CHECK(error);
CHECK(ts_parser__push(self, version, error, ts_parse_state_error));
CHECK(ts_parser__push(self, version, next_token, next_state));
return ParseActionUpdated;
}
}
CHECK(array_push(&invalid_trees, next_token));
next_token = following_token;
}
Reduction reduction;
for (size_t i = 0; i < self->reduce_actions.size; i++) {
ReduceAction repair = self->reduce_actions.contents[i];
reduction = ts_parser__reduce(self, version, repair.symbol, repair.count,
false, true);
CHECK(reduction.status != ReduceFailed);
assert(reduction.status == ReduceSucceeded);
CHECK(ts_parser__shift(self, reduction.slice.version, ts_parse_state_error,
lookahead, false));
}
if (has_shift_action) {
CHECK(
ts_parser__shift(self, version, ts_parse_state_error, lookahead, false));
} else {
ts_stack_renumber_version(self->stack, reduction.slice.version, version);
}
return true;
error:
return false;
}
static bool ts_parser__recover(TSParser *self, StackVersion version,
TSStateId state, TSTree *lookahead) {
size_t error_length = ts_stack_error_length(self->stack, version);
bool has_repaired = false;
for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++)
if (i != version && ts_stack_error_length(self->stack, i) == 0 &&
ts_stack_last_repaired_error_size(self->stack, i) <= error_length) {
has_repaired = true;
break;
}
if (has_repaired) {
LOG_ACTION("final_recover state:%u, error_length:%lu ", state, error_length);
} else {
StackVersion new_version = ts_stack_duplicate_version(self->stack, version);
CHECK(new_version != STACK_VERSION_NONE);
CHECK(ts_parser__shift(
self, new_version, ts_parse_state_error, lookahead,
ts_language_symbol_metadata(self->language, lookahead->symbol).extra));
LOG_ACTION("recover_and_discard state:%u, error_length:%lu", state,
error_length);
}
CHECK(ts_parser__shift(self, version, state, lookahead, false));
return true;
error:
return false;
}
static bool ts_parser__recover_eof(TSParser *self, StackVersion version) {
TreeArray children = array_new();
TSTree *parent = ts_tree_make_error_node(&children);
return ts_parser__push(self, version, parent, 1);
}
static ParseActionResult ts_parser__consume_lookahead(TSParser *self,
StackVersion version,
TSTree *lookahead) {
@ -727,21 +781,40 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self,
}
if (ts_stack_version_count(self->stack) == 1 && !self->finished_tree) {
return ts_parser__handle_error(self, version, lookahead);
LOG_ACTION("handle_error");
CHECK(ts_parser__handle_error(self, version, state, lookahead));
return ParseActionUpdated;
} else {
ts_parser__remove_version(self, version);
LOG_ACTION("bail version:%d", version);
ts_stack_remove_version(self->stack, version);
return ParseActionRemoved;
}
}
case TSParseActionTypeShift: {
CHECK(ts_parser__shift(self, version,
action.extra ? state : action.data.to_state,
lookahead, action.extra));
TSStateId next_state;
if (action.extra) {
next_state = state;
LOG_ACTION("shift_extra");
} else {
next_state = action.data.to_state;
LOG_ACTION("shift state:%u", next_state);
}
CHECK(ts_parser__shift(self, version, next_state, lookahead,
action.extra));
return ParseActionUpdated;
}
case TSParseActionTypeReduce: {
if (action.extra) {
LOG_ACTION("reduce_extra");
} else {
LOG_ACTION("reduce sym:%s, child_count:%u, fragile:%s",
SYM_NAME(action.data.symbol), action.data.child_count,
BOOL_STRING(action.fragile));
}
Reduction reduction = ts_parser__reduce(
self, version, action.data.symbol, action.data.child_count,
action.extra, action.fragile);
@ -756,6 +829,7 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self,
error_repair_depth =
ts_tree_array_essential_count(&reduction.slice.trees);
LOG_ACTION("repair count_above_error:%lu", error_repair_depth);
switch (ts_parser__repair_error(self, reduction.slice, lookahead,
actions, action_count)) {
case RepairFailed:
@ -774,9 +848,21 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self,
}
case TSParseActionTypeAccept: {
LOG_ACTION("accept");
CHECK(ts_parser__accept(self, version));
return ParseActionRemoved;
}
case TSParseActionTypeRecover: {
if (lookahead->symbol == ts_builtin_sym_end) {
LOG_ACTION("recover_eof");
CHECK(ts_parser__recover_eof(self, version));
} else {
CHECK(ts_parser__recover(self, version, action.data.to_state,
lookahead));
}
return ParseActionUpdated;
}
}
}
@ -792,13 +878,13 @@ bool ts_parser_init(TSParser *self) {
ts_lexer_init(&self->lexer);
self->finished_tree = NULL;
self->stack = NULL;
array_init(&self->error_repairs);
array_init(&self->reduce_actions);
self->stack = ts_stack_new();
if (!self->stack)
goto error;
if (!array_grow(&self->error_repairs, 4))
if (!array_grow(&self->reduce_actions, 4))
goto error;
return true;
@ -808,16 +894,16 @@ error:
ts_stack_delete(self->stack);
self->stack = NULL;
}
if (self->error_repairs.contents)
array_delete(&self->error_repairs);
if (self->reduce_actions.contents)
array_delete(&self->reduce_actions);
return false;
}
void ts_parser_destroy(TSParser *self) {
if (self->stack)
ts_stack_delete(self->stack);
if (self->error_repairs.contents)
array_delete(&self->error_repairs);
if (self->reduce_actions.contents)
array_delete(&self->reduce_actions);
}
TSDebugger ts_parser_debugger(const TSParser *self) {

6
src/runtime/parser.h
View file

@ -7,15 +7,13 @@ extern "C" {
#include "runtime/stack.h"
#include "runtime/array.h"
typedef struct ErrorRepair ErrorRepair;
typedef Array(ErrorRepair) ErrorRepairArray;
#include "runtime/reduce_action.h"
typedef struct {
TSLexer lexer;
Stack *stack;
const TSLanguage *language;
ErrorRepairArray error_repairs;
ReduceActionSet reduce_actions;
TSTree *finished_tree;
bool is_split;
bool print_debugging_graphs;

32
src/runtime/reduce_action.h Normal file
View file

@ -0,0 +1,32 @@
#ifndef RUNTIME_REDUCE_ACTION_H_
#define RUNTIME_REDUCE_ACTION_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "runtime/array.h"
#include "tree_sitter/runtime.h"
typedef struct {
TSSymbol symbol;
size_t count;
} ReduceAction;
typedef Array(ReduceAction) ReduceActionSet;
static inline bool ts_reduce_action_set_add(ReduceActionSet *self,
ReduceAction new_action) {
for (size_t i = 0; i < self->size; i++) {
ReduceAction action = self->contents[i];
if (action.symbol == new_action.symbol && action.count == new_action.count)
return true;
}
return array_push(self, new_action);
}
#ifdef __cplusplus
}
#endif
#endif // RUNTIME_REDUCE_ACTION_H_

96
src/runtime/stack.c
View file

@ -26,6 +26,7 @@ struct StackNode {
StackLink links[MAX_LINK_COUNT];
short unsigned int link_count;
short unsigned int ref_count;
size_t error_length;
};
typedef struct {
@ -38,6 +39,7 @@ typedef struct {
typedef struct {
size_t goal_tree_count;
bool found_error;
bool found_valid_path;
} StackPopSession;
typedef Array(StackNode *) StackNodeArray;
@ -90,6 +92,7 @@ static StackNode *stack_node_new(StackNode *next, TSTree *tree, bool is_pending,
.links = {},
.state = state,
.position = position,
.error_length = (state == ts_parse_state_error) ? 1 : 0,
};
if (next) {
@ -97,6 +100,7 @@ static StackNode *stack_node_new(StackNode *next, TSTree *tree, bool is_pending,
stack_node_retain(next);
node->link_count = 1;
node->links[0] = (StackLink){ next, tree, is_pending };
node->error_length += next->error_length;
}
return node;
@ -168,8 +172,9 @@ INLINE StackPopResult stack__iter(Stack *self, StackVersion version,
StackNode *node = path->node;
bool is_done = node == self->base_node;
StackIterateAction action = callback(
payload, node->state, path->tree_count, is_done, path->is_pending);
StackIterateAction action =
callback(payload, node->state, &path->trees, path->tree_count, is_done,
path->is_pending);
bool should_pop = action & StackIteratePop;
bool should_stop = action & StackIterateStop || node->link_count == 0;
@ -207,7 +212,7 @@ INLINE StackPopResult stack__iter(Stack *self, StackVersion version,
next_path->node = link.node;
if (!link.is_pending)
next_path->is_pending = false;
if (!link.tree->extra && link.tree->symbol != ts_builtin_sym_error)
if (!link.tree->extra)
next_path->tree_count++;
if (!array_push(&next_path->trees, link.tree))
goto error;
@ -302,6 +307,24 @@ TSLength ts_stack_top_position(const Stack *self, StackVersion version) {
return (*array_get(&self->heads, version))->position;
}
size_t ts_stack_error_length(const Stack *self, StackVersion version) {
return (*array_get(&self->heads, version))->error_length;
}
size_t ts_stack_last_repaired_error_size(const Stack *self,
StackVersion version) {
StackNode *node = (*array_get(&self->heads, version));
for (;;) {
if (node->link_count == 0)
break;
TSTree *tree = node->links[0].tree;
if (tree->error_size > 0)
return ts_tree_last_error_size(tree);
node = node->links[0].node;
}
return 0;
}
bool ts_stack_push(Stack *self, StackVersion version, TSTree *tree,
bool is_pending, TSStateId state) {
StackNode *node = *array_get(&self->heads, version);
@ -321,40 +344,48 @@ StackPopResult ts_stack_iterate(Stack *self, StackVersion version,
}
INLINE StackIterateAction pop_count_callback(void *payload, TSStateId state,
size_t tree_count, bool is_done,
bool is_pending) {
TreeArray *trees, size_t tree_count,
bool is_done, bool is_pending) {
StackPopSession *pop_session = (StackPopSession *)payload;
if (pop_session->found_error)
return StackIterateStop;
if (tree_count == pop_session->goal_tree_count)
if (tree_count == pop_session->goal_tree_count) {
pop_session->found_valid_path = true;
return StackIteratePop | StackIterateStop;
}
if (state == ts_parse_state_error) {
pop_session->found_error = true;
return StackIteratePop | StackIterateStop;
if (pop_session->found_valid_path || pop_session->found_error) {
return StackIterateStop;
} else {
pop_session->found_error = true;
return StackIteratePop | StackIterateStop;
}
}
return StackIterateNone;
}
StackPopResult ts_stack_pop_count(Stack *self, StackVersion version,
size_t count) {
StackPopSession session = {.goal_tree_count = count, .found_error = false };
StackPopSession session = {
.goal_tree_count = count, .found_error = false, .found_valid_path = false,
};
StackPopResult pop = stack__iter(self, version, pop_count_callback, &session);
if (pop.status && session.found_error) {
pop.status = StackPopStoppedAtError;
StackSlice stopped_slice = array_pop(&pop.slices);
for (size_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
ts_tree_array_delete(&slice.trees);
ts_stack_remove_version(self, slice.version);
stopped_slice.version--;
if (session.found_valid_path) {
StackSlice error_slice = pop.slices.contents[0];
ts_stack_remove_version(self, error_slice.version);
ts_tree_array_delete(&error_slice.trees);
array_erase(&pop.slices, 0);
pop.slices.contents[0].version--;
} else {
pop.status = StackPopStoppedAtError;
}
pop.slices.size = 1;
pop.slices.contents[0] = stopped_slice;
}
return pop;
}
INLINE StackIterateAction pop_pending_callback(void *payload, TSStateId state,
TreeArray *trees,
size_t tree_count, bool is_done,
bool is_pending) {
if (tree_count >= 1) {
@ -378,8 +409,8 @@ StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) {
}
INLINE StackIterateAction pop_all_callback(void *payload, TSStateId state,
size_t tree_count, bool is_done,
bool is_pending) {
TreeArray *trees, size_t tree_count,
bool is_done, bool is_pending) {
return is_done ? (StackIteratePop | StackIterateStop) : StackIterateNone;
}
@ -401,6 +432,14 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
array_erase(&self->heads, v1);
}
StackVersion ts_stack_duplicate_version(Stack *self, StackVersion version) {
assert(version < self->heads.size);
if (!array_push(&self->heads, self->heads.contents[version]))
return STACK_VERSION_NONE;
stack_node_retain(*array_back(&self->heads));
return self->heads.size - 1;
}
void ts_stack_merge_from(Stack *self, StackVersion start_version) {
for (size_t i = start_version + 1; i < self->heads.size; i++) {
StackNode *node = self->heads.contents[i];
@ -408,11 +447,16 @@ void ts_stack_merge_from(Stack *self, StackVersion start_version) {
StackNode *prior_node = self->heads.contents[j];
if (prior_node->state == node->state &&
prior_node->position.chars == node->position.chars) {
for (size_t k = 0; k < node->link_count; k++) {
StackLink link = node->links[k];
stack_node_add_link(prior_node, link);
if (prior_node->error_length < node->error_length) {
ts_stack_remove_version(self, i);
} else if (node->error_length < prior_node->error_length) {
ts_stack_remove_version(self, j);
} else {
for (size_t k = 0; k < node->link_count; k++)
stack_node_add_link(prior_node, node->links[k]);
ts_stack_remove_version(self, i);
}
ts_stack_remove_version(self, i--);
i--;
break;
}
}

7
src/runtime/stack.h
View file

@ -41,6 +41,7 @@ enum {
typedef unsigned int StackIterateAction;
typedef StackIterateAction (*StackIterateCallback)(void *, TSStateId state,
TreeArray *trees,
size_t tree_count,
bool is_done,
bool is_pending);
@ -72,6 +73,10 @@ TSStateId ts_stack_top_state(const Stack *, StackVersion);
*/
TSLength ts_stack_top_position(const Stack *, StackVersion);
size_t ts_stack_error_length(const Stack *, StackVersion);
size_t ts_stack_last_repaired_error_size(const Stack *, StackVersion);
/*
* Push a tree and state onto the given head of the stack. This could cause
* the version to merge with an existing version.
@ -100,6 +105,8 @@ void ts_stack_merge(Stack *);
void ts_stack_renumber_version(Stack *, StackVersion, StackVersion);
StackVersion ts_stack_duplicate_version(Stack *, StackVersion);
/*
* Remove the given version from the stack.
*/

39
src/runtime/tree.c
View file

@ -107,6 +107,31 @@ recur:
}
}
static void ts_tree_total_tokens(const TSTree *self, size_t *result) {
recur:
if (self->child_count == 0) {
(*result)++;
} else {
for (size_t i = 1; i < self->child_count; i++)
ts_tree_total_tokens(self->children[i], result);
self = self->children[0];
goto recur;
}
}
size_t ts_tree_last_error_size(const TSTree *self) {
if (self->symbol == ts_builtin_sym_error)
return self->error_size;
for (size_t i = self->child_count - 1; i + 1 > 0; i--) {
size_t result = ts_tree_last_error_size(self->children[i]);
if (result > 0)
return result;
}
return 0;
}
void ts_tree_set_children(TSTree *self, size_t child_count, TSTree **children) {
if (self->child_count > 0)
ts_free(self->children);
@ -115,7 +140,7 @@ void ts_tree_set_children(TSTree *self, size_t child_count, TSTree **children) {
self->child_count = child_count;
self->named_child_count = 0;
self->visible_child_count = 0;
size_t error_size = 0;
self->error_size = 0;
for (size_t i = 0; i < child_count; i++) {
TSTree *child = children[i];
@ -127,6 +152,8 @@ void ts_tree_set_children(TSTree *self, size_t child_count, TSTree **children) {
self->size = ts_length_add(self->size, ts_tree_total_size(child));
}
self->error_size += child->error_size;
if (child->visible) {
self->visible_child_count++;
if (child->named)
@ -139,15 +166,13 @@ void ts_tree_set_children(TSTree *self, size_t child_count, TSTree **children) {
if (child->symbol == ts_builtin_sym_error) {
self->fragile_left = self->fragile_right = true;
self->parse_state = TS_TREE_STATE_ERROR;
} else {
error_size += child->error_size;
}
}
if (self->symbol == ts_builtin_sym_error)
self->error_size = self->size.chars;
else
self->error_size = error_size;
if (self->symbol == ts_builtin_sym_error) {
self->error_size = 0;
ts_tree_total_tokens(self, &self->error_size);
}
if (child_count > 0) {
self->lex_state = children[0]->lex_state;

1
src/runtime/tree.h
View file

@ -67,6 +67,7 @@ void ts_tree_assign_parents(TSTree *);
void ts_tree_edit(TSTree *, TSInputEdit);
void ts_tree_steal_padding(TSTree *, TSTree *);
char *ts_tree_string(const TSTree *, const TSLanguage *, bool include_all);
size_t ts_tree_last_error_size(const TSTree *);
static inline size_t ts_tree_total_chars(const TSTree *self) {
return self->padding.chars + self->size.chars;