traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Consider multiple error repairs on the same path of the stack

Browse source 
This changes the API to the stack_iterate function so that you can pop
from the stack without stopping iteration
This commit is contained in:
Max Brunsfeld 2016-04-15 21:28:00 -07:00
parent 3778c63277
commit cad663b144
8 changed files with 203 additions and 216 deletions
Download patch file Download diff file Expand all files Collapse all files

6
spec/runtime/stack_spec.cc
View file

@ -72,15 +72,15 @@ struct StackEntry {
vector<StackEntry> get_stack_entries(Stack *stack, StackVersion version) {
vector<StackEntry> result;
ts_stack_pop_until(
ts_stack_iterate(
stack,
version,
[](void *payload, TSStateId state, size_t tree_count, bool is_done, bool is_pending) {
[](void *payload, TSStateId state, size_t tree_count, bool is_done, bool is_pending) -> StackIterateAction {
auto entries = static_cast<vector<StackEntry> *>(payload);
StackEntry entry = {state, tree_count};
if (find(entries->begin(), entries->end(), entry) == entries->end())
entries->push_back(entry);
return StackIterateContinue;
return StackIterateNone;
}, &result);
return result;
}

4
src/compiler/build_tables/build_parse_table.cc
View file

@ -262,8 +262,8 @@ class ParseTableBuilder {
}
void remove_duplicate_parse_states() {
auto replacements = remove_duplicate_states<ParseState, ParseAction>(
&parse_table.states);
auto replacements =
remove_duplicate_states<ParseState, ParseAction>(&parse_table.states);
for (auto &pair : parse_table.out_of_context_state_indices) {
auto replacement = replacements.find(pair.second);

3
src/compiler/build_tables/item_set_closure.cc
View file

@ -37,7 +37,8 @@ ParseItemSet item_set_closure(const ParseItemSet &input_item_set,
// If the next symbol in the production is not a non-terminal, skip to the
// next item.
Symbol next_symbol = item.next_symbol();
if (next_symbol == NONE() || next_symbol.is_token || next_symbol.is_built_in())
if (next_symbol == NONE() || next_symbol.is_token ||
next_symbol.is_built_in())
continue;
// If the next symbol is the last symbol in the item's production, then the

8
src/compiler/generate_code/c_code.cc
View file

@ -228,8 +228,9 @@ class CCodeGenerator {
if (symbol.is_built_in())
continue;
auto iter = parse_table.out_of_context_state_indices.find(symbol);
string state = (iter != parse_table.out_of_context_state_indices.end()) ?
to_string(iter->second) : "ts_parse_state_error";
string state = (iter != parse_table.out_of_context_state_indices.end())
? to_string(iter->second)
: "ts_parse_state_error";
line("[" + symbol_id(symbol) + "] = " + state + ",");
}
});
@ -277,7 +278,8 @@ class CCodeGenerator {
for (const ParseState &state : parse_table.states) {
if (!state.in_progress_symbols.empty()) {
line("[" + to_string(state_id) + "] = ");
add(to_string(add_in_progress_symbol_list_id(state.in_progress_symbols)));
add(to_string(
add_in_progress_symbol_list_id(state.in_progress_symbols)));
add(",");
}
state_id++;

12
src/runtime/array.h
View file

@ -55,11 +55,13 @@ extern "C" {
#define array_reverse(self) \
array__reverse((VoidArray *)(self), array__elem_size(self))
#define array_copy(self) \
{ \
memcpy(ts_calloc((self)->capacity, array__elem_size(self)), \
(self)->contents, (self)->size *array__elem_size(self)), \
(self)->size, (self)->capacity, \
#define array_copy(self) \
{ \
(self)->contents \
? memcpy(ts_calloc((self)->capacity, array__elem_size(self)), \
(self)->contents, (self)->size *array__elem_size(self)) \
: NULL, \
(self)->size, (self)->capacity, \
}
// Private

133
src/runtime/parser.c
View file

@ -52,18 +52,20 @@ typedef struct {
} ReusableNode;
struct ErrorRepair {
TSParseAction action;
TSStateId next_state;
TSSymbol symbol;
size_t in_progress_state_count;
size_t essential_tree_count;
size_t count_below_error;
};
typedef struct {
ErrorRepairArray *repairs;
size_t count_above_error;
int best_repair_index;
TSParser *parser;
TSSymbol lookahead_symbol;
ErrorRepairArray *repairs;
bool found_repair;
ErrorRepair best_repair;
TSStateId best_repair_state;
TSStateId best_repair_next_state;
size_t best_repair_skip_count;
} ErrorRepairSession;
typedef enum {
@ -384,28 +386,39 @@ static StackIterateAction ts_parser__error_repair_callback(void *payload,
bool is_pending) {
ErrorRepairSession *session = (ErrorRepairSession *)payload;
const TSParser *self = session->parser;
size_t count_above_error = session->count_above_error;
const TSLanguage *language = self->language;
TSSymbol lookahead_symbol = session->lookahead_symbol;
if (session->best_repair_index >= 0)
return StackIterateAbort;
StackIterateAction result = StackIterateNone;
for (size_t i = 0; i < session->repairs->size; i++) {
ErrorRepair *repair = &session->repairs->contents[i];
TSSymbol symbol = repair->action.data.symbol;
size_t child_count = repair->action.data.child_count;
TSSymbol symbol = repair->symbol;
if (tree_count + count_above_error >= child_count &&
repair->in_progress_state_count > 0) {
TSParseAction new_action =
ts_language_last_action(self->language, state, symbol);
if (new_action.type == TSParseActionTypeShift) {
repair->next_state = new_action.data.to_state;
if (ts_language_has_action(self->language, repair->next_state,
lookahead_symbol)) {
session->best_repair_index = i;
repair->essential_tree_count = tree_count;
return StackIteratePop;
if (tree_count >= repair->count_below_error) {
size_t skip_count = tree_count - repair->count_below_error;
if (session->found_repair && skip_count > session->best_repair_skip_count) {
array_erase(session->repairs, i--);
continue;
}
if (repair->in_progress_state_count > 0) {
TSParseAction action = ts_language_last_action(language, state, symbol);
if (action.type == TSParseActionTypeShift) {
TSStateId next_state = action.data.to_state;
if (ts_language_has_action(language, next_state, lookahead_symbol) &&
(!session->found_repair ||
repair->in_progress_state_count >
session->best_repair.in_progress_state_count)) {
result |= StackIteratePop;
session->found_repair = true;
session->best_repair = *repair;
session->best_repair_state = state;
session->best_repair_skip_count = skip_count;
session->best_repair_next_state = next_state;
array_erase(session->repairs, i--);
continue;
}
}
}
}
@ -416,75 +429,85 @@ static StackIterateAction ts_parser__error_repair_callback(void *payload,
repair->in_progress_state_count = 0;
}
return StackIterateContinue;
if (session->repairs->size == 0)
result |= StackIterateStop;
return result;
}
static RepairResult ts_parser__repair_error(TSParser *self, StackSlice slice,
TSTree *lookahead,
const TSParseAction *actions,
size_t action_count) {
size_t count_above_error = ts_tree_array_essential_count(&slice.trees);
ErrorRepairSession session = {
.count_above_error = ts_tree_array_essential_count(&slice.trees),
.best_repair_index = -1,
.repairs = &self->error_repairs,
.lookahead_symbol = lookahead->symbol,
.parser = self,
.lookahead_symbol = lookahead->symbol,
.repairs = &self->error_repairs,
.found_repair = false,
};
array_clear(&self->error_repairs);
for (size_t i = 0; i < action_count; i++)
if (actions[i].type == TSParseActionTypeReduce &&
actions[i].data.child_count > session.count_above_error)
CHECK(array_push(&self->error_repairs, ((ErrorRepair){
.action = actions[i],
.in_progress_state_count = 0,
.essential_tree_count = 0,
})));
actions[i].data.child_count > count_above_error)
CHECK(array_push(
&self->error_repairs,
((ErrorRepair){
.symbol = actions[i].data.symbol,
.count_below_error = actions[i].data.child_count - count_above_error,
.in_progress_state_count = 0,
})));
StackPopResult pop = ts_stack_pop_until(
StackPopResult pop = ts_stack_iterate(
self->stack, slice.version, ts_parser__error_repair_callback, &session);
if (!pop.slices.size) {
CHECK(pop.status);
if (!session.found_repair) {
ts_tree_array_delete(&slice.trees);
return RepairNoneFound;
}
ts_stack_renumber_version(self->stack, pop.slices.contents[0].version,
slice.version);
ErrorRepair repair = self->error_repairs.contents[session.best_repair_index];
size_t count_needed_below =
repair.action.data.child_count - session.count_above_error;
TreeArray children_below = pop.slices.contents[0].trees;
size_t count_skipped_below = repair.essential_tree_count - count_needed_below;
TSSymbol symbol = repair.action.data.symbol;
ErrorRepair repair = session.best_repair;
TSStateId next_state = session.best_repair_next_state;
size_t skip_count = session.best_repair_skip_count;
TSSymbol symbol = repair.symbol;
StackSlice new_slice = array_pop(&pop.slices);
TreeArray children_below = new_slice.trees;
ts_stack_renumber_version(self->stack, new_slice.version, slice.version);
while (pop.slices.size) {
StackSlice other_slice = array_pop(&pop.slices);
ts_tree_array_delete(&other_slice.trees);
ts_stack_remove_version(self->stack, other_slice.version);
}
LOG_ACTION(
"repair_found sym:%s, child_count:%d, match_count:%lu, skipped:%lu",
SYM_NAME(symbol), repair.action.data.child_count,
repair.in_progress_state_count, count_skipped_below);
"repair_found sym:%s, child_count:%lu, match_count:%lu, skipped:%lu",
SYM_NAME(symbol), repair.count_below_error, repair.in_progress_state_count,
skip_count);
if (count_skipped_below > 0) {
if (skip_count > 0) {
TreeArray skipped_children = array_new();
CHECK(array_grow(&skipped_children, count_skipped_below));
for (size_t i = count_needed_below; i < children_below.size; i++)
CHECK(array_grow(&skipped_children, skip_count));
for (size_t i = repair.count_below_error; i < children_below.size; i++)
array_push(&skipped_children, children_below.contents[i]);
TSTree *error = ts_tree_make_error_node(&skipped_children);
CHECK(error);
children_below.size = count_needed_below;
children_below.size = repair.count_below_error;
array_push(&children_below, error);
}
for (size_t i = 0; i < slice.trees.size; i++)
array_push(&children_below, slice.trees.contents[i]);
array_delete(&slice);
array_delete(&slice.trees);
TSTree *parent =
ts_tree_make_node(symbol, children_below.size, children_below.contents,
ts_language_symbol_metadata(self->language, symbol));
CHECK(parent);
CHECK(ts_stack_push(self->stack, slice.version, parent, false,
repair.next_state));
ts_tree_release(parent);
CHECK(ts_parser__push(self, slice.version, parent, next_state));
return RepairSucceeded;
error:

237
src/runtime/stack.c
View file

@ -11,7 +11,7 @@
#define STARTING_TREE_CAPACITY 10
#define MAX_NODE_POOL_SIZE 50
#define ALWAYS_INLINE __attribute__((always_inline))
#define INLINE static inline __attribute__((always_inline))
typedef struct StackNode StackNode;
@ -31,9 +31,8 @@ struct StackNode {
typedef struct {
TreeArray trees;
size_t essential_tree_count;
size_t tree_count;
StackNode *node;
bool is_done;
bool is_pending;
} PopPath;
@ -58,10 +57,6 @@ static const char *COLORS[] = {
"red", "blue", "orange", "green", "purple",
};
/*
* Section: Manipulating nodes (Private)
*/
static void stack_node_retain(StackNode *self) {
if (!self)
return;
@ -137,10 +132,6 @@ static void stack_node_add_link(StackNode *self, StackLink link) {
}
}
/*
* Section: Stack lifecycle
*/
static int ts_stack__default_tree_selection(void *p, TSTree *t1, TSTree *t2) {
return 0;
}
@ -194,10 +185,6 @@ error:
return NULL;
}
/*
* Section: Reading from the stack
*/
TSStateId ts_stack_top_state(const Stack *self, StackVersion version) {
return (*array_get(&self->heads, version))->state;
}
@ -210,39 +197,6 @@ size_t ts_stack_version_count(const Stack *self) {
return self->heads.size;
}
static void ts_stack__merge_slice(Stack *self, StackSlice *slice,
StackSlice *new_slice) {
bool should_update = false;
if (slice->trees.size < new_slice->trees.size) {
should_update = true;
} else if (slice->trees.size == new_slice->trees.size) {
for (size_t i = 0; i < slice->trees.size; i++) {
TSTree *tree = slice->trees.contents[i];
TSTree *new_tree = new_slice->trees.contents[i];
int comparison = self->tree_selection_function(
self->tree_selection_payload, tree, new_tree);
if (comparison < 0) {
break;
} else if (comparison > 0) {
should_update = true;
break;
}
}
}
if (should_update) {
ts_tree_array_delete(&slice->trees);
slice->trees = new_slice->trees;
slice->trees.size = new_slice->trees.size;
} else {
ts_tree_array_delete(&new_slice->trees);
}
}
/*
* Section: Mutating the stack (Private)
*/
static StackVersion ts_stack__add_version(Stack *self, StackNode *node) {
if (!array_push(&self->heads, node))
return STACK_VERSION_NONE;
@ -256,9 +210,51 @@ void ts_stack_remove_version(Stack *self, StackVersion version) {
array_erase(&self->heads, version);
}
/*
* Section: Mutating the stack (Public)
*/
static void ts_stack__merge_slice(Stack *self, StackSlice *slice,
TreeArray *trees) {
bool should_update = false;
if (slice->trees.size < trees->size) {
should_update = true;
} else if (slice->trees.size == trees->size) {
for (size_t i = 0; i < slice->trees.size; i++) {
TSTree *tree = slice->trees.contents[i];
TSTree *new_tree = trees->contents[i];
int comparison = self->tree_selection_function(
self->tree_selection_payload, tree, new_tree);
if (comparison < 0) {
break;
} else if (comparison > 0) {
should_update = true;
break;
}
}
}
if (should_update) {
ts_tree_array_delete(&slice->trees);
slice->trees = *trees;
} else {
ts_tree_array_delete(trees);
}
}
static bool ts_stack__add_slice(Stack *self, size_t previous_version_count,
StackNode *node, TreeArray *trees) {
for (size_t i = 0; i < self->slices.size; i++) {
StackSlice *previous_slice = &self->slices.contents[i];
size_t version_index = previous_version_count + i;
if (self->heads.contents[version_index] == node) {
ts_stack__merge_slice(self, previous_slice, trees);
return true;
}
}
StackVersion version = ts_stack__add_version(self, node);
if (version == STACK_VERSION_NONE)
return false;
StackSlice slice = {.version = version, .trees = *trees };
return array_push(&self->slices, slice);
}
bool ts_stack_push(Stack *self, StackVersion version, TSTree *tree,
bool is_pending, TSStateId state) {
@ -299,57 +295,53 @@ void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
array_erase(&self->heads, v1);
}
static inline ALWAYS_INLINE StackPopResult
stack__pop(Stack *self, StackVersion version, StackIterateCallback callback,
void *payload) {
INLINE StackPopResult stack__iter(Stack *self, StackVersion version,
StackIterateCallback callback, void *payload) {
size_t previous_version_count = self->heads.size;
array_clear(&self->slices);
PopPath pop_path = {
.node = *array_get(&self->heads, version),
.trees = array_new(),
.essential_tree_count = 0,
.is_done = false,
.tree_count = 0,
.is_pending = true,
};
array_clear(&self->pop_paths);
if (!array_push(&self->pop_paths, pop_path))
goto error;
bool all_paths_done = false;
while (!all_paths_done) {
all_paths_done = true;
while (self->pop_paths.size > 0) {
for (size_t i = 0, size = self->pop_paths.size; i < size; i++) {
PopPath *path = &self->pop_paths.contents[i];
if (path->is_done)
continue;
StackNode *node = path->node;
size_t link_count = node->link_count;
bool is_done = node == self->base_node;
switch (callback(payload, node->state, path->essential_tree_count,
node == self->base_node, path->is_pending)) {
case StackIteratePop:
path->is_done = true;
continue;
case StackIterateAbort:
link_count = 0;
break;
default:
break;
StackIterateAction action = callback(
payload, node->state, path->tree_count, is_done, path->is_pending);
bool should_pop = action & StackIteratePop;
bool should_stop = action & StackIterateStop || node->link_count == 0;
if (should_pop) {
TreeArray trees =
should_stop ? path->trees : ts_tree_array_copy(&path->trees);
array_reverse(&trees);
if (!ts_stack__add_slice(self, previous_version_count, node, &trees))
goto error;
}
if (link_count == 0) {
ts_tree_array_delete(&path->trees);
if (should_stop) {
if (!should_pop)
ts_tree_array_delete(&path->trees);
array_erase(&self->pop_paths, i);
i--, size--;
continue;
}
all_paths_done = false;
for (size_t j = 1; j <= link_count; j++) {
for (size_t j = 1; j <= node->link_count; j++) {
PopPath *next_path;
StackLink link;
if (j == link_count) {
if (j == node->link_count) {
link = node->links[0];
next_path = &self->pop_paths.contents[i];
} else {
@ -364,8 +356,7 @@ static inline ALWAYS_INLINE StackPopResult
if (!link.is_pending)
next_path->is_pending = false;
if (!link.tree->extra && link.tree->symbol != ts_builtin_sym_error)
next_path->essential_tree_count++;
next_path->tree_count++;
if (!array_push(&next_path->trees, link.tree))
goto error;
ts_tree_retain(link.tree);
@ -373,36 +364,7 @@ static inline ALWAYS_INLINE StackPopResult
}
}
array_clear(&self->slices);
for (size_t i = 0; i < self->pop_paths.size; i++) {
PopPath *path = &self->pop_paths.contents[i];
if (!path->is_done)
continue;
StackSlice slice = {.trees = path->trees, .version = STACK_VERSION_NONE };
array_reverse(&slice.trees);
bool merged = false;
for (size_t j = 0; j < self->slices.size; j++) {
StackSlice *prior_slice = &self->slices.contents[j];
StackNode *prior_node = self->heads.contents[prior_slice->version];
if (prior_node == path->node) {
ts_stack__merge_slice(self, prior_slice, &slice);
merged = true;
break;
}
}
if (!merged) {
slice.version = ts_stack__add_version(self, path->node);
if (slice.version == STACK_VERSION_NONE)
goto error;
if (!array_push(&self->slices, slice))
goto error;
}
}
return (StackPopResult){.status = StackPopSucceeded, .slices = self->slices };
return (StackPopResult){ StackPopSucceeded, self->slices };
error:
for (size_t i = 0; i < self->pop_paths.size; i++)
@ -411,33 +373,30 @@ error:
return (StackPopResult){.status = StackPopFailed };
}
StackPopResult ts_stack_pop_until(Stack *self, StackVersion version,
StackIterateCallback callback, void *payload) {
return stack__pop(self, version, callback, payload);
StackPopResult ts_stack_iterate(Stack *self, StackVersion version,
StackIterateCallback callback, void *payload) {
return stack__iter(self, version, callback, payload);
}
static inline ALWAYS_INLINE StackIterateAction
stack__pop_count_callback(void *payload, TSStateId state, size_t tree_count,
bool is_done, bool is_pending) {
INLINE StackIterateAction pop_count_callback(void *payload, TSStateId state,
size_t tree_count, bool is_done,
bool is_pending) {
StackPopSession *pop_session = (StackPopSession *)payload;
if (pop_session->found_error)
return StackIterateAbort;
return StackIterateStop;
if (tree_count == pop_session->goal_tree_count)
return StackIteratePop;
return StackIteratePop | StackIterateStop;
if (state == ts_parse_state_error) {
pop_session->found_error = true;
return StackIteratePop;
return StackIteratePop | StackIterateStop;
}
return StackIterateContinue;
return StackIterateNone;
}
StackPopResult ts_stack_pop_count(Stack *self, StackVersion version,
size_t count) {
StackPopSession session = {.goal_tree_count = count, .found_error = false };
StackPopResult pop =
stack__pop(self, version, stack__pop_count_callback, &session);
StackPopResult pop = stack__iter(self, version, pop_count_callback, &session);
if (pop.status && session.found_error) {
pop.status = StackPopStoppedAtError;
array_reverse(&pop.slices);
@ -447,27 +406,25 @@ StackPopResult ts_stack_pop_count(Stack *self, StackVersion version,
ts_stack_remove_version(self, slice.version);
}
}
return pop;
}
static inline ALWAYS_INLINE StackIterateAction
stack__pop_pending_callback(void *payload, TSStateId state, size_t tree_count,
bool is_done, bool is_pending) {
INLINE StackIterateAction pop_pending_callback(void *payload, TSStateId state,
size_t tree_count, bool is_done,
bool is_pending) {
if (tree_count >= 1) {
if (is_pending) {
return StackIteratePop;
return StackIteratePop | StackIterateStop;
} else {
return StackIterateAbort;
return StackIterateStop;
}
} else {
return StackIterateContinue;
return StackIterateNone;
}
}
StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) {
StackPopResult pop =
stack__pop(self, version, stack__pop_pending_callback, NULL);
StackPopResult pop = stack__iter(self, version, pop_pending_callback, NULL);
if (pop.slices.size > 0) {
ts_stack_renumber_version(self, pop.slices.contents[0].version, version);
pop.slices.contents[0].version = version;
@ -475,14 +432,14 @@ StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) {
return pop;
}
static inline ALWAYS_INLINE StackIterateAction
stack__pop_all_callback(void *payload, TSStateId state, size_t tree_count,
bool is_done, bool is_pending) {
return is_done ? StackIteratePop : StackIterateContinue;
INLINE StackIterateAction pop_all_callback(void *payload, TSStateId state,
size_t tree_count, bool is_done,
bool is_pending) {
return is_done ? (StackIteratePop | StackIterateStop) : StackIterateNone;
}
TreeArray ts_stack_pop_all(Stack *self, StackVersion version) {
StackPopResult pop = stack__pop(self, version, stack__pop_all_callback, NULL);
StackPopResult pop = stack__iter(self, version, pop_all_callback, NULL);
if (pop.status != StackPopSucceeded)
return (TreeArray)array_new();
assert(pop.slices.size == 1);

16
src/runtime/stack.h
View file

@ -32,11 +32,13 @@ typedef struct {
StackSliceArray slices;
} StackPopResult;
typedef enum {
StackIterateContinue,
StackIterateAbort,
StackIteratePop,
} StackIterateAction;
enum {
StackIterateNone,
StackIterateStop = 1 << 0,
StackIteratePop = 1 << 1,
};
typedef unsigned int StackIterateAction;
typedef StackIterateAction (*StackIterateCallback)(void *, TSStateId state,
size_t tree_count,
@ -87,8 +89,8 @@ bool ts_stack_push(Stack *, StackVersion, TSTree *, bool, TSStateId);
*/
StackPopResult ts_stack_pop_count(Stack *, StackVersion, size_t count);
StackPopResult ts_stack_pop_until(Stack *, StackVersion, StackIterateCallback,
void *);
StackPopResult ts_stack_iterate(Stack *, StackVersion, StackIterateCallback,
void *);
StackPopResult ts_stack_pop_pending(Stack *, StackVersion);