traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Halt stack pops at all error states, not just error trees

Browse source
This commit is contained in:
Max Brunsfeld 2016-03-03 11:05:37 -08:00
parent 5f9be9d9b2
commit c0595c21c5
4 changed files with 127 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

113
spec/runtime/stack_spec.cc
View file

@ -132,61 +132,68 @@ describe("Stack", [&]() {
it("removes the given number of nodes from the stack", [&]() {
// . <──0── A*
StackSliceArray slices = ts_stack_pop(stack, 0, 2, false);
AssertThat(slices.size, Equals<size_t>(1));
StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = slices.contents[0];
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[1], trees[2] })));
AssertThat(*ts_stack_head(stack, 0), Equals<StackEntry>({stateA, tree_len}));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
// .*
slices = ts_stack_pop(stack, 0, 1, false);
AssertThat(slices.size, Equals<size_t>(1));
pop_result = ts_stack_pop(stack, 0, 1, false);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
slice = slices.contents[0];
slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[0] })));
AssertThat(ts_stack_head(stack, 0), Equals<const StackEntry *>(nullptr));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
it("does not count 'extra' trees toward the count", [&]() {
trees[1]->extra = true;
// .*
StackSliceArray slices = ts_stack_pop(stack, 0, 2, false);
AssertThat(slices.size, Equals<size_t>(1));
StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = slices.contents[0];
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[0], trees[1], trees[2] })));
AssertThat(ts_stack_head(stack, 0), Equals<const StackEntry *>(nullptr));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
it("pops the entire stack when given a negative count", [&]() {
// .*
StackSliceArray slices = ts_stack_pop(stack, 0, -1, false);
AssertThat(slices.size, Equals<size_t>(1));
StackPopResult pop_result = ts_stack_pop(stack, 0, -1, false);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = slices.contents[0];
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[0], trees[1], trees[2] })));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
it("stops immediately when removing an error", [&]() {
trees[2]->symbol = ts_builtin_sym_error;
it("stops immediately after popping a node with the error state", [&]() {
// . <──0── A <──1── B <──2── C <──3── ? <──4── D*
ts_stack_push(stack, 0, trees[3], ts_parse_state_error);
ts_stack_push(stack, 0, trees[4], stateD);
StackSliceArray slices = ts_stack_pop(stack, 0, 2, false);
AssertThat(slices.size, Equals<size_t>(1));
StackPopResult pop_result = ts_stack_pop(stack, 0, 3, false);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopStoppedAtError));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[2] })));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateB));
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[3], trees[4] })));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateC));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
});
@ -209,15 +216,15 @@ describe("Stack", [&]() {
// ↑
// └─*
ts_stack_push(stack, 0, trees[3], stateD);
StackSliceArray slices = ts_stack_pop(stack, 1, 1, false);
StackPopResult pop_result = ts_stack_pop(stack, 1, 1, false);
AssertThat(ts_stack_head_count(stack), Equals(2));
AssertThat(*ts_stack_head(stack, 0), Equals<StackEntry>({stateD, tree_len * 4}));
AssertThat(*ts_stack_head(stack, 1), Equals<StackEntry>({stateB, tree_len * 2}));
AssertThat(slices.size, Equals<size_t>(1));
StackSlice slice = slices.contents[0];
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees.size, Equals<size_t>(1));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
// . <──0── A <──1── B <──2── C <──3── D*
// ↑
@ -342,15 +349,15 @@ describe("Stack", [&]() {
// . <──0── A <──1── B <──2── C*
// ↑
// └───5─── F*
StackSliceArray slices = ts_stack_pop(stack, 0, 2, false);
StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false);
AssertThat(slices.size, Equals<size_t>(2));
StackSlice slice1 = slices.contents[0];
AssertThat(pop_result.slices.size, Equals<size_t>(2));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.head_index, Equals(0));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateC));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[3], trees[4] })));
StackSlice slice2 = slices.contents[1];
StackSlice slice2 = pop_result.slices.contents[1];
AssertThat(slice2.head_index, Equals(1));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateF));
AssertThat(slice2.trees, Equals(vector<TSTree *>({ trees[6], trees[7] })));
@ -359,7 +366,7 @@ describe("Stack", [&]() {
AssertThat(*ts_stack_head(stack, 0), Equals<StackEntry>({stateC, tree_len * 3}));
AssertThat(*ts_stack_head(stack, 1), Equals<StackEntry>({stateF, tree_len * 3}));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
});
@ -375,16 +382,16 @@ describe("Stack", [&]() {
// . <──0── A <──1── B <──2── C <──3── D <──4── E*
// ↑ |
// └───5─── F <──6── G <──7───┘
StackSliceArray slices = ts_stack_pop(stack, 0, 1, false);
AssertThat(slices.size, Equals<size_t>(1));
StackSlice slice1 = slices.contents[0];
StackPopResult pop_result = ts_stack_pop(stack, 0, 1, false);
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.head_index, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[8] })));
AssertThat(ts_stack_head_count(stack), Equals(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateE));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
});
@ -393,17 +400,17 @@ describe("Stack", [&]() {
it("returns one entry for that head, with the first path of trees", [&]() {
tree_selection_spy.tree_to_return = trees[2];
// . <──0── A <──1── B*
StackSliceArray slices = ts_stack_pop(stack, 0, 3, false);
// . <──0── A <──1── B*
StackPopResult pop_result = ts_stack_pop(stack, 0, 3, false);
AssertThat(ts_stack_head_count(stack), Equals(1));
AssertThat(*ts_stack_head(stack, 0), Equals<StackEntry>({stateB, tree_len * 2}));
AssertThat(slices.size, Equals<size_t>(1));
StackSlice slice1 = slices.contents[0];
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.head_index, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[2], trees[3], trees[4] })));
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
});
@ -411,17 +418,17 @@ describe("Stack", [&]() {
it("returns one entry for that head, with the second path of trees", [&]() {
tree_selection_spy.tree_to_return = trees[4];
// . <──0── A <──1── B*
StackSliceArray slices = ts_stack_pop(stack, 0, 3, false);
// . <──0── A <──1── B*
StackPopResult pop_result = ts_stack_pop(stack, 0, 3, false);
AssertThat(ts_stack_head_count(stack), Equals(1));
AssertThat(*ts_stack_head(stack, 0), Equals<StackEntry>({stateB, tree_len * 2}));
AssertThat(slices.size, Equals<size_t>(1));
StackSlice slice1 = slices.contents[0];
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.head_index, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[5], trees[6], trees[7] })))
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
});
});
@ -460,27 +467,27 @@ describe("Stack", [&]() {
// ├───4─── E <──5── F*
// |
// └───7─── G <──8── H*
StackSliceArray slices = ts_stack_pop(stack, 0, 2, false);
StackPopResult pop_result = ts_stack_pop(stack, 0, 2, false);
AssertThat(ts_stack_head_count(stack), Equals(3));
AssertThat(slices.size, Equals<size_t>(3));
AssertThat(pop_result.slices.size, Equals<size_t>(3));
StackSlice slice1 = slices.contents[0];
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(ts_stack_top_state(stack, 0), Equals(stateC));
AssertThat(slice1.head_index, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[3], trees[10] })))
StackSlice slice2 = slices.contents[1];
StackSlice slice2 = pop_result.slices.contents[1];
AssertThat(ts_stack_top_state(stack, 1), Equals(stateF));
AssertThat(slice2.head_index, Equals(1));
AssertThat(slice2.trees, Equals(vector<TSTree *>({ trees[6], trees[10] })))
StackSlice slice3 = slices.contents[2];
StackSlice slice3 = pop_result.slices.contents[2];
AssertThat(ts_stack_top_state(stack, 2), Equals(stateH));
AssertThat(slice3.head_index, Equals(2));
AssertThat(slice3.trees, Equals(vector<TSTree *>({ trees[9], trees[10] })))
free_slice_array(&slices);
free_slice_array(&pop_result.slices);
});
});
});

90
src/runtime/parser.c
View file

@ -52,7 +52,8 @@ struct LookaheadState {
typedef struct {
TSSymbol symbol;
TSStateId next_state;
size_t depth;
size_t stack_depth;
size_t child_count;
size_t skipped_subtree_count;
size_t in_progress_state_count;
} ErrorRepair;
@ -75,15 +76,16 @@ typedef enum {
*/
static StackSlice ts_parser__pop_one(TSParser *self, int head_index, int count) {
StackSliceArray slices = ts_stack_pop(self->stack, head_index, count, true);
assert(slices.size > 0);
assert(slices.contents[0].head_index == head_index);
for (size_t i = 1; i < slices.size; i++) {
ts_tree_array_clear(&slices.contents[i].trees);
array_delete(&slices.contents[i].trees);
ts_stack_remove_head(self->stack, slices.contents[i].head_index);
StackPopResult pop = ts_stack_pop(self->stack, head_index, count, true);
assert(pop.status == StackPopSucceeded);
assert(pop.slices.size > 0);
assert(pop.slices.contents[0].head_index == head_index);
for (size_t i = 1; i < pop.slices.size; i++) {
ts_tree_array_clear(&pop.slices.contents[i].trees);
array_delete(&pop.slices.contents[i].trees);
ts_stack_remove_head(self->stack, pop.slices.contents[i].head_index);
}
return slices.contents[0];
return pop.slices.contents[0];
}
static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self,
@ -91,18 +93,18 @@ static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self,
TSTree *last_child = NULL;
do {
StackSliceArray slices = ts_stack_pop(self->stack, head, 1, false);
if (!slices.size)
StackPopResult pop = ts_stack_pop(self->stack, head, 1, false);
if (!pop.slices.size)
return ParseActionFailed;
assert(slices.size > 0);
assert(pop.slices.size > 0);
/*
* Since only one entry (not counting extra trees) is being popped from the
* stack, there should only be one possible array of removed trees.
*/
for (size_t i = 0; i < slices.size; i++) {
StackSlice slice = slices.contents[i];
for (size_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
TreeArray removed_trees = slice.trees;
TSTree *parent = *array_front(&removed_trees);
int head_index = slice.head_index;
@ -349,28 +351,26 @@ static ReduceResult ts_parser__reduce(TSParser *self, int head, TSSymbol symbol,
int child_count, bool extra, bool fragile) {
array_clear(&self->reduce_parents);
TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol);
StackSliceArray slices =
ts_stack_pop(self->stack, head, child_count, false);
if (!slices.size)
StackPopResult pop = ts_stack_pop(self->stack, head, child_count, false);
if (!pop.slices.size)
return ReduceFailed;
if (slices.contents[0].trees.size &&
slices.contents[0].trees.contents[0]->symbol == ts_builtin_sym_error) {
if (pop.status == StackPopStoppedAtError) {
if (self->partial_pop.size) {
ts_tree_array_clear(&self->partial_pop);
array_delete(&self->partial_pop);
}
self->partial_pop = slices.contents[0].trees;
for (size_t i = 1; i < slices.size; i++) {
ts_tree_array_clear(&slices.contents[i].trees);
array_delete(&slices.contents[i].trees);
self->partial_pop = pop.slices.contents[0].trees;
for (size_t i = 1; i < pop.slices.size; i++) {
ts_tree_array_clear(&pop.slices.contents[i].trees);
array_delete(&pop.slices.contents[i].trees);
}
return ReduceStoppedAtError;
}
size_t removed_heads = 0;
for (size_t i = 0; i < slices.size; i++) {
StackSlice slice = slices.contents[i];
for (size_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
/*
* If the same set of trees led to a previous stack head, reuse the parent
@ -492,7 +492,7 @@ static ReduceResult ts_parser__reduce(TSParser *self, int head, TSSymbol symbol,
ts_tree_release(*parent);
}
if (removed_heads < slices.size)
if (removed_heads < pop.slices.size)
return ReduceSucceeded;
else
return ReduceMerged;
@ -555,8 +555,11 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index,
const TSParseAction *actions,
size_t action_count) {
StackEntry *entry_below = ts_stack_head(self->stack, head_index);
while (entry_below && entry_below->state == ts_parse_state_error)
size_t initial_depth = 0;
while (entry_below && entry_below->state == ts_parse_state_error) {
entry_below = ts_stack_entry_next(entry_below, 0);
initial_depth++;
}
bool has_repair = false;
ErrorRepair best_repair;
@ -575,7 +578,7 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index,
continue;
StackEntry *entry = entry_below;
size_t depth = 0;
size_t depth = initial_depth;
size_t child_count = 0;
size_t in_progress_state_count = 0;
@ -599,16 +602,12 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index,
ErrorRepair repair = {
.symbol = action.data.symbol,
.next_state = next_state,
.depth = depth,
.child_count = action.data.child_count,
.stack_depth = depth,
.skipped_subtree_count = child_count + count_above_error - action.data.child_count,
.in_progress_state_count = in_progress_state_count,
};
LOG_ACTION("REPAIR_GOOD sym:%s, in_progress:%lu, skipped:%lu!",
SYM_NAME(action.data.symbol),
in_progress_state_count,
repair.skipped_subtree_count);
if (!has_repair ||
ts_parser__error_repair_is_better(best_repair, repair)) {
has_repair = true;
@ -637,6 +636,15 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index,
return ParseActionRemoved;
}
LOG_ACTION(
"repair_found sym:%s, child_count:%lu, skip_count:%lu, match_count:%lu, DEPTH:%lu",
SYM_NAME(best_repair.symbol),
best_repair.child_count,
best_repair.in_progress_state_count,
best_repair.skipped_subtree_count,
best_repair.stack_depth
);
// Pop any trees that were skipped. Make a new extra error node that contains
// them and the error leaf node.
StackSlice slice = ts_parser__pop_one(self, head_index, best_repair.skipped_subtree_count);
@ -650,7 +658,7 @@ static ParseActionResult ts_parser__repair_error(TSParser *self, int head_index,
// Pop any additional trees that are needed for the chosen reduce action. Make
// a new wrapper node of the chosen symbol that contains them, the error node,
// and the trees that were popped above the error node.
slice = ts_parser__pop_one(self, head_index, best_repair.depth - (error->child_count - 1));
slice = ts_parser__pop_one(self, head_index, best_repair.stack_depth - (error->child_count - 1));
array_push(&slice.trees, error);
for (size_t i = 1; i < self->partial_pop.size; i++)
array_push(&slice.trees, self->partial_pop.contents[i]);
@ -698,12 +706,12 @@ static ParseActionResult ts_parser__start(TSParser *self, TSInput input,
}
static ParseActionResult ts_parser__accept(TSParser *self, int head) {
StackSliceArray slices = ts_stack_pop(self->stack, head, -1, true);
if (!slices.size)
StackPopResult pop = ts_stack_pop(self->stack, head, -1, true);
if (!pop.slices.size)
goto error;
for (size_t j = 0; j < slices.size; j++) {
StackSlice slice = slices.contents[j];
for (size_t j = 0; j < pop.slices.size; j++) {
StackSlice slice = pop.slices.contents[j];
TreeArray trees = slice.trees;
for (size_t i = trees.size - 1; i + 1 > 0; i--) {
@ -732,8 +740,8 @@ static ParseActionResult ts_parser__accept(TSParser *self, int head) {
return ParseActionRemoved;
error:
if (slices.size) {
StackSlice slice = *array_front(&slices);
if (pop.slices.size) {
StackSlice slice = *array_front(&pop.slices);
for (size_t i = 0; i < slice.trees.size; i++)
ts_tree_release(slice.trees.contents[i]);
array_delete(&slice.trees);

15
src/runtime/stack.c
View file

@ -304,7 +304,7 @@ int ts_stack_split(Stack *self, int head_index) {
return ts_stack__add_head(self, head);
}
StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count,
StackPopResult ts_stack_pop(Stack *self, int head_index, int child_count,
bool count_extra) {
array_clear(&self->slices);
array_clear(&self->pop_paths);
@ -327,6 +327,7 @@ StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count,
* of child trees have been collected along every path.
*/
bool all_paths_done = false;
int status = StackPopSucceeded;
while (!all_paths_done) {
all_paths_done = true;
@ -362,14 +363,16 @@ StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count,
next_path->is_shared = false;
}
if (successor.tree->extra && !count_extra)
if (!count_extra && successor.tree->extra)
next_path->goal_tree_count++;
ts_tree_retain(successor.tree);
if (!array_push(&next_path->trees, successor.tree))
goto error;
next_path->node = successor.node;
if (successor.tree->symbol == ts_builtin_sym_error && !count_extra) {
if (!count_extra && node->entry.state == ts_parse_state_error) {
status = StackPopStoppedAtError;
next_path->goal_tree_count = next_path->trees.size;
}
}
@ -416,13 +419,11 @@ StackSliceArray ts_stack_pop(Stack *self, int head_index, int child_count,
}
stack_node_release(previous_head, &self->node_pool);
return self->slices;
return (StackPopResult){ .status = status, .slices = self->slices };
error:
array_delete(&initial_path.trees);
StackSliceArray slices;
array_init(&slices);
return slices;
return (StackPopResult){StackPopFailed, self->slices};
}
void ts_stack_shrink(Stack *self, int head_index, int count) {

12
src/runtime/stack.h
View file

@ -29,6 +29,15 @@ typedef enum {
StackPushContinued,
} StackPushResult;
typedef struct {
enum {
StackPopFailed,
StackPopStoppedAtError,
StackPopSucceeded,
} status;
StackSliceArray slices;
} StackPopResult;
typedef int (*TreeSelectionFunction)(void *, TSTree *, TSTree *);
/*
@ -85,8 +94,7 @@ StackPushResult ts_stack_push(Stack *, int head_index, TSTree *, TSStateId);
* which had previously been merged. It returns a struct that indicates the
* index of each revealed head and the trees removed from that head.
*/
StackSliceArray ts_stack_pop(Stack *, int head_index, int count,
bool count_extra);
StackPopResult ts_stack_pop(Stack *, int head_index, int count, bool count_extra);
/*
* Remove the given number of entries from the given head of the stack.